def bbopt_actor(env):
_coconut_match_to_0 = env
_coconut_match_check_0 = False
if _coconut.isinstance(_coconut_match_to_0, _coconut.abc.Mapping):
_coconut_match_temp_0 = _coconut_match_to_0.get(bb_name, _coconut_sentinel)
if _coconut_match_temp_0 is not _coconut_sentinel:
bb = _coconut_match_temp_0
_coconut_match_check_0 = True
if _coconut_match_check_0 and not (bb is not None):
_coconut_match_check_0 = False
if _coconut_match_check_0:
if isinstance(util_func, Str):
util = env[util_func]
else:
util = util_func(env)
bb.maximize(util)
else:
bb = BlackBoxOptimizer(file=file, tag=env["game"].name + "_" + name)
env[bb_name] = bb
bb.run(alg=alg if not env["game"].final_step else None)
if print_chosen_alg:
chosen_alg = bb.get_current_run()["values"].get(meta_opt_alg_var)
if chosen_alg is not None:
print("\nusing BBopt alg =", chosen_alg)
return tunable_actor(bb, env)
def __init__(self):
super().__init__(create_parser(usage))
self.bb = BlackBoxOptimizer(file=self.args.trials_name)
if not self.test:
data = TrainData.from_both(self.args.tags_file,
self.args.tags_folder, self.args.folder)
_, self.test = data.load(False, True)
from keras.callbacks import ModelCheckpoint
for i in list(self.callbacks):
if isinstance(i, ModelCheckpoint):
self.callbacks.remove(i)
"""
Simple example using json instead of pickle for enhanced cross-platform compatibility.
To run this example, just run:
> bbopt ./skopt_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__, protocol="json")
if __name__ == "__main__":
bb.run()
# Set up log uniform and log normal parameters.
x0 = bb.loguniform("x0", 1, 10, guess=5)
x1 = bb.lognormvariate("x1", 0, 1, guess=1)
# Set the goal to be the sum.
y = x0 + x1
bb.minimize(y)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(y))
import sys
import os
from argparse import ArgumentParser
from pprint import pprint
import numpy as np
from sklearn import datasets
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
from keras.utils import to_categorical
from keras.regularizers import l1_l2
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
# Load data into X and y:
iris = datasets.load_iris()
X = iris.data
y = to_categorical(iris.target)
# Split data into training, validation, and testing:
train_split = int(.6 * len(X))
validate_split = train_split + int(.2 * len(X))
X_train, X_validate, X_test = X[:train_split], X[
train_split:validate_split], X[validate_split:]
y_train, y_validate, y_test = y[:train_split], y[
train_split:validate_split], y[validate_split:]
"""
Example of using BBopt with conditional parameters that only appear
during some runs depending on the value(s) of other parameters.
To run this example, just run:
> bbopt ./conditional_hyperopt_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run(alg="tree_structured_parzen_estimator")
# We set the x parameter conditional on the use_high parameter.
use_high = bb.randbool("use high", guess=False)
assert isinstance(use_high, bool)
if use_high:
x = bb.randrange("x high", 10, 20)
else:
x = bb.randrange("x low", 10)
# We set x as the thing we want to optimize.
bb.maximize(x)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(x))
class TrainOptimizeScript(TrainScript):
Usage('''
Use black box optimization to tune model hyperparameters
:-t --trials-name str -
Filename to save hyperparameter optimization trials in
'.bbopt.json' will automatically be appended
:-c --cycles int 20
Number of cycles of optimization to run
:-m --model str .cache/optimized.net
Model to load from
...
''') | TrainScript.usage
def __init__(self, args):
super().__init__(args)
from bbopt import BlackBoxOptimizer
self.bb = BlackBoxOptimizer(file=self.args.trials_name)
if not self.test:
data = TrainData.from_both(self.args.tags_file,
self.args.tags_folder, self.args.folder)
_, self.test = data.load(False, True)
from keras.callbacks import ModelCheckpoint
for i in list(self.callbacks):
if isinstance(i, ModelCheckpoint):
self.callbacks.remove(i)
def process_args(self, args: Any):
model_parts = glob(splitext(args.model)[0] + '.*')
if len(model_parts) < 5:
for name in model_parts:
if isfile(name):
remove(name)
else:
rmtree(name)
args.trials_name = args.trials_name.replace('.bbopt.json',
'').replace('.json', '')
if not args.trials_name:
if isfile(join('.cache', 'trials.bbopt.json')):
remove(join('.cache', 'trials.bbopt.json'))
args.trials_name = join('.cache', 'trials')
def run(self):
print('Writing to:', self.args.trials_name + '.bbopt.json')
for i in range(self.args.cycles):
self.bb.run(backend="random")
print("\n= %d = (example #%d)" %
(i + 1, len(self.bb.get_data()["examples"]) + 1))
params = ModelParams(recurrent_units=self.bb.randint("units",
1,
70,
guess=50),
dropout=self.bb.uniform("dropout",
0.1,
0.9,
guess=0.6),
extra_metrics=self.args.extra_metrics,
skip_acc=self.args.no_validation,
loss_bias=1.0 - self.args.sensitivity)
print('Testing with:', params)
model = create_model(self.args.model, params)
model.fit(*self.sampled_data,
batch_size=self.args.batch_size,
epochs=self.epoch + self.args.epochs,
validation_data=self.test *
(not self.args.no_validation),
callbacks=self.callbacks,
initial_epoch=self.epoch)
resp = model.evaluate(*self.test, batch_size=self.args.batch_size)
if not isinstance(resp, (list, tuple)):
resp = [resp, None]
test_loss, test_acc = resp
predictions = model.predict(self.test[0],
batch_size=self.args.batch_size)
num_false_positive = numpy.sum(predictions *
(1 - self.test[1]) > 0.5)
num_false_negative = numpy.sum(
(1 - predictions) * self.test[1] > 0.5)
false_positives = num_false_positive / numpy.sum(
self.test[1] < 0.5)
false_negatives = num_false_negative / numpy.sum(
self.test[1] > 0.5)
from math import exp
param_score = 1.0 / (1.0 + exp(
(model.count_params() - 11000) / 2000))
fitness = param_score * (1.0 - 0.2 * false_negatives -
0.8 * false_positives)
self.bb.remember({
"test loss": test_loss,
"test accuracy": test_acc,
"false positive%": false_positives,
"false negative%": false_negatives,
"fitness": fitness
})
print("False positive: ", false_positives * 100, "%")
self.bb.maximize(fitness)
pprint(self.bb.get_current_run())
best_example = self.bb.get_optimal_run()
print("\n= BEST = (example #%d)" %
self.bb.get_data()["examples"].index(best_example))
pprint(best_example)
"""
Example of using the mixture backend's remove_erroring_algs feature.
To run this example, just run:
> bbopt ./remove_erroring_algs_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run_backend(
"mixture",
distribution=[
("gaussian_process", float("inf")),
("tree_structured_parzen_estimator", 1),
],
remove_erroring_algs=True,
)
# Set some parameters that skopt supports.
x0 = bb.randint("x0", 1, 10, guess=5)
x1 = bb.choice("x1", [-10, -1, 0, 1, 10])
# Set a parameter that only hyperopt supports.
x2 = bb.normalvariate("x2", mu=0, sigma=1)
if not bb.is_serving:
assert bb.backend.selected_alg == "tree_structured_parzen_estimator", bb.backend.selected_alg
# Set the goal.
"""
Example of using BBopt with run_meta.
To run this example, just run:
> bbopt ./meta_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run_meta(
algs=(
"random",
"tree_structured_parzen_estimator",
"gaussian_process",
),
meta_alg="epsilon_greedy",
)
# We set u ~ uniform(0, 1) * sin(uniform(0, 1)).
from math import sin
u = bb.random("x0") * sin(bb.random("x1"))
# Set u as the thing to minimize.
bb.minimize(u)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(u))
import os
from argparse import ArgumentParser
from pprint import pprint
import numpy as np
from matplotlib import pyplot as plt
from sklearn import datasets
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
from keras.utils import to_categorical
from keras.regularizers import l1_l2
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
# Load data into X and y:
iris = datasets.load_iris()
X = iris.data
y = to_categorical(iris.target)
# Split data into training, validation, and testing:
train_split = int(.6 * len(X))
validate_split = train_split + int(.2 * len(X))
X_train, X_validate, X_test = X[:train_split], X[
train_split:validate_split], X[validate_split:]
y_train, y_validate, y_test = y[:train_split], y[
train_split:validate_split], y[validate_split:]
"""
Example of using the pySOT backend and stochastic_radial_basis_function algorithm with BBopt.
To run this example, just run:
> bbopt ./skopt_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
def run_trial(serving=False):
"""Run one trial of black box optimization."""
if serving:
bb.run(alg="serving")
else:
bb.run(alg="stochastic_radial_basis_function")
# We'll define some parameters
x0 = bb.randrange("x0", 1, 11, guess=5)
x1 = bb.uniform("x1", 0, 1)
x2 = bb.choice("x2", [-10, -1, 0, 1, 10])
# And set our goal
y = x0 + x1 * x2
bb.minimize(y)
return y
def main(num_trials=10):
"""
Example of using the hyperopt backend with BBopt.
To run this example, just run:
> bbopt ./hyperopt_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run(alg="tree_structured_parzen_estimator")
# Let's use some parameters!
x0 = bb.randint("x0", 1, 10, guess=5)
x1 = bb.normalvariate("x1", mu=0, sigma=1)
x2 = bb.choice("x2", [-10, -1, 0, 1, 10])
# And let's set our goal!
y = x0 + x1 * x2
bb.minimize(y)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(y))
"""
Example of using the default "any_fast" algorithm.
To run this example, just run:
> bbopt ./any_fast_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run() # alg="any_fast" should be the default
# We set u ~ dist(0, 1) * sin(dist(0, 1)) where dist is uniform or normal.
from math import sin
dist = bb.choice("dist", ["uniform", "normal"])
if dist == "normal":
u = bb.normalvariate("x0_n", 0, 1) * sin(bb.normalvariate("x1_n", 0, 1))
else:
u = bb.random("x0_u") * sin(bb.random("x1_u"))
# If we used hyperopt-only parameters, we shouldn't have skopt.
if hasattr(bb.backend, "selected_backend"):
bb.remember({"backend": bb.backend.selected_backend})
if dist == "normal":
assert bb.backend.selected_backend != "scikit-optimize", bb.backend.selected_backend
else:
bb.remember({"backend": bb.backend.backend_name})
# Set u as the thing to minimize.
bb.minimize(u)
from torch.autograd import Variable
from tqdm import tqdm
import utils
import datetime
import model.net as net
# import model.data_loader as data_loader
import model.data_generator as data_generator
from evaluate import evaluate
from tensorboardX import SummaryWriter
from shutil import copy
# boilerplate code so that bbopt can run this file
from bbopt import BlackBoxOptimizer
# will save to avi_hyperparam_train.bbopt.json
bb = BlackBoxOptimizer(file=__file__, protocol='json')
### TODO: update the defaults to the actual directories
parser = argparse.ArgumentParser()
parser.add_argument('--data_dir',
default='data/64x64_SIGNS',
help="File containing directory containing datasets")
# parser.add_argument('--data_dir_list', default=None,
# help="File contating list of dataset directories data_dirs")
parser.add_argument('--model_dir',
default='experiments/base_model',
help="Directory containing params.json")
parser.add_argument('--tensorboard_prefix',
default='',
help="prefix for tensorboard logging")
parser.add_argument('--prefix',
"""
Example of using the random backend with BBopt.
To run this example, just run:
> bbopt ./random_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run(alg="random")
# Let's use some parameters!
x = bb.randint("x", 1, 10)
# And let's set our goal!
bb.maximize(x)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(x)
"""
Example of using some of the array-based parameter definition methods.
To run this example, just run:
> bbopt ./skopt_example.py
"""
import numpy as np
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run()
# Generate 1 x 5 and 5 x 1 random vectors.
x0 = bb.rand("x0", 1, 5, guess=np.zeros((1, 5))) # entries uniform in [0, 1)
x1 = bb.randn("x1", 5, 1, guess=np.zeros((5, 1))) # entries standard normal
# Set the loss to be their dot product.
y = float(x0.dot(x1))
bb.minimize(y)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(y))
"""
Example of using the scikit-optimize backend and gaussian_process algorithm with BBopt.
To run this example, just run:
> bbopt ./skopt_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run(alg="gaussian_process")
# Let's use some parameters!
x0 = bb.randrange("x0", 1, 11, guess=5)
x1 = bb.uniform("x1", 0, 1)
x2 = bb.choice("x2", [-10, -1, 0, 1, 10])
# And let's set our goal!
y = x0 + x1*x2
bb.minimize(y)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(y))
"""
Example using a mixture distribution over many different possible algorithms.
To run this example, just run:
> bbopt ./mixture_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run_backend("mixture", [
("random", 1),
("tree_structured_parzen_estimator", 1),
("annealing", 1),
("gaussian_process", 1),
("random_forest", 1),
("extra_trees", 1),
("gradient_boosted_regression_trees", 1),
])
# If we're not serving, store which algorithm the
# mixture backend has selected.
from bbopt.backends.mixture import MixtureBackend
if isinstance(bb.backend, MixtureBackend):
bb.remember({
"alg": bb.backend.selected_alg,
})
# Set up a parameter from a choice and a random sample.
xs = bb.sample("xs", range(10), 5, guess=[3, 4, 5, 6, 7])
"""
Example of using BBopt with conditional parameters and randomness
while leveraging the bayes-skopt backend.
To run this example, just run:
> bbopt ./bask_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run(alg="bask_gaussian_process")
# We set the x parameter conditional on the use_high parameter and add randomness.
import random
use_high = bb.randbool("use high", guess=False)
assert isinstance(use_high, bool), type(use_high)
if use_high:
x = bb.randrange("x high", 10, 20) * random.random()
else:
x = bb.randrange("x low", 10) * random.random()
# We set x as the thing we want to optimize.
bb.maximize(x)
# Finally, we'll print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(x))
"""
Example of using the default "tpe_or_gp" algorithm.
To run this example, just run:
> bbopt ./tpe_or_gp_example.py
"""
# BBopt setup:
from bbopt import BlackBoxOptimizer
bb = BlackBoxOptimizer(file=__file__)
if __name__ == "__main__":
bb.run() # alg="tpe_or_gp" should be the default
# We set u ~ normal(0, 1) * sin(normal(0, 1)).
from math import sin
u = bb.normalvariate("x0", 0, 1) * sin(bb.normalvariate("x1", 0, 1))
# Since we used hyperopt-only parameters, we shouldn't have skopt.
if hasattr(bb.backend, "selected_backend"):
bb.remember({"backend": bb.backend.selected_backend})
assert bb.backend.selected_backend != "scikit-optimize", bb.backend.selected_backend
else:
bb.remember({"backend": bb.backend.backend_name})
# Set u as the thing to minimize.
bb.minimize(u)
# Print out the value we used for debugging purposes.
if __name__ == "__main__":
print(repr(u))
