def bbopt_actor(env): _coconut_match_to_0 = env _coconut_match_check_0 = False if _coconut.isinstance(_coconut_match_to_0, _coconut.abc.Mapping): _coconut_match_temp_0 = _coconut_match_to_0.get(bb_name, _coconut_sentinel) if _coconut_match_temp_0 is not _coconut_sentinel: bb = _coconut_match_temp_0 _coconut_match_check_0 = True if _coconut_match_check_0 and not (bb is not None): _coconut_match_check_0 = False if _coconut_match_check_0: if isinstance(util_func, Str): util = env[util_func] else: util = util_func(env) bb.maximize(util) else: bb = BlackBoxOptimizer(file=file, tag=env["game"].name + "_" + name) env[bb_name] = bb bb.run(alg=alg if not env["game"].final_step else None) if print_chosen_alg: chosen_alg = bb.get_current_run()["values"].get(meta_opt_alg_var) if chosen_alg is not None: print("\nusing BBopt alg =", chosen_alg) return tunable_actor(bb, env)
def __init__(self): super().__init__(create_parser(usage)) self.bb = BlackBoxOptimizer(file=self.args.trials_name) if not self.test: data = TrainData.from_both(self.args.tags_file, self.args.tags_folder, self.args.folder) _, self.test = data.load(False, True) from keras.callbacks import ModelCheckpoint for i in list(self.callbacks): if isinstance(i, ModelCheckpoint): self.callbacks.remove(i)
""" Simple example using json instead of pickle for enhanced cross-platform compatibility. To run this example, just run: > bbopt ./skopt_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__, protocol="json") if __name__ == "__main__": bb.run() # Set up log uniform and log normal parameters. x0 = bb.loguniform("x0", 1, 10, guess=5) x1 = bb.lognormvariate("x1", 0, 1, guess=1) # Set the goal to be the sum. y = x0 + x1 bb.minimize(y) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(y))
import sys import os from argparse import ArgumentParser from pprint import pprint import numpy as np from sklearn import datasets from keras.models import Sequential from keras.layers import Dense from keras.optimizers import SGD from keras.utils import to_categorical from keras.regularizers import l1_l2 # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) # Load data into X and y: iris = datasets.load_iris() X = iris.data y = to_categorical(iris.target) # Split data into training, validation, and testing: train_split = int(.6 * len(X)) validate_split = train_split + int(.2 * len(X)) X_train, X_validate, X_test = X[:train_split], X[ train_split:validate_split], X[validate_split:] y_train, y_validate, y_test = y[:train_split], y[ train_split:validate_split], y[validate_split:]
""" Example of using BBopt with conditional parameters that only appear during some runs depending on the value(s) of other parameters. To run this example, just run: > bbopt ./conditional_hyperopt_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run(alg="tree_structured_parzen_estimator") # We set the x parameter conditional on the use_high parameter. use_high = bb.randbool("use high", guess=False) assert isinstance(use_high, bool) if use_high: x = bb.randrange("x high", 10, 20) else: x = bb.randrange("x low", 10) # We set x as the thing we want to optimize. bb.maximize(x) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(x))
class TrainOptimizeScript(TrainScript): Usage(''' Use black box optimization to tune model hyperparameters :-t --trials-name str - Filename to save hyperparameter optimization trials in '.bbopt.json' will automatically be appended :-c --cycles int 20 Number of cycles of optimization to run :-m --model str .cache/optimized.net Model to load from ... ''') | TrainScript.usage def __init__(self, args): super().__init__(args) from bbopt import BlackBoxOptimizer self.bb = BlackBoxOptimizer(file=self.args.trials_name) if not self.test: data = TrainData.from_both(self.args.tags_file, self.args.tags_folder, self.args.folder) _, self.test = data.load(False, True) from keras.callbacks import ModelCheckpoint for i in list(self.callbacks): if isinstance(i, ModelCheckpoint): self.callbacks.remove(i) def process_args(self, args: Any): model_parts = glob(splitext(args.model)[0] + '.*') if len(model_parts) < 5: for name in model_parts: if isfile(name): remove(name) else: rmtree(name) args.trials_name = args.trials_name.replace('.bbopt.json', '').replace('.json', '') if not args.trials_name: if isfile(join('.cache', 'trials.bbopt.json')): remove(join('.cache', 'trials.bbopt.json')) args.trials_name = join('.cache', 'trials') def run(self): print('Writing to:', self.args.trials_name + '.bbopt.json') for i in range(self.args.cycles): self.bb.run(backend="random") print("\n= %d = (example #%d)" % (i + 1, len(self.bb.get_data()["examples"]) + 1)) params = ModelParams(recurrent_units=self.bb.randint("units", 1, 70, guess=50), dropout=self.bb.uniform("dropout", 0.1, 0.9, guess=0.6), extra_metrics=self.args.extra_metrics, skip_acc=self.args.no_validation, loss_bias=1.0 - self.args.sensitivity) print('Testing with:', params) model = create_model(self.args.model, params) model.fit(*self.sampled_data, batch_size=self.args.batch_size, epochs=self.epoch + self.args.epochs, validation_data=self.test * (not self.args.no_validation), callbacks=self.callbacks, initial_epoch=self.epoch) resp = model.evaluate(*self.test, batch_size=self.args.batch_size) if not isinstance(resp, (list, tuple)): resp = [resp, None] test_loss, test_acc = resp predictions = model.predict(self.test[0], batch_size=self.args.batch_size) num_false_positive = numpy.sum(predictions * (1 - self.test[1]) > 0.5) num_false_negative = numpy.sum( (1 - predictions) * self.test[1] > 0.5) false_positives = num_false_positive / numpy.sum( self.test[1] < 0.5) false_negatives = num_false_negative / numpy.sum( self.test[1] > 0.5) from math import exp param_score = 1.0 / (1.0 + exp( (model.count_params() - 11000) / 2000)) fitness = param_score * (1.0 - 0.2 * false_negatives - 0.8 * false_positives) self.bb.remember({ "test loss": test_loss, "test accuracy": test_acc, "false positive%": false_positives, "false negative%": false_negatives, "fitness": fitness }) print("False positive: ", false_positives * 100, "%") self.bb.maximize(fitness) pprint(self.bb.get_current_run()) best_example = self.bb.get_optimal_run() print("\n= BEST = (example #%d)" % self.bb.get_data()["examples"].index(best_example)) pprint(best_example)
""" Example of using the mixture backend's remove_erroring_algs feature. To run this example, just run: > bbopt ./remove_erroring_algs_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run_backend( "mixture", distribution=[ ("gaussian_process", float("inf")), ("tree_structured_parzen_estimator", 1), ], remove_erroring_algs=True, ) # Set some parameters that skopt supports. x0 = bb.randint("x0", 1, 10, guess=5) x1 = bb.choice("x1", [-10, -1, 0, 1, 10]) # Set a parameter that only hyperopt supports. x2 = bb.normalvariate("x2", mu=0, sigma=1) if not bb.is_serving: assert bb.backend.selected_alg == "tree_structured_parzen_estimator", bb.backend.selected_alg # Set the goal.
""" Example of using BBopt with run_meta. To run this example, just run: > bbopt ./meta_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run_meta( algs=( "random", "tree_structured_parzen_estimator", "gaussian_process", ), meta_alg="epsilon_greedy", ) # We set u ~ uniform(0, 1) * sin(uniform(0, 1)). from math import sin u = bb.random("x0") * sin(bb.random("x1")) # Set u as the thing to minimize. bb.minimize(u) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(u))
import os from argparse import ArgumentParser from pprint import pprint import numpy as np from matplotlib import pyplot as plt from sklearn import datasets from keras.models import Sequential from keras.layers import Dense from keras.optimizers import SGD from keras.utils import to_categorical from keras.regularizers import l1_l2 # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) # Load data into X and y: iris = datasets.load_iris() X = iris.data y = to_categorical(iris.target) # Split data into training, validation, and testing: train_split = int(.6 * len(X)) validate_split = train_split + int(.2 * len(X)) X_train, X_validate, X_test = X[:train_split], X[ train_split:validate_split], X[validate_split:] y_train, y_validate, y_test = y[:train_split], y[ train_split:validate_split], y[validate_split:]
""" Example of using the pySOT backend and stochastic_radial_basis_function algorithm with BBopt. To run this example, just run: > bbopt ./skopt_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) def run_trial(serving=False): """Run one trial of black box optimization.""" if serving: bb.run(alg="serving") else: bb.run(alg="stochastic_radial_basis_function") # We'll define some parameters x0 = bb.randrange("x0", 1, 11, guess=5) x1 = bb.uniform("x1", 0, 1) x2 = bb.choice("x2", [-10, -1, 0, 1, 10]) # And set our goal y = x0 + x1 * x2 bb.minimize(y) return y def main(num_trials=10):
""" Example of using the hyperopt backend with BBopt. To run this example, just run: > bbopt ./hyperopt_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run(alg="tree_structured_parzen_estimator") # Let's use some parameters! x0 = bb.randint("x0", 1, 10, guess=5) x1 = bb.normalvariate("x1", mu=0, sigma=1) x2 = bb.choice("x2", [-10, -1, 0, 1, 10]) # And let's set our goal! y = x0 + x1 * x2 bb.minimize(y) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(y))
""" Example of using the default "any_fast" algorithm. To run this example, just run: > bbopt ./any_fast_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run() # alg="any_fast" should be the default # We set u ~ dist(0, 1) * sin(dist(0, 1)) where dist is uniform or normal. from math import sin dist = bb.choice("dist", ["uniform", "normal"]) if dist == "normal": u = bb.normalvariate("x0_n", 0, 1) * sin(bb.normalvariate("x1_n", 0, 1)) else: u = bb.random("x0_u") * sin(bb.random("x1_u")) # If we used hyperopt-only parameters, we shouldn't have skopt. if hasattr(bb.backend, "selected_backend"): bb.remember({"backend": bb.backend.selected_backend}) if dist == "normal": assert bb.backend.selected_backend != "scikit-optimize", bb.backend.selected_backend else: bb.remember({"backend": bb.backend.backend_name}) # Set u as the thing to minimize. bb.minimize(u)
from torch.autograd import Variable from tqdm import tqdm import utils import datetime import model.net as net # import model.data_loader as data_loader import model.data_generator as data_generator from evaluate import evaluate from tensorboardX import SummaryWriter from shutil import copy # boilerplate code so that bbopt can run this file from bbopt import BlackBoxOptimizer # will save to avi_hyperparam_train.bbopt.json bb = BlackBoxOptimizer(file=__file__, protocol='json') ### TODO: update the defaults to the actual directories parser = argparse.ArgumentParser() parser.add_argument('--data_dir', default='data/64x64_SIGNS', help="File containing directory containing datasets") # parser.add_argument('--data_dir_list', default=None, # help="File contating list of dataset directories data_dirs") parser.add_argument('--model_dir', default='experiments/base_model', help="Directory containing params.json") parser.add_argument('--tensorboard_prefix', default='', help="prefix for tensorboard logging") parser.add_argument('--prefix',
""" Example of using the random backend with BBopt. To run this example, just run: > bbopt ./random_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run(alg="random") # Let's use some parameters! x = bb.randint("x", 1, 10) # And let's set our goal! bb.maximize(x) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(x)
""" Example of using some of the array-based parameter definition methods. To run this example, just run: > bbopt ./skopt_example.py """ import numpy as np # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run() # Generate 1 x 5 and 5 x 1 random vectors. x0 = bb.rand("x0", 1, 5, guess=np.zeros((1, 5))) # entries uniform in [0, 1) x1 = bb.randn("x1", 5, 1, guess=np.zeros((5, 1))) # entries standard normal # Set the loss to be their dot product. y = float(x0.dot(x1)) bb.minimize(y) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(y))
""" Example of using the scikit-optimize backend and gaussian_process algorithm with BBopt. To run this example, just run: > bbopt ./skopt_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run(alg="gaussian_process") # Let's use some parameters! x0 = bb.randrange("x0", 1, 11, guess=5) x1 = bb.uniform("x1", 0, 1) x2 = bb.choice("x2", [-10, -1, 0, 1, 10]) # And let's set our goal! y = x0 + x1*x2 bb.minimize(y) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(y))
""" Example using a mixture distribution over many different possible algorithms. To run this example, just run: > bbopt ./mixture_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run_backend("mixture", [ ("random", 1), ("tree_structured_parzen_estimator", 1), ("annealing", 1), ("gaussian_process", 1), ("random_forest", 1), ("extra_trees", 1), ("gradient_boosted_regression_trees", 1), ]) # If we're not serving, store which algorithm the # mixture backend has selected. from bbopt.backends.mixture import MixtureBackend if isinstance(bb.backend, MixtureBackend): bb.remember({ "alg": bb.backend.selected_alg, }) # Set up a parameter from a choice and a random sample. xs = bb.sample("xs", range(10), 5, guess=[3, 4, 5, 6, 7])
""" Example of using BBopt with conditional parameters and randomness while leveraging the bayes-skopt backend. To run this example, just run: > bbopt ./bask_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run(alg="bask_gaussian_process") # We set the x parameter conditional on the use_high parameter and add randomness. import random use_high = bb.randbool("use high", guess=False) assert isinstance(use_high, bool), type(use_high) if use_high: x = bb.randrange("x high", 10, 20) * random.random() else: x = bb.randrange("x low", 10) * random.random() # We set x as the thing we want to optimize. bb.maximize(x) # Finally, we'll print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(x))
""" Example of using the default "tpe_or_gp" algorithm. To run this example, just run: > bbopt ./tpe_or_gp_example.py """ # BBopt setup: from bbopt import BlackBoxOptimizer bb = BlackBoxOptimizer(file=__file__) if __name__ == "__main__": bb.run() # alg="tpe_or_gp" should be the default # We set u ~ normal(0, 1) * sin(normal(0, 1)). from math import sin u = bb.normalvariate("x0", 0, 1) * sin(bb.normalvariate("x1", 0, 1)) # Since we used hyperopt-only parameters, we shouldn't have skopt. if hasattr(bb.backend, "selected_backend"): bb.remember({"backend": bb.backend.selected_backend}) assert bb.backend.selected_backend != "scikit-optimize", bb.backend.selected_backend else: bb.remember({"backend": bb.backend.backend_name}) # Set u as the thing to minimize. bb.minimize(u) # Print out the value we used for debugging purposes. if __name__ == "__main__": print(repr(u))