def make_instrumentation(prefix, args, domain_classifier, target_classifier, debug=False): """ Create the possibilities for all of our hyperparameters """ # Possible argument values #batch = inst.var.OrderedDiscrete([2**i for i in range(7, 10)]) # 128 to 512 by powers of 2 lr = inst.var.OrderedDiscrete([10.0**(-i) for i in range(2, 6) ]) # 0.01 to 0.00001 by powers of 10 lr_domain = inst.var.OrderedDiscrete([10.0**(-i) for i in range(2, 6) ]) # same as above lr_target = inst.var.OrderedDiscrete([10.0**(-i) for i in range(2, 6) ]) # same as above # Our "function" (neural net training with output of max validation accuracy) # is a function of the above hyperparameters if not domain_classifier and not target_classifier: instrum = inst.Instrumentation(lr=lr, prefix=prefix, args=args) elif domain_classifier and not target_classifier: instrum = inst.Instrumentation(lr=lr, lr_domain=lr_domain, prefix=prefix, args=args) elif domain_classifier and target_classifier: instrum = inst.Instrumentation(lr=lr, lr_domain=lr_domain, lr_target=lr_target, prefix=prefix, args=args) else: raise NotImplementedError( "unsupported combination of domain/target classifier?") return instrum
def optimize_utility(self, labels, predictions): """ Main function for optimization of a threshold given labels and predictions. """ if isinstance(predictions, pd.Series): predictions = predictions.values # We only want scores correspondent with labels scores = self.scores.loc[labels.index].values # Give bounds instrum = inst.Instrumentation( *[inst.var.Array(1).asscalar().bounded(-0.2, 0.2)]) # Set optimizer optimizer = optimizerlib.TwoPointsDE(instrumentation=instrum, budget=self.budget, num_workers=self.num_workers) # Precompute the inaction and perfect scores inaction_score = scores[:, 0].sum() perfect_score = scores[:, [0, 1]].max(axis=1).sum() # Optimize recommendation = optimizer.optimize(lambda thresh: -self.score_func( scores, predictions, inaction_score, perfect_score, thresh=thresh)) # Get the threshold and return the score threshold = recommendation.args[0] score = self.score_func(scores, predictions, inaction_score, perfect_score, thresh=threshold) return threshold, score
def __init__(self, transform: Optional[str] = None) -> None: super().__init__( self._get_pixel_value, inst.Instrumentation(p.Scalar(), p.Scalar()).set_name("standard")) self.register_initialization(transform=transform) self._image = datasets.get_data("Landscape") if transform == "gaussian": variables = list( p.TransitionChoice(list(range(x))) for x in self._image.shape) self.parametrization = inst.Instrumentation( *variables).with_name("gaussian") elif transform == "square": stds = (np.array(self._image.shape) - 1.) / 2. variables2 = list(inst.var.Gaussian(s, s) for s in stds) self.parametrization = inst.Instrumentation(*variables2).with_name( "square") # maybe buggy, try again? elif transform is not None: raise ValueError(f"Unknown transform {transform}") self._max = float(self._image.max())
def get_nvg_dimensions(api_config): """Help routine to setup nevergrad search space in constructor. Take api_config as argument so this can be static. """ # The ordering of iteration prob makes no difference, but just to be # safe and consistnent with space.py, I will make sorted. param_list = sorted(api_config.keys()) all_args = {} all_prewarp = {} for param_name in param_list: param_config = api_config[param_name] param_type = param_config["type"] param_space = param_config.get("space", None) param_range = param_config.get("range", None) param_values = param_config.get("values", None) prewarp = None if param_type == "cat": assert param_space is None assert param_range is None arg = inst.var.SoftmaxCategorical(param_values) elif param_type == "bool": assert param_space is None assert param_range is None assert param_values is None arg = inst.var.OrderedDiscrete([False, True]) elif param_values is not None: assert param_type in ("int", "ordinal", "real") arg = inst.var.OrderedDiscrete(param_values) # We are throwing away information here, but OrderedDiscrete # appears to be invariant to monotonic transformation anyway. elif param_type == "int": assert param_values is None # Need +1 since API in inclusive choices = range(int(param_range[0]), int(param_range[-1]) + 1) arg = inst.var.OrderedDiscrete(choices) # We are throwing away information here, but OrderedDiscrete # appears to be invariant to monotonic transformation anyway. elif param_type == "real": assert param_values is None assert param_range is not None # Will need to warp to this space sep. arg = inst.var.Gaussian(mean=0, std=1) prewarp = Real(warp=param_space, range_=param_range) else: assert False, "type %s not handled in API" % param_type all_args[param_name] = arg all_prewarp[param_name] = prewarp instrum = inst.Instrumentation(**all_args) return instrum, all_prewarp
def optimize_utility_threshold(predictions, scores=None, idxs=None, budget=200, num_workers=1): """Optimizes the cutoff threshold to maximise the utility score. Sepsis predictions must be binary labels. Dependent on where these are in a patients time-series, we achieve a different utility score for that prediction. Our current methodology involves regressing against the utility function such that our output predictions are number in $\mathbb{R}$ with the expectation that larger values correspond to a higher likelihood of sepsis. To convert onto binary predictions we must choose some cutoff value, `thresh` with which to predict 1 (sepsis) if `pred > thresh` else 0. Given a set of predictions, this function optimizes that `thresh` value such that we would achieve the maximum utility score. This `thresh` value can now be used as the final step in the full model. This function would take a huge amount of time to compute if we did not compute for every patient, the utility of a zero or of a one prediction at each time-point in their series. The downside of this is that we must always specify the indexes from the full dataset from which the predictions correspond to, as we load in this precomputed scores tensor and query at the indexes to get the score. Args: predictions (torch.Tensor): The predictions (or a subset of the predictions) on the data. NOTE: if idxs is specified, predictions must be specified as the subset of predictions corresponding to those indexes. That is predictions[idxs]. idxs (torch.Tensor): The index locations of the predictions in the full dataset. budget (int): The number optimizer iterations. num_workers (int): The number of parallel workers in the optimization. Returns: float: The estimation of the optimal cutoff threshold for which to predict sepsis. """ # Load the full version of the scores if not pre-specified if scores is None: scores = load_pickle(DATA_DIR + '/processed/labels/full_scores.pickle').values if idxs is not None: scores = scores[idxs] # Set optimizer and instrumentation (bounds) instrum = inst.Instrumentation( *[inst.var.Array(1).asscalar().bounded(-0.2, 0.2)]) optimizer = optimizerlib.TwoPointsDE(instrumentation=instrum, budget=budget, num_workers=num_workers) # Optimize recommendation = optimizer.optimize( lambda thresh: -compute_utility(scores, predictions, thresh)) # Get the threshold and return the score threshold = recommendation.args[0] return threshold
def __init__(self, module: nn.Module, deterministic: bool = True, instrumentation_std: float = 0.1) -> None: super().__init__() self.deterministic = deterministic self.module = module kwargs = { name: inst.var.Array(*value.shape).affined(a=instrumentation_std).bounded(-10, 10, transform="arctan") for name, value in module.state_dict().items() # type: ignore } # bounded to avoid overflows self.instrumentation = inst.Instrumentation(**kwargs)
def optimize(self, idx): # Set optimizer and instrumentation (bounds) instrum = inst.Instrumentation(*[inst.var.Array(1).asscalar().bounded(-0.2, 0.2)]) optimizer = optimizerlib.TwoPointsDE(instrumentation=instrum, budget=self.budget, num_workers=self.num_workers) # Optimize recommendation = optimizer.optimize( lambda thresh: -self.score(thresh, idx=idx) ) # Get the threshold and return the score threshold = recommendation.args[0] return threshold
def initialize(self, parameter_names, popsize, rounds, **params): self.tested_parameters = [] self.errors = [] for param in params: if param not in parameter_names: raise ValueError("Parameter %s must be defined as a parameter " "in the model" % param) bounds = calc_bounds(parameter_names, **params) instruments = [] for i, name in enumerate(parameter_names): assert len(bounds[i]) == 2 instrumentation = inst.var.Array(1).asscalar().bounded(np.array([bounds[i][0]]), np.array([bounds[i][1]])) instruments.append(instrumentation) instrum = inst.Instrumentation(*instruments) nevergrad_method = optimizerlib.registry[self.method] if nevergrad_method.no_parallelization and popsize > 1: logger.warn(f'Sample size {popsize} requested, but Nevergrad\'s ' f'\'{self.method}\' algorithm does not support ' f'parallelization. Will run the algorithm ' f'sequentially.', name_suffix='no_parallelization') popsize = 1 budget = rounds*popsize self.optim = nevergrad_method(instrumentation=instrum, num_workers=popsize, budget=budget, **self.kwds) if hasattr(self.optim, 'llambda'): optimizer_pop_size = self.optim.llambda elif hasattr(self.optim, 'es') and hasattr(self.optim.es, 'popsize'): # For CMA algorithm optimizer_pop_size = self.optim.es.popsize else: optimizer_pop_size = popsize if optimizer_pop_size != popsize: logger.warn(f'Sample size {popsize} requested, but Nevergrad\'s ' f'\'{self.method}\' algorithm will use ' f'{optimizer_pop_size}.', name_suffix='sample_size') return optimizer_pop_size
def initialize(self, parameter_names, popsize, **params): self.tested_parameters = [] self.errors = [] for param in params.keys(): if param not in parameter_names: raise ValueError("Parameter %s must be defined as a parameter " "in the model" % param) bounds = calc_bounds(parameter_names, **params) instruments = [] for i, name in enumerate(parameter_names): assert len(bounds[i]) == 2 vars()[name] = inst.var.Array(1).asfloat().bounded(np.array([bounds[i][0]]), np.array([bounds[i][1]])) instruments.append(vars()[name]) instrum = inst.Instrumentation(*instruments) self.optim = optimizerlib.registry[self.method](instrumentation=instrum, **self.kwds) self.optim._llambda = popsize # TODO: more elegant way once possible
def __init__(self) -> None: super().__init__(self._func, inst.Instrumentation())
''' Test the optimizer class ''' import numpy as np from numpy.testing.utils import assert_equal, assert_raises from brian2modelfitting import Optimizer, NevergradOptimizer, SkoptOptimizer, calc_bounds from skopt import Optimizer as SOptimizer from nevergrad import instrumentation as inst from nevergrad.optimization.base import Optimizer as NOptimzer from nevergrad.optimization.base import CandidateMaker arg1 = inst.var.Array(1).bounded(0, 1).asscalar() arg2 = inst.var.Array(1).bounded(0, 2).asscalar() arg3 = inst.var.Array(1).bounded(0, 3).asscalar() instrum = inst.Instrumentation(arg1, arg2, arg3) CM = CandidateMaker(instrum) def test_init(): # Optimizer() NevergradOptimizer() SkoptOptimizer() NevergradOptimizer(method='DE') SkoptOptimizer(method='GP') def test_init_wrong_method(): assert_raises(AssertionError, NevergradOptimizer, method='foo') assert_raises(AssertionError, SkoptOptimizer, method='foo')
def find_super_params(): # Instrumentation # argument transformation """ def find_super_params(patch_size=8, num_patches=1, loc_hidden=256, glimpse_hidden=128, num_glimpses=6, std=0.17, M=10, valid_size=0.1, batch_size=256, batchnorm_phi=True, batchnorm_l=True, batchnorm_g=True, batchnorm_h=True, glimpse_scale=2, weight_decay=0, dropout_phi=0, dropout_l=0, dropout_g=0, dropout_h=0): When optimizing hyperparameters as e.g. in machine learning. If you don't know what variables (see instrumentation) to use: use SoftmaxCategorical for discrete variables use TwoPointsDE with num_workers equal to the number of workers available to you. See the machine learning example for more. Or if you want something more aimed at robustly outperforming random search in highly parallel settings (one-shot): use OrderedDiscrete for discrete variables, taking care that the default value is in the middle. Use ScrHammersleySearchPlusMiddlePoint (PlusMiddlePoint only if you have continuous parameters or good default values for discrete parameters). """ # dicrete batch_size = inst.var.SoftmaxCategorical([256, 512, 1024]) patch_size = inst.var.SoftmaxCategorical(np.arange(10, 17, 2).tolist()) #num_patches = inst.var.SoftmaxCategorical(np.arange(1,3).tolist()) num_glimpses = inst.var.SoftmaxCategorical(np.arange(5, 17, 2).tolist()) glimpse_hidden = inst.var.SoftmaxCategorical([128, 256, 512]) loc_hidden = inst.var.SoftmaxCategorical([128, 256]) #batchnorm_phi = inst.var.SoftmaxCategorical(["True", "False"]) #batchnorm_l = inst.var.SoftmaxCategorical(["True", "False"]) #batchnorm_g = inst.var.SoftmaxCategorical(["True", "False"]) #batchnorm_h = inst.var.SoftmaxCategorical(["True", "False"]) # not available in built-in RNN #glimpse_scale = inst.var.SoftmaxCategorical(np.arange(1,3,1).tolist()) # weight_decay = inst.var.SoftmaxCategorical(np.arange(0.0001,0.05,0.0005).tolist()) # dropout_phi = inst.var.SoftmaxCategorical(np.arange(0.1,0.3,0.1).tolist()) # dropout_l = inst.var.SoftmaxCategorical(np.arange(0,0.4,0.1).tolist()) #dropout_g = inst.var.SoftmaxCategorical(np.arange(0,0.5,0.1).tolist()) # dropout_h = inst.var.SoftmaxCategorical([0,0.3]) # alpha = inst.var.SoftmaxCategorical(np.arange(1,2,0.1).tolist()) # gamma = inst.var.SoftmaxCategorical(np.arange(0.1,2,0.1).tolist()) # continuous; the Gaussian method does not have truncated version, so it is unavailable # glimpse_scale = inst.var.Gaussian(mean=2, std=2) # weight_decay = inst.var.Gaussian(mean=0.001, std=0.001) # dropout_phi = inst.var.Gaussian(mean=0.5, std=2) # dropout_l = inst.var.Gaussian(mean=0.5, std=2) # dropout_g = inst.var.Gaussian(mean=0.5, std=2) # dropout_h = inst.var.Gaussian(mean=0.5, std=2) """ def find_super_params(patch_size=8, num_patches=1, loc_hidden=256, glimpse_hidden=128, num_glimpses=6, std=0.17, M=10, valid_size=0.1, batch_size=256, batchnorm_phi=True, batchnorm_l=True, batchnorm_g=True, batchnorm_h=True, glimpse_scale=2, weight_decay=0, dropout_phi=0, dropout_l=0, dropout_g=0, dropout_h=0, , alpha=1.4, gamma=0.9): """ # create the instrumented function # put them in order, if it is discrete varible, only give the variable name; if it is continuous, give a pair; # if it is constant, only give the constant instrum = inst.Instrumentation(patch_size, 2, loc_hidden, glimpse_hidden, num_glimpses, 0.17, 10, 0.1, batch_size, 'True', 'True', 'True', 'True', 2, 0, 0, 0, 0, 0, 1.4, 0.9) print(instrum.dimension) #Converts data to arguments to check #print(instrum.data_to_arguments([1, -80, -80, 80, 3])) # prints (args, kwargs): (('b', 'e', 'blublu'), {'value': 7}) # b is selected because 1 > 0 (the threshold is 0 here since there are 2 values. # e is selected because proba(e) = exp(80) / (exp(80) + exp(-80) + exp(-80)) # value=7 because 3 * std + mean = 7 # create the instrumented function using the "Instrumentation" instance above ifunc = instrum.instrument(call_rva) print(ifunc.dimension) # dimensional space as above # you can still access the instrumentation instance will ifunc.instrumentation optimizer = optimizerlib.PortfolioDiscreteOnePlusOne( dimension=ifunc.dimension, budget=20, num_workers=2) #TwoPointsDE # simple example #recommendation = optimizer.optimize(ifunc) # using several workers from concurrent import futures with futures.ProcessPoolExecutor( max_workers=optimizer.num_workers) as executor: recommendation = optimizer.optimize(ifunc, executor=executor, batch_mode=False) # # recover the arguments this way (don't forget deteriministic=True) # args, kwargs = ifunc.data_to_arguments(recommendation, deterministic=True) # print(args) # should print ["b", "e", "blublu"] # print(kwargs) # should print {"value": 0} because -.5 * std + mean = 0 # but be careful, since some variables are stochastic (SoftmaxCategorical ones are), setting deterministic=False may yield different results # The following will print more information on the conversion to your arguments: print(ifunc.get_summary(recommendation))
latent_size=instru.var.OrderedDiscrete([64, 128, 256, 512]) n_frames=instru.var.OrderedDiscrete([300, 400, 500, 600, 800]) model=instru.var.OrderedDiscrete(['resnet_mfcc', 'resnet_34', 'resnet_lstm', 'resnet_qrnn', 'resnet_stats', 'resnet_large', 'resnet_small', 'TDNN', 'TDNN_att', 'TDNN_multihead', 'TDNN_lstm', 'TDNN_aspp', 'TDNN_mod', 'transformer']) if args.model=='all' else args.model ncoef=args.ncoef epochs=args.epochs batch_size=args.batch_size valid_batch_size=args.valid_batch_size n_workers=args.workers cuda=args.cuda train_hdf_file=args.train_hdf_file valid_hdf_file=args.valid_hdf_file slurm_sub_file=args.slurm_sub_file checkpoint_path=args.checkpoint_path softmax=instru.var.OrderedDiscrete(['softmax', 'am_softmax']) delta=instru.var.OrderedDiscrete([True, False]) logdir=args.logdir tmp_dir = os.getcwd() + '/' + args.temp_folder + '/' if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) instrum=instru.Instrumentation(lr, l2, max_gnorm, momentum, margin, lambda_, swap, latent_size, n_frames, model, ncoef, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, slurm_sub_file, tmp_dir, checkpoint_path, softmax, delta, logdir) hp_optimizer=optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2)) shutil.rmtree(tmp_dir)
lr = instru.var.OrderedDiscrete([0.5, 0.1, 0.01, 0.001]) l2 = instru.var.OrderedDiscrete([1e-2, 1e-3, 1e-4, 1e-5]) momentum = instru.var.OrderedDiscrete([0.1, 0.5, 0.9]) smoothing = instru.var.OrderedDiscrete([0.01, 0.1, 0.2]) patience = instru.var.OrderedDiscrete([1, 5, 10, 20]) n_hidden = instru.var.OrderedDiscrete([2, 3, 4, 5]) hidden_size = instru.var.OrderedDiscrete([128, 256, 350, 512]) dropout_prob = instru.var.OrderedDiscrete([0.01, 0.1, 0.2]) model = args.model epochs = args.epochs batch_size = args.batch_size valid_batch_size = args.valid_batch_size n_workers = args.n_workers cuda = args.cuda data_path = args.data_path valid_data_path = args.valid_data_path checkpoint_path = args.checkpoint_path softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax']) instrum = instru.Instrumentation(lr, l2, momentum, smoothing, patience, model, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path, softmax) hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget) print(hp_optimizer.optimize(train, verbosity=2))
valid_hdf_path = args.valid_hdf_path if args.valid_hdf_path else 'none' sub_file = args.sub_file checkpoint_path = args.checkpoint_path softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax']) n_classes = args.nclasses pretrained = args.pretrained max_gnorm = instru.var.OrderedDiscrete([10, 30, 100]) lr_factor = instru.var.OrderedDiscrete([0.1, 0.3, 0.5, 0.8]) r_proj = instru.var.OrderedDiscrete([0., 1., 1.5, 1.8]) logdir = args.logdir ablation = args.ablation tmp_dir = os.getcwd() + '/' + args.temp_folder + '/' if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) instrum = instru.Instrumentation( lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, hdf_path, valid_hdf_path, sub_file, checkpoint_path, softmax, n_classes, pretrained, max_gnorm, lr_factor, r_proj, logdir, ablation) hp_optimizer = optimization.optimizerlib.RandomSearch( instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2)) shutil.rmtree(tmp_dir)
data_path = args.data_path if args.data_path is not None else 'none' hdf_path = args.hdf_path if args.hdf_path is not None else 'none' valid_data_path = args.valid_data_path if args.valid_data_path is not None else 'none' valid_hdf_path = args.valid_hdf_path if args.valid_hdf_path is not None else 'none' checkpoint_path = args.checkpoint_path softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax']) pretrained = args.pretrained if len(args.pretrained_path) > 1: pretrained_path = instru.var.OrderedDiscrete(args.pretrained_path) elif len(args.pretrained_path) == 1: pretrained_path = args.pretrained_path else: pretrained_path = 'none' max_gnorm = instru.var.OrderedDiscrete([10, 50, 100]) stats = args.stats log_dir = args.logdir if args.logdir else 'none' eval_every = args.eval_every ablation = args.ablation instrum = instru.Instrumentation( lr, l2, momentum, smoothing, warmup, model, emb_size, n_hidden, hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, hdf_path, valid_hdf_path, checkpoint_path, softmax, pretrained, pretrained_path, max_gnorm, stats, log_dir, eval_every, ablation) hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget) print(hp_optimizer.optimize(train, verbosity=2))
lr = instru.var.Array(1).asfloat().bounded(1, 4).exponentiated(base=10, coeff=-1) l2 = instru.var.Array(1).asfloat().bounded(1, 5).exponentiated(base=10, coeff=-1) momentum = instru.var.Array(1).asfloat().bounded(0.10, 0.95) margin = instru.var.Array(1).asfloat().bounded(0.10, 1.00) lambda_ = instru.var.Array(1).asfloat().bounded(1, 5).exponentiated(base=10, coeff=-1) patience = instru.var.Array(1).asfloat().bounded(1, 100) swap = instru.var.OrderedDiscrete([True, False]) model = args.model epochs = args.epochs batch_size = args.batch_size valid_batch_size = args.valid_batch_size n_workers = args.n_workers cuda = args.cuda data_path = args.data_path valid_data_path = args.valid_data_path checkpoint_path = args.checkpoint_path instrum = instru.Instrumentation(lr, l2, momentum, margin, lambda_, patience, swap, model, epochs, batch_size, valid_batch_size, n_workers, cuda, data_path, valid_data_path, checkpoint_path) hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget) print(hp_optimizer.optimize(train, verbosity=2))
max_gnorm = instru.var.OrderedDiscrete([10.0, 100.0, 1.0]) warmup = instru.var.OrderedDiscrete([1, 500, 2000, 5000]) input_size = args.input_size n_hidden = instru.var.OrderedDiscrete([1, 2, 3, 4, 5, 6, 8, 10]) hidden_size = instru.var.OrderedDiscrete([64, 128, 256, 512, 1024]) dropout_prob = instru.var.OrderedDiscrete([0.01, 0.1, 0.2, 0.4]) smoothing = instru.var.OrderedDiscrete([0.0, 0.01, 0.1, 0.2]) n_cycles = args.n_cycles epochs = args.epochs batch_size = args.batch_size valid_batch_size = args.valid_batch_size n_workers = args.workers cuda = args.cuda train_hdf_path = args.train_hdf_path valid_hdf_path = args.valid_hdf_path cp_path = args.checkpoint_path logdir = args.logdir instrum = instru.Instrumentation(lr, l2, momentum, max_gnorm, warmup, input_size, n_hidden, hidden_size, dropout_prob, smoothing, n_cycles, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_path, valid_hdf_path, cp_path, logdir) hp_optimizer = optimization.optimizerlib.RandomSearch( instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2))
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. from pathlib import Path from nevergrad import optimization from nevergrad import instrumentation as inst if __name__ == "__main__": folder = Path(__file__).parents[1] / "instrumentation" / "examples" func = inst.FolderFunction(folder, ["python", "examples/script.py"], clean_copy=True) instrumentation = inst.Instrumentation(value1=inst.var.Array(1).asscalar(), value2=12, string=inst.var.SoftmaxCategorical( ["plop", "blublu", "plouf"])) opt = optimization.registry["OnePlusOne"](instrumentation, budget=4) opt.optimize(func)
batch_size = args.batch_size valid_batch_size = args.valid_batch_size n_workers = args.workers cuda = args.cuda train_hdf_file = args.train_hdf_file valid_hdf_file = args.valid_hdf_file sub_file = args.sub_file checkpoint_path = args.checkpoint_path softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax']) max_gnorm = instru.var.OrderedDiscrete([10.0, 20.0, 50.0]) logdir = args.logdir tmp_dir = os.getcwd() + '/' + args.temp_folder + '/' if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) instrum = instru.Instrumentation( lr, l2, momentum, smoothing, warmup, latent_size, n_hidden, hidden_size, n_frames, model, ndiscriminators, rproj_size, ncoef, dropout_prob, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, sub_file, tmp_dir, checkpoint_path, softmax, max_gnorm, logdir) hp_optimizer = optimization.optimizerlib.RandomSearch( instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2)) shutil.rmtree(tmp_dir)
model = instru.var.OrderedDiscrete([ 'resnet_mfcc', 'resnet_lstm', 'resnet_stats', 'inception_mfcc', 'resnet_large', 'resnet_small' ]) if args.model == 'all' else args.model ncoef = args.ncoef epochs = args.epochs batch_size = args.batch_size n_workers = args.workers cuda = args.cuda train_hdf_file = args.train_hdf_file data_info_path = args.data_info_path valid_hdf_file = args.valid_hdf_file n_cycles = args.n_cycles valid_n_cycles = args.valid_n_cycles checkpoint_path = args.checkpoint_path softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax']) delta = instru.var.OrderedDiscrete([True, False]) instrum = instru.Instrumentation(lr, l2, momentum, margin, lambda_, patience, swap, latent_size, n_frames, model, ncoef, epochs, batch_size, n_workers, cuda, train_hdf_file, data_info_path, valid_hdf_file, n_cycles, valid_n_cycles, checkpoint_path, softmax, delta) hp_optimizer = optimization.optimizerlib.RandomSearch( instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor))
hidden_size=instru.var.Array(1).asfloat().bounded(64, 512) dropout_prob=instru.var.Array(1).asfloat().bounded(0.01, 0.50) n_frames=instru.var.Array(1).asfloat().bounded(600, 1000) model=instru.var.OrderedDiscrete(['resnet_mfcc', 'resnet_lstm', 'resnet_stats', 'resnet_small']) if args.model=='all' else args.model ncoef=args.ncoef epochs=args.epochs batch_size=args.batch_size n_workers=args.workers cuda=args.cuda train_hdf_file=args.train_hdf_file valid_hdf_file=args.valid_hdf_file n_cycles=args.n_cycles valid_n_cycles=args.valid_n_cycles sub_file=args.sub_file checkpoint_path=args.checkpoint_path softmax=instru.var.OrderedDiscrete(['softmax', 'am_softmax']) tmp_dir = os.getcwd() + '/' + args.temp_folder + '/' if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) instrum=instru.Instrumentation(lr, l2, momentum, patience, latent_size, n_hidden, hidden_size, n_frames, model, ncoef, dropout_prob, epochs, batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, n_cycles, valid_n_cycles, sub_file, tmp_dir, checkpoint_path, softmax) hp_optimizer=optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2)) shutil.rmtree(tmp_dir)
lr = instru.var.Array(1).asfloat().bounded(1, 4).exponentiated(base=10, coeff=-1) l2 = instru.var.Array(1).asfloat().bounded(1, 5).exponentiated(base=10, coeff=-1) momentum = instru.var.Array(1).asfloat().bounded(0.01, 0.99) slack = instru.var.Array(1).asfloat().bounded(1.01, 3.00) patience = instru.var.Array(1).asfloat().bounded(1, 100) model = instru.var.OrderedDiscrete(['vgg', 'resnet', 'densenet' ]) if args.model == 'all' else args.model epochs = args.epochs batch_size = args.batch_size valid_batch_size = args.valid_batch_size train_mode = instru.var.OrderedDiscrete( ['vanilla', 'hyper']) if args.train_mode == 'all' else args.train_mode n_workers = args.n_workers cuda = args.cuda data_path = args.data_path valid_data_path = args.valid_data_path checkpoint_path = args.checkpoint_path instrum = instru.Instrumentation(lr, l2, momentum, slack, patience, model, epochs, batch_size, valid_batch_size, train_mode, n_workers, cuda, data_path, valid_data_path, checkpoint_path) hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget) print(hp_optimizer.optimize(train, verbosity=2))