def make_instrumentation(prefix,
args,
domain_classifier,
target_classifier,
debug=False):
""" Create the possibilities for all of our hyperparameters """
# Possible argument values
#batch = inst.var.OrderedDiscrete([2**i for i in range(7, 10)]) # 128 to 512 by powers of 2
lr = inst.var.OrderedDiscrete([10.0**(-i) for i in range(2, 6)
]) # 0.01 to 0.00001 by powers of 10
lr_domain = inst.var.OrderedDiscrete([10.0**(-i) for i in range(2, 6)
]) # same as above
lr_target = inst.var.OrderedDiscrete([10.0**(-i) for i in range(2, 6)
]) # same as above
# Our "function" (neural net training with output of max validation accuracy)
# is a function of the above hyperparameters
if not domain_classifier and not target_classifier:
instrum = inst.Instrumentation(lr=lr, prefix=prefix, args=args)
elif domain_classifier and not target_classifier:
instrum = inst.Instrumentation(lr=lr,
lr_domain=lr_domain,
prefix=prefix,
args=args)
elif domain_classifier and target_classifier:
instrum = inst.Instrumentation(lr=lr,
lr_domain=lr_domain,
lr_target=lr_target,
prefix=prefix,
args=args)
else:
raise NotImplementedError(
"unsupported combination of domain/target classifier?")
return instrum
def optimize_utility(self, labels, predictions):
""" Main function for optimization of a threshold given labels and predictions. """
if isinstance(predictions, pd.Series):
predictions = predictions.values
# We only want scores correspondent with labels
scores = self.scores.loc[labels.index].values
# Give bounds
instrum = inst.Instrumentation(
*[inst.var.Array(1).asscalar().bounded(-0.2, 0.2)])
# Set optimizer
optimizer = optimizerlib.TwoPointsDE(instrumentation=instrum,
budget=self.budget,
num_workers=self.num_workers)
# Precompute the inaction and perfect scores
inaction_score = scores[:, 0].sum()
perfect_score = scores[:, [0, 1]].max(axis=1).sum()
# Optimize
recommendation = optimizer.optimize(lambda thresh: -self.score_func(
scores, predictions, inaction_score, perfect_score, thresh=thresh))
# Get the threshold and return the score
threshold = recommendation.args[0]
score = self.score_func(scores,
predictions,
inaction_score,
perfect_score,
thresh=threshold)
return threshold, score
def __init__(self, transform: Optional[str] = None) -> None:
super().__init__(
self._get_pixel_value,
inst.Instrumentation(p.Scalar(), p.Scalar()).set_name("standard"))
self.register_initialization(transform=transform)
self._image = datasets.get_data("Landscape")
if transform == "gaussian":
variables = list(
p.TransitionChoice(list(range(x))) for x in self._image.shape)
self.parametrization = inst.Instrumentation(
*variables).with_name("gaussian")
elif transform == "square":
stds = (np.array(self._image.shape) - 1.) / 2.
variables2 = list(inst.var.Gaussian(s, s) for s in stds)
self.parametrization = inst.Instrumentation(*variables2).with_name(
"square") # maybe buggy, try again?
elif transform is not None:
raise ValueError(f"Unknown transform {transform}")
self._max = float(self._image.max())
def get_nvg_dimensions(api_config):
"""Help routine to setup nevergrad search space in constructor.
Take api_config as argument so this can be static.
"""
# The ordering of iteration prob makes no difference, but just to be
# safe and consistnent with space.py, I will make sorted.
param_list = sorted(api_config.keys())
all_args = {}
all_prewarp = {}
for param_name in param_list:
param_config = api_config[param_name]
param_type = param_config["type"]
param_space = param_config.get("space", None)
param_range = param_config.get("range", None)
param_values = param_config.get("values", None)
prewarp = None
if param_type == "cat":
assert param_space is None
assert param_range is None
arg = inst.var.SoftmaxCategorical(param_values)
elif param_type == "bool":
assert param_space is None
assert param_range is None
assert param_values is None
arg = inst.var.OrderedDiscrete([False, True])
elif param_values is not None:
assert param_type in ("int", "ordinal", "real")
arg = inst.var.OrderedDiscrete(param_values)
# We are throwing away information here, but OrderedDiscrete
# appears to be invariant to monotonic transformation anyway.
elif param_type == "int":
assert param_values is None
# Need +1 since API in inclusive
choices = range(int(param_range[0]), int(param_range[-1]) + 1)
arg = inst.var.OrderedDiscrete(choices)
# We are throwing away information here, but OrderedDiscrete
# appears to be invariant to monotonic transformation anyway.
elif param_type == "real":
assert param_values is None
assert param_range is not None
# Will need to warp to this space sep.
arg = inst.var.Gaussian(mean=0, std=1)
prewarp = Real(warp=param_space, range_=param_range)
else:
assert False, "type %s not handled in API" % param_type
all_args[param_name] = arg
all_prewarp[param_name] = prewarp
instrum = inst.Instrumentation(**all_args)
return instrum, all_prewarp
def optimize_utility_threshold(predictions,
scores=None,
idxs=None,
budget=200,
num_workers=1):
"""Optimizes the cutoff threshold to maximise the utility score.
Sepsis predictions must be binary labels. Dependent on where these are in a patients time-series, we achieve a
different utility score for that prediction. Our current methodology involves regressing against the utility
function such that our output predictions are number in $\mathbb{R}$ with the expectation that larger values
correspond to a higher likelihood of sepsis. To convert onto binary predictions we must choose some cutoff value,
`thresh` with which to predict 1 (sepsis) if `pred > thresh` else 0. Given a set of predictions, this function
optimizes that `thresh` value such that we would achieve the maximum utility score. This `thresh` value can now be
used as the final step in the full model.
This function would take a huge amount of time to compute if we did not compute for every patient, the utility
of a zero or of a one prediction at each time-point in their series. The downside of this is that we must always
specify the indexes from the full dataset from which the predictions correspond to, as we load in this precomputed
scores tensor and query at the indexes to get the score.
Args:
predictions (torch.Tensor): The predictions (or a subset of the predictions) on the data. NOTE: if idxs is
specified, predictions must be specified as the subset of predictions corresponding to those indexes. That
is predictions[idxs].
idxs (torch.Tensor): The index locations of the predictions in the full dataset.
budget (int): The number optimizer iterations.
num_workers (int): The number of parallel workers in the optimization.
Returns:
float: The estimation of the optimal cutoff threshold for which to predict sepsis.
"""
# Load the full version of the scores if not pre-specified
if scores is None:
scores = load_pickle(DATA_DIR +
'/processed/labels/full_scores.pickle').values
if idxs is not None:
scores = scores[idxs]
# Set optimizer and instrumentation (bounds)
instrum = inst.Instrumentation(
*[inst.var.Array(1).asscalar().bounded(-0.2, 0.2)])
optimizer = optimizerlib.TwoPointsDE(instrumentation=instrum,
budget=budget,
num_workers=num_workers)
# Optimize
recommendation = optimizer.optimize(
lambda thresh: -compute_utility(scores, predictions, thresh))
# Get the threshold and return the score
threshold = recommendation.args[0]
return threshold
def __init__(self, module: nn.Module,
deterministic: bool = True,
instrumentation_std: float = 0.1) -> None:
super().__init__()
self.deterministic = deterministic
self.module = module
kwargs = {
name: inst.var.Array(*value.shape).affined(a=instrumentation_std).bounded(-10, 10, transform="arctan")
for name, value in module.state_dict().items() # type: ignore
} # bounded to avoid overflows
self.instrumentation = inst.Instrumentation(**kwargs)
def optimize(self, idx):
# Set optimizer and instrumentation (bounds)
instrum = inst.Instrumentation(*[inst.var.Array(1).asscalar().bounded(-0.2, 0.2)])
optimizer = optimizerlib.TwoPointsDE(instrumentation=instrum, budget=self.budget, num_workers=self.num_workers)
# Optimize
recommendation = optimizer.optimize(
lambda thresh: -self.score(thresh, idx=idx)
)
# Get the threshold and return the score
threshold = recommendation.args[0]
return threshold
def initialize(self, parameter_names, popsize, rounds, **params):
self.tested_parameters = []
self.errors = []
for param in params:
if param not in parameter_names:
raise ValueError("Parameter %s must be defined as a parameter "
"in the model" % param)
bounds = calc_bounds(parameter_names, **params)
instruments = []
for i, name in enumerate(parameter_names):
assert len(bounds[i]) == 2
instrumentation = inst.var.Array(1).asscalar().bounded(np.array([bounds[i][0]]),
np.array([bounds[i][1]]))
instruments.append(instrumentation)
instrum = inst.Instrumentation(*instruments)
nevergrad_method = optimizerlib.registry[self.method]
if nevergrad_method.no_parallelization and popsize > 1:
logger.warn(f'Sample size {popsize} requested, but Nevergrad\'s '
f'\'{self.method}\' algorithm does not support '
f'parallelization. Will run the algorithm '
f'sequentially.',
name_suffix='no_parallelization')
popsize = 1
budget = rounds*popsize
self.optim = nevergrad_method(instrumentation=instrum,
num_workers=popsize,
budget=budget,
**self.kwds)
if hasattr(self.optim, 'llambda'):
optimizer_pop_size = self.optim.llambda
elif hasattr(self.optim, 'es') and hasattr(self.optim.es, 'popsize'):
# For CMA algorithm
optimizer_pop_size = self.optim.es.popsize
else:
optimizer_pop_size = popsize
if optimizer_pop_size != popsize:
logger.warn(f'Sample size {popsize} requested, but Nevergrad\'s '
f'\'{self.method}\' algorithm will use '
f'{optimizer_pop_size}.', name_suffix='sample_size')
return optimizer_pop_size
def initialize(self, parameter_names, popsize, **params):
self.tested_parameters = []
self.errors = []
for param in params.keys():
if param not in parameter_names:
raise ValueError("Parameter %s must be defined as a parameter "
"in the model" % param)
bounds = calc_bounds(parameter_names, **params)
instruments = []
for i, name in enumerate(parameter_names):
assert len(bounds[i]) == 2
vars()[name] = inst.var.Array(1).asfloat().bounded(np.array([bounds[i][0]]),
np.array([bounds[i][1]]))
instruments.append(vars()[name])
instrum = inst.Instrumentation(*instruments)
self.optim = optimizerlib.registry[self.method](instrumentation=instrum,
**self.kwds)
self.optim._llambda = popsize # TODO: more elegant way once possible
def __init__(self) -> None:
super().__init__(self._func, inst.Instrumentation())
'''
Test the optimizer class
'''
import numpy as np
from numpy.testing.utils import assert_equal, assert_raises
from brian2modelfitting import Optimizer, NevergradOptimizer, SkoptOptimizer, calc_bounds
from skopt import Optimizer as SOptimizer
from nevergrad import instrumentation as inst
from nevergrad.optimization.base import Optimizer as NOptimzer
from nevergrad.optimization.base import CandidateMaker
arg1 = inst.var.Array(1).bounded(0, 1).asscalar()
arg2 = inst.var.Array(1).bounded(0, 2).asscalar()
arg3 = inst.var.Array(1).bounded(0, 3).asscalar()
instrum = inst.Instrumentation(arg1, arg2, arg3)
CM = CandidateMaker(instrum)
def test_init():
# Optimizer()
NevergradOptimizer()
SkoptOptimizer()
NevergradOptimizer(method='DE')
SkoptOptimizer(method='GP')
def test_init_wrong_method():
assert_raises(AssertionError, NevergradOptimizer, method='foo')
assert_raises(AssertionError, SkoptOptimizer, method='foo')
def find_super_params():
# Instrumentation
# argument transformation
"""
def find_super_params(patch_size=8, num_patches=1, loc_hidden=256, glimpse_hidden=128,
num_glimpses=6, std=0.17, M=10, valid_size=0.1, batch_size=256, batchnorm_phi=True,
batchnorm_l=True, batchnorm_g=True, batchnorm_h=True, glimpse_scale=2, weight_decay=0,
dropout_phi=0, dropout_l=0, dropout_g=0, dropout_h=0):
When optimizing hyperparameters as e.g. in machine learning. If you don't know what variables (see instrumentation) to use:
use SoftmaxCategorical for discrete variables
use TwoPointsDE with num_workers equal to the number of workers available to you. See the machine learning example for more.
Or if you want something more aimed at robustly outperforming random search in highly parallel settings (one-shot):
use OrderedDiscrete for discrete variables, taking care that the default value is in the middle.
Use ScrHammersleySearchPlusMiddlePoint (PlusMiddlePoint only if you have continuous parameters or good default values for discrete parameters).
"""
# dicrete
batch_size = inst.var.SoftmaxCategorical([256, 512, 1024])
patch_size = inst.var.SoftmaxCategorical(np.arange(10, 17, 2).tolist())
#num_patches = inst.var.SoftmaxCategorical(np.arange(1,3).tolist())
num_glimpses = inst.var.SoftmaxCategorical(np.arange(5, 17, 2).tolist())
glimpse_hidden = inst.var.SoftmaxCategorical([128, 256, 512])
loc_hidden = inst.var.SoftmaxCategorical([128, 256])
#batchnorm_phi = inst.var.SoftmaxCategorical(["True", "False"])
#batchnorm_l = inst.var.SoftmaxCategorical(["True", "False"])
#batchnorm_g = inst.var.SoftmaxCategorical(["True", "False"])
#batchnorm_h = inst.var.SoftmaxCategorical(["True", "False"]) # not available in built-in RNN
#glimpse_scale = inst.var.SoftmaxCategorical(np.arange(1,3,1).tolist())
# weight_decay = inst.var.SoftmaxCategorical(np.arange(0.0001,0.05,0.0005).tolist())
# dropout_phi = inst.var.SoftmaxCategorical(np.arange(0.1,0.3,0.1).tolist())
# dropout_l = inst.var.SoftmaxCategorical(np.arange(0,0.4,0.1).tolist())
#dropout_g = inst.var.SoftmaxCategorical(np.arange(0,0.5,0.1).tolist())
# dropout_h = inst.var.SoftmaxCategorical([0,0.3])
# alpha = inst.var.SoftmaxCategorical(np.arange(1,2,0.1).tolist())
# gamma = inst.var.SoftmaxCategorical(np.arange(0.1,2,0.1).tolist())
# continuous; the Gaussian method does not have truncated version, so it is unavailable
# glimpse_scale = inst.var.Gaussian(mean=2, std=2)
# weight_decay = inst.var.Gaussian(mean=0.001, std=0.001)
# dropout_phi = inst.var.Gaussian(mean=0.5, std=2)
# dropout_l = inst.var.Gaussian(mean=0.5, std=2)
# dropout_g = inst.var.Gaussian(mean=0.5, std=2)
# dropout_h = inst.var.Gaussian(mean=0.5, std=2)
"""
def find_super_params(patch_size=8, num_patches=1, loc_hidden=256, glimpse_hidden=128,
num_glimpses=6, std=0.17, M=10, valid_size=0.1, batch_size=256, batchnorm_phi=True,
batchnorm_l=True, batchnorm_g=True, batchnorm_h=True, glimpse_scale=2, weight_decay=0,
dropout_phi=0, dropout_l=0, dropout_g=0, dropout_h=0, , alpha=1.4, gamma=0.9):
"""
# create the instrumented function
# put them in order, if it is discrete varible, only give the variable name; if it is continuous, give a pair;
# if it is constant, only give the constant
instrum = inst.Instrumentation(patch_size, 2, loc_hidden, glimpse_hidden,
num_glimpses, 0.17, 10, 0.1, batch_size,
'True', 'True', 'True', 'True', 2, 0, 0, 0,
0, 0, 1.4, 0.9)
print(instrum.dimension)
#Converts data to arguments to check
#print(instrum.data_to_arguments([1, -80, -80, 80, 3]))
# prints (args, kwargs): (('b', 'e', 'blublu'), {'value': 7})
# b is selected because 1 > 0 (the threshold is 0 here since there are 2 values.
# e is selected because proba(e) = exp(80) / (exp(80) + exp(-80) + exp(-80))
# value=7 because 3 * std + mean = 7
# create the instrumented function using the "Instrumentation" instance above
ifunc = instrum.instrument(call_rva)
print(ifunc.dimension) # dimensional space as above
# you can still access the instrumentation instance will ifunc.instrumentation
optimizer = optimizerlib.PortfolioDiscreteOnePlusOne(
dimension=ifunc.dimension, budget=20, num_workers=2) #TwoPointsDE
# simple example
#recommendation = optimizer.optimize(ifunc)
# using several workers
from concurrent import futures
with futures.ProcessPoolExecutor(
max_workers=optimizer.num_workers) as executor:
recommendation = optimizer.optimize(ifunc,
executor=executor,
batch_mode=False)
# # recover the arguments this way (don't forget deteriministic=True)
# args, kwargs = ifunc.data_to_arguments(recommendation, deterministic=True)
# print(args) # should print ["b", "e", "blublu"]
# print(kwargs) # should print {"value": 0} because -.5 * std + mean = 0
# but be careful, since some variables are stochastic (SoftmaxCategorical ones are), setting deterministic=False may yield different results
# The following will print more information on the conversion to your arguments:
print(ifunc.get_summary(recommendation))
latent_size=instru.var.OrderedDiscrete([64, 128, 256, 512]) n_frames=instru.var.OrderedDiscrete([300, 400, 500, 600, 800]) model=instru.var.OrderedDiscrete(['resnet_mfcc', 'resnet_34', 'resnet_lstm', 'resnet_qrnn', 'resnet_stats', 'resnet_large', 'resnet_small', 'TDNN', 'TDNN_att', 'TDNN_multihead', 'TDNN_lstm', 'TDNN_aspp', 'TDNN_mod', 'transformer']) if args.model=='all' else args.model ncoef=args.ncoef epochs=args.epochs batch_size=args.batch_size valid_batch_size=args.valid_batch_size n_workers=args.workers cuda=args.cuda train_hdf_file=args.train_hdf_file valid_hdf_file=args.valid_hdf_file slurm_sub_file=args.slurm_sub_file checkpoint_path=args.checkpoint_path softmax=instru.var.OrderedDiscrete(['softmax', 'am_softmax']) delta=instru.var.OrderedDiscrete([True, False]) logdir=args.logdir tmp_dir = os.getcwd() + '/' + args.temp_folder + '/' if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) instrum=instru.Instrumentation(lr, l2, max_gnorm, momentum, margin, lambda_, swap, latent_size, n_frames, model, ncoef, epochs, batch_size, valid_batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, slurm_sub_file, tmp_dir, checkpoint_path, softmax, delta, logdir) hp_optimizer=optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2)) shutil.rmtree(tmp_dir)
lr = instru.var.OrderedDiscrete([0.5, 0.1, 0.01, 0.001])
l2 = instru.var.OrderedDiscrete([1e-2, 1e-3, 1e-4, 1e-5])
momentum = instru.var.OrderedDiscrete([0.1, 0.5, 0.9])
smoothing = instru.var.OrderedDiscrete([0.01, 0.1, 0.2])
patience = instru.var.OrderedDiscrete([1, 5, 10, 20])
n_hidden = instru.var.OrderedDiscrete([2, 3, 4, 5])
hidden_size = instru.var.OrderedDiscrete([128, 256, 350, 512])
dropout_prob = instru.var.OrderedDiscrete([0.01, 0.1, 0.2])
model = args.model
epochs = args.epochs
batch_size = args.batch_size
valid_batch_size = args.valid_batch_size
n_workers = args.n_workers
cuda = args.cuda
data_path = args.data_path
valid_data_path = args.valid_data_path
checkpoint_path = args.checkpoint_path
softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax'])
instrum = instru.Instrumentation(lr, l2, momentum, smoothing, patience, model,
n_hidden, hidden_size, dropout_prob, epochs,
batch_size, valid_batch_size, n_workers, cuda,
data_path, valid_data_path, checkpoint_path,
softmax)
hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum,
budget=args.budget)
print(hp_optimizer.optimize(train, verbosity=2))
valid_hdf_path = args.valid_hdf_path if args.valid_hdf_path else 'none'
sub_file = args.sub_file
checkpoint_path = args.checkpoint_path
softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax'])
n_classes = args.nclasses
pretrained = args.pretrained
max_gnorm = instru.var.OrderedDiscrete([10, 30, 100])
lr_factor = instru.var.OrderedDiscrete([0.1, 0.3, 0.5, 0.8])
r_proj = instru.var.OrderedDiscrete([0., 1., 1.5, 1.8])
logdir = args.logdir
ablation = args.ablation
tmp_dir = os.getcwd() + '/' + args.temp_folder + '/'
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
instrum = instru.Instrumentation(
lr, l2, momentum, smoothing, patience, model, emb_size, n_hidden,
hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers,
cuda, data_path, valid_data_path, hdf_path, valid_hdf_path, sub_file,
checkpoint_path, softmax, n_classes, pretrained, max_gnorm, lr_factor,
r_proj, logdir, ablation)
hp_optimizer = optimization.optimizerlib.RandomSearch(
instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers)
with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor:
print(hp_optimizer.optimize(train, executor=executor, verbosity=2))
shutil.rmtree(tmp_dir)
data_path = args.data_path if args.data_path is not None else 'none'
hdf_path = args.hdf_path if args.hdf_path is not None else 'none'
valid_data_path = args.valid_data_path if args.valid_data_path is not None else 'none'
valid_hdf_path = args.valid_hdf_path if args.valid_hdf_path is not None else 'none'
checkpoint_path = args.checkpoint_path
softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax'])
pretrained = args.pretrained
if len(args.pretrained_path) > 1:
pretrained_path = instru.var.OrderedDiscrete(args.pretrained_path)
elif len(args.pretrained_path) == 1:
pretrained_path = args.pretrained_path
else:
pretrained_path = 'none'
max_gnorm = instru.var.OrderedDiscrete([10, 50, 100])
stats = args.stats
log_dir = args.logdir if args.logdir else 'none'
eval_every = args.eval_every
ablation = args.ablation
instrum = instru.Instrumentation(
lr, l2, momentum, smoothing, warmup, model, emb_size, n_hidden,
hidden_size, dropout_prob, epochs, batch_size, valid_batch_size, n_workers,
cuda, data_path, valid_data_path, hdf_path, valid_hdf_path,
checkpoint_path, softmax, pretrained, pretrained_path, max_gnorm, stats,
log_dir, eval_every, ablation)
hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum,
budget=args.budget)
print(hp_optimizer.optimize(train, verbosity=2))
lr = instru.var.Array(1).asfloat().bounded(1, 4).exponentiated(base=10,
coeff=-1)
l2 = instru.var.Array(1).asfloat().bounded(1, 5).exponentiated(base=10,
coeff=-1)
momentum = instru.var.Array(1).asfloat().bounded(0.10, 0.95)
margin = instru.var.Array(1).asfloat().bounded(0.10, 1.00)
lambda_ = instru.var.Array(1).asfloat().bounded(1, 5).exponentiated(base=10,
coeff=-1)
patience = instru.var.Array(1).asfloat().bounded(1, 100)
swap = instru.var.OrderedDiscrete([True, False])
model = args.model
epochs = args.epochs
batch_size = args.batch_size
valid_batch_size = args.valid_batch_size
n_workers = args.n_workers
cuda = args.cuda
data_path = args.data_path
valid_data_path = args.valid_data_path
checkpoint_path = args.checkpoint_path
instrum = instru.Instrumentation(lr, l2, momentum, margin, lambda_, patience,
swap, model, epochs, batch_size,
valid_batch_size, n_workers, cuda, data_path,
valid_data_path, checkpoint_path)
hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum,
budget=args.budget)
print(hp_optimizer.optimize(train, verbosity=2))
max_gnorm = instru.var.OrderedDiscrete([10.0, 100.0, 1.0])
warmup = instru.var.OrderedDiscrete([1, 500, 2000, 5000])
input_size = args.input_size
n_hidden = instru.var.OrderedDiscrete([1, 2, 3, 4, 5, 6, 8, 10])
hidden_size = instru.var.OrderedDiscrete([64, 128, 256, 512, 1024])
dropout_prob = instru.var.OrderedDiscrete([0.01, 0.1, 0.2, 0.4])
smoothing = instru.var.OrderedDiscrete([0.0, 0.01, 0.1, 0.2])
n_cycles = args.n_cycles
epochs = args.epochs
batch_size = args.batch_size
valid_batch_size = args.valid_batch_size
n_workers = args.workers
cuda = args.cuda
train_hdf_path = args.train_hdf_path
valid_hdf_path = args.valid_hdf_path
cp_path = args.checkpoint_path
logdir = args.logdir
instrum = instru.Instrumentation(lr, l2, momentum, max_gnorm, warmup,
input_size, n_hidden, hidden_size,
dropout_prob, smoothing, n_cycles, epochs,
batch_size, valid_batch_size, n_workers, cuda,
train_hdf_path, valid_hdf_path, cp_path,
logdir)
hp_optimizer = optimization.optimizerlib.RandomSearch(
instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers)
with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor:
print(hp_optimizer.optimize(train, executor=executor, verbosity=2))
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from pathlib import Path
from nevergrad import optimization
from nevergrad import instrumentation as inst
if __name__ == "__main__":
folder = Path(__file__).parents[1] / "instrumentation" / "examples"
func = inst.FolderFunction(folder, ["python", "examples/script.py"],
clean_copy=True)
instrumentation = inst.Instrumentation(value1=inst.var.Array(1).asscalar(),
value2=12,
string=inst.var.SoftmaxCategorical(
["plop", "blublu", "plouf"]))
opt = optimization.registry["OnePlusOne"](instrumentation, budget=4)
opt.optimize(func)
batch_size = args.batch_size
valid_batch_size = args.valid_batch_size
n_workers = args.workers
cuda = args.cuda
train_hdf_file = args.train_hdf_file
valid_hdf_file = args.valid_hdf_file
sub_file = args.sub_file
checkpoint_path = args.checkpoint_path
softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax'])
max_gnorm = instru.var.OrderedDiscrete([10.0, 20.0, 50.0])
logdir = args.logdir
tmp_dir = os.getcwd() + '/' + args.temp_folder + '/'
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir)
instrum = instru.Instrumentation(
lr, l2, momentum, smoothing, warmup, latent_size, n_hidden, hidden_size,
n_frames, model, ndiscriminators, rproj_size, ncoef, dropout_prob, epochs,
batch_size, valid_batch_size, n_workers, cuda, train_hdf_file,
valid_hdf_file, sub_file, tmp_dir, checkpoint_path, softmax, max_gnorm,
logdir)
hp_optimizer = optimization.optimizerlib.RandomSearch(
instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers)
with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor:
print(hp_optimizer.optimize(train, executor=executor, verbosity=2))
shutil.rmtree(tmp_dir)
model = instru.var.OrderedDiscrete([
'resnet_mfcc', 'resnet_lstm', 'resnet_stats', 'inception_mfcc',
'resnet_large', 'resnet_small'
]) if args.model == 'all' else args.model
ncoef = args.ncoef
epochs = args.epochs
batch_size = args.batch_size
n_workers = args.workers
cuda = args.cuda
train_hdf_file = args.train_hdf_file
data_info_path = args.data_info_path
valid_hdf_file = args.valid_hdf_file
n_cycles = args.n_cycles
valid_n_cycles = args.valid_n_cycles
checkpoint_path = args.checkpoint_path
softmax = instru.var.OrderedDiscrete(['softmax', 'am_softmax'])
delta = instru.var.OrderedDiscrete([True, False])
instrum = instru.Instrumentation(lr, l2, momentum, margin, lambda_, patience,
swap, latent_size, n_frames, model, ncoef,
epochs, batch_size, n_workers, cuda,
train_hdf_file, data_info_path,
valid_hdf_file, n_cycles, valid_n_cycles,
checkpoint_path, softmax, delta)
hp_optimizer = optimization.optimizerlib.RandomSearch(
instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers)
with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor:
print(hp_optimizer.optimize(train, executor=executor))
hidden_size=instru.var.Array(1).asfloat().bounded(64, 512) dropout_prob=instru.var.Array(1).asfloat().bounded(0.01, 0.50) n_frames=instru.var.Array(1).asfloat().bounded(600, 1000) model=instru.var.OrderedDiscrete(['resnet_mfcc', 'resnet_lstm', 'resnet_stats', 'resnet_small']) if args.model=='all' else args.model ncoef=args.ncoef epochs=args.epochs batch_size=args.batch_size n_workers=args.workers cuda=args.cuda train_hdf_file=args.train_hdf_file valid_hdf_file=args.valid_hdf_file n_cycles=args.n_cycles valid_n_cycles=args.valid_n_cycles sub_file=args.sub_file checkpoint_path=args.checkpoint_path softmax=instru.var.OrderedDiscrete(['softmax', 'am_softmax']) tmp_dir = os.getcwd() + '/' + args.temp_folder + '/' if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) instrum=instru.Instrumentation(lr, l2, momentum, patience, latent_size, n_hidden, hidden_size, n_frames, model, ncoef, dropout_prob, epochs, batch_size, n_workers, cuda, train_hdf_file, valid_hdf_file, n_cycles, valid_n_cycles, sub_file, tmp_dir, checkpoint_path, softmax) hp_optimizer=optimization.optimizerlib.RandomSearch(instrumentation=instrum, budget=args.budget, num_workers=args.hp_workers) with futures.ThreadPoolExecutor(max_workers=args.hp_workers) as executor: print(hp_optimizer.optimize(train, executor=executor, verbosity=2)) shutil.rmtree(tmp_dir)
lr = instru.var.Array(1).asfloat().bounded(1, 4).exponentiated(base=10,
coeff=-1)
l2 = instru.var.Array(1).asfloat().bounded(1, 5).exponentiated(base=10,
coeff=-1)
momentum = instru.var.Array(1).asfloat().bounded(0.01, 0.99)
slack = instru.var.Array(1).asfloat().bounded(1.01, 3.00)
patience = instru.var.Array(1).asfloat().bounded(1, 100)
model = instru.var.OrderedDiscrete(['vgg', 'resnet', 'densenet'
]) if args.model == 'all' else args.model
epochs = args.epochs
batch_size = args.batch_size
valid_batch_size = args.valid_batch_size
train_mode = instru.var.OrderedDiscrete(
['vanilla', 'hyper']) if args.train_mode == 'all' else args.train_mode
n_workers = args.n_workers
cuda = args.cuda
data_path = args.data_path
valid_data_path = args.valid_data_path
checkpoint_path = args.checkpoint_path
instrum = instru.Instrumentation(lr, l2, momentum, slack, patience, model,
epochs, batch_size, valid_batch_size,
train_mode, n_workers, cuda, data_path,
valid_data_path, checkpoint_path)
hp_optimizer = optimization.optimizerlib.RandomSearch(instrumentation=instrum,
budget=args.budget)
print(hp_optimizer.optimize(train, verbosity=2))
