def get_hyperspaces(parameters = None, sampler=None, n_samples=None):
'''
* `sampler` [str, default=None]
Random sampling scheme for optimizer's initial runs.
Options:
- "lhs": latin hypercube sampling
* `n_samples` [int, default=None]
Number of random samples to be drawn from the `sampler`.
- Required if you would like to use `sampler`.
- Must be <= the number of elements in the smallest hyperparameter bound's set.
'''
hyperspaces = create_hyperspace(hyperparameters)
with open('/home/dakka/spaces.txt', 'wb') as fp: # add the location an argument save known place
pickle.dump(hyperspaces, fp)
if sampler and not n_samples:
raise ValueError(f'Sampler requires n_samples > 0. Got {n_samples}')
elif sampler and n_samples:
# if samples is false, its value is returned, otherwise n_samples is evaluated and
# the resulting value is returned
hyperbounds = create_hyperbounds(hyperparameters)
return hyperspaces
def hyperbelt(objective,
hyperparameters,
results_path,
max_iter=100,
eta=3,
verbose=True,
n_evaluations=None,
random_state=0):
"""
Distributed HyperBand with SMBO - one hyperspace per node.
Parameters
----------
* `objective` [function]:
User defined function which calls a learner
and returns a metric of interest.
* `hyperparameters` [list, shape=(n_hyperparameters,)]:
* `results_path` [string]
Path to save optimization results
* `n_iterations` [int, default=50]
Number of optimization iterations
* `verbose` [bool, default=False]
Verbosity of optimization.
* `random_state` [int, default=0]
Random state for reproducibility.
"""
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Setup savefile
if rank < 10:
# Ensure results are sorted by rank
filename = 'hyperspace' + str(0) + str(rank)
else:
filename = 'hyperspace' + str(rank)
if not os.path.exists(results_path):
os.makedirs(results_path, exist_ok=True)
savefile = os.path.join(results_path, filename)
if rank == 0:
hyperspace = create_hyperspace(hyperparameters)
else:
hyperspace = None
space = comm.scatter(hyperspace, root=0)
result = hyperband(objective, space, max_iter, eta, random_state, verbose,
n_evaluations, rank)
# Each worker will independently write their results to disk
dump(result, savefile)
def hyperdrive(objective, hyperparameters, results_path, model="GP", n_iterations=50, verbose=False,
checkpoints_path=None, deadline=None, sampler=None, n_samples=None, random_state=0):
"""
Distributed optimization - one optimization per node.
Parameters
----------
* `objective` [function]:
User defined function which calls a learner
and returns a metric of interest.
* `hyperparameters` [list, shape=(n_hyperparameters,)]:
* `results_path` [string]
Path to save optimization results
* `checkpoint_path` [string]
Path to previously saved results. Used to resume optimization.
* `model` [string, default="GP"]
Probilistic learner used to model our objective function.
Options:
- "GP": Gaussian process
- "RF": Random forest
- "GBRT": Gradient boosted regression trees
- "RAND": Random search
* `n_iterations` [int, default=50]
Number of optimization iterations
* `verbose` [bool, default=False]
Verbosity of optimization.
* `checkpoints` [bool, default=False]
Whether to checkpoint at each step of the optimization.
* `deadline` [int, optional]
Deadline (seconds) for the optimization to finish within.
* `sampler` [str, default=None]
Random sampling scheme for optimizer's initial runs.
Options:
- "lhs": latin hypercube sampling
* `n_samples` [int, default=None]
Number of random samples to be drawn from the `sampler`.
- Required if you would like to use `sampler`.
- Must be <= the number of elements in the smallest hyperparameter bound's set.
* `random_state` [int, default=0]
Random state for reproducibility.
"""
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if checkpoints_path and sampler:
raise ValueError('Cannot use both a restart from a previous run and ' \
'use latin hypercube sampling for initial search points!')
# Setup savefile
if rank < 10:
# Ensure results are sorted by rank
filename = 'hyperspace' + str(0) + str(rank)
else:
filename = 'hyperspace' + str(rank)
savefile = os.path.join(results_path, filename)
# Create hyperspaces, and either sampling bounds or checkpoints
hyperspace = create_hyperspace(hyperparameters)
space = hyperspace[rank]
# Latin hypercube sampling
if sampler and not n_samples:
raise ValueError(f'Sampler requires n_samples > 0. Got {n_samples}')
elif sampler and n_samples:
hyperbounds = create_hyperbounds(hyperparameters)
bounds = hyperbounds[rank]
# Get initial points in domain via latin hypercube sampling
init_points = lhs_start(bounds, n_samples)
init_response = None
n_rand = 10 - len(init_points)
else:
init_points = None
init_response = None
n_rand = 10
# Resuming from checkpoint
if checkpoints_path:
checkpoint = _load_checkpoint(checkpoints_path, rank)
try:
init_points = checkpoint.x_iters
init_response = checkpoint.func_vals
n_rand = 10 - len(init_points)
except AttributeError:
# Missing saves won't have initial values.
init_points = None
init_response = None
n_rand = 10
callbacks = []
if deadline:
deadline = DeadlineStopper(deadline)
callbacks.append(deadline)
if checkpoints_path:
checkpoint_callback = CheckpointSaver(checkpoints_path, filename)
callbacks.append(checkpoint_callback)
# Thanks Guido for refusing to believe in switch statements.
# Case 0
if model == "GP":
# Verbose mode should only run on node 0.
if verbose and rank == 0:
result = gp_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
else:
result = gp_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
# Case 1
elif model == "RF":
if verbose and rank == 0:
result = forest_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
else:
result = forest_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
# Case 2
elif model == "GBRT":
if verbose and rank == 0:
result = gbrt_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
else:
result = gbrt_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
n_random_starts=n_rand, random_state=random_state)
# Case 3
elif model == "RAND":
if verbose and rank == 0:
result = dummy_minimize(objective, space, n_calls=n_iterations, verbose=verbose,
callback=callbacks, x0=init_points, y0=init_response,
random_state=random_state)
else:
result = dummy_minimize(objective, space, n_calls=n_iterations,
callback=callbacks, x0=init_points, y0=init_response,
random_state=random_state)
else:
raise ValueError("Invalid model {}. Read the documentation for "
"supported models.".format(model))
# Each worker will independently write their results to disk
dump(result, savefile)
def get_hyperspaces(parameters = None):
hyperspaces = create_hyperspace(hyperparameters)
with open('/home/dakka/spaces.txt', 'wb') as fp: # add the location an argument save known place
pickle.dump(hyperspaces, fp)
return hyperspaces
def hyperdrive(objective,
hyperparameters,
results_path,
model="GP",
n_iterations=50,
verbose=False,
deadline=None,
sampler=None,
n_samples=None,
random_state=0):
"""
Distributed optimization - one optimization per node.
Parameters
----------
* `objective` [function]:
User defined function which calls a learner
and returns a metric of interest.
* `hyperparameters` [list, shape=(n_hyperparameters,)]:
* `results_path` [string]
Path to save optimization results
* `model` [string, default="GP"]
Probilistic learner used to model our objective function.
Options:
- "GP": Gaussian process
- "RF": Random forest
- "GBRT": Gradient boosted regression trees
- "RAND": Random search
* `n_iterations` [int, default=50]
Number of optimization iterations
* `verbose` [bool, default=False]
Verbosity of optimization.
* `deadline` [int, optional]
Deadline (seconds) for the optimization to finish within.
* `sampler` [str, default=None]
Random sampling scheme for optimizer's initial runs.
Options:
- "lhs": latin hypercube sampling
* `n_samples` [int, default=None]
Number of random samples to be drawn from the `sampler`.
- Required if you would like to use `sampler`.
- Must be <= the number of elements in the smallest hyperparameter bound's set.
* `random_state` [int, default=0]
Random state for reproducibility.
"""
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
if rank == 0:
hyperspace = create_hyperspace(hyperparameters)
if sampler and not n_samples:
raise ValueError(
'Sampler requires n_samples > 0. Got {}'.format(n_samples))
elif sampler and n_samples:
hyperbounds = create_hyperbounds(hyperparameters)
else:
hyperspace = None
if sampler is not None:
hyperbounds = None
space = comm.scatter(hyperspace, root=0)
if sampler:
bounds = comm.scatter(hyperbounds, root=0)
# Get initial points in the obj. function domain via latin hypercube sampling
init_points = lhs_start(bounds, n_samples)
n_rand = 10 - len(init_points)
else:
init_points = None
n_rand = 10
if deadline:
deadline = DeadlineStopper(deadline)
# Thanks Guido for refusing to believe in switch statements.
# Case 0
if model == "GP":
# Verbose mode should only run on node 0.
if verbose and rank == 0:
result = gp_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = gp_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
# Case 1
elif model == "RF":
if verbose and rank == 0:
result = forest_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = forest_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
# Case 2
elif model == "GRBRT":
if verbose and rank == 0:
result = gbrt_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = gbrt_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
# Case 3
elif model == "RAND":
if verbose and rank == 0:
result = dummy_minimize(objective,
space,
n_calls=n_iterations,
verbose=verbose,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
result = dummy_minimize(objective,
space,
n_calls=n_iterations,
callback=deadline,
x0=init_points,
n_random_starts=n_rand,
random_state=random_state)
else:
raise ValueError("Invalid model {}. Read the documentation for "
"supported models.".format(model))
# Each worker will independently write their results to disk
dump(result, results_path + '/hyperspace' + str(rank))