def set_basic_conf(self):
from dragonfly.opt.gp_bandit import EuclideanGPBandit
from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
from dragonfly import load_config
def cost(space, reporter):
height, width = space["point"]
reporter(loss=(height - 14)**2 - abs(width - 3))
domain_vars = [{
"name": "height",
"type": "float",
"min": -10,
"max": 10
}, {
"name": "width",
"type": "float",
"min": 0,
"max": 20
}]
domain_config = load_config({"domain": domain_vars})
func_caller = EuclideanFunctionCaller(
None, domain_config.domain.list_of_domains[0])
optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
search_alg = DragonflySearch(
optimizer,
metric="loss",
mode="min",
max_concurrent=1000, # Here to avoid breaking back-compat.
)
return search_alg, cost
def main():
""" Main function. """
disc_euc_items = list(np.random.random((1000, 3)))
domain_vars = [
{
'type': 'discrete_euclidean',
'items': disc_euc_items
},
{
'type': 'float',
'min': 0,
'max': 11,
'dim': 2
},
{
'type': 'int',
'min': 0,
'max': 114
},
]
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_num_evals = 100
opt_val, opt_pt, history = maximise_function(hartmann6_3,
config.domain,
max_num_evals,
config=config)
print(opt_pt, opt_val)
def main():
""" Main function. """
# First Specify all parameters.
# See examples/synthetic/multiobjective_hartmann/in_code_demo.py for speciying
# domain via a JSON file.
domain_vars = [
{
'type': 'float',
'min': -5,
'max': 10,
'dim': 1
},
{
'type': 'float',
'min': 0,
'max': 15,
'dim': 1
},
]
config_params = {'domain': domain_vars}
config = load_config(config_params)
# Specify objectives -- either of the following options could work.
# 1. compute_objectives returns a list of objective values, num_objectives is the number
# of objectives. This has to be a 2-tuple.
moo_objectives = (compute_objectives, num_objectives)
# 2. Specify each function separately. This has to be a list.
# moo_objectives = [branin, currin_exp]
# Optimise
max_num_evals = 100 # Optimisation budget (max number of evaluations)
pareto_opt_vals, pareto_opt_pts, history = multiobjective_maximise_functions(
moo_objectives, config.domain, max_num_evals, config=config)
print(pareto_opt_pts)
print(pareto_opt_vals)
def main():
""" Main function. """
domain_vars = [{
'name': 'hubble_constant',
'type': 'float',
'min': 60,
'max': 80
}, {
'name': 'omega_m',
'type': 'float',
'min': 0,
'max': 1
}, {
'name': 'omega_l',
'type': 'float',
'min': 0,
'max': 1
}]
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_capital = 2 * 60 * 60 # Optimisation budget in seconds
# Optimise
opt_pt, opt_val, history = maximise_function(snls_objective,
config.domain,
max_capital,
capital_type='realtime',
config=config)
def load_config_dict(d):
"""Load Dragonfly config from dict d.
d should be what you'd normally put in a JSON file, i.e.,
with open(fn) as f: load_config_dict(json.load(f)) is equivalent
to dragonfly.load_config_file(fn). This function is useful when d
does not come directly from a file.
"""
return load_config(load_parameters(d))
def main():
""" Main function. """
# First Specify all parameters
disc_num_items_x1 = [4, 10, 23, 45, 78, 87.1, 91.8, 99, 75.7, 28.1, 3.141593]
domain_vars = [
{'name': 'x0', 'type': 'discrete_numeric', 'items': '0:0.05:1', 'dim': 2},
{'name': 'x1', 'type': 'discrete_numeric', 'items': disc_num_items_x1},
{'name': 'x2', 'type': 'float', 'min': 10, 'max': 16},
]
domain_constraints = [
{'name': 'dc1', 'constraint': 'sum(x0) + (x2 - 10)/6.0 <= 2.1'},
{'name': 'dc2', 'constraint': dc2_constraint}
]
disc_items_z2 = ['a', 'ab', 'abc', 'abcd', 'abcde', 'abcdef', 'abcdefg', 'abcdefg',
'abcdefgh', 'abcdefghi']
fidel_vars = [{'name': 'z1', 'type': 'discrete', 'items': disc_items_z2, 'dim': 2},
{'name': 'z2', 'type': 'float', 'min': 21.3, 'max': 243.9},
]
fidel_to_opt = [["abcdefghi", "abcdefghi"], 200.1]
# Budget of evaluations
max_num_evals = 100 # Optimisation budget (max number of evaluations)
max_mf_capital = max_num_evals * mf_cost(fidel_to_opt) # Multi-fideltiy capital
# First do the MF version
config_params = {'domain': domain_vars, 'fidel_space': fidel_vars,
'domain_constraints': domain_constraints,
'fidel_to_opt': fidel_to_opt}
config = load_config(config_params)
# Optimise
mf_opt_val, mf_opt_pt, history = maximise_multifidelity_function(mf_objective,
config.fidel_space, config.domain,
config.fidel_to_opt, mf_cost,
max_mf_capital, config=config)
print(mf_opt_pt, mf_opt_val)
# Non-MF version
config_params = {'domain': domain_vars, 'domain_constraints': domain_constraints}
config = load_config(config_params)
max_capital = 100 # Optimisation budget (max number of evaluations)
# Optimise
opt_val, opt_pt, history = maximise_function(objective, config.domain,
max_num_evals, config=config)
print(opt_pt, opt_val)
def main():
""" Main function. """
# First Specify all parameters
domain_vars = [{'name': 'x', 'type': 'float', 'min': 0, 'max': 1, 'dim': 3}]
domain_constraints = [
{'name': 'quadrant', 'constraint': 'np.linalg.norm(x[0:2]) <= 0.5'},
]
fidel_vars = [{'name': 'z', 'type': 'float', 'min': 0, 'max': 10}]
fidel_space_constraints = [{'name': 'fsc1', 'constraint': fsc1_constraint}]
fidel_to_opt = [9.1]
print('fsc1_constraint(fidel_to_opt)', fsc1_constraint(fidel_to_opt))
# Budget of evaluations
max_num_evals = 100 # Optimisation budget (max number of evaluations)
max_mf_capital = max_num_evals * mf_cost(fidel_to_opt) # Multi-fideltiy capital
# Non-MF version
config_params = {'domain': domain_vars, 'domain_constraints': domain_constraints}
config = load_config(config_params)
max_capital = 100 # Optimisation budget (max number of evaluations)
# Optimise
opt_val, opt_pt, history = maximise_function(objective, config.domain,
max_num_evals, config=config)
print(opt_pt, opt_val)
# MF version
config_params = {'domain': domain_vars, 'fidel_space': fidel_vars,
'domain_constraints': domain_constraints,
'fidel_space_constraints': fidel_space_constraints,
'fidel_to_opt': fidel_to_opt}
config = load_config(config_params)
# Optimise
mf_opt_val, mf_opt_pt, history = maximise_multifidelity_function(mf_objective,
config.fidel_space, config.domain,
config.fidel_to_opt, mf_cost,
max_mf_capital, config=config)
print(mf_opt_pt, mf_opt_val)
def main():
""" Main function. """
size = 1000
dim = 3
disc_euc_items = list(np.random.random((size, dim)))
domain_vars = [
{'type': 'discrete_euclidean', 'items': disc_euc_items},
]
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_num_evals = 100
# specify optimisation method
# opt_method = 'bo' # Bayesian optimisation
opt_method = 'ea' # evolutionary algorithm
# opt_method = 'rand' # random search
opt_val, opt_pt, history = maximise_function(objective, config.domain, max_num_evals,
config=config, opt_method=opt_method)
print(opt_pt, opt_val)
def main():
""" Main function. """
size = 1000
dim = 3
disc_euc_items = list(np.random.random((size, dim)))
domain_vars = [
{
'type': 'discrete_euclidean',
'items': disc_euc_items
},
]
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_num_evals = 100
opt_pt, opt_val, history = maximise_function(objective,
config.domain,
max_num_evals,
config=config)
print(opt_pt, opt_val)
def optimization_loop(capital=10):
vals_limits = read_vars('limits.sh')
vals = read_vars('ml-perf-harness.conf', vals=vals_limits)
vals_default = read_vars('default.conf', vals=vals_limits)
domain = [
#{'name': 'READ_LAT_NSEC', 'type': 'int', 'min': 0, 'max': 100000000, 'dim': 1},
#{'name': 'WRITE_LAT_NSEC', 'type': 'int', 'min': 0, 'max': 100000000, 'dim': 1},
#{'name': 'NR_REQUESTS', 'type': 'int', 'min': 4, 'max': 10000, 'dim': 1}, #queue depth
#{'name': 'MAX_SECTORS_KB', 'type': 'int', 'min': 128, 'max': 1280, 'dim': 1}, #max IO size sent to device
{
'name': 'READ_AHEAD_KB',
'type': 'int',
'min': 0,
'max': 10000,
'dim': 1
}, #amount of IO to read ahead into cache
#{'name': 'WBT_LAT_USEC', 'type': 'int', 'min': 0, 'max': 10000, 'dim': 1}, #target latency for reads. throttle writes otherwise
#{'name': 'DIRTY_RATIO', 'type': 'int', 'min': 0, 'max': 100, 'dim': 1},
#{'name': 'DIRTY_BACKGROUND_RATIO', 'type': 'int', 'min': 0, 'max': 100, 'dim': 1},
#{'name': 'SWAPPINESS', 'type': 'int', 'min': 0, 'max': 100, 'dim': 1},
]
config = load_config({'domain': domain})
objective_partial = lambda x: objective(
x, domain=domain, default_vals=vals_default)
if not os.path.exists(LOC):
os.makedirs(LOC)
val, point, history = maximize_function(objective_partial,
config.domain,
capital,
config=config)
return val, point, history
{'type': 'discrete', 'items': place_pruned_graph_list},
{'type': 'discrete', 'items': enable_bfloat16_sendrecv_list},
{'type': 'discrete', 'items': do_common_subexpression_elimination_list},
{'type': 'discrete_numeric', 'items': max_folded_constant_list},
{'type': 'discrete', 'items': do_function_inlining_list},
{'type': 'discrete_numeric', 'items': global_jit_level_list},
{'type': 'discrete', 'items': optimizer_list}
]
dragonfly_args = [
get_option_specs('report_results_every', False, 2, 'Path to the json or pb config file. '),
get_option_specs('init_capital', False, None, 'Path to the json or pb config file. '),
get_option_specs('init_capital_frac', False, 0.07, 'Path to the json or pb config file. '),
get_option_specs('num_init_evals', False, 2, 'Path to the json or pb config file. ')]
options = load_options(dragonfly_args)
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_num_evals = 60*60*12
moo_objectives = [runtime_eval, acc_eval]
pareto_opt_vals, pareto_opt_pts, history = multiobjective_maximise_functions(moo_objectives, config.domain,max_num_evals,capital_type='realtime',config=config,options=options)
f = open("./output.log","w+")
print(pareto_opt_pts,file=f)
print("\n",file=f)
print(pareto_opt_vals,file=f)
print("\n",file=f)
print(history,file=f)
def get_config(self, budget):
"""Function to sample a new configuration
This function is called inside BOHB to query a new configuration
Parameters:
-----------
budget: float
the budget for which this configuration is scheduled
Returns
-------
config
return a valid configuration with parameters and budget
"""
if not self.is_moo:
return self.get_config_old(budget)
logger.debug('start sampling a new configuration.')
if not self.configs:
print(
f"[vincent] self.configs is empty! Use a random config instead."
)
sample = self.configspace.sample_configuration()
sample = ConfigSpace.util.deactivate_inactive_hyperparameters(
configuration_space=self.configspace,
configuration=sample.get_dictionary()).get_dictionary()
sample['TRIAL_BUDGET'] = budget
return sample
domain_vars = list()
for name in self.search_space.keys():
if isinstance(self.search_space[name][0], (float, int)):
var_type = 'discrete_numeric'
else:
var_type = 'discrete'
domain_var = {'type': var_type, 'items': self.search_space[name]}
domain_vars.append(domain_var)
points = list()
vals = list()
true_vals = list()
print(f"[vincent] self.configs:{self.configs} budget:{budget}")
print(f"{list(self.search_space.keys())}")
for conf_array in self.configs[0]:
first, second = [], []
for i in range(len(conf_array)):
item = self.search_space[list(
self.search_space.keys())[i]][int(conf_array[i])]
if isinstance(item, (float, int)):
second.append(item)
else:
first.append(item)
points.append([first, second])
for idx in range(len(self.losses[0])):
vals.append([-self.losses[0][idx], -self.runtime[0][idx]])
true_vals.append([-self.losses[0][idx], -self.runtime[0][idx]])
print(f"[vincent] len of points:{len(points)}")
if len(points) > 10:
vals_array = np.array(vals)
pareto_index = is_pareto_efficient_simple(vals_array)
p_idx = []
np_idx = []
np_items = []
for j in range(len(pareto_index)):
if pareto_index[j] == True:
p_idx.append(j)
else:
np_idx.append(j)
np_items.append(vals[j])
print(f"[vincent] pareto_index:{p_idx}")
print(f"[vincent] not pareto_index:{np_idx}")
if len(p_idx) >= 5:
tmp_idx = []
for j in range(5):
tmp_idx.append(p_idx[j])
points = [points[i] for i in tmp_idx]
vals = [vals[i] for i in tmp_idx]
true_vals = [true_vals[i] for i in tmp_idx]
else:
num_diff = 5 - len(p_idx)
print(f"[vincent] diff num:{num_diff}")
print(f"[vincent] search space:{self.search_space}")
if self.search_space['PREFERENCE'][0] == "accuracy":
acc_items = [-item[0] for item in np_items]
sort_n_idx = np.argsort(acc_items)
for i in range(num_diff):
p_idx.append(sort_n_idx[i])
print(f"[vincent] final pareto_index:{p_idx}")
points = [points[i] for i in p_idx]
vals = [vals[i] for i in p_idx]
true_vals = [true_vals[i] for i in p_idx]
elif self.search_space['PREFERENCE'][0] == "runtime":
time_items = [-item[1] for item in np_items]
sort_n_idx = np.argsort(time_items)
for i in range(num_diff):
p_idx.append(sort_n_idx[i])
print(f"[vincent] final pareto_index:{p_idx}")
points = [points[i] for i in p_idx]
vals = [vals[i] for i in p_idx]
true_vals = [true_vals[i] for i in p_idx]
# import random
# idx_list = random.sample(range(len(points)), 10)
# print(f"[vincent] random selections list idx_list:{idx_list}")
# points = [points[i] for i in idx_list]
# vals = [vals[i] for i in idx_list]
# true_vals = [true_vals[i] for i in idx_list]
## vals = [[acc,-spent time],[acc,-spent time]]
## load from memory
previous_eval = {'qinfos': []}
for i in range(len(points)):
tmp = Namespace(point=points[i],
val=vals[i],
true_val=true_vals[i])
previous_eval['qinfos'].append(tmp)
p = Namespace(**previous_eval)
load_args = [
get_option_specs('init_capital', False, 1,
'Path to the json or pb config file. '),
get_option_specs(
'init_capital_frac', False, None,
'The fraction of the total capital to be used for initialisation.'
),
get_option_specs(
'num_init_evals', False, 1,
'The number of evaluations for initialisation. If <0, will use default.'
),
get_option_specs('prev_evaluations', False, p,
'Data for any previous evaluations.')
]
options = load_options(load_args)
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_num_evals = 1
self.dragonfly_config = None
def fake_func(x):
if not self.dragonfly_config:
self.dragonfly_config = x
print(
f"[vincent] x is assigned to self.dragonfly_config:{self.dragonfly_config}"
)
return 0
moo_objectives = [fake_func, fake_func]
_, _, _ = multiobjective_maximise_functions(moo_objectives,
config.domain,
max_num_evals,
capital_type='num_evals',
config=config,
options=options)
print(
f"[vincent] self.dragonfly_config after dragonfly:{self.dragonfly_config}"
)
## load prev from the file
# data_to_save = {'points': points,
# 'vals': vals,
# 'true_vals': true_vals}
# print(f"[vincent] data_to_save:{data_to_save}")
# temp_save_path = './dragonfly.saved'
# with open(temp_save_path, 'wb') as save_file_handle:
# pickle.dump(data_to_save, save_file_handle)
# load_args = [
# get_option_specs('progress_load_from', False, temp_save_path,
# 'Load progress (from possibly a previous run) from this file.')
# ]
# options = load_options(load_args)
# config_params = {'domain': domain_vars}
# config = load_config(config_params)
# max_num_evals = 1
# self.dragonfly_config = None
# def fake_func(x):
# if not self.dragonfly_config:
# self.dragonfly_config = x
# print(f"[vincent] x is assigned to self.dragonfly_config:{self.dragonfly_config}")
# return 0
# moo_objectives = [fake_func, fake_func]
# _, _, _ = multiobjective_maximise_functions(moo_objectives, config.domain,max_num_evals,capital_type='num_evals',config=config,options=options)
# print(f"[vincent] self.dragonfly_config after dragonfly:{self.dragonfly_config}")
# import os
# if os.path.exists(temp_save_path):
# os.remove(temp_save_path)
if not self.dragonfly_config:
print(
f"[vincent] Get empty config from dragonfly! Use a random config instead."
)
sample = self.configspace.sample_configuration()
else:
sample = dict()
df_idx = 0
for name in self.search_space.keys():
sample[name] = self.dragonfly_config[df_idx]
df_idx += 1
logger.debug('done sampling a new configuration.')
sample['TRIAL_BUDGET'] = budget
print(f'[vincent] sample from get_config:{sample}')
return sample
def main():
""" Main function. """
# First Specify all parameters
domain_vars = [
{
'type': 'int',
'min': 224,
'max': 324,
'dim': 1
},
{
'type': 'float',
'min': 0,
'max': 10,
'dim': 2
},
{
'type': 'float',
'min': 0,
'max': 1,
'dim': 1
},
{
'type': 'int',
'min': 0,
'max': 92,
'dim': 2
},
]
fidel_vars = [
{
'type': 'float',
'min': 1234.9,
'max': 9467.18,
'dim': 2
},
{
'type': 'discrete',
'items': ['a', 'bc', 'def', 'ghij']
},
{
'type': 'int',
'min': 123,
'max': 234,
'dim': 1
},
]
fidel_to_opt = [[9467.18, 9452.8], "def", [234]]
# Budget of evaluations
max_num_evals = 100 # Optimisation budget (max number of evaluations)
max_mf_capital = max_num_evals * mf_cost(
fidel_to_opt) # Multi-fideltiy capital
# First do the MF version
config_params = {
'domain': domain_vars,
'fidel_space': fidel_vars,
'fidel_to_opt': fidel_to_opt
}
config = load_config(config_params)
# Optimise
mf_opt_val, mf_opt_pt, history = maximise_multifidelity_function(
mf_objective,
config.fidel_space,
config.domain,
config.fidel_to_opt,
mf_cost,
max_mf_capital,
config=config)
print(mf_opt_pt, mf_opt_val)
# Non-MF version
config_params = {'domain': domain_vars}
config = load_config(config_params)
max_capital = 100 # Optimisation budget (max number of evaluations)
# Optimise
opt_val, opt_pt, history = maximise_function(objective,
config.domain,
max_num_evals,
config=config)
print(opt_pt, opt_val)
def hparams(algorithm, scheduler, num_samples, tensorboard, bare):
from glob import glob
import tensorflow.summary
from tensorflow import random as tfrandom, int64 as tfint64
from ray import init as init_ray, shutdown as shutdown_ray
from ray import tune
from wandb.ray import WandbLogger
from wandb import sweep as wandbsweep
from wandb.apis import CommError as wandbCommError
# less summaries are logged if MLENCRYPT_TB is TRUE (for efficiency)
# TODO: use tf.summary.record_if?
environ["MLENCRYPT_TB"] = str(tensorboard).upper()
environ["MLENCRYPT_BARE"] = str(bare).upper()
if getenv('MLENCRYPT_TB', 'FALSE') == 'TRUE' and \
getenv('MLENCRYPT_BARE', 'FALSE') == 'TRUE':
raise ValueError('TensorBoard logging cannot be enabled in bare mode.')
logdir = f'logs/hparams/{datetime.now()}'
# "These results show that K = 3 is the optimal choice for the
# cryptographic application of neural synchronization. K = 1 and K = 2 are
# too insecure in regard to the geometric attack. And for K > 3 the effort
# of A and B grows exponentially with increasing L, while the simple attack
# is quite successful in the limit K -> infinity. Consequently, one should
# only use Tree Parity Machines with three hidden units for the neural
# key-exchange protocol." (Ruttor, 2006)
# https://arxiv.org/pdf/0711.2411.pdf#page=59
update_rules = [
'random-same',
# 'random-different-A-B-E', 'random-different-A-B',
'hebbian',
'anti_hebbian',
'random_walk'
]
K_bounds = {'min': 4, 'max': 8}
N_bounds = {'min': 4, 'max': 8}
L_bounds = {'min': 4, 'max': 8}
# TODO: don't use *_bounds.values() since .values doesn't preserve order
def get_session_num(logdir):
current_runs = glob(join(logdir, "run-*"))
if current_runs:
last_run_path = current_runs[-1]
last_run_session_num = int(last_run_path.split('-')[-1])
return last_run_session_num + 1
else: # there are no runs yet, start at 0
return 0
def trainable(config, reporter):
"""
Args:
config (dict): Parameters provided from the search algorithm
or variant generation.
"""
if not isinstance(config['update_rule'], str):
update_rule = update_rules[int(config['update_rule'])]
else:
update_rule = config['update_rule']
K, N, L = int(config['K']), int(config['N']), int(config['L'])
run_name = f"run-{get_session_num(logdir)}"
run_logdir = join(logdir, run_name)
# for each attack, the TPMs should start with the same weights
initial_weights_tensors = get_initial_weights(K, N, L)
training_steps_ls = {}
eve_scores_ls = {}
losses_ls = {}
# for each attack, the TPMs should use the same inputs
seed = tfrandom.uniform([],
minval=0,
maxval=tfint64.max,
dtype=tfint64).numpy()
for attack in ['none', 'geometric']:
initial_weights = {
tpm: weights_tensor_to_variable(weights, tpm)
for tpm, weights in initial_weights_tensors.items()
}
tfrandom.set_seed(seed)
if tensorboard:
attack_logdir = join(run_logdir, attack)
attack_writer = tensorflow.summary.create_file_writer(
attack_logdir)
with attack_writer.as_default():
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
else:
training_steps, sync_scores, loss = run(
update_rule, K, N, L, attack, initial_weights)
training_steps_ls[attack] = training_steps
eve_scores_ls[attack] = sync_scores
losses_ls[attack] = loss
avg_training_steps = tensorflow.math.reduce_mean(
list(training_steps_ls.values()))
avg_eve_score = tensorflow.math.reduce_mean(
list(eve_scores_ls.values()))
mean_loss = tensorflow.math.reduce_mean(list(losses_ls.values()))
reporter(
avg_training_steps=avg_training_steps.numpy(),
avg_eve_score=avg_eve_score.numpy(),
mean_loss=mean_loss.numpy(),
done=True,
)
if algorithm == 'hyperopt':
from hyperopt import hp as hyperopt
from hyperopt.pyll.base import scope
from ray.tune.suggest.hyperopt import HyperOptSearch
space = {
'update_rule': hyperopt.choice(
'update_rule',
update_rules,
),
'K': scope.int(hyperopt.quniform('K', *K_bounds.values(), q=1)),
'N': scope.int(hyperopt.quniform('N', *N_bounds.values(), q=1)),
'L': scope.int(hyperopt.quniform('L', *L_bounds.values(), q=1)),
}
algo = HyperOptSearch(
space,
metric='mean_loss',
mode='min',
points_to_evaluate=[
{
'update_rule': 0,
'K': 3,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 8
},
{
'update_rule': 0,
'K': 8,
'N': 16,
'L': 128
},
],
)
elif algorithm == 'bayesopt':
from ray.tune.suggest.bayesopt import BayesOptSearch
space = {
'update_rule': (0, len(update_rules)),
'K': tuple(K_bounds.values()),
'N': tuple(N_bounds.values()),
'L': tuple(L_bounds.values()),
}
algo = BayesOptSearch(
space,
metric="mean_loss",
mode="min",
# TODO: what is utility_kwargs for and why is it needed?
utility_kwargs={
"kind": "ucb",
"kappa": 2.5,
"xi": 0.0
})
elif algorithm == 'nevergrad':
from ray.tune.suggest.nevergrad import NevergradSearch
from nevergrad import optimizers
from nevergrad import p as ngp
algo = NevergradSearch(
optimizers.TwoPointsDE(
ngp.Instrumentation(
update_rule=ngp.Choice(update_rules),
K=ngp.Scalar(lower=K_bounds['min'],
upper=K_bounds['max']).set_integer_casting(),
N=ngp.Scalar(lower=N_bounds['min'],
upper=N_bounds['max']).set_integer_casting(),
L=ngp.Scalar(lower=L_bounds['min'],
upper=L_bounds['max']).set_integer_casting(),
)),
None, # since the optimizer is already instrumented with kwargs
metric="mean_loss",
mode="min")
elif algorithm == 'skopt':
from skopt import Optimizer
from ray.tune.suggest.skopt import SkOptSearch
optimizer = Optimizer([
update_rules,
tuple(K_bounds.values()),
tuple(N_bounds.values()),
tuple(L_bounds.values())
])
algo = SkOptSearch(
optimizer,
["update_rule", "K", "N", "L"],
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'dragonfly':
# TODO: doesn't work
from ray.tune.suggest.dragonfly import DragonflySearch
from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
from dragonfly.opt.gp_bandit import EuclideanGPBandit
# from dragonfly.exd.experiment_caller import CPFunctionCaller
# from dragonfly.opt.gp_bandit import CPGPBandit
from dragonfly import load_config
domain_config = load_config({
"domain": [
{
"name": "update_rule",
"type": "discrete",
"dim": 1,
"items": update_rules
},
{
"name": "K",
"type": "int",
"min": K_bounds['min'],
"max": K_bounds['max'],
# "dim": 1
},
{
"name": "N",
"type": "int",
"min": N_bounds['min'],
"max": N_bounds['max'],
# "dim": 1
},
{
"name": "L",
"type": "int",
"min": L_bounds['min'],
"max": L_bounds['max'],
# "dim": 1
}
]
})
func_caller = EuclideanFunctionCaller(
None, domain_config.domain.list_of_domains[0])
optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
algo = DragonflySearch(
optimizer,
metric="mean_loss",
mode="min",
points_to_evaluate=[
['random-same', 3, 16, 8],
['random-same', 8, 16, 8],
['random-same', 8, 16, 128],
],
)
elif algorithm == 'bohb':
from ConfigSpace import ConfigurationSpace
from ConfigSpace import hyperparameters as CSH
from ray.tune.suggest.bohb import TuneBOHB
config_space = ConfigurationSpace()
config_space.add_hyperparameter(
CSH.CategoricalHyperparameter("update_rule", choices=update_rules))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='K',
lower=K_bounds['min'],
upper=K_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='N',
lower=N_bounds['min'],
upper=N_bounds['max']))
config_space.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='L',
lower=L_bounds['min'],
upper=L_bounds['max']))
algo = TuneBOHB(config_space, metric="mean_loss", mode="min")
elif algorithm == 'zoopt':
from ray.tune.suggest.zoopt import ZOOptSearch
from zoopt import ValueType
space = {
"update_rule":
(ValueType.DISCRETE, range(0, len(update_rules)), False),
"K": (ValueType.DISCRETE,
range(K_bounds['min'], K_bounds['max'] + 1), True),
"N": (ValueType.DISCRETE,
range(N_bounds['min'], N_bounds['max'] + 1), True),
"L": (ValueType.DISCRETE,
range(L_bounds['min'], L_bounds['max'] + 1), True),
}
# TODO: change budget to a large value
algo = ZOOptSearch(budget=10,
dim_dict=space,
metric="mean_loss",
mode="min")
# TODO: use more appropriate arguments for schedulers:
# https://docs.ray.io/en/master/tune/api_docs/schedulers.html
if scheduler == 'fifo':
sched = None # Tune defaults to FIFO
elif scheduler == 'pbt':
from ray.tune.schedulers import PopulationBasedTraining
from random import randint
sched = PopulationBasedTraining(
metric="mean_loss",
mode="min",
hyperparam_mutations={
"update_rule": update_rules,
"K": lambda: randint(K_bounds['min'], K_bounds['max']),
"N": lambda: randint(N_bounds['min'], N_bounds['max']),
"L": lambda: randint(L_bounds['min'], L_bounds['max']),
})
elif scheduler == 'ahb' or scheduler == 'asha':
# https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#asha-tune-schedulers-ashascheduler
from ray.tune.schedulers import AsyncHyperBandScheduler
sched = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
elif scheduler == 'hb':
from ray.tune.schedulers import HyperBandScheduler
sched = HyperBandScheduler(metric="mean_loss", mode="min")
elif algorithm == 'bohb' or scheduler == 'bohb':
from ray.tune.schedulers import HyperBandForBOHB
sched = HyperBandForBOHB(metric="mean_loss", mode="min")
elif scheduler == 'msr':
from ray.tune.schedulers import MedianStoppingRule
sched = MedianStoppingRule(metric="mean_loss", mode="min")
init_ray(
address=getenv("ip_head"),
redis_password=getenv('redis_password'),
)
analysis = tune.run(
trainable,
name='mlencrypt_research',
config={
"monitor": True,
"env_config": {
"wandb": {
"project": "mlencrypt-research",
"sync_tensorboard": True,
},
},
},
# resources_per_trial={"cpu": 1, "gpu": 3},
local_dir='./ray_results',
export_formats=['csv'], # TODO: add other formats?
num_samples=num_samples,
loggers=[
tune.logger.JsonLogger, tune.logger.CSVLogger,
tune.logger.TBXLogger, WandbLogger
],
search_alg=algo,
scheduler=sched,
queue_trials=True,
)
try:
wandbsweep(analysis)
except wandbCommError:
# see https://docs.wandb.com/sweeps/ray-tune#feature-compatibility
pass
best_config = analysis.get_best_config(metric='mean_loss', mode='min')
print(f"Best config: {best_config}")
shutdown_ray()
def main():
""" Main function. """
domain_vars = [{
'name': 'hubble_constant',
'type': 'float',
'min': 60,
'max': 80
}, {
'name': 'omega_m',
'type': 'float',
'min': 0,
'max': 1
}, {
'name': 'omega_l',
'type': 'float',
'min': 0,
'max': 1
}]
fidel_vars = [{
'name': 'log10_resolution',
'type': 'float',
'min': 2,
'max': 5
}, {
'name': 'num_obs_to_use',
'type': 'int',
'min': 50,
'max': 192
}]
fidel_to_opt = [5, 192]
max_capital = 2 * 60 * 60 # Optimisation budget in seconds
# A parallel set up where we will evaluate the function in three different threads.
num_workers = 3
# Optimise without multi-fidelity
config_params = {'domain': domain_vars}
config = load_config(config_params)
opt_val, opt_pt, history = maximise_function(snls_objective,
config.domain,
max_capital,
num_workers=num_workers,
capital_type='realtime',
config=config)
print(opt_pt, opt_val)
# Optimise with multi-fidelity
config_params = {
'domain': domain_vars,
'fidel_space': fidel_vars,
'fidel_to_opt': fidel_to_opt
}
config = load_config(config_params)
# Optimise
mf_opt_val, mf_opt_pt, history = maximise_multifidelity_function(
snls_mf_objective,
config.fidel_space,
config.domain,
config.fidel_to_opt,
snls_mf_cost,
max_capital,
config=config)
print(mf_opt_pt, mf_opt_val)
"type": "float",
"min": 0,
"max": 7
}, {
"name": "Li2SO4_vol",
"type": "float",
"min": 0,
"max": 7
}, {
"name": "NaClO4_vol",
"type": "float",
"min": 0,
"max": 7
}]
domain_config = load_config({"domain": domain_vars})
func_caller = EuclideanFunctionCaller(
None, domain_config.domain.list_of_domains[0])
optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
algo = DragonflySearch(optimizer,
max_concurrent=4,
metric="objective",
mode="max")
scheduler = AsyncHyperBandScheduler(metric="objective", mode="max")
run(objective,
name="dragonfly_search",
search_alg=algo,
scheduler=scheduler,
**config)
def _setup_dragonfly(self):
"""Setup dragonfly when no optimizer has been passed."""
assert not self._opt, "Optimizer already set."
from dragonfly import load_config
from dragonfly.exd.experiment_caller import (
CPFunctionCaller,
EuclideanFunctionCaller,
)
from dragonfly.opt.blackbox_optimiser import BlackboxOptimiser
from dragonfly.opt.random_optimiser import (
CPRandomOptimiser,
EuclideanRandomOptimiser,
)
from dragonfly.opt.cp_ga_optimiser import CPGAOptimiser
from dragonfly.opt.gp_bandit import CPGPBandit, EuclideanGPBandit
if not self._space:
raise ValueError(
"You have to pass a `space` when initializing dragonfly, or "
"pass a search space definition to the `config` parameter "
"of `tune.run()`.")
if not self._domain:
raise ValueError(
"You have to set a `domain` when initializing dragonfly. "
"Choose one of [Cartesian, Euclidean].")
self._point_parameter_names = [param["name"] for param in self._space]
if self._random_state_seed is not None:
np.random.seed(self._random_state_seed)
if self._domain.lower().startswith("cartesian"):
function_caller_cls = CPFunctionCaller
elif self._domain.lower().startswith("euclidean"):
function_caller_cls = EuclideanFunctionCaller
else:
raise ValueError("Dragonfly's `domain` argument must be one of "
"[Cartesian, Euclidean].")
optimizer_cls = None
if inspect.isclass(self._opt_arg) and issubclass(
self._opt_arg, BlackboxOptimiser):
optimizer_cls = self._opt_arg
elif isinstance(self._opt_arg, str):
if self._opt_arg.lower().startswith("random"):
if function_caller_cls == CPFunctionCaller:
optimizer_cls = CPRandomOptimiser
else:
optimizer_cls = EuclideanRandomOptimiser
elif self._opt_arg.lower().startswith("bandit"):
if function_caller_cls == CPFunctionCaller:
optimizer_cls = CPGPBandit
else:
optimizer_cls = EuclideanGPBandit
elif self._opt_arg.lower().startswith("genetic"):
if function_caller_cls == CPFunctionCaller:
optimizer_cls = CPGAOptimiser
else:
raise ValueError(
"Currently only the `cartesian` domain works with "
"the `genetic` optimizer.")
else:
raise ValueError(
"Invalid optimizer specification. Either pass a full "
"dragonfly optimizer, or a string "
"in [random, bandit, genetic].")
assert optimizer_cls, "No optimizer could be determined."
domain_config = load_config({"domain": self._space})
function_caller = function_caller_cls(
None, domain_config.domain.list_of_domains[0])
self._opt = optimizer_cls(function_caller, ask_tell_mode=True)
self.init_dragonfly()
def main():
""" Main function. """
# First Specify all parameters
domain_vars = [
{
'name': 'rw',
'type': 'float',
'min': 0.05,
'max': 0.15,
'dim': 1
},
{
'name': 'L_Kw',
'type': 'float',
'min': 0,
'max': 1,
'dim': 2
},
{
'name': 'Tu',
'type': 'int',
'min': 63070,
'max': 115600,
'dim': ''
},
{
'name': 'Tl',
'type': 'float',
'min': 63.1,
'max': 116
},
{
'name': 'Hu_Hl',
'type': 'int',
'min': 0,
'max': 240,
'dim': 2
},
{
'name': 'r',
'type': 'float',
'min': 100,
'max': 50000
},
]
domain_constraints = [{
'constraint': 'np.sqrt(rw[0]) + L_Kw[1] <= 0.9'
}, {
'constraint': 'r/100.0 + Hu_Hl[1] < 200'
}]
fidel_vars = [
{
'name': 'fidel_0',
'type': 'float',
'min': 0.05,
'max': 0.25
},
{
'name': 'fidel_1',
'type': 'discrete_numeric',
'items': "0.1:0.05:1.01"
},
]
fidel_space_constraints = [{
'name': 'fsc1',
'constraint': 'fidel_0 + fidel_1 <= 0.9'
}]
fidel_to_opt = [0.1, 0.75]
# Budget of evaluations
max_num_evals = 100 # Optimisation budget (max number of evaluations)
max_mf_capital = max_num_evals * mf_cost(
fidel_to_opt) # Multi-fideltiy capital
# First do the MF version
config_params = {
'domain': domain_vars,
'fidel_space': fidel_vars,
'domain_constraints': domain_constraints,
'fidel_space_constraints': fidel_space_constraints,
'fidel_to_opt': fidel_to_opt
}
config = load_config(config_params)
# Optimise
mf_opt_pt, mf_opt_val, history = maximise_multifidelity_function(
mf_objective,
config.fidel_space,
config.domain,
config.fidel_to_opt,
mf_cost,
max_mf_capital,
config=config)
print(mf_opt_pt, mf_opt_val)
# Non-MF version
config_params = {
'domain': domain_vars,
'domain_constraints': domain_constraints
}
config = load_config(config_params)
max_capital = 100 # Optimisation budget (max number of evaluations)
# Optimise
opt_pt, opt_val, history = maximise_function(objective,
config.domain,
max_num_evals,
config=config)
print(opt_pt, opt_val)