def test_fetch_results_all_completed(client):
defaults = {'a': 1000, 'b': 1001}
params = {'c': 2, 'd': 3, 'epoch': 5}
defaults.update(params)
medians = ['a']
num_items = 2
configs = generate(range(num_items), 'ab', defaults=defaults)
namespace = 'test'
register(client, foo, namespace, configs)
print(configs)
worker = TrialWorker(URI, DATABASE, 0, None)
worker.max_retry = 0
worker.timeout = 1
worker.run()
print(fetch_vars_stats(client, namespace))
data = fetch_results(client, namespace, configs, medians, params, defaults)
assert data.medians == ['a']
assert data.noise.values.tolist() == ['a', 'b']
assert data.params.values.tolist() == ['c', 'd']
assert data.order.values.tolist() == [0, 1]
assert data.epoch.values.tolist() == list(range(params['epoch'] + 1))
assert data.uid.shape == (3, 2)
assert data.seed.values.tolist(
) == data.noise.values.tolist() + ['reference']
assert data.a.values.tolist() == [[0, 1000, 1000], [1, 1000, 1000]]
assert data.b.values.tolist() == [[1001, 0, 1001], [1001, 1, 1001]]
assert data.c.values.tolist() == [[2, 2, 2], [2, 2, 2]]
assert data.d.values.tolist() == [[3, 3, 3], [3, 3, 3]]
assert (data.obj.loc[dict(order=0, seed='a')].values.tolist() == list(
range(2002, 2002 + params['epoch'] + 1)))
def test_consolidate_results(client):
num_experiments = 5
num_replicates = 10
space = {
'a': 'uniform(lower=-1, upper=1)',
'b': 'uniform(lower=-1, upper=1)',
'c': 'uniform(lower=-1, upper=1)'
}
variables = {'d': 5}
defaults = {'e': 2, 'epoch': 5}
seed = 2
hpo = 'random_search'
objective = 'obj'
fidelity = Fidelity(5, 5, name='epoch').to_dict()
surrogate_budget = 10
hpo_budget = 5
configs = generate_hpos(list(range(num_experiments)), [hpo],
budget=surrogate_budget,
fidelity=fidelity,
search_space=space,
namespace=NAMESPACE,
defaults=defaults)
to_replicate = get_configs_to_replicate(configs, num_experiments)
reset_pool_size(configs['random_search'])
randomize_seeds(configs['random_search'], variables, seed)
variable_names = list(sorted(variables.keys()))
hpo_stats = fetch_all_hpo_stats(client, NAMESPACE)
namespaces = register_hpos(client, NAMESPACE, foo, configs, defaults,
hpo_stats)
worker = TrialWorker(URI, DATABASE, 0, None)
worker.max_retry = 0
worker.timeout = 0.02
worker.run()
data = defaultdict(dict)
hpos_ready, remainings = fetch_hpos_valid_curves(client, namespaces,
variable_names, data)
ready_configs = get_ready_configs(hpos_ready, configs, to_replicate)
replicates = generate_replicates(ready_configs,
data,
variables,
objective,
hpo_budget,
num_replicates,
early_stopping=False)
register(client, foo, NAMESPACE, replicates)
worker = TrialWorker(URI, DATABASE, 0, None)
worker.max_retry = 0
worker.timeout = 0.02
worker.run()
print(fetch_vars_stats(client, NAMESPACE))
data = fetch_hpos_replicates(client, configs, replicates, variable_names,
space, data)
data = consolidate_results(data)
assert len(data) == 1
assert len(data['random_search']) == 4
hpo_reps = data['random_search']
assert hpo_reps['ideal'].obj.shape == (6, surrogate_budget,
num_experiments)
assert hpo_reps['biased'].obj.shape == (6, num_replicates, num_experiments)
assert hpo_reps['simul-fix'].obj.shape == (6, num_replicates,
num_experiments)
assert hpo_reps['simul-free'].obj.shape == (6, num_replicates,
num_experiments)
def count_unique(attr):
return len(set(attr.values.reshape(-1).tolist()))
# Test sources of variation
# NOTE: In ideal, source of variation will vary across ideal after consolidation
# but it stays fixed during the HPO itself
assert count_unique(hpo_reps['ideal']['d']) == num_experiments
assert count_unique(hpo_reps['biased']['d']) == (num_replicates *
num_experiments)
assert count_unique(hpo_reps['simul-free']['d']) == (num_replicates *
num_experiments)
assert count_unique(hpo_reps['simul-fix']['d']) == num_experiments
# Test HPs
assert count_unique(hpo_reps['ideal']['a']) == (num_experiments *
surrogate_budget)
assert count_unique(hpo_reps['biased']['a']) == num_experiments
assert count_unique(hpo_reps['simul-free']['a']) == (num_replicates *
num_experiments)
assert count_unique(hpo_reps['simul-fix']['a']) == (num_replicates *
num_experiments)
assert numpy.allclose(hpo_reps['simul-free']['a'].values,
hpo_reps['simul-fix']['a'].values)
def get_stats(variables):
trial_stats = fetch_vars_stats(client, namespace)
print(trial_stats)
return {v: trial_stats[f'test-{v}'] for v in variables}
def count_registered():
status = fetch_vars_stats(client, NAMESPACE)
return sum(status[key]['count'] for key in status.keys())