def test_is_exsited(self):
def get_space():
space = HyperSpace()
with space.as_default():
id1 = Identity(p1=Choice(['a', 'b']),
p2=Int(1, 100),
p3=Real(0, 1.0))
return space
th = TrailHistory('min')
sample = get_space()
sample.random_sample()
trail = Trail(sample, 1, 0.99, 100)
th.append(trail)
sample2 = get_space()
sample2.assign_by_vectors(sample.vectors)
assert th.is_existed(sample2)
t = th.get_trail(sample2)
assert t.reward == 0.99
assert t.elapsed == 100
assert t.trail_no == 1
sample3 = get_space()
sample3.random_sample()
assert not th.is_existed(sample3)
def run_searcher(searcher,
max_trails=None,
max_time_budget=5e6,
use_meta_learner=True):
history = TrailHistory('max')
if use_meta_learner:
disk_trail_store = DiskTrailStore(f'{test_output_dir}/trail_store')
disk_trail_store.clear_history()
meta_learner = MetaLearner(history, 'nas_bench_101', disk_trail_store)
searcher.set_meta_learner(meta_learner)
nasbench.reset_budget_counters()
times, best_valids, best_tests = [0.0], [0.0], [0.0]
trail_no = 0
while True:
trail_no += 1
if max_trails is not None and trail_no > max_trails:
break
sample = searcher.sample()
matrix, ops = hyn_nasbench.sample2spec(sample)
model_spec = api.ModelSpec(matrix=matrix, ops=ops)
data = nasbench.query(model_spec)
if data['validation_accuracy'] > best_valids[-1]:
best_valids.append(data['validation_accuracy'])
best_tests.append(data['test_accuracy'])
else:
best_valids.append(best_valids[-1])
best_tests.append(best_tests[-1])
time_spent, _ = nasbench.get_budget_counters()
times.append(time_spent)
reward = data['test_accuracy']
trail = Trail(sample, trail_no, reward, data['training_time'])
history.append(trail)
searcher.update_result(sample, reward)
if time_spent > max_time_budget:
# Break the first time we exceed the budget.
break
return times, best_valids, best_tests
def test_mcts_searcher_parallelize(self):
searcher = MCTSSearcher(self.get_space, max_node_space=10)
history = TrailHistory(OptimizeDirection.Maximize)
disk_trail_store = DiskTrailStore(f'{test_output_dir}/trail_store')
meta_learner = MetaLearner(history, 'test_mcts_searcher_parallelize',
disk_trail_store)
searcher.set_meta_learner(meta_learner)
trail_no = 1
running_samples = []
for i in range(10):
space_sample = searcher.sample()
running_samples.append(space_sample)
assert searcher.tree.root.visits == 10
for sample in running_samples:
reward = np.random.uniform(0.1, 0.9)
trail = Trail(sample, trail_no, reward, 10)
history.append(trail)
searcher.update_result(sample, reward)
trail_no += 1
assert searcher.tree.root.visits == 10
assert len(searcher.nodes_map.items()) == 10
running_samples = []
for i in range(10):
space_sample = searcher.sample()
running_samples.append(space_sample)
assert searcher.tree.root.visits == 20
for sample in running_samples:
reward = np.random.uniform(0.1, 0.9)
trail = Trail(sample, trail_no, reward, 10)
history.append(trail)
searcher.update_result(sample, reward)
assert searcher.tree.root.visits == 20
def test_save_load(self):
def get_space():
space = HyperSpace()
with space.as_default():
id1 = Identity(p1=Choice(['a', 'b']),
p2=Int(1, 100),
p3=Real(0, 1.0))
return space
th = TrailHistory('min')
sample = get_space()
sample.assign_by_vectors([0, 1, 0.1])
trail = Trail(sample, 1, 0.99, 100)
th.append(trail)
sample = get_space()
sample.assign_by_vectors([1, 2, 0.2])
trail = Trail(sample, 2, 0.9, 50)
th.append(trail)
sample = get_space()
sample.assign_by_vectors([0, 3, 0.3])
trail = Trail(sample, 3, 0.7, 200)
th.append(trail)
filepath = f'{test_output_dir}/history.txt'
th.save(filepath)
with open(filepath) as f:
lines = f.readlines()
# assert lines == ['min\n', '1|[0, 1, 0.1]|0.99|100\n', '2|[1, 2, 0.2]|0.9|50\n', '3|[0, 3, 0.3]|0.7|200\n']
history = TrailHistory.load_history(get_space, filepath)
assert history.optimize_direction == 'min'
assert len(history.history) == 3
assert history.history[0].space_sample.vectors == [0, 1, 0.1]
assert history.history[0].elapsed == 100.0
assert history.history[0].reward == 0.99
assert history.history[0].trail_no == 1
trajectories = history.get_trajectories()
assert trajectories == ([0.0, 100.0, 150.0,
350.0], [0.0, 0.99, 0.99,
0.99], [0.0, 0.99, 0.9,
0.7], 1, 100.0)