def test_is_exsited(self): def get_space(): space = HyperSpace() with space.as_default(): id1 = Identity(p1=Choice(['a', 'b']), p2=Int(1, 100), p3=Real(0, 1.0)) return space th = TrailHistory('min') sample = get_space() sample.random_sample() trail = Trail(sample, 1, 0.99, 100) th.append(trail) sample2 = get_space() sample2.assign_by_vectors(sample.vectors) assert th.is_existed(sample2) t = th.get_trail(sample2) assert t.reward == 0.99 assert t.elapsed == 100 assert t.trail_no == 1 sample3 = get_space() sample3.random_sample() assert not th.is_existed(sample3)
def run_searcher(searcher, max_trails=None, max_time_budget=5e6, use_meta_learner=True): history = TrailHistory('max') if use_meta_learner: disk_trail_store = DiskTrailStore(f'{test_output_dir}/trail_store') disk_trail_store.clear_history() meta_learner = MetaLearner(history, 'nas_bench_101', disk_trail_store) searcher.set_meta_learner(meta_learner) nasbench.reset_budget_counters() times, best_valids, best_tests = [0.0], [0.0], [0.0] trail_no = 0 while True: trail_no += 1 if max_trails is not None and trail_no > max_trails: break sample = searcher.sample() matrix, ops = hyn_nasbench.sample2spec(sample) model_spec = api.ModelSpec(matrix=matrix, ops=ops) data = nasbench.query(model_spec) if data['validation_accuracy'] > best_valids[-1]: best_valids.append(data['validation_accuracy']) best_tests.append(data['test_accuracy']) else: best_valids.append(best_valids[-1]) best_tests.append(best_tests[-1]) time_spent, _ = nasbench.get_budget_counters() times.append(time_spent) reward = data['test_accuracy'] trail = Trail(sample, trail_no, reward, data['training_time']) history.append(trail) searcher.update_result(sample, reward) if time_spent > max_time_budget: # Break the first time we exceed the budget. break return times, best_valids, best_tests
def test_mcts_searcher_parallelize(self): searcher = MCTSSearcher(self.get_space, max_node_space=10) history = TrailHistory(OptimizeDirection.Maximize) disk_trail_store = DiskTrailStore(f'{test_output_dir}/trail_store') meta_learner = MetaLearner(history, 'test_mcts_searcher_parallelize', disk_trail_store) searcher.set_meta_learner(meta_learner) trail_no = 1 running_samples = [] for i in range(10): space_sample = searcher.sample() running_samples.append(space_sample) assert searcher.tree.root.visits == 10 for sample in running_samples: reward = np.random.uniform(0.1, 0.9) trail = Trail(sample, trail_no, reward, 10) history.append(trail) searcher.update_result(sample, reward) trail_no += 1 assert searcher.tree.root.visits == 10 assert len(searcher.nodes_map.items()) == 10 running_samples = [] for i in range(10): space_sample = searcher.sample() running_samples.append(space_sample) assert searcher.tree.root.visits == 20 for sample in running_samples: reward = np.random.uniform(0.1, 0.9) trail = Trail(sample, trail_no, reward, 10) history.append(trail) searcher.update_result(sample, reward) assert searcher.tree.root.visits == 20
def test_save_load(self): def get_space(): space = HyperSpace() with space.as_default(): id1 = Identity(p1=Choice(['a', 'b']), p2=Int(1, 100), p3=Real(0, 1.0)) return space th = TrailHistory('min') sample = get_space() sample.assign_by_vectors([0, 1, 0.1]) trail = Trail(sample, 1, 0.99, 100) th.append(trail) sample = get_space() sample.assign_by_vectors([1, 2, 0.2]) trail = Trail(sample, 2, 0.9, 50) th.append(trail) sample = get_space() sample.assign_by_vectors([0, 3, 0.3]) trail = Trail(sample, 3, 0.7, 200) th.append(trail) filepath = f'{test_output_dir}/history.txt' th.save(filepath) with open(filepath) as f: lines = f.readlines() # assert lines == ['min\n', '1|[0, 1, 0.1]|0.99|100\n', '2|[1, 2, 0.2]|0.9|50\n', '3|[0, 3, 0.3]|0.7|200\n'] history = TrailHistory.load_history(get_space, filepath) assert history.optimize_direction == 'min' assert len(history.history) == 3 assert history.history[0].space_sample.vectors == [0, 1, 0.1] assert history.history[0].elapsed == 100.0 assert history.history[0].reward == 0.99 assert history.history[0].trail_no == 1 trajectories = history.get_trajectories() assert trajectories == ([0.0, 100.0, 150.0, 350.0], [0.0, 0.99, 0.99, 0.99], [0.0, 0.99, 0.9, 0.7], 1, 100.0)