def test_fetch_trials(self, storage): """Test fetch experiment trials""" with OrionState(experiments=[base_experiment], trials=generate_trials(), database=storage) as cfg: storage = cfg.storage() experiment = cfg.get_experiment('default_name', 'default_user', version=None) trials1 = storage.fetch_trials(experiment=experiment) trials2 = storage.fetch_trials(uid=experiment._id) with pytest.raises(MissingArguments): storage.fetch_trials() with pytest.raises(AssertionError): storage.fetch_trials(experiment=experiment, uid='123') assert len(trials1) == len(cfg.trials), 'trial count should match' assert len(trials2) == len(cfg.trials), 'trial count should match'
def test_good_set_before_init_no_hit(self, random_dt, new_config): """Trying to set, overwrite everything from input.""" with OrionState(experiments=[], trials=[]): exp = experiment_builder.build(**new_config) found_config = list(get_storage().fetch_experiments({ 'name': 'supernaekei', 'metadata.user': '******' })) new_config['metadata']['datetime'] = exp.metadata['datetime'] assert len(found_config) == 1 _id = found_config[0].pop('_id') assert _id != 'fasdfasfa' assert exp._id == _id new_config['refers'] = {} new_config.pop('_id') new_config.pop('something_to_be_ignored') new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 new_config['algorithms']['dumbalgo']['seed'] = None new_config['refers'] = { 'adapter': [], 'parent_id': None, 'root_id': _id } assert found_config[0] == new_config assert exp.name == new_config['name'] assert exp.configuration['refers'] == new_config['refers'] assert exp.metadata == new_config['metadata'] assert exp.pool_size == new_config['pool_size'] assert exp.max_trials == new_config['max_trials'] assert exp.working_dir == new_config['working_dir'] assert exp.version == new_config['version'] assert exp.algorithms.configuration == new_config['algorithms']
def test_build_from_args_hit(old_config_file, script_path, new_config): """Try building experiment when in db (no branch)""" cmdargs = { 'name': 'supernaekei', 'config': old_config_file, 'user_args': [script_path, '--mini-batch~uniform(32, 256, discrete=True)'] } with OrionState(experiments=[new_config], trials=[]): # Test that experiment already exists experiment_builder.build_view_from_args(cmdargs) exp = experiment_builder.build_from_args(cmdargs) assert exp._id == new_config['_id'] assert exp.name == new_config['name'] assert exp.version == 1 assert exp.configuration['refers'] == new_config['refers'] assert exp.metadata == new_config['metadata'] assert exp.max_trials == new_config['max_trials'] assert exp.algorithms.configuration == new_config['algorithms']
def test_optimizer_actually_optimize(monkeypatch): """Check if Bayesian Optimizer has better optimization than random search.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) best_random_search = 23.403275057472825 with OrionState(experiments=[], trials=[]): orion.core.cli.main([ "hunt", "--config", "./benchmark/meshadaptivedirectsearch.yaml", "./benchmark/rosenbrock.py", "--max-trials", 100, "-x~uniform(-50, 50)", ]) exp = create_experiment(name="exp") objective = exp.stats["best_evaluation"] assert best_random_search > objective
def retrieve_result(self, storage, generated_result): """Test retrieve result""" results_file = tempfile.NamedTemporaryFile(mode='w', prefix='results_', suffix='.log', dir='.', delete=True) # Generate fake result with open(results_file.name, 'w') as file: json.dump([generated_result], file) # -- with OrionState(experiments=[], trials=[], database=storage) as cfg: storage = cfg.storage() trial = Trial(**base_trial) trial = storage.retrieve_result(trial, results_file) results = trial.results assert len(results) == 1 assert results[0].to_dict() == generated_result
def test_is_done_property_with_pending(algorithm): """Check experiment stopping conditions when there is pending trials.""" completed = ['completed'] * 10 reserved = ['reserved'] * 5 with OrionState(trials=generate_trials(completed + reserved)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 10 assert exp.is_done exp.max_trials = 15 # There is only 10 completed trials assert not exp.is_done exp.algorithms.algorithm.done = True # Algorithm is done but 5 trials are pending assert not exp.is_done
def test_optimizer_actually_optimize(monkeypatch): """Check if Bayesian Optimizer has better optimization than random search.""" monkeypatch.chdir(os.path.dirname(os.path.abspath(__file__))) best_random_search = 23.403275057472825 with OrionState(experiments=[], trials=[]): orion.core.cli.main([ "hunt", "--name", "exp", "--max-trials", "20", "--config", "./benchmark/hyperband.yaml", "./benchmark/rosenbrock.py", "-x~uniform(-50, 50)" ]) with open("./benchmark/hyperband.yaml", "r") as f: exp = ExperimentBuilder().build_view_from({ 'name': 'exp', 'config': f }) objective = exp.stats['best_evaluation'] assert best_random_search > objective
def test_view_is_done_property_no_pending(algorithm): """Check experiment stopping conditions from view when there is no pending trials.""" completed = ['completed'] * 10 broken = ['broken'] * 5 with OrionState(trials=generate_trials(completed + broken)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 100 exp_view = ExperimentView(exp) exp.algorithms = algorithm exp.max_trials = 15 # There is only 10 completed trials and algo not done. assert not exp_view.is_done exp.algorithms.algorithm.done = True # Algorithm is done and no pending trials assert exp_view.is_done
def test_fix_lost_trials_configurable_hb(self): """Test that heartbeat is correctly being configured.""" trial = copy.deepcopy(base_trial) trial['status'] = 'reserved' trial['heartbeat'] = datetime.datetime.utcnow() - datetime.timedelta( seconds=180) with OrionState(trials=[trial]) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert len(exp.fetch_trials_by_status('reserved')) == 1 orion.core.config.worker.heartbeat = 360 exp.fix_lost_trials() assert len(exp.fetch_trials_by_status('reserved')) == 1 orion.core.config.worker.heartbeat = 180 exp.fix_lost_trials() assert len(exp.fetch_trials_by_status('reserved')) == 0
def test_good_set_before_init_hit_no_diffs_exc_max_trials( self, new_config): """Trying to set, and NO differences were found from the config pulled from db. Everything is normal, nothing changes. Experiment is resumed, perhaps with more trials to evaluate (an exception is 'max_trials'). """ with OrionState(experiments=[new_config], trials=[]): new_config['max_trials'] = 5000 exp = experiment_builder.build(**new_config) # Deliver an external configuration to finalize init new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 new_config['algorithms']['dumbalgo']['seed'] = None new_config['producer']['strategy'] = "NoParallelStrategy" new_config.pop('something_to_be_ignored') assert exp.configuration == new_config
def test_fix_only_lost_trials(self): """Test that an old trial is set to interrupted but not a recent one.""" lost_trial, running_trial = generate_trials(['reserved'] * 2) lost_trial['heartbeat'] = datetime.datetime.utcnow( ) - datetime.timedelta(seconds=360) running_trial['heartbeat'] = datetime.datetime.utcnow() with OrionState(trials=[lost_trial, running_trial]) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert len(exp.fetch_trials_by_status('reserved')) == 2 exp.fix_lost_trials() reserved_trials = exp.fetch_trials_by_status('reserved') assert len(reserved_trials) == 1 assert reserved_trials[0].to_dict( )['params'] == running_trial['params'] failedover_trials = exp.fetch_trials_by_status('interrupted') assert len(failedover_trials) == 1 assert failedover_trials[0].to_dict( )['params'] == lost_trial['params']
def test_try_set_after_race_condition(self, new_config, monkeypatch): """Cannot set a configuration after init if it looses a race condition. The experiment from process which first writes to db is initialized properly. The experiment which looses the race condition cannot be initialized and needs to be rebuilt. """ with OrionState(experiments=[new_config], trials=[]): experiment_count_before = count_experiments() def insert_race_condition(*args, **kwargs): if insert_race_condition.count == 0: data = {} else: data = new_config insert_race_condition.count += 1 return data insert_race_condition.count = 0 monkeypatch.setattr(experiment_builder, 'fetch_config_from_db', insert_race_condition) experiment_builder.build(**new_config) assert experiment_count_before == count_experiments() # Should be called # - once in build(), # -> then register fails, # - then called once again in build, # - then called in build_view to evaluate the conflicts assert insert_race_condition.count == 3
def test_race_condition_wout_version(self, monkeypatch): """Test that an experiment loosing the race condition during version increment raises RaceCondition if version number was not specified. """ name = 'parent' space = {'x': 'uniform(0,10)'} with OrionState(experiments=[], trials=[]): parent = experiment_builder.build(name, space=space) child = experiment_builder.build(name=name, space={'x': 'loguniform(1,10)'}) assert child.name == parent.name assert parent.version == 1 assert child.version == 2 # Either # 1. # fetch_config_from_db only fetch parent # test_version finds other child # -> Detect race condition looking at conflicts # 2. # fetch_config_from_db only fetch parent # test_version do not find other child # -> DuplicateKeyError def insert_race_condition_1(self, query): is_auto_version_query = (query == { 'name': name, 'refers.parent_id': parent.id }) if is_auto_version_query: data = [child.configuration] # First time the query returns no other child elif insert_race_condition_1.count < 1: data = [parent.configuration] else: data = [parent.configuration, child.configuration] insert_race_condition_1.count += int(is_auto_version_query) return data insert_race_condition_1.count = 0 monkeypatch.setattr(get_storage().__class__, 'fetch_experiments', insert_race_condition_1) with pytest.raises(RaceCondition) as exc_info: experiment_builder.build(name=name, space={'x': 'loguniform(1,10)'}) assert 'There was likely a race condition during version' in str( exc_info.value) def insert_race_condition_2(self, query): is_auto_version_query = (query == { 'name': name, 'refers.parent_id': parent.id }) # First time the query returns no other child if is_auto_version_query: data = [] elif insert_race_condition_2.count < 1: data = [parent.configuration] else: data = [parent.configuration, child.configuration] insert_race_condition_2.count += int(is_auto_version_query) return data insert_race_condition_2.count = 0 monkeypatch.setattr(get_storage().__class__, 'fetch_experiments', insert_race_condition_2) with pytest.raises(RaceCondition) as exc_info: experiment_builder.build(name=name, space={'x': 'loguniform(1,10)'}) assert 'There was a race condition during branching.' in str( exc_info.value)
def test_reserve_none(self): """Find nothing, return None.""" with OrionState(experiments=[], trials=[]): exp = Experiment('supernaekei') trial = exp.reserve_trial() assert trial is None
def test_register_lie(self, storage): """Test register lie""" with OrionState(experiments=[base_experiment], storage=storage) as cfg: storage = cfg.storage() storage.register_lie(Trial(**base_trial))
def test_race_condition_w_version(self, monkeypatch): """Test that an experiment loosing the race condition during version increment cannot be resolved automatically if a version number was specified. Note that if we would raise RaceCondition, the conflict would still occur since the version number fetched will not be the new one from the resolution but the requested one. Therefore raising and handling RaceCondition would lead to infinite recursion in the experiment builder. """ name = 'parent' space = {'x': 'uniform(0,10)'} with OrionState(experiments=[], trials=[]): parent = experiment_builder.build(name, space=space) child = experiment_builder.build(name=name, space={'x': 'loguniform(1,10)'}) assert child.name == parent.name assert parent.version == 1 assert child.version == 2 # Either # 1. # fetch_config_from_db only fetch parent # test_version finds other child # -> Detect race condition looking at conflicts # 2. # fetch_config_from_db only fetch parent # test_version do not find other child # -> DuplicateKeyError def insert_race_condition_1(self, query): is_auto_version_query = (query == { 'name': name, 'refers.parent_id': parent.id }) if is_auto_version_query: data = [child.configuration] # First time the query returns no other child elif insert_race_condition_1.count < 1: data = [parent.configuration] else: data = [parent.configuration, child.configuration] insert_race_condition_1.count += int(is_auto_version_query) return data insert_race_condition_1.count = 0 monkeypatch.setattr(get_storage().__class__, 'fetch_experiments', insert_race_condition_1) with pytest.raises(BranchingEvent) as exc_info: experiment_builder.build(name=name, version=1, space={'x': 'loguniform(1,10)'}) assert 'Configuration is different and generates' in str( exc_info.value) def insert_race_condition_2(self, query): is_auto_version_query = (query == { 'name': name, 'refers.parent_id': parent.id }) # First time the query returns no other child if is_auto_version_query: data = [] elif insert_race_condition_2.count < 1: data = [parent.configuration] else: data = [parent.configuration, child.configuration] insert_race_condition_2.count += int(is_auto_version_query) return data insert_race_condition_2.count = 0 monkeypatch.setattr(get_storage().__class__, 'fetch_experiments', insert_race_condition_2) with pytest.raises(RaceCondition) as exc_info: experiment_builder.build(name=name, version=1, space={'x': 'loguniform(1,10)'}) assert 'There was a race condition during branching.' in str( exc_info.value)
def test_fetch_config_from_db_no_hit(): """Verify that fetch_config_from_db returns an empty dict when the experiment is not in db""" with OrionState(experiments=[], trials=[]): db_config = experiment_builder.fetch_config_from_db(name='supernaekei') assert db_config == {}
def test_old_experiment_wout_version(self, parent_version_config): """Create an already existing experiment without a version.""" with OrionState(experiments=[parent_version_config]): exp = experiment_builder.build(name=parent_version_config["name"]) assert exp.version == 1
def test_build_no_commandline_config(): """Try building experiment with no commandline configuration.""" with OrionState(experiments=[], trials=[]): with pytest.raises(NoConfigurationError): experiment_builder.build('supernaekei')