def test_update_completed_trial(random_dt): """Successfully push a completed trial into database.""" with OrionState(trials=generate_trials(["new"])) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] trial = exp.reserve_trial() results_file = tempfile.NamedTemporaryFile(mode="w", prefix="results_", suffix=".log", dir=".", delete=True) # Generate fake result with open(results_file.name, "w") as file: json.dump([{ "name": "loss", "type": "objective", "value": 2 }], file) # -- exp.update_completed_trial(trial, results_file=results_file) yo = get_storage().fetch_trials(exp)[0].to_dict() assert len(yo["results"]) == len(trial.results) assert yo["results"][0] == trial.results[0].to_dict() assert yo["status"] == "completed" assert yo["end_time"] == random_dt results_file.close()
def test_update_completed_trial(random_dt): """Successfully push a completed trial into database.""" with OrionState(trials=generate_trials(['new'])) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] trial = exp.reserve_trial() results_file = tempfile.NamedTemporaryFile(mode='w', prefix='results_', suffix='.log', dir='.', delete=True) # Generate fake result with open(results_file.name, 'w') as file: json.dump([{ 'name': 'loss', 'type': 'objective', 'value': 2 }], file) # -- exp.update_completed_trial(trial, results_file=results_file) yo = get_storage().fetch_trials(exp)[0].to_dict() assert len(yo['results']) == len(trial.results) assert yo['results'][0] == trial.results[0].to_dict() assert yo['status'] == 'completed' assert yo['end_time'] == random_dt results_file.close()
def test_register_trials(random_dt): """Register a list of newly proposed trials/parameters.""" with OrionState(): exp = Experiment('supernaekei') exp._id = 0 trials = [ Trial(params=[{ 'name': 'a', 'type': 'integer', 'value': 5 }]), Trial(params=[{ 'name': 'b', 'type': 'integer', 'value': 6 }]), ] for trial in trials: exp.register_trial(trial) yo = list( map(lambda trial: trial.to_dict(), get_storage().fetch_trials(exp))) assert len(yo) == len(trials) assert yo[0]['params'] == list( map(lambda x: x.to_dict(), trials[0]._params)) assert yo[1]['params'] == list( map(lambda x: x.to_dict(), trials[1]._params)) assert yo[0]['status'] == 'new' assert yo[1]['status'] == 'new' assert yo[0]['submit_time'] == random_dt assert yo[1]['submit_time'] == random_dt
def test_fix_lost_trials_race_condition(self, monkeypatch, caplog): """Test that a lost trial fixed by a concurrent process does not cause error.""" trial = copy.deepcopy(base_trial) trial['status'] = 'interrupted' trial['heartbeat'] = datetime.datetime.utcnow() - datetime.timedelta( seconds=360) with OrionState(trials=[trial]) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert len(exp.fetch_trials_by_status('interrupted')) == 1 assert len(exp._storage.fetch_lost_trials(exp)) == 0 def fetch_lost_trials(self, query): trial_object = Trial(**trial) trial_object.status = 'reserved' return [trial_object] # Force the fetch of a trial marked as reserved (and lost) while actually interrupted # (as if already failed-over by another process). with monkeypatch.context() as m: m.setattr(exp._storage.__class__, 'fetch_lost_trials', fetch_lost_trials) assert len(exp._storage.fetch_lost_trials(exp)) == 1 with caplog.at_level(logging.DEBUG): exp.fix_lost_trials() assert caplog.records[-1].levelname == 'DEBUG' assert caplog.records[-1].msg == 'failed' assert len(exp.fetch_trials_by_status('interrupted')) == 1 assert len(exp.fetch_trials_by_status('reserved')) == 0
def test_register_trials(random_dt): """Register a list of newly proposed trials/parameters.""" with OrionState(): exp = Experiment("supernaekei", mode="x") exp._id = 0 trials = [ Trial(params=[{ "name": "a", "type": "integer", "value": 5 }]), Trial(params=[{ "name": "b", "type": "integer", "value": 6 }]), ] for trial in trials: exp.register_trial(trial) yo = list( map(lambda trial: trial.to_dict(), get_storage().fetch_trials(exp))) assert len(yo) == len(trials) assert yo[0]["params"] == list( map(lambda x: x.to_dict(), trials[0]._params)) assert yo[1]["params"] == list( map(lambda x: x.to_dict(), trials[1]._params)) assert yo[0]["status"] == "new" assert yo[1]["status"] == "new" assert yo[0]["submit_time"] == random_dt assert yo[1]["submit_time"] == random_dt
def hacked_exp(with_user_dendi, random_dt, clean_db, create_db_instance): """Return an `Experiment` instance with hacked _id to find trials in fake database. """ exp = Experiment('supernaedo2-dendi') exp._id = 'supernaedo2-dendi' # white box hack return exp
def test_fetch_all_trials(): """Fetch a list of all trials""" with OrionState(trials=generate_trials(["new", "reserved", "completed"])) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] trials = list(map(lambda trial: trial.to_dict(), exp.fetch_trials({}))) assert trials == cfg.trials
def test_reserve_when_exhausted(self): """Return None once all the trials have been allocated""" stati = ["new", "reserved", "interrupted", "completed", "broken"] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] assert exp.reserve_trial() is not None assert exp.reserve_trial() is not None assert exp.reserve_trial() is None
def test_fetch_all_trials(): """Fetch a list of all trials""" with OrionState( trials=generate_trials(['new', 'reserved', 'completed'])) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] trials = list(map(lambda trial: trial.to_dict(), exp.fetch_trials({}))) assert trials == cfg.trials
def test_broken_property(): """Check experiment stopping conditions for maximum number of broken.""" MAX_BROKEN = 3 orion.core.config.worker.max_broken = MAX_BROKEN stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN - 1)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert not exp.is_broken stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert exp.is_broken
def test_reserve_when_exhausted(self): """Return None once all the trials have been allocated""" stati = ['new', 'reserved', 'interrupted', 'completed', 'broken'] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert exp.reserve_trial() is not None assert exp.reserve_trial() is not None assert exp.reserve_trial() is None
def test_fetch_completed_trials_from_view(): """Fetch a list of the unseen yet completed trials.""" non_completed_stati = ['new', 'interrupted', 'suspended', 'reserved'] stati = non_completed_stati + ['completed'] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp_view = ExperimentView(exp) trials = exp_view.fetch_trials_by_status('completed') assert len(trials) == 1 assert trials[0].status == 'completed'
def test_broken_property(): """Check experiment stopping conditions for maximum number of broken.""" MAX_BROKEN = 5 stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN - 1)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.max_broken = MAX_BROKEN assert not exp.is_broken stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.max_broken = MAX_BROKEN assert exp.is_broken
def test_fix_lost_trials(self): """Test that a running trial with an old heartbeat is set to interrupted.""" trial = copy.deepcopy(base_trial) trial["status"] = "reserved" trial["heartbeat"] = datetime.datetime.utcnow() - datetime.timedelta( seconds=60 * 10) with OrionState(trials=[trial]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] assert len(exp.fetch_trials_by_status("reserved")) == 1 exp.fix_lost_trials() assert len(exp.fetch_trials_by_status("reserved")) == 0
def test_fix_lost_trials(self): """Test that a running trial with an old heartbeat is set to interrupted.""" trial = copy.deepcopy(base_trial) trial['status'] = 'reserved' trial['heartbeat'] = datetime.datetime.utcnow() - datetime.timedelta( seconds=360) with OrionState(trials=[trial]) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert len(exp.fetch_trials_by_status('reserved')) == 1 exp.fix_lost_trials() assert len(exp.fetch_trials_by_status('reserved')) == 0
def create_experiment(name, version, space, **kwargs): """Instantiate the experiment and its attribute objects All unspecified arguments will be replaced by system's defaults (orion.core.config.*). Parameters ---------- name: str Name of the experiment. version: int Version of the experiment. space: dict or Space object Optimization space of the algorithm. If dict, should have the form `dict(name='<prior>(args)')`. algorithms: str or dict, optional Algorithm used for optimization. strategy: str or dict, optional Parallel strategy to use to parallelize the algorithm. max_trials: int, optional Maximum number or trials before the experiment is considered done. storage: dict, optional Configuration of the storage backend. """ experiment = Experiment(name=name, version=version) experiment._id = kwargs.get('_id', None) # pylint:disable=protected-access experiment.pool_size = kwargs.get('pool_size') if experiment.pool_size is None: experiment.pool_size = orion.core.config.experiment.get( 'pool_size', deprecated='ignore') experiment.max_trials = kwargs.get('max_trials', orion.core.config.experiment.max_trials) experiment.space = _instantiate_space(space) experiment.algorithms = _instantiate_algo(experiment.space, kwargs.get('algorithms')) experiment.producer = kwargs.get('producer', {}) experiment.producer['strategy'] = _instantiate_strategy( experiment.producer.get('strategy')) experiment.working_dir = kwargs.get( 'working_dir', orion.core.config.experiment.working_dir) experiment.metadata = kwargs.get( 'metadata', {'user': kwargs.get('user', getpass.getuser())}) experiment.refers = kwargs.get('refers', { 'parent_id': None, 'root_id': None, 'adapter': [] }) experiment.refers['adapter'] = _instantiate_adapters( experiment.refers.get('adapter', [])) return experiment
def test_fetch_non_completed_trials(): """Fetch a list of the trials that are not completed trials.status in ['new', 'interrupted', 'suspended', 'broken'] """ non_completed_stati = ["new", "interrupted", "suspended", "reserved"] stati = non_completed_stati + ["completed"] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] trials = exp.fetch_noncompleted_trials() assert len(trials) == 4 assert set(trial.status for trial in trials) == set(non_completed_stati)
def hacked_exp(with_user_dendi, random_dt, clean_db): """Return an `Experiment` instance with hacked _id to find trials in fake database. """ try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass exp = Experiment('supernaedo2') exp._id = 'supernaedo2' # white box hack return exp
def test_configurable_broken_property(): """Check if max_broken changes after configuration.""" MAX_BROKEN = 3 orion.core.config.worker.max_broken = MAX_BROKEN stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert exp.is_broken orion.core.config.worker.max_broken += 1 assert not exp.is_broken
def test_experiment_stats(): """Check that property stats is returning a proper summary of experiment's results.""" NUM_COMPLETED = 3 stati = (["completed"] * NUM_COMPLETED) + (["reserved"] * 2) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.metadata = {"datetime": datetime.datetime.utcnow()} stats = exp.stats assert stats.trials_completed == NUM_COMPLETED assert stats.best_trials_id == cfg.trials[3]["_id"] assert stats.best_evaluation == 0 assert stats.start_time == exp.metadata["datetime"] assert stats.finish_time == cfg.trials[0]["end_time"] assert stats.duration == stats.finish_time - stats.start_time
def test_acquire_algorithm_lock_with_different_config(self, new_config, algorithm): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 algorithm_original_config = algorithm.configuration exp.algorithms = algorithm # Setting attribute to algorithm inside the wrapper algorithm.algorithm.seed = 10 assert algorithm.configuration != algorithm_original_config with pytest.raises( RuntimeError, match="Algorithm configuration changed since" ): with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): pass
def test_acquire_algorithm_lock_timeout(self, new_config, algorithm, mocker): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 exp.algorithms = algorithm storage_acquisition_mock = mocker.spy( cfg.storage(), "acquire_algorithm_lock" ) with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): pass storage_acquisition_mock.assert_called_with( experiment=exp, timeout=0.2, retry_interval=0.1 )
def test_configurable_broken_property(): """Check if max_broken changes after configuration.""" MAX_BROKEN = 5 stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.max_broken = MAX_BROKEN assert exp.is_broken exp.max_broken += 1 assert not exp.is_broken
def test_experiment_stats(): """Check that property stats is returning a proper summary of experiment's results.""" NUM_COMPLETED = 3 stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.metadata = {'datetime': datetime.datetime.utcnow()} stats = exp.stats assert stats['trials_completed'] == NUM_COMPLETED assert stats['best_trials_id'] == cfg.trials[3]['_id'] assert stats['best_evaluation'] == 0 assert stats['start_time'] == exp.metadata['datetime'] assert stats['finish_time'] == cfg.trials[0]['end_time'] assert stats['duration'] == stats['finish_time'] - stats['start_time'] assert len(stats) == 6
def test_reserve_success(self, random_dt): """Successfully find new trials in db and reserve the first one""" storage_config = {"type": "legacy", "database": {"type": "EphemeralDB"}} with OrionState( trials=generate_trials(["new", "reserved"]), storage=storage_config ) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] trial = exp.reserve_trial() # Trials are sorted according to hash and 'new' gets position second cfg.trials[1]["status"] = "reserved" cfg.trials[1]["start_time"] = random_dt cfg.trials[1]["heartbeat"] = random_dt assert trial.to_dict() == cfg.trials[1]
def test_is_done_property_no_pending(algorithm): """Check experiment stopping conditions when there is no pending trials.""" completed = ['completed'] * 10 broken = ['broken'] * 5 with OrionState(trials=generate_trials(completed + broken)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 15 # There is only 10 completed trials and algo not done. assert not exp.is_done exp.algorithms.algorithm.done = True # Algorithm is done and no pending trials assert exp.is_done
def test_experiment_pickleable(): """Test experiment instance is pickleable""" with OrionState(trials=generate_trials(["new"])) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp_trials = exp.fetch_trials() assert len(exp_trials) > 0 from orion.storage.base import storage_factory exp_bytes = pickle.dumps(exp) new_exp = pickle.loads(exp_bytes) assert [trial.to_dict() for trial in exp_trials ] == [trial.to_dict() for trial in new_exp.fetch_trials()]
def test_experiment_view_stats(): """Check that property stats from view is consistent.""" NUM_COMPLETED = 3 stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.metadata = {'datetime': datetime.datetime.utcnow()} exp_view = ExperimentView(exp) stats = exp_view.stats assert stats['trials_completed'] == NUM_COMPLETED assert stats['best_trials_id'] == cfg.trials[3]['_id'] assert stats['best_evaluation'] == 0 assert stats['start_time'] == exp_view.metadata['datetime'] assert stats['finish_time'] == cfg.trials[0]['end_time'] assert stats['duration'] == stats['finish_time'] - stats['start_time'] assert len(stats) == 6
def test_reserve_success(self, random_dt): """Successfully find new trials in db and reserve the first one""" storage_config = { 'type': 'legacy', 'database': { 'type': 'EphemeralDB' } } with OrionState(trials=generate_trials(['new', 'reserved']), storage=storage_config) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] trial = exp.reserve_trial() # Trials are sorted according to hash and 'new' gets position second cfg.trials[1]['status'] = 'reserved' cfg.trials[1]['start_time'] = random_dt cfg.trials[1]['heartbeat'] = random_dt assert trial.to_dict() == cfg.trials[1]
def test_is_done_property_with_pending(algorithm): """Check experiment stopping conditions when there is pending trials.""" completed = ['completed'] * 10 reserved = ['reserved'] * 5 with OrionState(trials=generate_trials(completed + reserved)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 10 assert exp.is_done exp.max_trials = 15 # There is only 10 completed trials assert not exp.is_done exp.algorithms.algorithm.done = True # Algorithm is done but 5 trials are pending assert not exp.is_done