def build_from_config(self, config): """Build a fully configured (and writable) experiment based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.Experiment` for more information on the experiment object. """ log.info(config) # Pop out configuration concerning databases and resources config.pop('database', None) config.pop('resources', None) experiment = Experiment(config['name'], config.get('user', None)) # Finish experiment's configuration and write it to database. try: experiment.configure(config) except AttributeError as ex: if 'user_script' not in config['metadata']: raise NoConfigurationError from ex return experiment
def build_from_config(self, config): """Build a fully configured (and writable) experiment based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.Experiment` for more information on the experiment object. """ log.info(config) # Pop out configuration concerning databases and resources config.pop('database', None) config.pop('resources', None) experiment = Experiment(config['name'], config.get('user', None), config.get('version', None)) # TODO: Handle both from cmdline and python APIs. if 'priors' not in config['metadata'] and 'user_args' not in config['metadata']: raise NoConfigurationError # Parse to generate priors if 'user_args' in config['metadata']: parser = OrionCmdlineParser(orion.core.config.user_script_config) parser.parse(config['metadata']['user_args']) config['metadata']['parser'] = parser.get_state_dict() config['metadata']['priors'] = dict(parser.priors) # Finish experiment's configuration and write it to database. experiment.configure(config) return experiment
def test_good_set_before_init_no_hit(self, random_dt, database, new_config): """Trying to set, overwrite everything from input.""" exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True found_config = list( database.experiments.find({ 'name': 'supernaekei', 'metadata.user': '******' })) assert len(found_config) == 1 _id = found_config[0].pop('_id') assert _id != 'fasdfasfa' assert exp._id == _id new_config['refers'] = None new_config['status'] = 'pending' new_config.pop('_id') new_config.pop('something_to_be_ignored') new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 assert found_config[0] == new_config assert exp.name == new_config['name'] assert exp.refers is None assert exp.metadata == new_config['metadata'] assert exp.pool_size == new_config['pool_size'] assert exp.max_trials == new_config['max_trials'] assert exp.status == new_config['status']
def test_register_trials(random_dt): """Register a list of newly proposed trials/parameters.""" with OrionState(): exp = Experiment('supernaekei') exp._id = 0 trials = [ Trial(params=[{ 'name': 'a', 'type': 'integer', 'value': 5 }]), Trial(params=[{ 'name': 'b', 'type': 'integer', 'value': 6 }]), ] for trial in trials: exp.register_trial(trial) yo = list( map(lambda trial: trial.to_dict(), get_storage().fetch_trials(exp))) assert len(yo) == len(trials) assert yo[0]['params'] == list( map(lambda x: x.to_dict(), trials[0]._params)) assert yo[1]['params'] == list( map(lambda x: x.to_dict(), trials[1]._params)) assert yo[0]['status'] == 'new' assert yo[1]['status'] == 'new' assert yo[0]['submit_time'] == random_dt assert yo[1]['submit_time'] == random_dt
def hacked_exp(with_user_dendi, random_dt, clean_db, create_db_instance): """Return an `Experiment` instance with hacked _id to find trials in fake database. """ exp = Experiment('supernaedo2-dendi') exp._id = 'supernaedo2-dendi' # white box hack return exp
def test_register_trials(random_dt): """Register a list of newly proposed trials/parameters.""" with OrionState(): exp = Experiment("supernaekei", mode="x") exp._id = 0 trials = [ Trial(params=[{ "name": "a", "type": "integer", "value": 5 }]), Trial(params=[{ "name": "b", "type": "integer", "value": 6 }]), ] for trial in trials: exp.register_trial(trial) yo = list( map(lambda trial: trial.to_dict(), get_storage().fetch_trials(exp))) assert len(yo) == len(trials) assert yo[0]["params"] == list( map(lambda x: x.to_dict(), trials[0]._params)) assert yo[1]["params"] == list( map(lambda x: x.to_dict(), trials[1]._params)) assert yo[0]["status"] == "new" assert yo[1]["status"] == "new" assert yo[0]["submit_time"] == random_dt assert yo[1]["submit_time"] == random_dt
def test_workon(database): """Test scenario having a configured experiment already setup.""" try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass experiment = Experiment('voila_voici') config = experiment.configuration config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}} config['pool_size'] = 1 config['max_trials'] = 100 config['metadata']['user_script'] = os.path.abspath( os.path.join(os.path.dirname(__file__), "black_box.py")) config['metadata']['user_args'] = ["-x~uniform(-50, 50)"] experiment.configure(config) workon(experiment) exp = list(database.experiments.find({'name': 'voila_voici'})) assert len(exp) == 1 exp = exp[0] assert '_id' in exp exp_id = exp['_id'] assert exp['name'] == 'voila_voici' assert exp['pool_size'] == 1 assert exp['max_trials'] == 100 assert exp['algorithms'] == { 'gradient_descent': { 'learning_rate': 0.1, 'dx_tolerance': 1e-7 } } assert 'user' in exp['metadata'] assert 'datetime' in exp['metadata'] assert 'user_script' in exp['metadata'] assert os.path.isabs(exp['metadata']['user_script']) assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)'] trials = list(database.trials.find({'experiment': exp_id})) assert len(trials) <= 15 trials = list(sorted(trials, key=lambda trial: trial['submit_time'])) assert trials[-1]['status'] == 'completed' for result in trials[-1]['results']: assert result['type'] != 'constraint' if result['type'] == 'objective': assert abs(result['value'] - 23.4) < 1e-6 assert result['name'] == 'example_objective' elif result['type'] == 'gradient': res = numpy.asarray(result['value']) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result['name'] == 'example_gradient' params = trials[-1]['params'] assert len(params) == 1 assert params[0]['name'] == '/x' assert params[0]['type'] == 'real' assert (params[0]['value'] - 34.56789) < 1e-5
def test_fetch_all_trials(): """Fetch a list of all trials""" with OrionState(trials=generate_trials(["new", "reserved", "completed"])) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] trials = list(map(lambda trial: trial.to_dict(), exp.fetch_trials({}))) assert trials == cfg.trials
def test_inconsistent_2_set_before_init_no_hit(self, random_dt, new_config): """Test inconsistent configuration because of user.""" exp = Experiment(new_config['name']) new_config['metadata']['user'] = '******' with pytest.raises(ValueError) as exc_info: exp.configure(new_config) assert 'inconsistent' in str(exc_info.value)
def test_fetch_all_trials(): """Fetch a list of all trials""" with OrionState( trials=generate_trials(['new', 'reserved', 'completed'])) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] trials = list(map(lambda trial: trial.to_dict(), exp.fetch_trials({}))) assert trials == cfg.trials
def test_reserve_when_exhausted(self): """Return None once all the trials have been allocated""" stati = ["new", "reserved", "interrupted", "completed", "broken"] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] assert exp.reserve_trial() is not None assert exp.reserve_trial() is not None assert exp.reserve_trial() is None
def test_reserve_when_exhausted(self): """Return None once all the trials have been allocated""" stati = ['new', 'reserved', 'interrupted', 'completed', 'broken'] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert exp.reserve_trial() is not None assert exp.reserve_trial() is not None assert exp.reserve_trial() is None
def create_experiment(exp_name, expconfig, cmdconfig, cmdargs): """Create an experiment based on configuration. Configuration is a combination of command line, experiment configuration file, experiment configuration in database and orion configuration files. Precedence of configurations is: `cmdargs` > `cmdconfig` > `dbconfig` > `expconfig` This means `expconfig` values would be overwritten by `dbconfig` and so on. Parameters ---------- exp_name: str Name of the experiment expconfig: dict Configuration coming from default configuration files. cmdconfig: dict Configuration coming from configuration file. cmdargs: dict Configuration coming from command line arguments. """ # Initialize experiment object. # Check for existing name and fetch configuration. experiment = Experiment(exp_name) dbconfig = experiment.configuration log.debug("DB config") log.debug(dbconfig) expconfig = resolve_config.merge_orion_config(expconfig, dbconfig, cmdconfig, cmdargs) # Infer rest information about the process + versioning expconfig['metadata'] = infer_versioning_metadata(expconfig['metadata']) # Pop out configuration concerning databases and resources expconfig.pop('database', None) expconfig.pop('resources', None) expconfig.pop('status', None) log.info(expconfig) # Finish experiment's configuration and write it to database. try: experiment.configure(expconfig) except DuplicateKeyError: # Fails if concurrent experiment with identical (name, metadata.user) # is written first in the database. # Next infer_experiment() should either load experiment from database # and run smoothly if identical or trigger an experiment fork. # In other words, there should not be more than 1 level of recursion. experiment = create_experiment(exp_name, expconfig, cmdconfig, cmdargs) return experiment
def test_reserve_none(self): """Find nothing, return None.""" try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass exp = Experiment('supernaekei') trial = exp.reserve_trial() assert trial is None
def test_fetch_completed_trials_from_view(): """Fetch a list of the unseen yet completed trials.""" non_completed_stati = ['new', 'interrupted', 'suspended', 'reserved'] stati = non_completed_stati + ['completed'] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp_view = ExperimentView(exp) trials = exp_view.fetch_trials_by_status('completed') assert len(trials) == 1 assert trials[0].status == 'completed'
def test_fetch_non_completed_trials(): """Fetch a list of the trials that are not completed trials.status in ['new', 'interrupted', 'suspended', 'broken'] """ non_completed_stati = ["new", "interrupted", "suspended", "reserved"] stati = non_completed_stati + ["completed"] with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] trials = exp.fetch_noncompleted_trials() assert len(trials) == 4 assert set(trial.status for trial in trials) == set(non_completed_stati)
def hacked_exp(with_user_dendi, random_dt, clean_db): """Return an `Experiment` instance with hacked _id to find trials in fake database. """ try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass exp = Experiment('supernaedo2') exp._id = 'supernaedo2' # white box hack return exp
def test_fix_lost_trials_race_condition(self, monkeypatch, caplog): """Test that a lost trial fixed by a concurrent process does not cause error.""" trial = copy.deepcopy(base_trial) trial['status'] = 'interrupted' trial['heartbeat'] = datetime.datetime.utcnow() - datetime.timedelta( seconds=360) with OrionState(trials=[trial]) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert len(exp.fetch_trials_by_status('interrupted')) == 1 assert len(exp._storage.fetch_lost_trials(exp)) == 0 def fetch_lost_trials(self, query): trial_object = Trial(**trial) trial_object.status = 'reserved' return [trial_object] # Force the fetch of a trial marked as reserved (and lost) while actually interrupted # (as if already failed-over by another process). with monkeypatch.context() as m: m.setattr(exp._storage.__class__, 'fetch_lost_trials', fetch_lost_trials) assert len(exp._storage.fetch_lost_trials(exp)) == 1 with caplog.at_level(logging.DEBUG): exp.fix_lost_trials() assert caplog.records[-1].levelname == 'DEBUG' assert caplog.records[-1].msg == 'failed' assert len(exp.fetch_trials_by_status('interrupted')) == 1 assert len(exp.fetch_trials_by_status('reserved')) == 0
def test_experiment_with_parent(self, create_db_instance, random_dt, exp_config): """Configure an existing experiment with parent.""" exp = Experiment('supernaedo2.1') exp.algorithms = {'random': {}} exp.configure(exp.configuration) assert exp._init_done is True assert exp._db is create_db_instance assert exp._id is not None assert exp.name == 'supernaedo2.1' assert exp.configuration['refers'] == exp_config[0][4]['refers'] assert exp.metadata == exp_config[0][4]['metadata'] assert exp.pool_size == 2 assert exp.max_trials == 1000 assert exp.configuration['algorithms'] == {'random': {}}
def test_algorithm_config_with_just_a_string(self, exp_config): """Test that configuring an algorithm with just a string is OK.""" new_config = copy.deepcopy(exp_config[0][1]) new_config['algorithms'] = 'dumbalgo' exp = Experiment('supernaedo3') exp.configure(new_config) new_config['algorithms'] = dict() new_config['algorithms']['dumbalgo'] = dict() new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 assert exp._id == new_config.pop('_id') assert exp.configuration['algorithms'] == new_config['algorithms']
def test_experiment_stats(): """Check that property stats is returning a proper summary of experiment's results.""" NUM_COMPLETED = 3 stati = (["completed"] * NUM_COMPLETED) + (["reserved"] * 2) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.metadata = {"datetime": datetime.datetime.utcnow()} stats = exp.stats assert stats.trials_completed == NUM_COMPLETED assert stats.best_trials_id == cfg.trials[3]["_id"] assert stats.best_evaluation == 0 assert stats.start_time == exp.metadata["datetime"] assert stats.finish_time == cfg.trials[0]["end_time"] assert stats.duration == stats.finish_time - stats.start_time
def test_good_set_before_init_hit_with_diffs(self, exp_config): """Trying to set, and differences were found from the config pulled from db. In this case: 1. Force renaming of experiment, prompt user for new name. 2. Fork from experiment with previous name. New experiments refers to the old one, if user wants to. 3. Overwrite elements with the ones from input. .. warning:: Currently, not implemented. """ new_config = copy.deepcopy(exp_config[0][1]) new_config['metadata']['user_version'] = 1.2 exp = Experiment('supernaedo2') exp.configure(new_config)
def test_existing_experiment(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = Experiment('supernaedo2') assert exp._init_done is False assert exp._db is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.refers == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.algorithms == exp_config[0][0]['algorithms'] with pytest.raises(AttributeError): exp.this_is_not_in_config = 5
def test_configurable_broken_property(): """Check if max_broken changes after configuration.""" MAX_BROKEN = 3 orion.core.config.worker.max_broken = MAX_BROKEN stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] assert exp.is_broken orion.core.config.worker.max_broken += 1 assert not exp.is_broken
def test_acquire_algorithm_lock_with_different_config(self, new_config, algorithm): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 algorithm_original_config = algorithm.configuration exp.algorithms = algorithm # Setting attribute to algorithm inside the wrapper algorithm.algorithm.seed = 10 assert algorithm.configuration != algorithm_original_config with pytest.raises( RuntimeError, match="Algorithm configuration changed since" ): with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): pass
def test_experiment_stats(): """Check that property stats is returning a proper summary of experiment's results.""" NUM_COMPLETED = 3 stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.metadata = {'datetime': datetime.datetime.utcnow()} stats = exp.stats assert stats['trials_completed'] == NUM_COMPLETED assert stats['best_trials_id'] == cfg.trials[3]['_id'] assert stats['best_evaluation'] == 0 assert stats['start_time'] == exp.metadata['datetime'] assert stats['finish_time'] == cfg.trials[0]['end_time'] assert stats['duration'] == stats['finish_time'] - stats['start_time'] assert len(stats) == 6
def test_empty(self, space): """Test panda frame creation when there is no trials""" with OrionState(): exp = Experiment("supernaekei", mode="x") exp.space = space assert exp.to_pandas().shape == (0, 8) assert list(exp.to_pandas().columns) == [ "id", "experiment_id", "status", "suggested", "reserved", "completed", "objective", "/index", ]
def test_configurable_broken_property(): """Check if max_broken changes after configuration.""" MAX_BROKEN = 5 stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN)) with OrionState(trials=generate_trials(stati)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.max_broken = MAX_BROKEN assert exp.is_broken exp.max_broken += 1 assert not exp.is_broken
def test_new_experiment_due_to_name(self, create_db_instance, random_dt): """Hit user name, but exp_name does not hit the db, create new entry.""" exp = Experiment('supernaekei') assert exp._init_done is False assert exp._db is create_db_instance assert exp._id is None assert exp.name == 'supernaekei' assert exp.refers == {} assert exp.metadata['user'] == 'tsirif' assert exp._last_fetched == random_dt assert len(exp.metadata) == 1 assert exp.pool_size is None assert exp.max_trials is None assert exp.algorithms is None with pytest.raises(AttributeError): exp.this_is_not_in_config = 5
def test_acquire_algorithm_lock_timeout(self, new_config, algorithm, mocker): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 exp.algorithms = algorithm storage_acquisition_mock = mocker.spy( cfg.storage(), "acquire_algorithm_lock" ) with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): pass storage_acquisition_mock.assert_called_with( experiment=exp, timeout=0.2, retry_interval=0.1 )