def test_try_set_after_init(self, exp_config): """Cannot set a configuration after init (currently).""" exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp.configure(exp_config[0][0]) assert exp._init_done is True with pytest.raises(RuntimeError) as exc_info: exp.configure(exp_config[0][0]) assert 'cannot reset' in str(exc_info.value)
def test_workon(database): """Test scenario having a configured experiment already setup.""" try: Database(of_type='MongoDB', name='orion_test', username='******', password='******') except (TypeError, ValueError): pass experiment = Experiment('voila_voici') config = experiment.configuration config['algorithms'] = { 'gradient_descent': { 'learning_rate': 0.1 } } config['pool_size'] = 1 config['max_trials'] = 100 config['metadata']['user_script'] = os.path.abspath(os.path.join( os.path.dirname(__file__), "black_box.py")) config['metadata']['user_args'] = ["-x~uniform(-50, 50)"] experiment.configure(config) workon(experiment) exp = list(database.experiments.find({'name': 'voila_voici'})) assert len(exp) == 1 exp = exp[0] assert '_id' in exp exp_id = exp['_id'] assert exp['name'] == 'voila_voici' assert exp['pool_size'] == 1 assert exp['max_trials'] == 100 assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1, 'dx_tolerance': 1e-7}} assert 'user' in exp['metadata'] assert 'datetime' in exp['metadata'] assert 'user_script' in exp['metadata'] assert os.path.isabs(exp['metadata']['user_script']) assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)'] trials = list(database.trials.find({'experiment': exp_id})) assert len(trials) < 15 assert trials[-1]['status'] == 'completed' for result in trials[-1]['results']: assert result['type'] != 'constraint' if result['type'] == 'objective': assert abs(result['value'] - 23.4) < 1e-6 assert result['name'] == 'example_objective' elif result['type'] == 'gradient': res = numpy.asarray(result['value']) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result['name'] == 'example_gradient' params = trials[-1]['params'] assert len(params) == 1 assert params[0]['name'] == '/x' assert params[0]['type'] == 'real' assert (params[0]['value'] - 34.56789) < 1e-5
def create_experiment(exp_name, expconfig, cmdconfig, cmdargs): """Create an experiment based on configuration. Configuration is a combination of command line, experiment configuration file, experiment configuration in database and orion configuration files. Precedence of configurations is: `cmdargs` > `cmdconfig` > `dbconfig` > `expconfig` This means `expconfig` values would be overwritten by `dbconfig` and so on. Parameters ---------- exp_name: str Name of the experiment expconfig: dict Configuration coming from default configuration files. cmdconfig: dict Configuration coming from configuration file. cmdargs: dict Configuration coming from command line arguments. """ # Initialize experiment object. # Check for existing name and fetch configuration. experiment = Experiment(exp_name) dbconfig = experiment.configuration log.debug("DB config") log.debug(dbconfig) expconfig = resolve_config.merge_orion_config(expconfig, dbconfig, cmdconfig, cmdargs) # Infer rest information about the process + versioning expconfig['metadata'] = infer_versioning_metadata(expconfig['metadata']) # Pop out configuration concerning databases and resources expconfig.pop('database', None) expconfig.pop('resources', None) expconfig.pop('status', None) log.info(expconfig) # Finish experiment's configuration and write it to database. try: experiment.configure(expconfig) except DuplicateKeyError: # Fails if concurrent experiment with identical (name, metadata.user) # is written first in the database. # Next infer_experiment() should either load experiment from database # and run smoothly if identical or trigger an experiment fork. # In other words, there should not be more than 1 level of recursion. experiment = create_experiment(exp_name, expconfig, cmdconfig, cmdargs) return experiment
def test_experiment_non_interactive_branching(self, create_db_instance, random_dt, exp_config, monkeypatch): """Configure an existing experiment with parent.""" monkeypatch.setattr('sys.__stdin__.isatty', lambda: True) exp = Experiment('supernaedo2.1') exp.algorithms = {'dumbalgo': {}} with pytest.raises(OSError): exp.configure(exp.configuration) monkeypatch.undo() with pytest.raises(ValueError) as exc_info: exp.configure(exp.configuration) assert "Configuration is different and generates a branching" in str( exc_info.value)
def test_algorithm_config_with_just_a_string(self, exp_config): """Test that configuring an algorithm with just a string is OK.""" new_config = copy.deepcopy(exp_config[0][1]) new_config['algorithms'] = 'dumbalgo' exp = Experiment('supernaedo3') exp.configure(new_config) new_config['algorithms'] = dict() new_config['algorithms']['dumbalgo'] = dict() new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 assert exp._id == new_config.pop('_id') assert exp.configuration['algorithms'] == new_config['algorithms']
def test_status_is_pending_when_increase_max_trials(self, exp_config): """Attribute exp.algorithms become objects after init.""" exp = Experiment('supernaedo4') # Deliver an external configuration to finalize init exp.configure(exp_config[0][2]) assert exp.is_done exp = Experiment('supernaedo4') # Deliver an external configuration to finalize init exp_config[0][2]['max_trials'] = 1000 exp.configure(exp_config[0][2]) assert not exp.is_done
def test_experiment_with_parent(self, create_db_instance, random_dt, exp_config): """Configure an existing experiment with parent.""" exp = Experiment('supernaedo2.1') exp.algorithms = {'random': {}} exp.configure(exp.configuration) assert exp._init_done is True assert exp._db is create_db_instance assert exp._id is not None assert exp.name == 'supernaedo2.1' assert exp.configuration['refers'] == exp_config[0][4]['refers'] assert exp.metadata == exp_config[0][4]['metadata'] assert exp.pool_size == 2 assert exp.max_trials == 1000 assert exp.configuration['algorithms'] == {'random': {}}
def test_good_set_before_init_hit_with_diffs(self, exp_config): """Trying to set, and differences were found from the config pulled from db. In this case: 1. Force renaming of experiment, prompt user for new name. 2. Fork from experiment with previous name. New experiments refers to the old one, if user wants to. 3. Overwrite elements with the ones from input. .. warning:: Currently, not implemented. """ new_config = copy.deepcopy(exp_config[0][1]) new_config['metadata']['user_version'] = 1.2 exp = Experiment('supernaedo2') exp.configure(new_config)
def test_good_set_before_init_hit_no_diffs_exc_pool_size(self, exp_config): """Trying to set, and NO differences were found from the config pulled from db. Everything is normal, nothing changes. Experiment is resumed, perhaps with more workers that evaluate (an exception is 'pool_size'). """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp_config[0][0]['pool_size'] = 10 exp.configure(exp_config[0][0]) exp_config[0][0]['algorithms']['dumbalgo']['done'] = False exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0 exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5 assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0]
def test_new_experiment_with_parent(self, create_db_instance, random_dt, exp_config): """Configure a branch experiment.""" exp = Experiment('supernaedo2.6') exp.metadata = exp_config[0][4]['metadata'] exp.refers = exp_config[0][4]['refers'] exp.algorithms = exp_config[0][4]['algorithms'] exp.configure(exp.configuration) assert exp._init_done is True assert_protocol(exp, create_db_instance) assert exp._id is not None assert exp.name == 'supernaedo2.6' assert exp.configuration['refers'] == exp_config[0][4]['refers'] exp_config[0][4]['metadata']['datetime'] = random_dt assert exp.metadata == exp_config[0][4]['metadata'] assert exp.pool_size is None assert exp.max_trials is None assert exp.configuration['algorithms'] == {'random': {'seed': None}}
def test_forcing_user(exp_config): """Trying to set by forcing user so that NO differences are found.""" assert getpass.getuser() == 'bouthilx' exp = Experiment('supernaedo2') assert exp.metadata['user'] == 'bouthilx' exp = Experiment('supernaedo2', 'tsirif') # Deliver an external configuration to finalize init exp_config[0][0]['max_trials'] = 5000 exp.configure(exp_config[0][0]) exp_config[0][0]['algorithms']['dumbalgo']['done'] = False exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0 exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5 exp_config[0][0]['algorithms']['dumbalgo']['seed'] = None exp_config[0][0]['producer']['strategy'] = "NoParallelStrategy" assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0]
def test_good_set_before_init_hit_no_diffs_exc_max_trials( self, exp_config): """Trying to set, and NO differences were found from the config pulled from db. Everything is normal, nothing changes. Experiment is resumed, perhaps with more trials to evaluate (an exception is 'max_trials'). """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp_config[0][0]['max_trials'] = 5000 exp.configure(exp_config[0][0]) exp_config[0][0]['algorithms']['dumbalgo']['done'] = False exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0 exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5 exp_config[0][0]['producer']['strategy'] = "NoParallelStrategy" assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0]
def test_get_after_init_plus_hit_no_diffs(self, exp_config): """Return a configuration dict according to an experiment object. Before initialization is done, it can be the case that the pair (`name`, user's name) has not hit the database. return a yaml compliant form of current state, to be used with :mod:`orion.core.cli.esolve_config`. """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init experiment_count_before = exp._db.count("experiments") exp.configure(exp_config[0][0]) assert exp._init_done is True exp_config[0][0]['algorithms']['dumbalgo']['done'] = False exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0 exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5 assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0] assert experiment_count_before == exp._db.count("experiments")
def test_try_reset_after_race_condition(self, exp_config, new_config): """Cannot set a configuration after init if it looses a race condition, but can set it if reloaded. The experiment from process which first writes to db is initialized properly. The experiment which looses the race condition cannot be initialized and needs to be rebuilt. """ exp = Experiment(new_config['name']) # Another experiment gets configured first experiment_count_before = exp._db.count("experiments") naughty_little_exp = Experiment(new_config['name']) naughty_little_exp.configure(new_config) assert naughty_little_exp._init_done is True assert exp._init_done is False assert (experiment_count_before + 1) == exp._db.count("experiments") # First experiment won't be able to be configured with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'duplicate key error' in str(exc_info.value) # Still not more experiment in DB assert (experiment_count_before + 1) == exp._db.count("experiments") # Retry configuring the experiment new_config['metadata']['datetime'] = naughty_little_exp.metadata[ 'datetime'] exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True assert (experiment_count_before + 1) == exp._db.count("experiments") assert exp.configuration == naughty_little_exp.configuration
def test_working_dir_is_correctly_set(self, database, new_config): """Check if working_dir is correctly changed.""" exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True database.experiments.update_one( { 'name': 'supernaekei', 'metadata.user': '******' }, {'$set': { 'working_dir': './' }}) found_config = list( database.experiments.find({ 'name': 'supernaekei', 'metadata.user': '******' })) found_config = found_config[0] exp = Experiment(found_config['name']) exp.configure(found_config) assert exp.working_dir == './'
def test_working_dir_works_when_db_absent(self, database, new_config): """Check if working_dir is correctly when absent from the database.""" exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True database.experiments.update_one( { 'name': 'supernaekei', 'metadata.user': '******' }, {'$unset': { 'working_dir': '' }}) found_config = list( database.experiments.find({ 'name': 'supernaekei', 'metadata.user': '******' })) found_config = found_config[0] exp = Experiment(found_config['name']) exp.configure(found_config) assert exp.working_dir is None
def test_good_set_before_init_no_hit(self, random_dt, database, new_config): """Trying to set, overwrite everything from input.""" exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True found_config = list( database.experiments.find({ 'name': 'supernaekei', 'metadata.user': '******' })) new_config['metadata']['datetime'] = exp.metadata['datetime'] assert len(found_config) == 1 _id = found_config[0].pop('_id') assert _id != 'fasdfasfa' assert exp._id == _id new_config['refers'] = {} new_config.pop('_id') new_config.pop('something_to_be_ignored') new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 new_config['algorithms']['dumbalgo']['seed'] = None new_config['refers'] = { 'adapter': [], 'parent_id': None, 'root_id': _id } assert found_config[0] == new_config assert exp.name == new_config['name'] assert exp.configuration['refers'] == new_config['refers'] assert exp.metadata == new_config['metadata'] assert exp.pool_size == new_config['pool_size'] assert exp.max_trials == new_config['max_trials'] assert exp.working_dir == new_config['working_dir']
def build_from_config(self, config): """Build a fully configured (and writable) experiment based on full configuration. .. seealso:: `orion.core.io.experiment_builder` for more information on the hierarchy of configurations. :class:`orion.core.worker.experiment.Experiment` for more information on the experiment object. """ log.info(config) # Pop out configuration concerning databases and resources config.pop('database', None) config.pop('resources', None) experiment = Experiment(config['name']) # Finish experiment's configuration and write it to database. experiment.configure(config) return experiment
def test_try_set_after_race_condition(self, exp_config, new_config): """Cannot set a configuration after init if it looses a race condition. The experiment from process which first writes to db is initialized properly. The experiment which looses the race condition cannot be initialized and needs to be rebuilt. """ exp = Experiment(new_config['name']) assert exp.id is None # Another experiment gets configured first experiment_count_before = exp._db.count("experiments") naughty_little_exp = Experiment(new_config['name']) assert naughty_little_exp.id is None naughty_little_exp.configure(new_config) assert naughty_little_exp._init_done is True assert exp._init_done is False assert (experiment_count_before + 1) == exp._db.count("experiments") # First experiment won't be able to be configured with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'duplicate key error' in str(exc_info.value) assert (experiment_count_before + 1) == exp._db.count("experiments")
def test_try_set_after_race_condition_with_hit(self, exp_config, new_config): """Cannot set a configuration after init if config is built from no-hit (without up-to-date db info) and new exp is hit The experiment from process which first writes to db is initialized properly. The experiment which looses the race condition cannot be initialized and needs to be rebuilt. """ # Another experiment gets configured first naughty_little_exp = Experiment(new_config['name']) assert naughty_little_exp.id is None experiment_count_before = naughty_little_exp._db.count("experiments") naughty_little_exp.configure(copy.deepcopy(new_config)) assert naughty_little_exp._init_done is True exp = Experiment(new_config['name']) assert exp._init_done is False assert (experiment_count_before + 1) == exp._db.count("experiments") # Experiment with hit won't be able to be configured with config without db info with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'Cannot register an existing experiment with a new config' in str( exc_info.value) assert (experiment_count_before + 1) == exp._db.count("experiments") new_config['metadata']['datetime'] = naughty_little_exp.metadata[ 'datetime'] exp = Experiment(new_config['name']) assert exp._init_done is False assert (experiment_count_before + 1) == exp._db.count("experiments") # New experiment will be able to be configured exp.configure(new_config) assert (experiment_count_before + 1) == exp._db.count("experiments")
def test_not_inconsistent_3_set_before_init_no_hit(self, random_dt, new_config): """Test inconsistent configuration because of datetime.""" exp = Experiment(new_config['name']) new_config['metadata']['datetime'] = 123 exp.configure(new_config)
def test_after_init_algorithms_are_objects(self, exp_config): """Attribute exp.algorithms become objects after init.""" exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp.configure(exp_config[0][0]) assert isinstance(exp.algorithms, BaseAlgorithm)