def test_try_reset_after_race_condition(self, exp_config, new_config): """Cannot set a configuration after init if it looses a race condition, but can set it if reloaded. The experiment from process which first writes to db is initialized properly. The experiment which looses the race condition cannot be initialized and needs to be rebuilt. """ exp = Experiment(new_config['name']) # Another experiment gets configured first experiment_count_before = exp._db.count("experiments") naughty_little_exp = Experiment(new_config['name']) naughty_little_exp.configure(new_config) assert naughty_little_exp._init_done is True assert exp._init_done is False assert (experiment_count_before + 1) == exp._db.count("experiments") # First experiment won't be able to be configured with pytest.raises(DuplicateKeyError) as exc_info: exp.configure(new_config) assert 'duplicate key error' in str(exc_info.value) # Still not more experiment in DB assert (experiment_count_before + 1) == exp._db.count("experiments") # Retry configuring the experiment exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True assert (experiment_count_before + 1) == exp._db.count("experiments") assert exp.configuration == naughty_little_exp.configuration
def test_inconsistent_3_set_before_init_no_hit(self, random_dt, new_config): """Test inconsistent configuration because of datetime.""" exp = Experiment(new_config['name']) new_config['metadata']['datetime'] = 123 with pytest.raises(ValueError) as exc_info: exp.configure(new_config) assert 'inconsistent' in str(exc_info.value)
def test_good_set_before_init_no_hit(self, random_dt, database, new_config): """Trying to set, overwrite everything from input.""" exp = Experiment(new_config['name']) exp.configure(new_config) assert exp._init_done is True found_config = list(database.experiments.find({'name': 'supernaekei', 'metadata.user': '******'})) assert len(found_config) == 1 _id = found_config[0].pop('_id') assert _id != 'fasdfasfa' assert exp._id == _id new_config['refers'] = None new_config['status'] = 'pending' new_config.pop('_id') new_config.pop('something_to_be_ignored') new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 assert found_config[0] == new_config assert exp.name == new_config['name'] assert exp.refers is None assert exp.metadata == new_config['metadata'] assert exp.pool_size == new_config['pool_size'] assert exp.max_trials == new_config['max_trials'] assert exp.status == new_config['status']
def test_get_before_init_has_hit(self, exp_config, random_dt): """Return a configuration dict according to an experiment object. Assuming that experiment's (exp's name, user's name) has hit the database. """ exp = Experiment('supernaedo2') exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0]
def test_try_set_after_init(self, exp_config): """Cannot set a configuration after init (currently).""" exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp.configure(exp_config[0][0]) assert exp._init_done is True with pytest.raises(RuntimeError) as exc_info: exp.configure(exp_config[0][0]) assert 'cannot reset' in str(exc_info.value)
def test_reserve_none(self): """Find nothing, return None.""" try: Database(of_type='MongoDB', name='metaopt_test', username='******', password='******') except (TypeError, ValueError): pass exp = Experiment('supernaekei') trial = exp.reserve_trial() assert trial is None
def test_workon(database): """Test scenario having a configured experiment already setup.""" try: Database(of_type='MongoDB', name='metaopt_test', username='******', password='******') except (TypeError, ValueError): pass experiment = Experiment('voila_voici') config = experiment.configuration config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}} config['pool_size'] = 1 config['max_trials'] = 100 config['metadata']['user_script'] = os.path.abspath( os.path.join(os.path.dirname(__file__), "black_box.py")) config['metadata']['user_args'] = ["-x~uniform(-50, 50)"] experiment.configure(config) workon(experiment) exp = list(database.experiments.find({'name': 'voila_voici'})) assert len(exp) == 1 exp = exp[0] assert '_id' in exp exp_id = exp['_id'] assert exp['name'] == 'voila_voici' assert exp['pool_size'] == 1 assert exp['max_trials'] == 100 assert exp['status'] == 'done' assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1}} assert 'user' in exp['metadata'] assert 'datetime' in exp['metadata'] assert 'user_script' in exp['metadata'] assert os.path.isabs(exp['metadata']['user_script']) assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)'] trials = list(database.trials.find({'experiment': exp_id})) assert len(trials) < 15 assert trials[-1]['status'] == 'completed' for result in trials[-1]['results']: assert result['type'] != 'constraint' if result['type'] == 'objective': assert abs(result['value'] - 23.4) < 1e-6 assert result['name'] == 'example_objective' elif result['type'] == 'gradient': res = numpy.asarray(result['value']) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result['name'] == 'example_gradient' params = trials[-1]['params'] assert len(params) == 1 assert params[0]['name'] == '/x' assert params[0]['type'] == 'real' assert (params[0]['value'] - 34.56789) < 1e-5
def test_good_set_before_init_hit_no_diffs_exc_pool_size(self, exp_config): """Trying to set, and NO differences were found from the config pulled from db. Everything is normal, nothing changes. Experiment is resumed, perhaps with more workers that evaluate (an exception is 'pool_size'). """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp_config[0][1]['pool_size'] = 10 exp_config[0][1]['status'] = 'pending' exp.configure(exp_config[0][1]) assert exp.configuration == exp_config[0][1]
def test_get_after_init_plus_hit_no_diffs(self, exp_config): """Return a configuration dict according to an experiment object. Before initialization is done, it can be the case that the pair (`name`, user's name) has not hit the database. return a yaml compliant form of current state, to be used with :mod:`metaopt.core.resolve_config`. """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp.configure(exp_config[0][1]) assert exp._init_done is True exp_config[0][1]['status'] = 'pending' assert exp.configuration == exp_config[0][1]
def create_experiment(exp_name, expconfig, cmdconfig, cmdargs): """Create an experiment based on configuration. Configuration is a combination of command line, experiment configuration file, experiment configuration in database and metaopt configuration files. Precedence of configurations is: `cmdargs` > `cmdconfig` > `dbconfig` > `expconfig` This means `expconfig` values would be overwritten by `dbconfig` and so on. Parameters ---------- exp_name: str Name of the experiment expconfig: dict Configuration coming from default configuration files. cmdconfig: dict Configuration coming from configuration file. cmdargs: dict Configuration coming from command line arguments. """ # Initialize experiment object. # Check for existing name and fetch configuration. experiment = Experiment(exp_name) dbconfig = experiment.configuration expconfig = resolve_config.merge_mopt_config(expconfig, dbconfig, cmdconfig, cmdargs) # Infer rest information about the process + versioning expconfig['metadata'] = infer_versioning_metadata(expconfig['metadata']) # Pop out configuration concerning databases and resources expconfig.pop('database', None) expconfig.pop('resources', None) expconfig.pop('status', None) # Finish experiment's configuration and write it to database. try: experiment.configure(expconfig) except DuplicateKeyError: # Fails if concurrent experiment with identical (name, metadata.user) # is written first in the database. # Next infer_experiment() should either load experiment from database # and run smoothly if identical or trigger an experiment fork. # In other words, there should not be more than 1 level of recursion. experiment = create_experiment(exp_name, expconfig, cmdconfig, cmdargs) return experiment
def hacked_exp(with_user_dendi, random_dt, clean_db): """Return an `Experiment` instance with hacked _id to find trials in fake database. """ try: Database(of_type='MongoDB', name='metaopt_test', username='******', password='******') except (TypeError, ValueError): pass exp = Experiment('supernaedo2') exp._id = 'supernaedo2' # white box hack return exp
def test_good_set_before_init_hit_with_diffs(self, exp_config): """Trying to set, and differences were found from the config pulled from db. In this case: 1. Force renaming of experiment, prompt user for new name. 2. Fork from experiment with previous name. New experiments refers to the old one, if user wants to. 3. Overwrite elements with the ones from input. .. warning:: Currently, not implemented. """ new_config = {'metadata': {'user_version': 1.2}} exp = Experiment('supernaedo2') exp.configure(new_config)
def test_algorithm_config_with_just_a_string(self, exp_config): """Test that configuring an algorithm with just a string is OK.""" new_config = copy.deepcopy(exp_config[0][1]) new_config['algorithms'] = 'dumbalgo' exp = Experiment('supernaedo3') exp.configure(new_config) new_config['status'] = 'pending' new_config['algorithms'] = dict() new_config['algorithms']['dumbalgo'] = dict() new_config['algorithms']['dumbalgo']['done'] = False new_config['algorithms']['dumbalgo']['judgement'] = None new_config['algorithms']['dumbalgo']['scoring'] = 0 new_config['algorithms']['dumbalgo']['suspend'] = False new_config['algorithms']['dumbalgo']['value'] = 5 assert exp._id == new_config.pop('_id') assert exp.configuration == new_config
def test_existing_experiment(self, create_db_instance, exp_config): """Hit exp_name + user's name in the db, fetch most recent entry.""" exp = Experiment('supernaedo2') assert exp._init_done is False assert exp._db is create_db_instance assert exp._id == exp_config[0][0]['_id'] assert exp.name == exp_config[0][0]['name'] assert exp.refers == exp_config[0][0]['refers'] assert exp.metadata == exp_config[0][0]['metadata'] assert exp._last_fetched == exp_config[0][0]['metadata']['datetime'] assert exp.pool_size == exp_config[0][0]['pool_size'] assert exp.max_trials == exp_config[0][0]['max_trials'] assert exp.status == exp_config[0][0]['status'] assert exp.algorithms == exp_config[0][0]['algorithms'] with pytest.raises(AttributeError): exp.this_is_not_in_config = 5
def test_new_experiment_due_to_username(self, create_db_instance, random_dt): """Hit exp_name, but user's name does not hit the db, create new entry.""" exp = Experiment('supernaedo2') assert exp._init_done is False assert exp._db is create_db_instance assert exp._id is None assert exp.name == 'supernaedo2' assert exp.refers is None assert exp.metadata['user'] == 'bouthilx' assert exp.metadata['datetime'] == random_dt assert exp._last_fetched == random_dt assert len(exp.metadata) == 2 assert exp.pool_size is None assert exp.max_trials is None assert exp.status is None assert exp.algorithms is None with pytest.raises(AttributeError): exp.this_is_not_in_config = 5
def test_good_set_before_init_hit_no_diffs_exc_pool_size(self, exp_config): """Trying to set, and NO differences were found from the config pulled from db. Everything is normal, nothing changes. Experiment is resumed, perhaps with more workers that evaluate (an exception is 'pool_size'). """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp_config[0][0]['pool_size'] = 10 exp_config[0][0]['status'] = 'pending' exp.configure(exp_config[0][0]) exp_config[0][0]['algorithms']['dumbalgo']['done'] = False exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0 exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5 assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0]
def test_get_before_init_no_hit(self, exp_config, random_dt): """Return a configuration dict according to an experiment object. Before initialization is done, it can be the case that the pair (`name`, user's name) has not hit the database. return a yaml compliant form of current state, to be used with :mod:`metaopt.core.resolve_config`. """ exp = Experiment('supernaekei') cfg = exp.configuration assert cfg['name'] == 'supernaekei' assert cfg['refers'] is None assert cfg['metadata']['user'] == 'tsirif' assert cfg['metadata']['datetime'] == random_dt assert len(cfg['metadata']) == 2 assert cfg['pool_size'] is None assert cfg['max_trials'] is None assert cfg['status'] is None assert cfg['algorithms'] is None assert len(cfg) == 7
def test_get_after_init_plus_hit_no_diffs(self, exp_config): """Return a configuration dict according to an experiment object. Before initialization is done, it can be the case that the pair (`name`, user's name) has not hit the database. return a yaml compliant form of current state, to be used with :mod:`metaopt.core.resolve_config`. """ exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init experiment_count_before = exp._db.count("experiments") exp.configure(exp_config[0][0]) assert exp._init_done is True exp_config[0][0]['status'] = 'pending' exp_config[0][0]['algorithms']['dumbalgo']['done'] = False exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0 exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5 assert exp._id == exp_config[0][0].pop('_id') assert exp.configuration == exp_config[0][0] assert experiment_count_before == exp._db.count("experiments")
def test_after_init_algorithms_are_objects(self, exp_config): """Attribute exp.algorithms become objects after init.""" exp = Experiment('supernaedo2') # Deliver an external configuration to finalize init exp.configure(exp_config[0][0]) assert isinstance(exp.algorithms, BaseAlgorithm)