def test_good_set_before_init_no_hit(self, random_dt, database, new_config):
"""Trying to set, overwrite everything from input."""
exp = Experiment(new_config['name'])
exp.configure(new_config)
assert exp._init_done is True
found_config = list(database.experiments.find({'name': 'supernaekei',
'metadata.user': '******'}))
assert len(found_config) == 1
_id = found_config[0].pop('_id')
assert _id != 'fasdfasfa'
assert exp._id == _id
new_config['refers'] = None
new_config['status'] = 'pending'
new_config.pop('_id')
new_config.pop('something_to_be_ignored')
new_config['algorithms']['dumbalgo']['done'] = False
new_config['algorithms']['dumbalgo']['judgement'] = None
new_config['algorithms']['dumbalgo']['scoring'] = 0
new_config['algorithms']['dumbalgo']['suspend'] = False
new_config['algorithms']['dumbalgo']['value'] = 5
assert found_config[0] == new_config
assert exp.name == new_config['name']
assert exp.refers is None
assert exp.metadata == new_config['metadata']
assert exp.pool_size == new_config['pool_size']
assert exp.max_trials == new_config['max_trials']
assert exp.status == new_config['status']
def test_inconsistent_3_set_before_init_no_hit(self, random_dt, new_config):
"""Test inconsistent configuration because of datetime."""
exp = Experiment(new_config['name'])
new_config['metadata']['datetime'] = 123
with pytest.raises(ValueError) as exc_info:
exp.configure(new_config)
assert 'inconsistent' in str(exc_info.value)
def test_reserve_none(self):
"""Find nothing, return None."""
try:
Database(of_type='MongoDB', name='metaopt_test',
username='******', password='******')
except (TypeError, ValueError):
pass
exp = Experiment('supernaekei')
trial = exp.reserve_trial()
assert trial is None
def test_workon(database):
"""Test scenario having a configured experiment already setup."""
try:
Database(of_type='MongoDB',
name='metaopt_test',
username='******',
password='******')
except (TypeError, ValueError):
pass
experiment = Experiment('voila_voici')
config = experiment.configuration
config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}}
config['pool_size'] = 1
config['max_trials'] = 100
config['metadata']['user_script'] = os.path.abspath(
os.path.join(os.path.dirname(__file__), "black_box.py"))
config['metadata']['user_args'] = ["-x~uniform(-50, 50)"]
experiment.configure(config)
workon(experiment)
exp = list(database.experiments.find({'name': 'voila_voici'}))
assert len(exp) == 1
exp = exp[0]
assert '_id' in exp
exp_id = exp['_id']
assert exp['name'] == 'voila_voici'
assert exp['pool_size'] == 1
assert exp['max_trials'] == 100
assert exp['status'] == 'done'
assert exp['algorithms'] == {'gradient_descent': {'learning_rate': 0.1}}
assert 'user' in exp['metadata']
assert 'datetime' in exp['metadata']
assert 'user_script' in exp['metadata']
assert os.path.isabs(exp['metadata']['user_script'])
assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)']
trials = list(database.trials.find({'experiment': exp_id}))
assert len(trials) < 15
assert trials[-1]['status'] == 'completed'
for result in trials[-1]['results']:
assert result['type'] != 'constraint'
if result['type'] == 'objective':
assert abs(result['value'] - 23.4) < 1e-6
assert result['name'] == 'example_objective'
elif result['type'] == 'gradient':
res = numpy.asarray(result['value'])
assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7
assert result['name'] == 'example_gradient'
params = trials[-1]['params']
assert len(params) == 1
assert params[0]['name'] == '/x'
assert params[0]['type'] == 'real'
assert (params[0]['value'] - 34.56789) < 1e-5
def test_good_set_before_init_hit_no_diffs_exc_pool_size(self, exp_config):
"""Trying to set, and NO differences were found from the config pulled from db.
Everything is normal, nothing changes. Experiment is resumed,
perhaps with more workers that evaluate (an exception is 'pool_size').
"""
exp = Experiment('supernaedo2')
# Deliver an external configuration to finalize init
exp_config[0][1]['pool_size'] = 10
exp_config[0][1]['status'] = 'pending'
exp.configure(exp_config[0][1])
assert exp.configuration == exp_config[0][1]
def test_get_after_init_plus_hit_no_diffs(self, exp_config):
"""Return a configuration dict according to an experiment object.
Before initialization is done, it can be the case that the pair (`name`,
user's name) has not hit the database. return a yaml compliant form
of current state, to be used with :mod:`metaopt.core.resolve_config`.
"""
exp = Experiment('supernaedo2')
# Deliver an external configuration to finalize init
exp.configure(exp_config[0][1])
assert exp._init_done is True
exp_config[0][1]['status'] = 'pending'
assert exp.configuration == exp_config[0][1]
def create_experiment(exp_name, expconfig, cmdconfig, cmdargs):
"""Create an experiment based on configuration.
Configuration is a combination of command line, experiment configuration
file, experiment configuration in database and metaopt configuration files.
Precedence of configurations is:
`cmdargs` > `cmdconfig` > `dbconfig` > `expconfig`
This means `expconfig` values would be overwritten by `dbconfig` and so on.
Parameters
----------
exp_name: str
Name of the experiment
expconfig: dict
Configuration coming from default configuration files.
cmdconfig: dict
Configuration coming from configuration file.
cmdargs: dict
Configuration coming from command line arguments.
"""
# Initialize experiment object.
# Check for existing name and fetch configuration.
experiment = Experiment(exp_name)
dbconfig = experiment.configuration
expconfig = resolve_config.merge_mopt_config(expconfig, dbconfig,
cmdconfig, cmdargs)
# Infer rest information about the process + versioning
expconfig['metadata'] = infer_versioning_metadata(expconfig['metadata'])
# Pop out configuration concerning databases and resources
expconfig.pop('database', None)
expconfig.pop('resources', None)
expconfig.pop('status', None)
# Finish experiment's configuration and write it to database.
try:
experiment.configure(expconfig)
except DuplicateKeyError:
# Fails if concurrent experiment with identical (name, metadata.user)
# is written first in the database.
# Next infer_experiment() should either load experiment from database
# and run smoothly if identical or trigger an experiment fork.
# In other words, there should not be more than 1 level of recursion.
experiment = create_experiment(exp_name, expconfig, cmdconfig, cmdargs)
return experiment
def hacked_exp(with_user_dendi, random_dt, clean_db):
"""Return an `Experiment` instance with hacked _id to find trials in
fake database.
"""
try:
Database(of_type='MongoDB',
name='metaopt_test',
username='******',
password='******')
except (TypeError, ValueError):
pass
exp = Experiment('supernaedo2')
exp._id = 'supernaedo2' # white box hack
return exp
def test_good_set_before_init_hit_with_diffs(self, exp_config):
"""Trying to set, and differences were found from the config pulled from db.
In this case:
1. Force renaming of experiment, prompt user for new name.
2. Fork from experiment with previous name. New experiments refers to the
old one, if user wants to.
3. Overwrite elements with the ones from input.
.. warning:: Currently, not implemented.
"""
new_config = {'metadata': {'user_version': 1.2}}
exp = Experiment('supernaedo2')
exp.configure(new_config)
def test_existing_experiment(self, create_db_instance, exp_config):
"""Hit exp_name + user's name in the db, fetch most recent entry."""
exp = Experiment('supernaedo2')
assert exp._init_done is False
assert exp._db is create_db_instance
assert exp._id == exp_config[0][0]['_id']
assert exp.name == exp_config[0][0]['name']
assert exp.refers == exp_config[0][0]['refers']
assert exp.metadata == exp_config[0][0]['metadata']
assert exp._last_fetched == exp_config[0][0]['metadata']['datetime']
assert exp.pool_size == exp_config[0][0]['pool_size']
assert exp.max_trials == exp_config[0][0]['max_trials']
assert exp.status == exp_config[0][0]['status']
assert exp.algorithms == exp_config[0][0]['algorithms']
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
def test_algorithm_config_with_just_a_string(self, exp_config):
"""Test that configuring an algorithm with just a string is OK."""
new_config = copy.deepcopy(exp_config[0][1])
new_config['algorithms'] = 'dumbalgo'
exp = Experiment('supernaedo3')
exp.configure(new_config)
new_config['status'] = 'pending'
new_config['algorithms'] = dict()
new_config['algorithms']['dumbalgo'] = dict()
new_config['algorithms']['dumbalgo']['done'] = False
new_config['algorithms']['dumbalgo']['judgement'] = None
new_config['algorithms']['dumbalgo']['scoring'] = 0
new_config['algorithms']['dumbalgo']['suspend'] = False
new_config['algorithms']['dumbalgo']['value'] = 5
assert exp._id == new_config.pop('_id')
assert exp.configuration == new_config
def test_get_before_init_has_hit(self, exp_config, random_dt):
"""Return a configuration dict according to an experiment object.
Assuming that experiment's (exp's name, user's name) has hit the database.
"""
exp = Experiment('supernaedo2')
exp_config[0][0].pop('_id')
assert exp.configuration == exp_config[0][0]
def test_new_experiment_due_to_username(self, create_db_instance, random_dt):
"""Hit exp_name, but user's name does not hit the db, create new entry."""
exp = Experiment('supernaedo2')
assert exp._init_done is False
assert exp._db is create_db_instance
assert exp._id is None
assert exp.name == 'supernaedo2'
assert exp.refers is None
assert exp.metadata['user'] == 'bouthilx'
assert exp.metadata['datetime'] == random_dt
assert exp._last_fetched == random_dt
assert len(exp.metadata) == 2
assert exp.pool_size is None
assert exp.max_trials is None
assert exp.status is None
assert exp.algorithms is None
with pytest.raises(AttributeError):
exp.this_is_not_in_config = 5
def test_good_set_before_init_hit_no_diffs_exc_pool_size(self, exp_config):
"""Trying to set, and NO differences were found from the config pulled from db.
Everything is normal, nothing changes. Experiment is resumed,
perhaps with more workers that evaluate (an exception is 'pool_size').
"""
exp = Experiment('supernaedo2')
# Deliver an external configuration to finalize init
exp_config[0][0]['pool_size'] = 10
exp_config[0][0]['status'] = 'pending'
exp.configure(exp_config[0][0])
exp_config[0][0]['algorithms']['dumbalgo']['done'] = False
exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None
exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0
exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False
exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5
assert exp._id == exp_config[0][0].pop('_id')
assert exp.configuration == exp_config[0][0]
def test_get_after_init_plus_hit_no_diffs(self, exp_config):
"""Return a configuration dict according to an experiment object.
Before initialization is done, it can be the case that the pair (`name`,
user's name) has not hit the database. return a yaml compliant form
of current state, to be used with :mod:`metaopt.core.resolve_config`.
"""
exp = Experiment('supernaedo2')
# Deliver an external configuration to finalize init
experiment_count_before = exp._db.count("experiments")
exp.configure(exp_config[0][0])
assert exp._init_done is True
exp_config[0][0]['status'] = 'pending'
exp_config[0][0]['algorithms']['dumbalgo']['done'] = False
exp_config[0][0]['algorithms']['dumbalgo']['judgement'] = None
exp_config[0][0]['algorithms']['dumbalgo']['scoring'] = 0
exp_config[0][0]['algorithms']['dumbalgo']['suspend'] = False
exp_config[0][0]['algorithms']['dumbalgo']['value'] = 5
assert exp._id == exp_config[0][0].pop('_id')
assert exp.configuration == exp_config[0][0]
assert experiment_count_before == exp._db.count("experiments")
def test_try_set_after_init(self, exp_config):
"""Cannot set a configuration after init (currently)."""
exp = Experiment('supernaedo2')
# Deliver an external configuration to finalize init
exp.configure(exp_config[0][0])
assert exp._init_done is True
with pytest.raises(RuntimeError) as exc_info:
exp.configure(exp_config[0][0])
assert 'cannot reset' in str(exc_info.value)
def test_get_before_init_no_hit(self, exp_config, random_dt):
"""Return a configuration dict according to an experiment object.
Before initialization is done, it can be the case that the pair (`name`,
user's name) has not hit the database. return a yaml compliant form
of current state, to be used with :mod:`metaopt.core.resolve_config`.
"""
exp = Experiment('supernaekei')
cfg = exp.configuration
assert cfg['name'] == 'supernaekei'
assert cfg['refers'] is None
assert cfg['metadata']['user'] == 'tsirif'
assert cfg['metadata']['datetime'] == random_dt
assert len(cfg['metadata']) == 2
assert cfg['pool_size'] is None
assert cfg['max_trials'] is None
assert cfg['status'] is None
assert cfg['algorithms'] is None
assert len(cfg) == 7
def test_try_set_after_race_condition(self, exp_config, new_config):
"""Cannot set a configuration after init if it looses a race
condition.
The experiment from process which first writes to db is initialized
properly. The experiment which looses the race condition cannot be
initialized and needs to be rebuilt.
"""
exp = Experiment(new_config['name'])
# Another experiment gets configured first
experiment_count_before = exp._db.count("experiments")
naughty_little_exp = Experiment(new_config['name'])
naughty_little_exp.configure(new_config)
assert naughty_little_exp._init_done is True
assert exp._init_done is False
assert (experiment_count_before + 1) == exp._db.count("experiments")
# First experiment won't be able to be configured
with pytest.raises(DuplicateKeyError) as exc_info:
exp.configure(new_config)
assert 'duplicate key error' in str(exc_info.value)
assert (experiment_count_before + 1) == exp._db.count("experiments")
def test_after_init_algorithms_are_objects(self, exp_config):
"""Attribute exp.algorithms become objects after init."""
exp = Experiment('supernaedo2')
# Deliver an external configuration to finalize init
exp.configure(exp_config[0][0])
assert isinstance(exp.algorithms, BaseAlgorithm)
