def test_rename_missing_changed(self, parent_config, missing_config):
"""Test if renaming a dimension to another with different prior solves both conflicts but
creates a new one which is not solved
"""
missing_config['metadata']['user_args'].append('-w_d~normal(0,1)')
backward.populate_priors(missing_config['metadata'])
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
assert len(conflicts.get()) == 4
branch_builder.rename_dimension('x', 'w_d')
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[
2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[
1].resolution.new_dimension_conflict.dimension.name == '/w_d'
def changed_cli_config(child_config):
"""Create a child config with a changed dimension"""
child_config['metadata']['user_args'] += [
'-u=0', '--another=test', 'positional'
]
backward.populate_priors(child_config['metadata'])
return child_config
def test_rename_missing_changed_marked(self, parent_config, child_config):
"""Test if renaming is automatically applied with all conflicts resolved including
the new one caused by prior change
"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~+normal(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_b'
backward.populate_priors(child_config['metadata'])
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name='x')[0].is_resolved
assert conflicts.get([NewDimensionConflict],
dimension_name='w_b')[0].is_resolved
assert conflicts.get([ChangedDimensionConflict],
dimension_name='w_b')[0].is_resolved
assert not conflicts.get(dimension_name='w_a')[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 4
assert resolved_conflicts[1].resolution is resolved_conflicts[
2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[
1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def cli_conflict(old_config, new_config):
"""Generate a commandline conflict"""
new_config = copy.deepcopy(new_config)
new_config['metadata']['user_args'].append("--some-new=args")
new_config['metadata']['user_args'].append("--bool-arg")
backward.populate_priors(new_config['metadata'])
return conflicts.CommandLineConflict(old_config, new_config)
def changed_config(child_config):
"""Create a child config with a changed dimension"""
second_element = child_config['metadata']['user_args'][2]
second_element = second_element.replace('normal', 'uniform')
child_config['metadata']['user_args'][2] = second_element
backward.populate_priors(child_config['metadata'])
return child_config
def test_add_bad_default(self, parent_config, new_config):
"""Test if new dimension conflict raises an error if marked with invalid default value"""
new_config['metadata']['user_args'][
-1] = '-w_d~+normal(0,1,default_value=\'a\')'
backward.populate_priors(new_config['metadata'])
with pytest.raises(TypeError) as exc:
detect_conflicts(parent_config, new_config)
assert "Parameter \'/w_d\': Incorrect arguments." in str(exc.value)
def config(exp_config):
"""Return a configuration."""
config = exp_config[0][0]
config['metadata']['user_args'] = ['--x~uniform(-50, 50)']
config['name'] = 'exp'
config['working_dir'] = "/tmp/orion"
backward.populate_priors(config['metadata'])
return config
def list_arg_with_equals_cli_config(child_config):
"""Create a child config with an argument of the
form --args=1 --args=2 --args=3
"""
child_config['metadata']['user_args'] += [
'--args=1', '--args=2', '--args=3'
]
backward.populate_priors(child_config['metadata'])
return child_config
def same_userconfig_config(user_config, child_config):
"""Create a child config with a changed dimension"""
config_file_path = "./same_config.yaml"
with open(config_file_path, 'w') as f:
yaml.dump(user_config, f)
child_config['metadata']['user_args'][
-1] = '--config=%s' % config_file_path
backward.populate_priors(child_config['metadata'])
yield child_config
os.remove(config_file_path)
def test_rename_invalid(self, parent_config, child_config):
"""Test if renaming to invalid dimension raises an error"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~uniform(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_c'
backward.populate_priors(child_config['metadata'])
conflicts = detect_conflicts(parent_config, child_config)
with pytest.raises(ValueError) as exc:
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def exp_config():
"""Load an example database."""
with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'experiment.yaml')) as f:
exp_config = list(yaml.safe_load_all(f))
for config in exp_config[0]:
backward.populate_priors(config['metadata'])
return exp_config
def test_comparison(self, yaml_config, yaml_diff_config):
"""Test that different configs are detected as conflict."""
old_config = {'metadata': {'user_args': yaml_config}}
new_config = {'metadata': {'user_args': yaml_diff_config}}
backward.populate_priors(old_config['metadata'])
backward.populate_priors(new_config['metadata'])
conflicts = list(
conflict.ScriptConfigConflict.detect(old_config, new_config))
assert len(conflicts) == 1
def test_workon(database):
"""Test scenario having a configured experiment already setup."""
experiment = Experiment('voila_voici')
config = experiment.configuration
config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}}
config['pool_size'] = 1
config['max_trials'] = 100
config['metadata']['user_script'] = os.path.abspath(
os.path.join(os.path.dirname(__file__), "black_box.py"))
config['metadata']['user_args'] = ["-x~uniform(-50, 50)"]
backward.populate_priors(config['metadata'])
experiment.configure(config)
workon(experiment)
exp = list(database.experiments.find({'name': 'voila_voici'}))
assert len(exp) == 1
exp = exp[0]
assert '_id' in exp
exp_id = exp['_id']
assert exp['name'] == 'voila_voici'
assert exp['pool_size'] == 1
assert exp['max_trials'] == 100
assert exp['algorithms'] == {
'gradient_descent': {
'learning_rate': 0.1,
'dx_tolerance': 1e-7
}
}
assert 'user' in exp['metadata']
assert 'datetime' in exp['metadata']
assert 'user_script' in exp['metadata']
assert os.path.isabs(exp['metadata']['user_script'])
assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)']
trials = list(database.trials.find({'experiment': exp_id}))
assert len(trials) <= 15
trials = list(sorted(trials, key=lambda trial: trial['submit_time']))
assert trials[-1]['status'] == 'completed'
for result in trials[-1]['results']:
assert result['type'] != 'constraint'
if result['type'] == 'objective':
assert abs(result['value'] - 23.4) < 1e-6
assert result['name'] == 'example_objective'
elif result['type'] == 'gradient':
res = numpy.asarray(result['value'])
assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7
assert result['name'] == 'example_gradient'
params = trials[-1]['params']
assert len(params) == 1
assert params[0]['name'] == '/x'
assert params[0]['type'] == 'real'
assert (params[0]['value'] - 34.56789) < 1e-5
def changed_userconfig_config(user_config, child_config):
"""Create a child config with a changed dimension"""
config_file_path = './changed_config.yaml'
user_config['b'] = 'orion~uniform(-20, 0)'
user_config['some_other'] = 'hello'
with open(config_file_path, 'w') as f:
yaml.dump(user_config, f)
child_config['metadata']['user_args'][
-1] = '--config=%s' % config_file_path
backward.populate_priors(child_config['metadata'])
yield child_config
os.remove(config_file_path)
def test_add_single_hit(self, parent_config, new_config):
"""Test if adding a dimension only touches the correct status"""
del new_config['metadata']['user_args'][1]
backward.populate_priors(new_config['metadata'])
conflicts = detect_conflicts(parent_config, new_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
branch_builder.add_dimension('w_d')
assert len(conflicts.get()) == 3
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_adapter_rename_missing(self, parent_config, cl_config):
"""Test if a DimensionRenaming is created when solving a new conflict"""
cl_config['metadata']['user_args'] = ['-x~>w_d', '-w_d~+uniform(0,1)']
backward.populate_priors(cl_config['metadata'])
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 3
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
def bad_exp_parent_config():
"""Generate a new experiment configuration"""
config = dict(_id='test',
name='test',
metadata={
'user': '******',
'user_args': ['--x~normal(0,1)']
},
version=1,
algorithms='random')
backward.populate_priors(config['metadata'])
return config
def test_remove_missing_bad_default(self, parent_config, child_config):
"""Test if missing dimension conflict raises an error if marked with invalid default"""
child_config['metadata']['user_args'][1] = '-x~--100'
backward.populate_priors(child_config['metadata'])
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
conflict = conflicts.get()[1]
assert not conflict.is_resolved
assert isinstance(conflict, MissingDimensionConflict)
def test_remove_missing(self, parent_config, child_config):
"""Test if missing dimension conflict is automatically resolved"""
child_config['metadata']['user_args'][1] = '-x~-'
backward.populate_priors(child_config['metadata'])
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 2
conflict = conflicts.get_resolved()[1]
assert conflict.is_resolved
assert isinstance(conflict.resolution,
conflict.RemoveDimensionResolution)
def test_comparison_idem(self, yaml_config):
"""Test that identical configs are not detected as conflict."""
old_config = {'metadata': {'user_args': yaml_config}}
new_config = {
'metadata': {
'user_args': yaml_config + ['--other', 'args']
}
}
backward.populate_priors(old_config['metadata'])
backward.populate_priors(new_config['metadata'])
assert list(
conflict.ScriptConfigConflict.detect(old_config,
new_config)) == []
def new_config():
"""Generate a new experiment configuration"""
config = dict(name='test',
algorithms='fancy',
version=1,
metadata={
'VCS': 'to be changed',
'user_script': 'abs_path/black_box.py',
'user_args':
['--new~normal(0,2)', '--changed~normal(0,2)'],
'user': '******'
})
backward.populate_priors(config['metadata'])
return config
def test_add_new_default(self, parent_config, new_config):
"""Test if new dimension conflict is automatically resolved"""
new_config['metadata']['user_args'][
-1] = '-w_d~+normal(0,1,default_value=0)'
backward.populate_priors(new_config['metadata'])
conflicts = detect_conflicts(parent_config, new_config)
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 2
conflict = conflicts.get_resolved()[1]
assert conflict.is_resolved
assert isinstance(conflict.resolution, conflict.AddDimensionResolution)
assert conflict.resolution.default_value == 0.0
def exp_config():
"""Load an example database."""
with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'unittests', 'core', 'experiment.yaml')) as f:
exp_config = list(yaml.safe_load_all(f))
for i, t_dict in enumerate(exp_config[1]):
exp_config[1][i] = Trial(**t_dict).to_dict()
for config in exp_config[0]:
config["metadata"]["user_script"] = os.path.join(
os.path.dirname(__file__), config["metadata"]["user_script"])
backward.populate_priors(config['metadata'])
config['version'] = 1
return exp_config
def cl_config(create_db_instance):
"""Create a child config with markers for commandline solving"""
config = dict(name='test',
branch='test2',
algorithms='random',
metadata={
'hash_commit':
'old',
'user_script':
'abs_path/black_box.py',
'user_args': [
'--nameless=option', '-x~>w_d', '-w_d~+normal(0,1)',
'-y~+uniform(0,1)', '-z~-', '--omega~+normal(0,1)'
],
'user':
'******'
})
backward.populate_priors(config['metadata'])
return config
def test_name_experiment(self, bad_exp_parent_config, bad_exp_child_config,
create_db_instance):
"""Test if having the same experiment name does not create a conflict."""
backward.populate_priors(bad_exp_parent_config['metadata'])
backward.populate_priors(bad_exp_child_config['metadata'])
create_db_instance.write('experiments', bad_exp_parent_config)
create_db_instance.write('experiments', bad_exp_child_config)
conflicts = detect_conflicts(bad_exp_parent_config,
bad_exp_parent_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
assert len(conflicts.get()) == 1
assert len(conflicts.get_resolved()) == 0
conflict = conflicts.get([ExperimentNameConflict])[0]
assert conflict.new_config['name'] == 'test'
assert not conflict.is_resolved
branch_builder.change_experiment_name('test2')
assert len(conflicts.get_resolved()) == 1
assert conflict.new_config['name'] == 'test2'
assert conflict.is_resolved
def parent_config(user_config):
"""Create a configuration that will not hit the database."""
config = dict(_id='test',
name='test',
algorithms='random',
version=1,
metadata={
'VCS': {
"type": "git",
"is_dirty": False,
"HEAD_sha": "test",
"active_branch": None,
"diff_sha": "diff",
},
'user_script':
'abs_path/black_box.py',
'user_args': [
'--nameless=option', '-x~uniform(0,1)',
'-y~normal(0,1)', '-z~uniform(0,10)',
'--manual-resolution'
],
'user':
'******'
},
refers={})
config_file_path = "./parent_config.yaml"
with open(config_file_path, 'w') as f:
yaml.dump(user_config, f)
config['metadata']['user_args'].append('--config=%s' % config_file_path)
backward.populate_priors(config['metadata'])
yield config
os.remove(config_file_path)
def old_config(create_db_instance):
"""Generate an old experiment configuration"""
config = dict(
name='test',
algorithms='random',
version=1,
metadata={
'VCS': {
"type": "git",
"is_dirty": False,
"HEAD_sha": "test",
"active_branch": None,
"diff_sha": "diff",
},
'user_script': 'abs_path/black_box.py',
'user_args':
['--missing~uniform(-10,10)', '--changed~uniform(-10,10)'],
'user': '******'
})
backward.populate_priors(config['metadata'])
create_db_instance.write('experiments', config)
return config
def new_config(child_config):
"""Create a child config with a new dimension"""
child_config['metadata']['user_args'].append('-w_d~normal(0,1)')
backward.populate_priors(child_config['metadata'])
return child_config
def add_priors(experiment):
"""Add priors to metadata if not present"""
backward.populate_priors(experiment['metadata'])
def missing_config(child_config):
"""Create a child config with a missing dimension"""
del (child_config['metadata']['user_args'][1]) # del -x
del (child_config['metadata']['user_args'][1]) # del -y
backward.populate_priors(child_config['metadata'])
return child_config
