def changed_config(child_config):
"""Create a child config with a changed dimension"""
second_element = child_config['metadata']['user_args'][3]
second_element = second_element.replace('normal', 'uniform')
child_config['metadata']['user_args'][3] = second_element
backward.populate_space(child_config)
return child_config
def parent_version_config(script_path):
"""Return a configuration for an experiment."""
config = dict(
_id="parent_config",
name="old_experiment",
version=1,
algorithms="random",
metadata={
"user": "******",
"datetime": datetime.datetime.utcnow(),
"user_args": ["--x~normal(0,1)"],
"user_script": script_path,
"VCS": {
"type": "git",
"is_dirty": False,
"HEAD_sha": "test",
"active_branch": None,
"diff_sha": "diff",
},
"orion_version": "XYZ",
},
)
backward.populate_space(config)
return config
def new_config(random_dt):
"""Create a configuration that will not hit the database."""
new_config = dict(
name="supernaekei",
# refers is missing on purpose
metadata={
"user": "******",
"orion_version": 0.1,
"user_script": "tests/functional/demo/black_box.py",
"user_config": "abs_path/hereitis.yaml",
"user_args": ["--mini-batch~uniform(32, 256, discrete=True)"],
"VCS": {
"type": "git",
"is_dirty": False,
"HEAD_sha": "fsa7df7a8sdf7a8s7",
"active_branch": None,
"diff_sha": "diff",
},
},
version=1,
pool_size=10,
max_trials=1000,
max_broken=5,
working_dir=None,
algorithms={"dumbalgo": {}},
producer={"strategy": "NoParallelStrategy"},
# attrs starting with '_' also
# _id='fasdfasfa',
# and in general anything which is not in Experiment's slots
something_to_be_ignored="asdfa",
)
backward.populate_space(new_config)
return new_config
def list_arg_with_equals_cli_config(child_config):
"""Create a child config with an argument of the
form --args=1 --args=2 --args=3
"""
child_config["metadata"]["user_args"] += ["--args=1", "--args=2", "--args=3"]
backward.populate_space(child_config)
return child_config
def new_config(random_dt):
"""Create a configuration that will not hit the database."""
new_config = dict(
name='supernaekei',
# refers is missing on purpose
metadata={
'user': '******',
'orion_version': 0.1,
'user_script': 'abs_path/to_yoyoy.py',
'user_config': 'abs_path/hereitis.yaml',
'user_args': ['--mini-batch~uniform(32, 256, discrete=True)'],
'VCS': {
"type": "git",
"is_dirty": False,
"HEAD_sha": "fsa7df7a8sdf7a8s7",
"active_branch": None,
"diff_sha": "diff"
}
},
version=1,
pool_size=10,
max_trials=1000,
working_dir=None,
algorithms={'dumbalgo': {}},
producer={'strategy': 'NoParallelStrategy'},
# attrs starting with '_' also
# _id='fasdfasfa',
# and in general anything which is not in Experiment's slots
something_to_be_ignored='asdfa')
backward.populate_space(new_config)
return new_config
def test_rename_missing_changed_marked(self, parent_config, child_config):
"""Test if renaming is automatically applied with all conflicts resolved including
the new one caused by prior change
"""
child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)")
child_config["metadata"]["user_args"].append("-w_b~+normal(0,1)")
child_config["metadata"]["user_args"][2] = "-x~>w_b"
backward.populate_space(child_config)
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name="x")[0].is_resolved
assert conflicts.get([NewDimensionConflict], dimension_name="w_b")[
0
].is_resolved
assert conflicts.get([ChangedDimensionConflict], dimension_name="w_b")[
0
].is_resolved
assert not conflicts.get(dimension_name="w_a")[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 4
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(
resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution,
)
assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x"
assert (
resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name
== "/w_b"
)
def test_add_bad_default(self, parent_config, new_config):
"""Test if new dimension conflict raises an error if marked with invalid default value"""
new_config["metadata"]["user_args"][-1] = "-w_d~+normal(0,1,default_value='a')"
backward.populate_space(new_config)
with pytest.raises(TypeError) as exc:
detect_conflicts(parent_config, new_config)
assert "Parameter '/w_d': Incorrect arguments." in str(exc.value)
def old_config(create_db_instance):
"""Generate an old experiment configuration"""
user_script = "tests/functional/demo/black_box.py"
config = dict(
name="test",
algorithms="random",
version=1,
metadata={
"VCS": {
"type": "git",
"is_dirty": False,
"HEAD_sha": "test",
"active_branch": None,
"diff_sha": "diff",
},
"user_script":
user_script,
"user_args": [
user_script,
"--missing~uniform(-10,10)",
"--changed~uniform(-10,10)",
],
"user":
"******",
"orion_version":
"XYZ",
},
)
backward.populate_space(config)
create_db_instance.write("experiments", config)
return config
def cl_config(create_db_instance):
"""Create a child config with markers for commandline solving"""
user_script = "tests/functional/demo/black_box.py"
config = dict(
name="test",
branch="test2",
algorithms="random",
metadata={
"hash_commit": "old",
"user_script": user_script,
"user_args": [
user_script,
"--nameless=option",
"-x~>w_d",
"-w_d~+normal(0,1)",
"-y~+uniform(0,1)",
"-z~-",
"--omega~+normal(0,1)",
],
"user": "******",
"orion_version": "XYZ",
},
)
backward.populate_space(config)
return config
def parent_config(user_config):
"""Create a configuration that will not hit the database."""
user_script = 'abs_path/black_box.py'
config = dict(
_id='test',
name='test',
algorithms='random',
version=1,
metadata={'VCS': {"type": "git",
"is_dirty": False,
"HEAD_sha": "test",
"active_branch": None,
"diff_sha": "diff",
},
'user_script': user_script,
'user_args': [
user_script, '--nameless=option', '-x~uniform(0,1)', '-y~normal(0,1)',
'-z~uniform(0,10)', '--manual-resolution'],
'user': '******'},
refers={})
config_file_path = "./parent_config.yaml"
with open(config_file_path, 'w') as f:
yaml.dump(user_config, f)
config['metadata']['user_args'].append('--config=%s' % config_file_path)
backward.populate_space(config)
yield config
os.remove(config_file_path)
def cli_conflict(old_config, new_config):
"""Generate a commandline conflict"""
new_config = copy.deepcopy(new_config)
new_config["metadata"]["user_args"].append("--some-new=args")
new_config["metadata"]["user_args"].append("--bool-arg")
backward.populate_space(new_config)
return conflicts.CommandLineConflict(old_config, new_config)
def old_config(create_db_instance):
"""Generate an old experiment configuration"""
user_script = 'abs_path/black_box.py'
config = dict(name='test',
algorithms='random',
version=1,
metadata={
'VCS': {
"type": "git",
"is_dirty": False,
"HEAD_sha": "test",
"active_branch": None,
"diff_sha": "diff",
},
'user_script':
user_script,
'user_args': [
user_script, '--missing~uniform(-10,10)',
'--changed~uniform(-10,10)'
],
'user':
'******'
})
backward.populate_space(config)
create_db_instance.write('experiments', config)
return config
def fetch_config_from_db(name, version=None):
"""Fetch configuration from database
Parameters
----------
name: str
Name of the experiment to fetch
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
"""
configs = get_storage().fetch_experiments({'name': name})
if not configs:
return {}
config = _fetch_config_version(configs, version)
if len(configs) > 1:
log.info(
"Many versions for experiment %s have been found. Using latest "
"version %s.", name, config['version'])
backward.populate_space(config)
return config
def changed_config(child_config):
"""Create a child config with a changed dimension"""
second_element = child_config["metadata"]["user_args"][3]
second_element = second_element.replace("normal", "uniform")
child_config["metadata"]["user_args"][3] = second_element
backward.populate_space(child_config)
return child_config
def test_rename_missing_changed(self, parent_config, child_config):
"""Test if renaming is automatically applied with both conflicts resolved,
but not the new one because of prior change
"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~normal(0,1)')
child_config['metadata']['user_args'][2] = '-x~>w_b'
backward.populate_space(child_config)
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name='x')[0].is_resolved
assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved
assert not conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved
assert not conflicts.get(dimension_name='w_a')[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def fetch_config_from_db(name, version=None):
"""Fetch configuration from database
Parameters
----------
name: str
Name of the experiment to fetch
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
"""
configs = get_storage().fetch_experiments({"name": name})
if not configs:
return {}
config = _fetch_config_version(configs, version)
if len(configs) > 1 and version is None:
log.info(
"Many versions for experiment %s have been found. Using latest "
"version %s.",
name,
config["version"],
)
log.debug("Config found in DB:\n%s", pprint.pformat(config))
backward.populate_space(config, force_update=False)
backward.update_max_broken(config)
return config
def test_rename_missing_changed(self, parent_config, missing_config):
"""Test if renaming a dimension to another with different prior solves both conflicts but
creates a new one which is not solved
"""
missing_config["metadata"]["user_args"].append("-w_d~normal(0,1)")
backward.populate_space(missing_config)
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 4
branch_builder.rename_dimension("x", "w_d")
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(
resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution,
)
assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x"
assert (
resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name
== "/w_d"
)
def list_arg_with_equals_cli_config(child_config):
"""Create a child config with an argument of the
form --args=1 --args=2 --args=3
"""
child_config['metadata']['user_args'] += ['--args=1',
'--args=2', '--args=3']
backward.populate_space(child_config)
return child_config
def same_userconfig_config(user_config, child_config):
"""Create a child config with a changed dimension"""
config_file_path = "./same_config.yaml"
with open(config_file_path, "w") as f:
yaml.dump(user_config, f)
child_config["metadata"]["user_args"][-1] = "--config=%s" % config_file_path
backward.populate_space(child_config)
yield child_config
os.remove(config_file_path)
def config(exp_config):
"""Return a configuration."""
config = exp_config[0][0]
config['metadata']['user_args'] = ['--x~uniform(-50, 50)']
config['name'] = 'exp'
config['working_dir'] = "/tmp/orion"
backward.populate_space(config)
config['space'] = config['metadata']['priors']
return config
def test_rename_invalid(self, parent_config, child_config):
"""Test if renaming to invalid dimension raises an error"""
child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)")
child_config["metadata"]["user_args"].append("-w_b~uniform(0,1)")
child_config["metadata"]["user_args"][2] = "-x~>w_c"
backward.populate_space(child_config)
conflicts = detect_conflicts(parent_config, child_config)
with pytest.raises(ValueError) as exc:
ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def child_version_config(parent_version_config):
"""Return a configuration for an experiment."""
config = copy.deepcopy(parent_version_config)
config['_id'] = 'child_config'
config['version'] = 2
config['refers'] = {'parent_id': 'parent_config'}
config['metadata']['datetime'] = datetime.datetime.utcnow()
config['metadata']['user_args'].append('--y~+normal(0,1)')
backward.populate_space(config)
return config
def child_version_config(parent_version_config):
"""Return a configuration for an experiment."""
config = copy.deepcopy(parent_version_config)
config["_id"] = "child_config"
config["version"] = 2
config["refers"] = {"parent_id": "parent_config"}
config["metadata"]["datetime"] = datetime.datetime.utcnow()
config["metadata"]["user_args"].append("--y~+normal(0,1)")
backward.populate_space(config)
return config
def changed_userconfig_config(user_config, child_config):
"""Create a child config with a changed dimension"""
config_file_path = './changed_config.yaml'
user_config['b'] = 'orion~uniform(-20, 0, precision=None)'
user_config['some_other'] = 'hello'
with open(config_file_path, 'w') as f:
yaml.dump(user_config, f)
child_config['metadata']['user_args'][-1] = '--config=%s' % config_file_path
backward.populate_space(child_config)
yield child_config
os.remove(config_file_path)
def config(exp_config):
"""Return a configuration."""
config = exp_config[0][0]
config["metadata"]["user_args"] = ["--x~uniform(-50, 50)"]
config["metadata"]["VCS"] = resolve_config.infer_versioning_metadata(
config["metadata"]["user_script"])
config["name"] = "exp"
config["working_dir"] = "/tmp/orion"
backward.populate_space(config)
config["space"] = config["metadata"]["priors"]
return config
def changed_userconfig_config(user_config, child_config):
"""Create a child config with a changed dimension"""
config_file_path = "./changed_config.yaml"
user_config["b"] = "orion~uniform(-20, 0, precision=None)"
user_config["some_other"] = "hello"
with open(config_file_path, "w") as f:
yaml.dump(user_config, f)
child_config["metadata"]["user_args"][-1] = "--config=%s" % config_file_path
backward.populate_space(child_config)
yield child_config
os.remove(config_file_path)
def test_comparison(self, yaml_config, yaml_diff_config):
"""Test that different configs are detected as conflict."""
old_config = {'metadata': {'user_args': yaml_config}}
new_config = {'metadata': {'user_args': yaml_diff_config}}
backward.populate_space(old_config)
backward.populate_space(new_config)
conflicts = list(
conflict.ScriptConfigConflict.detect(old_config, new_config))
assert len(conflicts) == 1
def exp_config():
"""Load an example database."""
with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'experiment.yaml')) as f:
exp_config = list(yaml.safe_load_all(f))
for config in exp_config[0]:
backward.populate_space(config)
return exp_config
def test_remove_with_id(self, exp_config, database, orion_db):
"""Query using ``_id`` key."""
filt = {'_id': exp_config[0][0]['_id']}
count_before = database['experiments'].count()
# call interface
assert orion_db.remove('experiments', filt) == 1
assert database['experiments'].count() == count_before - 1
loaded_configs = database['experiments'].find()
for loaded_config in loaded_configs:
backward.populate_space(loaded_config)
assert loaded_configs == exp_config[0][1:]
def test_remove_many_default(self, exp_config, database, orion_db):
"""Should match existing entries, and delete them all."""
filt = {'metadata.user': '******'}
count_before = database['experiments'].count()
count_filt = database['experiments'].count(filt)
# call interface
assert orion_db.remove('experiments', filt) == count_filt
assert database['experiments'].count() == count_before - count_filt
assert database['experiments'].count() == 1
loaded_config = list(database['experiments'].find())
backward.populate_space(loaded_config[0])
assert loaded_config == [exp_config[0][0]]
