def test_bad_name_experiment(self, parent_config, child_config,
monkeypatch):
"""Test if changing the experiment names does not work for invalid name and revert
to old one
"""
def _is_unique(self, *args, **kwargs):
return False
def _versions(self, *args, **kwargs):
return True
monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution,
"_name_is_unique", _is_unique)
monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution,
"_check_for_greater_versions", _versions)
conflicts = detect_conflicts(parent_config, child_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
assert len(conflicts.get()) == 1
assert len(conflicts.get_resolved()) == 0
conflict = conflicts.get([ExperimentNameConflict])[0]
assert not conflict.is_resolved
branch_builder.change_experiment_name('test2')
assert len(conflicts.get_resolved()) == 0
assert conflict.new_config['name'] == 'test'
assert not conflict.is_resolved
def _branch_config(self, conflicts, branching_configuration):
"""Ask for a different identifier for this experiment. Set :attr:`refers`
key to previous experiment's name, the one that we branched from.
:param config: Conflicting configuration that will change based on prompt.
"""
experiment_brancher = ExperimentBranchBuilder(conflicts,
branching_configuration)
needs_manual_resolution = (not experiment_brancher.is_resolved
or experiment_brancher.auto_resolution)
if needs_manual_resolution:
branching_prompt = BranchingPrompt(experiment_brancher)
if not sys.__stdin__.isatty():
raise ValueError(
"Configuration is different and generates a branching event:\n{}"
.format(branching_prompt.get_status()))
branching_prompt.cmdloop()
if branching_prompt.abort or not experiment_brancher.is_resolved:
sys.exit()
adapter = experiment_brancher.create_adapters()
self._instantiate_config(experiment_brancher.conflicting_config)
self.refers['adapter'] = adapter
self.refers['parent_id'] = self._id
def test_rename_missing(self, parent_config, missing_config):
"""Test if renaming a dimension to another solves both conflicts"""
missing_config['metadata']['user_args'].append('-w_d~uniform(0,1)')
backward.populate_priors(missing_config['metadata'])
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
branch_builder.rename_dimension('x', 'w_d')
assert len(conflicts.get()) == 4
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[
2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[
1].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_rename_missing_changed(self, parent_config, missing_config):
"""Test if renaming a dimension to another with different prior solves both conflicts but
creates a new one which is not solved
"""
missing_config['metadata']['user_args'].append('-w_d~normal(0,1)')
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 4
branch_builder.rename_dimension('x', 'w_d')
assert len(conflicts.get()) == 5
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 2
assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution
assert isinstance(resolved_conflicts[0].resolution,
resolved_conflicts[0].RenameDimensionResolution)
assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_rename_missing_changed(self, parent_config, missing_config):
"""Test if renaming a dimension to another with different prior solves both conflicts but
creates a new one which is not solved
"""
missing_config["metadata"]["user_args"].append("-w_d~normal(0,1)")
backward.populate_space(missing_config)
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 4
branch_builder.rename_dimension("x", "w_d")
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(
resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution,
)
assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x"
assert (
resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name
== "/w_d"
)
def _branch_experiment(experiment, conflicts, version, branching_arguments):
"""Create a new branch experiment with adapters for the given conflicts"""
experiment_brancher = ExperimentBranchBuilder(conflicts,
**branching_arguments)
needs_manual_resolution = (not experiment_brancher.is_resolved
or experiment_brancher.manual_resolution)
if not experiment_brancher.is_resolved:
name_conflict = conflicts.get([ExperimentNameConflict])[0]
if not name_conflict.is_resolved and not version:
raise RaceCondition(
'There was likely a race condition during version increment.')
if needs_manual_resolution:
# TODO: This should only be possible when using cmdline API
branching_prompt = BranchingPrompt(experiment_brancher)
if not sys.__stdin__.isatty():
raise BranchingEvent(branching_prompt.get_status())
branching_prompt.cmdloop()
if branching_prompt.abort or not experiment_brancher.is_resolved:
sys.exit()
config = experiment_brancher.conflicting_config
config['refers']['adapter'] = experiment_brancher.create_adapters(
).configuration
config['refers']['parent_id'] = experiment.id
config.pop('_id')
return create_experiment(**config)
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config):
"""Test if giving an invalid change-type prints error message and do nothing"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
capsys.readouterr()
branch_builder.set_script_config_change_type("bad-type")
out, err = capsys.readouterr()
assert "Invalid script's config change type" in out.split("\n")[-3]
assert len(conflicts.get()) == 4
assert len(conflicts.get_resolved()) == 1
def test_add_single_hit(self, parent_config, new_config):
"""Test if adding a dimension only touches the correct status"""
del new_config['metadata']['user_args'][1]
conflicts = detect_conflicts(parent_config, new_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
branch_builder.add_dimension('w_d')
assert len(conflicts.get()) == 3
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_bad_cli_change(self, capsys, parent_config, changed_cli_config):
"""Test if giving an invalid change-type prints error message and do nothing"""
conflicts = detect_conflicts(parent_config, changed_cli_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
capsys.readouterr()
branch_builder.set_cli_change_type('bad-type')
out, err = capsys.readouterr()
assert 'Invalid cli change type' in out.split("\n")[-3]
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 0
def test_add_single_hit(self, parent_config, new_config):
"""Test if adding a dimension only touches the correct status"""
del new_config["metadata"]["user_args"][2]
backward.populate_space(new_config)
conflicts = detect_conflicts(parent_config, new_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
branch_builder.add_dimension("w_d")
assert len(conflicts.get()) == 3
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_adapter_rename_missing(self, parent_config, cl_config):
"""Test if a DimensionRenaming is created when solving a new conflict"""
cl_config['metadata']['user_args'] = ['-x~>w_d', '-w_d~+uniform(0,1)']
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 3
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
def test_adapter_add_new(self, parent_config, cl_config):
"""Test if a DimensionAddition is created when solving a new conflict"""
cl_config["metadata"]["user_args"] = ["-w_d~+normal(0,1)"]
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 2
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionAddition)
def test_adapter_add_changed(self, parent_config, cl_config):
"""Test if a DimensionPriorChange is created when solving a new conflict"""
cl_config['metadata']['user_args'] = ['-y~+uniform(0,1)']
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 2
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionPriorChange)
def test_adapter_remove_missing(self, parent_config, cl_config):
"""Test if a DimensionDeletion is created when solving a new conflict"""
cl_config['metadata']['user_args'] = ['-z~-']
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 2
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionDeletion)
def test_remove_missing(self, parent_config, missing_config):
"""Test if removing a missing dimension solves the conflict"""
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
branch_builder.remove_dimension('x')
assert len(conflicts.get()) == 3
assert len(conflicts.get_resolved()) == 1
conflict = conflicts.get_resolved()[0]
assert conflict.is_resolved
assert isinstance(conflict.resolution, conflict.RemoveDimensionResolution)
def test_add_changed(self, parent_config, changed_config):
"""Test if adding a changed dimension solves the conflict"""
conflicts = detect_conflicts(parent_config, changed_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
branch_builder.add_dimension('y')
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
conflict = conflicts.get_resolved()[0]
assert conflict.is_resolved
assert isinstance(conflict.resolution, conflict.ChangeDimensionResolution)
def test_add_new(self, parent_config, new_config):
"""Test if adding a new dimension solves the conflict"""
conflicts = detect_conflicts(parent_config, new_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
branch_builder.add_dimension("w_d")
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 2
conflict = conflicts.get_resolved()[1]
assert conflict.is_resolved
assert isinstance(conflict.resolution, conflict.AddDimensionResolution)
def test_adapter_rename_missing(self, parent_config, cl_config):
"""Test if a DimensionRenaming is created when solving a new conflict"""
cl_config["metadata"]["user_args"] = ["-x~>w_d", "-w_d~+uniform(0,1)"]
backward.populate_space(cl_config)
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 3
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
def test_adapter_rename_different_prior(self, parent_config, cl_config):
"""Test if a DimensionRenaming is created when solving a new conflict"""
cl_config["metadata"]["user_args"] = ["-x~>w_d", "-w_d~+normal(0,1)"]
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 4
assert len(adapters) == 2
assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
assert isinstance(adapters[1], evc.adapters.DimensionPriorChange)
def test_cli_change(self, parent_config, changed_cli_config):
"""Test if giving a proper change-type solves the command line conflict"""
conflicts = detect_conflicts(parent_config, changed_cli_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 0
branch_builder.set_cli_change_type(evc.adapters.CommandLineChange.types[0])
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
conflict = conflicts.get_resolved()[0]
assert conflict.is_resolved
assert isinstance(conflict, CommandLineConflict)
def test_config_change(self, parent_config, changed_userconfig_config):
"""Test if giving a proper change-type solves the user script config conflict"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 4
assert len(conflicts.get_resolved()) == 0
branch_builder.set_script_config_change_type(evc.adapters.ScriptConfigChange.types[0])
assert len(conflicts.get()) == 4
assert len(conflicts.get_resolved()) == 1
conflict = conflicts.get_resolved()[0]
assert conflict.is_resolved
assert isinstance(conflict, ScriptConfigConflict)
def test_code_change(self, parent_config, changed_code_config):
"""Test if giving a proper change-type solves the code conflict"""
conflicts = detect_conflicts(parent_config, changed_code_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
branch_builder.set_code_change_type(evc.adapters.CodeChange.types[0])
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 2
conflict = conflicts.get_resolved()[0]
assert conflict.is_resolved
assert isinstance(conflict, CodeConflict)
def test_name_experiment(self, parent_config, child_config, create_db_instance):
"""Test if changing the experiment names work for valid name"""
conflicts = detect_conflicts(parent_config, child_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 1
assert len(conflicts.get_resolved()) == 0
conflict = conflicts.get([ExperimentNameConflict])[0]
assert conflict.new_config['name'] == 'test'
assert not conflict.is_resolved
branch_builder.change_experiment_name('test2')
assert len(conflicts.get_resolved()) == 1
assert conflict.new_config['name'] == 'test2'
assert conflict.is_resolved
def test_rename_missing_changed(self, parent_config, child_config):
"""Test if renaming is automatically applied with both conflicts resolved,
but not the new one because of prior change
"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~normal(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_b'
backward.populate_priors(child_config['metadata'])
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name='x')[0].is_resolved
assert conflicts.get([NewDimensionConflict],
dimension_name='w_b')[0].is_resolved
assert not conflicts.get([ChangedDimensionConflict],
dimension_name='w_b')[0].is_resolved
assert not conflicts.get(dimension_name='w_a')[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[
2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[
1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_rename_missing_changed_marked(self, parent_config, child_config):
"""Test if renaming is automatically applied with all conflicts resolved including
the new one caused by prior change
"""
child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)")
child_config["metadata"]["user_args"].append("-w_b~+normal(0,1)")
child_config["metadata"]["user_args"][2] = "-x~>w_b"
backward.populate_space(child_config)
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name="x")[0].is_resolved
assert conflicts.get([NewDimensionConflict], dimension_name="w_b")[
0
].is_resolved
assert conflicts.get([ChangedDimensionConflict], dimension_name="w_b")[
0
].is_resolved
assert not conflicts.get(dimension_name="w_a")[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 4
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(
resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution,
)
assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x"
assert (
resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name
== "/w_b"
)
def test_rename_missing_changed_marked(self, parent_config, child_config):
"""Test if renaming is automatically applied with all conflicts resolved including
the new one caused by prior change
"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~+normal(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_b'
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 5
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name='x')[0].is_resolved
assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved
assert conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved
assert not conflicts.get(dimension_name='w_a')[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution
assert isinstance(resolved_conflicts[0].resolution,
resolved_conflicts[0].RenameDimensionResolution)
assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_algo_change(self, parent_config, changed_algo_config):
"""Test if setting the algorithm conflict solves it"""
conflicts = detect_conflicts(parent_config, changed_algo_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 0
branch_builder.set_algo()
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
conflict = conflicts.get_resolved()[0]
assert conflict.is_resolved
assert isinstance(conflict.resolution, conflict.AlgorithmResolution)
def test_orion_version_change(self, parent_config, changed_orion_version_config):
"""Test if setting the orion version conflict solves it"""
conflicts = detect_conflicts(parent_config, changed_orion_version_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
branch_builder.set_orion_version()
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 2
conflict = conflicts.get_resolved()[1]
assert conflict.is_resolved
assert isinstance(conflict.resolution, conflict.OrionVersionResolution)
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config):
"""Test if giving an invalid change-type prints error message and do nothing"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
orion.core.config.evc.config_change_type = "bad-type"
ExperimentBranchBuilder(conflicts)
assert len(conflicts.get()) == 4
assert len(conflicts.get_resolved()) == 3
def test_rename_invalid(self, parent_config, child_config):
"""Test if renaming to invalid dimension raises an error"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~uniform(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_c'
conflicts = detect_conflicts(parent_config, child_config)
with pytest.raises(ValueError) as exc:
ExperimentBranchBuilder(conflicts, {})
assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
