def test_add_bad_default(self, parent_config, new_config):
"""Test if new dimension conflict raises an error if marked with invalid default value"""
new_config["metadata"]["user_args"][-1] = "-w_d~+normal(0,1,default_value='a')"
backward.populate_space(new_config)
with pytest.raises(TypeError) as exc:
detect_conflicts(parent_config, new_config)
assert "Parameter '/w_d': Incorrect arguments." in str(exc.value)
def test_add_bad_default(self, parent_config, new_config):
"""Test if new dimension conflict raises an error if marked with invalid default value"""
new_config['metadata']['user_args'][
-1] = '-w_d~+normal(0,1,default_value=\'a\')'
with pytest.raises(TypeError) as exc:
detect_conflicts(parent_config, new_config)
assert "Parameter \'/w_d\': Incorrect arguments." in str(exc.value)
def test_bad_name_experiment(self, parent_config, child_config,
monkeypatch):
"""Test if changing the experiment names does not work for invalid name and revert
to old one
"""
def _is_unique(self, *args, **kwargs):
return False
def _versions(self, *args, **kwargs):
return True
monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution,
"_name_is_unique", _is_unique)
monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution,
"_check_for_greater_versions", _versions)
conflicts = detect_conflicts(parent_config, child_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
assert len(conflicts.get()) == 1
assert len(conflicts.get_resolved()) == 0
conflict = conflicts.get([ExperimentNameConflict])[0]
assert not conflict.is_resolved
branch_builder.change_experiment_name('test2')
assert len(conflicts.get_resolved()) == 0
assert conflict.new_config['name'] == 'test'
assert not conflict.is_resolved
def test_rename_missing_changed_marked(self, parent_config, child_config):
"""Test if renaming is automatically applied with all conflicts resolved including
the new one caused by prior change
"""
child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)")
child_config["metadata"]["user_args"].append("-w_b~+normal(0,1)")
child_config["metadata"]["user_args"][2] = "-x~>w_b"
backward.populate_space(child_config)
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name="x")[0].is_resolved
assert conflicts.get([NewDimensionConflict], dimension_name="w_b")[
0
].is_resolved
assert conflicts.get([ChangedDimensionConflict], dimension_name="w_b")[
0
].is_resolved
assert not conflicts.get(dimension_name="w_a")[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 4
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(
resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution,
)
assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x"
assert (
resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name
== "/w_b"
)
def test_rename_missing_changed(self, parent_config, child_config):
"""Test if renaming is automatically applied with both conflicts resolved,
but not the new one because of prior change
"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~normal(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_b'
backward.populate_priors(child_config['metadata'])
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {'manual_resolution': True})
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name='x')[0].is_resolved
assert conflicts.get([NewDimensionConflict],
dimension_name='w_b')[0].is_resolved
assert not conflicts.get([ChangedDimensionConflict],
dimension_name='w_b')[0].is_resolved
assert not conflicts.get(dimension_name='w_a')[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[
2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[
1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_rename_missing_changed(self, parent_config, missing_config):
"""Test if renaming a dimension to another with different prior solves both conflicts but
creates a new one which is not solved
"""
missing_config['metadata']['user_args'].append('-w_d~normal(0,1)')
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 4
branch_builder.rename_dimension('x', 'w_d')
assert len(conflicts.get()) == 5
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 2
assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution
assert isinstance(resolved_conflicts[0].resolution,
resolved_conflicts[0].RenameDimensionResolution)
assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_rename_missing_changed(self, parent_config, missing_config):
"""Test if renaming a dimension to another with different prior solves both conflicts but
creates a new one which is not solved
"""
missing_config["metadata"]["user_args"].append("-w_d~normal(0,1)")
backward.populate_space(missing_config)
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert len(conflicts.get()) == 4
branch_builder.rename_dimension("x", "w_d")
assert len(conflicts.get()) == 5
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution
assert isinstance(
resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution,
)
assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x"
assert (
resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name
== "/w_d"
)
def test_ignore_code_conflict(self, parent_config, changed_code_config):
"""Test if ignored code commit hash change is detected as a conflict"""
conflicts = detect_conflicts(
parent_config, changed_code_config, {"ignore_code_changes": True}
)
assert len(conflicts.get()) == 1
def test_rename_missing_changed_marked(self, parent_config, child_config):
"""Test if renaming is automatically applied with all conflicts resolved including
the new one caused by prior change
"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~+normal(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_b'
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {})
assert len(conflicts.get()) == 5
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get(dimension_name='x')[0].is_resolved
assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved
assert conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved
assert not conflicts.get(dimension_name='w_a')[0].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution
assert isinstance(resolved_conflicts[0].resolution,
resolved_conflicts[0].RenameDimensionResolution)
assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_rename_missing(self, parent_config, missing_config):
"""Test if renaming a dimension to another solves both conflicts"""
missing_config['metadata']['user_args'].append('-w_d~uniform(0,1)')
backward.populate_priors(missing_config['metadata'])
conflicts = detect_conflicts(parent_config, missing_config)
branch_builder = ExperimentBranchBuilder(conflicts,
{'manual_resolution': True})
branch_builder.rename_dimension('x', 'w_d')
assert len(conflicts.get()) == 4
assert conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert conflicts.get([MissingDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[1].is_resolved
resolved_conflicts = conflicts.get_resolved()
assert len(resolved_conflicts) == 3
assert resolved_conflicts[1].resolution is resolved_conflicts[
2].resolution
assert isinstance(resolved_conflicts[1].resolution,
resolved_conflicts[1].RenameDimensionResolution)
assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x'
assert resolved_conflicts[
1].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_config_non_dim_conflict(self, parent_config, changed_userconfig_config):
"""Test if changed configuration file is detected as a conflict"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
assert len(conflicts.get()) == 4
assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert not conflicts.get([ScriptConfigConflict])[0].is_resolved
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config):
"""Test if giving an invalid change-type prints error message and do nothing"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
orion.core.config.evc.config_change_type = "bad-type"
ExperimentBranchBuilder(conflicts)
assert len(conflicts.get()) == 4
assert len(conflicts.get_resolved()) == 3
def test_cli_conflict(self, parent_config, changed_cli_config):
"""Test if changed command line call is detected as a conflict"""
conflicts = detect_conflicts(parent_config, changed_cli_config)
assert len(conflicts.get()) == 2
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert not conflicts.get([CommandLineConflict])[0].is_resolved
def test_rename_invalid(self, parent_config, child_config):
"""Test if renaming to invalid dimension raises an error"""
child_config['metadata']['user_args'].append('-w_a~uniform(0,1)')
child_config['metadata']['user_args'].append('-w_b~uniform(0,1)')
child_config['metadata']['user_args'][1] = '-x~>w_c'
conflicts = detect_conflicts(parent_config, child_config)
with pytest.raises(ValueError) as exc:
ExperimentBranchBuilder(conflicts, {})
assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def test_cli_ignored_conflict(self, parent_config, changed_cli_config):
"""Test if ignored changed command line call is detected as a conflict"""
changed_cli_config['metadata']['user_args'].pop()
conflicts = detect_conflicts(parent_config, changed_cli_config,
{'non_monitored_arguments': ['u', 'another']})
assert len(conflicts.get()) == 1
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
def test_bad_cli_change(self, capsys, parent_config, changed_cli_config):
"""Test if giving an invalid change-type fails the the resolution"""
conflicts = detect_conflicts(parent_config, changed_cli_config)
orion.core.config.evc.cli_change_type = "bad-type"
ExperimentBranchBuilder(conflicts)
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 1
orion.core.config.evc.cli_change_type = "break"
def test_changed_dim_userconfig_conflict(self, parent_config, changed_userconfig_config):
"""Test if changed dimension from user's config is currently detected"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
assert len(conflicts.get()) == 4
conflict = conflicts.get()[0]
assert conflict.is_resolved is False
assert conflict.dimension.name == '/b'
assert isinstance(conflict, ChangedDimensionConflict)
def test_new_dim_conflict(self, parent_config, new_config):
"""Test if new dimension is currently detected"""
conflicts = detect_conflicts(parent_config, new_config)
assert len(conflicts.get()) == 2
conflict = conflicts.get()[1]
assert conflict.is_resolved is False
assert conflict.dimension.name == '/w_d'
assert isinstance(conflict, NewDimensionConflict)
def test_rename_invalid(self, parent_config, child_config):
"""Test if renaming to invalid dimension raises an error"""
child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)")
child_config["metadata"]["user_args"].append("-w_b~uniform(0,1)")
child_config["metadata"]["user_args"][2] = "-x~>w_c"
backward.populate_space(child_config)
conflicts = detect_conflicts(parent_config, child_config)
with pytest.raises(ValueError) as exc:
ExperimentBranchBuilder(conflicts, manual_resolution=True)
assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def _get_conflicts(experiment, branching):
"""Get conflicts between current experiment and corresponding configuration in database"""
db_experiment = build_view(experiment.name, experiment.version)
conflicts = detect_conflicts(db_experiment.configuration,
experiment.configuration, branching)
# elif must_branch and not enable_branching:
# raise ValueError("Configuration is different and generate a branching event")
return conflicts
def test_missing_dim_conflict(self, parent_config, missing_config):
"""Test if missing dimension is currently detected"""
conflicts = detect_conflicts(parent_config, missing_config)
assert len(conflicts.get()) == 3
conflict = conflicts.get()[1]
assert conflict.is_resolved is False
assert conflict.dimension.name == '/x'
assert isinstance(conflict, MissingDimensionConflict)
def test_bad_name_experiment(self, parent_config, child_config, create_db_instance):
"""Test if experiment name conflict is not resolved when invalid name is marked"""
new_name = 'test'
create_db_instance.write('experiments', parent_config)
conflicts = detect_conflicts(parent_config, child_config)
ExperimentBranchBuilder(conflicts, {'branch': new_name})
assert len(conflicts.get()) == 1
assert len(conflicts.get_resolved()) == 0
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config):
"""Test if giving an invalid change-type prints error message and do nothing"""
conflicts = detect_conflicts(parent_config, changed_userconfig_config)
branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True)
capsys.readouterr()
branch_builder.set_script_config_change_type("bad-type")
out, err = capsys.readouterr()
assert "Invalid script's config change type" in out.split("\n")[-3]
assert len(conflicts.get()) == 4
assert len(conflicts.get_resolved()) == 1
def test_orion_version_conflict(self, parent_config, changed_orion_version_config):
"""Test if orion version changes are currently detected"""
conflicts = detect_conflicts(parent_config, changed_orion_version_config)
assert len(conflicts.get()) == 2
conflict = conflicts.get()[1]
assert conflict.is_resolved is False
assert conflict.old_config["metadata"]["orion_version"] == "XYZ"
assert conflict.new_config["metadata"]["orion_version"] == "UVW"
assert isinstance(conflict, OrionVersionConflict)
def test_bad_cli_change(self, capsys, parent_config, changed_cli_config):
"""Test if giving an invalid change-type prints error message and do nothing"""
conflicts = detect_conflicts(parent_config, changed_cli_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
capsys.readouterr()
branch_builder.set_cli_change_type('bad-type')
out, err = capsys.readouterr()
assert 'Invalid cli change type' in out.split("\n")[-3]
assert len(conflicts.get()) == 2
assert len(conflicts.get_resolved()) == 0
def test_algo_conflict(self, parent_config, changed_algo_config):
"""Test if algorithm changes are currently detected"""
conflicts = detect_conflicts(parent_config, changed_algo_config)
assert len(conflicts.get()) == 2
conflict = conflicts.get()[0]
assert conflict.is_resolved is False
assert conflict.old_config['algorithms'] == 'random'
assert conflict.new_config['algorithms'] == 'stupid-grid'
assert isinstance(conflict, AlgorithmConflict)
def test_add_single_hit(self, parent_config, new_config):
"""Test if adding a dimension only touches the correct status"""
del new_config['metadata']['user_args'][1]
conflicts = detect_conflicts(parent_config, new_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
branch_builder.add_dimension('w_d')
assert len(conflicts.get()) == 3
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
assert conflicts.get([NewDimensionConflict])[0].is_resolved
assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_code_conflict(self, parent_config, changed_code_config):
"""Test if code commit hash change is currently detected"""
conflicts = detect_conflicts(parent_config, changed_code_config)
assert len(conflicts.get()) == 2
conflict = conflicts.get()[0]
assert conflict.is_resolved is False
assert conflict.old_config['metadata']['VCS']['HEAD_sha'] == 'test'
assert conflict.new_config['metadata']['VCS']['HEAD_sha'] == 'new_test'
assert isinstance(conflict, CodeConflict)
def test_config_new_name_no_conflict(self, parent_config, same_userconfig_config):
"""Test if same configuration file with a different name is not detected as a conflict"""
conflicts = detect_conflicts(parent_config, same_userconfig_config)
assert parent_config['metadata']['user_args'][-1].startswith('--config=')
assert same_userconfig_config['metadata']['user_args'][-1].startswith('--config=')
assert (parent_config['metadata']['user_args'][-1] !=
same_userconfig_config['metadata']['user_args'][-1])
assert len(conflicts.get()) == 2
assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
def test_adapter_rename_missing(self, parent_config, cl_config):
"""Test if a DimensionRenaming is created when solving a new conflict"""
cl_config['metadata']['user_args'] = ['-x~>w_d', '-w_d~+uniform(0,1)']
conflicts = detect_conflicts(parent_config, cl_config)
branch_builder = ExperimentBranchBuilder(conflicts, {})
adapters = branch_builder.create_adapters().adapters
assert len(conflicts.get_resolved()) == 3
assert len(adapters) == 1
assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
