def test_add_bad_default(self, parent_config, new_config): """Test if new dimension conflict raises an error if marked with invalid default value""" new_config["metadata"]["user_args"][-1] = "-w_d~+normal(0,1,default_value='a')" backward.populate_space(new_config) with pytest.raises(TypeError) as exc: detect_conflicts(parent_config, new_config) assert "Parameter '/w_d': Incorrect arguments." in str(exc.value)
def test_add_bad_default(self, parent_config, new_config): """Test if new dimension conflict raises an error if marked with invalid default value""" new_config['metadata']['user_args'][ -1] = '-w_d~+normal(0,1,default_value=\'a\')' with pytest.raises(TypeError) as exc: detect_conflicts(parent_config, new_config) assert "Parameter \'/w_d\': Incorrect arguments." in str(exc.value)
def test_bad_name_experiment(self, parent_config, child_config, monkeypatch): """Test if changing the experiment names does not work for invalid name and revert to old one """ def _is_unique(self, *args, **kwargs): return False def _versions(self, *args, **kwargs): return True monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution, "_name_is_unique", _is_unique) monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution, "_check_for_greater_versions", _versions) conflicts = detect_conflicts(parent_config, child_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 1 assert len(conflicts.get_resolved()) == 0 conflict = conflicts.get([ExperimentNameConflict])[0] assert not conflict.is_resolved branch_builder.change_experiment_name('test2') assert len(conflicts.get_resolved()) == 0 assert conflict.new_config['name'] == 'test' assert not conflict.is_resolved
def test_rename_missing_changed_marked(self, parent_config, child_config): """Test if renaming is automatically applied with all conflicts resolved including the new one caused by prior change """ child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)") child_config["metadata"]["user_args"].append("-w_b~+normal(0,1)") child_config["metadata"]["user_args"][2] = "-x~>w_b" backward.populate_space(child_config) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, manual_resolution=True) assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name="x")[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name="w_b")[ 0 ].is_resolved assert conflicts.get([ChangedDimensionConflict], dimension_name="w_b")[ 0 ].is_resolved assert not conflicts.get(dimension_name="w_a")[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 4 assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution assert isinstance( resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution, ) assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x" assert ( resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name == "/w_b" )
def test_rename_missing_changed(self, parent_config, child_config): """Test if renaming is automatically applied with both conflicts resolved, but not the new one because of prior change """ child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~normal(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_b' backward.populate_priors(child_config['metadata']) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name='x')[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get(dimension_name='w_a')[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[ 2].resolution assert isinstance(resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution) assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[ 1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_rename_missing_changed(self, parent_config, missing_config): """Test if renaming a dimension to another with different prior solves both conflicts but creates a new one which is not solved """ missing_config['metadata']['user_args'].append('-w_d~normal(0,1)') conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 4 branch_builder.rename_dimension('x', 'w_d') assert len(conflicts.get()) == 5 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 2 assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution assert isinstance(resolved_conflicts[0].resolution, resolved_conflicts[0].RenameDimensionResolution) assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_rename_missing_changed(self, parent_config, missing_config): """Test if renaming a dimension to another with different prior solves both conflicts but creates a new one which is not solved """ missing_config["metadata"]["user_args"].append("-w_d~normal(0,1)") backward.populate_space(missing_config) conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) assert len(conflicts.get()) == 4 branch_builder.rename_dimension("x", "w_d") assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution assert isinstance( resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution, ) assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x" assert ( resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name == "/w_d" )
def test_ignore_code_conflict(self, parent_config, changed_code_config): """Test if ignored code commit hash change is detected as a conflict""" conflicts = detect_conflicts( parent_config, changed_code_config, {"ignore_code_changes": True} ) assert len(conflicts.get()) == 1
def test_rename_missing_changed_marked(self, parent_config, child_config): """Test if renaming is automatically applied with all conflicts resolved including the new one caused by prior change """ child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~+normal(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_b' conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 5 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name='x')[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved assert conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get(dimension_name='w_a')[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution assert isinstance(resolved_conflicts[0].resolution, resolved_conflicts[0].RenameDimensionResolution) assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_rename_missing(self, parent_config, missing_config): """Test if renaming a dimension to another solves both conflicts""" missing_config['metadata']['user_args'].append('-w_d~uniform(0,1)') backward.populate_priors(missing_config['metadata']) conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) branch_builder.rename_dimension('x', 'w_d') assert len(conflicts.get()) == 4 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[ 2].resolution assert isinstance(resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution) assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[ 1].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_config_non_dim_conflict(self, parent_config, changed_userconfig_config): """Test if changed configuration file is detected as a conflict""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) assert len(conflicts.get()) == 4 assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert not conflicts.get([ScriptConfigConflict])[0].is_resolved
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config): """Test if giving an invalid change-type prints error message and do nothing""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) orion.core.config.evc.config_change_type = "bad-type" ExperimentBranchBuilder(conflicts) assert len(conflicts.get()) == 4 assert len(conflicts.get_resolved()) == 3
def test_cli_conflict(self, parent_config, changed_cli_config): """Test if changed command line call is detected as a conflict""" conflicts = detect_conflicts(parent_config, changed_cli_config) assert len(conflicts.get()) == 2 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert not conflicts.get([CommandLineConflict])[0].is_resolved
def test_rename_invalid(self, parent_config, child_config): """Test if renaming to invalid dimension raises an error""" child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~uniform(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_c' conflicts = detect_conflicts(parent_config, child_config) with pytest.raises(ValueError) as exc: ExperimentBranchBuilder(conflicts, {}) assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def test_cli_ignored_conflict(self, parent_config, changed_cli_config): """Test if ignored changed command line call is detected as a conflict""" changed_cli_config['metadata']['user_args'].pop() conflicts = detect_conflicts(parent_config, changed_cli_config, {'non_monitored_arguments': ['u', 'another']}) assert len(conflicts.get()) == 1 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
def test_bad_cli_change(self, capsys, parent_config, changed_cli_config): """Test if giving an invalid change-type fails the the resolution""" conflicts = detect_conflicts(parent_config, changed_cli_config) orion.core.config.evc.cli_change_type = "bad-type" ExperimentBranchBuilder(conflicts) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 orion.core.config.evc.cli_change_type = "break"
def test_changed_dim_userconfig_conflict(self, parent_config, changed_userconfig_config): """Test if changed dimension from user's config is currently detected""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) assert len(conflicts.get()) == 4 conflict = conflicts.get()[0] assert conflict.is_resolved is False assert conflict.dimension.name == '/b' assert isinstance(conflict, ChangedDimensionConflict)
def test_new_dim_conflict(self, parent_config, new_config): """Test if new dimension is currently detected""" conflicts = detect_conflicts(parent_config, new_config) assert len(conflicts.get()) == 2 conflict = conflicts.get()[1] assert conflict.is_resolved is False assert conflict.dimension.name == '/w_d' assert isinstance(conflict, NewDimensionConflict)
def test_rename_invalid(self, parent_config, child_config): """Test if renaming to invalid dimension raises an error""" child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)") child_config["metadata"]["user_args"].append("-w_b~uniform(0,1)") child_config["metadata"]["user_args"][2] = "-x~>w_c" backward.populate_space(child_config) conflicts = detect_conflicts(parent_config, child_config) with pytest.raises(ValueError) as exc: ExperimentBranchBuilder(conflicts, manual_resolution=True) assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def _get_conflicts(experiment, branching): """Get conflicts between current experiment and corresponding configuration in database""" db_experiment = build_view(experiment.name, experiment.version) conflicts = detect_conflicts(db_experiment.configuration, experiment.configuration, branching) # elif must_branch and not enable_branching: # raise ValueError("Configuration is different and generate a branching event") return conflicts
def test_missing_dim_conflict(self, parent_config, missing_config): """Test if missing dimension is currently detected""" conflicts = detect_conflicts(parent_config, missing_config) assert len(conflicts.get()) == 3 conflict = conflicts.get()[1] assert conflict.is_resolved is False assert conflict.dimension.name == '/x' assert isinstance(conflict, MissingDimensionConflict)
def test_bad_name_experiment(self, parent_config, child_config, create_db_instance): """Test if experiment name conflict is not resolved when invalid name is marked""" new_name = 'test' create_db_instance.write('experiments', parent_config) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {'branch': new_name}) assert len(conflicts.get()) == 1 assert len(conflicts.get_resolved()) == 0
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config): """Test if giving an invalid change-type prints error message and do nothing""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) capsys.readouterr() branch_builder.set_script_config_change_type("bad-type") out, err = capsys.readouterr() assert "Invalid script's config change type" in out.split("\n")[-3] assert len(conflicts.get()) == 4 assert len(conflicts.get_resolved()) == 1
def test_orion_version_conflict(self, parent_config, changed_orion_version_config): """Test if orion version changes are currently detected""" conflicts = detect_conflicts(parent_config, changed_orion_version_config) assert len(conflicts.get()) == 2 conflict = conflicts.get()[1] assert conflict.is_resolved is False assert conflict.old_config["metadata"]["orion_version"] == "XYZ" assert conflict.new_config["metadata"]["orion_version"] == "UVW" assert isinstance(conflict, OrionVersionConflict)
def test_bad_cli_change(self, capsys, parent_config, changed_cli_config): """Test if giving an invalid change-type prints error message and do nothing""" conflicts = detect_conflicts(parent_config, changed_cli_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) capsys.readouterr() branch_builder.set_cli_change_type('bad-type') out, err = capsys.readouterr() assert 'Invalid cli change type' in out.split("\n")[-3] assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 0
def test_algo_conflict(self, parent_config, changed_algo_config): """Test if algorithm changes are currently detected""" conflicts = detect_conflicts(parent_config, changed_algo_config) assert len(conflicts.get()) == 2 conflict = conflicts.get()[0] assert conflict.is_resolved is False assert conflict.old_config['algorithms'] == 'random' assert conflict.new_config['algorithms'] == 'stupid-grid' assert isinstance(conflict, AlgorithmConflict)
def test_add_single_hit(self, parent_config, new_config): """Test if adding a dimension only touches the correct status""" del new_config['metadata']['user_args'][1] conflicts = detect_conflicts(parent_config, new_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) branch_builder.add_dimension('w_d') assert len(conflicts.get()) == 3 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_code_conflict(self, parent_config, changed_code_config): """Test if code commit hash change is currently detected""" conflicts = detect_conflicts(parent_config, changed_code_config) assert len(conflicts.get()) == 2 conflict = conflicts.get()[0] assert conflict.is_resolved is False assert conflict.old_config['metadata']['VCS']['HEAD_sha'] == 'test' assert conflict.new_config['metadata']['VCS']['HEAD_sha'] == 'new_test' assert isinstance(conflict, CodeConflict)
def test_config_new_name_no_conflict(self, parent_config, same_userconfig_config): """Test if same configuration file with a different name is not detected as a conflict""" conflicts = detect_conflicts(parent_config, same_userconfig_config) assert parent_config['metadata']['user_args'][-1].startswith('--config=') assert same_userconfig_config['metadata']['user_args'][-1].startswith('--config=') assert (parent_config['metadata']['user_args'][-1] != same_userconfig_config['metadata']['user_args'][-1]) assert len(conflicts.get()) == 2 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved
def test_adapter_rename_missing(self, parent_config, cl_config): """Test if a DimensionRenaming is created when solving a new conflict""" cl_config['metadata']['user_args'] = ['-x~>w_d', '-w_d~+uniform(0,1)'] conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 3 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionRenaming)