def test_bad_name_experiment(self, parent_config, child_config, monkeypatch): """Test if changing the experiment names does not work for invalid name and revert to old one """ def _is_unique(self, *args, **kwargs): return False def _versions(self, *args, **kwargs): return True monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution, "_name_is_unique", _is_unique) monkeypatch.setattr(ExperimentNameConflict.ExperimentNameResolution, "_check_for_greater_versions", _versions) conflicts = detect_conflicts(parent_config, child_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 1 assert len(conflicts.get_resolved()) == 0 conflict = conflicts.get([ExperimentNameConflict])[0] assert not conflict.is_resolved branch_builder.change_experiment_name('test2') assert len(conflicts.get_resolved()) == 0 assert conflict.new_config['name'] == 'test' assert not conflict.is_resolved
def _branch_config(self, conflicts, branching_configuration): """Ask for a different identifier for this experiment. Set :attr:`refers` key to previous experiment's name, the one that we branched from. :param config: Conflicting configuration that will change based on prompt. """ experiment_brancher = ExperimentBranchBuilder(conflicts, branching_configuration) needs_manual_resolution = (not experiment_brancher.is_resolved or experiment_brancher.auto_resolution) if needs_manual_resolution: branching_prompt = BranchingPrompt(experiment_brancher) if not sys.__stdin__.isatty(): raise ValueError( "Configuration is different and generates a branching event:\n{}" .format(branching_prompt.get_status())) branching_prompt.cmdloop() if branching_prompt.abort or not experiment_brancher.is_resolved: sys.exit() adapter = experiment_brancher.create_adapters() self._instantiate_config(experiment_brancher.conflicting_config) self.refers['adapter'] = adapter self.refers['parent_id'] = self._id
def test_rename_missing(self, parent_config, missing_config): """Test if renaming a dimension to another solves both conflicts""" missing_config['metadata']['user_args'].append('-w_d~uniform(0,1)') backward.populate_priors(missing_config['metadata']) conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) branch_builder.rename_dimension('x', 'w_d') assert len(conflicts.get()) == 4 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[ 2].resolution assert isinstance(resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution) assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[ 1].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_rename_missing_changed(self, parent_config, missing_config): """Test if renaming a dimension to another with different prior solves both conflicts but creates a new one which is not solved """ missing_config['metadata']['user_args'].append('-w_d~normal(0,1)') conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 4 branch_builder.rename_dimension('x', 'w_d') assert len(conflicts.get()) == 5 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 2 assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution assert isinstance(resolved_conflicts[0].resolution, resolved_conflicts[0].RenameDimensionResolution) assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_d'
def test_rename_missing_changed(self, parent_config, missing_config): """Test if renaming a dimension to another with different prior solves both conflicts but creates a new one which is not solved """ missing_config["metadata"]["user_args"].append("-w_d~normal(0,1)") backward.populate_space(missing_config) conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) assert len(conflicts.get()) == 4 branch_builder.rename_dimension("x", "w_d") assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution assert isinstance( resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution, ) assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x" assert ( resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name == "/w_d" )
def _branch_experiment(experiment, conflicts, version, branching_arguments): """Create a new branch experiment with adapters for the given conflicts""" experiment_brancher = ExperimentBranchBuilder(conflicts, **branching_arguments) needs_manual_resolution = (not experiment_brancher.is_resolved or experiment_brancher.manual_resolution) if not experiment_brancher.is_resolved: name_conflict = conflicts.get([ExperimentNameConflict])[0] if not name_conflict.is_resolved and not version: raise RaceCondition( 'There was likely a race condition during version increment.') if needs_manual_resolution: # TODO: This should only be possible when using cmdline API branching_prompt = BranchingPrompt(experiment_brancher) if not sys.__stdin__.isatty(): raise BranchingEvent(branching_prompt.get_status()) branching_prompt.cmdloop() if branching_prompt.abort or not experiment_brancher.is_resolved: sys.exit() config = experiment_brancher.conflicting_config config['refers']['adapter'] = experiment_brancher.create_adapters( ).configuration config['refers']['parent_id'] = experiment.id config.pop('_id') return create_experiment(**config)
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config): """Test if giving an invalid change-type prints error message and do nothing""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) capsys.readouterr() branch_builder.set_script_config_change_type("bad-type") out, err = capsys.readouterr() assert "Invalid script's config change type" in out.split("\n")[-3] assert len(conflicts.get()) == 4 assert len(conflicts.get_resolved()) == 1
def test_add_single_hit(self, parent_config, new_config): """Test if adding a dimension only touches the correct status""" del new_config['metadata']['user_args'][1] conflicts = detect_conflicts(parent_config, new_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) branch_builder.add_dimension('w_d') assert len(conflicts.get()) == 3 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_bad_cli_change(self, capsys, parent_config, changed_cli_config): """Test if giving an invalid change-type prints error message and do nothing""" conflicts = detect_conflicts(parent_config, changed_cli_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) capsys.readouterr() branch_builder.set_cli_change_type('bad-type') out, err = capsys.readouterr() assert 'Invalid cli change type' in out.split("\n")[-3] assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 0
def test_add_single_hit(self, parent_config, new_config): """Test if adding a dimension only touches the correct status""" del new_config["metadata"]["user_args"][2] backward.populate_space(new_config) conflicts = detect_conflicts(parent_config, new_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) branch_builder.add_dimension("w_d") assert len(conflicts.get()) == 3 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_adapter_rename_missing(self, parent_config, cl_config): """Test if a DimensionRenaming is created when solving a new conflict""" cl_config['metadata']['user_args'] = ['-x~>w_d', '-w_d~+uniform(0,1)'] conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 3 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
def test_adapter_add_new(self, parent_config, cl_config): """Test if a DimensionAddition is created when solving a new conflict""" cl_config["metadata"]["user_args"] = ["-w_d~+normal(0,1)"] conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 2 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionAddition)
def test_adapter_add_changed(self, parent_config, cl_config): """Test if a DimensionPriorChange is created when solving a new conflict""" cl_config['metadata']['user_args'] = ['-y~+uniform(0,1)'] conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 2 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionPriorChange)
def test_adapter_remove_missing(self, parent_config, cl_config): """Test if a DimensionDeletion is created when solving a new conflict""" cl_config['metadata']['user_args'] = ['-z~-'] conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 2 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionDeletion)
def test_remove_missing(self, parent_config, missing_config): """Test if removing a missing dimension solves the conflict""" conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) branch_builder.remove_dimension('x') assert len(conflicts.get()) == 3 assert len(conflicts.get_resolved()) == 1 conflict = conflicts.get_resolved()[0] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.RemoveDimensionResolution)
def test_add_changed(self, parent_config, changed_config): """Test if adding a changed dimension solves the conflict""" conflicts = detect_conflicts(parent_config, changed_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) branch_builder.add_dimension('y') assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 conflict = conflicts.get_resolved()[0] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.ChangeDimensionResolution)
def test_add_new(self, parent_config, new_config): """Test if adding a new dimension solves the conflict""" conflicts = detect_conflicts(parent_config, new_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) branch_builder.add_dimension("w_d") assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 2 conflict = conflicts.get_resolved()[1] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.AddDimensionResolution)
def test_adapter_rename_missing(self, parent_config, cl_config): """Test if a DimensionRenaming is created when solving a new conflict""" cl_config["metadata"]["user_args"] = ["-x~>w_d", "-w_d~+uniform(0,1)"] backward.populate_space(cl_config) conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 3 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
def test_adapter_rename_different_prior(self, parent_config, cl_config): """Test if a DimensionRenaming is created when solving a new conflict""" cl_config["metadata"]["user_args"] = ["-x~>w_d", "-w_d~+normal(0,1)"] conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 4 assert len(adapters) == 2 assert isinstance(adapters[0], evc.adapters.DimensionRenaming) assert isinstance(adapters[1], evc.adapters.DimensionPriorChange)
def test_cli_change(self, parent_config, changed_cli_config): """Test if giving a proper change-type solves the command line conflict""" conflicts = detect_conflicts(parent_config, changed_cli_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 0 branch_builder.set_cli_change_type(evc.adapters.CommandLineChange.types[0]) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 conflict = conflicts.get_resolved()[0] assert conflict.is_resolved assert isinstance(conflict, CommandLineConflict)
def test_config_change(self, parent_config, changed_userconfig_config): """Test if giving a proper change-type solves the user script config conflict""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 4 assert len(conflicts.get_resolved()) == 0 branch_builder.set_script_config_change_type(evc.adapters.ScriptConfigChange.types[0]) assert len(conflicts.get()) == 4 assert len(conflicts.get_resolved()) == 1 conflict = conflicts.get_resolved()[0] assert conflict.is_resolved assert isinstance(conflict, ScriptConfigConflict)
def test_code_change(self, parent_config, changed_code_config): """Test if giving a proper change-type solves the code conflict""" conflicts = detect_conflicts(parent_config, changed_code_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 branch_builder.set_code_change_type(evc.adapters.CodeChange.types[0]) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 2 conflict = conflicts.get_resolved()[0] assert conflict.is_resolved assert isinstance(conflict, CodeConflict)
def test_name_experiment(self, parent_config, child_config, create_db_instance): """Test if changing the experiment names work for valid name""" conflicts = detect_conflicts(parent_config, child_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 1 assert len(conflicts.get_resolved()) == 0 conflict = conflicts.get([ExperimentNameConflict])[0] assert conflict.new_config['name'] == 'test' assert not conflict.is_resolved branch_builder.change_experiment_name('test2') assert len(conflicts.get_resolved()) == 1 assert conflict.new_config['name'] == 'test2' assert conflict.is_resolved
def test_rename_missing_changed(self, parent_config, child_config): """Test if renaming is automatically applied with both conflicts resolved, but not the new one because of prior change """ child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~normal(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_b' backward.populate_priors(child_config['metadata']) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name='x')[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get(dimension_name='w_a')[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[ 2].resolution assert isinstance(resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution) assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[ 1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_rename_missing_changed_marked(self, parent_config, child_config): """Test if renaming is automatically applied with all conflicts resolved including the new one caused by prior change """ child_config["metadata"]["user_args"].append("-w_a~uniform(0,1)") child_config["metadata"]["user_args"].append("-w_b~+normal(0,1)") child_config["metadata"]["user_args"][2] = "-x~>w_b" backward.populate_space(child_config) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, manual_resolution=True) assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name="x")[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name="w_b")[ 0 ].is_resolved assert conflicts.get([ChangedDimensionConflict], dimension_name="w_b")[ 0 ].is_resolved assert not conflicts.get(dimension_name="w_a")[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 4 assert resolved_conflicts[1].resolution is resolved_conflicts[2].resolution assert isinstance( resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution, ) assert resolved_conflicts[1].resolution.conflict.dimension.name == "/x" assert ( resolved_conflicts[1].resolution.new_dimension_conflict.dimension.name == "/w_b" )
def test_rename_missing_changed_marked(self, parent_config, child_config): """Test if renaming is automatically applied with all conflicts resolved including the new one caused by prior change """ child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~+normal(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_b' conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 5 assert not conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name='x')[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved assert conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get(dimension_name='w_a')[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[0].resolution is resolved_conflicts[1].resolution assert isinstance(resolved_conflicts[0].resolution, resolved_conflicts[0].RenameDimensionResolution) assert resolved_conflicts[0].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[0].resolution.new_dimension_conflict.dimension.name == '/w_b'
def test_algo_change(self, parent_config, changed_algo_config): """Test if setting the algorithm conflict solves it""" conflicts = detect_conflicts(parent_config, changed_algo_config) branch_builder = ExperimentBranchBuilder(conflicts, {}) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 0 branch_builder.set_algo() assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 conflict = conflicts.get_resolved()[0] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.AlgorithmResolution)
def test_orion_version_change(self, parent_config, changed_orion_version_config): """Test if setting the orion version conflict solves it""" conflicts = detect_conflicts(parent_config, changed_orion_version_config) branch_builder = ExperimentBranchBuilder(conflicts, manual_resolution=True) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 branch_builder.set_orion_version() assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 2 conflict = conflicts.get_resolved()[1] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.OrionVersionResolution)
def test_bad_config_change(self, capsys, parent_config, changed_userconfig_config): """Test if giving an invalid change-type prints error message and do nothing""" conflicts = detect_conflicts(parent_config, changed_userconfig_config) orion.core.config.evc.config_change_type = "bad-type" ExperimentBranchBuilder(conflicts) assert len(conflicts.get()) == 4 assert len(conflicts.get_resolved()) == 3
def test_rename_invalid(self, parent_config, child_config): """Test if renaming to invalid dimension raises an error""" child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~uniform(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_c' conflicts = detect_conflicts(parent_config, child_config) with pytest.raises(ValueError) as exc: ExperimentBranchBuilder(conflicts, {}) assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)