def test_rename_missing_changed(self, parent_config, missing_config): """Test if renaming a dimension to another with different prior solves both conflicts but creates a new one which is not solved """ missing_config['metadata']['user_args'].append('-w_d~normal(0,1)') backward.populate_priors(missing_config['metadata']) conflicts = detect_conflicts(parent_config, missing_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 4 branch_builder.rename_dimension('x', 'w_d') assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert conflicts.get([MissingDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[1].is_resolved assert not conflicts.get([ChangedDimensionConflict])[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 3 assert resolved_conflicts[1].resolution is resolved_conflicts[ 2].resolution assert isinstance(resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution) assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[ 1].resolution.new_dimension_conflict.dimension.name == '/w_d'
def changed_cli_config(child_config): """Create a child config with a changed dimension""" child_config['metadata']['user_args'] += [ '-u=0', '--another=test', 'positional' ] backward.populate_priors(child_config['metadata']) return child_config
def test_rename_missing_changed_marked(self, parent_config, child_config): """Test if renaming is automatically applied with all conflicts resolved including the new one caused by prior change """ child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~+normal(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_b' backward.populate_priors(child_config['metadata']) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 5 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get(dimension_name='x')[0].is_resolved assert conflicts.get([NewDimensionConflict], dimension_name='w_b')[0].is_resolved assert conflicts.get([ChangedDimensionConflict], dimension_name='w_b')[0].is_resolved assert not conflicts.get(dimension_name='w_a')[0].is_resolved resolved_conflicts = conflicts.get_resolved() assert len(resolved_conflicts) == 4 assert resolved_conflicts[1].resolution is resolved_conflicts[ 2].resolution assert isinstance(resolved_conflicts[1].resolution, resolved_conflicts[1].RenameDimensionResolution) assert resolved_conflicts[1].resolution.conflict.dimension.name == '/x' assert resolved_conflicts[ 1].resolution.new_dimension_conflict.dimension.name == '/w_b'
def cli_conflict(old_config, new_config): """Generate a commandline conflict""" new_config = copy.deepcopy(new_config) new_config['metadata']['user_args'].append("--some-new=args") new_config['metadata']['user_args'].append("--bool-arg") backward.populate_priors(new_config['metadata']) return conflicts.CommandLineConflict(old_config, new_config)
def changed_config(child_config): """Create a child config with a changed dimension""" second_element = child_config['metadata']['user_args'][2] second_element = second_element.replace('normal', 'uniform') child_config['metadata']['user_args'][2] = second_element backward.populate_priors(child_config['metadata']) return child_config
def test_add_bad_default(self, parent_config, new_config): """Test if new dimension conflict raises an error if marked with invalid default value""" new_config['metadata']['user_args'][ -1] = '-w_d~+normal(0,1,default_value=\'a\')' backward.populate_priors(new_config['metadata']) with pytest.raises(TypeError) as exc: detect_conflicts(parent_config, new_config) assert "Parameter \'/w_d\': Incorrect arguments." in str(exc.value)
def config(exp_config): """Return a configuration.""" config = exp_config[0][0] config['metadata']['user_args'] = ['--x~uniform(-50, 50)'] config['name'] = 'exp' config['working_dir'] = "/tmp/orion" backward.populate_priors(config['metadata']) return config
def list_arg_with_equals_cli_config(child_config): """Create a child config with an argument of the form --args=1 --args=2 --args=3 """ child_config['metadata']['user_args'] += [ '--args=1', '--args=2', '--args=3' ] backward.populate_priors(child_config['metadata']) return child_config
def same_userconfig_config(user_config, child_config): """Create a child config with a changed dimension""" config_file_path = "./same_config.yaml" with open(config_file_path, 'w') as f: yaml.dump(user_config, f) child_config['metadata']['user_args'][ -1] = '--config=%s' % config_file_path backward.populate_priors(child_config['metadata']) yield child_config os.remove(config_file_path)
def test_rename_invalid(self, parent_config, child_config): """Test if renaming to invalid dimension raises an error""" child_config['metadata']['user_args'].append('-w_a~uniform(0,1)') child_config['metadata']['user_args'].append('-w_b~uniform(0,1)') child_config['metadata']['user_args'][1] = '-x~>w_c' backward.populate_priors(child_config['metadata']) conflicts = detect_conflicts(parent_config, child_config) with pytest.raises(ValueError) as exc: ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert "Dimension name 'w_c' not found in conflicts" in str(exc.value)
def exp_config(): """Load an example database.""" with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'experiment.yaml')) as f: exp_config = list(yaml.safe_load_all(f)) for config in exp_config[0]: backward.populate_priors(config['metadata']) return exp_config
def test_comparison(self, yaml_config, yaml_diff_config): """Test that different configs are detected as conflict.""" old_config = {'metadata': {'user_args': yaml_config}} new_config = {'metadata': {'user_args': yaml_diff_config}} backward.populate_priors(old_config['metadata']) backward.populate_priors(new_config['metadata']) conflicts = list( conflict.ScriptConfigConflict.detect(old_config, new_config)) assert len(conflicts) == 1
def test_workon(database): """Test scenario having a configured experiment already setup.""" experiment = Experiment('voila_voici') config = experiment.configuration config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}} config['pool_size'] = 1 config['max_trials'] = 100 config['metadata']['user_script'] = os.path.abspath( os.path.join(os.path.dirname(__file__), "black_box.py")) config['metadata']['user_args'] = ["-x~uniform(-50, 50)"] backward.populate_priors(config['metadata']) experiment.configure(config) workon(experiment) exp = list(database.experiments.find({'name': 'voila_voici'})) assert len(exp) == 1 exp = exp[0] assert '_id' in exp exp_id = exp['_id'] assert exp['name'] == 'voila_voici' assert exp['pool_size'] == 1 assert exp['max_trials'] == 100 assert exp['algorithms'] == { 'gradient_descent': { 'learning_rate': 0.1, 'dx_tolerance': 1e-7 } } assert 'user' in exp['metadata'] assert 'datetime' in exp['metadata'] assert 'user_script' in exp['metadata'] assert os.path.isabs(exp['metadata']['user_script']) assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)'] trials = list(database.trials.find({'experiment': exp_id})) assert len(trials) <= 15 trials = list(sorted(trials, key=lambda trial: trial['submit_time'])) assert trials[-1]['status'] == 'completed' for result in trials[-1]['results']: assert result['type'] != 'constraint' if result['type'] == 'objective': assert abs(result['value'] - 23.4) < 1e-6 assert result['name'] == 'example_objective' elif result['type'] == 'gradient': res = numpy.asarray(result['value']) assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7 assert result['name'] == 'example_gradient' params = trials[-1]['params'] assert len(params) == 1 assert params[0]['name'] == '/x' assert params[0]['type'] == 'real' assert (params[0]['value'] - 34.56789) < 1e-5
def changed_userconfig_config(user_config, child_config): """Create a child config with a changed dimension""" config_file_path = './changed_config.yaml' user_config['b'] = 'orion~uniform(-20, 0)' user_config['some_other'] = 'hello' with open(config_file_path, 'w') as f: yaml.dump(user_config, f) child_config['metadata']['user_args'][ -1] = '--config=%s' % config_file_path backward.populate_priors(child_config['metadata']) yield child_config os.remove(config_file_path)
def test_add_single_hit(self, parent_config, new_config): """Test if adding a dimension only touches the correct status""" del new_config['metadata']['user_args'][1] backward.populate_priors(new_config['metadata']) conflicts = detect_conflicts(parent_config, new_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) branch_builder.add_dimension('w_d') assert len(conflicts.get()) == 3 assert conflicts.get([ExperimentNameConflict])[0].is_resolved assert conflicts.get([NewDimensionConflict])[0].is_resolved assert not conflicts.get([MissingDimensionConflict])[0].is_resolved
def test_adapter_rename_missing(self, parent_config, cl_config): """Test if a DimensionRenaming is created when solving a new conflict""" cl_config['metadata']['user_args'] = ['-x~>w_d', '-w_d~+uniform(0,1)'] backward.populate_priors(cl_config['metadata']) conflicts = detect_conflicts(parent_config, cl_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) adapters = branch_builder.create_adapters().adapters assert len(conflicts.get_resolved()) == 3 assert len(adapters) == 1 assert isinstance(adapters[0], evc.adapters.DimensionRenaming)
def bad_exp_parent_config(): """Generate a new experiment configuration""" config = dict(_id='test', name='test', metadata={ 'user': '******', 'user_args': ['--x~normal(0,1)'] }, version=1, algorithms='random') backward.populate_priors(config['metadata']) return config
def test_remove_missing_bad_default(self, parent_config, child_config): """Test if missing dimension conflict raises an error if marked with invalid default""" child_config['metadata']['user_args'][1] = '-x~--100' backward.populate_priors(child_config['metadata']) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 1 conflict = conflicts.get()[1] assert not conflict.is_resolved assert isinstance(conflict, MissingDimensionConflict)
def test_remove_missing(self, parent_config, child_config): """Test if missing dimension conflict is automatically resolved""" child_config['metadata']['user_args'][1] = '-x~-' backward.populate_priors(child_config['metadata']) conflicts = detect_conflicts(parent_config, child_config) ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 2 conflict = conflicts.get_resolved()[1] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.RemoveDimensionResolution)
def test_comparison_idem(self, yaml_config): """Test that identical configs are not detected as conflict.""" old_config = {'metadata': {'user_args': yaml_config}} new_config = { 'metadata': { 'user_args': yaml_config + ['--other', 'args'] } } backward.populate_priors(old_config['metadata']) backward.populate_priors(new_config['metadata']) assert list( conflict.ScriptConfigConflict.detect(old_config, new_config)) == []
def new_config(): """Generate a new experiment configuration""" config = dict(name='test', algorithms='fancy', version=1, metadata={ 'VCS': 'to be changed', 'user_script': 'abs_path/black_box.py', 'user_args': ['--new~normal(0,2)', '--changed~normal(0,2)'], 'user': '******' }) backward.populate_priors(config['metadata']) return config
def test_add_new_default(self, parent_config, new_config): """Test if new dimension conflict is automatically resolved""" new_config['metadata']['user_args'][ -1] = '-w_d~+normal(0,1,default_value=0)' backward.populate_priors(new_config['metadata']) conflicts = detect_conflicts(parent_config, new_config) ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 2 assert len(conflicts.get_resolved()) == 2 conflict = conflicts.get_resolved()[1] assert conflict.is_resolved assert isinstance(conflict.resolution, conflict.AddDimensionResolution) assert conflict.resolution.default_value == 0.0
def exp_config(): """Load an example database.""" with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), 'unittests', 'core', 'experiment.yaml')) as f: exp_config = list(yaml.safe_load_all(f)) for i, t_dict in enumerate(exp_config[1]): exp_config[1][i] = Trial(**t_dict).to_dict() for config in exp_config[0]: config["metadata"]["user_script"] = os.path.join( os.path.dirname(__file__), config["metadata"]["user_script"]) backward.populate_priors(config['metadata']) config['version'] = 1 return exp_config
def cl_config(create_db_instance): """Create a child config with markers for commandline solving""" config = dict(name='test', branch='test2', algorithms='random', metadata={ 'hash_commit': 'old', 'user_script': 'abs_path/black_box.py', 'user_args': [ '--nameless=option', '-x~>w_d', '-w_d~+normal(0,1)', '-y~+uniform(0,1)', '-z~-', '--omega~+normal(0,1)' ], 'user': '******' }) backward.populate_priors(config['metadata']) return config
def test_name_experiment(self, bad_exp_parent_config, bad_exp_child_config, create_db_instance): """Test if having the same experiment name does not create a conflict.""" backward.populate_priors(bad_exp_parent_config['metadata']) backward.populate_priors(bad_exp_child_config['metadata']) create_db_instance.write('experiments', bad_exp_parent_config) create_db_instance.write('experiments', bad_exp_child_config) conflicts = detect_conflicts(bad_exp_parent_config, bad_exp_parent_config) branch_builder = ExperimentBranchBuilder(conflicts, {'manual_resolution': True}) assert len(conflicts.get()) == 1 assert len(conflicts.get_resolved()) == 0 conflict = conflicts.get([ExperimentNameConflict])[0] assert conflict.new_config['name'] == 'test' assert not conflict.is_resolved branch_builder.change_experiment_name('test2') assert len(conflicts.get_resolved()) == 1 assert conflict.new_config['name'] == 'test2' assert conflict.is_resolved
def parent_config(user_config): """Create a configuration that will not hit the database.""" config = dict(_id='test', name='test', algorithms='random', version=1, metadata={ 'VCS': { "type": "git", "is_dirty": False, "HEAD_sha": "test", "active_branch": None, "diff_sha": "diff", }, 'user_script': 'abs_path/black_box.py', 'user_args': [ '--nameless=option', '-x~uniform(0,1)', '-y~normal(0,1)', '-z~uniform(0,10)', '--manual-resolution' ], 'user': '******' }, refers={}) config_file_path = "./parent_config.yaml" with open(config_file_path, 'w') as f: yaml.dump(user_config, f) config['metadata']['user_args'].append('--config=%s' % config_file_path) backward.populate_priors(config['metadata']) yield config os.remove(config_file_path)
def old_config(create_db_instance): """Generate an old experiment configuration""" config = dict( name='test', algorithms='random', version=1, metadata={ 'VCS': { "type": "git", "is_dirty": False, "HEAD_sha": "test", "active_branch": None, "diff_sha": "diff", }, 'user_script': 'abs_path/black_box.py', 'user_args': ['--missing~uniform(-10,10)', '--changed~uniform(-10,10)'], 'user': '******' }) backward.populate_priors(config['metadata']) create_db_instance.write('experiments', config) return config
def new_config(child_config): """Create a child config with a new dimension""" child_config['metadata']['user_args'].append('-w_d~normal(0,1)') backward.populate_priors(child_config['metadata']) return child_config
def add_priors(experiment): """Add priors to metadata if not present""" backward.populate_priors(experiment['metadata'])
def missing_config(child_config): """Create a child config with a missing dimension""" del (child_config['metadata']['user_args'][1]) # del -x del (child_config['metadata']['user_args'][1]) # del -y backward.populate_priors(child_config['metadata']) return child_config