def build(name, version=None, branching=None, **config): """Build an experiment object If new, `space` argument must be provided, else all arguments are fetched from the database based on (name, version). If any argument given does not match the corresponding ones in the database for given (name, version), than the version is incremented and the experiment will be a child of the previous version. Parameters ---------- name: str Name of the experiment to build version: int, optional Version to select. If None, last version will be selected. If version given is larger than largest version available, the largest version will be selected. branch_from: str, optional Name of the experiment to branch from. The new experiment will have access to all trials from the parent experiment it has been branched from. space: dict, optional Optimization space of the algorithm. Should have the form `dict(name='<prior>(args)')`. algorithms: str or dict, optional Algorithm used for optimization. strategy: str or dict, optional Parallel strategy to use to parallelize the algorithm. max_trials: int, optional Maximum number or trials before the experiment is considered done. storage: dict, optional Configuration of the storage backend. branching: dict, optional Arguments to control the branching. branch_from: str, optional Name of the experiment to branch from. manual_resolution: bool, optional Starts the prompt to resolve manually the conflicts. Defaults to False. non_monitored_arguments: list of str, optional Will ignore these arguments while looking for differences. Defaults to []. ignore_code_changes: bool, optional Will ignore code changes while looking for differences. Defaults to False. algorithm_change: bool, optional Whether to automatically solve the algorithm conflict (change of algo config). Defaults to True. code_change_type: str, optional How to resolve code change automatically. Must be one of 'noeffect', 'unsure' or 'break'. Defaults to 'break'. cli_change_type: str, optional How to resolve cli change automatically. Must be one of 'noeffect', 'unsure' or 'break'. Defaults to 'break'. config_change_type: str, optional How to resolve config change automatically. Must be one of 'noeffect', 'unsure' or 'break'. Defaults to 'break'. """ config = copy.deepcopy(config) for key, value in list(config.items()): if key.startswith('_') or value is None: config.pop(key) if 'strategy' in config: config['producer'] = {'strategy': config.pop('strategy')} if branching is None: branching = {} if branching.get('branch_from'): branching.setdefault('branch_to', name) name = branching['branch_from'] db_config = fetch_config_from_db(name, version) new_config = config config = resolve_config.merge_configs(db_config, config) metadata = resolve_config.fetch_metadata(config.get('user'), config.get('user_args')) config = resolve_config.merge_configs(db_config, config, {'metadata': metadata}) # TODO: Find a better solution if isinstance(config.get('algorithms'), dict) and len(config['algorithms']) > 1: config['algorithms'] = new_config['algorithms'] config.setdefault('name', name) config.setdefault('version', version) if 'space' not in config: raise NoConfigurationError( 'Experiment {} does not exist in DB and space was not defined.'. format(name)) if len(config['space']) == 0: raise NoConfigurationError( "No prior found. Please include at least one.") experiment = create_experiment(**copy.deepcopy(config)) if experiment.id is None: try: _register_experiment(experiment) except DuplicateKeyError: experiment = build(branching=branching, **config) return experiment conflicts = _get_conflicts(experiment, branching) must_branch = len(conflicts.get()) > 1 or branching.get('branch_to') if must_branch: branched_experiment = _branch_experiment(experiment, conflicts, version, branching) try: _register_experiment(branched_experiment) except DuplicateKeyError as e: raise RaceCondition( 'There was a race condition during branching.') from e return branched_experiment _update_experiment(experiment) return experiment
def test_fetch_metadata_non_executable_users_script(): """Verify executable user script keeps given path""" script_path = 'tests/functional/demo/orion_config.yaml' metadata = resolve_config.fetch_metadata(user_args=[script_path]) assert metadata['user_script'] == script_path
def test_fetch_metadata_python_users_script(script_path): """Verify user script is correctly infered if called with python""" metadata = resolve_config.fetch_metadata( user_args=['python', script_path, 'some', 'args']) assert metadata['user_script'] == script_path
def test_fetch_metadata_orion_version(): """Verify orion version""" metadata = resolve_config.fetch_metadata() assert metadata['orion_version'] == 'XYZ'
def test_fetch_metadata_executable_users_script(script_path): """Verify executable user script with absolute path""" metadata = resolve_config.fetch_metadata(user_args=[script_path]) assert metadata['user_script'] == os.path.abspath(script_path)
def test_fetch_metadata_user_tsirif(): """Verify user name""" metadata = resolve_config.fetch_metadata() assert metadata['user'] == "tsirif"
def test_fetch_metadata(): """Verify no additional data is stored in metadata""" metadata = resolve_config.fetch_metadata() len(metadata) == 4
def test_fetch_metadata_user_args(script_path): """Verify user args""" user_args = [os.path.abspath(script_path)] + list(map(str, range(10))) metadata = resolve_config.fetch_metadata(user_args=user_args) assert metadata['user_script'] == user_args[0] assert metadata['user_args'] == user_args
def fetch_metadata(self, cmdargs): """Infer rest information about the process + versioning""" return resolve_config.fetch_metadata(cmdargs)
def test_fetch_metadata_python_users_script(script_path): """Verify user script is correctly infered if called with python""" metadata = resolve_config.fetch_metadata( user_args=["python", script_path, "some", "args"]) assert metadata["user_script"] == script_path
def test_fetch_metadata_not_existed_path(): """Verfiy the raise of error when user_script path does not exist""" path = "dummy/path" with pytest.raises(OSError) as exc_info: resolve_config.fetch_metadata(user_args=[path]) assert "The path specified for the script does not exist" in str(exc_info.value)
def test_fetch_metadata_non_executable_users_script(): """Verify executable user script keeps given path""" cmdargs = {'user_args': ['tests/functional/demo/orion_config.yaml']} metadata = resolve_config.fetch_metadata(cmdargs) assert metadata['user_script'] == 'tests/functional/demo/orion_config.yaml'