def build(name, version=None, branching=None, **config):
"""Build an experiment object
If new, `space` argument must be provided, else all arguments are fetched from the database
based on (name, version). If any argument given does not match the corresponding ones in the
database for given (name, version), than the version is incremented and the experiment will be a
child of the previous version.
Parameters
----------
name: str
Name of the experiment to build
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
branch_from: str, optional
Name of the experiment to branch from. The new experiment will have access to all trials
from the parent experiment it has been branched from.
space: dict, optional
Optimization space of the algorithm. Should have the form `dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Parallel strategy to use to parallelize the algorithm.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
storage: dict, optional
Configuration of the storage backend.
branching: dict, optional
Arguments to control the branching.
branch_from: str, optional
Name of the experiment to branch from.
manual_resolution: bool, optional
Starts the prompt to resolve manually the conflicts. Defaults to False.
non_monitored_arguments: list of str, optional
Will ignore these arguments while looking for differences. Defaults to [].
ignore_code_changes: bool, optional
Will ignore code changes while looking for differences. Defaults to False.
algorithm_change: bool, optional
Whether to automatically solve the algorithm conflict (change of algo config).
Defaults to True.
code_change_type: str, optional
How to resolve code change automatically. Must be one of 'noeffect', 'unsure' or
'break'. Defaults to 'break'.
cli_change_type: str, optional
How to resolve cli change automatically. Must be one of 'noeffect', 'unsure' or 'break'.
Defaults to 'break'.
config_change_type: str, optional
How to resolve config change automatically. Must be one of 'noeffect', 'unsure' or
'break'. Defaults to 'break'.
"""
config = copy.deepcopy(config)
for key, value in list(config.items()):
if key.startswith('_') or value is None:
config.pop(key)
if 'strategy' in config:
config['producer'] = {'strategy': config.pop('strategy')}
if branching is None:
branching = {}
if branching.get('branch_from'):
branching.setdefault('branch_to', name)
name = branching['branch_from']
db_config = fetch_config_from_db(name, version)
new_config = config
config = resolve_config.merge_configs(db_config, config)
metadata = resolve_config.fetch_metadata(config.get('user'),
config.get('user_args'))
config = resolve_config.merge_configs(db_config, config,
{'metadata': metadata})
# TODO: Find a better solution
if isinstance(config.get('algorithms'),
dict) and len(config['algorithms']) > 1:
config['algorithms'] = new_config['algorithms']
config.setdefault('name', name)
config.setdefault('version', version)
if 'space' not in config:
raise NoConfigurationError(
'Experiment {} does not exist in DB and space was not defined.'.
format(name))
if len(config['space']) == 0:
raise NoConfigurationError(
"No prior found. Please include at least one.")
experiment = create_experiment(**copy.deepcopy(config))
if experiment.id is None:
try:
_register_experiment(experiment)
except DuplicateKeyError:
experiment = build(branching=branching, **config)
return experiment
conflicts = _get_conflicts(experiment, branching)
must_branch = len(conflicts.get()) > 1 or branching.get('branch_to')
if must_branch:
branched_experiment = _branch_experiment(experiment, conflicts,
version, branching)
try:
_register_experiment(branched_experiment)
except DuplicateKeyError as e:
raise RaceCondition(
'There was a race condition during branching.') from e
return branched_experiment
_update_experiment(experiment)
return experiment
def test_fetch_metadata_non_executable_users_script():
"""Verify executable user script keeps given path"""
script_path = 'tests/functional/demo/orion_config.yaml'
metadata = resolve_config.fetch_metadata(user_args=[script_path])
assert metadata['user_script'] == script_path
def test_fetch_metadata_python_users_script(script_path):
"""Verify user script is correctly infered if called with python"""
metadata = resolve_config.fetch_metadata(
user_args=['python', script_path, 'some', 'args'])
assert metadata['user_script'] == script_path
def test_fetch_metadata_orion_version():
"""Verify orion version"""
metadata = resolve_config.fetch_metadata()
assert metadata['orion_version'] == 'XYZ'
def test_fetch_metadata_executable_users_script(script_path):
"""Verify executable user script with absolute path"""
metadata = resolve_config.fetch_metadata(user_args=[script_path])
assert metadata['user_script'] == os.path.abspath(script_path)
def test_fetch_metadata_user_tsirif():
"""Verify user name"""
metadata = resolve_config.fetch_metadata()
assert metadata['user'] == "tsirif"
def test_fetch_metadata():
"""Verify no additional data is stored in metadata"""
metadata = resolve_config.fetch_metadata()
len(metadata) == 4
def test_fetch_metadata_user_args(script_path):
"""Verify user args"""
user_args = [os.path.abspath(script_path)] + list(map(str, range(10)))
metadata = resolve_config.fetch_metadata(user_args=user_args)
assert metadata['user_script'] == user_args[0]
assert metadata['user_args'] == user_args
def fetch_metadata(self, cmdargs):
"""Infer rest information about the process + versioning"""
return resolve_config.fetch_metadata(cmdargs)
def test_fetch_metadata_python_users_script(script_path):
"""Verify user script is correctly infered if called with python"""
metadata = resolve_config.fetch_metadata(
user_args=["python", script_path, "some", "args"])
assert metadata["user_script"] == script_path
def test_fetch_metadata_not_existed_path():
"""Verfiy the raise of error when user_script path does not exist"""
path = "dummy/path"
with pytest.raises(OSError) as exc_info:
resolve_config.fetch_metadata(user_args=[path])
assert "The path specified for the script does not exist" in str(exc_info.value)
def test_fetch_metadata_non_executable_users_script():
"""Verify executable user script keeps given path"""
cmdargs = {'user_args': ['tests/functional/demo/orion_config.yaml']}
metadata = resolve_config.fetch_metadata(cmdargs)
assert metadata['user_script'] == 'tests/functional/demo/orion_config.yaml'
