def load(name, version=None, mode="r"):
"""Load experiment from database
An experiment view provides all reading operations of standard experiment but prevents the
modification of the experiment and its trials.
Parameters
----------
name: str
Name of the experiment to build
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
mode: str, optional
The access rights of the experiment on the database.
'r': read access only
'w': can read and write to database
Default is 'r'
"""
assert mode in set("rw")
log.debug(
f"Loading experiment {name} (version={version}) from database in mode `{mode}`"
)
db_config = fetch_config_from_db(name, version)
if not db_config:
message = (
"No experiment with given name '%s' and version '%s' inside database, "
"no view can be created." % (name, version if version else "*")
)
raise NoConfigurationError(message)
db_config.setdefault("version", 1)
return create_experiment(mode=mode, **db_config)
def build(name, version=None, branching=None, **config):
"""Build an experiment object
If new, `space` argument must be provided, else all arguments are fetched from the database
based on (name, version). If any argument given does not match the corresponding ones in the
database for given (name, version), than the version is incremented and the experiment will be a
child of the previous version.
Parameters
----------
name: str
Name of the experiment to build
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
branch_from: str, optional
Name of the experiment to branch from. The new experiment will have access to all trials
from the parent experiment it has been branched from.
space: dict, optional
Optimization space of the algorithm. Should have the form `dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Parallel strategy to use to parallelize the algorithm.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
storage: dict, optional
Configuration of the storage backend.
branching: dict, optional
Arguments to control the branching.
branch_from: str, optional
Name of the experiment to branch from.
manual_resolution: bool, optional
Starts the prompt to resolve manually the conflicts. Defaults to False.
non_monitored_arguments: list of str, optional
Will ignore these arguments while looking for differences. Defaults to [].
ignore_code_changes: bool, optional
Will ignore code changes while looking for differences. Defaults to False.
algorithm_change: bool, optional
Whether to automatically solve the algorithm conflict (change of algo config).
Defaults to True.
code_change_type: str, optional
How to resolve code change automatically. Must be one of 'noeffect', 'unsure' or
'break'. Defaults to 'break'.
cli_change_type: str, optional
How to resolve cli change automatically. Must be one of 'noeffect', 'unsure' or 'break'.
Defaults to 'break'.
config_change_type: str, optional
How to resolve config change automatically. Must be one of 'noeffect', 'unsure' or
'break'. Defaults to 'break'.
"""
config = copy.deepcopy(config)
for key, value in list(config.items()):
if key.startswith('_') or value is None:
config.pop(key)
if 'strategy' in config:
config['producer'] = {'strategy': config.pop('strategy')}
if branching is None:
branching = {}
if branching.get('branch_from'):
branching.setdefault('branch_to', name)
name = branching['branch_from']
db_config = fetch_config_from_db(name, version)
new_config = config
config = resolve_config.merge_configs(db_config, config)
metadata = resolve_config.fetch_metadata(config.get('user'),
config.get('user_args'))
config = resolve_config.merge_configs(db_config, config,
{'metadata': metadata})
# TODO: Find a better solution
if isinstance(config.get('algorithms'),
dict) and len(config['algorithms']) > 1:
config['algorithms'] = new_config['algorithms']
config.setdefault('name', name)
config.setdefault('version', version)
if 'space' not in config:
raise NoConfigurationError(
'Experiment {} does not exist in DB and space was not defined.'.
format(name))
if len(config['space']) == 0:
raise NoConfigurationError(
"No prior found. Please include at least one.")
experiment = create_experiment(**copy.deepcopy(config))
if experiment.id is None:
try:
_register_experiment(experiment)
except DuplicateKeyError:
experiment = build(branching=branching, **config)
return experiment
conflicts = _get_conflicts(experiment, branching)
must_branch = len(conflicts.get()) > 1 or branching.get('branch_to')
if must_branch:
branched_experiment = _branch_experiment(experiment, conflicts,
version, branching)
try:
_register_experiment(branched_experiment)
except DuplicateKeyError as e:
raise RaceCondition(
'There was a race condition during branching.') from e
return branched_experiment
_update_experiment(experiment)
return experiment
def build(name, version=None, branching=None, **config):
"""Build an experiment object
If new, ``space`` argument must be provided, else all arguments are fetched from the database
based on (name, version). If any argument given does not match the corresponding ones in the
database for given (name, version), than the version is incremented and the experiment will be a
child of the previous version.
Parameters
----------
name: str
Name of the experiment to build
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
space: dict, optional
Optimization space of the algorithm. Should have the form ``dict(name='<prior>(args)')``.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Deprecated and will be remove in v0.4. It should now be set in algorithm configuration
directly if it supports it.
max_trials: int, optional
Maximum number of trials before the experiment is considered done.
max_broken: int, optional
Number of broken trials for the experiment to be considered broken.
storage: dict, optional
Configuration of the storage backend.
branching: dict, optional
Arguments to control the branching.
branch_from: str, optional
Name of the experiment to branch from.
manual_resolution: bool, optional
Starts the prompt to resolve manually the conflicts. Defaults to False.
non_monitored_arguments: list of str, optional
Will ignore these arguments while looking for differences. Defaults to [].
ignore_code_changes: bool, optional
Will ignore code changes while looking for differences. Defaults to False.
algorithm_change: bool, optional
Whether to automatically solve the algorithm conflict (change of algo config).
Defaults to True.
orion_version_change: bool, optional
Whether to automatically solve the orion version conflict.
Defaults to True.
code_change_type: str, optional
How to resolve code change automatically. Must be one of 'noeffect', 'unsure' or
'break'. Defaults to 'break'.
cli_change_type: str, optional
How to resolve cli change automatically. Must be one of 'noeffect', 'unsure' or 'break'.
Defaults to 'break'.
config_change_type: str, optional
How to resolve config change automatically. Must be one of 'noeffect', 'unsure' or
'break'. Defaults to 'break'.
"""
log.debug(f"Building experiment {name} with {version}")
log.debug(" Passed experiment config:\n%s", pprint.pformat(config))
log.debug(" Branching config:\n%s", pprint.pformat(branching))
name, config, branching = clean_config(name, config, branching)
config = consolidate_config(name, version, config)
if "space" not in config:
raise NoConfigurationError(
"Experiment {} does not exist in DB and space was not defined.".
format(name))
if len(config["space"]) == 0:
raise NoConfigurationError(
"No prior found. Please include at least one.")
experiment = create_experiment(mode="x", **copy.deepcopy(config))
if experiment.id is None:
log.debug(
"Experiment not found in DB. Now attempting registration in DB.")
try:
_register_experiment(experiment)
log.debug("Experiment successfully registered in DB.")
except DuplicateKeyError:
log.debug(
"Experiment registration failed. This is likely due to a race condition. "
"Now rolling back and re-attempting building it.")
experiment = build(branching=branching, **config)
return experiment
log.debug(
f"Experiment {config['name']}-v{config['version']} already existed.")
conflicts = _get_conflicts(experiment, branching)
must_branch = len(conflicts.get()) > 1 or branching.get("branch_to")
if must_branch and branching.get("enable", orion.core.config.evc.enable):
return _attempt_branching(conflicts, experiment, version, branching)
elif must_branch:
log.warning(
"Running experiment in a different state:\n%s",
_get_branching_status_string(conflicts, branching),
)
log.debug("No branching required.")
_update_experiment(experiment)
return experiment
def get_or_create_benchmark(
name, algorithms=None, targets=None, storage=None, debug=False
):
"""
Create or get a benchmark object.
Parameters
----------
name: str
Name of the benchmark
algorithms: list, optional
Algorithms used for benchmark, each algorithm can be a string or dict.
targets: list, optional
Targets for the benchmark, each target will be a dict with two keys.
assess: list
Assessment objects
task: list
Task objects
storage: dict, optional
Configuration of the storage backend.
debug: bool, optional
If using in debug mode, the storage config is overrided with legacy:EphemeralDB.
Defaults to False.
Returns
-------
An instance of `orion.benchmark.Benchmark`
"""
setup_storage(storage=storage, debug=debug)
# fetch benchmark from db
db_config = _fetch_benchmark(name)
benchmark_id = None
input_configure = None
if db_config:
if algorithms or targets:
input_benchmark = Benchmark(name, algorithms, targets)
input_configure = input_benchmark.configuration
benchmark_id, algorithms, targets = _resolve_db_config(db_config)
if not algorithms or not targets:
raise NoConfigurationError(
"Benchmark {} does not exist in DB, "
"algorithms and targets space was not defined.".format(name)
)
benchmark = _create_benchmark(name, algorithms, targets)
if input_configure and input_benchmark.configuration != benchmark.configuration:
logger.warn(
"Benchmark with same name is found but has different configuration, "
"which will be used for this creation.\n{}".format(benchmark.configuration)
)
if benchmark_id is None:
logger.debug("Benchmark not found in DB. Now attempting registration in DB.")
try:
_register_benchmark(benchmark)
logger.debug("Benchmark successfully registered in DB.")
except DuplicateKeyError:
logger.info(
"Benchmark registration failed. This is likely due to a race condition. "
"Now rolling back and re-attempting building it."
)
get_or_create_benchmark(name, algorithms, targets, storage, debug)
return benchmark