def create_experiment(mode, space, algorithm):
experiment = Experiment("supernaekei", mode=mode)
experiment.space = space
experiment.algorithms = algorithm
experiment.max_broken = 5
experiment.max_trials = 5
return experiment
def create_experiment(name, version, space, **kwargs):
"""Instantiate the experiment and its attribute objects
All unspecified arguments will be replaced by system's defaults (orion.core.config.*).
Parameters
----------
name: str
Name of the experiment.
version: int
Version of the experiment.
space: dict or Space object
Optimization space of the algorithm. If dict, should have the form
`dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Parallel strategy to use to parallelize the algorithm.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
storage: dict, optional
Configuration of the storage backend.
"""
experiment = Experiment(name=name, version=version)
experiment._id = kwargs.get('_id', None) # pylint:disable=protected-access
experiment.pool_size = kwargs.get('pool_size')
if experiment.pool_size is None:
experiment.pool_size = orion.core.config.experiment.get(
'pool_size', deprecated='ignore')
experiment.max_trials = kwargs.get('max_trials',
orion.core.config.experiment.max_trials)
experiment.space = _instantiate_space(space)
experiment.algorithms = _instantiate_algo(experiment.space,
kwargs.get('algorithms'))
experiment.producer = kwargs.get('producer', {})
experiment.producer['strategy'] = _instantiate_strategy(
experiment.producer.get('strategy'))
experiment.working_dir = kwargs.get(
'working_dir', orion.core.config.experiment.working_dir)
experiment.metadata = kwargs.get(
'metadata', {'user': kwargs.get('user', getpass.getuser())})
experiment.refers = kwargs.get('refers', {
'parent_id': None,
'root_id': None,
'adapter': []
})
experiment.refers['adapter'] = _instantiate_adapters(
experiment.refers.get('adapter', []))
return experiment
def test_view_is_done_property_no_pending(algorithm):
"""Check experiment stopping conditions from view when there is no pending trials."""
completed = ['completed'] * 10
broken = ['broken'] * 5
with OrionState(trials=generate_trials(completed + broken)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.algorithms = algorithm
exp.max_trials = 100
exp_view = ExperimentView(exp)
exp.algorithms = algorithm
exp.max_trials = 15
# There is only 10 completed trials and algo not done.
assert not exp_view.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done and no pending trials
assert exp_view.is_done
def test_experiment_non_interactive_branching(self, create_db_instance,
random_dt, exp_config,
monkeypatch):
"""Configure an existing experiment with parent."""
monkeypatch.setattr('sys.__stdin__.isatty', lambda: True)
exp = Experiment('supernaedo2.1')
exp.algorithms = {'dumbalgo': {}}
with pytest.raises(OSError):
exp.configure(exp.configuration)
monkeypatch.undo()
with pytest.raises(ValueError) as exc_info:
exp.configure(exp.configuration)
assert "Configuration is different and generates a branching" in str(
exc_info.value)
def test_experiment_with_parent(self, create_db_instance, random_dt,
exp_config):
"""Configure an existing experiment with parent."""
exp = Experiment('supernaedo2.1')
exp.algorithms = {'random': {}}
exp.configure(exp.configuration)
assert exp._init_done is True
assert exp._db is create_db_instance
assert exp._id is not None
assert exp.name == 'supernaedo2.1'
assert exp.configuration['refers'] == exp_config[0][4]['refers']
assert exp.metadata == exp_config[0][4]['metadata']
assert exp.pool_size == 2
assert exp.max_trials == 1000
assert exp.configuration['algorithms'] == {'random': {}}
def test_acquire_algorithm_lock_timeout(self, new_config, algorithm, mocker):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
exp.algorithms = algorithm
storage_acquisition_mock = mocker.spy(
cfg.storage(), "acquire_algorithm_lock"
)
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
pass
storage_acquisition_mock.assert_called_with(
experiment=exp, timeout=0.2, retry_interval=0.1
)
def test_acquire_algorithm_lock_with_different_config(self, new_config, algorithm):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
algorithm_original_config = algorithm.configuration
exp.algorithms = algorithm
# Setting attribute to algorithm inside the wrapper
algorithm.algorithm.seed = 10
assert algorithm.configuration != algorithm_original_config
with pytest.raises(
RuntimeError, match="Algorithm configuration changed since"
):
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
pass
def test_new_experiment_with_parent(self, create_db_instance, random_dt,
exp_config):
"""Configure a branch experiment."""
exp = Experiment('supernaedo2.6')
exp.metadata = exp_config[0][4]['metadata']
exp.refers = exp_config[0][4]['refers']
exp.algorithms = exp_config[0][4]['algorithms']
exp.configure(exp.configuration)
assert exp._init_done is True
assert_protocol(exp, create_db_instance)
assert exp._id is not None
assert exp.name == 'supernaedo2.6'
assert exp.configuration['refers'] == exp_config[0][4]['refers']
exp_config[0][4]['metadata']['datetime'] = random_dt
assert exp.metadata == exp_config[0][4]['metadata']
assert exp.pool_size is None
assert exp.max_trials is None
assert exp.configuration['algorithms'] == {'random': {'seed': None}}
def test_is_done_property_no_pending(algorithm):
"""Check experiment stopping conditions when there is no pending trials."""
completed = ["completed"] * 10
broken = ["broken"] * 5
with OrionState(trials=generate_trials(completed + broken)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.algorithms = algorithm
exp.max_trials = 15
# There is only 10 completed trials and algo not done.
assert not exp.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done and no pending trials
assert exp.is_done
def test_is_done_property_with_pending(algorithm):
"""Check experiment stopping conditions when there is pending trials."""
completed = ['completed'] * 10
reserved = ['reserved'] * 5
with OrionState(trials=generate_trials(completed + reserved)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.algorithms = algorithm
exp.max_trials = 10
assert exp.is_done
exp.max_trials = 15
# There is only 10 completed trials
assert not exp.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done but 5 trials are pending
assert not exp.is_done
def test_acquire_algorithm_lock_successful(self, new_config, algorithm):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
exp.algorithms = algorithm
state_dict = algorithm.state_dict
# No state_dict in DB
with exp.acquire_algorithm_lock(
timeout=0.2, retry_interval=0.1) as locked_algorithm:
assert locked_algorithm is algorithm
assert algorithm.state_dict == state_dict
algorithm.suggest(1)
assert algorithm.state_dict != state_dict
new_state_dict = algorithm.state_dict
algorithm.set_state(state_dict)
assert algorithm.configuration != new_state_dict
# State_dict in DB used to set algorithm state.
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
assert algorithm.state_dict == new_state_dict
def create_experiment(name, version, mode, space, **kwargs):
"""Instantiate the experiment and its attribute objects
All unspecified arguments will be replaced by system's defaults (orion.core.config.*).
Parameters
----------
name: str
Name of the experiment.
version: int
Version of the experiment.
mode: str
The access rights of the experiment on the database.
'r': read access only
'w': can read and write to database
'x': can read and write to database, algo is instantiated and can execute optimization
space: dict or Space object
Optimization space of the algorithm. If dict, should have the form
`dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Parallel strategy to use to parallelize the algorithm.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
max_broken: int, optional
Number of broken trials for the experiment to be considered broken.
storage: dict, optional
Configuration of the storage backend.
"""
experiment = Experiment(name=name, version=version, mode=mode)
experiment._id = kwargs.get("_id", None) # pylint:disable=protected-access
experiment.max_trials = kwargs.get("max_trials",
orion.core.config.experiment.max_trials)
experiment.max_broken = kwargs.get("max_broken",
orion.core.config.experiment.max_broken)
experiment.space = _instantiate_space(space)
experiment.algorithms = _instantiate_algo(
experiment.space,
experiment.max_trials,
kwargs.get("algorithms"),
ignore_unavailable=mode != "x",
)
# TODO: Remove for v0.4
_instantiate_strategy(kwargs.get("producer", {}).get("strategy"))
experiment.working_dir = kwargs.get(
"working_dir", orion.core.config.experiment.working_dir)
experiment.metadata = kwargs.get(
"metadata", {"user": kwargs.get("user", getpass.getuser())})
experiment.refers = kwargs.get("refers", {
"parent_id": None,
"root_id": None,
"adapter": []
})
experiment.refers["adapter"] = _instantiate_adapters(
experiment.refers.get("adapter", []))
log.debug("Created experiment with config:\n%s",
pprint.pformat(experiment.configuration))
return experiment
