def create_experiment(mode, space, algorithm): experiment = Experiment("supernaekei", mode=mode) experiment.space = space experiment.algorithms = algorithm experiment.max_broken = 5 experiment.max_trials = 5 return experiment
def create_experiment(name, version, space, **kwargs): """Instantiate the experiment and its attribute objects All unspecified arguments will be replaced by system's defaults (orion.core.config.*). Parameters ---------- name: str Name of the experiment. version: int Version of the experiment. space: dict or Space object Optimization space of the algorithm. If dict, should have the form `dict(name='<prior>(args)')`. algorithms: str or dict, optional Algorithm used for optimization. strategy: str or dict, optional Parallel strategy to use to parallelize the algorithm. max_trials: int, optional Maximum number or trials before the experiment is considered done. storage: dict, optional Configuration of the storage backend. """ experiment = Experiment(name=name, version=version) experiment._id = kwargs.get('_id', None) # pylint:disable=protected-access experiment.pool_size = kwargs.get('pool_size') if experiment.pool_size is None: experiment.pool_size = orion.core.config.experiment.get( 'pool_size', deprecated='ignore') experiment.max_trials = kwargs.get('max_trials', orion.core.config.experiment.max_trials) experiment.space = _instantiate_space(space) experiment.algorithms = _instantiate_algo(experiment.space, kwargs.get('algorithms')) experiment.producer = kwargs.get('producer', {}) experiment.producer['strategy'] = _instantiate_strategy( experiment.producer.get('strategy')) experiment.working_dir = kwargs.get( 'working_dir', orion.core.config.experiment.working_dir) experiment.metadata = kwargs.get( 'metadata', {'user': kwargs.get('user', getpass.getuser())}) experiment.refers = kwargs.get('refers', { 'parent_id': None, 'root_id': None, 'adapter': [] }) experiment.refers['adapter'] = _instantiate_adapters( experiment.refers.get('adapter', [])) return experiment
def test_view_is_done_property_no_pending(algorithm): """Check experiment stopping conditions from view when there is no pending trials.""" completed = ['completed'] * 10 broken = ['broken'] * 5 with OrionState(trials=generate_trials(completed + broken)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 100 exp_view = ExperimentView(exp) exp.algorithms = algorithm exp.max_trials = 15 # There is only 10 completed trials and algo not done. assert not exp_view.is_done exp.algorithms.algorithm.done = True # Algorithm is done and no pending trials assert exp_view.is_done
def test_experiment_non_interactive_branching(self, create_db_instance, random_dt, exp_config, monkeypatch): """Configure an existing experiment with parent.""" monkeypatch.setattr('sys.__stdin__.isatty', lambda: True) exp = Experiment('supernaedo2.1') exp.algorithms = {'dumbalgo': {}} with pytest.raises(OSError): exp.configure(exp.configuration) monkeypatch.undo() with pytest.raises(ValueError) as exc_info: exp.configure(exp.configuration) assert "Configuration is different and generates a branching" in str( exc_info.value)
def test_experiment_with_parent(self, create_db_instance, random_dt, exp_config): """Configure an existing experiment with parent.""" exp = Experiment('supernaedo2.1') exp.algorithms = {'random': {}} exp.configure(exp.configuration) assert exp._init_done is True assert exp._db is create_db_instance assert exp._id is not None assert exp.name == 'supernaedo2.1' assert exp.configuration['refers'] == exp_config[0][4]['refers'] assert exp.metadata == exp_config[0][4]['metadata'] assert exp.pool_size == 2 assert exp.max_trials == 1000 assert exp.configuration['algorithms'] == {'random': {}}
def test_acquire_algorithm_lock_timeout(self, new_config, algorithm, mocker): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 exp.algorithms = algorithm storage_acquisition_mock = mocker.spy( cfg.storage(), "acquire_algorithm_lock" ) with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): pass storage_acquisition_mock.assert_called_with( experiment=exp, timeout=0.2, retry_interval=0.1 )
def test_acquire_algorithm_lock_with_different_config(self, new_config, algorithm): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 algorithm_original_config = algorithm.configuration exp.algorithms = algorithm # Setting attribute to algorithm inside the wrapper algorithm.algorithm.seed = 10 assert algorithm.configuration != algorithm_original_config with pytest.raises( RuntimeError, match="Algorithm configuration changed since" ): with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): pass
def test_new_experiment_with_parent(self, create_db_instance, random_dt, exp_config): """Configure a branch experiment.""" exp = Experiment('supernaedo2.6') exp.metadata = exp_config[0][4]['metadata'] exp.refers = exp_config[0][4]['refers'] exp.algorithms = exp_config[0][4]['algorithms'] exp.configure(exp.configuration) assert exp._init_done is True assert_protocol(exp, create_db_instance) assert exp._id is not None assert exp.name == 'supernaedo2.6' assert exp.configuration['refers'] == exp_config[0][4]['refers'] exp_config[0][4]['metadata']['datetime'] = random_dt assert exp.metadata == exp_config[0][4]['metadata'] assert exp.pool_size is None assert exp.max_trials is None assert exp.configuration['algorithms'] == {'random': {'seed': None}}
def test_is_done_property_no_pending(algorithm): """Check experiment stopping conditions when there is no pending trials.""" completed = ["completed"] * 10 broken = ["broken"] * 5 with OrionState(trials=generate_trials(completed + broken)) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = cfg.trials[0]["experiment"] exp.algorithms = algorithm exp.max_trials = 15 # There is only 10 completed trials and algo not done. assert not exp.is_done exp.algorithms.algorithm.done = True # Algorithm is done and no pending trials assert exp.is_done
def test_is_done_property_with_pending(algorithm): """Check experiment stopping conditions when there is pending trials.""" completed = ['completed'] * 10 reserved = ['reserved'] * 5 with OrionState(trials=generate_trials(completed + reserved)) as cfg: exp = Experiment('supernaekei') exp._id = cfg.trials[0]['experiment'] exp.algorithms = algorithm exp.max_trials = 10 assert exp.is_done exp.max_trials = 15 # There is only 10 completed trials assert not exp.is_done exp.algorithms.algorithm.done = True # Algorithm is done but 5 trials are pending assert not exp.is_done
def test_acquire_algorithm_lock_successful(self, new_config, algorithm): with OrionState(experiments=[new_config]) as cfg: exp = Experiment("supernaekei", mode="x") exp._id = 0 exp.algorithms = algorithm state_dict = algorithm.state_dict # No state_dict in DB with exp.acquire_algorithm_lock( timeout=0.2, retry_interval=0.1) as locked_algorithm: assert locked_algorithm is algorithm assert algorithm.state_dict == state_dict algorithm.suggest(1) assert algorithm.state_dict != state_dict new_state_dict = algorithm.state_dict algorithm.set_state(state_dict) assert algorithm.configuration != new_state_dict # State_dict in DB used to set algorithm state. with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1): assert algorithm.state_dict == new_state_dict
def create_experiment(name, version, mode, space, **kwargs): """Instantiate the experiment and its attribute objects All unspecified arguments will be replaced by system's defaults (orion.core.config.*). Parameters ---------- name: str Name of the experiment. version: int Version of the experiment. mode: str The access rights of the experiment on the database. 'r': read access only 'w': can read and write to database 'x': can read and write to database, algo is instantiated and can execute optimization space: dict or Space object Optimization space of the algorithm. If dict, should have the form `dict(name='<prior>(args)')`. algorithms: str or dict, optional Algorithm used for optimization. strategy: str or dict, optional Parallel strategy to use to parallelize the algorithm. max_trials: int, optional Maximum number or trials before the experiment is considered done. max_broken: int, optional Number of broken trials for the experiment to be considered broken. storage: dict, optional Configuration of the storage backend. """ experiment = Experiment(name=name, version=version, mode=mode) experiment._id = kwargs.get("_id", None) # pylint:disable=protected-access experiment.max_trials = kwargs.get("max_trials", orion.core.config.experiment.max_trials) experiment.max_broken = kwargs.get("max_broken", orion.core.config.experiment.max_broken) experiment.space = _instantiate_space(space) experiment.algorithms = _instantiate_algo( experiment.space, experiment.max_trials, kwargs.get("algorithms"), ignore_unavailable=mode != "x", ) # TODO: Remove for v0.4 _instantiate_strategy(kwargs.get("producer", {}).get("strategy")) experiment.working_dir = kwargs.get( "working_dir", orion.core.config.experiment.working_dir) experiment.metadata = kwargs.get( "metadata", {"user": kwargs.get("user", getpass.getuser())}) experiment.refers = kwargs.get("refers", { "parent_id": None, "root_id": None, "adapter": [] }) experiment.refers["adapter"] = _instantiate_adapters( experiment.refers.get("adapter", [])) log.debug("Created experiment with config:\n%s", pprint.pformat(experiment.configuration)) return experiment