def test_setup_storage_bad(): """Test how setup fails when configuring with non-existant backends""" update_singletons() with pytest.raises(NotImplementedError) as exc: setup_storage({'type': 'idontexist'}) assert exc.match('idontexist')
def test_get_storage_uninitiated(): """Test that get storage fails if no storage singleton exist""" update_singletons() with pytest.raises(SingletonNotInstantiatedError) as exc: get_storage() assert exc.match('No singleton instance of \(type: Storage\) was created')
def test_setup_storage_default(): """Test that storage is setup using default config""" update_singletons() setup_storage() storage = Storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB)
def test_build_view_from_args_debug_mode(script_path): """Try building experiment view in debug mode""" update_singletons() # Can't build view if none exist. It's fine we only want to test the storage creation. with pytest.raises(ValueError): experiment_builder.build_view_from_args({'name': 'whatever'}) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) update_singletons() # Can't build view if none exist. It's fine we only want to test the storage creation. with pytest.raises(ValueError): experiment_builder.build_view_from_args({ 'name': 'whatever', 'debug': True }) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, EphemeralDB)
def test_build_from_args_debug_mode(script_path): """Try building experiment in debug mode""" update_singletons() experiment_builder.build_from_args({ 'name': 'whatever', 'user_args': [script_path, '--mini-batch~uniform(32, 256)'] }) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) update_singletons() experiment_builder.build_from_args({ 'name': 'whatever', 'user_args': [script_path, '--mini-batch~uniform(32, 256)'], 'debug': True }) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, EphemeralDB)
def test_get_storage(): """Test that get storage gets the singleton""" update_singletons() setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}}) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) assert get_storage() == storage
def test_setup_storage_custom_legacy_emtpy(): """Test setup with local configuration with legacy but no config""" update_singletons() setup_storage({'type': 'legacy'}) storage = Storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) assert storage._db.host == orion.core.config.storage.database.host
def test_setup_storage_custom_type_missing(): """Test setup with local configuration with type missing""" update_singletons() setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}}) storage = Storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) assert storage._db.host == 'test.pkl'
def test_setup_storage_bad_config_override(): """Test setup with different config than existing singleton""" update_singletons() setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}}) storage = Storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) with pytest.raises(SingletonAlreadyInstantiatedError): setup_storage({'database': {'type': 'mongodb'}})
def test_create_experiment_bad_storage(self): """Test error message if storage is not configured properly""" name = 'oopsie_bad_storage' # Make sure there is no existing storage singleton update_singletons() with pytest.raises(NotImplementedError) as exc: create_experiment(name=name, storage={'type': 'legacy', 'database': {'type': 'idontexist'}}) assert "Could not find implementation of AbstractDB, type = 'idontexist'" in str(exc.value)
def setup_pickleddb_database(): """Configure the database""" update_singletons() temporary_file = tempfile.NamedTemporaryFile() os.environ['ORION_DB_TYPE'] = "pickleddb" os.environ['ORION_DB_ADDRESS'] = temporary_file.name yield temporary_file.close() del os.environ['ORION_DB_TYPE'] del os.environ['ORION_DB_ADDRESS']
def workon(function, space, name='loop', algorithms=None, max_trials=None): """Optimize a function over a given search space This will create a new experiment with an in-memory storage and optimize the given function until `max_trials` is reached or the `algorithm` is done (some algorithms like random search are never done). For informations on how to fetch results, see :py:class:`orion.client.experiment.ExperimentClient`. .. note:: Each call to this function will create a separate in-memory storage. Parameters ---------- name: str Name of the experiment version: int, optional Version of the experiment. Defaults to last existing version for a given `name` or 1 for new experiment. space: dict, optional Optimization space of the algorithm. Should have the form `dict(name='<prior>(args)')`. algorithms: str or dict, optional Algorithm used for optimization. max_trials: int, optional Maximum number or trials before the experiment is considered done. Raises ------ `NotImplementedError` If the algorithm specified is not properly installed. """ # Clear singletons and keep pointers to restore them. singletons = update_singletons() setup_storage(storage={'type': 'legacy', 'database': {'type': 'EphemeralDB'}}) experiment = experiment_builder.build( name, version=1, space=space, algorithms=algorithms, strategy='NoParallelStrategy', max_trials=max_trials) producer = Producer(experiment) experiment_client = ExperimentClient(experiment, producer) experiment_client.workon(function, max_trials=max_trials) # Restore singletons update_singletons(singletons) return experiment_client
def test_setup_storage_bad_override(): """Test setup with different type than existing singleton""" update_singletons() setup_storage({ 'type': 'legacy', 'database': { 'type': 'pickleddb', 'host': 'test.pkl' } }) storage = Storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) with pytest.raises(SingletonAlreadyInstantiatedError) as exc: setup_storage({'type': 'track'}) assert exc.match('A singleton instance of \(type: Storage\)')
def test_create_experiment_debug_mode(self): """Test that EphemeralDB is used in debug mode whatever the storage config given""" update_singletons() create_experiment( config['name'], space={'x': 'uniform(0, 10)'}, storage={'type': 'legacy', 'database': {'type': 'pickleddb'}}) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, PickledDB) update_singletons() create_experiment( config['name'], space={'x': 'uniform(0, 10)'}, storage={'type': 'legacy', 'database': {'type': 'pickleddb'}}, debug=True) storage = get_storage() assert isinstance(storage, Legacy) assert isinstance(storage._db, EphemeralDB)
def test_create_experiment_no_storage(self, monkeypatch): """Test creation if storage is not configured""" name = 'oopsie_forgot_a_storage' host = orion.core.config.storage.database.host with OrionState(storage=orion.core.config.storage.to_dict()) as cfg: # Reset the Storage and drop instances so that get_storage() would fail. cfg.cleanup() cfg.singletons = update_singletons() # Make sure storage must be instantiated during `create_experiment()` with pytest.raises(SingletonNotInstantiatedError): get_storage() experiment = create_experiment(name=name, space={'x': 'uniform(0, 10)'}) assert isinstance(experiment._experiment._storage, Legacy) assert isinstance(experiment._experiment._storage._db, PickledDB) assert experiment._experiment._storage._db.host == host
def stress_test(storage, space_type, workers, size): """Spawn workers and run stress test with verifications Parameters ---------- storage: str See `get_experiment`. space_type: str See `get_experiment`. workers: int Number of workers to run in parallel. size: int See `get_experiment`. Returns ------- `list` of `orion.core.worker.trial.Trial` List of all trials at the end of the stress test """ if storage == 'pickleddb': if os.path.exists(DB_FILE): os.remove(DB_FILE) elif storage == 'mongodb': client = MongoClient(username='******', password='******', authSource='stress') database = client.stress database.experiments.drop() database.lying_trials.drop() database.trials.drop() database.workers.drop() database.resources.drop() client.close() update_singletons() print('Worker | Point') with Pool(workers) as p: results = p.starmap( worker, zip(range(workers), [storage] * workers, [space_type] * workers, [size] * workers)) assert None not in results, 'A worker crashed unexpectedly. See logs for the error messages.' assert all(n > 0 for n in results), 'A worker could not execute any trial.' if space_type in ['discrete', 'real-seeded']: assert sum(results) == size, results else: assert sum(results) >= size, results experiment = get_experiment(storage, space_type, size) trials = experiment.fetch_trials() if storage == 'pickleddb': os.remove(DB_FILE) elif storage == 'mongodb': client = MongoClient(username='******', password='******', authSource='stress') database = client.stress database.experiments.drop() database.lying_trials.drop() database.trials.drop() database.workers.drop() database.resources.drop() client.close() update_singletons() return trials