def test_setup_storage_bad():
"""Test how setup fails when configuring with non-existant backends"""
update_singletons()
with pytest.raises(NotImplementedError) as exc:
setup_storage({'type': 'idontexist'})
assert exc.match('idontexist')
def test_get_storage_uninitiated():
"""Test that get storage fails if no storage singleton exist"""
update_singletons()
with pytest.raises(SingletonNotInstantiatedError) as exc:
get_storage()
assert exc.match('No singleton instance of \(type: Storage\) was created')
def test_setup_storage_default():
"""Test that storage is setup using default config"""
update_singletons()
setup_storage()
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
def test_build_view_from_args_debug_mode(script_path):
"""Try building experiment view in debug mode"""
update_singletons()
# Can't build view if none exist. It's fine we only want to test the storage creation.
with pytest.raises(ValueError):
experiment_builder.build_view_from_args({'name': 'whatever'})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
# Can't build view if none exist. It's fine we only want to test the storage creation.
with pytest.raises(ValueError):
experiment_builder.build_view_from_args({
'name': 'whatever',
'debug': True
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_build_from_args_debug_mode(script_path):
"""Try building experiment in debug mode"""
update_singletons()
experiment_builder.build_from_args({
'name':
'whatever',
'user_args': [script_path, '--mini-batch~uniform(32, 256)']
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
experiment_builder.build_from_args({
'name':
'whatever',
'user_args': [script_path, '--mini-batch~uniform(32, 256)'],
'debug':
True
})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_get_storage():
"""Test that get storage gets the singleton"""
update_singletons()
setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert get_storage() == storage
def test_setup_storage_custom_legacy_emtpy():
"""Test setup with local configuration with legacy but no config"""
update_singletons()
setup_storage({'type': 'legacy'})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == orion.core.config.storage.database.host
def test_setup_storage_custom_type_missing():
"""Test setup with local configuration with type missing"""
update_singletons()
setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == 'test.pkl'
def test_setup_storage_bad_config_override():
"""Test setup with different config than existing singleton"""
update_singletons()
setup_storage({'database': {'type': 'pickleddb', 'host': 'test.pkl'}})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError):
setup_storage({'database': {'type': 'mongodb'}})
def test_create_experiment_bad_storage(self):
"""Test error message if storage is not configured properly"""
name = 'oopsie_bad_storage'
# Make sure there is no existing storage singleton
update_singletons()
with pytest.raises(NotImplementedError) as exc:
create_experiment(name=name, storage={'type': 'legacy',
'database': {'type': 'idontexist'}})
assert "Could not find implementation of AbstractDB, type = 'idontexist'" in str(exc.value)
def setup_pickleddb_database():
"""Configure the database"""
update_singletons()
temporary_file = tempfile.NamedTemporaryFile()
os.environ['ORION_DB_TYPE'] = "pickleddb"
os.environ['ORION_DB_ADDRESS'] = temporary_file.name
yield
temporary_file.close()
del os.environ['ORION_DB_TYPE']
del os.environ['ORION_DB_ADDRESS']
def workon(function, space, name='loop', algorithms=None, max_trials=None):
"""Optimize a function over a given search space
This will create a new experiment with an in-memory storage and optimize the given function
until `max_trials` is reached or the `algorithm` is done
(some algorithms like random search are never done).
For informations on how to fetch results, see
:py:class:`orion.client.experiment.ExperimentClient`.
.. note::
Each call to this function will create a separate in-memory storage.
Parameters
----------
name: str
Name of the experiment
version: int, optional
Version of the experiment. Defaults to last existing version for a given `name`
or 1 for new experiment.
space: dict, optional
Optimization space of the algorithm. Should have the form `dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
Raises
------
`NotImplementedError`
If the algorithm specified is not properly installed.
"""
# Clear singletons and keep pointers to restore them.
singletons = update_singletons()
setup_storage(storage={'type': 'legacy', 'database': {'type': 'EphemeralDB'}})
experiment = experiment_builder.build(
name, version=1, space=space, algorithms=algorithms,
strategy='NoParallelStrategy', max_trials=max_trials)
producer = Producer(experiment)
experiment_client = ExperimentClient(experiment, producer)
experiment_client.workon(function, max_trials=max_trials)
# Restore singletons
update_singletons(singletons)
return experiment_client
def test_setup_storage_bad_override():
"""Test setup with different type than existing singleton"""
update_singletons()
setup_storage({
'type': 'legacy',
'database': {
'type': 'pickleddb',
'host': 'test.pkl'
}
})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError) as exc:
setup_storage({'type': 'track'})
assert exc.match('A singleton instance of \(type: Storage\)')
def test_create_experiment_debug_mode(self):
"""Test that EphemeralDB is used in debug mode whatever the storage config given"""
update_singletons()
create_experiment(
config['name'], space={'x': 'uniform(0, 10)'},
storage={'type': 'legacy', 'database': {'type': 'pickleddb'}})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
create_experiment(
config['name'], space={'x': 'uniform(0, 10)'},
storage={'type': 'legacy', 'database': {'type': 'pickleddb'}}, debug=True)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_create_experiment_no_storage(self, monkeypatch):
"""Test creation if storage is not configured"""
name = 'oopsie_forgot_a_storage'
host = orion.core.config.storage.database.host
with OrionState(storage=orion.core.config.storage.to_dict()) as cfg:
# Reset the Storage and drop instances so that get_storage() would fail.
cfg.cleanup()
cfg.singletons = update_singletons()
# Make sure storage must be instantiated during `create_experiment()`
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
experiment = create_experiment(name=name, space={'x': 'uniform(0, 10)'})
assert isinstance(experiment._experiment._storage, Legacy)
assert isinstance(experiment._experiment._storage._db, PickledDB)
assert experiment._experiment._storage._db.host == host
def stress_test(storage, space_type, workers, size):
"""Spawn workers and run stress test with verifications
Parameters
----------
storage: str
See `get_experiment`.
space_type: str
See `get_experiment`.
workers: int
Number of workers to run in parallel.
size: int
See `get_experiment`.
Returns
-------
`list` of `orion.core.worker.trial.Trial`
List of all trials at the end of the stress test
"""
if storage == 'pickleddb':
if os.path.exists(DB_FILE):
os.remove(DB_FILE)
elif storage == 'mongodb':
client = MongoClient(username='******',
password='******',
authSource='stress')
database = client.stress
database.experiments.drop()
database.lying_trials.drop()
database.trials.drop()
database.workers.drop()
database.resources.drop()
client.close()
update_singletons()
print('Worker | Point')
with Pool(workers) as p:
results = p.starmap(
worker,
zip(range(workers), [storage] * workers, [space_type] * workers,
[size] * workers))
assert None not in results, 'A worker crashed unexpectedly. See logs for the error messages.'
assert all(n > 0 for n in results), 'A worker could not execute any trial.'
if space_type in ['discrete', 'real-seeded']:
assert sum(results) == size, results
else:
assert sum(results) >= size, results
experiment = get_experiment(storage, space_type, size)
trials = experiment.fetch_trials()
if storage == 'pickleddb':
os.remove(DB_FILE)
elif storage == 'mongodb':
client = MongoClient(username='******',
password='******',
authSource='stress')
database = client.stress
database.experiments.drop()
database.lying_trials.drop()
database.trials.drop()
database.workers.drop()
database.resources.drop()
client.close()
update_singletons()
return trials