def test_setup_storage_default():
"""Test that storage is setup using default config"""
update_singletons()
setup_storage()
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
def check_env_var_config(self, tmp_path, monkeypatch):
"""Check that env vars overrides global configuration"""
update_singletons()
assert orion.core.config.storage.to_dict() == {
"type": self.env_vars["ORION_STORAGE_TYPE"],
"database": {
"name": self.env_vars["ORION_DB_NAME"],
"type": self.env_vars["ORION_DB_TYPE"],
"host": self.env_vars["ORION_DB_ADDRESS"],
"port": int(self.env_vars["ORION_DB_PORT"]),
},
}
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
command = f"hunt --exp-max-trials 0 -n test python {script} -x~uniform(0,1)"
orion.core.cli.main(command.split(" "))
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == os.path.abspath(
self.env_vars["ORION_DB_ADDRESS"])
def test_create_experiment_debug_mode(self, tmp_path, benchmark_config_py):
"""Test that EphemeralDB is used in debug mode whatever the storage config given"""
update_singletons()
conf_file = str(tmp_path / "db.pkl")
config = copy.deepcopy(benchmark_config_py)
config["storage"] = {
"type": "legacy",
"database": {"type": "pickleddb", "host": conf_file},
}
get_or_create_benchmark(**config)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
config["storage"] = {"type": "legacy", "database": {"type": "pickleddb"}}
config["debug"] = True
get_or_create_benchmark(**config)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_setup_database_bad():
"""Test how setup fails when configuring with non-existant backends"""
update_singletons()
with pytest.raises(NotImplementedError) as exc:
setup_database({"type": "idontexist"})
assert exc.match("idontexist")
def test_setup_storage_stateless():
"""Test that passed configuration dictionary is not modified by the fonction"""
update_singletons()
config = {"database": {"type": "pickleddb", "host": "test.pkl"}}
passed_config = copy.deepcopy(config)
setup_storage(passed_config)
assert config == passed_config
def test_get_database_uninitiated():
"""Test that get database fails if no database singleton exist"""
update_singletons()
with pytest.raises(SingletonNotInstantiatedError) as exc:
get_database()
assert exc.match("No singleton instance of \(type: Database\) was created")
def test_get_database():
"""Test that get database gets the singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = get_database()
assert isinstance(database, PickledDB)
assert get_database() == database
def test_setup_database_custom():
"""Test setup with local configuration"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = database_factory.create()
assert isinstance(database, PickledDB)
assert database.host == os.path.abspath("test.pkl")
def test_build_from_args_debug_mode(script_path):
"""Try building experiment in debug mode"""
update_singletons()
experiment_builder.build_from_args(
{
"name": "whatever",
"user_args": [script_path, "--mini-batch~uniform(32, 256)"],
}
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
experiment_builder.build_from_args(
{
"name": "whatever",
"user_args": [script_path, "--mini-batch~uniform(32, 256)"],
"debug": True,
}
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_get_storage():
"""Test that get storage gets the singleton"""
update_singletons()
setup_storage({"database": {"type": "pickleddb", "host": "test.pkl"}})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert get_storage() == storage
def test_setup_storage_custom_type_missing():
"""Test setup with local configuration with type missing"""
update_singletons()
setup_storage({"database": {"type": "pickleddb", "host": "test.pkl"}})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == "test.pkl"
def test_setup_database_bad_config_override():
"""Test setup with different config than existing singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = database_factory.create()
assert isinstance(database, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError):
setup_database({"type": "pickleddb", "host": "other.pkl"})
def test_get_storage_uninitiated():
"""Test that get storage fails if no storage singleton exist"""
update_singletons()
with pytest.raises(SingletonNotInstantiatedError) as exc:
get_storage()
assert exc.match(
"No singleton instance of \(type: BaseStorageProtocol\) was created")
def test_setup_storage_custom_legacy_emtpy():
"""Test setup with local configuration with legacy but no config"""
update_singletons()
setup_storage({"type": "legacy"})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == orion.core.config.storage.database.host
def test_setup_storage_bad_config_override():
"""Test setup with different config than existing singleton"""
update_singletons()
setup_storage({"database": {"type": "pickleddb", "host": "test.pkl"}})
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError):
setup_storage({"database": {"type": "mongodb"}})
def test_setup_database_bad_override():
"""Test setup with different type than existing singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = database_factory.create()
assert isinstance(database, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError) as exc:
setup_database({"type": "mongodb"})
assert exc.match("A singleton instance of \(type: Database\)")
def setup_pickleddb_database():
"""Configure the database"""
update_singletons()
temporary_file = tempfile.NamedTemporaryFile()
os.environ["ORION_DB_TYPE"] = "pickleddb"
os.environ["ORION_DB_ADDRESS"] = temporary_file.name
yield
temporary_file.close()
del os.environ["ORION_DB_TYPE"]
del os.environ["ORION_DB_ADDRESS"]
def test_setup_storage_bad_override():
"""Test setup with different type than existing singleton"""
update_singletons()
setup_storage(
{"type": "legacy", "database": {"type": "pickleddb", "host": "test.pkl"}}
)
storage = Storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError) as exc:
setup_storage({"type": "track"})
assert exc.match("A singleton instance of \(type: Storage\)")
def __getstate__(self):
"""Remove storage instance during experiment serialization"""
state = dict()
for entry in self.__slots__:
state[entry] = getattr(self, entry)
# TODO: This should be removed when singletons and `get_storage()` are removed.
# See https://github.com/Epistimio/orion/issues/606
singletons = update_singletons()
state["singletons"] = singletons
update_singletons(singletons)
return state
def setup_tmp_storage(host):
# Clear singletons
update_singletons()
setup_storage(storage={
"type": "legacy",
"database": {
"type": "pickleddb",
"host": host,
},
})
return get_storage()
def test_setup_storage_custom():
"""Test setup with local configuration"""
update_singletons()
setup_storage({
"type": "legacy",
"database": {
"type": "pickleddb",
"host": "test.pkl"
}
})
storage = storage_factory.create()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == os.path.abspath("test.pkl")
def check_global_config(self, tmp_path, monkeypatch):
"""Check that global configuration is set properly"""
update_singletons()
assert orion.core.config.database.to_dict() == self.config["database"]
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
command = f"hunt --exp-max-trials 0 -n test python {script} -x~uniform(0,1)"
orion.core.cli.main(command.split(" "))
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == "dbhere.pkl"
def test_create_benchmark_bad_storage(self, benchmark_config_py):
"""Test error message if storage is not configured properly"""
name = "oopsie_bad_storage"
# Make sure there is no existing storage singleton
update_singletons()
with pytest.raises(NotImplementedError) as exc:
benchmark_config_py["storage"] = {
"type": "legacy",
"database": {
"type": "idontexist"
},
}
get_or_create_benchmark(**benchmark_config_py).close()
assert "Could not find implementation of Database, type = 'idontexist'" in str(
exc.value)
def test_workon_fail(self, monkeypatch):
"""Verify that storage is reverted if workon fails"""
def foo(x):
return [dict(name="result", type="objective", value=x * 2)]
def build_fail(*args, **kwargs):
raise RuntimeError("You shall not build!")
monkeypatch.setattr("orion.core.io.experiment_builder.build",
build_fail)
# Flush storage singleton
update_singletons()
with pytest.raises(RuntimeError) as exc:
experiment = workon(foo,
space={"x": "uniform(0, 10)"},
max_trials=5,
name="voici")
assert exc.match("You shall not build!")
# Verify that tmp storage was cleared
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
# Now test with a prior storage
with OrionState(storage={
"type": "legacy",
"database": {
"type": "EphemeralDB"
}
}):
storage = get_storage()
with pytest.raises(RuntimeError) as exc:
workon(foo,
space={"x": "uniform(0, 10)"},
max_trials=5,
name="voici")
assert exc.match("You shall not build!")
assert get_storage() is storage
def test_create_experiment_bad_storage(self):
"""Test error message if storage is not configured properly"""
name = "oopsie_bad_storage"
# Make sure there is no existing storage singleton
update_singletons()
with pytest.raises(NotImplementedError) as exc:
create_experiment(
name=name,
storage={
"type": "legacy",
"database": {
"type": "idontexist"
}
},
)
assert "Could not find implementation of Database, type = 'idontexist'" in str(
exc.value)
def test_create_experiment_debug_mode(self, tmp_path):
"""Test that EphemeralDB is used in debug mode whatever the storage config given"""
update_singletons()
conf_file = str(tmp_path / "db.pkl")
create_experiment(
config["name"],
space={"x": "uniform(0, 10)"},
storage={
"type": "legacy",
"database": {
"type": "pickleddb",
"host": conf_file
},
},
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
create_experiment(
config["name"],
space={"x": "uniform(0, 10)"},
storage={
"type": "legacy",
"database": {
"type": "pickleddb"
}
},
debug=True,
)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_get_from_args_debug_mode(script_path):
"""Try building experiment view in debug mode"""
update_singletons()
# Can't build view if none exist. It's fine we only want to test the storage creation.
with pytest.raises(NoConfigurationError):
experiment_builder.get_from_args({"name": "whatever"})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
update_singletons()
# Can't build view if none exist. It's fine we only want to test the storage creation.
with pytest.raises(NoConfigurationError):
experiment_builder.get_from_args({"name": "whatever", "debug": True})
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, EphemeralDB)
def test_create_experiment_no_storage(self, monkeypatch):
"""Test creation if storage is not configured"""
name = "oopsie_forgot_a_storage"
host = orion.core.config.storage.database.host
with OrionState(storage=orion.core.config.storage.to_dict()) as cfg:
# Reset the Storage and drop instances so that get_storage() would fail.
cfg.cleanup()
cfg.singletons = update_singletons()
# Make sure storage must be instantiated during `create_experiment()`
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
experiment = create_experiment(name=name,
space={"x": "uniform(0, 10)"})
assert isinstance(experiment._experiment._storage, Legacy)
assert isinstance(experiment._experiment._storage._db, PickledDB)
assert experiment._experiment._storage._db.host == host
def test_create_benchmark_no_storage(self, benchmark_config_py):
"""Test creation if storage is not configured"""
name = "oopsie_forgot_a_storage"
host = orion.core.config.storage.database.host
with OrionState(storage=orion.core.config.storage.to_dict()) as cfg:
# Reset the Storage and drop instances so that get_storage() would fail.
cfg.cleanup()
cfg.singletons = update_singletons()
# Make sure storage must be instantiated during `get_or_create_benchmark()`
with pytest.raises(SingletonNotInstantiatedError):
get_storage()
get_or_create_benchmark(**benchmark_config_py)
storage = get_storage()
assert isinstance(storage, Legacy)
assert isinstance(storage._db, PickledDB)
assert storage._db.host == host
def workon(
function, space, name="loop", algorithms=None, max_trials=None, max_broken=None
):
"""Optimize a function over a given search space
This will create a new experiment with an in-memory storage and optimize the given function
until `max_trials` is reached or the `algorithm` is done
(some algorithms like random search are never done).
For informations on how to fetch results, see
:py:class:`orion.client.experiment.ExperimentClient`.
.. note::
Each call to this function will create a separate in-memory storage.
Parameters
----------
name: str
Name of the experiment
version: int, optional
Version of the experiment. Defaults to last existing version for a given `name`
or 1 for new experiment.
space: dict, optional
Optimization space of the algorithm. Should have the form `dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
max_broken: int, optional
Number of broken trials for the experiment to be considered broken.
Raises
------
`NotImplementedError`
If the algorithm specified is not properly installed.
"""
# Clear singletons and keep pointers to restore them.
singletons = update_singletons()
try:
setup_storage(storage={"type": "legacy", "database": {"type": "EphemeralDB"}})
experiment = experiment_builder.build(
name,
version=1,
space=space,
algorithms=algorithms,
max_trials=max_trials,
max_broken=max_broken,
)
producer = Producer(experiment)
experiment_client = ExperimentClient(experiment, producer)
with experiment_client.tmp_executor("singleexecutor", n_workers=1):
experiment_client.workon(function, n_workers=1, max_trials=max_trials)
finally:
# Restore singletons
update_singletons(singletons)
return experiment_client
