def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = experiment_builder.build(name="test_single_exp")
x = {"name": "/x", "type": "real", "value": 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id, params=[x], status="completed", results=[results])
database_factory.create().write("trials", trial.to_dict())
return exp.configuration
def test_empty_first_call(self):
"""Should not be able to make first call without any arguments.
Hegelian Ontology Primer
------------------------
Type indeterminate <-> type abstracted from its property <-> No type
"""
with pytest.raises(SingletonNotInstantiatedError):
database_factory.create()
def with_experiment_missing_conf_file(monkeypatch, one_experiment):
"""Create an experiment without trials."""
exp = experiment_builder.build(name="test_single_exp", version=1)
conf_file = "idontexist.yaml"
exp.metadata["user_config"] = conf_file
exp.metadata["user_args"] += ["--config", conf_file]
database_factory.create().write(
"experiments", exp.configuration, query={"_id": exp.id}
)
return exp
def test_instantiation_and_singleton(self):
"""Test create just one object, that object persists between calls."""
database = database_factory.create(of_type="PickledDB",
name="orion_test")
assert isinstance(database, PickledDB)
assert database is database_factory.create()
with pytest.raises(SingletonAlreadyInstantiatedError):
database_factory.create("fire", [],
{"it_matters": "it's singleton"})
def three_family_with_trials(three_experiments_family, family_with_trials):
"""Create three experiments, all related, two direct children, with all types of trials."""
exp = experiment_builder.build(name="test_double_exp_child2")
x = {"name": "/x", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value + 0.75 # To avoid duplicates
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, z], status=status)
x_value += 1
database_factory.create().write("trials", trial.to_dict())
def single_without_success(one_experiment):
"""Create an experiment without a successful trial."""
statuses = list(Trial.allowed_stati)
statuses.remove("completed")
exp = experiment_builder.build(name="test_single_exp")
x = {"name": "/x", "type": "real"}
x_value = 0
for status in statuses:
x["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x_value += 1
database_factory.create().write("trials", trial.to_dict())
def test_singleton(self):
"""Test that MongoDB class is a singleton."""
orion_db = database_factory.create(
of_type="mongodb",
host="mongodb://localhost",
port=27017,
name="orion_test",
username="******",
password="******",
)
# reinit connection does not change anything
orion_db.initiate_connection()
orion_db.close_connection()
assert database_factory.create() is orion_db
def test_setup_database_custom():
"""Test setup with local configuration"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = database_factory.create()
assert isinstance(database, PickledDB)
assert database.host == os.path.abspath("test.pkl")
def __init__(self, database=None, setup=True):
if database is not None:
setup_database(database)
self._db = database_factory.create()
if setup:
self._setup_db()
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name="test_double_exp")
exp2 = experiment_builder.build(name="test_double_exp_child")
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x["value"] = x_value + 0.5 # To avoid duplicates
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
database_factory.create().write("trials", trial.to_dict())
database_factory.create().write("trials", trial2.to_dict())
def test_setup_database_bad_config_override():
"""Test setup with different config than existing singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = database_factory.create()
assert isinstance(database, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError):
setup_database({"type": "pickleddb", "host": "other.pkl"})
def test_setup_database_bad_override():
"""Test setup with different type than existing singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = database_factory.create()
assert isinstance(database, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError) as exc:
setup_database({"type": "mongodb"})
assert exc.match("A singleton instance of \(type: Database\)")
def three_family_branch_with_trials(
three_experiments_family_branch, family_with_trials
):
"""Create three experiments, all related, one child and one grandchild,
with all types of trials.
"""
exp = experiment_builder.build(name="test_double_exp_grand_child")
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value + 0.25 # To avoid duplicates
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, y, z], status=status)
x_value += 1
database_factory.create().write("trials", trial.to_dict())
def check_database_creation(self):
"""Check if database of specified type can be created."""
database = self.p_stage.db_config
db_type = database.pop("type")
try:
db = database_factory.create(db_type, **database)
except ValueError as ex:
raise CheckError(str(ex))
if not db.is_connected:
raise CheckError("Database failed to connect after creation.")
self.instance = db
return "Success", ""
def get_database():
"""Return current database
This is a wrapper around the Database Singleton object to provide
better error message when it is used without being initialized.
Raises
------
RuntimeError
If the underlying database was not initialized prior to calling this function
Notes
-----
To initialize the underlying database you must first call `Database(...)`
with the appropriate arguments for the chosen backend
"""
return database_factory.create()
def setup_database(config=None):
"""Create the Database instance from a configuration.
Parameters
----------
config: dict
Configuration for the database backend. If not defined, global configuration
is used.
"""
if config is None:
# TODO: How could we support orion.core.config.storage.database as well?
config = orion.core.config.database.to_dict()
db_opts = config
dbtype = db_opts.pop("type")
log.debug("Creating %s database client with args: %s", dbtype, db_opts)
return database_factory.create(dbtype, **db_opts)
def three_experiments_same_name_with_trials(two_experiments_same_name, storage):
"""Create three experiments with the same name but different versions."""
orion.core.cli.main(
[
"hunt",
"--init-only",
"--enable-evc",
"-n",
"test_single_exp",
"./black_box.py",
"--x~uniform(0,1)",
"--y~normal(0,1)",
"--z~+normal(0,1)",
]
)
ensure_deterministic_id("test_single_exp", storage, version=3)
exp = experiment_builder.build(name="test_single_exp", version=1)
exp2 = experiment_builder.build(name="test_single_exp", version=2)
exp3 = experiment_builder.build(name="test_single_exp", version=3)
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value + 0.1 # To avoid duplicates
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x], status=status)
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
trial3 = Trial(experiment=exp3.id, params=[x, y, z], status=status)
database_factory.create().write("trials", trial.to_dict())
database_factory.create().write("trials", trial2.to_dict())
database_factory.create().write("trials", trial3.to_dict())
x_value += 1
def test_setup_database_default(monkeypatch):
"""Test that database is setup using default config"""
update_singletons()
setup_database()
database = database_factory.create()
assert isinstance(database, PickledDB)
def unrelated_with_trials(family_with_trials, single_with_trials):
"""Create two unrelated experiments with all types of trials."""
exp = experiment_builder.build(name="test_double_exp_child")
database_factory.create().remove("trials", {"experiment": exp.id})
database_factory.create().remove("experiments", {"_id": exp.id})
def test_notfound_type_first_call(self):
"""Raise when supplying not implemented wrapper name."""
with pytest.raises(NotImplementedError) as exc_info:
database_factory.create("notfound")
assert "Database" in str(exc_info.value)