def build_view_from(self, cmdargs):
"""Build an experiment view based on full configuration.
.. seealso::
`orion.core.io.experiment_builder` for more information on the hierarchy of
configurations.
:class:`orion.core.worker.experiment.ExperimentView` for more information on the
experiment view object.
"""
local_config = self.fetch_full_config(cmdargs, use_db=False)
db_opts = local_config['database']
dbtype = db_opts.pop('type')
if local_config.get("debug"):
dbtype = "EphemeralDB"
# Information should be enough to infer experiment's name.
log.debug("Creating %s database client with args: %s", dbtype, db_opts)
try:
Database(of_type=dbtype, **db_opts)
except ValueError:
if Database().__class__.__name__.lower() != dbtype.lower():
raise
exp_name = local_config['name']
if exp_name is None:
raise RuntimeError("Could not infer experiment's name. "
"Please use either `name` cmd line arg or provide "
"one in orion's configuration file.")
return ExperimentView(local_config["name"], local_config.get('user', None))
def three_experiments_same_name_with_trials(two_experiments_same_name,
storage):
"""Create three experiments with the same name but different versions."""
orion.core.cli.main([
"hunt",
"--init-only",
"-n",
"test_single_exp",
"./black_box.py",
"--x~uniform(0,1)",
"--y~normal(0,1)",
"--z~+normal(0,1)",
])
ensure_deterministic_id("test_single_exp", storage, version=3)
exp = experiment_builder.build(name="test_single_exp", version=1)
exp2 = experiment_builder.build(name="test_single_exp", version=2)
exp3 = experiment_builder.build(name="test_single_exp", version=3)
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x], status=status)
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
trial3 = Trial(experiment=exp3.id, params=[x, y, z], status=status)
Database().write("trials", trial.to_dict())
Database().write("trials", trial2.to_dict())
Database().write("trials", trial3.to_dict())
x_value += 1
def __init__(self, database=None, setup=True):
if database is not None:
setup_database(database)
self._db = Database()
if setup:
self._setup_db()
def __init__(self, config=None, setup=True):
if config is not None:
setup_database(config)
self._db = Database()
if setup:
self._setup_db()
def create_db_instance(null_db_instances, clean_db):
"""Create and save a singleton database instance."""
try:
db = Database(of_type='MongoDB', name='orion_test',
username='******', password='******')
except ValueError:
db = Database()
return db
def __init__(self, name, user=None):
"""Initialize an Experiment object with primary key (:attr:`name`, :attr:`user`).
Try to find an entry in `Database` with such a key and config this object
from it import, if successful. Else, init with default/empty values and
insert new entry with this object's attributes in database.
.. note::
Practically initialization has not finished until `config`'s setter
is called.
:param name: Describe a configuration with a unique identifier per :attr:`user`.
:type name: str
"""
log.debug("Creating Experiment object with name: %s", name)
self._init_done = False
self._db = Database() # fetch database instance
self._setup_db() # build indexes for collections
self._id = None
self.name = name
self._node = None
self.refers = {}
if user is None:
user = getpass.getuser()
self.metadata = {'user': user}
self.pool_size = None
self.max_trials = None
self.algorithms = None
self.working_dir = None
self.producer = {'strategy': None}
config = self._db.read('experiments', {
'name': name,
'metadata.user': user
})
if config:
log.debug(
"Found existing experiment, %s, under user, %s, registered in database.",
name, user)
if len(config) > 1:
log.warning(
"Many (%s) experiments for (%s, %s) are available but "
"only the most recent one can be accessed. "
"Experiment branches will be supported soon.", len(config),
name, user)
config = sorted(config,
key=lambda x: x['metadata']['datetime'],
reverse=True)[0]
for attrname in self.__slots__:
if not attrname.startswith('_') and attrname in config:
setattr(self, attrname, config[attrname])
self._id = config['_id']
self._last_fetched = self.metadata.get("datetime",
datetime.datetime.utcnow())
def test_instantiation_and_singleton(self):
"""Test create just one object, that object persists between calls."""
database = Database(of_type="PickledDB", name="orion_test")
assert isinstance(database, PickledDB)
assert database is PickledDB()
assert database is Database()
with pytest.raises(SingletonAlreadyInstantiatedError):
Database("fire", [], {"it_matters": "it's singleton"})
def db_instance(null_db_instances):
"""Create and save a singleton database instance."""
try:
db = Database(of_type="MongoDB",
name="orion_test",
username="******",
password="******")
except ValueError:
db = Database()
return db
def test_instatiation_and_singleton(self):
"""Test create just one object, that object persists between calls."""
database = Database(of_type='MongoDB', name='orion_test',
username='******', password='******')
assert isinstance(database, MongoDB)
assert database is MongoDB()
assert database is Database()
with pytest.raises(ValueError) as exc_info:
Database('fire', [], {'it_matters': 'it\'s singleton'})
assert 'singleton' in str(exc_info.value)
def test_fix_lost_trials(self, hacked_exp, random_dt):
"""Test that a running trial with an old heartbeat is set to interrupted."""
exp_query = {'experiment': hacked_exp.id}
trial = hacked_exp.fetch_trials(exp_query)[0]
heartbeat = random_dt - datetime.timedelta(seconds=180)
Database().write('trials', {
'status': 'reserved',
'heartbeat': heartbeat
}, {
'experiment': hacked_exp.id,
'_id': trial.id
})
exp_query['status'] = 'reserved'
exp_query['_id'] = trial.id
assert len(hacked_exp.fetch_trials(exp_query)) == 1
hacked_exp.fix_lost_trials()
assert len(hacked_exp.fetch_trials(exp_query)) == 0
exp_query['status'] = 'interrupted'
assert len(hacked_exp.fetch_trials(exp_query)) == 1
def insert_trials(experiment_name, points, cmdconfig=None, raise_exc=True):
"""Insert sets of parameters manually, defined in `points`, as new trials
for the experiment name, `experiment_name`.
:param experiment_name: Name of the experiment which the new trials are
going to be associated with
:param points: list of tuples in agreement with experiment's parameter space
:param raise_exc: whether an inappropriate tuple of parameters will raise
an exception or it will be ignored
.. note:: If `raise_exc` is True, no set of parameters will be inserted. If
it is False, only the valid ones will be inserted; the rest will be ignored.
.. note:: This cannot be used to prepopulate a future experiment. So,
an experiment with `experiment_name` should already be configured in
the database.
"""
cmdconfig = cmdconfig if cmdconfig else {}
config = resolve_config.fetch_default_options(
) # Get database perhaps from default locs
config = resolve_config.merge_env_vars(
config) # Get database perhaps from env vars
tmpconfig = resolve_config.merge_orion_config(config, dict(), cmdconfig,
dict())
db_opts = tmpconfig['database']
db_type = db_opts.pop('type')
try:
Database(of_type=db_type, **db_opts)
except SingletonError:
pass
experiment = Experiment(experiment_name)
# Configuration is completely taken from the database
if experiment.status is None:
raise ValueError(
"No experiment named '{}' could be found.".format(experiment_name))
experiment.configure(experiment.configuration)
valid_points = []
print(experiment.space)
for point in points:
try:
assert point in experiment.space
valid_points.append(point)
except AssertionError:
if raise_exc:
raise
if not valid_points:
return
new_trials = list(
map(lambda data: format_trials.tuple_to_trial(data, experiment.space),
valid_points))
experiment.register_trials(new_trials)
def _infer_experiment(cmd_args, file_config):
# Initialize configuration dictionary.
# Fetch info from defaults and configurations from default locations.
expconfig = resolve_config.fetch_default_options()
# Fetch orion system variables (database and resource information)
# See :const:`orion.core.io.resolve_config.ENV_VARS` for environmental
# variables used
expconfig = resolve_config.merge_env_vars(expconfig)
# Initialize singleton database object
tmpconfig = resolve_config.merge_orion_config(expconfig, dict(),
file_config, cmd_args)
db_opts = tmpconfig['database']
dbtype = db_opts.pop('type')
log.debug("Creating %s database client with args: %s", dbtype, db_opts)
Database(of_type=dbtype, **db_opts)
# Information should be enough to infer experiment's name.
exp_name = tmpconfig['name']
if exp_name is None:
raise RuntimeError("Could not infer experiment's name. "
"Please use either `name` cmd line arg or provide "
"one in orion's configuration file.")
experiment = create_experiment(exp_name, expconfig, file_config, cmd_args)
return experiment
def main(args):
"""List all experiments inside database."""
builder = ExperimentBuilder()
config = builder.fetch_full_config(args, use_db=False)
builder.setup_database(config)
query = {}
if args['name']:
query['name'] = args['name']
experiments = Database().read("experiments", query)
if args['name']:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp['refers'].get('root_id', exp['_id']) == exp['_id']
]
for root_experiment in root_experiments:
root = EVCBuilder().build_view_from({
'name': root_experiment['name']
}).node
print_tree(root)
def consume(self, trial):
"""Execute user's script as a block box using the options contained
within `trial`.
:type trial: `orion.core.worker.trial.Trial`
"""
log.debug("### Create new directory at '%s':", self.working_dir)
temp_dir = self.experiment.working_dir is None
prefix = self.experiment.name + "_"
suffix = trial.id
with WorkingDir(self.working_dir, temp_dir,
prefix=prefix, suffix=suffix) as workdirname:
log.debug("## New consumer context: %s", workdirname)
completed_trial = self._consume(trial, workdirname)
if completed_trial is not None:
log.debug("### Register successfully evaluated %s.", completed_trial)
self.experiment.push_completed_trial(completed_trial)
else:
log.debug("### Save %s as broken.", trial)
trial.status = 'broken'
Database().write('trials', trial.to_dict(),
query={'_id': trial.id})
def infer_experiment():
"""Use `orion.core.resolve_config` to organize how configuration is built."""
# Fetch experiment name, user's script path and command line arguments
# Use `-h` option to show help
cmdargs, cmdconfig = resolve_config.fetch_orion_args(CLI_DOC_HEADER)
# Initialize configuration dictionary.
# Fetch info from defaults and configurations from default locations.
expconfig = resolve_config.fetch_default_options()
# Fetch orion system variables (database and resource information)
# See :const:`orion.core.io.resolve_config.ENV_VARS` for environmental
# variables used
expconfig = resolve_config.merge_env_vars(expconfig)
# Initialize singleton database object
tmpconfig = resolve_config.merge_orion_config(expconfig, dict(),
cmdconfig, cmdargs)
db_opts = tmpconfig['database']
dbtype = db_opts.pop('type')
log.debug("Creating %s database client with args: %s", dbtype, db_opts)
Database(of_type=dbtype, **db_opts)
# Information should be enough to infer experiment's name.
exp_name = tmpconfig['name']
if exp_name is None:
raise RuntimeError("Could not infer experiment's name. "
"Please use either `name` cmd line arg or provide "
"one in orion's configuration file.")
experiment = create_experiment(exp_name, expconfig, cmdconfig, cmdargs)
return experiment
def test_setup_database_custom():
"""Test setup with local configuration"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = Database()
assert isinstance(database, PickledDB)
assert database.host == "test.pkl"
def test_workon(database):
"""Test scenario having a configured experiment already setup."""
try:
Database(of_type='MongoDB',
name='orion_test',
username='******',
password='******')
except (TypeError, ValueError):
pass
experiment = Experiment('voila_voici')
config = experiment.configuration
config['algorithms'] = {'gradient_descent': {'learning_rate': 0.1}}
config['pool_size'] = 1
config['max_trials'] = 100
config['metadata']['user_script'] = os.path.abspath(
os.path.join(os.path.dirname(__file__), "black_box.py"))
config['metadata']['user_args'] = ["-x~uniform(-50, 50)"]
experiment.configure(config)
workon(experiment)
exp = list(database.experiments.find({'name': 'voila_voici'}))
assert len(exp) == 1
exp = exp[0]
assert '_id' in exp
exp_id = exp['_id']
assert exp['name'] == 'voila_voici'
assert exp['pool_size'] == 1
assert exp['max_trials'] == 100
assert exp['algorithms'] == {
'gradient_descent': {
'learning_rate': 0.1,
'dx_tolerance': 1e-7
}
}
assert 'user' in exp['metadata']
assert 'datetime' in exp['metadata']
assert 'user_script' in exp['metadata']
assert os.path.isabs(exp['metadata']['user_script'])
assert exp['metadata']['user_args'] == ['-x~uniform(-50, 50)']
trials = list(database.trials.find({'experiment': exp_id}))
assert len(trials) <= 15
trials = list(sorted(trials, key=lambda trial: trial['submit_time']))
assert trials[-1]['status'] == 'completed'
for result in trials[-1]['results']:
assert result['type'] != 'constraint'
if result['type'] == 'objective':
assert abs(result['value'] - 23.4) < 1e-6
assert result['name'] == 'example_objective'
elif result['type'] == 'gradient':
res = numpy.asarray(result['value'])
assert 0.1 * numpy.sqrt(res.dot(res)) < 1e-7
assert result['name'] == 'example_gradient'
params = trials[-1]['params']
assert len(params) == 1
assert params[0]['name'] == '/x'
assert params[0]['type'] == 'real'
assert (params[0]['value'] - 34.56789) < 1e-5
def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = ExperimentBuilder().build_from({'name': 'test_single_exp'})
x = {'name': '/x', 'type': 'real', 'value': 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results])
Database().write('trials', trial.to_dict())
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name="test_double_exp")
exp2 = experiment_builder.build(name="test_double_exp_child")
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x["value"] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
Database().write("trials", trial2.to_dict())
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name='test_double_exp')
exp2 = experiment_builder.build(name='test_double_exp_child')
x = {'name': '/x', 'type': 'real'}
y = {'name': '/y', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
y['value'] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x['value'] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
Database().write('trials', trial2.to_dict())
def test_setup_database_bad_config_override():
"""Test setup with different config than existing singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = Database()
assert isinstance(database, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError):
setup_database({"type": "pickleddb", "host": "other.pkl"})
def with_experiment_missing_conf_file(monkeypatch, one_experiment):
"""Create an experiment without trials."""
exp = experiment_builder.build(name="test_single_exp", version=1)
conf_file = "idontexist.yaml"
exp.metadata["user_config"] = conf_file
exp.metadata["user_args"] += ["--config", conf_file]
Database().write("experiments", exp.configuration, query={"_id": exp.id})
return exp
def test_setup_database_bad_override():
"""Test setup with different type than existing singleton"""
update_singletons()
setup_database({"type": "pickleddb", "host": "test.pkl"})
database = Database()
assert isinstance(database, PickledDB)
with pytest.raises(SingletonAlreadyInstantiatedError) as exc:
setup_database({"type": "mongodb"})
assert exc.match("A singleton instance of \(type: Database\)")
def test_empty_first_call(self):
"""Should not be able to make first call without any arguments.
Hegelian Ontology Primer
------------------------
Type indeterminate <-> type abstracted from its property <-> No type
"""
with pytest.raises(SingletonNotInstantiatedError):
Database()
def test_empty_first_call(self):
"""Should not be able to make first call without any arguments.
Hegelian Ontology Primer
------------------------
Type indeterminate <-> type abstracted from its property <-> No type
"""
with pytest.raises(TypeError) as exc_info:
Database()
assert 'positional argument' in str(exc_info.value)
def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = experiment_builder.build(name="test_single_exp")
x = {"name": "/x", "type": "real", "value": 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id,
params=[x],
status="completed",
results=[results])
Database().write("trials", trial.to_dict())
def trial(exp):
"""Return a Trial which is registered in DB."""
trial = tuple_to_trial((1.0, ), exp.space)
heartbeat = datetime.datetime.utcnow()
trial.experiment = exp.id
trial.status = 'reserved'
trial.heartbeat = heartbeat
Database().write('trials', trial.to_dict())
return trial
def setup_database(self, config):
"""Create the Database instance from a configuration.
Parameters
----------
config: dict
Configuration for the database.
"""
db_opts = config['database']
dbtype = db_opts.pop('type')
if config.get("debug"):
dbtype = "EphemeralDB"
log.debug("Creating %s database client with args: %s", dbtype, db_opts)
try:
Database(of_type=dbtype, **db_opts)
except ValueError:
if Database().__class__.__name__.lower() != dbtype.lower():
raise
def test_reserve_none(self):
"""Find nothing, return None."""
try:
Database(of_type='MongoDB',
name='orion_test',
username='******',
password='******')
except (TypeError, ValueError):
pass
exp = Experiment('supernaekei')
trial = exp.reserve_trial()
assert trial is None
def test_broken_property(hacked_exp):
"""Check experiment stopping conditions for maximum number of broken."""
assert not hacked_exp.is_broken
trials = hacked_exp.fetch_trials({})[:3]
query = {'experiment': hacked_exp.id}
for trial in trials:
query['_id'] = trial.id
Database().write('trials', {'status': 'broken'}, query)
assert hacked_exp.is_broken
