class Legacy(BaseStorageProtocol):
"""Legacy protocol, store all experiments and trials inside the Database()
Parameters
----------
config: Dict
configuration definition passed from experiment_builder
to storage factory to legacy constructor.
See `~orion.io.database.Database` for more details
setup: bool
Setup the database (create indexes)
"""
def __init__(self, database=None, setup=True):
if database is not None:
setup_database(database)
self._db = Database()
if setup:
self._setup_db()
def _setup_db(self):
"""Database index setup"""
if backward.db_is_outdated(self._db):
raise OutdatedDatabaseError(
"The database is outdated. You can upgrade it with the "
"command `orion db upgrade`.")
self._db.index_information("experiment")
self._db.ensure_index(
"experiments",
[("name", Database.ASCENDING), ("version", Database.ASCENDING)],
unique=True,
)
self._db.ensure_index("experiments", "metadata.datetime")
self._db.ensure_index("benchmarks", "name", unique=True)
self._db.ensure_index("trials", "experiment")
self._db.ensure_index("trials", "status")
self._db.ensure_index("trials", "results")
self._db.ensure_index("trials", "start_time")
self._db.ensure_index("trials", [("end_time", Database.DESCENDING)])
def create_benchmark(self, config):
"""Insert a new benchmark inside the database"""
return self._db.write("benchmarks", data=config, query=None)
def fetch_benchmark(self, query, selection=None):
"""Fetch all benchmarks that match the query"""
return self._db.read("benchmarks", query, selection)
def create_experiment(self, config):
"""See :func:`orion.storage.base.BaseStorageProtocol.create_experiment`"""
return self._db.write("experiments", data=config, query=None)
def delete_experiment(self, experiment=None, uid=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.delete_experiment`"""
uid = get_uid(experiment, uid)
return self._db.remove("experiments", query={"_id": uid})
def update_experiment(self,
experiment=None,
uid=None,
where=None,
**kwargs):
"""See :func:`orion.storage.base.BaseStorageProtocol.update_experiment`"""
uid = get_uid(experiment, uid)
if where is None:
where = dict()
if uid is not None:
where["_id"] = uid
return self._db.write("experiments", data=kwargs, query=where)
def fetch_experiments(self, query, selection=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_experiments`"""
return self._db.read("experiments", query, selection)
def fetch_trials(self, experiment=None, uid=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_trials`"""
uid = get_uid(experiment, uid)
return self._fetch_trials(dict(experiment=uid))
def _fetch_trials(self, query, selection=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_trials`"""
def sort_key(item):
submit_time = item.submit_time
if submit_time is None:
return 0
return submit_time
trials = Trial.build(
self._db.read("trials", query=query, selection=selection))
trials.sort(key=sort_key)
return trials
def register_trial(self, trial):
"""See :func:`orion.storage.base.BaseStorageProtocol.register_trial`"""
self._db.write("trials", trial.to_dict())
return trial
def delete_trials(self, experiment=None, uid=None, where=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.delete_trials`"""
uid = get_uid(experiment, uid)
if where is None:
where = dict()
if uid is not None:
where["experiment"] = uid
return self._db.remove("trials", query=where)
def register_lie(self, trial):
"""See :func:`orion.storage.base.BaseStorageProtocol.register_lie`"""
return self._db.write("lying_trials", trial.to_dict())
def retrieve_result(self, trial, results_file=None, **kwargs):
"""Parse the results file that was generated by the trial process.
Parameters
----------
trial: Trial
The trial object to be updated
results_file: str
the file handle to read the result from
Returns
-------
returns the updated trial object
Notes
-----
This does not update the database!
"""
if results_file is None:
return trial
try:
results = JSONConverter().parse(results_file.name)
except json.decoder.JSONDecodeError:
raise MissingResultFile()
trial.results = [
Trial.Result(name=res["name"],
type=res["type"],
value=res["value"]) for res in results
]
return trial
def get_trial(self, trial=None, uid=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.get_trial`"""
if trial is not None and uid is not None:
assert trial._id == uid
if uid is None:
if trial is None:
raise MissingArguments("Either `trial` or `uid` should be set")
uid = trial.id
result = self._db.read("trials", {"_id": uid})
if not result:
return None
return Trial(**result[0])
def update_trials(self, experiment=None, uid=None, where=None, **kwargs):
"""See :func:`orion.storage.base.BaseStorageProtocol.update_trials`"""
uid = get_uid(experiment, uid)
if where is None:
where = dict()
where["experiment"] = uid
return self._db.write("trials", data=kwargs, query=where)
def update_trial(self, trial=None, uid=None, where=None, **kwargs):
"""See :func:`orion.storage.base.BaseStorageProtocol.update_trial`"""
uid = get_uid(trial, uid)
if where is None:
where = dict()
where["_id"] = uid
return self._db.write("trials", data=kwargs, query=where)
def fetch_lost_trials(self, experiment):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_lost_trials`"""
heartbeat = orion.core.config.worker.heartbeat
threshold = datetime.datetime.utcnow() - datetime.timedelta(
seconds=heartbeat * 5)
lte_comparison = {"$lte": threshold}
query = {
"experiment": experiment._id,
"status": "reserved",
"heartbeat": lte_comparison,
}
return self._fetch_trials(query)
def push_trial_results(self, trial):
"""See :func:`orion.storage.base.BaseStorageProtocol.push_trial_results`"""
rc = self.update_trial(trial,
**trial.to_dict(),
where={
"_id": trial.id,
"status": "reserved"
})
if not rc:
raise FailedUpdate()
return rc
def set_trial_status(self, trial, status, heartbeat=None):
"""See :func:`orion.storage.base.BaseStorageProtocol.set_trial_status`"""
if heartbeat is None:
heartbeat = datetime.datetime.utcnow()
update = dict(status=status,
heartbeat=heartbeat,
experiment=trial.experiment)
validate_status(status)
rc = self.update_trial(trial,
**update,
where={
"status": trial.status,
"_id": trial.id
})
if not rc:
raise FailedUpdate()
trial.status = status
def fetch_pending_trials(self, experiment):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_pending_trials`"""
query = dict(
experiment=experiment._id,
status={"$in": ["new", "suspended", "interrupted"]},
)
return self._fetch_trials(query)
def reserve_trial(self, experiment):
"""See :func:`orion.storage.base.BaseStorageProtocol.reserve_trial`"""
query = dict(
experiment=experiment._id,
status={"$in": ["interrupted", "new", "suspended"]},
)
# read and write works on a single document
now = datetime.datetime.utcnow()
trial = self._db.read_and_write(
"trials",
query=query,
data=dict(status="reserved", start_time=now, heartbeat=now),
)
if trial is None:
return None
return Trial(**trial)
def fetch_noncompleted_trials(self, experiment):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_noncompleted_trials`"""
query = dict(experiment=experiment._id, status={"$ne": "completed"})
return self._fetch_trials(query)
def count_completed_trials(self, experiment):
"""See :func:`orion.storage.base.BaseStorageProtocol.count_completed_trials`"""
query = dict(experiment=experiment._id, status="completed")
return self._db.count("trials", query)
def count_broken_trials(self, experiment):
"""See :func:`orion.storage.base.BaseStorageProtocol.count_broken_trials`"""
query = dict(experiment=experiment._id, status="broken")
return self._db.count("trials", query)
def update_heartbeat(self, trial):
"""Update trial's heartbeat"""
return self.update_trial(trial,
heartbeat=datetime.datetime.utcnow(),
status="reserved")
def fetch_trials_by_status(self, experiment, status):
"""See :func:`orion.storage.base.BaseStorageProtocol.fetch_trials_by_status`"""
query = dict(experiment=experiment._id, status=status)
return self._fetch_trials(query)
class Experiment(object):
"""Represents an entry in database/experiments collection.
Attributes
----------
name : str
Unique identifier for this experiment per `user`.
refers : dict or list of `Experiment` objects, after initialization is done.
A dictionary pointing to a past `Experiment` name, ``refers[name]``, whose
trials we want to add in the history of completed trials we want to re-use.
metadata : dict
Contains managerial information about this `Experiment`.
pool_size : int
How many workers can participate asynchronously in this `Experiment`.
max_trials : int
How many trials must be evaluated, before considering this `Experiment` done.
This attribute can be updated if the rest of the experiment configuration
is the same. In that case, if trying to set to an already set experiment,
it will overwrite the previous one.
status : str
A keyword among {*'pending'*, *'done'*, *'broken'*} indicating
how **Oríon** considers the current `Experiment`. This attribute cannot
be set from an orion configuration.
* 'pending' : Denotes an experiment with valid configuration which is
currently being handled by **Oríon**.
* 'done' : Denotes an experiment which has completed `max_trials` number
of parameter evaluations and is not *pending*.
* 'broken' : Denotes an experiment which stopped unsuccessfully due to
unexpected behaviour.
algorithms : dict of dicts or an `PrimaryAlgo` object, after initialization is done.
Complete specification of the optimization and dynamical procedures taking
place in this `Experiment`.
Metadata
--------
user : str
System user currently owning this running process, the one who invoked **Oríon**.
datetime : `datetime.datetime`
When was this particular configuration submitted to the database.
orion_version : str
Version of **Oríon** which suggested this experiment. `user`'s current
**Oríon** version.
user_script : str
Full absolute path to `user`'s executable.
user_args : list of str
Contains separate arguments to be passed when invoking `user_script`,
possibly templated for **Oríon**.
user_vcs : str, optional
User's version control system for this executable's code repository.
user_version : str, optional
Current user's repository version.
user_commit_hash : str, optional
Current `Experiment`'s commit hash for **Oríon**'s invocation.
"""
__slots__ = ('name', 'refers', 'metadata', 'pool_size', 'max_trials',
'status', 'algorithms', '_db', '_init_done', '_id',
'_last_fetched')
# 'status' should not be in config
non_forking_attrs = ('status', 'pool_size', 'max_trials')
def __init__(self, name):
"""Initialize an Experiment object with primary key (:attr:`name`, :attr:`user`).
Try to find an entry in `Database` with such a key and config this object
from it import, if successful. Else, init with default/empty values and
insert new entry with this object's attributes in database.
.. note::
Practically initialization has not finished until `config`'s setter
is called.
:param name: Describe a configuration with a unique identifier per :attr:`user`.
:type name: str
"""
log.debug("Creating Experiment object with name: %s", name)
self._init_done = False
self._db = Database() # fetch database instance
self._setup_db() # build indexes for collections
self._id = None
self.name = name
self.refers = None
user = getpass.getuser()
stamp = datetime.datetime.utcnow()
self.metadata = {'user': user, 'datetime': stamp}
self.pool_size = None
self.max_trials = None
self.status = None
self.algorithms = None
config = self._db.read('experiments', {
'name': name,
'metadata.user': user
})
if config:
log.debug(
"Found existing experiment, %s, under user, %s, registered in database.",
name, user)
if len(config) > 1:
log.warning(
"Many (%s) experiments for (%s, %s) are available but "
"only the most recent one can be accessed. "
"Experiment forks will be supported soon.", len(config),
name, user)
config = sorted(config,
key=lambda x: x['metadata']['datetime'],
reverse=True)[0]
for attrname in self.__slots__:
if not attrname.startswith('_'):
setattr(self, attrname, config[attrname])
self._id = config['_id']
self._last_fetched = self.metadata['datetime']
def _setup_db(self):
self._db.ensure_index('experiments',
[('name', Database.ASCENDING),
('metadata.user', Database.ASCENDING)],
unique=True)
self._db.ensure_index('experiments', 'status')
self._db.ensure_index('trials', 'experiment')
self._db.ensure_index('trials', 'status')
self._db.ensure_index('trials', 'results')
self._db.ensure_index('trials', 'start_time')
self._db.ensure_index('trials', [('end_time', Database.DESCENDING)])
def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError(
"Argument `score_handle` must be callable with a `Trial`.")
query = dict(experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']})
new_trials = Trial.build(self._db.read('trials', query))
if not new_trials:
return None
if score_handle is not None and self.space:
scores = list(
map(score_handle,
map(lambda x: trial_to_tuple(x, self.space), new_trials)))
scored_trials = zip(scores, new_trials)
best_trials = filter(lambda st: st[0] == max(scores),
scored_trials)
new_trials = list(zip(*best_trials))[1]
elif score_handle is not None:
log.warning(
"While reserving trial: `score_handle` was provided, but "
"parameter space has not been defined yet.")
selected_trial = random.sample(new_trials, 1)[0]
# Query on status to ensure atomicity. If another process change the
# status meanwhile, read_and_write will fail, because query will fail.
query = {'_id': selected_trial.id, 'status': selected_trial.status}
update = dict(status='reserved')
if selected_trial.status == 'new':
update["start_time"] = datetime.datetime.utcnow()
selected_trial_dict = self._db.read_and_write('trials',
query=query,
data=update)
if selected_trial_dict is None:
selected_trial = self.reserve_trial(score_handle=score_handle)
else:
selected_trial = Trial(**selected_trial_dict)
return selected_trial
def push_completed_trial(self, trial):
"""Inform database about an evaluated `trial` with results.
:param trial: Corresponds to a successful evaluation of a particular run.
:type trial: `Trial`
.. note:: Change status from *reserved* to *completed*.
"""
trial.end_time = datetime.datetime.utcnow()
trial.status = 'completed'
self._db.write('trials', trial.to_dict(), query={'_id': trial.id})
def register_trials(self, trials):
"""Inform database about *new* suggested trial with specific parameter
values. Each of them correspond to a different possible run.
:type trials: list of `Trial`
"""
stamp = datetime.datetime.utcnow()
for trial in trials:
trial.experiment = self._id
trial.status = 'new'
trial.submit_time = stamp
trials_dicts = list(map(lambda x: x.to_dict(), trials))
self._db.write('trials', trials_dicts)
def fetch_completed_trials(self):
"""Fetch recent completed trials that this `Experiment` instance has not
yet seen.
.. note:: It will return only those with `Trial.end_time` after
`_last_fetched`, for performance reasons.
:return: list of completed `Trial` objects
"""
query = dict(experiment=self._id,
status='completed',
end_time={'$gte': self._last_fetched})
completed_trials = Trial.build(self._db.read('trials', query))
self._last_fetched = datetime.datetime.utcnow()
return completed_trials
@property
def is_done(self):
"""Return True, if this experiment is considered to be finished.
1. Count how many trials have been completed and compare with `max_trials`.
2. Ask `algorithms` if they consider there is a chance for further improvement.
.. note:: To be used as a terminating condition in a ``Worker``.
"""
query = dict(experiment=self._id, status='completed')
num_completed_trials = self._db.count('trials', query)
if num_completed_trials >= self.max_trials or \
(self._init_done and self.algorithms.is_done):
self._db.write('experiments', {'status': 'done'},
{'_id': self._id})
return True
return False
@property
def space(self):
"""Return problem's parameter `orion.algo.space.Space`.
.. note:: It will return None, if experiment init is not done.
"""
if self._init_done:
return self.algorithms.space
return None
@property
def configuration(self):
"""Return a copy of an `Experiment` configuration as a dictionary."""
config = dict()
for attrname in self.__slots__:
if attrname.startswith('_'):
continue
attribute = getattr(self, attrname)
if self._init_done and attrname == 'algorithms':
config[attrname] = attribute.configuration
else:
config[attrname] = attribute
# Reason for deepcopy is that some attributes are dictionaries
# themselves, we don't want to accidentally change the state of this
# object from a getter.
return copy.deepcopy(config)
def configure(self, config):
"""Set `Experiment` by overwriting current attributes.
If `Experiment` was already set and an overwrite is needed, a *fork*
is advised with a different :attr:`name` for this particular configuration.
.. note:: Calling this property is necessary for an experiment's
initialization process to be considered as done. But it can be called
only once.
"""
if self._init_done:
raise RuntimeError(
"Configuration is done; cannot reset an Experiment.")
# Copy and simulate instantiating given configuration
experiment = Experiment(self.name)
experiment._instantiate_config(self.configuration)
experiment._instantiate_config(config)
experiment._init_done = True
experiment.status = 'pending'
# If status is None in this object, then database did not hit a config
# with same (name, user's name) pair. Everything depends on the user's
# orion_config to set.
if self.status is None:
if config['name'] != self.name or \
config['metadata']['user'] != self.metadata['user'] or \
config['metadata']['datetime'] != self.metadata['datetime']:
raise ValueError(
"Configuration given is inconsistent with this Experiment."
)
is_new = True
else:
# Fork if it is needed
is_new = self._is_different_from(experiment.configuration)
if is_new:
experiment._fork_config(config)
final_config = experiment.configuration
self._instantiate_config(final_config)
self._init_done = True
self.status = 'pending'
# If everything is alright, push new config to database
if is_new:
# This will raise DuplicateKeyError if a concurrent experiment with
# identical (name, metadata.user) is written first in the database.
self._db.write('experiments', final_config)
# XXX: Reminder for future DB implementations:
# MongoDB, updates an inserted dict with _id, so should you :P
self._id = final_config['_id']
else:
# Writing the final config to an already existing experiment raises
# a DuplicatKeyError because of the embedding id `metadata.user`.
# To avoid this `final_config["name"]` is popped out before
# `db.write()`, thus seamingly breaking the compound index
# `(name, metadata.user)`
final_config.pop("name")
self._db.write('experiments', final_config, {'_id': self._id})
@property
def stats(self):
"""Calculate a stats dictionary for this particular experiment.
Returns
-------
stats : dict
Stats
-----
trials_completed : int
Number of completed trials
best_trials_id : int
Unique identifier of the `Trial` object in the database which achieved
the best known objective result.
best_evaluation : float
Evaluation score of the best trial
start_time : `datetime.datetime`
When Experiment was first dispatched and started running.
finish_time : `datetime.datetime`
When Experiment reached terminating condition and stopped running.
duration : `datetime.timedelta`
Elapsed time.
"""
query = dict(experiment=self._id, status='completed')
completed_trials = self._db.read('trials',
query,
selection={
'_id': 1,
'end_time': 1,
'results': 1
})
stats = dict()
stats['trials_completed'] = len(completed_trials)
stats['best_trials_id'] = None
trial = Trial(**completed_trials[0])
stats['best_evaluation'] = trial.objective.value
stats['best_trials_id'] = trial.id
stats['start_time'] = self.metadata['datetime']
stats['finish_time'] = stats['start_time']
for trial in completed_trials:
trial = Trial(**trial)
# All trials are going to finish certainly after the start date
# of the experiment they belong to
if trial.end_time > stats['finish_time']: # pylint:disable=no-member
stats['finish_time'] = trial.end_time
objective = trial.objective.value
if objective < stats['best_evaluation']:
stats['best_evaluation'] = objective
stats['best_trials_id'] = trial.id
stats['duration'] = stats['finish_time'] - stats['start_time']
return stats
def _instantiate_config(self, config):
"""Check before dispatching experiment whether configuration corresponds
to a executable experiment environment.
1. Check `refers` and instantiate `Experiment` objects from it. (TODO)
2. Try to build parameter space from user arguments.
3. Check whether configured algorithms correspond to [known]/valid
implementations of the ``Algorithm`` class. Instantiate these objects.
4. Check if experiment `is_done`, prompt for larger `max_trials` if it is. (TODO)
"""
# Just overwrite everything else given
for section, value in config.items():
if section == 'status':
continue
if section not in self.__slots__:
log.warning(
"Found section '%s' in configuration. Experiments "
"do not support this option. Ignoring.", section)
continue
if section.startswith('_'):
log.warning(
"Found section '%s' in configuration. "
"Cannot set private attributes. Ignoring.", section)
continue
setattr(self, section, value)
try:
space = SpaceBuilder().build_from(config['metadata']['user_args'])
if not space:
raise ValueError(
"Parameter space is empty. There is nothing to optimize.")
# Instantiate algorithms
self.algorithms = PrimaryAlgo(space, self.algorithms)
except KeyError:
pass
def _fork_config(self, config):
"""Ask for a different identifier for this experiment. Set :attr:`refers`
key to previous experiment's name, the one that we forked from.
:param config: Conflicting configuration that will change based on prompt.
"""
raise NotImplementedError()
def _is_different_from(self, config):
"""Return True, if current `Experiment`'s configuration as described by
its attributes is different from the one suggested in `config`.
"""
is_diff = False
for section, value in config.items():
if section in self.non_forking_attrs or \
section not in self.__slots__ or \
section.startswith('_'):
continue
item = getattr(self, section)
if item != value:
log.warning(
"Config given is different from config found in db at section: %s",
section)
log.warning("Config+ :\n%s", value)
log.warning("Config- :\n%s", item)
is_diff = True
break
return is_diff
class Legacy(BaseStorageProtocol):
"""Legacy protocol, store all experiments and trials inside the Database()
Parameters
----------
config: Dict
configuration definition passed from experiment_builder
to storage factory to legacy constructor.
See `~orion.io.database.Database` for more details
setup: bool
Setup the database (create indexes)
"""
def __init__(self, database=None, setup=True):
if database is not None:
setup_database(database)
self._db = Database()
if setup:
self._setup_db()
def _setup_db(self):
"""Database index setup"""
if backward.db_is_outdated(self._db):
raise OutdatedDatabaseError(
"The database is outdated. You can upgrade it with the "
"command `orion db upgrade`.")
self._db.index_information('experiment')
self._db.ensure_index('experiments', [('name', Database.ASCENDING),
('version', Database.ASCENDING)],
unique=True)
self._db.ensure_index('experiments', 'metadata.datetime')
self._db.ensure_index('trials', 'experiment')
self._db.ensure_index('trials', 'status')
self._db.ensure_index('trials', 'results')
self._db.ensure_index('trials', 'start_time')
self._db.ensure_index('trials', [('end_time', Database.DESCENDING)])
def create_experiment(self, config):
"""See :func:`~orion.storage.BaseStorageProtocol.create_experiment`"""
return self._db.write('experiments', data=config, query=None)
def update_experiment(self,
experiment=None,
uid=None,
where=None,
**kwargs):
"""See :func:`~orion.storage.BaseStorageProtocol.update_experiment`"""
if experiment is not None and uid is not None:
assert experiment._id == uid
if uid is None:
if experiment is None:
raise MissingArguments(
'Either `experiment` or `uid` should be set')
uid = experiment._id
if where is None:
where = dict()
where['_id'] = uid
return self._db.write('experiments', data=kwargs, query=where)
def fetch_experiments(self, query, selection=None):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_experiments`"""
return self._db.read('experiments', query, selection)
def fetch_trials(self, experiment=None, uid=None):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_trials`"""
if experiment is not None and uid is not None:
assert experiment._id == uid
if uid is None:
if experiment is None:
raise MissingArguments(
'Either `experiment` or `uid` should be set')
uid = experiment._id
return self._fetch_trials(dict(experiment=uid))
def _fetch_trials(self, query, selection=None):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_trials`"""
def sort_key(item):
submit_time = item.submit_time
if submit_time is None:
return 0
return submit_time
trials = Trial.build(
self._db.read('trials', query=query, selection=selection))
trials.sort(key=sort_key)
return trials
def register_trial(self, trial):
"""See :func:`~orion.storage.BaseStorageProtocol.register_trial`"""
self._db.write('trials', trial.to_dict())
return trial
def register_lie(self, trial):
"""See :func:`~orion.storage.BaseStorageProtocol.register_lie`"""
return self._db.write('lying_trials', trial.to_dict())
def retrieve_result(self, trial, results_file=None, **kwargs):
"""Parse the results file that was generated by the trial process.
Parameters
----------
trial: Trial
The trial object to be updated
results_file: str
the file handle to read the result from
Returns
-------
returns the updated trial object
Notes
-----
This does not update the database!
"""
if results_file is None:
return trial
try:
results = JSONConverter().parse(results_file.name)
except json.decoder.JSONDecodeError:
raise MissingResultFile()
trial.results = [
Trial.Result(name=res['name'],
type=res['type'],
value=res['value']) for res in results
]
return trial
def get_trial(self, trial=None, uid=None):
"""See :func:`~orion.storage.BaseStorageProtocol.get_trial`"""
if trial is not None and uid is not None:
assert trial._id == uid
if uid is None:
if trial is None:
raise MissingArguments('Either `trial` or `uid` should be set')
uid = trial.id
result = self._db.read('trials', {'_id': uid})
if not result:
return None
return Trial(**result[0])
def _update_trial(self, trial, where=None, **kwargs):
"""See :func:`~orion.storage.BaseStorageProtocol.update_trial`"""
if where is None:
where = dict()
where['_id'] = trial.id
return self._db.write('trials', data=kwargs, query=where)
def fetch_lost_trials(self, experiment):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_lost_trials`"""
heartbeat = orion.core.config.worker.heartbeat
threshold = datetime.datetime.utcnow() - datetime.timedelta(
seconds=heartbeat)
lte_comparison = {'$lte': threshold}
query = {
'experiment': experiment._id,
'status': 'reserved',
'heartbeat': lte_comparison
}
return self._fetch_trials(query)
def push_trial_results(self, trial):
"""See :func:`~orion.storage.BaseStorageProtocol.push_trial_results`"""
rc = self._update_trial(trial,
**trial.to_dict(),
where={
'_id': trial.id,
'status': 'reserved'
})
if not rc:
raise FailedUpdate()
return rc
def set_trial_status(self, trial, status, heartbeat=None):
"""See :func:`~orion.storage.BaseStorageProtocol.set_trial_status`"""
if heartbeat is None:
heartbeat = datetime.datetime.utcnow()
update = dict(status=status,
heartbeat=heartbeat,
experiment=trial.experiment)
validate_status(status)
rc = self._update_trial(trial,
**update,
where={
'status': trial.status,
'_id': trial.id
})
if not rc:
raise FailedUpdate()
trial.status = status
def fetch_pending_trials(self, experiment):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_pending_trials`"""
query = dict(experiment=experiment._id,
status={'$in': ['new', 'suspended', 'interrupted']})
return self._fetch_trials(query)
def reserve_trial(self, experiment):
"""See :func:`~orion.storage.BaseStorageProtocol.reserve_trial`"""
query = dict(experiment=experiment._id,
status={'$in': ['interrupted', 'new', 'suspended']})
# read and write works on a single document
now = datetime.datetime.utcnow()
trial = self._db.read_and_write('trials',
query=query,
data=dict(status='reserved',
start_time=now,
heartbeat=now))
if trial is None:
return None
return Trial(**trial)
def fetch_noncompleted_trials(self, experiment):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_noncompleted_trials`"""
query = dict(experiment=experiment._id, status={'$ne': 'completed'})
return self._fetch_trials(query)
def count_completed_trials(self, experiment):
"""See :func:`~orion.storage.BaseStorageProtocol.count_completed_trials`"""
query = dict(experiment=experiment._id, status='completed')
return self._db.count('trials', query)
def count_broken_trials(self, experiment):
"""See :func:`~orion.storage.BaseStorageProtocol.count_broken_trials`"""
query = dict(experiment=experiment._id, status='broken')
return self._db.count('trials', query)
def update_heartbeat(self, trial):
"""Update trial's heartbeat"""
return self._update_trial(trial,
heartbeat=datetime.datetime.utcnow(),
status='reserved')
def fetch_trials_by_status(self, experiment, status):
"""See :func:`~orion.storage.BaseStorageProtocol.fetch_trials_by_status`"""
query = dict(experiment=experiment._id, status=status)
return self._fetch_trials(query)
class Experiment(object):
"""Represents an entry in database/experiments collection.
Attributes
----------
name : str
Unique identifier for this experiment per `user`.
id: object
id of the experiment in the database if experiment is configured. Value is `None`
if the experiment is not configured.
refers : dict or list of `Experiment` objects, after initialization is done.
A dictionary pointing to a past `Experiment` id, ``refers[parent_id]``, whose
trials we want to add in the history of completed trials we want to re-use.
For convenience and database effiency purpose, all experiments of a common tree shares
`refers[root_id]`, with the root experiment refering to itself.
metadata : dict
Contains managerial information about this `Experiment`.
pool_size : int
How many workers can participate asynchronously in this `Experiment`.
max_trials : int
How many trials must be evaluated, before considering this `Experiment` done.
This attribute can be updated if the rest of the experiment configuration
is the same. In that case, if trying to set to an already set experiment,
it will overwrite the previous one.
algorithms : dict of dicts or an `PrimaryAlgo` object, after initialization is done.
Complete specification of the optimization and dynamical procedures taking
place in this `Experiment`.
Metadata
--------
user : str
System user currently owning this running process, the one who invoked **Oríon**.
datetime : `datetime.datetime`
When was this particular configuration submitted to the database.
orion_version : str
Version of **Oríon** which suggested this experiment. `user`'s current
**Oríon** version.
user_script : str
Full absolute path to `user`'s executable.
user_args : list of str
Contains separate arguments to be passed when invoking `user_script`,
possibly templated for **Oríon**.
user_vcs : str, optional
User's version control system for this executable's code repository.
user_version : str, optional
Current user's repository version.
user_commit_hash : str, optional
Current `Experiment`'s commit hash for **Oríon**'s invocation.
"""
__slots__ = ('name', 'refers', 'metadata', 'pool_size', 'max_trials',
'algorithms', 'producer', 'working_dir', '_db', '_init_done',
'_id', '_node', '_last_fetched')
non_branching_attrs = ('pool_size', 'max_trials')
def __init__(self, name, user=None):
"""Initialize an Experiment object with primary key (:attr:`name`, :attr:`user`).
Try to find an entry in `Database` with such a key and config this object
from it import, if successful. Else, init with default/empty values and
insert new entry with this object's attributes in database.
.. note::
Practically initialization has not finished until `config`'s setter
is called.
:param name: Describe a configuration with a unique identifier per :attr:`user`.
:type name: str
"""
log.debug("Creating Experiment object with name: %s", name)
self._init_done = False
self._db = Database() # fetch database instance
self._setup_db() # build indexes for collections
self._id = None
self.name = name
self._node = None
self.refers = {}
if user is None:
user = getpass.getuser()
self.metadata = {'user': user}
self.pool_size = None
self.max_trials = None
self.algorithms = None
self.working_dir = None
self.producer = {'strategy': None}
config = self._db.read('experiments', {
'name': name,
'metadata.user': user
})
if config:
log.debug(
"Found existing experiment, %s, under user, %s, registered in database.",
name, user)
if len(config) > 1:
log.warning(
"Many (%s) experiments for (%s, %s) are available but "
"only the most recent one can be accessed. "
"Experiment branches will be supported soon.", len(config),
name, user)
config = sorted(config,
key=lambda x: x['metadata']['datetime'],
reverse=True)[0]
for attrname in self.__slots__:
if not attrname.startswith('_') and attrname in config:
setattr(self, attrname, config[attrname])
self._id = config['_id']
self._last_fetched = self.metadata.get("datetime",
datetime.datetime.utcnow())
def _setup_db(self):
self._db.ensure_index('experiments',
[('name', Database.ASCENDING),
('metadata.user', Database.ASCENDING)],
unique=True)
self._db.ensure_index('experiments', 'metadata.datetime')
self._db.ensure_index('trials', 'experiment')
self._db.ensure_index('trials', 'status')
self._db.ensure_index('trials', 'results')
self._db.ensure_index('trials', 'start_time')
self._db.ensure_index('trials', [('end_time', Database.DESCENDING)])
def fetch_trials(self, query, selection=None):
"""Fetch trials of the experiment in the database
Trials are sorted based on `Trial.submit_time`
.. note::
The query is always updated with `{"experiment": self._id}`
.. seealso::
:meth:`orion.core.io.database.AbstractDB.read` for more information about the
arguments.
"""
query["experiment"] = self._id
trials = Trial.build(self._db.read('trials', query, selection))
def _get_submit_time(trial):
if trial.submit_time:
return trial.submit_time
return datetime.datetime.utcnow()
return list(sorted(trials, key=_get_submit_time))
def fetch_trials_tree(self, query, selection=None):
"""Fetch trials recursively in the EVC tree
.. seealso::
:meth:`orion.core.worker.Experiment.fetch_trials` for more information about the
arguments.
:class:`orion.core.evc.experiment.ExperimentNode` for more information about the EVC
tree.
"""
if self._node is None:
return self.fetch_trials(query, selection)
return self._node.fetch_trials(query, selection)
def connect_to_version_control_tree(self, node):
"""Connect the experiment to its node in a version control tree
.. seealso::
:class:`orion.core.evc.experiment.ExperimentNode`
:param node: Node giving access to the experiment version control tree.
:type name: None or `ExperimentNode`
"""
self._node = node
def reserve_trial(self, score_handle=None):
"""Find *new* trials that exist currently in database and select one of
them based on the highest score return from `score_handle` callable.
:param score_handle: A way to decide which trial out of the *new* ones to
to pick as *reserved*, defaults to a random choice.
:type score_handle: callable
:return: selected `Trial` object, None if could not find any.
"""
if score_handle is not None and not callable(score_handle):
raise ValueError(
"Argument `score_handle` must be callable with a `Trial`.")
query = dict(experiment=self._id,
status={'$in': ['new', 'suspended', 'interrupted']})
new_trials = self.fetch_trials(query)
if not new_trials:
return None
if score_handle is not None and self.space:
scores = list(
map(score_handle,
map(lambda x: trial_to_tuple(x, self.space), new_trials)))
scored_trials = zip(scores, new_trials)
best_trials = filter(lambda st: st[0] == max(scores),
scored_trials)
new_trials = list(zip(*best_trials))[1]
elif score_handle is not None:
log.warning(
"While reserving trial: `score_handle` was provided, but "
"parameter space has not been defined yet.")
selected_trial = random.sample(new_trials, 1)[0]
# Query on status to ensure atomicity. If another process change the
# status meanwhile, read_and_write will fail, because query will fail.
query = {'_id': selected_trial.id, 'status': selected_trial.status}
update = dict(status='reserved')
if selected_trial.status == 'new':
update["start_time"] = datetime.datetime.utcnow()
selected_trial_dict = self._db.read_and_write('trials',
query=query,
data=update)
if selected_trial_dict is None:
selected_trial = self.reserve_trial(score_handle=score_handle)
else:
selected_trial = Trial(**selected_trial_dict)
return selected_trial
def push_completed_trial(self, trial):
"""Inform database about an evaluated `trial` with results.
:param trial: Corresponds to a successful evaluation of a particular run.
:type trial: `Trial`
.. note::
Change status from *reserved* to *completed*.
"""
trial.end_time = datetime.datetime.utcnow()
trial.status = 'completed'
self._db.write('trials', trial.to_dict(), query={'_id': trial.id})
def register_lie(self, lying_trial):
"""Register a *fake* trial created by the strategist.
The main difference between fake trial and orignal ones is the addition of a fake objective
result, and status being set to completed. The id of the fake trial is different than the id
of the original trial, but the original id can be computed using the hashcode on parameters
of the fake trial. See mod:`orion.core.worker.strategy` for more information and the
Strategist object and generation of fake trials.
Parameters
----------
trials: `Trial` object
Fake trial to register in the database
Raises
------
orion.core.io.database.DuplicateKeyError
If a trial with the same id already exist in the database. Since the id is computed
based on a hashing of the trial, this should mean that an identical trial already exist
in the database.
"""
lying_trial.status = 'completed'
lying_trial.end_time = datetime.datetime.utcnow()
self._db.write('lying_trials', lying_trial.to_dict())
def register_trial(self, trial):
"""Register new trial in the database.
Inform database about *new* suggested trial with specific parameter values. Trials may only
be registered one at a time to avoid registration of duplicates.
Parameters
----------
trials: `Trial` object
Trial to register in the database
Raises
------
orion.core.io.database.DuplicateKeyError
If a trial with the same id already exist in the database. Since the id is computed
based on a hashing of the trial, this should mean that an identical trial already exist
in the database.
"""
stamp = datetime.datetime.utcnow()
trial.experiment = self._id
trial.status = 'new'
trial.submit_time = stamp
self._db.write('trials', trial.to_dict())
def fetch_completed_trials(self):
"""Fetch recent completed trials that this `Experiment` instance has not yet seen.
Trials are sorted based on `Trial.submit_time`
.. note::
It will return only those with `Trial.end_time` after `_last_fetched`, for performance
reasons.
:return: list of completed `Trial` objects
"""
query = dict(status='completed', end_time={'$gt': self._last_fetched})
completed_trials = self.fetch_trials_tree(query)
if completed_trials:
self._last_fetched = max(trial.end_time
for trial in completed_trials)
return completed_trials
def fetch_noncompleted_trials(self):
"""Fetch non-completed trials of this `Experiment` instance.
Trials are sorted based on `Trial.submit_time`
.. note::
It will return all non-completed trials, including new, reserved, suspended,
interruped and broken ones.
:return: list of non-completed `Trial` objects
"""
query = dict(status={'$ne': 'completed'})
return self.fetch_trials_tree(query)
# pylint: disable=invalid-name
@property
def id(self):
"""Id of the experiment in the database if configured.
Value is `None` if the experiment is not configured.
"""
return self._id
@property
def node(self):
"""Node of the experiment in the version control tree.
Value is `None` if the experiment is not connected to the version control tree.
.. seealso::
:py:meth:`orion.core.worker.experiment.Experiment.connect_to_version_control_tree`
"""
return self._node
@property
def is_done(self):
"""Return True, if this experiment is considered to be finished.
1. Count how many trials have been completed and compare with `max_trials`.
2. Ask `algorithms` if they consider there is a chance for further improvement.
.. note::
To be used as a terminating condition in a ``Worker``.
"""
query = dict(experiment=self._id, status='completed')
num_completed_trials = self._db.count('trials', query)
return ((num_completed_trials >= self.max_trials)
or (self._init_done and self.algorithms.is_done))
@property
def space(self):
"""Return problem's parameter `orion.algo.space.Space`.
.. note:: It will return None, if experiment init is not done.
"""
if self._init_done:
return self.algorithms.space
return None
@property
def configuration(self):
"""Return a copy of an `Experiment` configuration as a dictionary."""
config = dict()
for attrname in self.__slots__:
if attrname.startswith('_'):
continue
attribute = getattr(self, attrname)
if self._init_done and attrname == 'algorithms':
config[attrname] = attribute.configuration
else:
config[attrname] = attribute
if self._init_done and attrname == "refers" and attribute.get(
"adapter"):
config[attrname] = copy.deepcopy(config[attrname])
config[attrname]['adapter'] = config[attrname][
'adapter'].configuration
if self._init_done and attrname == "producer" and attribute.get(
"strategy"):
config[attrname] = copy.deepcopy(config[attrname])
config[attrname]['strategy'] = config[attrname][
'strategy'].configuration
# Reason for deepcopy is that some attributes are dictionaries
# themselves, we don't want to accidentally change the state of this
# object from a getter.
return copy.deepcopy(config)
def configure(self, config, enable_branching=True):
"""Set `Experiment` by overwriting current attributes.
If `Experiment` was already set and an overwrite is needed, a *branch*
is advised with a different :attr:`name` for this particular configuration.
.. note::
Calling this property is necessary for an experiment's initialization process to be
considered as done. But it can be called only once.
"""
if self._init_done:
raise RuntimeError(
"Configuration is done; cannot reset an Experiment.")
# Experiment was build using db, but config was build before experiment got in db.
# Fake a DuplicateKeyError to force reinstantiation of experiment with proper config.
if self._id is not None and "datetime" not in config['metadata']:
raise DuplicateKeyError(
"Cannot register an existing experiment with a new config")
# Copy and simulate instantiating given configuration
experiment = Experiment(self.name)
experiment._instantiate_config(self.configuration)
experiment._instantiate_config(config)
experiment._init_done = True
# If id is None in this object, then database did not hit a config
# with same (name, user's name) pair. Everything depends on the user's
# orion_config to set.
if self._id is None:
if config['name'] != self.name or \
config['metadata']['user'] != self.metadata['user']:
raise ValueError(
"Configuration given is inconsistent with this Experiment."
)
is_new = True
else:
# Branch if it is needed
# TODO: When refactoring experiment managenent, is_different_from
# will be used when EVC is not available.
# is_new = self._is_different_from(experiment.configuration)
branching_configuration = fetch_branching_configuration(config)
conflicts = detect_conflicts(self.configuration,
experiment.configuration)
is_new = len(
conflicts.get()) > 1 or branching_configuration.get('branch')
if is_new and not enable_branching:
raise ValueError("Configuration is different and generate a "
"branching event")
elif is_new:
experiment._branch_config(conflicts, branching_configuration)
final_config = experiment.configuration
self._instantiate_config(final_config)
self._init_done = True
# If everything is alright, push new config to database
if is_new:
final_config['metadata']['datetime'] = datetime.datetime.utcnow()
self.metadata['datetime'] = final_config['metadata']['datetime']
# This will raise DuplicateKeyError if a concurrent experiment with
# identical (name, metadata.user) is written first in the database.
self._db.write('experiments', final_config)
# XXX: Reminder for future DB implementations:
# MongoDB, updates an inserted dict with _id, so should you :P
self._id = final_config['_id']
# Update refers in db if experiment is root
if not self.refers:
self.refers = {
'root_id': self._id,
'parent_id': None,
'adapter': []
}
update = {'refers': self.refers}
query = {'_id': self._id}
self._db.write('experiments', data=update, query=query)
else:
# Writing the final config to an already existing experiment raises
# a DuplicatKeyError because of the embedding id `metadata.user`.
# To avoid this `final_config["name"]` is popped out before
# `db.write()`, thus seamingly breaking the compound index
# `(name, metadata.user)`
final_config.pop("name")
self._db.write('experiments', final_config, {'_id': self._id})
@property
def stats(self):
"""Calculate a stats dictionary for this particular experiment.
Returns
-------
stats : dict
Stats
-----
trials_completed : int
Number of completed trials
best_trials_id : int
Unique identifier of the `Trial` object in the database which achieved
the best known objective result.
best_evaluation : float
Evaluation score of the best trial
start_time : `datetime.datetime`
When Experiment was first dispatched and started running.
finish_time : `datetime.datetime`
When Experiment reached terminating condition and stopped running.
duration : `datetime.timedelta`
Elapsed time.
"""
query = dict(experiment=self._id, status='completed')
selection = {'end_time': 1, 'results': 1, 'experiment': 1, 'params': 1}
completed_trials = self.fetch_trials(query, selection)
if not completed_trials:
return dict()
stats = dict()
stats['trials_completed'] = len(completed_trials)
stats['best_trials_id'] = None
trial = completed_trials[0]
stats['best_evaluation'] = trial.objective.value
stats['best_trials_id'] = trial.id
stats['start_time'] = self.metadata['datetime']
stats['finish_time'] = stats['start_time']
for trial in completed_trials:
# All trials are going to finish certainly after the start date
# of the experiment they belong to
if trial.end_time > stats['finish_time']: # pylint:disable=no-member
stats['finish_time'] = trial.end_time
objective = trial.objective.value
if objective < stats['best_evaluation']:
stats['best_evaluation'] = objective
stats['best_trials_id'] = trial.id
stats['duration'] = stats['finish_time'] - stats['start_time']
return stats
def _instantiate_config(self, config):
"""Check before dispatching experiment whether configuration corresponds
to a executable experiment environment.
1. Check `refers` and instantiate `Adapter` objects from it.
2. Try to build parameter space from user arguments.
3. Check whether configured algorithms correspond to [known]/valid
implementations of the ``Algorithm`` class. Instantiate these objects.
4. Check if experiment `is_done`, prompt for larger `max_trials` if it is. (TODO)
"""
# Just overwrite everything else given
for section, value in config.items():
if section not in self.__slots__:
log.info(
"Found section '%s' in configuration. Experiments "
"do not support this option. Ignoring.", section)
continue
if section.startswith('_'):
log.info(
"Found section '%s' in configuration. "
"Cannot set private attributes. Ignoring.", section)
continue
# Copy sub configuration to value confusing side-effects
# Only copy at this level, not `config` directly to avoid TypeErrors if config contains
# non-serializable objects (copy.deepcopy complains otherwise).
if isinstance(value, dict):
value = copy.deepcopy(value)
setattr(self, section, value)
try:
space_builder = SpaceBuilder()
space = space_builder.build_from(config['metadata']['user_args'])
if not space:
raise ValueError(
"Parameter space is empty. There is nothing to optimize.")
# Instantiate algorithms
self.algorithms = PrimaryAlgo(space, self.algorithms)
except KeyError:
pass
if self.refers and not isinstance(self.refers.get('adapter'),
BaseAdapter):
self.refers['adapter'] = Adapter.build(self.refers['adapter'])
if not self.producer.get('strategy'):
self.producer = {
'strategy': Strategy(of_type="MaxParallelStrategy")
}
elif not isinstance(self.producer.get('strategy'),
BaseParallelStrategy):
self.producer = {
'strategy': Strategy(of_type=self.producer['strategy'])
}
def _branch_config(self, conflicts, branching_configuration):
"""Ask for a different identifier for this experiment. Set :attr:`refers`
key to previous experiment's name, the one that we branched from.
:param config: Conflicting configuration that will change based on prompt.
"""
experiment_brancher = ExperimentBranchBuilder(conflicts,
branching_configuration)
needs_manual_resolution = (not experiment_brancher.is_resolved
or experiment_brancher.auto_resolution)
if needs_manual_resolution:
branching_prompt = BranchingPrompt(experiment_brancher)
if not sys.__stdin__.isatty():
raise ValueError(
"Configuration is different and generates a branching event:\n{}"
.format(branching_prompt.get_status()))
branching_prompt.cmdloop()
if branching_prompt.abort or not experiment_brancher.is_resolved:
sys.exit()
adapter = experiment_brancher.create_adapters()
self._instantiate_config(experiment_brancher.conflicting_config)
self.refers['adapter'] = adapter
self.refers['parent_id'] = self._id
def _is_different_from(self, config):
"""Return True, if current `Experiment`'s configuration as described by
its attributes is different from the one suggested in `config`.
"""
is_diff = False
for section, value in config.items():
if section in self.non_branching_attrs or \
section not in self.__slots__ or \
section.startswith('_'):
continue
item = getattr(self, section)
if item != value:
log.warning(
"Config given is different from config found in db at section: %s",
section)
log.warning("Config+ :\n%s", value)
log.warning("Config- :\n%s", item)
is_diff = True
break
return is_diff
def __repr__(self):
"""Represent the object as a string."""
return "Experiment(name=%s, metadata.user=%s)" % (
self.name, self.metadata['user'])
