def test_update_completed_trial(random_dt):
"""Successfully push a completed trial into database."""
with OrionState(trials=generate_trials(["new"])) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trial = exp.reserve_trial()
results_file = tempfile.NamedTemporaryFile(mode="w",
prefix="results_",
suffix=".log",
dir=".",
delete=True)
# Generate fake result
with open(results_file.name, "w") as file:
json.dump([{
"name": "loss",
"type": "objective",
"value": 2
}], file)
# --
exp.update_completed_trial(trial, results_file=results_file)
yo = get_storage().fetch_trials(exp)[0].to_dict()
assert len(yo["results"]) == len(trial.results)
assert yo["results"][0] == trial.results[0].to_dict()
assert yo["status"] == "completed"
assert yo["end_time"] == random_dt
results_file.close()
def test_update_completed_trial(random_dt):
"""Successfully push a completed trial into database."""
with OrionState(trials=generate_trials(['new'])) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
trial = exp.reserve_trial()
results_file = tempfile.NamedTemporaryFile(mode='w',
prefix='results_',
suffix='.log',
dir='.',
delete=True)
# Generate fake result
with open(results_file.name, 'w') as file:
json.dump([{
'name': 'loss',
'type': 'objective',
'value': 2
}], file)
# --
exp.update_completed_trial(trial, results_file=results_file)
yo = get_storage().fetch_trials(exp)[0].to_dict()
assert len(yo['results']) == len(trial.results)
assert yo['results'][0] == trial.results[0].to_dict()
assert yo['status'] == 'completed'
assert yo['end_time'] == random_dt
results_file.close()
def test_register_trials(random_dt):
"""Register a list of newly proposed trials/parameters."""
with OrionState():
exp = Experiment('supernaekei')
exp._id = 0
trials = [
Trial(params=[{
'name': 'a',
'type': 'integer',
'value': 5
}]),
Trial(params=[{
'name': 'b',
'type': 'integer',
'value': 6
}]),
]
for trial in trials:
exp.register_trial(trial)
yo = list(
map(lambda trial: trial.to_dict(),
get_storage().fetch_trials(exp)))
assert len(yo) == len(trials)
assert yo[0]['params'] == list(
map(lambda x: x.to_dict(), trials[0]._params))
assert yo[1]['params'] == list(
map(lambda x: x.to_dict(), trials[1]._params))
assert yo[0]['status'] == 'new'
assert yo[1]['status'] == 'new'
assert yo[0]['submit_time'] == random_dt
assert yo[1]['submit_time'] == random_dt
def test_fix_lost_trials_race_condition(self, monkeypatch, caplog):
"""Test that a lost trial fixed by a concurrent process does not cause error."""
trial = copy.deepcopy(base_trial)
trial['status'] = 'interrupted'
trial['heartbeat'] = datetime.datetime.utcnow() - datetime.timedelta(
seconds=360)
with OrionState(trials=[trial]) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert len(exp.fetch_trials_by_status('interrupted')) == 1
assert len(exp._storage.fetch_lost_trials(exp)) == 0
def fetch_lost_trials(self, query):
trial_object = Trial(**trial)
trial_object.status = 'reserved'
return [trial_object]
# Force the fetch of a trial marked as reserved (and lost) while actually interrupted
# (as if already failed-over by another process).
with monkeypatch.context() as m:
m.setattr(exp._storage.__class__, 'fetch_lost_trials',
fetch_lost_trials)
assert len(exp._storage.fetch_lost_trials(exp)) == 1
with caplog.at_level(logging.DEBUG):
exp.fix_lost_trials()
assert caplog.records[-1].levelname == 'DEBUG'
assert caplog.records[-1].msg == 'failed'
assert len(exp.fetch_trials_by_status('interrupted')) == 1
assert len(exp.fetch_trials_by_status('reserved')) == 0
def test_register_trials(random_dt):
"""Register a list of newly proposed trials/parameters."""
with OrionState():
exp = Experiment("supernaekei", mode="x")
exp._id = 0
trials = [
Trial(params=[{
"name": "a",
"type": "integer",
"value": 5
}]),
Trial(params=[{
"name": "b",
"type": "integer",
"value": 6
}]),
]
for trial in trials:
exp.register_trial(trial)
yo = list(
map(lambda trial: trial.to_dict(),
get_storage().fetch_trials(exp)))
assert len(yo) == len(trials)
assert yo[0]["params"] == list(
map(lambda x: x.to_dict(), trials[0]._params))
assert yo[1]["params"] == list(
map(lambda x: x.to_dict(), trials[1]._params))
assert yo[0]["status"] == "new"
assert yo[1]["status"] == "new"
assert yo[0]["submit_time"] == random_dt
assert yo[1]["submit_time"] == random_dt
def hacked_exp(with_user_dendi, random_dt, clean_db, create_db_instance):
"""Return an `Experiment` instance with hacked _id to find trials in
fake database.
"""
exp = Experiment('supernaedo2-dendi')
exp._id = 'supernaedo2-dendi' # white box hack
return exp
def test_fetch_all_trials():
"""Fetch a list of all trials"""
with OrionState(trials=generate_trials(["new", "reserved", "completed"])) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trials = list(map(lambda trial: trial.to_dict(), exp.fetch_trials({})))
assert trials == cfg.trials
def test_reserve_when_exhausted(self):
"""Return None once all the trials have been allocated"""
stati = ["new", "reserved", "interrupted", "completed", "broken"]
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is None
def test_fetch_all_trials():
"""Fetch a list of all trials"""
with OrionState(
trials=generate_trials(['new', 'reserved', 'completed'])) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
trials = list(map(lambda trial: trial.to_dict(), exp.fetch_trials({})))
assert trials == cfg.trials
def test_broken_property():
"""Check experiment stopping conditions for maximum number of broken."""
MAX_BROKEN = 3
orion.core.config.worker.max_broken = MAX_BROKEN
stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN - 1))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert not exp.is_broken
stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert exp.is_broken
def test_reserve_when_exhausted(self):
"""Return None once all the trials have been allocated"""
stati = ['new', 'reserved', 'interrupted', 'completed', 'broken']
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is None
def test_fetch_completed_trials_from_view():
"""Fetch a list of the unseen yet completed trials."""
non_completed_stati = ['new', 'interrupted', 'suspended', 'reserved']
stati = non_completed_stati + ['completed']
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp_view = ExperimentView(exp)
trials = exp_view.fetch_trials_by_status('completed')
assert len(trials) == 1
assert trials[0].status == 'completed'
def test_broken_property():
"""Check experiment stopping conditions for maximum number of broken."""
MAX_BROKEN = 5
stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN - 1))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.max_broken = MAX_BROKEN
assert not exp.is_broken
stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.max_broken = MAX_BROKEN
assert exp.is_broken
def test_fix_lost_trials(self):
"""Test that a running trial with an old heartbeat is set to interrupted."""
trial = copy.deepcopy(base_trial)
trial["status"] = "reserved"
trial["heartbeat"] = datetime.datetime.utcnow() - datetime.timedelta(
seconds=60 * 10)
with OrionState(trials=[trial]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
assert len(exp.fetch_trials_by_status("reserved")) == 1
exp.fix_lost_trials()
assert len(exp.fetch_trials_by_status("reserved")) == 0
def test_fix_lost_trials(self):
"""Test that a running trial with an old heartbeat is set to interrupted."""
trial = copy.deepcopy(base_trial)
trial['status'] = 'reserved'
trial['heartbeat'] = datetime.datetime.utcnow() - datetime.timedelta(
seconds=360)
with OrionState(trials=[trial]) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert len(exp.fetch_trials_by_status('reserved')) == 1
exp.fix_lost_trials()
assert len(exp.fetch_trials_by_status('reserved')) == 0
def create_experiment(name, version, space, **kwargs):
"""Instantiate the experiment and its attribute objects
All unspecified arguments will be replaced by system's defaults (orion.core.config.*).
Parameters
----------
name: str
Name of the experiment.
version: int
Version of the experiment.
space: dict or Space object
Optimization space of the algorithm. If dict, should have the form
`dict(name='<prior>(args)')`.
algorithms: str or dict, optional
Algorithm used for optimization.
strategy: str or dict, optional
Parallel strategy to use to parallelize the algorithm.
max_trials: int, optional
Maximum number or trials before the experiment is considered done.
storage: dict, optional
Configuration of the storage backend.
"""
experiment = Experiment(name=name, version=version)
experiment._id = kwargs.get('_id', None) # pylint:disable=protected-access
experiment.pool_size = kwargs.get('pool_size')
if experiment.pool_size is None:
experiment.pool_size = orion.core.config.experiment.get(
'pool_size', deprecated='ignore')
experiment.max_trials = kwargs.get('max_trials',
orion.core.config.experiment.max_trials)
experiment.space = _instantiate_space(space)
experiment.algorithms = _instantiate_algo(experiment.space,
kwargs.get('algorithms'))
experiment.producer = kwargs.get('producer', {})
experiment.producer['strategy'] = _instantiate_strategy(
experiment.producer.get('strategy'))
experiment.working_dir = kwargs.get(
'working_dir', orion.core.config.experiment.working_dir)
experiment.metadata = kwargs.get(
'metadata', {'user': kwargs.get('user', getpass.getuser())})
experiment.refers = kwargs.get('refers', {
'parent_id': None,
'root_id': None,
'adapter': []
})
experiment.refers['adapter'] = _instantiate_adapters(
experiment.refers.get('adapter', []))
return experiment
def test_fetch_non_completed_trials():
"""Fetch a list of the trials that are not completed
trials.status in ['new', 'interrupted', 'suspended', 'broken']
"""
non_completed_stati = ["new", "interrupted", "suspended", "reserved"]
stati = non_completed_stati + ["completed"]
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trials = exp.fetch_noncompleted_trials()
assert len(trials) == 4
assert set(trial.status for trial in trials) == set(non_completed_stati)
def hacked_exp(with_user_dendi, random_dt, clean_db):
"""Return an `Experiment` instance with hacked _id to find trials in
fake database.
"""
try:
Database(of_type='MongoDB',
name='orion_test',
username='******',
password='******')
except (TypeError, ValueError):
pass
exp = Experiment('supernaedo2')
exp._id = 'supernaedo2' # white box hack
return exp
def test_configurable_broken_property():
"""Check if max_broken changes after configuration."""
MAX_BROKEN = 3
orion.core.config.worker.max_broken = MAX_BROKEN
stati = (['reserved'] * 10) + (['broken'] * (MAX_BROKEN))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert exp.is_broken
orion.core.config.worker.max_broken += 1
assert not exp.is_broken
def test_experiment_stats():
"""Check that property stats is returning a proper summary of experiment's results."""
NUM_COMPLETED = 3
stati = (["completed"] * NUM_COMPLETED) + (["reserved"] * 2)
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.metadata = {"datetime": datetime.datetime.utcnow()}
stats = exp.stats
assert stats.trials_completed == NUM_COMPLETED
assert stats.best_trials_id == cfg.trials[3]["_id"]
assert stats.best_evaluation == 0
assert stats.start_time == exp.metadata["datetime"]
assert stats.finish_time == cfg.trials[0]["end_time"]
assert stats.duration == stats.finish_time - stats.start_time
def test_acquire_algorithm_lock_with_different_config(self, new_config, algorithm):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
algorithm_original_config = algorithm.configuration
exp.algorithms = algorithm
# Setting attribute to algorithm inside the wrapper
algorithm.algorithm.seed = 10
assert algorithm.configuration != algorithm_original_config
with pytest.raises(
RuntimeError, match="Algorithm configuration changed since"
):
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
pass
def test_acquire_algorithm_lock_timeout(self, new_config, algorithm, mocker):
with OrionState(experiments=[new_config]) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = 0
exp.algorithms = algorithm
storage_acquisition_mock = mocker.spy(
cfg.storage(), "acquire_algorithm_lock"
)
with exp.acquire_algorithm_lock(timeout=0.2, retry_interval=0.1):
pass
storage_acquisition_mock.assert_called_with(
experiment=exp, timeout=0.2, retry_interval=0.1
)
def test_configurable_broken_property():
"""Check if max_broken changes after configuration."""
MAX_BROKEN = 5
stati = (["reserved"] * 10) + (["broken"] * (MAX_BROKEN))
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp.max_broken = MAX_BROKEN
assert exp.is_broken
exp.max_broken += 1
assert not exp.is_broken
def test_experiment_stats():
"""Check that property stats is returning a proper summary of experiment's results."""
NUM_COMPLETED = 3
stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2)
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.metadata = {'datetime': datetime.datetime.utcnow()}
stats = exp.stats
assert stats['trials_completed'] == NUM_COMPLETED
assert stats['best_trials_id'] == cfg.trials[3]['_id']
assert stats['best_evaluation'] == 0
assert stats['start_time'] == exp.metadata['datetime']
assert stats['finish_time'] == cfg.trials[0]['end_time']
assert stats['duration'] == stats['finish_time'] - stats['start_time']
assert len(stats) == 6
def test_reserve_success(self, random_dt):
"""Successfully find new trials in db and reserve the first one"""
storage_config = {"type": "legacy", "database": {"type": "EphemeralDB"}}
with OrionState(
trials=generate_trials(["new", "reserved"]), storage=storage_config
) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trial = exp.reserve_trial()
# Trials are sorted according to hash and 'new' gets position second
cfg.trials[1]["status"] = "reserved"
cfg.trials[1]["start_time"] = random_dt
cfg.trials[1]["heartbeat"] = random_dt
assert trial.to_dict() == cfg.trials[1]
def test_is_done_property_no_pending(algorithm):
"""Check experiment stopping conditions when there is no pending trials."""
completed = ['completed'] * 10
broken = ['broken'] * 5
with OrionState(trials=generate_trials(completed + broken)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.algorithms = algorithm
exp.max_trials = 15
# There is only 10 completed trials and algo not done.
assert not exp.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done and no pending trials
assert exp.is_done
def test_experiment_pickleable():
"""Test experiment instance is pickleable"""
with OrionState(trials=generate_trials(["new"])) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
exp_trials = exp.fetch_trials()
assert len(exp_trials) > 0
from orion.storage.base import storage_factory
exp_bytes = pickle.dumps(exp)
new_exp = pickle.loads(exp_bytes)
assert [trial.to_dict() for trial in exp_trials
] == [trial.to_dict() for trial in new_exp.fetch_trials()]
def test_experiment_view_stats():
"""Check that property stats from view is consistent."""
NUM_COMPLETED = 3
stati = (['completed'] * NUM_COMPLETED) + (['reserved'] * 2)
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.metadata = {'datetime': datetime.datetime.utcnow()}
exp_view = ExperimentView(exp)
stats = exp_view.stats
assert stats['trials_completed'] == NUM_COMPLETED
assert stats['best_trials_id'] == cfg.trials[3]['_id']
assert stats['best_evaluation'] == 0
assert stats['start_time'] == exp_view.metadata['datetime']
assert stats['finish_time'] == cfg.trials[0]['end_time']
assert stats['duration'] == stats['finish_time'] - stats['start_time']
assert len(stats) == 6
def test_reserve_success(self, random_dt):
"""Successfully find new trials in db and reserve the first one"""
storage_config = {
'type': 'legacy',
'database': {
'type': 'EphemeralDB'
}
}
with OrionState(trials=generate_trials(['new', 'reserved']),
storage=storage_config) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
trial = exp.reserve_trial()
# Trials are sorted according to hash and 'new' gets position second
cfg.trials[1]['status'] = 'reserved'
cfg.trials[1]['start_time'] = random_dt
cfg.trials[1]['heartbeat'] = random_dt
assert trial.to_dict() == cfg.trials[1]
def test_is_done_property_with_pending(algorithm):
"""Check experiment stopping conditions when there is pending trials."""
completed = ['completed'] * 10
reserved = ['reserved'] * 5
with OrionState(trials=generate_trials(completed + reserved)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
exp.algorithms = algorithm
exp.max_trials = 10
assert exp.is_done
exp.max_trials = 15
# There is only 10 completed trials
assert not exp.is_done
exp.algorithms.algorithm.done = True
# Algorithm is done but 5 trials are pending
assert not exp.is_done