def test_reserve_when_exhausted(self):
"""Return None once all the trials have been allocated"""
stati = ["new", "reserved", "interrupted", "completed", "broken"]
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is None
def test_reserve_when_exhausted(self):
"""Return None once all the trials have been allocated"""
stati = ['new', 'reserved', 'interrupted', 'completed', 'broken']
with OrionState(trials=generate_trials(stati)) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is not None
assert exp.reserve_trial() is None
def test_update_completed_trial(random_dt):
"""Successfully push a completed trial into database."""
with OrionState(trials=generate_trials(['new'])) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
trial = exp.reserve_trial()
results_file = tempfile.NamedTemporaryFile(mode='w',
prefix='results_',
suffix='.log',
dir='.',
delete=True)
# Generate fake result
with open(results_file.name, 'w') as file:
json.dump([{
'name': 'loss',
'type': 'objective',
'value': 2
}], file)
# --
exp.update_completed_trial(trial, results_file=results_file)
yo = get_storage().fetch_trials(exp)[0].to_dict()
assert len(yo['results']) == len(trial.results)
assert yo['results'][0] == trial.results[0].to_dict()
assert yo['status'] == 'completed'
assert yo['end_time'] == random_dt
results_file.close()
def test_update_completed_trial(random_dt):
"""Successfully push a completed trial into database."""
with OrionState(trials=generate_trials(["new"])) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trial = exp.reserve_trial()
results_file = tempfile.NamedTemporaryFile(mode="w",
prefix="results_",
suffix=".log",
dir=".",
delete=True)
# Generate fake result
with open(results_file.name, "w") as file:
json.dump([{
"name": "loss",
"type": "objective",
"value": 2
}], file)
# --
exp.update_completed_trial(trial, results_file=results_file)
yo = get_storage().fetch_trials(exp)[0].to_dict()
assert len(yo["results"]) == len(trial.results)
assert yo["results"][0] == trial.results[0].to_dict()
assert yo["status"] == "completed"
assert yo["end_time"] == random_dt
results_file.close()
def test_reserve_none(self):
"""Find nothing, return None."""
try:
Database(of_type='MongoDB',
name='orion_test',
username='******',
password='******')
except (TypeError, ValueError):
pass
exp = Experiment('supernaekei')
trial = exp.reserve_trial()
assert trial is None
def test_reserve_success(self, random_dt):
"""Successfully find new trials in db and reserve the first one"""
storage_config = {"type": "legacy", "database": {"type": "EphemeralDB"}}
with OrionState(
trials=generate_trials(["new", "reserved"]), storage=storage_config
) as cfg:
exp = Experiment("supernaekei", mode="x")
exp._id = cfg.trials[0]["experiment"]
trial = exp.reserve_trial()
# Trials are sorted according to hash and 'new' gets position second
cfg.trials[1]["status"] = "reserved"
cfg.trials[1]["start_time"] = random_dt
cfg.trials[1]["heartbeat"] = random_dt
assert trial.to_dict() == cfg.trials[1]
def test_reserve_success(self, random_dt):
"""Successfully find new trials in db and reserve the first one"""
storage_config = {
'type': 'legacy',
'database': {
'type': 'EphemeralDB'
}
}
with OrionState(trials=generate_trials(['new', 'reserved']),
storage=storage_config) as cfg:
exp = Experiment('supernaekei')
exp._id = cfg.trials[0]['experiment']
trial = exp.reserve_trial()
# Trials are sorted according to hash and 'new' gets position second
cfg.trials[1]['status'] = 'reserved'
cfg.trials[1]['start_time'] = random_dt
cfg.trials[1]['heartbeat'] = random_dt
assert trial.to_dict() == cfg.trials[1]
def test_reserve_none(self):
"""Find nothing, return None."""
with OrionState(experiments=[], trials=[]):
exp = Experiment('supernaekei')
trial = exp.reserve_trial()
assert trial is None
