def test_invalid_result(self, exp_config):
"""Test that invalid objectives cannot be set"""
t = Trial(**exp_config[1][1])
# Make sure valid ones pass
t.results = [
Trial.Result(name='asfda', type='constraint', value=None),
Trial.Result(name='asfdb', type='objective', value=1e-5)
]
# No results at all
with pytest.raises(ValueError) as exc:
t.results = []
assert 'No objective found' in str(exc.value)
# No objectives
with pytest.raises(ValueError) as exc:
t.results = [
Trial.Result(name='asfda', type='constraint', value=None)
]
assert 'No objective found' in str(exc.value)
# Bad objective type
with pytest.raises(ValueError) as exc:
t.results = [
Trial.Result(name='asfda', type='constraint', value=None),
Trial.Result(name='asfdb', type='objective', value=None)
]
assert 'Results must contain' in str(exc.value)
def test_invalid_result(self, trial_config):
"""Test that invalid objectives cannot be set"""
t = Trial(**trial_config)
# Make sure valid ones pass
t.results = [
Trial.Result(name="asfda", type="constraint", value=None),
Trial.Result(name="asfdb", type="objective", value=1e-5),
]
# No results at all
with pytest.raises(ValueError) as exc:
t.results = []
assert "No objective found" in str(exc.value)
# No objectives
with pytest.raises(ValueError) as exc:
t.results = [
Trial.Result(name="asfda", type="constraint", value=None)
]
assert "No objective found" in str(exc.value)
# Bad objective type
with pytest.raises(ValueError) as exc:
t.results = [
Trial.Result(name="asfda", type="constraint", value=None),
Trial.Result(name="asfdb", type="objective", value=None),
]
assert "Results must contain" in str(exc.value)
def lie(self, trial):
"""See ParallelStrategy.lie"""
result = super(MaxParallelStrategy, self).lie(trial)
if result:
return result
return Trial.Result(name="lie", type="objective", value=self.max_result)
def test_queue_completed_trials_for_promotions(self, space):
pbt = SpaceTransformAlgoWrapper(PBT, space).algorithm
trial = sample_trials(pbt.space,
num=1,
status="completed",
objective=1)[0]
pbt.register(trial)
# Should queue the trial itself
pbt._queue_trials_for_promotions([trial])
assert len(pbt._queue) == 1
assert pbt._queue[0].id == trial.id
new_trial = trial.branch(
params={"f": pbt.fidelities[trial.params["f"]]})
pbt.lineages.fork(trial, new_trial)
new_trial.status = "completed"
new_trial._results.append(
Trial.Result(name="objective", type="objective", value=1))
pbt.register(new_trial)
# Should queue the parent of the broken trial
pbt._queue_trials_for_promotions([new_trial])
assert len(pbt._queue) == 2
assert pbt._queue[1].id == new_trial.id
def test_value_not_allowed_type(self):
"""Other than `Trial.Result.allowed_types` are not allowed in `Trial.Result.type`.
Same for `Trial.Param`.
"""
with pytest.raises(ValueError):
v = Trial.Result(name='asfda', type='hoho')
v = Trial.Result()
with pytest.raises(ValueError):
v.type = 'asfda'
with pytest.raises(ValueError):
v = Trial.Param(name='asfda', type='hoho')
v = Trial.Param()
with pytest.raises(ValueError):
v.type = 'asfda'
def retrieve_result(self, trial, results_file=None, **kwargs):
"""Parse the results file that was generated by the trial process.
Parameters
----------
trial: Trial
The trial object to be updated
results_file: str
the file handle to read the result from
Returns
-------
returns the updated trial object
Note
----
This does not update the database!
"""
results = JSONConverter().parse(results_file.name)
trial.results = [
Trial.Result(
name=res['name'],
type=res['type'],
value=res['value']) for res in results
]
return trial
def lie(self, trial):
"""Construct a fake result for an incomplete trial
Parameters
----------
trial: `orion.core.worker.trial.Trial`
A trial object which is not supposed to be completed.
Returns
-------
``orion.core.worker.trial.Trial.Result``
The fake objective result corresponding to the trial given.
Notes
-----
If the trial has an objective even if not completed, a warning is printed to user
with a pointer to documentation to resolve the database corruption. The result returned is
the corresponding objective instead of the lie.
"""
objective = get_objective(trial)
if objective:
log.warning(CORRUPTED_DB_WARNING, trial.id)
return Trial.Result(name='lie', type='lie', value=objective)
return None
def lie(self, trial):
"""See BaseParallelStrategy.lie"""
result = super(MeanParallelStrategy, self).lie(trial)
if result:
return result
return Trial.Result(name="lie", type="lie", value=self.mean_result)
def retrieve_result(self, trial, results_file=None, **kwargs):
"""Parse the results file that was generated by the trial process.
Parameters
----------
trial: Trial
The trial object to be updated
results_file: str
the file handle to read the result from
Returns
-------
returns the updated trial object
Notes
-----
This does not update the database!
"""
if results_file is None:
return trial
try:
results = JSONConverter().parse(results_file.name)
except json.decoder.JSONDecodeError:
raise MissingResultFile()
trial.results = [
Trial.Result(name=res["name"],
type=res["type"],
value=res["value"]) for res in results
]
return trial
def lie(self, trial):
"""See BaseParallelStrategy.lie"""
if get_objective(trial):
raise RuntimeError(
"Trial {} is completed but should not be.".format(trial.id))
return Trial.Result(name='lie', type='lie', value=self.max_result)
def lie(self, trial):
"""See BaseParallelStrategy.lie"""
result = super(StubParallelStrategy, self).lie(trial)
if result:
return result
return Trial.Result(name='lie', type='lie', value=self.stub_value)
def test_objective_property(self):
"""Check property `Trial.objective`."""
base_trial = {"results": []}
base_trial["results"].append({
"name": "a",
"type": "gradient",
"value": 0.5
})
# 0 objective in `results` list
trial = Trial(**base_trial)
assert trial.objective is None
# 1 results in `results` list
expected = Trial.Result(name="b", type="objective", value=42)
base_trial["results"].append({
"name": "b",
"type": "objective",
"value": 42
})
trial = Trial(**base_trial)
assert trial.objective == expected
# >1 results in `results` list
base_trial["results"].append({
"name": "c",
"type": "objective",
"value": 42
})
trial = Trial(**base_trial)
assert trial.objective == expected
def lie(self, trial: Trial) -> Trial.Result | None:
"""See ParallelStrategy.lie"""
result = super().lie(trial)
if result:
return result
return Trial.Result(name="lie",
type="objective",
value=self.stub_value)
def lie(self, trial):
"""See BaseParallelStrategy.lie"""
if self._value:
value = self._value
else:
value = len(self._observed_points)
self._lie = lie = Trial.Result(name='lie', type='lie', value=value)
return lie
def observations():
"""10 objective observations"""
points = [i for i in range(10)]
results = [
Trial.Result(name='foo', type='objective', value=points[i])
for i in range(10)
]
return points, results
def test_statistics_property(self):
"""Tests the property for accessing statistics"""
base_trial = {"results": []}
# 0 result exist
trial = Trial(**base_trial)
assert trial.statistics == []
# 0 result of type 'statistic' exist
base_trial["results"].append({
"name": "a",
"type": "objective",
"value": 10
})
trial = Trial(**base_trial)
assert trial.statistics == []
# 1 result of type 'statistic' exist
expected = [Trial.Result(name="b", type="statistic", value=5)]
base_trial["results"].append({
"name": "b",
"type": "statistic",
"value": 5
})
trial = Trial(**base_trial)
assert expected == trial.statistics
# > 1 results of type 'statistic' exist
expected = [
Trial.Result(name="b", type="statistic", value=5),
Trial.Result(name="c", type="statistic", value=20),
]
base_trial["results"].append({
"name": "c",
"type": "statistic",
"value": 20
})
trial = Trial(**base_trial)
assert expected == trial.statistics
def test_push_trial_results_unreserved(self, storage=None):
"""Successfully push a completed trial into database."""
with OrionState(experiments=[], trials=[base_trial],
storage=storage) as cfg:
storage = cfg.storage()
trial = storage.get_trial(Trial(**base_trial))
results = [Trial.Result(name="loss", type="objective", value=2)]
trial.results = results
with pytest.raises(FailedUpdate):
storage.push_trial_results(trial)
def test_push_completed_trial(hacked_exp, database, random_dt):
"""Successfully push a completed trial into database."""
trial = hacked_exp.reserve_trial()
trial.results = [Trial.Result(name='yolo', type='objective', value=3)]
hacked_exp.push_completed_trial(trial)
yo = database.trials.find_one({'_id': trial.id})
assert len(yo['results']) == len(trial.results)
assert yo['results'][0] == trial.results[0].to_dict()
assert yo['status'] == 'completed'
assert yo['end_time'] == random_dt
def algo_observe(algo, trials, results):
"""Convert trials so that algo can observe with legacy format (trials, results)."""
for trial, trial_results in zip(trials, results):
for name, trial_result in trial_results.items():
if trial.exp_working_dir is None:
trial.exp_working_dir = "/nothing"
trial.status = "completed"
trial.results.append(
Trial.Result(name=name, type=name, value=trial_result))
algo.observe(trials)
def test_push_trial_results(self, storage):
"""Successfully push a completed trial into database."""
with OrionState(experiments=[], trials=[base_trial],
database=storage) as cfg:
storage = cfg.storage()
trial = storage.get_trial(Trial(**base_trial))
results = [Trial.Result(name='loss', type='objective', value=2)]
trial.results = results
assert storage.push_trial_results(
trial), 'should update successfully'
trial2 = storage.get_trial(trial)
assert trial2.results == results
def results(self):
"""See `~orion.core.worker.trial.Trial`"""
self._results = []
for k, values in self.storage.metrics.items():
result_type = "statistic"
if k == self.objective_key:
result_type = "objective"
if isinstance(values, dict):
items = list(values.items())
items.sort(key=lambda v: v[0])
val = items[-1][1]
self._results.append(
OrionTrial.Result(name=k, type=result_type, value=val))
elif isinstance(values, list):
self._results.append(
OrionTrial.Result(name=k,
type=result_type,
value=values[-1]))
return self._results
def _consume(self, trial, workdirname):
config_file = tempfile.NamedTemporaryFile(mode='w',
prefix='trial_',
suffix='.conf',
dir=workdirname,
delete=False)
config_file.close()
log.debug("## New temp config file: %s", config_file.name)
results_file = tempfile.NamedTemporaryFile(mode='w',
prefix='results_',
suffix='.log',
dir=workdirname,
delete=False)
results_file.close()
log.debug("## New temp results file: %s", results_file.name)
log.debug(
"## Building command line argument and configuration for trial.")
cmd_args = self.template_builder.build_to(config_file.name, trial,
self.experiment)
log.debug(
"## Launch user's script as a subprocess and wait for finish.")
script_process = self.launch_process(results_file.name, cmd_args)
if script_process is None:
return None
returncode = script_process.wait()
if returncode != 0:
log.error(
"Something went wrong. Check logs. Process "
"returned with code %d !", returncode)
if returncode == 2:
sys.exit(2)
return None
log.debug(
"## Parse results from file and fill corresponding Trial object.")
results = self.converter.parse(results_file.name)
trial.results = [
Trial.Result(name=res['name'],
type=res['type'],
value=res['value']) for res in results
]
return trial
def observe(self, trial, results):
"""Observe trial results
Experiment must be in executable ('x') mode.
Parameters
----------
trial: `orion.core.worker.trial.Trial`
Reserved trial to observe.
results: list
Results to be set for the new trial. Results must have the format
{name: <str>: type: <'objective', 'constraint' or 'gradient'>, value=<float>} otherwise
a ValueError will be raised. If the results are invalid, the trial will not be released.
Returns
-------
`orion.core.worker.trial.Trial`
The trial inserted in storage. If `reserve=True` and no results are given, the returned
trial will be in a `reserved` status.
Raises
------
`ValueError`
- If results have invalid format
- If the trial does not exist in storage.
`RuntimeError`
If reservation of the trial has been lost prior to releasing it.
`orion.core.utils.exceptions.UnsupportedOperation`
If the experiment was not loaded in executable mode.
"""
self._check_if_executable()
trial.results += [Trial.Result(**result) for result in results]
raise_if_unreserved = True
try:
self._experiment.update_completed_trial(trial)
except FailedUpdate as e:
if self.get_trial(trial) is None:
raise_if_unreserved = False
raise ValueError("Trial {} does not exist in database.".format(
trial.id)) from e
raise RuntimeError(
"Reservation for trial {} has been lost.".format(
trial.id)) from e
finally:
self._release_reservation(trial,
raise_if_unreserved=raise_if_unreserved)
def test_push_trial_results(self, storage=None):
"""Successfully push a completed trial into database."""
reserved_trial = copy.deepcopy(base_trial)
reserved_trial["status"] = "reserved"
with OrionState(experiments=[],
trials=[reserved_trial],
storage=storage) as cfg:
storage = cfg.storage()
trial = storage.get_trial(Trial(**reserved_trial))
results = [Trial.Result(name="loss", type="objective", value=2)]
trial.results = results
assert storage.push_trial_results(
trial), "should update successfully"
trial2 = storage.get_trial(trial)
assert trial2.results == results
def test_register_overwrite_with_results(self, space: Space):
"""Tests that registering a trial with the same params overwrites the existing trial."""
registry = Registry()
trial = space.sample(1)[0]
registered_id = registry.register(trial)
assert len(registry) == 1
assert list(registry) == [trial]
assert registry[registered_id] == trial
same_but_with_results = copy.deepcopy(trial)
same_but_with_results._results.append(
Trial.Result(name="objective", type="objective", value=1))
same_id = registry.register(same_but_with_results)
assert same_id == registered_id
assert len(registry) == 1
assert list(registry) == [same_but_with_results]
def test_triage_root_ready(self, status, space):
pbt = SpaceTransformAlgoWrapper(PBT, space).algorithm
trial = sample_trials(pbt.space, num=1, status="new")[0]
pbt.register(trial)
trial.status = status
trial._results.append(
Trial.Result(name="objective", type="objective", value=1))
trials_to_verify = pbt._triage([trial])
assert trials_to_verify == [trial]
assert pbt.has_suggested(trial)
assert pbt.has_observed(trial)
assert len(pbt.lineages) == 1
def test_suggest_initial_points(self, tpe: TPE, monkeypatch):
"""Test that initial points can be sampled correctly"""
_points = [(i, i - 6, "c") for i in range(1, 12)]
_trials = [
format_trials.tuple_to_trial(point, space=tpe.space)
for point in _points
]
index = 0
def sample(num: int = 1, seed=None) -> list[Trial]:
nonlocal index
result = _trials[index:index + num]
index += num
return result
monkeypatch.setattr(tpe.space, "sample", sample)
tpe.n_initial_points = 10
results = numpy.random.random(10)
for i in range(1, 11):
trials = tpe.suggest(1)
assert trials is not None
trial = trials[0]
assert trial.params == _trials[i]
point = format_trials.trial_to_tuple(trial, space=tpe.space)
assert point == (i, i - 6, "c")
trial.results = [
Trial.Result(name="objective",
type="objective",
value=results[i - 1])
]
tpe.observe([trial])
trials = tpe.suggest(1)
assert trials is not None
trial = trials[0]
assert trial == _trials[-1]
# BUG: This is failing. We expect this trial to be sampled from the model, not from the
# search space.
assert format_trials.trial_to_tuple(trial,
space=tpe.space) != (11, 5, "c")
def observe(self, trial, results):
"""Observe trial results
Parameters
----------
trial: `orion.core.worker.trial.Trial`
Reserved trial to observe.
results: list
Results to be set for the new trial. Results must have the format
{name: <str>: type: <'objective', 'constraint' or 'gradient'>, value=<float>} otherwise
a ValueError will be raised. If the results are invalid, the trial will not be released.
Returns
-------
`orion.core.worker.trial.Trial`
The trial inserted in storage. If `reserve=True` and no results are given, the returned
trial will be in a `reserved` status.
Raises
------
`ValueError`
- If results have invalid format
- If the trial does not exist in storage.
`RuntimeError`
If reservation of the trial has been lost prior to releasing it.
"""
trial.results += [Trial.Result(**result) for result in results]
try:
self._experiment.update_completed_trial(trial)
self.release(trial, 'completed')
except FailedUpdate as e:
if self.get_trial(trial) is None:
raise ValueError('Trial {} does not exist in database.'.format(
trial.id)) from e
self._release_reservation(trial)
raise RuntimeError(
'Reservation for trial {} has been lost.'.format(
trial.id)) from e
def test_suggest_num_population_size_fork_completed(self, space, mocker):
population_size = 10
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
population_size=population_size,
exploit=ExploitStub(rval=None).configuration,
).algorithm
pbt_sample_mock = mocker.spy(pbt, "_sample")
pbt_fork_mock = mocker.spy(pbt, "_fork_lineages")
num = 4
trials = pbt.suggest(num)
assert len(trials) == num
pbt_sample_mock.assert_called_with(num)
pbt_fork_mock.assert_called_with(0)
n_completed = 3
for trial in trials[:n_completed]:
trial.exp_working_dir = "/nothing"
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=1))
pbt.observe(trials)
assert len(pbt._queue) == n_completed
# There are 4 trials sampled, out of which 3 are completed. Still missing 6 trials
# for base population.
assert len(pbt.suggest(num)) == num
pbt_sample_mock.assert_called_with(num)
pbt_fork_mock.assert_called_with(0)
# There are 8 trials sampled, out of which 3 are completed. Still missing 2 trials
# for base population.
assert len(pbt.suggest(num)) == num
pbt_sample_mock.assert_called_with(2)
pbt_fork_mock.assert_called_with(2)
def test_gradient_property(self):
"""Check property `Trial.gradient`."""
base_trial = {"results": []}
# 0 result exist
trial = Trial(**base_trial)
assert trial.gradient is None
# 0 result of type 'gradient' exist
base_trial["results"].append({
"name": "a",
"type": "objective",
"value": 10
})
trial = Trial(**base_trial)
assert trial.gradient is None
# 1 result of type 'gradient' exist
expected = Trial.Result(name="b", type="gradient", value=5)
base_trial["results"].append({
"name": "b",
"type": "gradient",
"value": 5
})
trial = Trial(**base_trial)
assert trial.gradient == expected
# >1 gradient result
base_trial["results"].append({
"name": "c",
"type": "gradient",
"value": [12, 15]
})
trial = Trial(**base_trial)
assert trial.gradient == expected
