def test_invalid_result(self, exp_config): """Test that invalid objectives cannot be set""" t = Trial(**exp_config[1][1]) # Make sure valid ones pass t.results = [ Trial.Result(name='asfda', type='constraint', value=None), Trial.Result(name='asfdb', type='objective', value=1e-5) ] # No results at all with pytest.raises(ValueError) as exc: t.results = [] assert 'No objective found' in str(exc.value) # No objectives with pytest.raises(ValueError) as exc: t.results = [ Trial.Result(name='asfda', type='constraint', value=None) ] assert 'No objective found' in str(exc.value) # Bad objective type with pytest.raises(ValueError) as exc: t.results = [ Trial.Result(name='asfda', type='constraint', value=None), Trial.Result(name='asfdb', type='objective', value=None) ] assert 'Results must contain' in str(exc.value)
def test_invalid_result(self, trial_config): """Test that invalid objectives cannot be set""" t = Trial(**trial_config) # Make sure valid ones pass t.results = [ Trial.Result(name="asfda", type="constraint", value=None), Trial.Result(name="asfdb", type="objective", value=1e-5), ] # No results at all with pytest.raises(ValueError) as exc: t.results = [] assert "No objective found" in str(exc.value) # No objectives with pytest.raises(ValueError) as exc: t.results = [ Trial.Result(name="asfda", type="constraint", value=None) ] assert "No objective found" in str(exc.value) # Bad objective type with pytest.raises(ValueError) as exc: t.results = [ Trial.Result(name="asfda", type="constraint", value=None), Trial.Result(name="asfdb", type="objective", value=None), ] assert "Results must contain" in str(exc.value)
def lie(self, trial): """See ParallelStrategy.lie""" result = super(MaxParallelStrategy, self).lie(trial) if result: return result return Trial.Result(name="lie", type="objective", value=self.max_result)
def test_queue_completed_trials_for_promotions(self, space): pbt = SpaceTransformAlgoWrapper(PBT, space).algorithm trial = sample_trials(pbt.space, num=1, status="completed", objective=1)[0] pbt.register(trial) # Should queue the trial itself pbt._queue_trials_for_promotions([trial]) assert len(pbt._queue) == 1 assert pbt._queue[0].id == trial.id new_trial = trial.branch( params={"f": pbt.fidelities[trial.params["f"]]}) pbt.lineages.fork(trial, new_trial) new_trial.status = "completed" new_trial._results.append( Trial.Result(name="objective", type="objective", value=1)) pbt.register(new_trial) # Should queue the parent of the broken trial pbt._queue_trials_for_promotions([new_trial]) assert len(pbt._queue) == 2 assert pbt._queue[1].id == new_trial.id
def test_value_not_allowed_type(self): """Other than `Trial.Result.allowed_types` are not allowed in `Trial.Result.type`. Same for `Trial.Param`. """ with pytest.raises(ValueError): v = Trial.Result(name='asfda', type='hoho') v = Trial.Result() with pytest.raises(ValueError): v.type = 'asfda' with pytest.raises(ValueError): v = Trial.Param(name='asfda', type='hoho') v = Trial.Param() with pytest.raises(ValueError): v.type = 'asfda'
def retrieve_result(self, trial, results_file=None, **kwargs): """Parse the results file that was generated by the trial process. Parameters ---------- trial: Trial The trial object to be updated results_file: str the file handle to read the result from Returns ------- returns the updated trial object Note ---- This does not update the database! """ results = JSONConverter().parse(results_file.name) trial.results = [ Trial.Result( name=res['name'], type=res['type'], value=res['value']) for res in results ] return trial
def lie(self, trial): """Construct a fake result for an incomplete trial Parameters ---------- trial: `orion.core.worker.trial.Trial` A trial object which is not supposed to be completed. Returns ------- ``orion.core.worker.trial.Trial.Result`` The fake objective result corresponding to the trial given. Notes ----- If the trial has an objective even if not completed, a warning is printed to user with a pointer to documentation to resolve the database corruption. The result returned is the corresponding objective instead of the lie. """ objective = get_objective(trial) if objective: log.warning(CORRUPTED_DB_WARNING, trial.id) return Trial.Result(name='lie', type='lie', value=objective) return None
def lie(self, trial): """See BaseParallelStrategy.lie""" result = super(MeanParallelStrategy, self).lie(trial) if result: return result return Trial.Result(name="lie", type="lie", value=self.mean_result)
def retrieve_result(self, trial, results_file=None, **kwargs): """Parse the results file that was generated by the trial process. Parameters ---------- trial: Trial The trial object to be updated results_file: str the file handle to read the result from Returns ------- returns the updated trial object Notes ----- This does not update the database! """ if results_file is None: return trial try: results = JSONConverter().parse(results_file.name) except json.decoder.JSONDecodeError: raise MissingResultFile() trial.results = [ Trial.Result(name=res["name"], type=res["type"], value=res["value"]) for res in results ] return trial
def lie(self, trial): """See BaseParallelStrategy.lie""" if get_objective(trial): raise RuntimeError( "Trial {} is completed but should not be.".format(trial.id)) return Trial.Result(name='lie', type='lie', value=self.max_result)
def lie(self, trial): """See BaseParallelStrategy.lie""" result = super(StubParallelStrategy, self).lie(trial) if result: return result return Trial.Result(name='lie', type='lie', value=self.stub_value)
def test_objective_property(self): """Check property `Trial.objective`.""" base_trial = {"results": []} base_trial["results"].append({ "name": "a", "type": "gradient", "value": 0.5 }) # 0 objective in `results` list trial = Trial(**base_trial) assert trial.objective is None # 1 results in `results` list expected = Trial.Result(name="b", type="objective", value=42) base_trial["results"].append({ "name": "b", "type": "objective", "value": 42 }) trial = Trial(**base_trial) assert trial.objective == expected # >1 results in `results` list base_trial["results"].append({ "name": "c", "type": "objective", "value": 42 }) trial = Trial(**base_trial) assert trial.objective == expected
def lie(self, trial: Trial) -> Trial.Result | None: """See ParallelStrategy.lie""" result = super().lie(trial) if result: return result return Trial.Result(name="lie", type="objective", value=self.stub_value)
def lie(self, trial): """See BaseParallelStrategy.lie""" if self._value: value = self._value else: value = len(self._observed_points) self._lie = lie = Trial.Result(name='lie', type='lie', value=value) return lie
def observations(): """10 objective observations""" points = [i for i in range(10)] results = [ Trial.Result(name='foo', type='objective', value=points[i]) for i in range(10) ] return points, results
def test_statistics_property(self): """Tests the property for accessing statistics""" base_trial = {"results": []} # 0 result exist trial = Trial(**base_trial) assert trial.statistics == [] # 0 result of type 'statistic' exist base_trial["results"].append({ "name": "a", "type": "objective", "value": 10 }) trial = Trial(**base_trial) assert trial.statistics == [] # 1 result of type 'statistic' exist expected = [Trial.Result(name="b", type="statistic", value=5)] base_trial["results"].append({ "name": "b", "type": "statistic", "value": 5 }) trial = Trial(**base_trial) assert expected == trial.statistics # > 1 results of type 'statistic' exist expected = [ Trial.Result(name="b", type="statistic", value=5), Trial.Result(name="c", type="statistic", value=20), ] base_trial["results"].append({ "name": "c", "type": "statistic", "value": 20 }) trial = Trial(**base_trial) assert expected == trial.statistics
def test_push_trial_results_unreserved(self, storage=None): """Successfully push a completed trial into database.""" with OrionState(experiments=[], trials=[base_trial], storage=storage) as cfg: storage = cfg.storage() trial = storage.get_trial(Trial(**base_trial)) results = [Trial.Result(name="loss", type="objective", value=2)] trial.results = results with pytest.raises(FailedUpdate): storage.push_trial_results(trial)
def test_push_completed_trial(hacked_exp, database, random_dt): """Successfully push a completed trial into database.""" trial = hacked_exp.reserve_trial() trial.results = [Trial.Result(name='yolo', type='objective', value=3)] hacked_exp.push_completed_trial(trial) yo = database.trials.find_one({'_id': trial.id}) assert len(yo['results']) == len(trial.results) assert yo['results'][0] == trial.results[0].to_dict() assert yo['status'] == 'completed' assert yo['end_time'] == random_dt
def algo_observe(algo, trials, results): """Convert trials so that algo can observe with legacy format (trials, results).""" for trial, trial_results in zip(trials, results): for name, trial_result in trial_results.items(): if trial.exp_working_dir is None: trial.exp_working_dir = "/nothing" trial.status = "completed" trial.results.append( Trial.Result(name=name, type=name, value=trial_result)) algo.observe(trials)
def test_push_trial_results(self, storage): """Successfully push a completed trial into database.""" with OrionState(experiments=[], trials=[base_trial], database=storage) as cfg: storage = cfg.storage() trial = storage.get_trial(Trial(**base_trial)) results = [Trial.Result(name='loss', type='objective', value=2)] trial.results = results assert storage.push_trial_results( trial), 'should update successfully' trial2 = storage.get_trial(trial) assert trial2.results == results
def results(self): """See `~orion.core.worker.trial.Trial`""" self._results = [] for k, values in self.storage.metrics.items(): result_type = "statistic" if k == self.objective_key: result_type = "objective" if isinstance(values, dict): items = list(values.items()) items.sort(key=lambda v: v[0]) val = items[-1][1] self._results.append( OrionTrial.Result(name=k, type=result_type, value=val)) elif isinstance(values, list): self._results.append( OrionTrial.Result(name=k, type=result_type, value=values[-1])) return self._results
def _consume(self, trial, workdirname): config_file = tempfile.NamedTemporaryFile(mode='w', prefix='trial_', suffix='.conf', dir=workdirname, delete=False) config_file.close() log.debug("## New temp config file: %s", config_file.name) results_file = tempfile.NamedTemporaryFile(mode='w', prefix='results_', suffix='.log', dir=workdirname, delete=False) results_file.close() log.debug("## New temp results file: %s", results_file.name) log.debug( "## Building command line argument and configuration for trial.") cmd_args = self.template_builder.build_to(config_file.name, trial, self.experiment) log.debug( "## Launch user's script as a subprocess and wait for finish.") script_process = self.launch_process(results_file.name, cmd_args) if script_process is None: return None returncode = script_process.wait() if returncode != 0: log.error( "Something went wrong. Check logs. Process " "returned with code %d !", returncode) if returncode == 2: sys.exit(2) return None log.debug( "## Parse results from file and fill corresponding Trial object.") results = self.converter.parse(results_file.name) trial.results = [ Trial.Result(name=res['name'], type=res['type'], value=res['value']) for res in results ] return trial
def observe(self, trial, results): """Observe trial results Experiment must be in executable ('x') mode. Parameters ---------- trial: `orion.core.worker.trial.Trial` Reserved trial to observe. results: list Results to be set for the new trial. Results must have the format {name: <str>: type: <'objective', 'constraint' or 'gradient'>, value=<float>} otherwise a ValueError will be raised. If the results are invalid, the trial will not be released. Returns ------- `orion.core.worker.trial.Trial` The trial inserted in storage. If `reserve=True` and no results are given, the returned trial will be in a `reserved` status. Raises ------ `ValueError` - If results have invalid format - If the trial does not exist in storage. `RuntimeError` If reservation of the trial has been lost prior to releasing it. `orion.core.utils.exceptions.UnsupportedOperation` If the experiment was not loaded in executable mode. """ self._check_if_executable() trial.results += [Trial.Result(**result) for result in results] raise_if_unreserved = True try: self._experiment.update_completed_trial(trial) except FailedUpdate as e: if self.get_trial(trial) is None: raise_if_unreserved = False raise ValueError("Trial {} does not exist in database.".format( trial.id)) from e raise RuntimeError( "Reservation for trial {} has been lost.".format( trial.id)) from e finally: self._release_reservation(trial, raise_if_unreserved=raise_if_unreserved)
def test_push_trial_results(self, storage=None): """Successfully push a completed trial into database.""" reserved_trial = copy.deepcopy(base_trial) reserved_trial["status"] = "reserved" with OrionState(experiments=[], trials=[reserved_trial], storage=storage) as cfg: storage = cfg.storage() trial = storage.get_trial(Trial(**reserved_trial)) results = [Trial.Result(name="loss", type="objective", value=2)] trial.results = results assert storage.push_trial_results( trial), "should update successfully" trial2 = storage.get_trial(trial) assert trial2.results == results
def test_register_overwrite_with_results(self, space: Space): """Tests that registering a trial with the same params overwrites the existing trial.""" registry = Registry() trial = space.sample(1)[0] registered_id = registry.register(trial) assert len(registry) == 1 assert list(registry) == [trial] assert registry[registered_id] == trial same_but_with_results = copy.deepcopy(trial) same_but_with_results._results.append( Trial.Result(name="objective", type="objective", value=1)) same_id = registry.register(same_but_with_results) assert same_id == registered_id assert len(registry) == 1 assert list(registry) == [same_but_with_results]
def test_triage_root_ready(self, status, space): pbt = SpaceTransformAlgoWrapper(PBT, space).algorithm trial = sample_trials(pbt.space, num=1, status="new")[0] pbt.register(trial) trial.status = status trial._results.append( Trial.Result(name="objective", type="objective", value=1)) trials_to_verify = pbt._triage([trial]) assert trials_to_verify == [trial] assert pbt.has_suggested(trial) assert pbt.has_observed(trial) assert len(pbt.lineages) == 1
def test_suggest_initial_points(self, tpe: TPE, monkeypatch): """Test that initial points can be sampled correctly""" _points = [(i, i - 6, "c") for i in range(1, 12)] _trials = [ format_trials.tuple_to_trial(point, space=tpe.space) for point in _points ] index = 0 def sample(num: int = 1, seed=None) -> list[Trial]: nonlocal index result = _trials[index:index + num] index += num return result monkeypatch.setattr(tpe.space, "sample", sample) tpe.n_initial_points = 10 results = numpy.random.random(10) for i in range(1, 11): trials = tpe.suggest(1) assert trials is not None trial = trials[0] assert trial.params == _trials[i] point = format_trials.trial_to_tuple(trial, space=tpe.space) assert point == (i, i - 6, "c") trial.results = [ Trial.Result(name="objective", type="objective", value=results[i - 1]) ] tpe.observe([trial]) trials = tpe.suggest(1) assert trials is not None trial = trials[0] assert trial == _trials[-1] # BUG: This is failing. We expect this trial to be sampled from the model, not from the # search space. assert format_trials.trial_to_tuple(trial, space=tpe.space) != (11, 5, "c")
def observe(self, trial, results): """Observe trial results Parameters ---------- trial: `orion.core.worker.trial.Trial` Reserved trial to observe. results: list Results to be set for the new trial. Results must have the format {name: <str>: type: <'objective', 'constraint' or 'gradient'>, value=<float>} otherwise a ValueError will be raised. If the results are invalid, the trial will not be released. Returns ------- `orion.core.worker.trial.Trial` The trial inserted in storage. If `reserve=True` and no results are given, the returned trial will be in a `reserved` status. Raises ------ `ValueError` - If results have invalid format - If the trial does not exist in storage. `RuntimeError` If reservation of the trial has been lost prior to releasing it. """ trial.results += [Trial.Result(**result) for result in results] try: self._experiment.update_completed_trial(trial) self.release(trial, 'completed') except FailedUpdate as e: if self.get_trial(trial) is None: raise ValueError('Trial {} does not exist in database.'.format( trial.id)) from e self._release_reservation(trial) raise RuntimeError( 'Reservation for trial {} has been lost.'.format( trial.id)) from e
def test_suggest_num_population_size_fork_completed(self, space, mocker): population_size = 10 pbt = SpaceTransformAlgoWrapper( PBT, space, population_size=population_size, exploit=ExploitStub(rval=None).configuration, ).algorithm pbt_sample_mock = mocker.spy(pbt, "_sample") pbt_fork_mock = mocker.spy(pbt, "_fork_lineages") num = 4 trials = pbt.suggest(num) assert len(trials) == num pbt_sample_mock.assert_called_with(num) pbt_fork_mock.assert_called_with(0) n_completed = 3 for trial in trials[:n_completed]: trial.exp_working_dir = "/nothing" trial.status = "completed" trial._results.append( Trial.Result(name="objective", type="objective", value=1)) pbt.observe(trials) assert len(pbt._queue) == n_completed # There are 4 trials sampled, out of which 3 are completed. Still missing 6 trials # for base population. assert len(pbt.suggest(num)) == num pbt_sample_mock.assert_called_with(num) pbt_fork_mock.assert_called_with(0) # There are 8 trials sampled, out of which 3 are completed. Still missing 2 trials # for base population. assert len(pbt.suggest(num)) == num pbt_sample_mock.assert_called_with(2) pbt_fork_mock.assert_called_with(2)
def test_gradient_property(self): """Check property `Trial.gradient`.""" base_trial = {"results": []} # 0 result exist trial = Trial(**base_trial) assert trial.gradient is None # 0 result of type 'gradient' exist base_trial["results"].append({ "name": "a", "type": "objective", "value": 10 }) trial = Trial(**base_trial) assert trial.gradient is None # 1 result of type 'gradient' exist expected = Trial.Result(name="b", type="gradient", value=5) base_trial["results"].append({ "name": "b", "type": "gradient", "value": 5 }) trial = Trial(**base_trial) assert trial.gradient == expected # >1 gradient result base_trial["results"].append({ "name": "c", "type": "gradient", "value": [12, 15] }) trial = Trial(**base_trial) assert trial.gradient == expected