def test_is_done_cardinality(monkeypatch, dumbalgo): """Check whether algorithm will stop with base algorithm cardinality check""" monkeypatch.delattr(dumbalgo, "is_done") space = Space() space.register(Integer("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(6) for i in range(1, 6): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 5 assert algo.is_done space = Space() space.register(Real("yolo1", "uniform", 1, 4)) algo = dumbalgo(space) algo.suggest(6) for i in range(1, 6): backward.algo_observe(algo, [format_trials.tuple_to_trial( (i, ), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 5 assert not algo.is_done
def test_reshape(self, space, rspace): """Verify that the dimension are reshaped properly, forward and backward""" trial = format_trials.tuple_to_trial( (numpy.arange(6).reshape(3, 2).tolist(), "3", [10]), space) rtrial = format_trials.tuple_to_trial( [0.0, 0.0, 1.0, 0.0] + [10] + numpy.array(trial.params["yolo"]).reshape(-1).tolist(), rspace, ) assert rspace.transform(trial).params == rtrial.params numpy.testing.assert_equal( rspace.reverse(rtrial).params["yolo"], trial.params["yolo"]) assert rspace.reverse(rtrial).params["yolo2"] == trial.params["yolo2"] assert rspace.reverse(rtrial).params["yolo3"] == trial.params["yolo3"]
def test_is_done_cardinality(self): """Test that algorithm will stop when cardinality is reached""" space = self.update_space( { "x": "uniform(0, 4, discrete=True)", "y": "choices(['a', 'b', 'c'])", "z": "loguniform(1, 6, discrete=True)", } ) space = self.create_space(space) assert space.cardinality == 5 * 3 * 6 algo = self.create_algo(space=space) for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))): assert not algo.is_done n = algo.n_suggested backward.algo_observe( algo, [format_trials.tuple_to_trial([x, y, z], space)], [dict(objective=i)], ) assert algo.n_suggested == n + 1 assert i + 1 == space.cardinality assert algo.is_done
def test_tuple_to_trial_to_tuple(space, trial, fixed_suggestion): """The two functions should be inverse.""" data = trial_to_tuple(tuple_to_trial(fixed_suggestion, space), space) assert data == fixed_suggestion t = tuple_to_trial(trial_to_tuple(trial, space), space) assert t.experiment is None assert t.status == 'new' assert t.worker is None assert t.submit_time is None assert t.start_time is None assert t.end_time is None assert t.results == [] assert len(t._params) == len(trial._params) for i in range(len(t._params)): assert t._params[i].to_dict() == trial._params[i].to_dict()
def sample(self, n_samples=1, seed=None): """Draw random samples from this space. Parameters ---------- n_samples : int, optional The number of samples to be drawn. Default is 1 sample. seed : None | int | ``numpy.random.RandomState`` instance, optional This parameter defines the RandomState object to use for drawing random variates. If None (or np.random), the **global** np.random state is used. If integer, it is used to seed a RandomState instance **just for the call of this function**. Default is None. Set random state to something other than None for reproducible results. Returns ------- trials: list of `orion.core.worker.trial.Trial` Each element is a separate sample of this space, a trial containing values associated with the corresponding dimension. """ rng = check_random_state(seed) samples = [dim.sample(n_samples, rng) for dim in self.values()] return [format_trials.tuple_to_trial(point, self) for point in zip(*samples)]
def get_id(point, ignore_fidelity=False, exp_id=None): trial = format_trials.tuple_to_trial(point, space) trial.experiment = exp_id return algo.get_id( trial, ignore_fidelity=ignore_fidelity, )
def test_code_changed(config, monkeypatch): """Check that trial has its working_dir attribute changed.""" exp = experiment_builder.build(**config) trial = tuple_to_trial((1.0,), exp.space) exp.register_trial(trial, status="reserved") con = Consumer(exp) def code_changed(user_script): return dict( type="git", is_dirty=True, HEAD_sha="changed", active_branch="new_branch", diff_sha="new_diff", ) monkeypatch.setattr(consumer, "infer_versioning_metadata", code_changed) with pytest.raises(BranchingEvent) as exc: con.consume(trial) assert exc.match("Code changed between execution of 2 trials")
def produce(self): """Create and register new trials.""" sampled_points = 0 start = time.time() while sampled_points < self.pool_size and not self.algorithm.is_done: if time.time() - start > self.max_idle_time: raise RuntimeError( "Algorithm could not sample new points in less than {} seconds" .format(self.max_idle_time)) log.debug("### Algorithm suggests new points.") new_points = self.naive_algorithm.suggest(self.pool_size) # Sync state of original algo so that state continues evolving. self.algorithm.set_state(self.naive_algorithm.state_dict) if new_points is None: log.info("### Algo opted out.") self.backoff() continue for new_point in new_points: log.debug("#### Convert point to `Trial` object.") new_trial = format_trials.tuple_to_trial(new_point, self.space) try: new_trial.parents = self.naive_trials_history.children log.debug("#### Register new trial to database: %s", new_trial) self.experiment.register_trial(new_trial) sampled_points += 1 except DuplicateKeyError: log.debug("#### Duplicate sample.") self.backoff() break
def test_is_done_cardinality(self): """Test that algorithm will stop when cardinality is reached""" space = SpaceBuilder().build({ "x": "uniform(0, 4, discrete=True)", "y": "choices(['a', 'b', 'c'])", "z": "loguniform(1, 6, discrete=True)", }) assert space.cardinality == 5 * 3 * 6 algo = self.create_algo(space=space) # Prevent the algo from exiting early because of a max_trials limit. algo.algorithm.max_trials = None i = 0 for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))): assert not algo.is_done n = algo.n_suggested backward.algo_observe( algo, [format_trials.tuple_to_trial([x, y, z], space)], [dict(objective=i)], ) assert algo.n_suggested == n + 1 assert i + 1 == space.cardinality assert algo.is_done
def insert_trials(experiment_name, points, cmdconfig=None, raise_exc=True): """Insert sets of parameters manually, defined in `points`, as new trials for the experiment name, `experiment_name`. :param experiment_name: Name of the experiment which the new trials are going to be associated with :param points: list of tuples in agreement with experiment's parameter space :param raise_exc: whether an inappropriate tuple of parameters will raise an exception or it will be ignored .. note:: If `raise_exc` is True, no set of parameters will be inserted. If it is False, only the valid ones will be inserted; the rest will be ignored. .. note:: This cannot be used to prepopulate a future experiment. So, an experiment with `experiment_name` should already be configured in the database. """ cmdconfig = cmdconfig if cmdconfig else {} config = resolve_config.fetch_default_options( ) # Get database perhaps from default locs config = resolve_config.merge_env_vars( config) # Get database perhaps from env vars tmpconfig = resolve_config.merge_orion_config(config, dict(), cmdconfig, dict()) db_opts = tmpconfig['database'] db_type = db_opts.pop('type') try: Database(of_type=db_type, **db_opts) except SingletonError: pass experiment = Experiment(experiment_name) # Configuration is completely taken from the database if experiment.status is None: raise ValueError( "No experiment named '{}' could be found.".format(experiment_name)) experiment.configure(experiment.configuration) valid_points = [] print(experiment.space) for point in points: try: assert point in experiment.space valid_points.append(point) except AssertionError: if raise_exc: raise if not valid_points: return new_trials = list( map(lambda data: format_trials.tuple_to_trial(data, experiment.space), valid_points)) experiment.register_trials(new_trials)
def register_trials(self, new_point): """Register a new set of sampled parameters into the DB guaranteeing their uniqueness Parameters ---------- new_point: tuple tuple of values representing the hyperparameters values """ # FIXME: Relying on DB to guarantee uniqueness # when the trial history will be held by that algo we can move that logic out of the DB log.debug("#### Convert point to `Trial` object.") new_trial = format_trials.tuple_to_trial(new_point, self.space) try: self._prevalidate_trial(new_trial) new_trial.parents = self.naive_trials_history.children log.debug("#### Register new trial to database: %s", new_trial) self.experiment.register_trial(new_trial) self._update_params_hashes([new_trial]) return 1 except DuplicateKeyError: log.debug("#### Duplicate sample.") self.backoff() return 0
def sample(self, n_samples=1, seed=None): return [ format_trials.tuple_to_trial( (numpy.log(values.pop()),), algo.transformed_space ) for _ in range(n_samples) ]
def test_is_done_cardinality(self): # TODO: Support correctly loguniform(discrete=True) # See https://github.com/Epistimio/orion/issues/566 space = self.update_space( { "x": "uniform(0, 4, discrete=True)", "y": "choices(['a', 'b', 'c'])", "z": "uniform(1, 6, discrete=True)", } ) space = self.create_space(space) assert space.cardinality == 5 * 3 * 6 algo = self.create_algo(space=space) for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))): assert not algo.is_done n = algo.n_suggested backward.algo_observe( algo, [format_trials.tuple_to_trial([x, y, z], space)], [dict(objective=i)], ) assert algo.n_suggested == n + 1 assert i + 1 == space.cardinality assert algo.is_done
def test_tuple_to_hierarchical_trial(hierarchical_space, hierarchical_trial, fixed_suggestion): """Check if sample is recovered successfully from hierarchical trial.""" t = tuple_to_trial(fixed_suggestion, hierarchical_space) assert len(t._params) == len(hierarchical_trial._params) for i in range(len(t._params)): assert t._params[i].to_dict() == hierarchical_trial._params[i].to_dict( )
def _array_to_trial(x: np.ndarray, space: Space, y: np.ndarray | None = None) -> Trial: trial = format_trials.tuple_to_trial(x, space=space) if y is not None: trial.results = [_result(y)] trial.status = "completed" return trial
def test_contains(self, tspace, rspace, seed): """Check method `transform`.""" ryo = format_trials.tuple_to_trial( numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist() + [10] + numpy.zeros(tspace["yolo"].shape).reshape(-1).tolist(), rspace, ) assert ryo in rspace
def test_state_dict(dumbalgo): """Check whether trials_info is in the state dict""" space = Space() dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "alpha", 0.9) space.register(dim) nested_algo = {"DumbAlgo": dict(value=6, scoring=5)} algo = dumbalgo(space, value=(1, 1)) algo.suggest(1) assert not algo.state_dict["registry"]["_trials"] backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 1 backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)], [dict(objective=3)]) assert len(algo.state_dict["registry"]["_trials"]) == 1
def test_reverse(self, space, tspace, rspace, seed): """Check method `reverse`.""" ryo = format_trials.tuple_to_trial( tuple( numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist() + [10] + numpy.zeros(tspace["yolo"].shape).reshape(-1).tolist()), rspace, ) yo = rspace.reverse(ryo) assert yo in space
def trial(exp): """Return a Trial which is registered in DB.""" trial = tuple_to_trial((1.0, ), exp.space) heartbeat = datetime.datetime.utcnow() trial.experiment = exp.id trial.status = "reserved" trial.heartbeat = heartbeat get_storage().register_trial(trial) return trial
def trial(exp): """Return a Trial which is registered in DB.""" trial = tuple_to_trial((1.0, ), exp.space) heartbeat = datetime.datetime.utcnow() trial.experiment = exp.id trial.status = 'reserved' trial.heartbeat = heartbeat Database().write('trials', trial.to_dict()) return trial
def produce(self): """Create and register new trials.""" log.debug("### Suggest new ones.") new_points = self.algorithm.suggest(self.num_new_trials) log.debug("### Convert them to `Trial` objects.") new_trials = list( map(lambda data: format_trials.tuple_to_trial(data, self.space), new_points)) log.debug("### Register to database: %s", new_trials) self.experiment.register_trials(new_trials)
def suggest(self, num): """Suggest a `num`ber of new sets of parameters. Perform a step towards negative gradient and suggest that point. """ num = 1 # Simple gradient descent only make sense for 1 point at a time. if not self.has_observed_once: return self.space.sample(1) self.current_point -= self.learning_rate * self.gradient return [format_trials.tuple_to_trial(self.current_point, self.space)]
def test_trial_working_dir_is_changed(config, monkeypatch): """Check that trial has its working_dir attribute changed.""" exp = experiment_builder.build(**config) trial = tuple_to_trial((1.0, ), exp.space) exp.register_trial(trial, status='reserved') con = Consumer(exp) con.consume(trial) assert trial.working_dir is not None assert trial.working_dir == con.working_dir + "/exp_" + trial.id
def test_tuple_to_trial(space, trial, fixed_suggestion): """Check if sample is recovered successfully from trial.""" t = tuple_to_trial(fixed_suggestion, space) assert t.experiment is None assert t.status == 'new' assert t.worker is None assert t.submit_time is None assert t.start_time is None assert t.end_time is None assert t.results == [] assert len(t._params) == len(trial.params) for i in range(len(t.params)): assert t._params[i].to_dict() == trial._params[i].to_dict()
def flatten_numpy(trials_array, flattened_space): """Flatten dimensions""" flattened_points = numpy.array([ format_trials.trial_to_tuple( flattened_space.transform( format_trials.tuple_to_trial(point[:-1], flattened_space.original)), flattened_space, ) for point in trials_array ]) return numpy.concatenate((flattened_points, trials_array[:, -1:]), axis=1)
def test_verify_trial(self, palgo, space): trial = format_trials.tuple_to_trial((["asdfa", 2], 0, 3.5), space) palgo._verify_trial(trial) with pytest.raises(ValueError, match="not contained in space:"): invalid_trial = format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space) palgo._verify_trial(invalid_trial) # transform space tspace = build_required_space( space, type_requirement="real", shape_requirement="flattened" ) # transform point ttrial = tspace.transform(trial) ttrial in tspace # Transformed point is not in original space with pytest.raises(ValueError, match="not contained in space:"): palgo._verify_trial(ttrial) # Transformed point is in transformed space palgo._verify_trial(ttrial, space=tspace)
def main(args): """Fetch config and insert new point""" command_line_user_args = args.pop('user_args', [None])[1:] experiment_view = ExperimentBuilder().build_view_from(args) transformed_args = _build_from(command_line_user_args) exp_space = experiment_view.space values = _create_tuple_from_values(transformed_args, exp_space) trial = tuple_to_trial(values, exp_space) ExperimentBuilder().build_from_config( experiment_view.configuration).register_trials([trial])
def test_register_and_contain(self): """Register bunch of dimensions, check if points/name are in space.""" space = Space() trial = Trial(params=[{"name": "no", "value": 0, "type": "integer"}]) assert "yolo" not in space assert trial not in space categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4} dim = Categorical("yolo", categories, shape=2) space.register(dim) dim = Integer("yolo2", "uniform", -3, 6) space.register(dim) dim = Real("yolo3", "norm", 0.9) space.register(dim) assert "yolo" in space assert "yolo2" in space assert "yolo3" in space assert format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space) in space assert format_trials.tuple_to_trial((("asdfa", 2), 7, 3.5), space) not in space
def main(args): """Fetch config and insert new point""" command_line_user_args = args.pop('user_args', [None])[1:] experiment_view = experiment_builder.build_view_from_args(args) experiment = experiment_builder.build(name=experiment_view.name, version=experiment_view.version) transformed_args = _build_from(command_line_user_args) exp_space = experiment.space values = _create_tuple_from_values(transformed_args, exp_space) trial = tuple_to_trial(values, exp_space) experiment.register_trial(trial)
def insert_trials(experiment_name, points, raise_exc=True): """Insert sets of parameters manually, defined in `points`, as new trials for the experiment name, `experiment_name`. .. warning:: This function is deprecated and will be removed in 0.3.0. You should use ExperimentClient.insert() instead. :param experiment_name: Name of the experiment which the new trials are going to be associated with :param points: list of tuples in agreement with experiment's parameter space :param raise_exc: whether an inappropriate tuple of parameters will raise an exception or it will be ignored .. note:: If `raise_exc` is True, no set of parameters will be inserted. If it is False, only the valid ones will be inserted; the rest will be ignored. .. note:: This cannot be used to prepopulate a future experiment. So, an experiment with `experiment_name` should already be configured in the database. """ log.warning("insert_trials() is deprecated and will be removed in 0.3.0. " "You should use ExperimentClient.insert() instead.") experiment = create_experiment(experiment_name) valid_points = [] for point in points: try: assert point in experiment.space valid_points.append(point) except AssertionError: if raise_exc: raise if not valid_points: return new_trials = list( map( lambda data: format_trials.tuple_to_trial(data, experiment.space), valid_points, )) for new_trial in new_trials: experiment.insert(new_trial.params)