def test_is_done_cardinality(monkeypatch, dumbalgo):
"""Check whether algorithm will stop with base algorithm cardinality check"""
monkeypatch.delattr(dumbalgo, "is_done")
space = Space()
space.register(Integer("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert algo.is_done
space = Space()
space.register(Real("yolo1", "uniform", 1, 4))
algo = dumbalgo(space)
algo.suggest(6)
for i in range(1, 6):
backward.algo_observe(algo,
[format_trials.tuple_to_trial(
(i, ), space)], [dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 5
assert not algo.is_done
def test_reshape(self, space, rspace):
"""Verify that the dimension are reshaped properly, forward and backward"""
trial = format_trials.tuple_to_trial(
(numpy.arange(6).reshape(3, 2).tolist(), "3", [10]), space)
rtrial = format_trials.tuple_to_trial(
[0.0, 0.0, 1.0, 0.0] + [10] +
numpy.array(trial.params["yolo"]).reshape(-1).tolist(),
rspace,
)
assert rspace.transform(trial).params == rtrial.params
numpy.testing.assert_equal(
rspace.reverse(rtrial).params["yolo"], trial.params["yolo"])
assert rspace.reverse(rtrial).params["yolo2"] == trial.params["yolo2"]
assert rspace.reverse(rtrial).params["yolo3"] == trial.params["yolo3"]
def test_is_done_cardinality(self):
"""Test that algorithm will stop when cardinality is reached"""
space = self.update_space(
{
"x": "uniform(0, 4, discrete=True)",
"y": "choices(['a', 'b', 'c'])",
"z": "loguniform(1, 6, discrete=True)",
}
)
space = self.create_space(space)
assert space.cardinality == 5 * 3 * 6
algo = self.create_algo(space=space)
for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))):
assert not algo.is_done
n = algo.n_suggested
backward.algo_observe(
algo,
[format_trials.tuple_to_trial([x, y, z], space)],
[dict(objective=i)],
)
assert algo.n_suggested == n + 1
assert i + 1 == space.cardinality
assert algo.is_done
def test_tuple_to_trial_to_tuple(space, trial, fixed_suggestion):
"""The two functions should be inverse."""
data = trial_to_tuple(tuple_to_trial(fixed_suggestion, space), space)
assert data == fixed_suggestion
t = tuple_to_trial(trial_to_tuple(trial, space), space)
assert t.experiment is None
assert t.status == 'new'
assert t.worker is None
assert t.submit_time is None
assert t.start_time is None
assert t.end_time is None
assert t.results == []
assert len(t._params) == len(trial._params)
for i in range(len(t._params)):
assert t._params[i].to_dict() == trial._params[i].to_dict()
def sample(self, n_samples=1, seed=None):
"""Draw random samples from this space.
Parameters
----------
n_samples : int, optional
The number of samples to be drawn. Default is 1 sample.
seed : None | int | ``numpy.random.RandomState`` instance, optional
This parameter defines the RandomState object to use for drawing
random variates. If None (or np.random), the **global**
np.random state is used. If integer, it is used to seed a
RandomState instance **just for the call of this function**.
Default is None.
Set random state to something other than None for reproducible
results.
Returns
-------
trials: list of `orion.core.worker.trial.Trial`
Each element is a separate sample of this space, a trial containing
values associated with the corresponding dimension.
"""
rng = check_random_state(seed)
samples = [dim.sample(n_samples, rng) for dim in self.values()]
return [format_trials.tuple_to_trial(point, self) for point in zip(*samples)]
def get_id(point, ignore_fidelity=False, exp_id=None):
trial = format_trials.tuple_to_trial(point, space)
trial.experiment = exp_id
return algo.get_id(
trial,
ignore_fidelity=ignore_fidelity,
)
def test_code_changed(config, monkeypatch):
"""Check that trial has its working_dir attribute changed."""
exp = experiment_builder.build(**config)
trial = tuple_to_trial((1.0,), exp.space)
exp.register_trial(trial, status="reserved")
con = Consumer(exp)
def code_changed(user_script):
return dict(
type="git",
is_dirty=True,
HEAD_sha="changed",
active_branch="new_branch",
diff_sha="new_diff",
)
monkeypatch.setattr(consumer, "infer_versioning_metadata", code_changed)
with pytest.raises(BranchingEvent) as exc:
con.consume(trial)
assert exc.match("Code changed between execution of 2 trials")
def produce(self):
"""Create and register new trials."""
sampled_points = 0
start = time.time()
while sampled_points < self.pool_size and not self.algorithm.is_done:
if time.time() - start > self.max_idle_time:
raise RuntimeError(
"Algorithm could not sample new points in less than {} seconds"
.format(self.max_idle_time))
log.debug("### Algorithm suggests new points.")
new_points = self.naive_algorithm.suggest(self.pool_size)
# Sync state of original algo so that state continues evolving.
self.algorithm.set_state(self.naive_algorithm.state_dict)
if new_points is None:
log.info("### Algo opted out.")
self.backoff()
continue
for new_point in new_points:
log.debug("#### Convert point to `Trial` object.")
new_trial = format_trials.tuple_to_trial(new_point, self.space)
try:
new_trial.parents = self.naive_trials_history.children
log.debug("#### Register new trial to database: %s",
new_trial)
self.experiment.register_trial(new_trial)
sampled_points += 1
except DuplicateKeyError:
log.debug("#### Duplicate sample.")
self.backoff()
break
def test_is_done_cardinality(self):
"""Test that algorithm will stop when cardinality is reached"""
space = SpaceBuilder().build({
"x": "uniform(0, 4, discrete=True)",
"y": "choices(['a', 'b', 'c'])",
"z": "loguniform(1, 6, discrete=True)",
})
assert space.cardinality == 5 * 3 * 6
algo = self.create_algo(space=space)
# Prevent the algo from exiting early because of a max_trials limit.
algo.algorithm.max_trials = None
i = 0
for i, (x, y,
z) in enumerate(itertools.product(range(5), "abc", range(1,
7))):
assert not algo.is_done
n = algo.n_suggested
backward.algo_observe(
algo,
[format_trials.tuple_to_trial([x, y, z], space)],
[dict(objective=i)],
)
assert algo.n_suggested == n + 1
assert i + 1 == space.cardinality
assert algo.is_done
def insert_trials(experiment_name, points, cmdconfig=None, raise_exc=True):
"""Insert sets of parameters manually, defined in `points`, as new trials
for the experiment name, `experiment_name`.
:param experiment_name: Name of the experiment which the new trials are
going to be associated with
:param points: list of tuples in agreement with experiment's parameter space
:param raise_exc: whether an inappropriate tuple of parameters will raise
an exception or it will be ignored
.. note:: If `raise_exc` is True, no set of parameters will be inserted. If
it is False, only the valid ones will be inserted; the rest will be ignored.
.. note:: This cannot be used to prepopulate a future experiment. So,
an experiment with `experiment_name` should already be configured in
the database.
"""
cmdconfig = cmdconfig if cmdconfig else {}
config = resolve_config.fetch_default_options(
) # Get database perhaps from default locs
config = resolve_config.merge_env_vars(
config) # Get database perhaps from env vars
tmpconfig = resolve_config.merge_orion_config(config, dict(), cmdconfig,
dict())
db_opts = tmpconfig['database']
db_type = db_opts.pop('type')
try:
Database(of_type=db_type, **db_opts)
except SingletonError:
pass
experiment = Experiment(experiment_name)
# Configuration is completely taken from the database
if experiment.status is None:
raise ValueError(
"No experiment named '{}' could be found.".format(experiment_name))
experiment.configure(experiment.configuration)
valid_points = []
print(experiment.space)
for point in points:
try:
assert point in experiment.space
valid_points.append(point)
except AssertionError:
if raise_exc:
raise
if not valid_points:
return
new_trials = list(
map(lambda data: format_trials.tuple_to_trial(data, experiment.space),
valid_points))
experiment.register_trials(new_trials)
def register_trials(self, new_point):
"""Register a new set of sampled parameters into the DB
guaranteeing their uniqueness
Parameters
----------
new_point: tuple
tuple of values representing the hyperparameters values
"""
# FIXME: Relying on DB to guarantee uniqueness
# when the trial history will be held by that algo we can move that logic out of the DB
log.debug("#### Convert point to `Trial` object.")
new_trial = format_trials.tuple_to_trial(new_point, self.space)
try:
self._prevalidate_trial(new_trial)
new_trial.parents = self.naive_trials_history.children
log.debug("#### Register new trial to database: %s", new_trial)
self.experiment.register_trial(new_trial)
self._update_params_hashes([new_trial])
return 1
except DuplicateKeyError:
log.debug("#### Duplicate sample.")
self.backoff()
return 0
def sample(self, n_samples=1, seed=None):
return [
format_trials.tuple_to_trial(
(numpy.log(values.pop()),), algo.transformed_space
)
for _ in range(n_samples)
]
def test_is_done_cardinality(self):
# TODO: Support correctly loguniform(discrete=True)
# See https://github.com/Epistimio/orion/issues/566
space = self.update_space(
{
"x": "uniform(0, 4, discrete=True)",
"y": "choices(['a', 'b', 'c'])",
"z": "uniform(1, 6, discrete=True)",
}
)
space = self.create_space(space)
assert space.cardinality == 5 * 3 * 6
algo = self.create_algo(space=space)
for i, (x, y, z) in enumerate(itertools.product(range(5), "abc", range(1, 7))):
assert not algo.is_done
n = algo.n_suggested
backward.algo_observe(
algo,
[format_trials.tuple_to_trial([x, y, z], space)],
[dict(objective=i)],
)
assert algo.n_suggested == n + 1
assert i + 1 == space.cardinality
assert algo.is_done
def test_tuple_to_hierarchical_trial(hierarchical_space, hierarchical_trial,
fixed_suggestion):
"""Check if sample is recovered successfully from hierarchical trial."""
t = tuple_to_trial(fixed_suggestion, hierarchical_space)
assert len(t._params) == len(hierarchical_trial._params)
for i in range(len(t._params)):
assert t._params[i].to_dict() == hierarchical_trial._params[i].to_dict(
)
def _array_to_trial(x: np.ndarray,
space: Space,
y: np.ndarray | None = None) -> Trial:
trial = format_trials.tuple_to_trial(x, space=space)
if y is not None:
trial.results = [_result(y)]
trial.status = "completed"
return trial
def test_contains(self, tspace, rspace, seed):
"""Check method `transform`."""
ryo = format_trials.tuple_to_trial(
numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist() + [10] +
numpy.zeros(tspace["yolo"].shape).reshape(-1).tolist(),
rspace,
)
assert ryo in rspace
def test_state_dict(dumbalgo):
"""Check whether trials_info is in the state dict"""
space = Space()
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "alpha", 0.9)
space.register(dim)
nested_algo = {"DumbAlgo": dict(value=6, scoring=5)}
algo = dumbalgo(space, value=(1, 1))
algo.suggest(1)
assert not algo.state_dict["registry"]["_trials"]
backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)],
[dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 1
backward.algo_observe(algo, [format_trials.tuple_to_trial((1, 2), space)],
[dict(objective=3)])
assert len(algo.state_dict["registry"]["_trials"]) == 1
def test_reverse(self, space, tspace, rspace, seed):
"""Check method `reverse`."""
ryo = format_trials.tuple_to_trial(
tuple(
numpy.zeros(tspace["yolo2"].shape).reshape(-1).tolist() +
[10] + numpy.zeros(tspace["yolo"].shape).reshape(-1).tolist()),
rspace,
)
yo = rspace.reverse(ryo)
assert yo in space
def trial(exp):
"""Return a Trial which is registered in DB."""
trial = tuple_to_trial((1.0, ), exp.space)
heartbeat = datetime.datetime.utcnow()
trial.experiment = exp.id
trial.status = "reserved"
trial.heartbeat = heartbeat
get_storage().register_trial(trial)
return trial
def trial(exp):
"""Return a Trial which is registered in DB."""
trial = tuple_to_trial((1.0, ), exp.space)
heartbeat = datetime.datetime.utcnow()
trial.experiment = exp.id
trial.status = 'reserved'
trial.heartbeat = heartbeat
Database().write('trials', trial.to_dict())
return trial
def produce(self):
"""Create and register new trials."""
log.debug("### Suggest new ones.")
new_points = self.algorithm.suggest(self.num_new_trials)
log.debug("### Convert them to `Trial` objects.")
new_trials = list(
map(lambda data: format_trials.tuple_to_trial(data, self.space),
new_points))
log.debug("### Register to database: %s", new_trials)
self.experiment.register_trials(new_trials)
def suggest(self, num):
"""Suggest a `num`ber of new sets of parameters.
Perform a step towards negative gradient and suggest that point.
"""
num = 1 # Simple gradient descent only make sense for 1 point at a time.
if not self.has_observed_once:
return self.space.sample(1)
self.current_point -= self.learning_rate * self.gradient
return [format_trials.tuple_to_trial(self.current_point, self.space)]
def test_trial_working_dir_is_changed(config, monkeypatch):
"""Check that trial has its working_dir attribute changed."""
exp = experiment_builder.build(**config)
trial = tuple_to_trial((1.0, ), exp.space)
exp.register_trial(trial, status='reserved')
con = Consumer(exp)
con.consume(trial)
assert trial.working_dir is not None
assert trial.working_dir == con.working_dir + "/exp_" + trial.id
def test_tuple_to_trial(space, trial, fixed_suggestion):
"""Check if sample is recovered successfully from trial."""
t = tuple_to_trial(fixed_suggestion, space)
assert t.experiment is None
assert t.status == 'new'
assert t.worker is None
assert t.submit_time is None
assert t.start_time is None
assert t.end_time is None
assert t.results == []
assert len(t._params) == len(trial.params)
for i in range(len(t.params)):
assert t._params[i].to_dict() == trial._params[i].to_dict()
def flatten_numpy(trials_array, flattened_space):
"""Flatten dimensions"""
flattened_points = numpy.array([
format_trials.trial_to_tuple(
flattened_space.transform(
format_trials.tuple_to_trial(point[:-1],
flattened_space.original)),
flattened_space,
) for point in trials_array
])
return numpy.concatenate((flattened_points, trials_array[:, -1:]), axis=1)
def test_verify_trial(self, palgo, space):
trial = format_trials.tuple_to_trial((["asdfa", 2], 0, 3.5), space)
palgo._verify_trial(trial)
with pytest.raises(ValueError, match="not contained in space:"):
invalid_trial = format_trials.tuple_to_trial((("asdfa", 2), 10, 3.5), space)
palgo._verify_trial(invalid_trial)
# transform space
tspace = build_required_space(
space, type_requirement="real", shape_requirement="flattened"
)
# transform point
ttrial = tspace.transform(trial)
ttrial in tspace
# Transformed point is not in original space
with pytest.raises(ValueError, match="not contained in space:"):
palgo._verify_trial(ttrial)
# Transformed point is in transformed space
palgo._verify_trial(ttrial, space=tspace)
def main(args):
"""Fetch config and insert new point"""
command_line_user_args = args.pop('user_args', [None])[1:]
experiment_view = ExperimentBuilder().build_view_from(args)
transformed_args = _build_from(command_line_user_args)
exp_space = experiment_view.space
values = _create_tuple_from_values(transformed_args, exp_space)
trial = tuple_to_trial(values, exp_space)
ExperimentBuilder().build_from_config(
experiment_view.configuration).register_trials([trial])
def test_register_and_contain(self):
"""Register bunch of dimensions, check if points/name are in space."""
space = Space()
trial = Trial(params=[{"name": "no", "value": 0, "type": "integer"}])
assert "yolo" not in space
assert trial not in space
categories = {"asdfa": 0.1, 2: 0.2, 3: 0.3, 4: 0.4}
dim = Categorical("yolo", categories, shape=2)
space.register(dim)
dim = Integer("yolo2", "uniform", -3, 6)
space.register(dim)
dim = Real("yolo3", "norm", 0.9)
space.register(dim)
assert "yolo" in space
assert "yolo2" in space
assert "yolo3" in space
assert format_trials.tuple_to_trial((("asdfa", 2), 0, 3.5), space) in space
assert format_trials.tuple_to_trial((("asdfa", 2), 7, 3.5), space) not in space
def main(args):
"""Fetch config and insert new point"""
command_line_user_args = args.pop('user_args', [None])[1:]
experiment_view = experiment_builder.build_view_from_args(args)
experiment = experiment_builder.build(name=experiment_view.name,
version=experiment_view.version)
transformed_args = _build_from(command_line_user_args)
exp_space = experiment.space
values = _create_tuple_from_values(transformed_args, exp_space)
trial = tuple_to_trial(values, exp_space)
experiment.register_trial(trial)
def insert_trials(experiment_name, points, raise_exc=True):
"""Insert sets of parameters manually, defined in `points`, as new trials
for the experiment name, `experiment_name`.
.. warning::
This function is deprecated and will be removed in 0.3.0.
You should use ExperimentClient.insert() instead.
:param experiment_name: Name of the experiment which the new trials are
going to be associated with
:param points: list of tuples in agreement with experiment's parameter space
:param raise_exc: whether an inappropriate tuple of parameters will raise
an exception or it will be ignored
.. note:: If `raise_exc` is True, no set of parameters will be inserted. If
it is False, only the valid ones will be inserted; the rest will be ignored.
.. note:: This cannot be used to prepopulate a future experiment. So,
an experiment with `experiment_name` should already be configured in
the database.
"""
log.warning("insert_trials() is deprecated and will be removed in 0.3.0. "
"You should use ExperimentClient.insert() instead.")
experiment = create_experiment(experiment_name)
valid_points = []
for point in points:
try:
assert point in experiment.space
valid_points.append(point)
except AssertionError:
if raise_exc:
raise
if not valid_points:
return
new_trials = list(
map(
lambda data: format_trials.tuple_to_trial(data, experiment.space),
valid_points,
))
for new_trial in new_trials:
experiment.insert(new_trial.params)
