def _generate_trials(self,
num_samples,
unresolved_spec,
output_path="",
points_to_evaluate=None):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
points_to_evaluate = points_to_evaluate or []
while points_to_evaluate:
config = points_to_evaluate.pop(0)
for resolved_vars, spec in get_preset_variants(
unresolved_spec, config):
trial_id = self._uuid_prefix + ("%05d" % self._counter)
experiment_tag = str(self._counter)
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
num_samples -= 1
if num_samples <= 0:
return
for _ in range(num_samples):
for resolved_vars, spec in generate_variants(unresolved_spec):
trial_id = self._uuid_prefix + ("%05d" % self._counter)
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(format_vars(resolved_vars))
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"], suggested_config)
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(str(self._counter),
format_vars(flattened_config))
yield create_trial_from_spec(spec,
output_path,
self._parser,
experiment_tag=tag,
trial_id=trial_id)
def _generate_trials(self, unresolved_spec, output_path=""):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
for _ in range(unresolved_spec.get("num_samples", 1)):
for resolved_vars, spec in generate_variants(unresolved_spec):
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(resolved_vars)
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=resolved_vars,
experiment_tag=experiment_tag)
def create_trial_if_possible(self, experiment_spec: Dict,
output_path: str) -> Optional[Trial]:
logger.debug("creating trial")
trial_id = Trial.generate_id()
suggested_config = self.searcher.suggest(trial_id)
if suggested_config == Searcher.FINISHED:
self._finished = True
logger.debug("Searcher has finished.")
return
if suggested_config is None:
return
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"],
copy.deepcopy(suggested_config))
# Create a new trial_id if duplicate trial is created
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(str(self._counter),
format_vars(flattened_config))
trial = create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_dict(suggested_config),
experiment_tag=tag,
trial_id=trial_id)
return trial
def _generate_trials(self, unresolved_spec, output_path=""):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
for _ in range(unresolved_spec.get("num_samples", 1)):
for resolved_vars, spec in generate_variants(unresolved_spec):
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(resolved_vars)
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
experiment_tag=experiment_tag)
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"], suggested_config)
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(
str(self._counter), format_vars(flattened_config))
yield create_trial_from_spec(
spec,
output_path,
self._parser,
experiment_tag=tag,
trial_id=trial_id)
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("repeat", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = suggested_config
yield create_trial_from_spec(
spec,
output_path,
self._parser,
experiment_tag=format_vars(spec["config"]),
trial_id=trial_id)
def _generate_next_trials(self):
self._next_trials = []
if self._unfinished_count > 0:
# Last round not finished
return
trials = []
raw_param_list, extra_arg_list = self._select()
if not extra_arg_list:
extra_arg_list = [None] * len(raw_param_list)
for exp in self.experiment_list:
for param_config, extra_arg in zip(raw_param_list, extra_arg_list):
tag = ""
new_spec = copy.deepcopy(exp.spec)
for path, value in param_config.items():
tag += "%s=%s-" % (path.split(".")[-1], value)
deep_insert(path.split("."), value, new_spec["config"])
trial = create_trial_from_spec(new_spec,
exp.dir_name,
self._parser,
experiment_tag=tag)
# AutoML specific fields set in Trial
trial.results = []
trial.best_result = None
trial.param_config = param_config
trial.extra_arg = extra_arg
trial.invalidate_json_state()
trials.append(trial)
self._running_trials[trial.trial_id] = trial
ntrial = len(trials)
self._iteration += 1
self._unfinished_count = ntrial
self._total_trial_num += ntrial
self._start_ts = time.time()
logger.info(
"=========== BEGIN Experiment-Round: %(round)s "
"[%(new)s NEW | %(total)s TOTAL] ===========",
{
"round": self._iteration,
"new": ntrial,
"total": self._total_trial_num
},
)
self._next_trials = trials
def create_trial(self, resolved_vars, spec):
trial_id = self.uuid_prefix + ("%05d" % self.counter)
experiment_tag = str(self.counter)
# Always append resolved vars to experiment tag?
if resolved_vars:
experiment_tag += "_{}".format(format_vars(resolved_vars))
self.counter += 1
return create_trial_from_spec(
spec,
self.output_path,
self.parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
# spec = copy.deepcopy(experiment_spec)
# spec["config"] = suggested_config
self._counter += 1
def resolve(spec):
res = {}
for k, v in spec.items():
if isinstance(v, dict):
for k_, v_ in resolve(v).items():
res[(k, ) + k_] = v_
else:
res[(k, )] = v
return res
resolved_config = resolve({"config": suggested_config})
tag = "{0}_{1}".format(str(self._counter),
format_vars(resolved_config))
spec = merge_dicts(experiment_spec, {"config": suggested_config})
yield create_trial_from_spec(spec,
output_path,
self._parser,
experiment_tag=tag,
trial_id=trial_id)
def _generate_trials(self, num_samples, unresolved_spec, output_path=""):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
for _ in range(num_samples):
# Iterate over list of configs
for unresolved_cfg in unresolved_spec["config"]:
unresolved_spec_variant = deepcopy(unresolved_spec)
unresolved_spec_variant["config"] = unresolved_cfg
resolved_base_vars = CustomVariantGenerator._extract_resolved_base_vars(unresolved_cfg,
unresolved_spec["config"])
print("Resolved base cfg vars", resolved_base_vars)
for resolved_vars, spec in generate_variants(unresolved_spec_variant):
resolved_vars.update(resolved_base_vars)
print("Resolved vars", resolved_vars)
trial_id = "%05d" % self._counter
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(
format_vars({k: v for k, v in resolved_vars.items() if "tag" in k}))
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
def _generate_trials(self, num_samples, unresolved_spec, output_path=""):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
for _ in range(num_samples):
for resolved_vars, spec in generate_variants(unresolved_spec):
while True:
if self._num_live_trials() >= self._max_concurrent:
yield None
else:
break
trial_id = "%05d" % self._counter
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(format_vars(resolved_vars))
self._counter += 1
self._live_trials.add(trial_id)
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag,
)
