def make_experiment_tag(orig_tag, config, mutations):
"""Appends perturbed params to the trial name to show in the console."""
resolved_vars = {}
for k in mutations.keys():
resolved_vars[("config", k)] = config[k]
return "{}@perturbed[{}]".format(orig_tag, format_vars(resolved_vars))
def _generate_trials(self, num_samples, unresolved_spec, output_path=""):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
for _ in range(num_samples):
for resolved_vars, spec in generate_variants(unresolved_spec):
trial_id = self._uuid_prefix + ("%05d" % self._counter)
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(format_vars(resolved_vars))
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"], suggested_config)
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(str(self._counter),
format_vars(flattened_config))
yield create_trial_from_spec(spec,
output_path,
self._parser,
experiment_tag=tag,
trial_id=trial_id)
def create_trial_if_possible(self, experiment_spec: Dict,
output_path: str) -> Optional[Trial]:
logger.debug("creating trial")
trial_id = Trial.generate_id()
suggested_config = self.searcher.suggest(trial_id)
if suggested_config == Searcher.FINISHED:
self._finished = True
logger.debug("Searcher has finished.")
return
if suggested_config is None:
return
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"],
copy.deepcopy(suggested_config))
# Create a new trial_id if duplicate trial is created
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(str(self._counter),
format_vars(flattened_config))
trial = create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_dict(suggested_config),
experiment_tag=tag,
trial_id=trial_id)
return trial
def make_experiment_tag(orig_tag, config, mutations):
"""Appends perturbed params to the trial name to show in the console."""
resolved_vars = {}
for k in mutations.keys():
resolved_vars[("config", k)] = config[k]
return "{}@perturbed[{}]".format(orig_tag, format_vars(resolved_vars))
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = merge_dicts(spec["config"], suggested_config)
flattened_config = resolve_nested_dict(spec["config"])
self._counter += 1
tag = "{0}_{1}".format(
str(self._counter), format_vars(flattened_config))
yield create_trial_from_spec(
spec,
output_path,
self._parser,
experiment_tag=tag,
trial_id=trial_id)
def _generate_resolved_specs(self, num_samples, unresolved_spec):
"""Needed for slurm_cluster.py
"""
for _ in range(num_samples):
# Iterate over list of configs
for unresolved_cfg in unresolved_spec["config"]:
unresolved_spec_variant = deepcopy(unresolved_spec)
unresolved_spec_variant["config"] = unresolved_cfg
resolved_base_vars = CustomVariantGenerator._extract_resolved_base_vars(unresolved_cfg,
unresolved_spec["config"])
print("Resolved base cfg vars", resolved_base_vars)
for resolved_vars, spec in generate_variants(unresolved_spec_variant):
resolved_vars.update(resolved_base_vars)
print("Resolved vars", resolved_vars)
trial_id = "%05d" % self._counter
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(
format_vars({k: v for k, v in resolved_vars.items() if "tag" in k}))
self._counter += 1
yield {
"spec": spec,
"evaluated_params": flatten_resolved_vars(resolved_vars),
"trial_id": trial_id,
"experiment_tag": experiment_tag
}
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("repeat", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
spec = copy.deepcopy(experiment_spec)
spec["config"] = suggested_config
yield create_trial_from_spec(
spec,
output_path,
self._parser,
experiment_tag=format_vars(spec["config"]),
trial_id=trial_id)
def create_trial(self, resolved_vars, spec):
trial_id = self.uuid_prefix + ("%05d" % self.counter)
experiment_tag = str(self.counter)
# Always append resolved vars to experiment tag?
if resolved_vars:
experiment_tag += "_{}".format(format_vars(resolved_vars))
self.counter += 1
return create_trial_from_spec(
spec,
self.output_path,
self.parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
def _generate_trials(self, experiment_spec, output_path=""):
"""Generates trials with configurations from `_suggest`.
Creates a trial_id that is passed into `_suggest`.
Yields:
Trial objects constructed according to `spec`
"""
if "run" not in experiment_spec:
raise TuneError("Must specify `run` in {}".format(experiment_spec))
for _ in range(experiment_spec.get("num_samples", 1)):
trial_id = Trial.generate_id()
while True:
suggested_config = self._suggest(trial_id)
if suggested_config is None:
yield None
else:
break
# spec = copy.deepcopy(experiment_spec)
# spec["config"] = suggested_config
self._counter += 1
def resolve(spec):
res = {}
for k, v in spec.items():
if isinstance(v, dict):
for k_, v_ in resolve(v).items():
res[(k, ) + k_] = v_
else:
res[(k, )] = v
return res
resolved_config = resolve({"config": suggested_config})
tag = "{0}_{1}".format(str(self._counter),
format_vars(resolved_config))
spec = merge_dicts(experiment_spec, {"config": suggested_config})
yield create_trial_from_spec(spec,
output_path,
self._parser,
experiment_tag=tag,
trial_id=trial_id)
def _generate_trials(self, num_samples, unresolved_spec, output_path=""):
"""Generates Trial objects with the variant generation process.
Uses a fixed point iteration to resolve variants. All trials
should be able to be generated at once.
See also: `ray.tune.suggest.variant_generator`.
Yields:
Trial object
"""
if "run" not in unresolved_spec:
raise TuneError("Must specify `run` in {}".format(unresolved_spec))
for _ in range(num_samples):
# Iterate over list of configs
for unresolved_cfg in unresolved_spec["config"]:
unresolved_spec_variant = deepcopy(unresolved_spec)
unresolved_spec_variant["config"] = unresolved_cfg
resolved_base_vars = CustomVariantGenerator._extract_resolved_base_vars(unresolved_cfg,
unresolved_spec["config"])
print("Resolved base cfg vars", resolved_base_vars)
for resolved_vars, spec in generate_variants(unresolved_spec_variant):
resolved_vars.update(resolved_base_vars)
print("Resolved vars", resolved_vars)
trial_id = "%05d" % self._counter
experiment_tag = str(self._counter)
if resolved_vars:
experiment_tag += "_{}".format(
format_vars({k: v for k, v in resolved_vars.items() if "tag" in k}))
self._counter += 1
yield create_trial_from_spec(
spec,
output_path,
self._parser,
evaluated_params=flatten_resolved_vars(resolved_vars),
trial_id=trial_id,
experiment_tag=experiment_tag)
