def _check_state_dims(state):
if not state.dim_names:
flag_desc = ", ".join(op_util.flag_assigns(state.proto_flags))
log.error(
"flags for batch (%s) do not contain any search "
"dimension - quitting", flag_desc)
raise batch_util.StopBatch(error=True)
def __call__(self, trial, state):
_check_state_dims(state)
next_trial_flags = self.seq_trial_cb(trial, state)
for run in trial.batch.seq_trial_runs():
if next_trial_flags == run.get("flags"):
flag_desc = ", ".join(op_util.flag_assigns(next_trial_flags))
log.warning(
"optimizer repeated trial (%s) - using random",
flag_desc)
next_trial_flags = self._random_trial_flags(state)
return next_trial_flags, {}
def __call__(self, trial, state):
next_trial_flags = self.seq_trial_cb(trial, state)
for run in trial.batch.seq_trial_runs():
if next_trial_flags == run.get("flags"):
flag_desc = " ".join(op_util.flag_assigns(next_trial_flags))
log.warning("optimizer repeated trial (%s) - using random",
flag_desc)
return (self._random_trial(state), {
"label": self._random_trial_label(trial, flag_desc)
})
return next_trial_flags, {}
def _row_for_print_run(run, flags, labels, status):
row = {"opspec": run_util.format_op_desc(run)}
if flags:
flag_vals = run.get("flags") or {}
flags_desc = " ".join(op_util.flag_assigns(flag_vals, quote=True))
row["flags"] = flags_desc
if labels:
row["label"] = run.get("label", "")
if status:
row["status"] = run.status
return row
def gen_trials(flags,
_runs,
max_trials=None,
random_seed=None,
label=None,
**kw):
"""Public interface for ipy."""
if kw:
log.warning("ignoring configuration %s", kw)
num_trials = max_trials or DEFAULT_TRIALS
dim_names, dims, _initials = skopt_util.flag_dims(flags)
if not dim_names:
flag_desc = ", ".join(op_util.flag_assigns(flags))
log.error(
"flags for batch (%s) do not contain any search "
"dimension - quitting", flag_desc)
raise batch_util.StopBatch(error=True)
trial_vals = _gen_trial_vals(dims, num_trials, random_seed)
trial_opts = {"label": label or "random"}
return [(_trial_flags(dim_names, dim_vals, flags), trial_opts)
for dim_vals in trial_vals]
def _popen_args(
opspec=None,
flags=None,
label=None,
run_dir=None,
restart=None,
stage=None,
rerun=None,
batch_files=None,
batch_label=None,
extra_env=None,
optimize=False,
optimizer=None,
minimize=None,
maximize=None,
opt_flags=None,
max_trials=None,
random_seed=None,
debug_sourcecode=None,
needed=False,
init_trials=False,
force_flags=False,
print_cmd=False,
print_trials=False,
save_trials=None,
guild_home=None,
cwd=None,
quiet=False,
debug=False,
test_sourcecode=False,
):
from guild import op_util
cwd = cwd or "."
flags = flags or {}
opt_flags = opt_flags or {}
args = [sys.executable, "-um", "guild.main_bootstrap"]
if debug:
args.append("--debug")
args.extend(["run", "-y"])
if opspec:
args.append(opspec)
if restart:
args.extend(["--restart", restart])
if stage:
args.append("--stage")
if rerun:
args.extend(["--rerun", rerun])
if label:
args.extend(['--label', label])
if batch_label:
args.extend(['--batch-label', batch_label])
args.extend(op_util.flag_assigns(flags))
args.extend(["@%s" % path for path in (batch_files or [])])
if run_dir:
args.extend(["--run-dir", run_dir])
if optimize:
args.append("--optimize")
if optimizer:
args.extend(["--optimizer", optimizer])
if minimize:
args.extend(["--minimize", minimize])
if maximize:
args.extend(["--maximize", maximize])
for name, val in sorted(opt_flags.items()):
args.extend(["--opt-flag", op_util.flag_assign(name, val)])
if max_trials is not None:
args.extend(["--max-trials", str(max_trials)])
if random_seed is not None:
args.extend(["--random-seed", str(random_seed)])
if debug_sourcecode:
args.extend(["--debug-sourcecode", debug_sourcecode])
if needed:
args.append("--needed")
if print_cmd:
args.append("--print-cmd")
if print_trials:
args.append("--print-trials")
if save_trials:
args.extend(["--save-trials", save_trials])
if init_trials:
args.append("--init-trials")
if force_flags:
args.append("--force-flags")
if quiet:
args.append("--quiet")
if test_sourcecode:
args.append("--test-sourcecode")
env = dict(os.environ)
env["NO_IMPORT_FLAGS_PROGRESS"] = "1"
if extra_env:
env.update(extra_env)
_apply_guild_home_env(env, guild_home)
_apply_python_path_env(env)
_apply_lang_env(env)
return args, cwd, env
def _popen_args(
opspec=None,
flags=None,
label=None,
tags=None,
comment=None,
run_dir=None,
restart=None,
start=None,
stage=None,
proto=None,
force_sourcecode=False,
batch_files=None,
batch_label=None,
batch_tags=None,
batch_comment=None,
extra_env=None,
optimize=False,
optimizer=None,
minimize=None,
maximize=None,
opt_flags=None,
max_trials=None,
random_seed=None,
debug_sourcecode=None,
needed=False,
stage_trials=False,
force_flags=False,
force_deps=False,
stop_after=None,
fail_on_trial_error=False,
print_cmd=False,
print_trials=False,
save_trials=None,
guild_home=None,
cwd=None,
quiet=False,
debug=False,
test_sourcecode=False,
gpus=None,
help_op=False,
run_id=None,
background=False,
keep_run=False,
keep_batch=False,
):
from guild import op_util
cwd = cwd or "."
flags = flags or {}
opt_flags = opt_flags or {}
tags = tags or []
batch_tags = batch_tags or []
args = [sys.executable, "-um", "guild.main_bootstrap"]
if debug:
args.append("--debug")
args.extend(["run", "-y"])
if opspec:
args.append(opspec)
if restart:
args.extend(["--restart", restart])
# start is a synonym for restart - pass this through as provided.
if start:
args.extend(["--start", start])
if stage:
args.append("--stage")
if proto:
args.extend(["--proto", proto])
if force_sourcecode:
args.append("--force-sourcecode")
if label:
args.extend(["--label", label])
for tag in tags:
args.extend(["--tag", tag])
if comment:
args.extend(["--comment", comment])
if batch_label:
args.extend(['--batch-label', batch_label])
for tag in batch_tags:
args.extend(["--batch-tag", tag])
if batch_comment:
args.extend(["--batch-comment", batch_comment])
if run_dir:
args.extend(["--run-dir", run_dir])
if optimize:
args.append("--optimize")
if optimizer:
args.extend(["--optimizer", optimizer])
if minimize:
args.extend(["--minimize", minimize])
if maximize:
args.extend(["--maximize", maximize])
for name, val in sorted(opt_flags.items()):
args.extend(["--opt-flag", op_util.flag_assign(name, val)])
if max_trials is not None:
args.extend(["--max-trials", str(max_trials)])
if random_seed is not None:
args.extend(["--random-seed", str(random_seed)])
if debug_sourcecode:
args.extend(["--debug-sourcecode", debug_sourcecode])
if needed:
args.append("--needed")
if print_cmd:
args.append("--print-cmd")
if print_trials:
args.append("--print-trials")
if save_trials:
args.extend(["--save-trials", save_trials])
if stage_trials:
args.append("--stage-trials")
if force_flags:
args.append("--force-flags")
if force_deps:
args.append("--force-deps")
if stop_after is not None:
args.extend(["--stop-after", str(stop_after)])
if fail_on_trial_error:
args.append("--fail-on-trial-error")
if quiet:
args.append("--quiet")
if test_sourcecode:
args.append("--test-sourcecode")
if gpus:
args.extend(["--gpus", str(gpus)])
if help_op:
args.append("--help-op")
if run_id:
args.extend(["--run-id", run_id])
if background:
args.append("--background")
if keep_run:
args.append("--keep-run")
if keep_batch:
args.append("--keep-batch")
args.extend(op_util.flag_assigns(flags))
args.extend(["@%s" % path for path in (batch_files or [])])
env = dict(os.environ)
env["NO_IMPORT_FLAGS_PROGRESS"] = "1"
if extra_env:
env.update(extra_env)
_apply_guild_home_env(env, guild_home)
_apply_python_path_env(env)
_apply_lang_env(env)
return args, cwd, env
def _flag_args(flags):
return tuple(op_util.flag_assigns(flags, skip_none=True))
def _flag_assigns(self):
return op_util.flag_assigns(self.flags)
