def _format_flag(name, val, opdef):
if val is None:
formatted = _null_label(name, opdef)
else:
formatted = util.find_apply(
[_try_format_function, flag_util.encode_flag_val], val)
return "%s: %s" % (name, formatted)
def _maybe_restart_rerun_run_dir(self, kw):
"""Return the run dir for a rerun or restart kw.
If kw contains either a rerun or restart spec, performs a
lookup within the project Guild home for a single matching run
and returns its directory.
This is used to identify the run directory prior to passing
rerunning/restarting it.
"""
for name in ("rerun", "restart"):
spec = kw.get(name)
if spec:
from guild import run_util
from guild.commands import run_impl
with configlib.SetGuildHome(self.guild_home):
run = util.find_apply(
[
run_util.marked_or_latest_run_for_opspec,
run_impl.one_run
],
spec,
)
return run.dir
return None
def _find_run(run_spec, args):
if args.remote:
return remote_impl_support.one_run(run_spec, args)
else:
return util.find_apply(
[_run_for_dir, marked_or_latest_run_for_spec, one_run,], run_spec
)
def _find_include(self, include):
path = util.find_apply(
[self._local_include, self._sys_path_include, self._gpkg_include],
include)
if path:
return path
raise GuildfileIncludeError(self, include)
def _find_template(name, opdef):
return util.find_apply([
lambda: _abs_template(name),
lambda: _default_template(name),
lambda: _project_template(name, opdef),
lambda: _cannot_find_template_error(name),
])
def _can_start(run, blocking, state):
return util.find_apply([
lambda: _check_gpu_mismatch(run, state),
lambda: _check_blocking(run, blocking, state),
lambda: _check_state_can_start(run, state),
lambda: True,
])
def _init_template(template, opdef, run_dest):
template_spec = util.find_apply([
lambda: template,
lambda: _opdef_template(opdef)
])
template_path = _find_template(template_spec, opdef)
return Template(template_path, run_dest)
def resolve_file(filename):
return util.find_apply([
_abs_file,
_cmd_file,
_model_file,
_cwd_file
], filename)
def _coerce_opts_data(data, opdef):
if "optimizer" in data and "optimizers" in data:
raise GuildfileError(
opdef.modeldef.guildfile,
"conflicting optimizer configuration in operation '%s' "
"- cannot define both 'optimizer' and 'optimizers'" % opdef.name,
)
opts_data = util.find_apply([
lambda: data.get("optimizers"),
lambda: _coerce_single_optimizer(data.get("optimizer"), opdef),
lambda: {},
])
if isinstance(opts_data, list):
opts_data = {name: {} for name in opts_data}
elif isinstance(opts_data, dict):
pass
else:
raise GuildfileError(
opdef.modeldef.guildfile,
"invalid optimizer config %r: expected list or mapping" % data,
)
return {
name: _coerce_opt_data_item(opt_data)
for name, opt_data in opts_data.items()
}
def parse_opspec(spec):
return util.find_apply([
_empty_spec,
_op_spec,
_model_op_spec,
_package_model_op_spec,
_package_op_spec,
], spec)
def _env_activate_cmd_lines(self):
return util.find_apply(
[
self._explicit_venv_activate,
self._conda_env_activate,
self._default_venv_activate,
]
)
def _batch_op_init_for_opdef_default_optimizer(opdef, S):
assert not S.batch_op
S.batch_op = Operation()
optdef = util.find_apply([
lambda: opdef.default_optimizer,
lambda: _default_optimizer(opdef),
])
_op_init_for_optimizer(optdef, S.batch_op)
def _try_implied_batch_op_init(user_op, S):
batch_opspec = util.find_apply([
lambda: _batch_opspec_for_flags(user_op._op_flag_vals),
lambda: _batch_opspec_for_trials(user_op._batch_trials),
])
if batch_opspec:
assert not S.batch_op
S.batch_op = Operation()
S.batch_op._opdef = _opdef_for_opspec(batch_opspec)
def _find_run_working_dir(run):
working_dir = util.find_apply([_opdef_sourcecode_root, _script_source],
run)
if not working_dir:
cli.error(
"cannot find working source code directory for run {run_id}\n"
"Try specifying the directory with 'guild diff {run_id} "
"--working-dir DIR'.".format(run_id=run.short_id))
return working_dir
def _init_guild_home(self, config):
return util.find_apply(
[
self._explicit_guild_home,
self._guild_home_from_conda_env,
self._guild_home_from_venv,
self._default_guild_home,
],
config,
)
def _resdef_for_dep(depdef, flag_vals):
if depdef.inline_resource:
return depdef.inline_resource, depdef.opdef.guildfile.dir
res_spec = util.resolve_refs(depdef.spec, flag_vals)
return util.find_apply([
_model_resource,
_guildfile_resource,
_package_resource,
_invalid_dependency_error,
], res_spec, depdef)
def _dashboard_address(args):
address = util.find_apply(
[
_false_flag_for_arg,
_port_int,
lambda s: s,
],
args.dashboard_address,
)
return _dashboard_address_for_typed_val(address)
def _run_op_compare(run, index):
"""Returns compare cols for run.
If we can get the current compare cols for the run op source
definition (in the Guild file) we use that, otherwise we use the
run "compare" attr.
"""
return util.find_apply(
[_try_guildfile_compare, _run_compare_attr, _default_run_compare], run,
index)
def _run_for_pid_arg(pid):
return util.find_apply(
[
_run_for_job_pidfile,
_run_for_pidfile,
_run_for_pid,
_handle_no_run_for_pid_arg,
],
pid,
)
def _remove_bound_method_self(op, op_flag_vals):
im_self = util.find_apply(
[lambda: getattr(op, "__self__", None), lambda: getattr(op, "im_self", None),]
)
if im_self:
for key, val in op_flag_vals.items():
if val is im_self:
del op_flag_vals[key]
break
else:
assert False, (op_flag_vals, im_self)
def _dependency_resource(spec, ctx):
res = util.find_apply(
[_model_resource,
_modelfile_resource,
_packaged_resource],
spec, ctx)
if res:
return res
raise DependencyError(
"invalid dependency '%s' in operation '%s'"
% (spec, ctx.opdef.fullname))
def _op_compare(run, index):
"""Returns compare cols for run.
If we can get the current compare cols for the run op source
definition (in the Guild file) we use that, otherwise we use the
run "compare" attr.
"""
return util.find_apply([
lambda: run_util.latest_compare(run),
lambda: _default_run_compare(run, index)
])
def _sys_arg_guild_exe_for_win():
arg0 = sys.argv[0]
arg0_basename = os.path.basename(arg0)
if arg0_basename in ("guild.cmd", "guild.exe"):
return arg0
if arg0_basename == "guild":
arg0_dir = os.path.dirname(arg0)
return util.find_apply([
lambda: _test_exe_path(os.path.join(arg0_dir, "guild.exe")),
lambda: _test_exe_path(os.path.join(arg0_dir, "guild.cmd")),
])
return None
def _all_runs_f(root, force_root):
root = root or runs_dir()
if force_root:
return _default_all_runs_f(root)
return util.find_apply(
[
_zipfile_all_runs_f,
_runs_under_parent_f,
_default_all_runs_f,
],
root,
)
def wsgi_app(logdir,
plugins,
reload_interval=None,
path_prefix="",
tensorboard_options=None):
f = util.find_apply(
[_wsgi_app_v1_f, _wsgi_app_v2_f, _wsgi_app_unsupported_error])
return f(
logdir,
plugins,
reload_interval=reload_interval,
path_prefix=path_prefix,
tensorboard_options=tensorboard_options,
)
def _coerce_opts_data(data, opdef):
if "optimizer" in data and "optimizers" in data:
raise GuildfileError(
opdef.modeldef.guildfile,
"conflicting optimizer configuration in operation '%s' "
"- cannot define both 'optimizer' and 'optimizers'" % opdef.name)
opts_data = util.find_apply([
lambda: data.get("optimizers"),
lambda: _coerce_single_optimizer(data.get("optimizer"), opdef),
lambda: {}
])
return {
name: _coerce_opt_data_item(opt_data)
for name, opt_data in opts_data.items()
}
def _run_dir_apply(self, kw):
"""Returns a run directory for kw, optionally apply it to kw.
If kw contains an explicit run directory, returns
it. Otherwise checks if kw is a restart/rerun and if so
returns the run directory associated with the
rerun/restart. If it's a normal run, creates a new run ID and
applies it to kw.
This scheme is used so that we know the run directory prior to
running an operation. This lets us return a corresponding run
object after the operation is finished.
"""
return util.find_apply([
lambda: kw.get("run_dir"),
lambda: self._maybe_restart_rerun_run_dir(kw),
lambda: self._init_run_dir_apply(kw),
])
def _dependency_resource(dep, flag_vals, ctx):
if dep.inline_resource:
return _inline_resource(dep.inline_resource, ctx)
spec = util.resolve_refs(dep.spec, flag_vals)
try:
res = util.find_apply(
[_model_resource, _guildfile_resource, _packaged_resource], spec,
ctx)
except DependencyError as e:
if spec in ctx.resource_config:
log.warning(str(e))
return ResourceProxy(dep, spec, ctx.resource_config[spec], ctx)
raise
if res:
res.dependency = spec
return res
raise DependencyError("invalid dependency '%s' in operation '%s'" %
(spec, ctx.opdef.fullname))
def _suggest_python_interpreter(user_reqs):
"""Returns best guess Python interpreter.
We look for a Python requirement in two places: a package def in
cwd Guild file and user requirements, which includes
requirements.txt by default.
A Python spec can be provided in a requirements file using a
special `python<spec>` comment. E.g. `python>=3.5`.
"""
python_requires = util.find_apply(
[_python_requires_for_pkg, lambda: _python_requires_for_reqs(user_reqs),]
)
if not python_requires:
return None
matching = util.find_python_interpreter(python_requires)
if matching:
_path, ver = matching
return "python%s" % ver
return None
def _load(self, name):
return util.find_apply([
self._load_yaml,
self._load_csv,
self._load_txt], name)
