def main(args):
if not os.path.exists(config.cwd()):
cli.error(
"%s does not exist\n"
"Try specifying a different directory."
% cmd_impl_support.cwd_desc(config.cwd()))
package_file = guildfile.guildfile_path()
if not os.path.exists(package_file):
cli.error(
"%s does not contain a guild.yml file\n"
"A guild.yml file is required when creating a package. Create one "
"in this directory first or try specifying a different directory."
% cmd_impl_support.cwd_desc(config.cwd()))
if args.upload:
_check_upload_support(package_file)
package.create_package(
package_file,
dist_dir=args.dist_dir,
upload_repo=args.upload,
sign=args.sign,
identity=args.identity,
user=args.user,
password=args.password,
skip_existing=args.skip_existing,
comment=args.comment)
def create_package(args):
package_file = os.path.join(config.cwd(), "PACKAGE")
if not os.path.exists(package_file):
cli.error("%s does not contain a PACKAGE file\n"
"Try specifying a different directory." %
cmd_impl_support.cwd_desc(config.cwd()))
package.create_package(package_file,
dist_dir=args.dist_dir,
upload_repo=args.upload,
sign=args.sign,
identity=args.identity,
user=args.user,
password=args.password,
comment=args.comment)
def config(self):
cwd = util.format_dir(config.cwd())
return {
"cwd": cwd,
"titleLabel": self._title_label(cwd),
"version": guild.version(),
}
def main(args):
package_file = os.path.join(config.cwd(), "guild.yml")
if not os.path.exists(package_file):
cli.error("%s does not contain a guild.yml file\n"
"Try specifying a different directory." %
cmd_impl_support.cwd_desc(config.cwd()))
_check_upload_config(package_file, args)
package.create_package(package_file,
dist_dir=args.dist_dir,
upload_repo=args.upload,
sign=args.sign,
identity=args.identity,
user=args.user,
password=args.password,
skip_existing=args.skip_existing,
comment=args.comment)
def _apply_guild_patch():
"""Look in config cwd for guild_patch.py and load if exists."""
patch_path = os.path.join(config.cwd(), "guild_patch.py")
if os.path.exists(patch_path):
from guild import python_util
python_util.exec_script(patch_path)
def _cwd_remote_run(run):
try:
gf = guildfile.for_dir(config.cwd())
except:
return False
else:
return gf.package and gf.package.name == run.opref.pkg_name
def _cwd_guildfile():
try:
return guildfile.for_dir(config.cwd())
except guildfile.NoModels as e:
return None
except guildfile.GuildfileError as e:
raise CwdGuildfileError(e)
def _guildfile_dir(opref):
from guild import config
gf_dir = os.path.dirname(opref.pkg_name)
relpath = os.path.relpath(gf_dir, config.cwd())
if relpath == ".":
return ""
return re.sub(r"\.\./(\.\./)+", ".../", _ensure_dot_path(relpath))
def _batch_desc_preview_part(op):
opt_name = op.opref.to_opspec(config.cwd())
if opt_name == "+":
return " as a batch"
elif opt_name in ("random", "skopt:random"):
return " with random search"
else:
return " with '%s' optimizer" % opt_name
def resolve_model_op(opspec):
path = os.path.join(config.cwd(), opspec)
if not os.path.exists(path):
return None
if not util.is_executable_file(path):
raise plugin.ModelOpResolutionError("must be an executable file")
model = ExecScriptModelProxy(path, opspec)
return model, model.op_name
def path_or_package_guildfile(path_or_package, ctx=None):
path_or_package = path_or_package or config.cwd()
if os.path.isdir(path_or_package):
return _dir_guildfile(path_or_package, ctx)
elif os.path.isfile(path_or_package):
return _guildfile(path_or_package)
else:
return _package_guildfile(path_or_package)
def resolve_model_op(self, opspec):
path = os.path.join(config.cwd(), opspec)
if not python_util.is_python_script(path):
return None
script = python_util.Script(path)
if not self.is_keras_script(script):
return None
model = KerasScriptModelProxy(opspec, script.src)
return model, model.op_name
def main(args):
path_or_package = args.path_or_package or config.cwd()
if os.path.isdir(path_or_package):
modelfile = _dir_modelfile(path_or_package)
desc = None
else:
modelfile, desc = _package_modelfile(path_or_package)
help = _format_modelfile_help(modelfile, desc, args)
_print_help(help, args)
def _guildfile_pkg_data():
src = os.path.join(config.cwd(), "guild.yml")
if not os.path.exists(src):
return None
data = _guildfile_data(src)
for top_level in data:
if isinstance(top_level, dict) and "package" in top_level:
return top_level
return None
def _index_template(args):
if not args.index_template:
return None
index_template = os.path.join(config.cwd(), args.index_template)
if not os.path.exists(index_template):
cli.error("index template '%s' does not exist" % index_template)
if os.path.isdir(index_template):
return _index_template_readme(index_template)
else:
return index_template
def _init_user_reqs(args):
# -r options may be used with --no-reqs, in which case -r
# takes precedence (--no-reqs may still be used to suppress
# installation of Guild package reqs)
if args.requirement:
return args.requirement
elif not args.no_reqs:
default_reqs = os.path.join(config.cwd(), "requirements.txt")
if os.path.exists(default_reqs):
return (default_reqs, )
return ()
def _diff_working(args, ctx):
assert len(args.runs) == 2, args
assert args.runs[0] is not None and args.runs[1] is None, args
run = runs_impl.one_run(OneRunArgs(args, args.runs[0]), ctx)
run_sourcecode_dir = run.guild_path("sourcecode")
if args.working_dir:
working_dir = os.path.join(config.cwd(), args.working_dir)
else:
assert args.working
working_dir = _find_run_working_dir(run)
_diff(run_sourcecode_dir, working_dir, args)
def cwd_modelfile(cwd=None):
"""Returns the modelfile of the context cwd.
Returns None if a modelfile doesn't exist in the cwd.
"""
cwd = cwd or config.cwd()
try:
importer = guild.model.ModelImporter(cwd)
except ImportError:
return None
else:
return getattr(importer.dist, "modelfile", None)
def resolve_model_op(self, opspec):
path = os.path.join(config.cwd(), opspec)
if not python_util.is_python_script(path):
self.log.debug("%s is not a python script", path)
return None
script = python_util.Script(path)
if not self.is_keras_script(script):
self.log.debug("%s is not a Keras script", path)
return None
model = KerasScriptModelProxy(script.src, opspec)
self.log.debug("%s is a Keras script", path)
return model, model.op_name
def _run_op_in_background(run, op, pidfile, quiet, stop_after, extra_env):
import daemonize
action = lambda: _run_op(run, op, quiet, stop_after, extra_env)
daemon = daemonize.Daemonize(
app="guild_op", action=action, pid=pidfile, chdir=config.cwd()
)
# Need to log before starting daemon, otherwise output isn't
# visible.
if not quiet:
log.info(
"%s started in background as %s (pidfile %s)",
run.opref.to_opspec(config.cwd()),
run.id,
pidfile,
)
try:
daemon.start()
except SystemExit:
op_util.clear_run_pending(run)
raise
def cwd_desc(cwd=None):
"""Returns a description for cwd.
If cwd the system cwd, returns "this directory" otherwise returns
the quoted cwd.
This is used in messages to the user where the cwd is referenced.
"""
cwd = cwd or config.cwd()
if os.path.abspath(cwd) == os.path.abspath(os.getcwd()):
return "the current directory"
else:
return "'%s'" % cwd
def cwd_guildfile(cwd=None):
"""Returns the guildfile of the context cwd.
Returns None if a guildfile doesn't exist in the cwd.
"""
import guild.model # expensive
cwd = cwd or config.cwd()
try:
importer = guild.model.ModelImporter(cwd)
except ImportError:
return None
else:
_check_bad_guildfile(importer.dist)
return getattr(importer.dist, "guildfile", None)
def _init_modeldef(self):
data = [{
"model": self.name,
"operations": {
self.op_name: {
"description": self.op_description,
"exec": "${python_exe} -um %s" % self.module_name,
"flag-encoder": self.flag_encoder,
"default-max-trials": self.default_max_trials,
"flags": self.flags_data,
}
}
}]
gf = guildfile.Guildfile(data, dir=config.cwd())
return gf.models[self.name]
def _init_modeldef(self):
data = [{
"model": self.name,
"operations": {
"queue": {
"description":
queue_description,
"exec": ("${python_exe} -um guild.plugins.queue_main "
"${flag_args}"),
"flags":
queue_flags_data,
}
}
}]
gf = guildfile.Guildfile(data, dir=config.cwd())
return gf.models["queue"]
def _init_modeldef(self):
flags_data = self._flags_data()
flags_dest = flags_data.pop("$dest", None)
data = [{
"model": self.name,
"operations": {
self.op_name: {
"exec": self._exec_attr(),
"flags": flags_data,
"flags-dest": flags_dest,
"output-scalars": self.output_scalars,
"objective": self.objective,
"disable-plugins": self.disable_plugins,
"sourcecode": self._sourcecode(),
}
}
}]
gf = guildfile.Guildfile(data, dir=config.cwd())
return gf.models[self.name]
def _init_project_name(guildfile):
"""Returns a project name for a guildfile distribution.
Guildfile distribution project names are of the format:
'.guildfile.' + ESCAPED_GUILDFILE_PATH
ESCAPED_GUILDFILE_PATH is a 'safe' project name (i.e. will not be
modified in a call to `pkg_resources.safe_name`) that, when
unescaped using `unescape_project_name`, is the relative path of
the directory containing the guildfile. The modefile name itself
(e.g. 'guild.yml') is not contained in the path.
Guildfile paths are relative to the current working directory
(i.e. the value of os.getcwd() at the time they are generated) and
always start with '.'.
"""
pkg_path = os.path.relpath(guildfile.dir, config.cwd())
if pkg_path[0] != ".":
pkg_path = os.path.join(".", pkg_path)
safe_path = escape_project_name(pkg_path)
return ".guildfile.%s" % safe_path
def _init_guild_pkg_reqs(args):
if args.no_reqs:
return ()
reqs = list(_iter_all_guild_pkg_reqs(config.cwd(), args.path))
return tuple(sorted(reqs))
def _is_guildfile_dir(path):
return os.path.abspath(path) == os.path.abspath(
config.cwd()) or guildfile.is_guildfile_dir(path)
def guildfile_path(*paths):
if not paths:
paths = (config.cwd(),)
return os.path.join(*(paths + (NAME,)))
def _styled_op_dir_suffix(op_dir, apply_style):
cwd = os.path.abspath(config.cwd())
if util.compare_paths(op_dir, cwd):
return _empty_style("", apply_style)
shortened_op_dir = run_util.shorten_op_dir(op_dir, cwd)
return _dim_style(" (%s)" % shortened_op_dir, apply_style)