def _main():
_init_sys_path()
_init_logging()
_init_warnings()
log.debug("cwd: %s", os.getcwd())
log.debug("sys.path: %s", os.path.pathsep.join(sys.path))
arg1, rest_args = _parse_args()
_apply_plugins()
_try_module(arg1, rest_args)
def _main():
_init_logging()
log.debug("cwd: %s", os.getcwd())
log.debug("sys.path: %s", os.path.pathsep.join(sys.path))
arg1, rest_args = _parse_args()
_apply_plugins()
if arg1[0] == "@":
_try_plugin(arg1[1:], rest_args)
else:
_try_module(arg1, rest_args)
def _run_step(step, parent_run):
step_run = _init_step_run(parent_run)
cmd = _init_step_cmd(step, step_run.path)
_link_to_step_run(step, step_run.path, parent_run.path)
env = dict(os.environ)
env["NO_WARN_RUNDIR"] = "1"
log.info("running %s: %s", step, _format_step_cmd(cmd))
log.debug("cmd for %s: %s", step, cmd)
returncode = subprocess.call(cmd, env=env, cwd=os.getenv("CMD_DIR"))
if returncode != 0:
sys.exit(returncode)
return step_run
def _run_plugin_op(plugin, op_spec, args):
log.debug("running plugin op '@%s:%s' %r", plugin.name, op_spec, args)
try:
plugin.run_op(op_spec, args)
except Exception as e:
from guild.plugin import NotSupported # expensive
if isinstance(e, NotSupported):
_error("plugin '%s' does not support operation '%s'" %
(plugin.name, op_spec))
raise
except SystemExit as e:
_handle_system_exit(e)
def _try_module(module_spec, args):
package_path, module = _parse_module_spec(module_spec)
if package_path:
package_path = _try_resolve_package_path(package_path)
log.debug("using package path '%s'", package_path)
sys.path.insert(0, package_path)
log.debug("finding module '%s'", module)
try:
module_info = _find_module(module)
except ImportError as e:
_error(str(e))
else:
_dispatch_module_exec(_flags_dest(args), module_info)
def _try_module(module_spec, args):
package_path, module_path = _split_module(module_spec)
if package_path:
log.debug("using package path '%s'", package_path)
sys.path.insert(0, package_path)
log.debug("finding module '%s'", module_path)
try:
module_info = imp.find_module(module_path)
except ImportError as e:
_error(str(e))
else:
_set_argv_for_module(module_info, args)
_module_main(module_info)
def _gen_exec(module_info, exec_cb):
f, path, desc = module_info
log.debug("loading module from '%s'", path)
if os.getenv("SET_TRACE"):
debugger = Debugger()
debugger.runcall(imp.load_module, "__main__", f, path, desc)
else:
# Use a closure to handle anything post load_module, which
# effectively refefines the current module namespace.
handle_interrupt = _interrupt_handler()
try:
exec_cb()
except KeyboardInterrupt:
if not handle_interrupt:
raise
handle_interrupt()
def select_copytree(src, dest, config, copy_filter=None):
if not isinstance(config, list):
raise ValueError("invalid config: expected list got %r" % config)
log.debug("copying files from %s to %s", src, dest)
to_copy = _select_files_to_copy(src, config, copy_filter)
if not to_copy:
log.debug("no files to copy")
return
for file_src, file_src_rel_path in to_copy:
file_dest = os.path.join(dest, file_src_rel_path)
log.debug("copying file %s to %s", file_src, file_dest)
ensure_dir(os.path.dirname(file_dest))
_try_copy_file(file_src, file_dest)
def _run_step(step, parent_run):
step_run = _init_step_run(parent_run)
cmd = _init_step_cmd(step, step_run.path, parent_run)
_link_to_step_run(step, step_run.path, parent_run.path)
env = dict(os.environ)
env["NO_WARN_RUNDIR"] = "1"
if step.isolate_runs:
env["GUILD_RUNS_PARENT"] = parent_run.id
cwd = os.getenv("PROJECT_DIR") or os.getenv("CMD_DIR")
log.info("running %s: %s", step, _format_step_cmd(cmd))
log.debug("step cwd %s", cwd)
log.debug("step command: %s", cmd)
log.debug("step env: %s", env)
returncode = subprocess.call(cmd, env=env, cwd=cwd)
if returncode != 0:
sys.exit(returncode)
return step_run
def gpu_available():
import ctypes
if "linux" in sys.platform:
lib = "libcublas.so"
elif sys.platform == "darwin":
lib = "libcublas.dylib"
elif sys.platform == "win32":
lib = "cublas.dll"
else:
log.warning("unable to detect GPU for platform '%s'", sys.platform)
lib = None
if lib:
log.debug("checking for GPU by loading %s", lib)
try:
ctypes.CDLL(lib)
except OSError as e:
log.debug("error loading '%s': %s", lib, e)
else:
log.debug("%s loaded", lib)
return True
return False
def _select_files_to_copy(src_dir, config, copy_filter):
to_copy = []
seen_dirs = set()
log.debug("generating file list from %s", src_dir)
for root, dirs, files in os.walk(src_dir, followlinks=True):
seen_dirs.add(os.path.realpath(root))
_del_excluded_select_copy_dirs(root, dirs, seen_dirs, copy_filter)
for name in files:
path = os.path.join(root, name)
if not os.path.isfile(path):
continue
rel_path = os.path.relpath(path, src_dir)
log.debug("considering file to copy %s", path)
if _select_to_copy(path, rel_path, config, copy_filter):
log.debug("seleted file to copy %s", path)
to_copy.append((path, rel_path))
# Sort before notifying copy_filter to have deterministic result.
to_copy.sort()
if copy_filter:
copy_filter.pre_copy(to_copy)
return to_copy
def _set_argv_for_module_with_args(module_info, args):
_, path, _ = module_info
sys.argv = [path] + args
log.debug("argv: %s", sys.argv)
def _apply_plugin(name):
plugin = _plugin_for_name(name)
log.debug("patching env with plugin %r", name)
plugin.patch_env()
def _module_main(module_info):
f, path, desc = module_info
log.debug("loading module from '%s'", path)
imp.load_module("__main__", f, path, desc)
