def _run_steps():
run = _init_run()
steps = _init_steps(run)
if not steps:
log.warning("no steps defined for run %s", run.id)
return
for step in steps:
step_run = _run_step(step, run)
_maybe_check_step_run(step, step_run)
def _try_copy_file(src, dest):
try:
shutil.copyfile(src, dest)
except (IOError, OSError) as e:
# This is not an error we want to stop an operation for. Log
# and continue.
if log.getEffectiveLevel() <= logging.DEBUG:
log.exception("copy %s to %s", src, dest)
else:
log.warning("could not copy source code file %s: %s", src, e)
def get_env(name, type, default=None):
try:
val = os.environ[name]
except KeyError:
return default
else:
try:
return type(val)
except Exception as e:
log.warning("error converting env %s to %s: %s", name, type, e)
return None
def _init_checks(data):
expect = data.get("expect") or []
if not isinstance(expect, list):
expect = [expect]
checks = []
for check_data in expect:
try:
check = run_check.init_check(check_data)
except ValueError as e:
log.warning("invalid check %r: %e", data, e)
else:
checks.append(check)
return checks
def _apply_template_transform(t, val):
parts = t.split(":", 1)
if len(parts) == 1:
name = parts[0]
arg = None
else:
name, arg = parts
if name[:1] == "%":
return _t_python_format(val, name)
elif name == "default":
return _t_default(val, arg)
elif name == "basename":
if arg:
log.warning("ignoring argment to baseline in %r", t)
return _t_basename(val)
else:
log.warning("unsupported template transform: %r", t)
return "#error#"
def _base_op_desc(opref, nowarn):
if opref.pkg_type == "guildfile":
return _format_guildfile_op(opref)
elif opref.pkg_type == "package":
return _format_package_op(opref)
elif opref.pkg_type == "script":
return _format_script_op(opref)
elif opref.pkg_type == "builtin":
return _format_builtin_op(opref)
elif opref.pkg_type == "pending":
return _format_pending_op(opref)
elif opref.pkg_type == "test":
return _format_test_op(opref)
else:
if not nowarn:
log.warning("cannot format op desc, unexpected pkg type: %s (%s)",
opref.pkg_type, opref.pkg_name)
return "?"
def _apply_data_params(data, ctx, run_spec):
"""Apply applicable data to params.
Warns if params contains unused values.
"""
defaults = {p.name: p.default for p in ctx.command.params}
for name, val in sorted(ctx.params.items()):
if name in STEP_USED_PARAMS:
data_name = name.replace("_", "-")
try:
data_val = data[data_name]
except KeyError:
pass
else:
if data_val != defaults[name]:
ctx.params[name] = data_val
else:
if val != defaults[name]:
log.warning("run parameter %s used in %r ignored", name, run_spec)
def gpu_available():
import ctypes
if "linux" in sys.platform:
lib = "libcublas.so"
elif sys.platform == "darwin":
lib = "libcublas.dylib"
elif sys.platform == "win32":
lib = "cublas.dll"
else:
log.warning("unable to detect GPU for platform '%s'", sys.platform)
lib = None
if lib:
log.debug("checking for GPU by loading %s", lib)
try:
ctypes.CDLL(lib)
except OSError as e:
log.debug("error loading '%s': %s", lib, e)
else:
log.debug("%s loaded", lib)
return True
return False
def op_flag_encoder(op):
import importlib
spec = op.opdef.flag_encoder
if not spec:
return None
parts = spec.split(":")
if len(parts) != 2:
log.warning("invalid flag decoder %r - must be MODULE:FUNCTION", spec)
return None
mod_name, fun_name = parts
try:
mod = importlib.import_module(mod_name)
except Exception as e:
if log.getEffectiveLevel() <= logging.DEBUG:
log.exception("importing %s", mod_name)
else:
log.warning("cannot load flag decoder %r: %s", spec, e)
return None
fun = getattr(mod, fun_name, None)
if fun is None:
log.warning(
"cannot load flag decoder %r: no such attribute in %s", spec, mod_name
)
return None
return fun
def _t_python_format(val, fmt):
try:
return fmt % val
except TypeError as e:
log.warning("error formatting %r with %r: %s", val, fmt, e)
return val
def safe_is_text_file(path, ignore_ext=False):
try:
return is_text_file(path, ignore_ext)
except OSError as e:
log.warning("could not check for text file %s: %s", path, e)
return False
def _anonymous_package_name(gf):
name = "gpkg.anonymous_%s" % _gf_digest(gf)
log.warning("package name not defined in %s - using %s", gf.src, name)
return name