def install_mod(raise_error=True):
modname = name.split(".")[0]
pkg = _MODULE_TO_PKG_NAME.get(modname, modname)
extra_flags = _MODULE_EXTRA_FLAGS.get(modname, [])
if version == LATEST_VERSION:
extra_flags.append("--upgrade")
elif version is not None:
pkg += version
cmd = config.INSTALL_CMD + [pkg] + extra_flags
G_LOGGER.info(
"{:} is required, but not installed. Attempting to install now.\n"
"Running: {:}".format(pkg, " ".join(cmd)))
status = sp.run(cmd)
if status.returncode != 0:
G_LOGGER.log(
"Could not automatically install required package: {:}. Please install it manually."
.format(pkg),
severity=G_LOGGER.CRITICAL
if raise_error else G_LOGGER.WARNING,
)
mod = importlib.import_module(name)
return mod
def log_output_stats(output,
info_hist=False,
runner_name=None,
hist_range=None):
ret = str_output_stats(output, runner_name)
G_LOGGER.info(ret)
with G_LOGGER.indent():
# Show histogram on failures.
G_LOGGER.log(lambda: str_histogram(output, hist_range),
severity=G_LOGGER.INFO if info_hist else G_LOGGER.VERBOSE)
def sort_artifacts(self, iteration, suffix=None):
"""
Run the check command and move artifacts into the correct subdirectory.
Args:
iteration (int):
The current iteration index. This is used to name artifacts
and display logging messages.
suffix (str):
A custom suffix to add to the artifact prior to moving it.
This will be applied in addition to the default suffix.
Returns:
bool: True if the command succeeded, False otherwise.
"""
def move_artifacts(subdir, returncode):
"""
Moves artifacts (args.artifacts) into the specified subdirectory or args.output and
appends an index and timestamp. Creates parent directories as required.
Args:
subdir (str): The destination path as a subdirectory of args.output.
index (int): The iteration index.
"""
for art in self.artifacts:
basename, ext = os.path.splitext(os.path.basename(art))
if suffix:
basename += suffix
name = "{:}_{:}_{:}_N{:}_ret{:}{:}".format(
basename, self.start_date, self.start_time, iteration,
returncode, ext)
dest = os.path.join(self.output, subdir, name)
if not os.path.exists(art):
G_LOGGER.error(
"Artifact: {:} does not exist, skipping.\n"
"Was the artifact supposed to be generated?".format(
art))
continue
if os.path.exists(dest):
G_LOGGER.error(
"Destination path: {:} already exists.\n"
"Refusing to overwrite. This artifact will be skipped!"
.format(dest))
continue
G_LOGGER.info("Moving {:} to {:}".format(art, dest))
dir_path = os.path.dirname(dest)
if dir_path:
dir_path = os.path.realpath(dir_path)
os.makedirs(dir_path, exist_ok=True)
shutil.move(art, dest)
def try_remove(path):
def func():
try:
os.remove(path)
except:
G_LOGGER.verbose("Could not remove: {:}".format(path))
return func
def is_success(status):
has_fail_regex = None
if self.fail_regexes is not None:
output = status.stdout.decode() + status.stderr.decode()
has_fail_regex = any(
regex.search(output) is not None
for regex in self.fail_regexes)
if self.fail_codes is not None:
# If a fail-code is specified, then we should also check has_fail_regex if provided.
failed = status.returncode in self.fail_codes
if has_fail_regex is not None:
failed &= has_fail_regex
else:
# If a fail-code is not specified, we should trigger failures even on 0-status
# if the fail regex is found.
failed = status.returncode != 0 if has_fail_regex is None else has_fail_regex
return not failed
with contextlib.ExitStack() as stack, G_LOGGER.indent():
if self.iter_artifact and self.remove_intermediate:
stack.callback(try_remove(self.iter_artifact))
if self.iteration_info:
util.save_json({"iteration": iteration}, self.iteration_info)
stack.callback(try_remove(self.iteration_info))
G_LOGGER.info("Running check command: {:}".format(" ".join(
self.check)))
status = sp.run(self.check, stdout=sp.PIPE, stderr=sp.PIPE)
success = is_success(status)
if self.show_output:
stderr_log_level = G_LOGGER.WARNING if success else G_LOGGER.ERROR
G_LOGGER.info(
"========== CAPTURED STDOUT ==========\n{:}".format(
status.stdout.decode()))
G_LOGGER.log(
"========== CAPTURED STDERR ==========\n{:}".format(
status.stderr.decode()),
severity=stderr_log_level)
if success:
move_artifacts("good", status.returncode)
G_LOGGER.finish("PASSED | Iteration {:}".format(iteration))
return True
else:
move_artifacts("bad", status.returncode)
G_LOGGER.error("FAILED | Iteration {:}".format(iteration))
return False
def log_module_info(module, name=None, severity=G_LOGGER.VERBOSE):
G_LOGGER.log(str_from_module_info(module, name), severity=severity)
