def save_results(self, results):
"""Save the results to the test specific results file and the general
pavilion results file.
:param dict results: The results dictionary.
"""
results_tmp_path = self.results_path.with_suffix('.tmp')
with PermissionsManager(results_tmp_path, self.group, self.umask), \
results_tmp_path.open('w') as results_file:
json.dump(results, results_file)
try:
self.results_path.unlink()
except OSError:
pass
results_tmp_path.rename(self.results_path)
self.result = results.get('result')
self.save_attributes()
result_logger = logging.getLogger('common_results')
if self._pav_cfg.get('flatten_results') and results.get('per_file'):
# Flatten 'per_file' results into separate result records.
base = results.copy()
del base['per_file']
for per_file, values in results['per_file'].items():
per_result = base.copy()
per_result['file'] = per_file
per_result.update(values)
result_logger.info(output.json_dumps(per_result))
else:
result_logger.info(output.json_dumps(results))
def run_cmd(pav_cfg, args):
try:
cmd = commands.get_command(args.command_name)
except KeyError:
output.fprint(
"Unknown command '{}'."
.format(args.command_name),
color=output.RED,
file=sys.stderr)
sys.exit(-1)
try:
sys.exit(cmd.run(pav_cfg, args))
except KeyboardInterrupt:
sys.exit(-1)
except Exception as err:
exc_info = {
'traceback': traceback.format_exc(),
'args': vars(args),
'config': pav_cfg,
}
json_data = output.json_dumps(exc_info)
logger = logging.getLogger('exceptions')
logger.error(json_data)
output.fprint(
"Unknown error running command {}: {}."
.format(args.command_name, err),
color=output.RED,
file=sys.stderr,
)
traceback.print_exc(file=sys.stderr)
output.fprint(
"Traceback logged to {}".format(pav_cfg.exception_log),
color=output.RED,
file=sys.stderr,
)
sys.exit(-1)
def main():
# Pavilion is compatible with python >= 3.4
if sys.version_info[0] != 3 or sys.version_info[1] < 4:
output.fprint("Pavilion requires python 3.4 or higher.",
color=output.RED,
file=sys.stderr)
sys.exit(-1)
# Get the config, and
try:
pav_cfg = config.find()
except Exception as err:
output.fprint(
"Error getting config, exiting: {}"
.format(err),
file=sys.stderr,
color=output.RED)
sys.exit(-1)
# Create the basic directories in the working directory and the .pavilion
# directory.
for path in [
config.USER_HOME_PAV,
config.USER_HOME_PAV/'working_dir',
pav_cfg.working_dir/'builds',
pav_cfg.working_dir/'downloads',
pav_cfg.working_dir/'series',
pav_cfg.working_dir/'test_runs',
pav_cfg.working_dir/'users']:
try:
path = path.expanduser()
path.mkdir(exist_ok=True)
except OSError as err:
output.fprint(
"Could not create base directory '{}': {}"
.format(path, err),
color=output.RED,
file=sys.stderr,
)
sys.exit(1)
root_logger = logging.getLogger()
# Set up a directory for tracebacks.
tracebacks_dir = Path('~/.pavilion/tracebacks').expanduser()
tracebacks_dir.mkdir(parents=True, exist_ok=True)
# Setup the logging records to contain host information, just like in
# the logging module example
old_factory = logging.getLogRecordFactory()
hostname = socket.gethostname()
def record_factory(*fargs, **kwargs):
record = old_factory(*fargs, **kwargs)
record.hostname = hostname
return record
# Setup the new record factory.
logging.setLogRecordFactory(record_factory)
# Put the log file in the lowest common pav config directory we can write
# to.
log_fn = pav_cfg.working_dir/'pav.log'
# Set up a rotating logfile than rotates when it gets larger
# than 1 MB.
try:
log_fn.touch()
except (PermissionError, FileNotFoundError) as err:
output.fprint("Could not write to pavilion log at '{}': {}"
.format(log_fn, err),
color=output.YELLOW,
file=sys.stderr,
)
else:
file_handler = RotatingFileHandler(filename=str(log_fn),
maxBytes=1024 ** 2,
backupCount=3)
file_handler.setFormatter(logging.Formatter(pav_cfg.log_format,
style='{'))
file_handler.setLevel(getattr(logging,
pav_cfg.log_level.upper()))
root_logger.addHandler(file_handler)
# The root logger should pass all messages, even if the handlers
# filter them.
root_logger.setLevel(logging.DEBUG)
# Setup the result logger.
# Results will be logged to both the main log and the result log.
try:
pav_cfg.result_log.touch()
except (PermissionError, FileNotFoundError) as err:
output.fprint(
"Could not write to result log at '{}': {}"
.format(pav_cfg.result_log, err),
color=output.YELLOW,
file=sys.stderr
)
sys.exit(1)
result_logger = logging.getLogger('results')
result_handler = RotatingFileHandler(filename=str(pav_cfg.result_log),
# 20 MB
maxBytes=20 * 1024 ** 2,
backupCount=3)
result_handler.setFormatter(logging.Formatter("{message}", style='{'))
result_logger.setLevel(logging.INFO)
result_logger.addHandler(result_handler)
# Setup the exception logger.
# Exceptions will be logged to this directory, along with other useful info.
exc_logger = logging.getLogger('exceptions')
try:
pav_cfg.exception_log.touch()
except (PermissionError, FileNotFoundError) as err:
output.fprint(
"Could not write to exception log at '{}': {}"
.format(pav_cfg.exception_log, err),
color=output.YELLOW,
file=sys.stderr
)
else:
exc_handler = RotatingFileHandler(
filename=pav_cfg.exception_log.as_posix(),
maxBytes=20 * 1024 ** 2,
backupCount=3,
)
exc_handler.setFormatter(logging.Formatter(
"{asctime} {message}",
style='{',
))
exc_logger.setLevel(logging.ERROR)
exc_logger.addHandler(exc_handler)
# Setup the yapsy logger to log to terminal. We need to know immediatly
# when yapsy encounters errors.
yapsy_logger = logging.getLogger('yapsy')
yapsy_handler = StreamHandler(stream=sys.stderr)
# Color all these error messages red.
yapsy_handler.setFormatter(
logging.Formatter("\x1b[31m{asctime} {message}\x1b[0m",
style='{'))
yapsy_logger.setLevel(logging.INFO)
yapsy_logger.addHandler(yapsy_handler)
# This has to be done before we initialize plugins
parser = arguments.get_parser()
# Initialize all the plugins
try:
plugins.initialize_plugins(pav_cfg)
except plugins.PluginError as err:
output.fprint(
"Error initializing plugins: {}"
.format(err),
color=output.RED,
file=sys.stderr)
sys.exit(-1)
pav_cfg.pav_vars = pav_vars.PavVars()
# Parse the arguments
try:
args = parser.parse_args()
except Exception:
# TODO: Handle argument parsing errors correctly.
raise
# Add a stream to stderr if we're in verbose mode, or if no other handler
# is defined.
if args.verbose or not root_logger.handlers:
verbose_handler = logging.StreamHandler(sys.stderr)
verbose_handler.setLevel(logging.DEBUG)
verbose_handler.setFormatter(logging.Formatter(pav_cfg.log_format,
style='{'))
root_logger.addHandler(result_handler)
if args.command_name is None:
parser.print_help()
sys.exit(0)
try:
cmd = commands.get_command(args.command_name)
except KeyError:
output.fprint(
"Unknown command '{}'."
.format(args.command_name),
color=output.RED,
file=sys.stderr)
sys.exit(-1)
try:
sys.exit(cmd.run(pav_cfg, args))
except Exception as err:
exc_info = {
'traceback': traceback.format_exc(),
'args': vars(args),
'config': pav_cfg,
}
json_data = output.json_dumps(exc_info)
logger = logging.getLogger('exceptions')
logger.error(json_data)
output.fprint(
"Unknown error running command {}: {}."
.format(args.command_name, err),
color=output.RED,
file=sys.stderr,
)
traceback.print_exc(file=sys.stderr)
output.fprint(
"Traceback logged to {}".format(pav_cfg.exception_log),
color=output.RED,
file=sys.stderr,
)
sys.exit(-1)
def _run(self, pav_cfg, test):
"""Run an already prepped test in the current environment.
"""
try:
sched = schedulers.get_plugin(test.scheduler)
except Exception:
test.status.set(STATES.BUILD_ERROR,
"Unknown error getting the scheduler. Refer to "
"the kickoff log.")
raise
# Re-add var sets that may have had deferred variables.
try:
var_man = VariableSetManager()
var_man.add_var_set('sys', system_variables.get_vars(defer=False))
sched_config = test.config[test.scheduler]
var_man.add_var_set('sched', sched.get_vars(sched_config))
except Exception:
test.status.set(STATES.RUN_ERROR,
"Unknown error getting pavilion variables at "
"run time.")
raise
try:
test.finalize(var_man)
except Exception:
test.status.set(STATES.RUN_ERROR,
"Unknown error finalizing test.")
raise
try:
if test.config['build']['on_nodes'] in ['true', 'True']:
if not test.build():
self.logger.warning(
"Test {t.id} failed to build:"
)
except Exception:
test.status.set(STATES.BUILD_ERROR,
"Unknown build error. Refer to the kickoff log.")
raise
# Optionally wait on other tests running under the same scheduler.
# This depends on the scheduler and the test configuration.
lock = sched.lock_concurrency(pav_cfg, test)
try:
run_result = test.run()
except TestRunError as err:
test.status.set(STATES.RUN_ERROR, err)
return 1
except TimeoutError:
return 1
except Exception:
test.status.set(
STATES.RUN_ERROR,
"Unknown error while running test. Refer to the kickoff log.")
raise
finally:
sched.unlock_concurrency(lock)
try:
rp_errors = []
# Make sure the result parsers have reasonable arguments.
# We check here because the parser code itself will likely assume
# the args are valid form _check_args, but those might not be
# checkable before kickoff due to deferred variables.
try:
result_parsers.check_args(test.config['results'])
except TestRunError as err:
rp_errors.append(str(err))
if rp_errors:
for msg in rp_errors:
test.status.set(STATES.RESULTS_ERROR, msg)
test.set_run_complete()
return 1
results = test.gather_results(run_result)
except result_parsers.ResultParserError as err:
self.logger.error("Unexpected error gathering results: %s", err)
test.status.set(STATES.RESULTS_ERROR,
"Error parsing results: {}".format(err))
return 1
try:
test.save_results(results)
result_logger = logging.getLogger('results')
result_logger.info(output.json_dumps(results))
except Exception:
test.status.set(
STATES.RESULTS_ERROR,
"Unknown error while saving results. Refer to the kickoff log.")
raise
try:
test.status.set(STATES.COMPLETE,
"The test completed with result: {}"
.format(results.get('result', '<unknown>')))
except Exception:
test.status.set(
STATES.UNKNOWN,
"Unknown error while setting test completion. Refer to the "
"kickoff log.")
raise
def main():
"""Setup Pavilion and run a command."""
# Pavilion is compatible with python >= 3.4
if sys.version_info[0] != 3 or sys.version_info[1] < 4:
output.fprint("Pavilion requires python 3.4 or higher.",
color=output.RED,
file=sys.stderr)
sys.exit(-1)
# Get the config, and
try:
pav_cfg = config.find()
except Exception as err:
output.fprint("Error getting config, exiting: {}".format(err),
file=sys.stderr,
color=output.RED)
sys.exit(-1)
# Create the basic directories in the working directory and the .pavilion
# directory.
for path in [
config.USER_HOME_PAV, config.USER_HOME_PAV / 'working_dir',
pav_cfg.working_dir / 'builds', pav_cfg.working_dir / 'series',
pav_cfg.working_dir / 'test_runs', pav_cfg.working_dir / 'users'
]:
try:
path = path.expanduser()
path.mkdir(exist_ok=True)
except OSError as err:
output.fprint(
"Could not create base directory '{}': {}".format(path, err),
color=output.RED,
file=sys.stderr,
)
sys.exit(1)
# Setup all the loggers for Pavilion
if not log_setup.setup_loggers(pav_cfg):
sys.exit(1)
# This has to be done before we initialize plugins
parser = arguments.get_parser()
# Initialize all the plugins
try:
plugins.initialize_plugins(pav_cfg)
except plugins.PluginError as err:
output.fprint("Error initializing plugins: {}".format(err),
color=output.RED,
file=sys.stderr)
sys.exit(-1)
pav_cfg.pav_vars = pavilion_variables.PavVars()
# Parse the arguments
try:
args = parser.parse_args()
except Exception:
# TODO: Handle argument parsing errors correctly.
raise
if args.command_name is None:
parser.print_help()
sys.exit(0)
try:
cmd = commands.get_command(args.command_name)
except KeyError:
output.fprint("Unknown command '{}'.".format(args.command_name),
color=output.RED,
file=sys.stderr)
sys.exit(-1)
try:
sys.exit(cmd.run(pav_cfg, args))
except Exception as err:
exc_info = {
'traceback': traceback.format_exc(),
'args': vars(args),
'config': pav_cfg,
}
json_data = output.json_dumps(exc_info)
logger = logging.getLogger('exceptions')
logger.error(json_data)
output.fprint(
"Unknown error running command {}: {}.".format(
args.command_name, err),
color=output.RED,
file=sys.stderr,
)
traceback.print_exc(file=sys.stderr)
output.fprint(
"Traceback logged to {}".format(pav_cfg.exception_log),
color=output.RED,
file=sys.stderr,
)
sys.exit(-1)
def _run(self, pav_cfg, test, sched):
"""Run an already prepped test in the current environment.
:param pav_cfg: The pavilion configuration object.
:param TestRun test: The test to run
:param sched: The scheduler we're running under.
:return:
"""
# Optionally wait on other tests running under the same scheduler.
# This depends on the scheduler and the test configuration.
lock = sched.lock_concurrency(pav_cfg, test)
try:
run_result = test.run()
except TestRunError as err:
test.status.set(STATES.RUN_ERROR, err)
return 1
except TimeoutError:
return 1
except Exception:
test.status.set(
STATES.RUN_ERROR,
"Unknown error while running test. Refer to the kickoff log.")
raise
finally:
sched.unlock_concurrency(lock)
try:
rp_errors = []
# Make sure the result parsers have reasonable arguments.
# We check here because the parser code itself will likely assume
# the args are valid form _check_args, but those might not be
# checkable before kickoff due to deferred variables.
try:
result_parsers.check_args(test.config['results'])
except TestRunError as err:
rp_errors.append(str(err))
if rp_errors:
for msg in rp_errors:
test.status.set(STATES.RESULTS_ERROR, msg)
test.set_run_complete()
return 1
results = test.gather_results(run_result)
except result_parsers.ResultParserError as err:
self.logger.error("Unexpected error gathering results: %s", err)
test.status.set(STATES.RESULTS_ERROR,
"Error parsing results: {}".format(err))
return 1
try:
test.save_results(results)
result_logger = logging.getLogger('results')
result_logger.info(output.json_dumps(results))
except Exception:
test.status.set(
STATES.RESULTS_ERROR,
"Unknown error while saving results. Refer to the kickoff log."
)
raise
try:
test.status.set(
STATES.COMPLETE, "The test completed with result: {}".format(
results.get('result', '<unknown>')))
except Exception:
test.status.set(
STATES.UNKNOWN,
"Unknown error while setting test completion. Refer to the "
"kickoff log.")
raise