def build(self, cancel_event=None):
"""Build the test using its builder object and symlink copy it to
it's final location. The build tracker will have the latest
information on any encountered errors.
:param threading.Event cancel_event: Event to tell builds when to die.
:returns: True if build successful
"""
if self.build_origin_path.exists():
raise RuntimeError(
"Whatever called build() is calling it for a second time."
"This should never happen for a given test run ({s.id})."
.format(s=self))
if cancel_event is None:
cancel_event = threading.Event()
if self.builder.build(cancel_event=cancel_event):
# Create the build origin path, to make tracking a test's build
# a bit easier.
with PermissionsManager(self.build_origin_path, self.group,
self.umask):
self.build_origin_path.symlink_to(self.builder.path)
with PermissionsManager(self.build_path, self.group, self.umask):
if not self.builder.copy_build(self.build_path):
cancel_event.set()
else:
self.builder.fail_path.rename(self.build_path)
return False
return True
def _save_series_id(self):
"""Save the series id to json file that tracks last series ran by user
on a per system basis."""
sys_vars = system_variables.get_vars(True)
sys_name = sys_vars['sys_name']
json_file = self.pav_cfg.working_dir/'users'
json_file /= '{}.json'.format(utils.get_login())
lockfile_path = json_file.with_suffix('.lock')
with LockFile(lockfile_path):
data = {}
try:
with json_file.open('r') as json_series_file:
try:
data = json.load(json_series_file)
except json.decoder.JSONDecodeError:
# File was empty, therefore json couldn't be loaded.
pass
with PermissionsManager(json_file, self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
json_file.open('w') as json_series_file:
data[sys_name] = self.sid
json_series_file.write(json.dumps(data))
except FileNotFoundError:
# File hadn't been created yet.
with PermissionsManager(json_file, self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
json_file.open('w') as json_series_file:
data[sys_name] = self.sid
json_series_file.write(json.dumps(data))
def __init__(self, pav_cfg, tests, _id=None):
"""Initialize the series.
:param pav_cfg: The pavilion configuration object.
:param list tests: The list of test objects that belong to this series.
:param int _id: The test id number. If this is given, it implies that
we're regenerating this series from saved files.
"""
self.pav_cfg = pav_cfg
self.tests = {test.id: test for test in tests}
series_path = self.pav_cfg.working_dir / 'series'
# We're creating this series from scratch.
if _id is None:
# Get the series id and path.
try:
self._id, self.path = dir_db.create_id_dir(
series_path, pav_cfg['shared_group'], pav_cfg['umask'])
except (OSError, TimeoutError) as err:
raise TestSeriesError(
"Could not get id or series directory in '{}': {}".format(
series_path, err))
perm_man = PermissionsManager(None, pav_cfg['shared_group'],
pav_cfg['umask'])
# Create a soft link to the test directory of each test in the
# series.
for test in tests:
link_path = dir_db.make_id_path(self.path, test.id)
try:
link_path.symlink_to(test.path)
perm_man.set_perms(link_path)
except OSError as err:
raise TestSeriesError(
"Could not link test '{}' in series at '{}': {}".
format(test.path, link_path, err))
# Update user.json to record last series run per sys_name
self._save_series_id()
else:
self._id = _id
self.path = dir_db.make_id_path(series_path, self._id)
self._logger = logging.getLogger(self.LOGGER_FMT.format(self._id))
def save_results(self, results):
"""Save the results to the test specific results file and the general
pavilion results file.
:param dict results: The results dictionary.
"""
results_tmp_path = self.results_path.with_suffix('.tmp')
with PermissionsManager(results_tmp_path, self.group, self.umask), \
results_tmp_path.open('w') as results_file:
json.dump(results, results_file)
try:
self.results_path.unlink()
except OSError:
pass
results_tmp_path.rename(self.results_path)
self.result = results.get('result')
self.save_attributes()
result_logger = logging.getLogger('common_results')
if self._pav_cfg.get('flatten_results') and results.get('per_file'):
# Flatten 'per_file' results into separate result records.
base = results.copy()
del base['per_file']
for per_file, values in results['per_file'].items():
per_result = base.copy()
per_result['file'] = per_file
per_result.update(values)
result_logger.info(output.json_dumps(per_result))
else:
result_logger.info(output.json_dumps(results))
def _hash_file(self, path, save=True):
"""Hash the given file (which is assumed to exist).
:param Path path: Path to the file to hash.
"""
stat = path.stat()
hash_fn = path.with_name('.' + path.name + '.hash')
# Read the has from the hashfile as long as it was created after
# our test source's last update.
if hash_fn.exists() and hash_fn.stat().st_mtime > stat.st_mtime:
try:
with hash_fn.open('rb') as hash_file:
return hash_file.read()
except OSError:
pass
hash_obj = hashlib.sha256()
with path.open('rb') as file:
chunk = file.read(self._BLOCK_SIZE)
while chunk:
hash_obj.update(chunk)
chunk = file.read(self._BLOCK_SIZE)
file_hash = hash_obj.digest()
if save:
# This should all be under the build lock.
with PermissionsManager(hash_fn, self._group, self._umask), \
hash_fn.open('wb') as hash_file:
hash_file.write(file_hash)
return file_hash
def run_series_background(self):
"""Run pav _series in background using subprocess module."""
# start subprocess
temp_args = ['pav', '_series', self.sid]
try:
series_out_path = self.path / SERIES_OUT_FN
with PermissionsManager(series_out_path,
self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
series_out_path.open('w') as series_out:
series_proc = subprocess.Popen(temp_args,
stdout=series_out,
stderr=series_out)
except OSError as err:
output.fprint("Could not kick off tests. Cancelling. \n{}.".format(
err.args[0]),
file=self.errfile,
color=output.RED)
return
# write pgid to a file (atomically)
series_pgid = os.getpgid(series_proc.pid)
series_pgid_path = self.path / SERIES_PGID_FN
try:
series_pgid_tmp = series_pgid_path.with_suffix('.tmp')
with PermissionsManager(series_pgid_tmp,
self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
series_pgid_tmp.open('w') as series_id_file:
series_id_file.write(str(series_pgid))
series_pgid_tmp.rename(series_pgid_path)
except OSError as err:
output.fprint("Warning: Could not write series PGID to a file.\n"
"To cancel, use `kill -14 -s{pgid}\n{err}".format(
err=err.args[0], pgid=series_pgid),
color=output.YELLOW,
file=self.outfile)
output.fprint(
"Started series {sid}.\n"
"Run `pav status {sid}` to view status.\n"
"PGID is {pgid}.\nTo kill, use `pav cancel {sid}`.".format(
sid=self.sid, pgid=series_pgid),
file=self.outfile)
def save_attributes(self):
"""Save the attributes to file in the test directory."""
attr_path = self.path/self.ATTR_FILE_NAME
with PermissionsManager(attr_path, self.group, self.umask), \
attr_path.open('w') as attr_file:
json.dump(self._attrs, attr_file)
def save_results(self, results):
"""Save the results to the results file.
:param dict results: The results dictionary.
"""
with self.results_path.open('w') as results_file, \
PermissionsManager(self.results_path, self.group, self.umask):
json.dump(results, results_file)
def job_id(self, job_id):
path = self.path / self.JOB_ID_FN
try:
with PermissionsManager(path, self.group, self.umask), \
path.open('w') as job_id_file:
job_id_file.write(job_id)
except (IOError, OSError) as err:
self.logger.error("Could not write jobid file '%s': %s", path, err)
self._job_id = job_id
def finalize(self, var_man):
"""Resolve any remaining deferred variables, and generate the final
run script."""
self.var_man.undefer(new_vars=var_man)
self.config = resolver.TestConfigResolver.resolve_deferred(
self.config, self.var_man)
self._save_config()
# Save our newly updated variables.
self.var_man.save(self._variables_path)
# Create files specified via run config key.
files_to_create = self.config['run'].get('create_files', [])
if files_to_create:
for file, contents in files_to_create.items():
file_path = Path(self.build_path / file)
# Prevent files from being written outside build directory.
if not utils.dir_contains(file_path, self.build_path):
raise TestRunError(
"'create_file: {}': file path"
" outside build context.".format(file_path))
# Prevent files from overwriting existing directories.
if file_path.is_dir():
raise TestRunError(
"'create_file: {}' clashes with"
" existing directory in build dir.".format(file_path))
# Create file parent directory(ies).
dirname = file_path.parent
(self.build_path / dirname).mkdir(parents=True, exist_ok=True)
# Don't try to overwrite a symlink without removing it first.
if file_path.is_symlink():
file_path.unlink()
# Write file.
with PermissionsManager(file_path, self.group, self.umask), \
file_path.open('w') as file_:
for line in contents:
file_.write("{}\n".format(line))
if not self.skipped:
self.skipped = self._get_skipped()
self.save_attributes()
self._write_script(
'run',
self.run_script_path,
self.config['run'],
)
def set_series_complete(self):
"""Write a file in the series directory that indicates that the series
has finished."""
series_complete_path = self.path / self.SERIES_COMPLETE_FN
series_complete_path_tmp = series_complete_path.with_suffix('.tmp')
with PermissionsManager(series_complete_path_tmp,
self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
series_complete_path_tmp.open('w') as series_complete:
json.dump({'complete': time.time()}, series_complete)
series_complete_path_tmp.rename(series_complete_path)
def save_build_name(self):
"""Save the builder's build name to the build name file for the test."""
try:
with PermissionsManager(self._build_name_fn, self.group,
self.umask), \
self._build_name_fn.open('w') as build_name_file:
build_name_file.write(self.builder.name)
except OSError as err:
raise TestRunError(
"Could not save build name to build name file at '{}': {}"
.format(self._build_name_fn, err)
)
def set_run_complete(self):
"""Write a file in the test directory that indicates that the test
has completed a run, one way or another. This should only be called
when we're sure their won't be any more status changes."""
# Write the current time to the file. We don't actually use the contents
# of the file, but it's nice to have another record of when this was
# run.
with (self.path/self.COMPLETE_FN).open('w') as run_complete, \
PermissionsManager(self.path/self.COMPLETE_FN,
self.group, self.umask):
json.dump({
'complete': datetime.datetime.now().isoformat(),
}, run_complete)
def save_series_config(self) -> None:
"""Saves series config to a file."""
series_config_path = self.path/CONFIG_FN
try:
series_config_tmp = series_config_path.with_suffix('.tmp')
with PermissionsManager(series_config_tmp,
self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
series_config_tmp.open('w') as config_file:
config_file.write(json.dumps(self.config))
series_config_path.with_suffix('.tmp').rename(series_config_path)
except OSError:
fprint("Could not write series config to file. Cancelling.",
color=output.RED)
def set_run_complete(self):
"""Write a file in the test directory that indicates that the test
has completed a run, one way or another. This should only be called
when we're sure their won't be any more status changes."""
# Write the current time to the file. We don't actually use the contents
# of the file, but it's nice to have another record of when this was
# run.
complete_path = self.path / self.COMPLETE_FN
complete_tmp_path = complete_path.with_suffix('.tmp')
with PermissionsManager(complete_tmp_path, self.group, self.umask), \
complete_tmp_path.open('w') as run_complete:
json.dump({'complete': time.time()}, run_complete)
complete_tmp_path.rename(complete_path)
self.complete = True
self.save_attributes()
def save_dep_graph(self):
"""Write dependency tree and config in series dir
:return:
"""
series_dep_path = self.path/DEPENDENCY_FN
series_dep_tmp = series_dep_path.with_suffix('.tmp')
try:
with PermissionsManager(series_dep_tmp,
self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
series_dep_tmp.open('w') as dep_file:
dep_file.write(json.dumps(self.dep_graph))
series_dep_path.with_suffix('.tmp').rename(series_dep_path)
except OSError:
fprint("Could not write dependency tree to file. Cancelling.",
color=output.RED, file=self.errfile)
def _save_config(self):
"""Save the configuration for this test to the test config file."""
config_path = self.path / 'config'
# make lock
tmp_path = config_path.with_suffix('.tmp')
try:
with PermissionsManager(config_path, self.group, self.umask), \
tmp_path.open('w') as json_file:
output.json_dump(self.config, json_file)
try:
config_path.unlink()
except OSError:
pass
tmp_path.rename(config_path)
except (OSError, IOError) as err:
raise TestRunError(
"Could not save TestRun ({}) config at {}: {}".format(
self.name, self.path, err))
except TypeError as err:
raise TestRunError("Invalid type in config for ({}): {}".format(
self.name, err))
def _create_kickoff_script(self, pav_cfg, test_obj: TestRun):
"""Function to accept a list of lines and generate a script that is
then submitted to the scheduler.
:param pavilion.test_config.TestRun test_obj:
"""
header = self._get_kickoff_script_header(test_obj)
script = scriptcomposer.ScriptComposer(header=header)
script.comment("Redirect all output to kickoff.log")
script.command("exec >{} 2>&1"
.format(test_obj.path/'kickoff.log'))
# Make sure the pavilion spawned
env_changes = {
'PATH': '{}:${{PATH}}'.format(pav_cfg.pav_root/'bin'),
'PAV_CONFIG_FILE': str(pav_cfg.pav_cfg_file),
}
if 'PAV_CONFIG_DIR' in os.environ:
env_changes['PAV_CONFIG_DIR'] = os.environ['PAV_CONFIG_DIR']
script.env_change(env_changes)
# Run Kickoff Env setup commands
for command in pav_cfg.env_setup:
script.command(command)
# Run the test via pavilion
script.command('pav _run {t.id}'.format(t=test_obj))
path = self._kickoff_script_path(test_obj)
with PermissionsManager(path, test_obj.group, test_obj.umask):
script.write(path)
return path
def save_attributes(self):
"""Save the attributes to file in the test directory."""
attr_path = self.path / self.ATTR_FILE_NAME
# Serialize all the values
attrs = {}
for key in self.list_attrs():
val = getattr(self, key)
if val is None:
continue
try:
attrs[key] = self.serializers.get(key, lambda v: v)(val)
except ValueError as err:
self.logger.warning(
"Error serializing attribute '%s' value '%s' for test run "
"'%s': %s", key, val, self.id, err.args[0])
with PermissionsManager(attr_path, self.group, self.umask):
tmp_path = attr_path.with_suffix('.tmp')
with tmp_path.open('w') as attr_file:
json.dump(attrs, attr_file)
tmp_path.rename(attr_path)
def __init__(self, pav_cfg, config,
build_tracker=None, var_man=None, _id=None,
rebuild=False, build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.load_ok = True
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir/'test_runs'
self.config = config
self.id = None # pylint: disable=invalid-name
self._attrs = {}
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
# If a test access group was given, make sure it exists and the
# current user is a member.
self.group = config.get('group')
if self.group is not None:
try:
group_data = grp.getgrnam(self.group)
user = utils.get_login()
if self.group != user and user not in group_data.gr_mem:
raise TestConfigError(
"Test specified group '{}', but the current user '{}' "
"is not a member of that group."
.format(self.group, user))
except KeyError as err:
raise TestConfigError(
"Test specified group '{}', but that group does not "
"exist on this system. {}"
.format(self.group, err))
self.umask = config.get('umask')
if self.umask is not None:
try:
self.umask = int(self.umask, 8)
except ValueError:
raise RuntimeError(
"Invalid umask. This should have been enforced by the "
"by the config format.")
self.build_only = build_only
self.rebuild = rebuild
self.suite_path = None
if self.config.get('suite_path') is not None:
try:
self.suite_path = Path(self.config['suite_path'])
except ValueError:
pass
# Get an id for the test, if we weren't given one.
if _id is None:
self.id, self.path = self.create_id_dir(tests_path)
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self._variables_path = self.path / 'variables'
self.var_man.save(self._variables_path)
self.save_attributes()
else:
self.id = _id
self.path = utils.make_id_path(tests_path, self.id)
self._variables_path = self.path / 'variables'
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path
)
except RuntimeError as err:
raise TestRunError(*err.args)
self.load_attributes()
name_parts = [
self.config.get('suite', '<unknown>'),
self.config.get('name', '<unnamed>'),
]
subtitle = self.config.get('subtitle')
# Don't add undefined or empty subtitles.
if subtitle:
name_parts.append(subtitle)
self.name = '.'.join(name_parts)
# Set a logger more specific to this test.
self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id))
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path/'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path/'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run', config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build', config.get('build', {}).get('timeout'))
self._attributes = {}
self.build_name = None
self.run_log = self.path/'run.log'
self.results_path = self.path/'results.json'
self.build_origin_path = self.path/'build_origin'
build_config = self.config.get('build', {})
if (build_config.get('source_path') is None and
build_config.get('source_url') is not None):
raise TestConfigError(
"Build source_url specified, but not a source_path.")
self.build_script_path = self.path/'build.sh' # type: Path
self.build_path = self.path/'build'
if _id is None:
self._write_script(
'build',
path=self.build_script_path,
config=build_config)
build_name = None
self._build_name_fn = self.path / 'build_name'
if _id is not None:
build_name = self._load_build_name()
try:
self.builder = builder.TestBuilder(
pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=build_name
)
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err)
)
self.save_build_name()
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path/'run.tmpl'
self.run_script_path = self.path/'run.sh'
if _id is None:
self._write_script(
'run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self._created = None
self.skipped = self._get_skipped()
def _write_script(self, stype, path, config):
"""Write a build or run script or template. The formats for each are
mostly identical.
:param str stype: The type of script (run or build).
:param Path path: Path to the template file to write.
:param dict config: Configuration dictionary for the script file.
:return:
"""
script = scriptcomposer.ScriptComposer()
verbose = config.get('verbose', 'false').lower() == 'true'
if verbose:
script.comment('# Echoing all commands to log.')
script.command('set -v')
script.newline()
pav_lib_bash = self._pav_cfg.pav_root / 'bin' / 'pav-lib.bash'
# If we include this directly, it breaks build hashing.
script.comment('The first (and only) argument of the build script is '
'the test id.')
script.env_change({
'TEST_ID': '${1:-0}', # Default to test id 0 if one isn't given.
'PAV_CONFIG_FILE': self._pav_cfg['pav_cfg_file']
})
script.command('source {}'.format(pav_lib_bash))
if config.get('preamble', []):
script.newline()
script.comment('Preamble commands')
for cmd in config['preamble']:
script.command(cmd)
if stype == 'build' and not self.build_local:
script.comment('To be built in an allocation.')
modules = config.get('modules', [])
if modules:
script.newline()
script.comment(
'Perform module related changes to the environment.')
for module in config.get('modules', []):
script.module_change(module, self.var_man)
env = config.get('env', {})
if env:
script.newline()
script.comment("Making any environment changes needed.")
script.env_change(config.get('env', {}))
if verbose:
script.newline()
script.comment('List all the module modules for posterity')
script.command("module -t list")
script.newline()
script.comment('Output the environment for posterity')
script.command("declare -p")
script.newline()
cmds = config.get('cmds', [])
if cmds:
script.comment("Perform the sequence of test commands.")
for line in config.get('cmds', []):
for split_line in line.split('\n'):
script.command(split_line)
else:
script.comment('No commands given for this script.')
with PermissionsManager(path, self.group, self.umask):
script.write(path)
def build(self, cancel_event=None):
"""Perform the build if needed, do a soft-link copy of the build
directory into our test directory, and note that we've used the given
build.
:param threading.Event cancel_event: Allows builds to tell each other
to die.
:return: True if these steps completed successfully.
"""
# Only try to do the build if it doesn't already exist and is finished.
if not self.finished_path.exists():
# Make sure another test doesn't try to do the build at
# the same time.
# Note cleanup of failed builds HAS to occur under this lock to
# avoid a race condition, even though it would be way simpler to
# do it in .build()
self.tracker.update(state=STATES.BUILD_WAIT,
note="Waiting on lock for build {}.".format(
self.name))
lock_path = self.path.with_suffix('.lock')
with lockfile.LockFile(lock_path,
group=self._pav_cfg.shared_group)\
as lock:
# Make sure the build wasn't created while we waited for
# the lock.
if not self.finished_path.exists():
self.tracker.update(state=STATES.BUILDING,
note="Starting build {}.".format(
self.name))
# If the build directory exists, we're assuming there was
# an incomplete build at this point.
if self.path.exists():
self.tracker.warn(
"Build lock acquired, but build exists that was "
"not marked as finished. Deleting...")
try:
shutil.rmtree(self.path)
except OSError as err:
self.tracker.error(
"Could not remove unfinished build.\n{}".
format(err.args[0]))
return False
# Attempt to perform the actual build, this shouldn't
# raise an exception unless something goes terribly
# wrong.
# This will also set the test status for
# non-catastrophic cases.
with PermissionsManager(self.path, self._group,
self._umask):
if not self._build(self.path, cancel_event, lock=lock):
try:
self.path.rename(self.fail_path)
except FileNotFoundError as err:
self.tracker.error(
"Failed to move build {} from {} to "
"failure path {}: {}".format(
self.name, self.path, self.fail_path,
err))
self.fail_path.mkdir()
if cancel_event is not None:
cancel_event.set()
return False
# Make a file with the test id of the building test.
built_by_path = self.path / '.built_by'
try:
with PermissionsManager(built_by_path, self._group,
self._umask | 0o222), \
built_by_path.open('w') as built_by:
built_by.write(str(self.test.id))
except OSError:
self.tracker.warn("Could not create built_by file.")
try:
with PermissionsManager(self.finished_path,
self._group, self._umask):
self.finished_path.touch()
except OSError:
self.tracker.warn("Could not touch '<build>.finished' "
"file.")
else:
self.tracker.update(
state=STATES.BUILD_REUSED,
note="Build {s.name} created while waiting for build "
"lock.".format(s=self))
else:
self.tracker.update(
note=("Test {s.name} run {s.test.id} reusing build.".format(
s=self)),
state=STATES.BUILD_REUSED)
return True
def __init__(self, pav_cfg, tests=None, _id=None, series_config=None,
dep_graph=None, outfile: TextIO = StringIO(),
errfile: TextIO = StringIO()):
"""Initialize the series.
:param pav_cfg: The pavilion configuration object.
:param list tests: The list of test objects that belong to this series.
:param _id: The test id number. If this is given, it implies that
we're regenerating this series from saved files.
:param series_config: Series config, if generated from a series file.
:param dep_graph: The saved dependency graph (when loading).
:param outfile: Where to send user output.
:param errfile: Where to send user error output.
"""
self.pav_cfg = pav_cfg
self.outfile = outfile
self.errfile = errfile
self.tests = {}
self.config = series_config
if not dep_graph:
self.dep_graph = {}
else:
self.dep_graph = dep_graph
self.test_sets = {} # type: Dict[str, TestSet]
self.test_objs = {}
if tests:
self.tests = {test.id: test for test in tests}
series_path = self.pav_cfg.working_dir/'series'
# We're creating this series from scratch.
if _id is None:
# Get the series id and path.
try:
self._id, self.path = dir_db.create_id_dir(
series_path,
pav_cfg['shared_group'],
pav_cfg['umask'])
except (OSError, TimeoutError) as err:
raise TestSeriesError(
"Could not get id or series directory in '{}': {}"
.format(series_path, err))
# Create self.dep_graph, apply ordered: True, check for circular
# dependencies
self.dep_graph = self.create_dependency_graph()
self.save_dep_graph()
# save series config
self.save_series_config()
perm_man = PermissionsManager(None, pav_cfg['shared_group'],
pav_cfg['umask'])
# Create a soft link to the test directory of each test in the
# series.
if tests:
for test in tests:
link_path = dir_db.make_id_path(self.path, test.id)
try:
link_path.symlink_to(test.path)
perm_man.set_perms(link_path)
except OSError as err:
raise TestSeriesError(
"Could not link test '{}' in series at '{}': {}"
.format(test.path, link_path, err))
# Update user.json to record last series run per sys_name
self._save_series_id()
# We're not creating this from scratch (an object was made ahead of
# time).
else:
self._id = _id
self.path = dir_db.make_id_path(series_path, self._id)
self.dep_graph, self.config = self.load_dep_graph()
self._logger = logging.getLogger(self.LOGGER_FMT.format(self._id))
def _run(self, pav_cfg, test, sched):
"""Run an already prepped test in the current environment.
:param pav_cfg: The pavilion configuration object.
:param TestRun test: The test to run
:param sched: The scheduler we're running under.
:return:
"""
# Optionally wait on other tests running under the same scheduler.
# This depends on the scheduler and the test configuration.
lock = sched.lock_concurrency(pav_cfg, test)
try:
run_result = test.run()
except TestRunError as err:
test.status.set(STATES.RUN_ERROR, err)
return 1
except TimeoutError:
return 1
except Exception:
test.status.set(
STATES.RUN_ERROR,
"Unknown error while running test. Refer to the kickoff log.")
raise
finally:
sched.unlock_concurrency(lock)
try:
# Make sure the result parsers have reasonable arguments.
# We check here because the parser code itself will likely assume
# the args are valid form _check_args, but those might not be
# check-able before kickoff due to deferred variables.
try:
result.check_config(test.config['result_parse'],
test.config['result_evaluate'])
except result.ResultError as err:
test.status.set(
STATES.RESULTS_ERROR,
"Error checking result parser configs: {}".format(
err.args[0]))
return 1
with PermissionsManager(test.results_log,
group=test.group, umask=test.umask), \
test.results_log.open('w') as log_file:
results = test.gather_results(run_result, log_file=log_file)
except Exception as err:
self.logger.error("Unexpected error gathering results: \n%s",
traceback.format_exc())
test.status.set(
STATES.RESULTS_ERROR,
"Unexpected error parsing results: {}. (This is a "
"bug, you should report it.)"
"See 'pav log kickoff {}' for the full error.".format(
err, test.id))
raise
try:
test.save_results(results)
except Exception:
test.status.set(
STATES.RESULTS_ERROR,
"Unknown error while saving results. Refer to the kickoff log."
)
raise
try:
test.status.set(
STATES.COMPLETE, "The test completed with result: {}".format(
results.get('result', '<unknown>')))
except Exception:
test.status.set(
STATES.UNKNOWN,
"Unknown error while setting test completion. Refer to the "
"kickoff log.")
raise
def setup_loggers(pav_cfg, verbose=False, err_out=sys.stderr):
"""Setup the loggers for the Pavilion command. This will include:
- The general log file (as a multi-process/host safe rotating logger).
- The result log file (also as a multi-process/host safe rotating logger).
- The exception log.
:param pav_cfg: The Pavilion configuration.
:param bool verbose: When verbose, setup the root logger to print to stderr
as well.
:param IO[str] err_out: Where to log errors meant for the terminal. This
exists primarily for testing.
"""
root_logger = logging.getLogger()
# Setup the new record factory.
logging.setLogRecordFactory(record_factory)
perm_man = PermissionsManager(None, pav_cfg['shared_group'],
pav_cfg['umask'])
# Put the log file in the lowest common pav config directory we can write
# to.
log_fn = pav_cfg.working_dir/'pav.log'
# Set up a rotating logfile than rotates when it gets larger
# than 1 MB.
try:
log_fn.touch()
except (PermissionError, FileNotFoundError) as err:
output.fprint("Could not write to pavilion log at '{}': {}"
.format(log_fn, err),
color=output.YELLOW,
file=err_out)
else:
file_handler = logging.FileHandler(filename=log_fn.as_posix())
file_handler.setFormatter(logging.Formatter(pav_cfg.log_format,
style='{'))
file_handler.setLevel(getattr(logging,
pav_cfg.log_level.upper()))
root_logger.addHandler(file_handler)
try:
perm_man.set_perms(log_fn)
except OSError:
pass
# The root logger should pass all messages, even if the handlers
# filter them.
root_logger.setLevel(logging.DEBUG)
# Setup the result logger.
# Results will be logged to both the main log and the result log.
try:
pav_cfg.result_log.touch()
except (PermissionError, FileNotFoundError) as err:
output.fprint(
"Could not write to result log at '{}': {}"
.format(pav_cfg.result_log, err),
color=output.YELLOW, file=err_out)
return False
try:
perm_man.set_perms(pav_cfg.result_log)
except OSError:
pass
result_logger = logging.getLogger('common_results')
result_handler = LockFileRotatingFileHandler(
file_name=str(pav_cfg.result_log),
# 20 MB
max_bytes=20 * 1024 ** 2,
backup_count=3)
result_handler.setFormatter(logging.Formatter("{message}", style='{'))
result_logger.setLevel(logging.INFO)
result_logger.addHandler(result_handler)
# Setup the exception logger.
# Exceptions will be logged to this directory, along with other useful info.
exc_logger = logging.getLogger('exceptions')
try:
pav_cfg.exception_log.touch()
except (PermissionError, FileNotFoundError) as err:
output.fprint(
"Could not write to exception log at '{}': {}"
.format(pav_cfg.exception_log, err),
color=output.YELLOW,
file=err_out
)
else:
exc_handler = LockFileRotatingFileHandler(
file_name=pav_cfg.exception_log.as_posix(),
max_bytes=20 * 1024 ** 2,
backup_count=3)
exc_handler.setFormatter(logging.Formatter(
"{asctime} {message}",
style='{',
))
exc_logger.setLevel(logging.ERROR)
exc_logger.addHandler(exc_handler)
try:
perm_man.set_perms(pav_cfg.exception_log)
except OSError:
pass
# Setup the yapsy logger to log to terminal. We need to know immediatly
# when yapsy encounters errors.
yapsy_logger = logging.getLogger('yapsy')
yapsy_handler = logging.StreamHandler(stream=err_out)
# Color all these error messages red.
yapsy_handler.setFormatter(
logging.Formatter("\x1b[31m{asctime} {message}\x1b[0m",
style='{'))
yapsy_logger.setLevel(logging.INFO)
yapsy_logger.addHandler(yapsy_handler)
# Add a stream to stderr if we're in verbose mode, or if no other handler
# is defined.
if verbose or not root_logger.handlers:
verbose_handler = logging.StreamHandler(err_out)
verbose_handler.setLevel(logging.DEBUG)
verbose_handler.setFormatter(logging.Formatter(pav_cfg.log_format,
style='{'))
root_logger.addHandler(result_handler)
return True
def __init__(self,
pav_cfg,
config,
build_tracker=None,
var_man=None,
_id=None,
rebuild=False,
build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir / 'test_runs'
self.config = config
group, umask = self.get_permissions(pav_cfg, config)
# Get an id for the test, if we weren't given one.
if _id is None:
id_tmp, run_path = dir_db.create_id_dir(tests_path, group, umask)
super().__init__(path=run_path, group=group, umask=umask)
# Set basic attributes
self.id = id_tmp
self.build_only = build_only
self.complete = False
self.created = dt.datetime.now()
self.name = self.make_name(config)
self.rebuild = rebuild
self.suite_path = Path(config.get('suite_path', '.'))
self.user = utils.get_login()
self.uuid = str(uuid.uuid4())
else:
# Load the test info from the given id path.
super().__init__(path=dir_db.make_id_path(tests_path, _id),
group=group,
umask=umask)
self.load_attributes()
self.test_version = config.get('test_version')
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
self._variables_path = self.path / 'variables'
if _id is None:
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self.var_man.save(self._variables_path)
self.sys_name = self.var_man.get('sys_name', '<unknown>')
else:
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path)
except RuntimeError as err:
raise TestRunError(*err.args)
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path / 'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path / 'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run',
config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build',
config.get('build', {}).get('timeout'))
self.run_log = self.path / 'run.log'
self.build_log = self.path / 'build.log'
self.results_log = self.path / 'results.log'
self.results_path = self.path / 'results.json'
self.build_origin_path = self.path / 'build_origin'
self.build_timeout_file = config.get('build', {}).get('timeout_file')
# Use run.log as the default run timeout file
self.timeout_file = self.run_log
run_timeout_file = config.get('run', {}).get('timeout_file')
if run_timeout_file is not None:
self.timeout_file = self.path / run_timeout_file
build_config = self.config.get('build', {})
self.build_script_path = self.path / 'build.sh' # type: Path
self.build_path = self.path / 'build'
if _id is None:
self._write_script('build',
path=self.build_script_path,
config=build_config)
try:
self.builder = builder.TestBuilder(pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=self.build_name)
self.build_name = self.builder.name
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err))
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path / 'run.tmpl'
self.run_script_path = self.path / 'run.sh'
if _id is None:
self._write_script('run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.save_attributes()
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self.skipped = self._get_skipped() # eval skip.
def run(self):
"""Run the test.
:rtype: bool
:returns: The return code of the test command.
:raises TimeoutError: When the run times out.
:raises TestRunError: We don't actually raise this, but might in the
future.
"""
if self.build_only:
self.status.set(STATES.RUN_ERROR,
"Tried to run a 'build_only' test object.")
return False
self.status.set(STATES.PREPPING_RUN,
"Converting run template into run script.")
with PermissionsManager(self.path, self.group, self.umask), \
self.run_log.open('wb') as run_log:
self.status.set(STATES.RUNNING, "Starting the run script.")
self.started = dt.datetime.now()
# Set the working directory to the build path, if there is one.
run_wd = None
if self.build_path is not None:
run_wd = self.build_path.as_posix()
# Run scripts take the test id as a first argument.
cmd = [self.run_script_path.as_posix(), str(self.id)]
proc = subprocess.Popen(cmd,
cwd=run_wd,
stdout=run_log,
stderr=subprocess.STDOUT)
self.status.set(STATES.RUNNING, "Currently running.")
# Run the test, but timeout if it doesn't produce any output every
# self._run_timeout seconds
timeout = self.run_timeout
ret = None
while ret is None:
try:
ret = proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
if self.timeout_file.exists():
timeout_file = self.timeout_file
else:
timeout_file = self.run_log
try:
out_stat = timeout_file.stat()
quiet_time = time.time() - out_stat.st_mtime
except OSError:
pass
# Has the output file changed recently?
if self.run_timeout < quiet_time:
# Give up on the build, and call it a failure.
proc.kill()
msg = ("Run timed out after {} seconds".format(
self.run_timeout))
self.status.set(STATES.RUN_TIMEOUT, msg)
self.finished = dt.datetime.now()
self.save_attributes()
raise TimeoutError(msg)
else:
# Only wait a max of run_silent_timeout next 'wait'
timeout = timeout - quiet_time
self.finished = dt.datetime.now()
self.save_attributes()
self.status.set(STATES.RUN_DONE, "Test run has completed.")
return ret