def __init__(self, pav_cfg, tests, _id=None):
"""Initialize the series.
:param pav_cfg: The pavilion configuration object.
:param list tests: The list of test objects that belong to this series.
:param int _id: The test id number. If this is given, it implies that
we're regenerating this series from saved files.
"""
self.pav_cfg = pav_cfg
self.tests = {test.id: test for test in tests}
series_path = self.pav_cfg.working_dir / 'series'
# We're creating this series from scratch.
if _id is None:
# Get the series id and path.
try:
self._id, self.path = dir_db.create_id_dir(
series_path, pav_cfg['shared_group'], pav_cfg['umask'])
except (OSError, TimeoutError) as err:
raise TestSeriesError(
"Could not get id or series directory in '{}': {}".format(
series_path, err))
perm_man = PermissionsManager(None, pav_cfg['shared_group'],
pav_cfg['umask'])
# Create a soft link to the test directory of each test in the
# series.
for test in tests:
link_path = dir_db.make_id_path(self.path, test.id)
try:
link_path.symlink_to(test.path)
perm_man.set_perms(link_path)
except OSError as err:
raise TestSeriesError(
"Could not link test '{}' in series at '{}': {}".
format(test.path, link_path, err))
# Update user.json to record last series run per sys_name
self._save_series_id()
else:
self._id = _id
self.path = dir_db.make_id_path(series_path, self._id)
self._logger = logging.getLogger(self.LOGGER_FMT.format(self._id))
def from_id(cls, pav_cfg, sid: str):
"""Load a series object from the given id, along with all of its
associated tests.
:raises TestSeriesError: From invalid series id or path.
"""
sid = cls.sid_to_id(sid)
series_path = pav_cfg.working_dir / 'series'
series_path = dir_db.make_id_path(series_path, sid)
if not series_path.exists():
raise TestSeriesError("No such series found: '{}' at '{}'".format(
sid, series_path))
logger = logging.getLogger(cls.LOGGER_FMT.format(sid))
tests = []
for path in dir_db.select(series_path):
try:
test_id = int(path.name)
except ValueError:
logger.info("Bad test id in series from dir '%s'", path)
continue
try:
tests.append(TestRun.load(pav_cfg, test_id=test_id))
except TestRunError as err:
logger.info("Error loading test %s: %s", test_id, err.args[0])
return cls(pav_cfg, tests, _id=sid)
def run(self, pav_cfg, args):
"""List the run directory for the given run."""
test_dir = pav_cfg.working_dir / 'test_runs'
job_dir = dir_db.make_id_path(test_dir, args.job_id)
if os.path.isdir(job_dir.as_posix()) is False:
output.fprint("directory '{}' does not exist.".format(job_dir),
file=sys.stderr, color=output.RED)
return errno.EEXIST
if args.path is True:
output.fprint(job_dir)
return 0
output.fprint(str(job_dir) + ':', file=self.outfile)
if args.tree is True:
level = 0
self.tree_(level, job_dir)
return 0
if args.subdir:
return self.ls_(job_dir / args.subdir[0])
else:
return self.ls_(job_dir)
def run(self, pav_cfg, args):
"""Run this command."""
test_dir = pav_cfg.working_dir / 'test_runs'
job_dir = dir_db.make_id_path(test_dir, args.job_id)
if os.path.isdir(job_dir.as_posix()) is False:
output.fprint("directory '{}' does not exist."
.format(job_dir.as_posix()),
file=sys.stderr, color=output.RED)
return errno.EEXIST
return self.print_file(job_dir / args.file)
def load(cls, pav_cfg, test_id):
"""Load an old TestRun object given a test id.
:param pav_cfg: The pavilion config
:param int test_id: The test's id number.
:rtype: TestRun
"""
path = dir_db.make_id_path(pav_cfg.working_dir / 'test_runs', test_id)
if not path.is_dir():
raise TestRunError("Test directory for test id {} does not exist "
"at '{}' as expected.".format(test_id, path))
config = cls._load_config(path)
return TestRun(pav_cfg, config, _id=test_id)
def path_from_id(cls, pav_cfg, sid: str):
"""Return the path to the series directory given a series id (in the
format 's[0-9]+'.
:raises TestSeriesError: For an invalid id.
"""
if not sid.startswith('s'):
raise TestSeriesError(
"Series id's must start with 's'. Got '{}'".format(sid))
try:
raw_id = int(sid[1:])
except ValueError:
raise TestSeriesError(
"Invalid series id '{}'. Series id's must be in the format "
"s[0-9]+".format(sid))
return dir_db.make_id_path(pav_cfg.working_dir / 'series', raw_id)
def add_tests(self, test_objs):
"""
Adds tests to existing series.
:param test_objs: List of test objects
:return: None
"""
for test in test_objs:
self.tests[test.id] = test
# attempt to make symlink
link_path = dir_db.make_id_path(self.path, test.id)
try:
link_path.symlink_to(test.path)
except OSError as err:
raise TestSeriesError(
"Could not link test '{}' in series at '{}': {}"
.format(test.path, link_path, err))
def from_id(cls,
pav_cfg,
sid: str,
outfile: TextIO = StringIO(),
errfile: TextIO = StringIO()):
"""Load a series object from the given id, along with all of its
associated tests.
:raises TestSeriesError: From invalid series id or path."""
sid = cls.sid_to_id(sid)
series_path = pav_cfg.working_dir / 'series'
series_path = dir_db.make_id_path(series_path, sid)
if not series_path.exists():
raise TestSeriesError("No such series found: '{}' at '{}'".format(
sid, series_path))
logger = logging.getLogger(cls.LOGGER_FMT.format(sid))
tests = []
for path in dir_db.select(series_path).paths:
try:
test_id = int(path.name)
except ValueError:
series_info_files = [
SERIES_OUT_FN, SERIES_PGID_FN, CONFIG_FN, DEPENDENCY_FN
]
if path.name not in series_info_files:
logger.info("Bad test id in series from dir '%s'", path)
continue
try:
tests.append(TestRun.load(pav_cfg, test_id=test_id))
except TestRunError as err:
logger.info("Error loading test %s: %s", test_id, err.args[0])
return cls(pav_cfg, tests, _id=sid, outfile=outfile, errfile=errfile)
def test_list_to_paths(pav_cfg, req_tests) -> List[Path]:
"""Given a list of test id's and series id's, return a list of paths
to those tests.
The keyword 'last' may also be given to get the last series run by
the current user on the current machine.
:param pav_cfg: The Pavilion config.
:param req_tests: A list of test id's, series id's, or 'last'.
:return: A list of test id's.
"""
test_paths = []
for test_id in req_tests:
if test_id == 'last':
test_id = series_util.load_user_series_id(pav_cfg)
if test_id.startswith('s'):
try:
test_paths.extend(
series_util.list_series_tests(pav_cfg, test_id))
except series_util.TestSeriesError:
raise ValueError("Invalid series id '{}'".format(test_id))
else:
try:
test_id = int(test_id)
except ValueError:
raise ValueError("Invalid test id '{}'".format(test_id))
test_dir = dir_db.make_id_path(pav_cfg.working_dir / 'test_runs',
test_id)
if not test_dir.exists():
raise ValueError("No such test '{}'".format(test_id))
test_paths.append(test_dir)
return test_paths
def get_pgid(pav_cfg, id_):
"""Returns pgid of series if it exists. """
try:
id_ = int(id_[1:])
except ValueError:
pass
series_path = pav_cfg.working_dir/'series'
series_path = dir_db.make_id_path(series_path, int(id_))
series_id_path = series_path/'series.pgid'
if not series_id_path.exists():
return False
with open(str(series_id_path), 'r') as series_id_file:
series_id = series_id_file.readline()
try:
series_id = int(series_id)
except ValueError:
return False
return series_id
def __init__(self, pav_cfg, tests=None, _id=None, series_config=None,
dep_graph=None, outfile: TextIO = StringIO(),
errfile: TextIO = StringIO()):
"""Initialize the series.
:param pav_cfg: The pavilion configuration object.
:param list tests: The list of test objects that belong to this series.
:param _id: The test id number. If this is given, it implies that
we're regenerating this series from saved files.
:param series_config: Series config, if generated from a series file.
:param dep_graph: The saved dependency graph (when loading).
:param outfile: Where to send user output.
:param errfile: Where to send user error output.
"""
self.pav_cfg = pav_cfg
self.outfile = outfile
self.errfile = errfile
self.tests = {}
self.config = series_config
if not dep_graph:
self.dep_graph = {}
else:
self.dep_graph = dep_graph
self.test_sets = {} # type: Dict[str, TestSet]
self.test_objs = {}
if tests:
self.tests = {test.id: test for test in tests}
series_path = self.pav_cfg.working_dir/'series'
# We're creating this series from scratch.
if _id is None:
# Get the series id and path.
try:
self._id, self.path = dir_db.create_id_dir(
series_path,
pav_cfg['shared_group'],
pav_cfg['umask'])
except (OSError, TimeoutError) as err:
raise TestSeriesError(
"Could not get id or series directory in '{}': {}"
.format(series_path, err))
# Create self.dep_graph, apply ordered: True, check for circular
# dependencies
self.dep_graph = self.create_dependency_graph()
self.save_dep_graph()
# save series config
self.save_series_config()
perm_man = PermissionsManager(None, pav_cfg['shared_group'],
pav_cfg['umask'])
# Create a soft link to the test directory of each test in the
# series.
if tests:
for test in tests:
link_path = dir_db.make_id_path(self.path, test.id)
try:
link_path.symlink_to(test.path)
perm_man.set_perms(link_path)
except OSError as err:
raise TestSeriesError(
"Could not link test '{}' in series at '{}': {}"
.format(test.path, link_path, err))
# Update user.json to record last series run per sys_name
self._save_series_id()
# We're not creating this from scratch (an object was made ahead of
# time).
else:
self._id = _id
self.path = dir_db.make_id_path(series_path, self._id)
self.dep_graph, self.config = self.load_dep_graph()
self._logger = logging.getLogger(self.LOGGER_FMT.format(self._id))
def __init__(self,
pav_cfg,
config,
build_tracker=None,
var_man=None,
_id=None,
rebuild=False,
build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir / 'test_runs'
self.config = config
group, umask = self.get_permissions(pav_cfg, config)
# Get an id for the test, if we weren't given one.
if _id is None:
id_tmp, run_path = dir_db.create_id_dir(tests_path, group, umask)
super().__init__(path=run_path, group=group, umask=umask)
# Set basic attributes
self.id = id_tmp
self.build_only = build_only
self.complete = False
self.created = dt.datetime.now()
self.name = self.make_name(config)
self.rebuild = rebuild
self.suite_path = Path(config.get('suite_path', '.'))
self.user = utils.get_login()
self.uuid = str(uuid.uuid4())
else:
# Load the test info from the given id path.
super().__init__(path=dir_db.make_id_path(tests_path, _id),
group=group,
umask=umask)
self.load_attributes()
self.test_version = config.get('test_version')
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
self._variables_path = self.path / 'variables'
if _id is None:
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self.var_man.save(self._variables_path)
self.sys_name = self.var_man.get('sys_name', '<unknown>')
else:
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path)
except RuntimeError as err:
raise TestRunError(*err.args)
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path / 'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path / 'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run',
config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build',
config.get('build', {}).get('timeout'))
self.run_log = self.path / 'run.log'
self.build_log = self.path / 'build.log'
self.results_log = self.path / 'results.log'
self.results_path = self.path / 'results.json'
self.build_origin_path = self.path / 'build_origin'
self.build_timeout_file = config.get('build', {}).get('timeout_file')
# Use run.log as the default run timeout file
self.timeout_file = self.run_log
run_timeout_file = config.get('run', {}).get('timeout_file')
if run_timeout_file is not None:
self.timeout_file = self.path / run_timeout_file
build_config = self.config.get('build', {})
self.build_script_path = self.path / 'build.sh' # type: Path
self.build_path = self.path / 'build'
if _id is None:
self._write_script('build',
path=self.build_script_path,
config=build_config)
try:
self.builder = builder.TestBuilder(pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=self.build_name)
self.build_name = self.builder.name
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err))
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path / 'run.tmpl'
self.run_script_path = self.path / 'run.sh'
if _id is None:
self._write_script('run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.save_attributes()
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self.skipped = self._get_skipped() # eval skip.
