def _save_series_id(self):
"""Save the series id to json file that tracks last series ran by user
on a per system basis."""
sys_vars = system_variables.get_vars(True)
sys_name = sys_vars['sys_name']
json_file = self.pav_cfg.working_dir/'users'
json_file /= '{}.json'.format(utils.get_login())
lockfile_path = json_file.with_suffix('.lock')
with LockFile(lockfile_path):
data = {}
try:
with json_file.open('r') as json_series_file:
try:
data = json.load(json_series_file)
except json.decoder.JSONDecodeError:
# File was empty, therefore json couldn't be loaded.
pass
with PermissionsManager(json_file, self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
json_file.open('w') as json_series_file:
data[sys_name] = self.sid
json_series_file.write(json.dumps(data))
except FileNotFoundError:
# File hadn't been created yet.
with PermissionsManager(json_file, self.pav_cfg['shared_group'],
self.pav_cfg['umask']), \
json_file.open('w') as json_series_file:
data[sys_name] = self.sid
json_series_file.write(json.dumps(data))
def get_permissions(pav_cfg, config) -> (str, int):
"""Get the permissions to use on file creation, either from the
pav_cfg or test config it that overrides.
:returns: A tuple of the group and umask.
"""
# If a test access group was given, make sure it exists and the
# current user is a member.
group = config.get('group', pav_cfg['shared_group'])
if group is not None:
try:
group_data = grp.getgrnam(group)
user = utils.get_login()
if group != user and user not in group_data.gr_mem:
raise TestConfigError(
"Test specified group '{}', but the current user '{}' "
"is not a member of that group.".format(group, user))
except KeyError as err:
raise TestConfigError(
"Test specified group '{}', but that group does not "
"exist on this system. {}".format(group, err))
umask = config.get('umask', pav_cfg['umask'])
if umask is not None:
try:
umask = int(umask, 8)
except ValueError:
raise RuntimeError(
"Invalid umask. This should have been enforced by the "
"by the config format.")
return group, umask
def test_sys_name_tracker(self):
"""Make sure the expected values are stored in the user.json file."""
user = utils.get_login()
sys_vars = system_variables.get_vars(True)
sys_name = sys_vars['sys_name']
arg_parser = arguments.get_parser()
args = arg_parser.parse_args(['run', 'hello_world'])
run_cmd = commands.get_command(args.command_name)
run_cmd.outfile = io.StringIO()
run_cmd.errfile = run_cmd.outfile
run_cmd.run(self.pav_cfg, args)
series = run_cmd.last_series
json_file = self.pav_cfg.working_dir / 'users'
json_file /= '{}.json'.format(user)
with json_file.open('r') as json_series_file:
data = json.load(json_series_file)
self.assertEqual(data[sys_name], series.sid)
def test_details(self):
"""Testing the ScriptDetails class."""
testPath = self.pav_cfg.working_dir / 'testScript.sh'
testGroup = 'anonymous'
testPerms = 0o531
# Testing valid uses.
# Testing initialization and defaults.
test_details = scriptcomposer.ScriptDetails()
self.assertEqual(test_details.group, utils.get_login())
self.assertEqual(test_details.perms, oct(0o770))
# Testing individual assignment.
test_details.path = testPath
test_details.group = testGroup
test_details.perms = testPerms
self.assertEqual(test_details.path, Path(testPath))
self.assertEqual(test_details.group, testGroup)
self.assertEqual(test_details.perms, oct(testPerms))
# Testing initialization assignment.
test_details = scriptcomposer.ScriptDetails(path=testPath,
group=testGroup,
perms=testPerms)
self.assertEqual(test_details.path, Path(testPath))
self.assertEqual(test_details.group, testGroup)
self.assertEqual(test_details.perms, oct(testPerms))
test_details = scriptcomposer.ScriptDetails()
# Testing invalid uses.
with self.assertRaises(TypeError):
test_details.path = True
with self.assertRaises(TypeError):
test_details.perms = 'string'
with self.assertRaises(TypeError):
test_details.perms = u'fail'
with self.assertRaises(TypeError):
test_details.perms = 7.5
# Testing invalid initialization.
with self.assertRaises(TypeError):
scriptcomposer.ScriptDetails(path=testPath,
group=testGroup,
perms='fail')
def _save_series_id(self):
"""Save the series id to the user's .pavilion directory."""
# Save the last series we created to the .pavilion directory
# in the user's home dir. Pavilion commands can use this so the
# user doesn't actually have to know the series_id of tests.
last_series_fn = self.pav_cfg.working_dir/'users'
last_series_fn /= '{}.series'.format(utils.get_login())
try:
with last_series_fn.open('w') as last_series_file:
last_series_file.write(self.id)
except (IOError, OSError):
# It's ok if we can't write this file.
self._logger.warning("Could not save series id to '%s'",
last_series_fn)
def load_user_series_id(cls, pav_cfg):
"""Load the last series id used by the current user."""
logger = logging.getLogger(cls.LOGGER_FMT.format('<unknown>'))
last_series_fn = pav_cfg.working_dir/'users'
last_series_fn /= '{}.series'.format(utils.get_login())
if not last_series_fn.exists():
return None
try:
with last_series_fn.open() as last_series_file:
return last_series_file.read().strip()
except (IOError, OSError) as err:
logger.warning("Failed to read series id file '%s': %s",
last_series_fn, err)
return None
def __init__(self, *args, **kwargs):
with self.PAV_CONFIG_PATH.open() as cfg_file:
raw_pav_cfg = config.PavilionConfigLoader().load(cfg_file)
raw_pav_cfg.config_dirs = [
self.TEST_DATA_ROOT / 'pav_config_dir', self.PAV_LIB_DIR
]
raw_pav_cfg.working_dir = Path('/tmp') / get_login() / 'pav_tests'
raw_pav_cfg.result_log = raw_pav_cfg.working_dir / 'results.log'
if not raw_pav_cfg.working_dir.exists():
raw_pav_cfg.working_dir.mkdir()
cfg_dir = raw_pav_cfg.working_dir / 'pav_cfgs'
if not cfg_dir.exists():
cfg_dir.mkdir()
cfg_path = Path(tempfile.mktemp(suffix='.yaml', dir=str(cfg_dir)))
with cfg_path.open('w') as pav_cfg_file:
config.PavilionConfigLoader().dump(pav_cfg_file, raw_pav_cfg)
with cfg_path.open() as cfg_file:
self.pav_cfg = config.PavilionConfigLoader().load(cfg_file)
self.pav_cfg.pav_cfg_file = cfg_path
# Create the basic directories in the working directory
for path in [
self.pav_cfg.working_dir, self.pav_cfg.working_dir / 'builds',
self.pav_cfg.working_dir / 'tests',
self.pav_cfg.working_dir / 'series',
self.pav_cfg.working_dir / 'users',
self.pav_cfg.working_dir / 'downloads'
]:
if not path.exists():
os.makedirs(str(path), exist_ok=True)
self.tmp_dir = tempfile.TemporaryDirectory()
# We have to get this to set up the base argument parser before
# plugins can add to it.
_ = arguments.get_parser()
super().__init__(*args, **kwargs)
def load_user_series_id(pav_cfg):
"""Load the last series id used by the current user."""
last_series_fn = pav_cfg.working_dir / 'users'
last_series_fn /= '{}.json'.format(utils.get_login())
sys_vars = system_variables.get_vars(True)
sys_name = sys_vars['sys_name']
if not last_series_fn.exists():
return None
try:
with last_series_fn.open() as last_series_file:
sys_name_series_dict = json.load(last_series_file)
return sys_name_series_dict[sys_name].strip()
except (IOError, OSError, KeyError) as err:
logger.warning("Failed to read series id file '%s': %s",
last_series_fn, err)
return None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Find a group that isn't the user's default group (or sudo), and
# use that as our default group.
login = utils.get_login()
def_gid = os.getgid()
candidates = [group for group in grp.getgrall() if
(login in group.gr_mem
and def_gid != group.gr_gid)]
if not candidates:
self.fail("Your user must be in at least two groups (other than "
"the user's group) to run this test.")
self.orig_group = grp.getgrgid(def_gid).gr_name
self.alt_group = candidates[0] # type: grp.struct_group
self.alt_group2 = candidates[1] # type: grp.struct_group
self.umask = 0o007
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Find a group that isn't the user's default group (or sudo), and
# use that as our default group.
login = utils.get_login()
def_gid = os.getgid()
candidates = [group for group in grp.getgrall() if
(login in group.gr_mem
and def_gid != group.gr_gid)]
if not candidates:
self.orig_group = None
self.alt_group = None
self.alt_group2 = None
else:
self.orig_group = grp.getgrgid(def_gid).gr_name
self.alt_group = candidates[0] # type: grp.struct_group
self.alt_group2 = candidates[1] # type: grp.struct_group
self.umask = 0o007
def _create_lockfile(cls, path, expires, lock_id, group_id=None):
"""Create and fill out a lockfile at the given path.
:param Path path: Where the file will be created.
:param int expires: How far in the future the lockfile expires.
:param str lock_id: The unique identifier for this lockfile.
:returns: None
:raises IOError: When the file cannot be written too.
:raises OSError: When the file cannot be opened or already exists.
"""
# DEV NOTE: This logic is separate so that we can create these files
# outside of the standard mechanisms for testing purposes.
# We're doing low level operations on the path, so we just need
# it as a string.
path = str(path)
file_num = os.open(path, os.O_EXCL | os.O_CREAT | os.O_RDWR)
file_note = ",".join(
[os.uname()[1],
utils.get_login(),
str(expires), lock_id])
file_note = file_note.encode('utf8')
os.write(file_num, file_note)
os.close(file_num)
try:
os.chmod(path, cls.LOCK_PERMS)
except OSError as err:
LOGGER.warning("Lockfile at '%s' could not set permissions: %s",
path, err)
if group_id is not None:
try:
os.chown(path, os.getuid(), group_id)
except OSError as err:
LOGGER.warning("Lockfile at '%s' could not set group: %s",
path, err)
from pavilion.test_run import TestRun
LOCAL_SYS_NAME = '<local_sys_name>'
TEST_FILTER_DEFAULTS = {
'complete': False,
'failed': False,
'incomplete': False,
'name': None,
'newer_than': time.time() - dt.timedelta(days=1).total_seconds(),
'older_than': None,
'passed': False,
'result_error': False,
'show_skipped': 'no',
'sort_by': '-created',
'sys_name': LOCAL_SYS_NAME,
'user': utils.get_login(),
'limit': None,
'disable_filter': False,
}
TEST_SORT_FUNCS = {
'created': lambda test: test['created'],
'finished': lambda test: test['finished'],
'name': lambda test: test['name'],
'started': lambda test: test['started'],
'user': lambda test: test['user'],
'id': lambda test: test['id'],
}
def add_common_filter_args(target: str, arg_parser: argparse.ArgumentParser,
def user(self):
"""The current user's login name."""
return utils.get_login()
def group(self, value):
if value is None:
value = utils.get_login()
self._group = str(value)
def __init__(self, pav_cfg, config,
build_tracker=None, var_man=None, _id=None,
rebuild=False, build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.load_ok = True
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir/'test_runs'
self.config = config
self.id = None # pylint: disable=invalid-name
self._attrs = {}
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
# If a test access group was given, make sure it exists and the
# current user is a member.
self.group = config.get('group')
if self.group is not None:
try:
group_data = grp.getgrnam(self.group)
user = utils.get_login()
if self.group != user and user not in group_data.gr_mem:
raise TestConfigError(
"Test specified group '{}', but the current user '{}' "
"is not a member of that group."
.format(self.group, user))
except KeyError as err:
raise TestConfigError(
"Test specified group '{}', but that group does not "
"exist on this system. {}"
.format(self.group, err))
self.umask = config.get('umask')
if self.umask is not None:
try:
self.umask = int(self.umask, 8)
except ValueError:
raise RuntimeError(
"Invalid umask. This should have been enforced by the "
"by the config format.")
self.build_only = build_only
self.rebuild = rebuild
self.suite_path = None
if self.config.get('suite_path') is not None:
try:
self.suite_path = Path(self.config['suite_path'])
except ValueError:
pass
# Get an id for the test, if we weren't given one.
if _id is None:
self.id, self.path = self.create_id_dir(tests_path)
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self._variables_path = self.path / 'variables'
self.var_man.save(self._variables_path)
self.save_attributes()
else:
self.id = _id
self.path = utils.make_id_path(tests_path, self.id)
self._variables_path = self.path / 'variables'
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path
)
except RuntimeError as err:
raise TestRunError(*err.args)
self.load_attributes()
name_parts = [
self.config.get('suite', '<unknown>'),
self.config.get('name', '<unnamed>'),
]
subtitle = self.config.get('subtitle')
# Don't add undefined or empty subtitles.
if subtitle:
name_parts.append(subtitle)
self.name = '.'.join(name_parts)
# Set a logger more specific to this test.
self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id))
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path/'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path/'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run', config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build', config.get('build', {}).get('timeout'))
self._attributes = {}
self.build_name = None
self.run_log = self.path/'run.log'
self.results_path = self.path/'results.json'
self.build_origin_path = self.path/'build_origin'
build_config = self.config.get('build', {})
if (build_config.get('source_path') is None and
build_config.get('source_url') is not None):
raise TestConfigError(
"Build source_url specified, but not a source_path.")
self.build_script_path = self.path/'build.sh' # type: Path
self.build_path = self.path/'build'
if _id is None:
self._write_script(
'build',
path=self.build_script_path,
config=build_config)
build_name = None
self._build_name_fn = self.path / 'build_name'
if _id is not None:
build_name = self._load_build_name()
try:
self.builder = builder.TestBuilder(
pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=build_name
)
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err)
)
self.save_build_name()
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path/'run.tmpl'
self.run_script_path = self.path/'run.sh'
if _id is None:
self._write_script(
'run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self._created = None
self.skipped = self._get_skipped()
def __init__(self,
pav_cfg,
config,
build_tracker=None,
var_man=None,
_id=None,
rebuild=False,
build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir / 'test_runs'
self.config = config
group, umask = self.get_permissions(pav_cfg, config)
# Get an id for the test, if we weren't given one.
if _id is None:
id_tmp, run_path = dir_db.create_id_dir(tests_path, group, umask)
super().__init__(path=run_path, group=group, umask=umask)
# Set basic attributes
self.id = id_tmp
self.build_only = build_only
self.complete = False
self.created = dt.datetime.now()
self.name = self.make_name(config)
self.rebuild = rebuild
self.suite_path = Path(config.get('suite_path', '.'))
self.user = utils.get_login()
self.uuid = str(uuid.uuid4())
else:
# Load the test info from the given id path.
super().__init__(path=dir_db.make_id_path(tests_path, _id),
group=group,
umask=umask)
self.load_attributes()
self.test_version = config.get('test_version')
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
self._variables_path = self.path / 'variables'
if _id is None:
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self.var_man.save(self._variables_path)
self.sys_name = self.var_man.get('sys_name', '<unknown>')
else:
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path)
except RuntimeError as err:
raise TestRunError(*err.args)
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path / 'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path / 'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run',
config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build',
config.get('build', {}).get('timeout'))
self.run_log = self.path / 'run.log'
self.build_log = self.path / 'build.log'
self.results_log = self.path / 'results.log'
self.results_path = self.path / 'results.json'
self.build_origin_path = self.path / 'build_origin'
self.build_timeout_file = config.get('build', {}).get('timeout_file')
# Use run.log as the default run timeout file
self.timeout_file = self.run_log
run_timeout_file = config.get('run', {}).get('timeout_file')
if run_timeout_file is not None:
self.timeout_file = self.path / run_timeout_file
build_config = self.config.get('build', {})
self.build_script_path = self.path / 'build.sh' # type: Path
self.build_path = self.path / 'build'
if _id is None:
self._write_script('build',
path=self.build_script_path,
config=build_config)
try:
self.builder = builder.TestBuilder(pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=self.build_name)
self.build_name = self.builder.name
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err))
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path / 'run.tmpl'
self.run_script_path = self.path / 'run.sh'
if _id is None:
self._write_script('run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.save_attributes()
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self.skipped = self._get_skipped() # eval skip.
def test_scriptComposer(self):
"""Testing ScriptComposer class variable setting."""
# Testing valid uses.
# Testing initialization defaults.
composer = scriptcomposer.ScriptComposer()
self.assertIsInstance(composer.header, scriptcomposer.ScriptHeader)
self.assertIsInstance(composer.details, scriptcomposer.ScriptDetails)
self.assertEqual(composer.header.shell_path, '#!/bin/bash')
self.assertEqual(composer.header.scheduler_headers, [])
self.assertEqual(composer.details.group, utils.get_login())
self.assertEqual(composer.details.perms, oct(0o770))
# Testing individual assignment
test_header_shell = "/usr/env/python"
test_header_scheduler = OrderedDict()
test_header_scheduler['-G'] = 'pam'
test_header_scheduler['-N'] = 'fam'
composer.newline()
composer.command(['taco', 'burrito', 'nachos'])
test_details_path = 'testPath'
test_details_group = 'groupies'
test_details_perms = 0o543
composer.header.shell_path = test_header_shell
composer.header.scheduler_headers = test_header_scheduler
composer.details.path = test_details_path
composer.details.group = test_details_group
composer.details.perms = test_details_perms
self.assertEqual(composer.header.shell_path, test_header_shell)
self.assertEqual(composer.header.scheduler_headers,
test_header_scheduler)
self.assertEqual(composer.details.path, Path(test_details_path))
self.assertEqual(composer.details.group, test_details_group)
self.assertEqual(composer.details.perms, oct(test_details_perms))
composer = scriptcomposer.ScriptComposer()
self.assertEqual(composer.header.shell_path, '#!/bin/bash')
self.assertEqual(composer.header.scheduler_headers, [])
self.assertEqual(composer.details.group, utils.get_login())
self.assertEqual(composer.details.perms, oct(0o770))
# Testing object assignment.
header = scriptcomposer.ScriptHeader(
shell_path=test_header_shell,
scheduler_headers=test_header_scheduler)
testDetailsObj = scriptcomposer.ScriptDetails(path=test_details_path,
group=test_details_group,
perms=test_details_perms)
composer.header = header
composer.details = testDetailsObj
self.assertEqual(composer.header.shell_path, test_header_shell)
self.assertEqual(composer.header.scheduler_headers,
test_header_scheduler)
self.assertEqual(composer.details.path, Path(test_details_path))
self.assertEqual(composer.details.group, test_details_group)
self.assertEqual(composer.details.perms, oct(test_details_perms))
