def _save_series_id(self): """Save the series id to json file that tracks last series ran by user on a per system basis.""" sys_vars = system_variables.get_vars(True) sys_name = sys_vars['sys_name'] json_file = self.pav_cfg.working_dir/'users' json_file /= '{}.json'.format(utils.get_login()) lockfile_path = json_file.with_suffix('.lock') with LockFile(lockfile_path): data = {} try: with json_file.open('r') as json_series_file: try: data = json.load(json_series_file) except json.decoder.JSONDecodeError: # File was empty, therefore json couldn't be loaded. pass with PermissionsManager(json_file, self.pav_cfg['shared_group'], self.pav_cfg['umask']), \ json_file.open('w') as json_series_file: data[sys_name] = self.sid json_series_file.write(json.dumps(data)) except FileNotFoundError: # File hadn't been created yet. with PermissionsManager(json_file, self.pav_cfg['shared_group'], self.pav_cfg['umask']), \ json_file.open('w') as json_series_file: data[sys_name] = self.sid json_series_file.write(json.dumps(data))
def get_permissions(pav_cfg, config) -> (str, int): """Get the permissions to use on file creation, either from the pav_cfg or test config it that overrides. :returns: A tuple of the group and umask. """ # If a test access group was given, make sure it exists and the # current user is a member. group = config.get('group', pav_cfg['shared_group']) if group is not None: try: group_data = grp.getgrnam(group) user = utils.get_login() if group != user and user not in group_data.gr_mem: raise TestConfigError( "Test specified group '{}', but the current user '{}' " "is not a member of that group.".format(group, user)) except KeyError as err: raise TestConfigError( "Test specified group '{}', but that group does not " "exist on this system. {}".format(group, err)) umask = config.get('umask', pav_cfg['umask']) if umask is not None: try: umask = int(umask, 8) except ValueError: raise RuntimeError( "Invalid umask. This should have been enforced by the " "by the config format.") return group, umask
def test_sys_name_tracker(self): """Make sure the expected values are stored in the user.json file.""" user = utils.get_login() sys_vars = system_variables.get_vars(True) sys_name = sys_vars['sys_name'] arg_parser = arguments.get_parser() args = arg_parser.parse_args(['run', 'hello_world']) run_cmd = commands.get_command(args.command_name) run_cmd.outfile = io.StringIO() run_cmd.errfile = run_cmd.outfile run_cmd.run(self.pav_cfg, args) series = run_cmd.last_series json_file = self.pav_cfg.working_dir / 'users' json_file /= '{}.json'.format(user) with json_file.open('r') as json_series_file: data = json.load(json_series_file) self.assertEqual(data[sys_name], series.sid)
def test_details(self): """Testing the ScriptDetails class.""" testPath = self.pav_cfg.working_dir / 'testScript.sh' testGroup = 'anonymous' testPerms = 0o531 # Testing valid uses. # Testing initialization and defaults. test_details = scriptcomposer.ScriptDetails() self.assertEqual(test_details.group, utils.get_login()) self.assertEqual(test_details.perms, oct(0o770)) # Testing individual assignment. test_details.path = testPath test_details.group = testGroup test_details.perms = testPerms self.assertEqual(test_details.path, Path(testPath)) self.assertEqual(test_details.group, testGroup) self.assertEqual(test_details.perms, oct(testPerms)) # Testing initialization assignment. test_details = scriptcomposer.ScriptDetails(path=testPath, group=testGroup, perms=testPerms) self.assertEqual(test_details.path, Path(testPath)) self.assertEqual(test_details.group, testGroup) self.assertEqual(test_details.perms, oct(testPerms)) test_details = scriptcomposer.ScriptDetails() # Testing invalid uses. with self.assertRaises(TypeError): test_details.path = True with self.assertRaises(TypeError): test_details.perms = 'string' with self.assertRaises(TypeError): test_details.perms = u'fail' with self.assertRaises(TypeError): test_details.perms = 7.5 # Testing invalid initialization. with self.assertRaises(TypeError): scriptcomposer.ScriptDetails(path=testPath, group=testGroup, perms='fail')
def _save_series_id(self): """Save the series id to the user's .pavilion directory.""" # Save the last series we created to the .pavilion directory # in the user's home dir. Pavilion commands can use this so the # user doesn't actually have to know the series_id of tests. last_series_fn = self.pav_cfg.working_dir/'users' last_series_fn /= '{}.series'.format(utils.get_login()) try: with last_series_fn.open('w') as last_series_file: last_series_file.write(self.id) except (IOError, OSError): # It's ok if we can't write this file. self._logger.warning("Could not save series id to '%s'", last_series_fn)
def load_user_series_id(cls, pav_cfg): """Load the last series id used by the current user.""" logger = logging.getLogger(cls.LOGGER_FMT.format('<unknown>')) last_series_fn = pav_cfg.working_dir/'users' last_series_fn /= '{}.series'.format(utils.get_login()) if not last_series_fn.exists(): return None try: with last_series_fn.open() as last_series_file: return last_series_file.read().strip() except (IOError, OSError) as err: logger.warning("Failed to read series id file '%s': %s", last_series_fn, err) return None
def __init__(self, *args, **kwargs): with self.PAV_CONFIG_PATH.open() as cfg_file: raw_pav_cfg = config.PavilionConfigLoader().load(cfg_file) raw_pav_cfg.config_dirs = [ self.TEST_DATA_ROOT / 'pav_config_dir', self.PAV_LIB_DIR ] raw_pav_cfg.working_dir = Path('/tmp') / get_login() / 'pav_tests' raw_pav_cfg.result_log = raw_pav_cfg.working_dir / 'results.log' if not raw_pav_cfg.working_dir.exists(): raw_pav_cfg.working_dir.mkdir() cfg_dir = raw_pav_cfg.working_dir / 'pav_cfgs' if not cfg_dir.exists(): cfg_dir.mkdir() cfg_path = Path(tempfile.mktemp(suffix='.yaml', dir=str(cfg_dir))) with cfg_path.open('w') as pav_cfg_file: config.PavilionConfigLoader().dump(pav_cfg_file, raw_pav_cfg) with cfg_path.open() as cfg_file: self.pav_cfg = config.PavilionConfigLoader().load(cfg_file) self.pav_cfg.pav_cfg_file = cfg_path # Create the basic directories in the working directory for path in [ self.pav_cfg.working_dir, self.pav_cfg.working_dir / 'builds', self.pav_cfg.working_dir / 'tests', self.pav_cfg.working_dir / 'series', self.pav_cfg.working_dir / 'users', self.pav_cfg.working_dir / 'downloads' ]: if not path.exists(): os.makedirs(str(path), exist_ok=True) self.tmp_dir = tempfile.TemporaryDirectory() # We have to get this to set up the base argument parser before # plugins can add to it. _ = arguments.get_parser() super().__init__(*args, **kwargs)
def load_user_series_id(pav_cfg): """Load the last series id used by the current user.""" last_series_fn = pav_cfg.working_dir / 'users' last_series_fn /= '{}.json'.format(utils.get_login()) sys_vars = system_variables.get_vars(True) sys_name = sys_vars['sys_name'] if not last_series_fn.exists(): return None try: with last_series_fn.open() as last_series_file: sys_name_series_dict = json.load(last_series_file) return sys_name_series_dict[sys_name].strip() except (IOError, OSError, KeyError) as err: logger.warning("Failed to read series id file '%s': %s", last_series_fn, err) return None
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Find a group that isn't the user's default group (or sudo), and # use that as our default group. login = utils.get_login() def_gid = os.getgid() candidates = [group for group in grp.getgrall() if (login in group.gr_mem and def_gid != group.gr_gid)] if not candidates: self.fail("Your user must be in at least two groups (other than " "the user's group) to run this test.") self.orig_group = grp.getgrgid(def_gid).gr_name self.alt_group = candidates[0] # type: grp.struct_group self.alt_group2 = candidates[1] # type: grp.struct_group self.umask = 0o007
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Find a group that isn't the user's default group (or sudo), and # use that as our default group. login = utils.get_login() def_gid = os.getgid() candidates = [group for group in grp.getgrall() if (login in group.gr_mem and def_gid != group.gr_gid)] if not candidates: self.orig_group = None self.alt_group = None self.alt_group2 = None else: self.orig_group = grp.getgrgid(def_gid).gr_name self.alt_group = candidates[0] # type: grp.struct_group self.alt_group2 = candidates[1] # type: grp.struct_group self.umask = 0o007
def _create_lockfile(cls, path, expires, lock_id, group_id=None): """Create and fill out a lockfile at the given path. :param Path path: Where the file will be created. :param int expires: How far in the future the lockfile expires. :param str lock_id: The unique identifier for this lockfile. :returns: None :raises IOError: When the file cannot be written too. :raises OSError: When the file cannot be opened or already exists. """ # DEV NOTE: This logic is separate so that we can create these files # outside of the standard mechanisms for testing purposes. # We're doing low level operations on the path, so we just need # it as a string. path = str(path) file_num = os.open(path, os.O_EXCL | os.O_CREAT | os.O_RDWR) file_note = ",".join( [os.uname()[1], utils.get_login(), str(expires), lock_id]) file_note = file_note.encode('utf8') os.write(file_num, file_note) os.close(file_num) try: os.chmod(path, cls.LOCK_PERMS) except OSError as err: LOGGER.warning("Lockfile at '%s' could not set permissions: %s", path, err) if group_id is not None: try: os.chown(path, os.getuid(), group_id) except OSError as err: LOGGER.warning("Lockfile at '%s' could not set group: %s", path, err)
from pavilion.test_run import TestRun LOCAL_SYS_NAME = '<local_sys_name>' TEST_FILTER_DEFAULTS = { 'complete': False, 'failed': False, 'incomplete': False, 'name': None, 'newer_than': time.time() - dt.timedelta(days=1).total_seconds(), 'older_than': None, 'passed': False, 'result_error': False, 'show_skipped': 'no', 'sort_by': '-created', 'sys_name': LOCAL_SYS_NAME, 'user': utils.get_login(), 'limit': None, 'disable_filter': False, } TEST_SORT_FUNCS = { 'created': lambda test: test['created'], 'finished': lambda test: test['finished'], 'name': lambda test: test['name'], 'started': lambda test: test['started'], 'user': lambda test: test['user'], 'id': lambda test: test['id'], } def add_common_filter_args(target: str, arg_parser: argparse.ArgumentParser,
def user(self): """The current user's login name.""" return utils.get_login()
def group(self, value): if value is None: value = utils.get_login() self._group = str(value)
def __init__(self, pav_cfg, config, build_tracker=None, var_man=None, _id=None, rebuild=False, build_only=False): """Create an new TestRun object. If loading an existing test instance, use the ``TestRun.from_id()`` method. :param pav_cfg: The pavilion configuration. :param dict config: The test configuration dictionary. :param builder.MultiBuildTracker build_tracker: Tracker for watching and managing the status of multiple builds. :param variables.VariableSetManager var_man: The variable set manager for this test. :param bool build_only: Only build this test run, do not run it. :param bool rebuild: After determining the build name, deprecate it and select a new, non-deprecated build. :param int _id: The test id of an existing test. (You should be using TestRun.load). """ # Just about every method needs this self._pav_cfg = pav_cfg self.load_ok = True self.scheduler = config['scheduler'] # Create the tests directory if it doesn't already exist. tests_path = pav_cfg.working_dir/'test_runs' self.config = config self.id = None # pylint: disable=invalid-name self._attrs = {} # Mark the run to build locally. self.build_local = config.get('build', {}) \ .get('on_nodes', 'false').lower() != 'true' # If a test access group was given, make sure it exists and the # current user is a member. self.group = config.get('group') if self.group is not None: try: group_data = grp.getgrnam(self.group) user = utils.get_login() if self.group != user and user not in group_data.gr_mem: raise TestConfigError( "Test specified group '{}', but the current user '{}' " "is not a member of that group." .format(self.group, user)) except KeyError as err: raise TestConfigError( "Test specified group '{}', but that group does not " "exist on this system. {}" .format(self.group, err)) self.umask = config.get('umask') if self.umask is not None: try: self.umask = int(self.umask, 8) except ValueError: raise RuntimeError( "Invalid umask. This should have been enforced by the " "by the config format.") self.build_only = build_only self.rebuild = rebuild self.suite_path = None if self.config.get('suite_path') is not None: try: self.suite_path = Path(self.config['suite_path']) except ValueError: pass # Get an id for the test, if we weren't given one. if _id is None: self.id, self.path = self.create_id_dir(tests_path) with PermissionsManager(self.path, self.group, self.umask): self._save_config() if var_man is None: var_man = variables.VariableSetManager() self.var_man = var_man self._variables_path = self.path / 'variables' self.var_man.save(self._variables_path) self.save_attributes() else: self.id = _id self.path = utils.make_id_path(tests_path, self.id) self._variables_path = self.path / 'variables' if not self.path.is_dir(): raise TestRunNotFoundError( "No test with id '{}' could be found.".format(self.id)) try: self.var_man = variables.VariableSetManager.load( self._variables_path ) except RuntimeError as err: raise TestRunError(*err.args) self.load_attributes() name_parts = [ self.config.get('suite', '<unknown>'), self.config.get('name', '<unnamed>'), ] subtitle = self.config.get('subtitle') # Don't add undefined or empty subtitles. if subtitle: name_parts.append(subtitle) self.name = '.'.join(name_parts) # Set a logger more specific to this test. self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id)) # This will be set by the scheduler self._job_id = None with PermissionsManager(self.path/'status', self.group, self.umask): # Setup the initial status file. self.status = StatusFile(self.path/'status') if _id is None: self.status.set(STATES.CREATED, "Test directory and status file created.") self.run_timeout = self.parse_timeout( 'run', config.get('run', {}).get('timeout')) self.build_timeout = self.parse_timeout( 'build', config.get('build', {}).get('timeout')) self._attributes = {} self.build_name = None self.run_log = self.path/'run.log' self.results_path = self.path/'results.json' self.build_origin_path = self.path/'build_origin' build_config = self.config.get('build', {}) if (build_config.get('source_path') is None and build_config.get('source_url') is not None): raise TestConfigError( "Build source_url specified, but not a source_path.") self.build_script_path = self.path/'build.sh' # type: Path self.build_path = self.path/'build' if _id is None: self._write_script( 'build', path=self.build_script_path, config=build_config) build_name = None self._build_name_fn = self.path / 'build_name' if _id is not None: build_name = self._load_build_name() try: self.builder = builder.TestBuilder( pav_cfg=pav_cfg, test=self, mb_tracker=build_tracker, build_name=build_name ) except builder.TestBuilderError as err: raise TestRunError( "Could not create builder for test {s.name} (run {s.id}): {err}" .format(s=self, err=err) ) self.save_build_name() run_config = self.config.get('run', {}) self.run_tmpl_path = self.path/'run.tmpl' self.run_script_path = self.path/'run.sh' if _id is None: self._write_script( 'run', path=self.run_tmpl_path, config=run_config) if _id is None: self.status.set(STATES.CREATED, "Test directory setup complete.") self._results = None self._created = None self.skipped = self._get_skipped()
def __init__(self, pav_cfg, config, build_tracker=None, var_man=None, _id=None, rebuild=False, build_only=False): """Create an new TestRun object. If loading an existing test instance, use the ``TestRun.from_id()`` method. :param pav_cfg: The pavilion configuration. :param dict config: The test configuration dictionary. :param builder.MultiBuildTracker build_tracker: Tracker for watching and managing the status of multiple builds. :param variables.VariableSetManager var_man: The variable set manager for this test. :param bool build_only: Only build this test run, do not run it. :param bool rebuild: After determining the build name, deprecate it and select a new, non-deprecated build. :param int _id: The test id of an existing test. (You should be using TestRun.load). """ # Just about every method needs this self._pav_cfg = pav_cfg self.scheduler = config['scheduler'] # Create the tests directory if it doesn't already exist. tests_path = pav_cfg.working_dir / 'test_runs' self.config = config group, umask = self.get_permissions(pav_cfg, config) # Get an id for the test, if we weren't given one. if _id is None: id_tmp, run_path = dir_db.create_id_dir(tests_path, group, umask) super().__init__(path=run_path, group=group, umask=umask) # Set basic attributes self.id = id_tmp self.build_only = build_only self.complete = False self.created = dt.datetime.now() self.name = self.make_name(config) self.rebuild = rebuild self.suite_path = Path(config.get('suite_path', '.')) self.user = utils.get_login() self.uuid = str(uuid.uuid4()) else: # Load the test info from the given id path. super().__init__(path=dir_db.make_id_path(tests_path, _id), group=group, umask=umask) self.load_attributes() self.test_version = config.get('test_version') if not self.path.is_dir(): raise TestRunNotFoundError( "No test with id '{}' could be found.".format(self.id)) # Mark the run to build locally. self.build_local = config.get('build', {}) \ .get('on_nodes', 'false').lower() != 'true' self._variables_path = self.path / 'variables' if _id is None: with PermissionsManager(self.path, self.group, self.umask): self._save_config() if var_man is None: var_man = variables.VariableSetManager() self.var_man = var_man self.var_man.save(self._variables_path) self.sys_name = self.var_man.get('sys_name', '<unknown>') else: try: self.var_man = variables.VariableSetManager.load( self._variables_path) except RuntimeError as err: raise TestRunError(*err.args) # This will be set by the scheduler self._job_id = None with PermissionsManager(self.path / 'status', self.group, self.umask): # Setup the initial status file. self.status = StatusFile(self.path / 'status') if _id is None: self.status.set(STATES.CREATED, "Test directory and status file created.") self.run_timeout = self.parse_timeout( 'run', config.get('run', {}).get('timeout')) self.build_timeout = self.parse_timeout( 'build', config.get('build', {}).get('timeout')) self.run_log = self.path / 'run.log' self.build_log = self.path / 'build.log' self.results_log = self.path / 'results.log' self.results_path = self.path / 'results.json' self.build_origin_path = self.path / 'build_origin' self.build_timeout_file = config.get('build', {}).get('timeout_file') # Use run.log as the default run timeout file self.timeout_file = self.run_log run_timeout_file = config.get('run', {}).get('timeout_file') if run_timeout_file is not None: self.timeout_file = self.path / run_timeout_file build_config = self.config.get('build', {}) self.build_script_path = self.path / 'build.sh' # type: Path self.build_path = self.path / 'build' if _id is None: self._write_script('build', path=self.build_script_path, config=build_config) try: self.builder = builder.TestBuilder(pav_cfg=pav_cfg, test=self, mb_tracker=build_tracker, build_name=self.build_name) self.build_name = self.builder.name except builder.TestBuilderError as err: raise TestRunError( "Could not create builder for test {s.name} (run {s.id}): {err}" .format(s=self, err=err)) run_config = self.config.get('run', {}) self.run_tmpl_path = self.path / 'run.tmpl' self.run_script_path = self.path / 'run.sh' if _id is None: self._write_script('run', path=self.run_tmpl_path, config=run_config) if _id is None: self.save_attributes() self.status.set(STATES.CREATED, "Test directory setup complete.") self._results = None self.skipped = self._get_skipped() # eval skip.
def test_scriptComposer(self): """Testing ScriptComposer class variable setting.""" # Testing valid uses. # Testing initialization defaults. composer = scriptcomposer.ScriptComposer() self.assertIsInstance(composer.header, scriptcomposer.ScriptHeader) self.assertIsInstance(composer.details, scriptcomposer.ScriptDetails) self.assertEqual(composer.header.shell_path, '#!/bin/bash') self.assertEqual(composer.header.scheduler_headers, []) self.assertEqual(composer.details.group, utils.get_login()) self.assertEqual(composer.details.perms, oct(0o770)) # Testing individual assignment test_header_shell = "/usr/env/python" test_header_scheduler = OrderedDict() test_header_scheduler['-G'] = 'pam' test_header_scheduler['-N'] = 'fam' composer.newline() composer.command(['taco', 'burrito', 'nachos']) test_details_path = 'testPath' test_details_group = 'groupies' test_details_perms = 0o543 composer.header.shell_path = test_header_shell composer.header.scheduler_headers = test_header_scheduler composer.details.path = test_details_path composer.details.group = test_details_group composer.details.perms = test_details_perms self.assertEqual(composer.header.shell_path, test_header_shell) self.assertEqual(composer.header.scheduler_headers, test_header_scheduler) self.assertEqual(composer.details.path, Path(test_details_path)) self.assertEqual(composer.details.group, test_details_group) self.assertEqual(composer.details.perms, oct(test_details_perms)) composer = scriptcomposer.ScriptComposer() self.assertEqual(composer.header.shell_path, '#!/bin/bash') self.assertEqual(composer.header.scheduler_headers, []) self.assertEqual(composer.details.group, utils.get_login()) self.assertEqual(composer.details.perms, oct(0o770)) # Testing object assignment. header = scriptcomposer.ScriptHeader( shell_path=test_header_shell, scheduler_headers=test_header_scheduler) testDetailsObj = scriptcomposer.ScriptDetails(path=test_details_path, group=test_details_group, perms=test_details_perms) composer.header = header composer.details = testDetailsObj self.assertEqual(composer.header.shell_path, test_header_shell) self.assertEqual(composer.header.scheduler_headers, test_header_scheduler) self.assertEqual(composer.details.path, Path(test_details_path)) self.assertEqual(composer.details.group, test_details_group) self.assertEqual(composer.details.perms, oct(test_details_perms))