def test_resolve_vars_in_vars(self): test = { 'permute_on': ['fruit', 'snacks'], 'variables': { 'fruit': ['apple', 'orange', 'banana'], 'snacks': ['{{fruit}}-x', '{{sys.soda}}'], 'stuff': 'y{{fruit}}-{{snacks}}', } } var_man = variables.VariableSetManager() var_man.add_var_set('sys', {'soda': 'pepsi'}) var_man.add_var_set('var', test['variables']) test, permuted = self.resolver.resolve_permutations(test, var_man) possible_stuff = [ 'yapple-apple-x', 'yapple-pepsi', 'yorange-orange-x', 'yorange-pepsi', 'ybanana-banana-x', 'ybanana-pepsi' ] stuff = [var_man['var.stuff'] for var_man in permuted] possible_stuff.sort() stuff.sort() self.assertEqual(possible_stuff, stuff)
def test_bad_data(self): """Make sure bad data causes issues on ingest.""" # Unknown vset name vsetm = variables.VariableSetManager() with self.assertRaises(ValueError): vsetm.add_var_set('blah', {}) # Duplicate vset name vsetm.add_var_set('var', {}) with self.assertRaises(ValueError): vsetm.add_var_set('var', {}) # Mismatched subvars data = { 'var4': [{'subvar1': 'subval0_1', 'subvar2': 'subval0_2'}, {'subvar1': 'subval1_1', 'subvar3': 'subval1_2'}] } with self.assertRaises(VariableError): vsetm.add_var_set('sys', data) slurm_data = { 'num_nodes': 45 } # Adding non-string data with self.assertRaises(VariableError): vsetm.add_var_set('sched', slurm_data)
def test_deferred(self): """Test deferred variables.""" data = { 'var1': 'val1', 'var3': {'subvar1': 'subval1', 'subvar2': 'subval2'}, } sys_data = { 'var1': variables.DeferredVariable(), } slurm_data = { 'num_nodes': '45' } var_man = variables.VariableSetManager() var_man.add_var_set('var', data) var_man.add_var_set('sys', sys_data) var_man.add_var_set('sched', slurm_data) with self.assertRaises(ValueError): var_man.len('sys', 'var1') for key in ( 'sys.var1', 'sys.var1.3', 'sys.var1.1.subvar1', 'sys.var1.noexist'): with self.assertRaises(KeyError): _ = var_man[key]
def test_good_queries(self): """Make sure all valid ways to lookup variables work.""" data = { 'var1': 'val1', 'var2': ['0', '1', '2'], 'var3': {'subvar1': 'subval1', 'subvar2': 'subval2'}, 'var4': [{'subvar1': 'subval0_1', 'subvar2': 'subval0_2'}, {'subvar1': 'subval1_1', 'subvar2': 'subval1_2'}] } sys_data = { 'var1': 'sys.val1' } slurm_data = { 'num_nodes': '45' } vsetm = variables.VariableSetManager() vsetm.add_var_set('var', data) vsetm.add_var_set('sys', sys_data) vsetm.add_var_set('sched', slurm_data) # Lookup without set name, this also conflicts across vsets and should # resolve correctly. self.assertEqual(vsetm['var1'], 'val1') # Lookup with set name self.assertEqual(vsetm['var.var1'], 'val1') # Explicit Index self.assertEqual(vsetm['var.var1.0'], 'val1') # Implicit Index self.assertEqual(vsetm['var2'], '0') # Explicit Index, set name self.assertEqual(vsetm['var.var2.2'], '2') # Negative Indexes are allowed (this one is at the edge of the range). self.assertEqual(vsetm['var.var2.-3'], '0') # Check the length of a variable list self.assertEqual(vsetm.len('var', 'var2'), 3) # Subkeys, when there's just one. self.assertEqual(vsetm['var3.subvar1'], 'subval1') self.assertEqual(vsetm['var3.subvar2'], 'subval2') # Subkey with explicit index self.assertEqual(vsetm['var3.0.subvar2'], 'subval2') # Multiple subkeys self.assertEqual(vsetm['var4.0.subvar1'], 'subval0_1') # Implicit index self.assertEqual(vsetm['var4.subvar1'], 'subval0_1') self.assertEqual(vsetm['var4.1.subvar1'], 'subval1_1') self.assertEqual(vsetm['var4.0.subvar2'], 'subval0_2') self.assertEqual(vsetm['var4.1.subvar2'], 'subval1_2') # Explicit access to conflicting variable self.assertEqual(vsetm['sys.var1'], 'sys.val1')
def test_deferred_errors(self): """Using deferred variables in inappropriate places should raise errors.""" test = { 'permute_on': ['sys.def'], 'variables': {}, } var_man = variables.VariableSetManager() var_man.add_var_set('sys', {'def': variables.DeferredVariable()}) with self.assertRaises(TestConfigError): self.resolver.resolve_permutations(test, var_man) test = {'permute_on': ['foo.1'], 'variables': {'foo': 'bleh'}} with self.assertRaises(TestConfigError): self.resolver.resolve_permutations(test, var_man) test = { 'permute_on': ['no_exist'], 'variables': {}, } with self.assertRaises(TestConfigError): self.resolver.resolve_permutations(test, var_man)
def test_finalize(self): cfg = self._quick_test_cfg() cfg['run']['cmds'] = ['echo "{{sys.host_name}}"'] cfg['result_parse'] = { 'regex': { 'foo': { 'regex': '{{sys.host_name}}' } } } test = self._quick_test(cfg, 'finalize_test', build=False, finalize=False) test.build() undefered_sys_vars = system_variables.SysVarDict(unique=True, ) fin_var_man = variables.VariableSetManager() fin_var_man.add_var_set('sys', undefered_sys_vars) resolver.TestConfigResolver.finalize(test, fin_var_man) results = test.gather_results(test.run()) test.save_results(results)
def test_unacceptable_queries(self): """Make sure all invalid variable lookups break.""" data = { 'var1': 'val1', 'var2': ['0', '1', '2'], 'var3': { 'subvar1': 'subval1', 'subvar2': 'subval2' }, 'var4': [{ 'subvar1': 'subval0_1', 'subvar2': 'subval0_2' }, { 'subvar1': 'subval1_1', 'subvar2': 'subval1_2' }] } vsetm = variables.VariableSetManager() vsetm.add_var_set('var', data) # Missing var self.assertRaises(KeyError, lambda: vsetm['var99']) # Too many parts (no vset) self.assertRaises(KeyError, lambda: vsetm['var1.0.a.b']) # Too many parts (w/ vset) self.assertRaises(KeyError, lambda: vsetm['var.var1.0.a.b']) # Empty key self.assertRaises(KeyError, lambda: vsetm['']) # vset only self.assertRaises(KeyError, lambda: vsetm['var']) # empty vset self.assertRaises(KeyError, lambda: vsetm['.var1']) # empty index/subvar self.assertRaises(KeyError, lambda: vsetm['var1.']) # empty index self.assertRaises(KeyError, lambda: vsetm['var3..subvar1']) # Out of range index self.assertRaises(KeyError, lambda: vsetm['var2.1000']) # Out of range negative index self.assertRaises(KeyError, lambda: vsetm['var2.-4']) # Empty subvar self.assertRaises(KeyError, lambda: vsetm['var1.0.']) # Unknown subvar self.assertRaises(KeyError, lambda: vsetm['var1.0.bleh']) # Out of range self.assertRaises(KeyError, lambda: vsetm['var1.1']) # Missing subvar self.assertRaises(KeyError, lambda: vsetm['var3.0.nope']) # Has subvar but none referenced self.assertRaises(KeyError, lambda: vsetm['var3.0']) # Len of invalid key self.assertRaises(KeyError, lambda: vsetm.len('var', 'var99')) # Keys must be unicode or a list/tuple self.assertRaises(TypeError, lambda: vsetm[1])
def __init__(self, pav_cfg): self.pav_cfg = pav_cfg self.base_var_man = variables.VariableSetManager() try: self.base_var_man.add_var_set( 'sys', system_variables.get_vars(defer=True)) except system_variables.SystemPluginError as err: raise TestConfigError("Error in system variables: {}".format(err)) self.base_var_man.add_var_set('pav', pavilion_variables.PavVars()) self.logger = logging.getLogger(__file__)
def test_resolve_template(self): tmpl_path = os.path.join(self.TEST_DATA_ROOT, 'resolve_template_good.tmpl') var_man = variables.VariableSetManager() var_man.add_var_set('sched', {'num_nodes': '3', 'partition': 'test'}) var_man.add_var_set('sys', { 'hostname': 'test.host.com', 'complicated': { 'a': 'yes', 'b': 'no' } }) script_path = tempfile.mktemp() PavTest.resolve_template(tmpl_path, script_path, var_man) good_path = os.path.join(self.TEST_DATA_ROOT, 'resolve_template_good.sh') with open(script_path) as gen_script,\ open(good_path) as ver_script: self.assertEqual(gen_script.read(), ver_script.read()) os.unlink(script_path) for bad_tmpl in ('resolve_template_keyerror.tmpl', 'resolve_template_bad_key.tmpl'): script_path = tempfile.mktemp() tmpl_path = os.path.join(self.TEST_DATA_ROOT, bad_tmpl) with self.assertRaises( KeyError, msg="Error not raised on bad file '{}'".format(bad_tmpl)): PavTest.resolve_template(tmpl_path, script_path, var_man) if os.path.exists(script_path): os.unlink(script_path) script_path = tempfile.mktemp() tmpl_path = os.path.join(self.TEST_DATA_ROOT, 'resolve_template_extra_escape.tmpl') with self.assertRaises( PavTestError, msg="Error not raised on bad file '{}'".format(bad_tmpl)): PavTest.resolve_template(tmpl_path, script_path, var_man) if os.path.exists(script_path): os.unlink(script_path)
def setUp(self) -> None: plugins.initialize_plugins(self.pav_cfg) self.var_man = variables.VariableSetManager() self.var_man.add_var_set( 'var', { 'int1': "1", 'int2': "2", 'float1': '1.1', 'str1': 'hello', 'ints': ['0', '1', '2', '3', '4', '5'], 'floats': ['0.1', '2.3'], 'more_ints': ['0', '1'], 'struct': { 'cpus': '200', 'flops': '2.1', 'name': 'earth_chicken', }, 'structs': [ { 'type': 'cat', 'bites': '3', 'evil_rating': '5.2' }, { 'type': 'dog', 'bites': '0', 'evil_rating': '0.2' }, { 'type': 'fish', 'bites': '1', 'evil_rating': '9.7' }, ] }) self.var_man.add_var_set('sys', system_variables.get_vars(defer=True))
def test_deferred(self): """Test deferred variables.""" data = { 'var1': 'val1', 'var3': {'subvar1': 'subval1', 'subvar2': 'subval2'}, } sys_data = { 'var1': variables.DeferredVariable('var1'), 'var3': variables.DeferredVariable('var3', sub_keys=['subvar1', 'subvar2']), } with self.assertRaises(ValueError): variables.DeferredVariable('test', var_set='var') slurm_data = { 'num_nodes': '45' } vsetm = variables.VariableSetManager() vsetm.add_var_set('var', data) vsetm.add_var_set('sys', sys_data) vsetm.add_var_set('sched', slurm_data) self.assertEqual(vsetm.len('sys', 'var1'), 1) self.assertEqual(vsetm['sys.var1'], '[\x1esys.var1\x1e]') self.assertEqual(vsetm['sys.var3.subvar1'], '[\x1esys.var3.subvar1\x1e]') for key in ( 'sys.var1.3', 'sys.var1.1.subvar1', 'sys.var3.noexist', 'sys.var1.noexist', 'sys.var3'): with self.assertRaises(KeyError): _ = vsetm[key]
def test_finalize(self): plugins.initialize_plugins(self.pav_cfg) cfg = self._quick_test_cfg() cfg['run']['cmds'] = ['echo "{{sys.host_name}}"'] cfg['results'] = { 'regex': [{ 'key': 'foo', 'regex': '{{sys.host_name}}', }] } test = self._quick_test(cfg, 'finalize_test', build=False, finalize=False) test.build() undefered_sys_vars = system_variables.SysVarDict( defer=False, unique=True, ) fin_var_man = variables.VariableSetManager() fin_var_man.add_var_set('sys', undefered_sys_vars) test.finalize(fin_var_man) results = test.gather_results(test.run()) test.save_results(results) plugins._reset_plugins()
def __init__(self, pav_cfg, config, build_tracker=None, var_man=None, _id=None, rebuild=False, build_only=False): """Create an new TestRun object. If loading an existing test instance, use the ``TestRun.from_id()`` method. :param pav_cfg: The pavilion configuration. :param dict config: The test configuration dictionary. :param builder.MultiBuildTracker build_tracker: Tracker for watching and managing the status of multiple builds. :param variables.VariableSetManager var_man: The variable set manager for this test. :param bool build_only: Only build this test run, do not run it. :param bool rebuild: After determining the build name, deprecate it and select a new, non-deprecated build. :param int _id: The test id of an existing test. (You should be using TestRun.load). """ # Just about every method needs this self._pav_cfg = pav_cfg self.load_ok = True self.scheduler = config['scheduler'] # Create the tests directory if it doesn't already exist. tests_path = pav_cfg.working_dir/'test_runs' self.config = config self.id = None # pylint: disable=invalid-name self._attrs = {} # Mark the run to build locally. self.build_local = config.get('build', {}) \ .get('on_nodes', 'false').lower() != 'true' # If a test access group was given, make sure it exists and the # current user is a member. self.group = config.get('group') if self.group is not None: try: group_data = grp.getgrnam(self.group) user = utils.get_login() if self.group != user and user not in group_data.gr_mem: raise TestConfigError( "Test specified group '{}', but the current user '{}' " "is not a member of that group." .format(self.group, user)) except KeyError as err: raise TestConfigError( "Test specified group '{}', but that group does not " "exist on this system. {}" .format(self.group, err)) self.umask = config.get('umask') if self.umask is not None: try: self.umask = int(self.umask, 8) except ValueError: raise RuntimeError( "Invalid umask. This should have been enforced by the " "by the config format.") self.build_only = build_only self.rebuild = rebuild self.suite_path = None if self.config.get('suite_path') is not None: try: self.suite_path = Path(self.config['suite_path']) except ValueError: pass # Get an id for the test, if we weren't given one. if _id is None: self.id, self.path = self.create_id_dir(tests_path) with PermissionsManager(self.path, self.group, self.umask): self._save_config() if var_man is None: var_man = variables.VariableSetManager() self.var_man = var_man self._variables_path = self.path / 'variables' self.var_man.save(self._variables_path) self.save_attributes() else: self.id = _id self.path = utils.make_id_path(tests_path, self.id) self._variables_path = self.path / 'variables' if not self.path.is_dir(): raise TestRunNotFoundError( "No test with id '{}' could be found.".format(self.id)) try: self.var_man = variables.VariableSetManager.load( self._variables_path ) except RuntimeError as err: raise TestRunError(*err.args) self.load_attributes() name_parts = [ self.config.get('suite', '<unknown>'), self.config.get('name', '<unnamed>'), ] subtitle = self.config.get('subtitle') # Don't add undefined or empty subtitles. if subtitle: name_parts.append(subtitle) self.name = '.'.join(name_parts) # Set a logger more specific to this test. self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id)) # This will be set by the scheduler self._job_id = None with PermissionsManager(self.path/'status', self.group, self.umask): # Setup the initial status file. self.status = StatusFile(self.path/'status') if _id is None: self.status.set(STATES.CREATED, "Test directory and status file created.") self.run_timeout = self.parse_timeout( 'run', config.get('run', {}).get('timeout')) self.build_timeout = self.parse_timeout( 'build', config.get('build', {}).get('timeout')) self._attributes = {} self.build_name = None self.run_log = self.path/'run.log' self.results_path = self.path/'results.json' self.build_origin_path = self.path/'build_origin' build_config = self.config.get('build', {}) if (build_config.get('source_path') is None and build_config.get('source_url') is not None): raise TestConfigError( "Build source_url specified, but not a source_path.") self.build_script_path = self.path/'build.sh' # type: Path self.build_path = self.path/'build' if _id is None: self._write_script( 'build', path=self.build_script_path, config=build_config) build_name = None self._build_name_fn = self.path / 'build_name' if _id is not None: build_name = self._load_build_name() try: self.builder = builder.TestBuilder( pav_cfg=pav_cfg, test=self, mb_tracker=build_tracker, build_name=build_name ) except builder.TestBuilderError as err: raise TestRunError( "Could not create builder for test {s.name} (run {s.id}): {err}" .format(s=self, err=err) ) self.save_build_name() run_config = self.config.get('run', {}) self.run_tmpl_path = self.path/'run.tmpl' self.run_script_path = self.path/'run.sh' if _id is None: self._write_script( 'run', path=self.run_tmpl_path, config=run_config) if _id is None: self.status.set(STATES.CREATED, "Test directory setup complete.") self._results = None self._created = None self.skipped = self._get_skipped()
def __init__(self, pav_cfg, config, build_tracker=None, var_man=None, _id=None, rebuild=False, build_only=False): """Create an new TestRun object. If loading an existing test instance, use the ``TestRun.from_id()`` method. :param pav_cfg: The pavilion configuration. :param dict config: The test configuration dictionary. :param builder.MultiBuildTracker build_tracker: Tracker for watching and managing the status of multiple builds. :param variables.VariableSetManager var_man: The variable set manager for this test. :param bool build_only: Only build this test run, do not run it. :param bool rebuild: After determining the build name, deprecate it and select a new, non-deprecated build. :param int _id: The test id of an existing test. (You should be using TestRun.load). """ # Just about every method needs this self._pav_cfg = pav_cfg self.scheduler = config['scheduler'] # Create the tests directory if it doesn't already exist. tests_path = pav_cfg.working_dir / 'test_runs' self.config = config group, umask = self.get_permissions(pav_cfg, config) # Get an id for the test, if we weren't given one. if _id is None: id_tmp, run_path = dir_db.create_id_dir(tests_path, group, umask) super().__init__(path=run_path, group=group, umask=umask) # Set basic attributes self.id = id_tmp self.build_only = build_only self.complete = False self.created = dt.datetime.now() self.name = self.make_name(config) self.rebuild = rebuild self.suite_path = Path(config.get('suite_path', '.')) self.user = utils.get_login() self.uuid = str(uuid.uuid4()) else: # Load the test info from the given id path. super().__init__(path=dir_db.make_id_path(tests_path, _id), group=group, umask=umask) self.load_attributes() self.test_version = config.get('test_version') if not self.path.is_dir(): raise TestRunNotFoundError( "No test with id '{}' could be found.".format(self.id)) # Mark the run to build locally. self.build_local = config.get('build', {}) \ .get('on_nodes', 'false').lower() != 'true' self._variables_path = self.path / 'variables' if _id is None: with PermissionsManager(self.path, self.group, self.umask): self._save_config() if var_man is None: var_man = variables.VariableSetManager() self.var_man = var_man self.var_man.save(self._variables_path) self.sys_name = self.var_man.get('sys_name', '<unknown>') else: try: self.var_man = variables.VariableSetManager.load( self._variables_path) except RuntimeError as err: raise TestRunError(*err.args) # This will be set by the scheduler self._job_id = None with PermissionsManager(self.path / 'status', self.group, self.umask): # Setup the initial status file. self.status = StatusFile(self.path / 'status') if _id is None: self.status.set(STATES.CREATED, "Test directory and status file created.") self.run_timeout = self.parse_timeout( 'run', config.get('run', {}).get('timeout')) self.build_timeout = self.parse_timeout( 'build', config.get('build', {}).get('timeout')) self.run_log = self.path / 'run.log' self.build_log = self.path / 'build.log' self.results_log = self.path / 'results.log' self.results_path = self.path / 'results.json' self.build_origin_path = self.path / 'build_origin' self.build_timeout_file = config.get('build', {}).get('timeout_file') # Use run.log as the default run timeout file self.timeout_file = self.run_log run_timeout_file = config.get('run', {}).get('timeout_file') if run_timeout_file is not None: self.timeout_file = self.path / run_timeout_file build_config = self.config.get('build', {}) self.build_script_path = self.path / 'build.sh' # type: Path self.build_path = self.path / 'build' if _id is None: self._write_script('build', path=self.build_script_path, config=build_config) try: self.builder = builder.TestBuilder(pav_cfg=pav_cfg, test=self, mb_tracker=build_tracker, build_name=self.build_name) self.build_name = self.builder.name except builder.TestBuilderError as err: raise TestRunError( "Could not create builder for test {s.name} (run {s.id}): {err}" .format(s=self, err=err)) run_config = self.config.get('run', {}) self.run_tmpl_path = self.path / 'run.tmpl' self.run_script_path = self.path / 'run.sh' if _id is None: self._write_script('run', path=self.run_tmpl_path, config=run_config) if _id is None: self.save_attributes() self.status.set(STATES.CREATED, "Test directory setup complete.") self._results = None self.skipped = self._get_skipped() # eval skip.
def run(self, sched_vars, sys_vars): """Run the test, returning True on success, False otherwise. :param dict sched_vars: The scheduler variables for resolving the build template. :param dict sys_vars: The system variables.""" self.status.set(STATES.PREPPING_RUN, "Resolving final run script.") if self.run_tmpl_path is not None: # Convert the run script template into the final run script. try: var_man = variables.VariableSetManager() var_man.add_var_set('sched', sched_vars) var_man.add_var_set('sys', sys_vars) self.resolve_template(self.run_tmpl_path, self.run_script_path, var_man) except KeyError as err: msg = ("Error converting run template '{}' into the final " "script: {}".format(self.run_tmpl_path, err)) self.logger.error(msg) self.status.set(STATES.RUN_ERROR, msg) return STATES.RUN_ERROR except PavTestError as err: self.logger.error(err) self.status.set(STATES.RUN_ERROR, err) return STATES.RUN_ERROR with self.run_log.open('wb') as run_log: self.status.set(STATES.RUNNING, "Starting the run script.") local_tz = tzlocal.get_localzone() self._started = local_tz.localize(datetime.datetime.now()) # TODO: There should always be a build directory, even if there # isn't a build. # Set the working directory to the build path, if there is one. run_wd = None if self.build_path is not None: run_wd = self.build_path.as_posix() # Run scripts take the test id as a first argument. cmd = [self.run_script_path.as_posix(), str(self.id)] proc = subprocess.Popen(cmd, cwd=run_wd, stdout=run_log, stderr=subprocess.STDOUT) # Run the test, but timeout if it doesn't produce any output every # RUN_SILENT_TIMEOUT seconds timeout = self.RUN_SILENT_TIMEOUT result = None while result is None: try: result = proc.wait(timeout=timeout) except subprocess.TimeoutExpired: out_stat = self.run_log.stat() quiet_time = time.time() - out_stat.st_mtime # Has the output file changed recently? if self.RUN_SILENT_TIMEOUT < quiet_time: # Give up on the build, and call it a failure. proc.kill() self.status.set( STATES.RUN_FAILED, "Run timed out after {} seconds.".format( self.RUN_SILENT_TIMEOUT)) self._finished = local_tz.localize( datetime.datetime.now()) return STATES.RUN_TIMEOUT else: # Only wait a max of BUILD_SILENT_TIMEOUT next 'wait' timeout = self.RUN_SILENT_TIMEOUT - quiet_time self._finished = local_tz.localize(datetime.datetime.now()) if result != 0: self.status.set(STATES.RUN_FAILED, "Test run failed.") return STATES.RUN_FAILED else: self.status.set(STATES.RUN_DONE, "Test run has completed successfully.") return STATES.RUN_DONE
class TestStringParser(PavTestCase): var_data = { 'var1': 'val1', 'sep': '-', 'var2': ['0', '1', '2'], 'var3': { 'subvar1': 'subval1', 'subvar2': 'subval2' }, 'var4': [{ 'subvar1': 'subval0_1', 'subvar2': 'subval0_2' }, { 'subvar1': 'subval1_1', 'subvar2': 'subval1_2' }] } pav_data = { 'var1': 'pval1', 'var2': ['p0', 'p1'], 'var3': { 'psubvar1': 'psubval1', 'psubvar2': 'psubval2' }, 'var4': [{ 'psubvar1': 'psubval0_1', 'psubvar2': 'psubval0_2' }, { 'psubvar1': 'psubval1_1', 'psubvar2': 'psubval1_2' }] } var_set_manager = variables.VariableSetManager() var_set_manager.add_var_set('var', var_data) var_set_manager.add_var_set('pav', pav_data) def test_parser(self): """Check string parsing and variable substitution.""" # Strings to test (unparsed string, expected result) test_strings = [ # The empty string is valid. ('', ''), # So are randomly generic strings. ('Hello you punks.', 'Hello you punks.'), # Checking that escapes are escaped. (r'\\\{\}\[\]\ \a', r'\{}[] a'), # Basic variable substitution ('Hello {{var1}} World', 'Hello val1 World'), # Variable with var_set. We'll rely on the variable tests for the full combinations, # as they use the same functions under the hood. ('Hello {{pav.var1}} World', 'Hello pval1 World'), # Substring substitution with spaces. ('Hello [~{{var2}}~ ] World.', 'Hello 0 1 2 World.'), # Substring substitution as last item. ('Hello [~{{var2}}~]', 'Hello 012'), # Substring substitution with multiple loop vars and a non-loop var. ('Hello [~{{var2}}{{sep}}{{pav.var2}}~ ] World.', 'Hello 0-p0 0-p1 1-p0 1-p1 2-p0 2-p1 World.'), # Substring substitution without spaces. ('Hello [~{{var2}}~]World.', 'Hello 012World.'), # Sub-strings with repeated usage ('Hello [~{{var4.subvar1}}-{{var4.subvar2}}~ ] World.', 'Hello subval0_1-subval0_2 subval1_1-subval1_2 World.'), # sub-sub strings ('Hello [~{{var2}}-[~{{var4.subvar1}}~-]~ ] World.', 'Hello 0-subval0_1-subval1_1 1-subval0_1-subval1_1 ' '2-subval0_1-subval1_1 World.'), # Default values ('Hello {{nope|World}}', 'Hello World'), ('No {{world|}} for you.', 'No for you.'), ] for test_str, answer_str in test_strings: self.assertEqual( string_parser.parse(test_str).resolve(self.var_set_manager), answer_str) def test_parser_errors(self): test_strings = [ # Missing close bracket on variable reference. ('Hello {{bleh World.', string_parser.ScanError), # Bad variable name ('Hello {{;;dasd}} World.', string_parser.ScanError), # Bad var_set name (raised by VariableSetManager, # re-caught in the tokenizer) ('Hello {{;.foo.bar}} World.', string_parser.ScanError), # Bad sub_var name ('Hello {{pav.bar.;-}} World.', string_parser.ScanError), # Extra close bracket ('Hello {{hello}}}} World.', string_parser.ScanError), # Strings cannot end with the escape character.for ('Hello \\', string_parser.ScanError), # The 'Unknown scanning error' exception shouldn't be reachable. # Missing close square bracket. ('Hello [~foo World', string_parser.ParseError), # The 'Unknown token of type' exception shouldn't be reachable. # Neither should the two RuntimeError's in Substring start and end. ] show_errors = False for test_str, error in test_strings: with self.assertRaises(error): string_parser.parse(test_str).resolve(self.var_set_manager) if show_errors: try: string_parser.parse(test_str).resolve(self.var_set_manager) except error: traceback.print_exc()
def test_resolve_all_vars(self): """Most of the variable resolution stuff is tested elsewhere, but it's good to have it all put together in one final test.""" test = { 'build': { 'cmds': [ "echo {{foo}} {{bar.p}}", "echo {{var.foo}}", "echo {{bar.q}}" ], 'env': [{ 'baz': '{{baz}}' }, { 'oof': '{{var.blarg.l}}-{{var.blarg.r}}' }, { 'pav': '{{pav.nope}}' }, { 'sys': '{{nope}}' }] }, 'variables': { 'baz': ['6'], 'blarg': [{ 'l': '7', 'r': '8' }], 'foo': ['1', '2'], 'bar': [ { 'p': '4', 'q': '4a' }, ], }, 'permute_on': ['foo', 'bar'], 'subtitle': None, } answer1 = { 'permute_on': ['foo', 'bar'], 'subtitle': '1-_bar_', 'build': { 'cmds': ["echo 1 4", "echo 1", "echo 4a"], 'env': [{ 'baz': '6' }, { 'oof': '7-8' }, { 'pav': '9' }, { 'sys': '10' }] } } # This is all that changes between the two. answer2 = copy.deepcopy(answer1) answer2['build']['cmds'] = ["echo 2 4", "echo 2", "echo 4a"] answer2['subtitle'] = '2-_bar_' answers = [answer1, answer2] var_man = variables.VariableSetManager() var_man.add_var_set('pav', {'nope': '9'}) var_man.add_var_set('sys', {'nope': '10'}) var_man.add_var_set('var', test['variables']) del test['variables'] test, permuted = self.resolver.resolve_permutations(test, var_man) self.assertEqual(len(permuted), 2) # Make sure each of our permuted results is in the list of answers. for var_man in permuted: out_test = self.resolver.resolve_test_vars(test, var_man) self.assertIn(out_test, answers) # Make sure we can successfully disallow deferred variables in a # section. test = { 'build': { 'cmds': ['echo {{foo}}'] }, 'permute_on': [], 'variables': {} } var_man = variables.VariableSetManager() var_man.add_var_set('sys', {'foo': variables.DeferredVariable()}) var_man.add_var_set('var', {}) test, permuted = self.resolver.resolve_permutations(test, var_man) with self.assertRaises(resolver.TestConfigError): # No deferred variables in the build section. self.resolver.resolve_test_vars(test, permuted[0])
def __init__(self, pav_cfg, config, build_tracker=None, var_man=None, _id=None, **options): """Create an new TestRun object. If loading an existing test instance, use the ``TestRun.from_id()`` method. :param pav_cfg: The pavilion configuration. :param dict config: The test configuration dictionary. :param builder.MultiBuildTracker build_tracker: Tracker for watching and managing the status of multiple builds. :param variables.VariableSetManager var_man: The variable set manager for this test. :param bool build_only: Only build this test run, do not run it. :param bool rebuild: After determining the build name, deprecate it and select a new, non-deprecated build. :param int _id: The test id of an existing test. (You should be using TestRun.load). """ # Just about every method needs this self._pav_cfg = pav_cfg self.load_ok = True # Compute the actual name of test, using the subtitle config parameter. self.name = '.'.join([ config.get('suite', '<unknown>'), config.get('name', '<unnamed>') ]) if 'subtitle' in config and config['subtitle']: self.name = self.name + '.' + config['subtitle'] self.scheduler = config['scheduler'] # Create the tests directory if it doesn't already exist. tests_path = pav_cfg.working_dir / 'test_runs' self.config = config self.id = None # pylint: disable=invalid-name # Mark the run to build locally. self.build_local = config.get('build', {}) \ .get('on_nodes', 'false').lower() != 'true' # Get an id for the test, if we weren't given one. if _id is None: self.id, self.path = self.create_id_dir(tests_path) self._save_config() if var_man is None: var_man = variables.VariableSetManager() self.var_man = var_man self._variables_path = self.path / 'variables' self.var_man.save(self._variables_path) self.opts = TestRunOptions(**options) self.opts.save(self) else: self.id = _id self.path = utils.make_id_path(tests_path, self.id) self._variables_path = self.path / 'variables' if not self.path.is_dir(): raise TestRunNotFoundError( "No test with id '{}' could be found.".format(self.id)) try: self.var_man = variables.VariableSetManager.load( self._variables_path) except RuntimeError as err: raise TestRunError(*err.args) self.opts = TestRunOptions.load(self) # Set a logger more specific to this test. self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id)) # This will be set by the scheduler self._job_id = None # Setup the initial status file. self.status = StatusFile(self.path / 'status') if _id is None: self.status.set(STATES.CREATED, "Test directory and status file created.") self.run_timeout = self.parse_timeout( 'run', config.get('run', {}).get('timeout')) self.build_timeout = self.parse_timeout( 'build', config.get('build', {}).get('timeout')) self._started = None self._finished = None self.build_name = None self.run_log = self.path / 'run.log' self.results_path = self.path / 'results.json' self.build_origin_path = self.path / 'build_origin' build_config = self.config.get('build', {}) # make sure build source_download_name is not set without # source_location try: if build_config['source_download_name'] is not None: if build_config['source_location'] is None: msg = "Test could not be built. Need 'source_location'." self.status.set( STATES.BUILD_ERROR, "'source_download_name is set without a " "'source_location'") raise TestConfigError(msg) except KeyError: # this is mostly for unit tests that create test configs without a # build section at all pass self.build_script_path = self.path / 'build.sh' # type: Path self.build_path = self.path / 'build' if _id is None: self._write_script('build', path=self.build_script_path, config=build_config) build_name = None self._build_name_fn = self.path / 'build_name' if _id is not None: build_name = self._load_build_name() try: self.builder = builder.TestBuilder(pav_cfg=pav_cfg, test=self, mb_tracker=build_tracker, build_name=build_name) except builder.TestBuilderError as err: raise TestRunError( "Could not create builder for test {s.name} (run {s.id}): {err}" .format(s=self, err=err)) self.save_build_name() run_config = self.config.get('run', {}) self.run_tmpl_path = self.path / 'run.tmpl' self.run_script_path = self.path / 'run.sh' if _id is None: self._write_script('run', path=self.run_tmpl_path, config=run_config) if _id is None: self.status.set(STATES.CREATED, "Test directory setup complete.") self._results = None self._created = None