def test_resolve_vars_in_vars(self):
test = {
'permute_on': ['fruit', 'snacks'],
'variables': {
'fruit': ['apple', 'orange', 'banana'],
'snacks': ['{{fruit}}-x', '{{sys.soda}}'],
'stuff': 'y{{fruit}}-{{snacks}}',
}
}
var_man = variables.VariableSetManager()
var_man.add_var_set('sys', {'soda': 'pepsi'})
var_man.add_var_set('var', test['variables'])
test, permuted = self.resolver.resolve_permutations(test, var_man)
possible_stuff = [
'yapple-apple-x', 'yapple-pepsi', 'yorange-orange-x',
'yorange-pepsi', 'ybanana-banana-x', 'ybanana-pepsi'
]
stuff = [var_man['var.stuff'] for var_man in permuted]
possible_stuff.sort()
stuff.sort()
self.assertEqual(possible_stuff, stuff)
def test_bad_data(self):
"""Make sure bad data causes issues on ingest."""
# Unknown vset name
vsetm = variables.VariableSetManager()
with self.assertRaises(ValueError):
vsetm.add_var_set('blah', {})
# Duplicate vset name
vsetm.add_var_set('var', {})
with self.assertRaises(ValueError):
vsetm.add_var_set('var', {})
# Mismatched subvars
data = {
'var4': [{'subvar1': 'subval0_1',
'subvar2': 'subval0_2'},
{'subvar1': 'subval1_1',
'subvar3': 'subval1_2'}]
}
with self.assertRaises(VariableError):
vsetm.add_var_set('sys', data)
slurm_data = {
'num_nodes': 45
}
# Adding non-string data
with self.assertRaises(VariableError):
vsetm.add_var_set('sched', slurm_data)
def test_deferred(self):
"""Test deferred variables."""
data = {
'var1': 'val1',
'var3': {'subvar1': 'subval1',
'subvar2': 'subval2'},
}
sys_data = {
'var1': variables.DeferredVariable(),
}
slurm_data = {
'num_nodes': '45'
}
var_man = variables.VariableSetManager()
var_man.add_var_set('var', data)
var_man.add_var_set('sys', sys_data)
var_man.add_var_set('sched', slurm_data)
with self.assertRaises(ValueError):
var_man.len('sys', 'var1')
for key in (
'sys.var1',
'sys.var1.3',
'sys.var1.1.subvar1',
'sys.var1.noexist'):
with self.assertRaises(KeyError):
_ = var_man[key]
def test_good_queries(self):
"""Make sure all valid ways to lookup variables work."""
data = {
'var1': 'val1',
'var2': ['0', '1', '2'],
'var3': {'subvar1': 'subval1',
'subvar2': 'subval2'},
'var4': [{'subvar1': 'subval0_1',
'subvar2': 'subval0_2'},
{'subvar1': 'subval1_1',
'subvar2': 'subval1_2'}]
}
sys_data = {
'var1': 'sys.val1'
}
slurm_data = {
'num_nodes': '45'
}
vsetm = variables.VariableSetManager()
vsetm.add_var_set('var', data)
vsetm.add_var_set('sys', sys_data)
vsetm.add_var_set('sched', slurm_data)
# Lookup without set name, this also conflicts across vsets and should
# resolve correctly.
self.assertEqual(vsetm['var1'], 'val1')
# Lookup with set name
self.assertEqual(vsetm['var.var1'], 'val1')
# Explicit Index
self.assertEqual(vsetm['var.var1.0'], 'val1')
# Implicit Index
self.assertEqual(vsetm['var2'], '0')
# Explicit Index, set name
self.assertEqual(vsetm['var.var2.2'], '2')
# Negative Indexes are allowed (this one is at the edge of the range).
self.assertEqual(vsetm['var.var2.-3'], '0')
# Check the length of a variable list
self.assertEqual(vsetm.len('var', 'var2'), 3)
# Subkeys, when there's just one.
self.assertEqual(vsetm['var3.subvar1'], 'subval1')
self.assertEqual(vsetm['var3.subvar2'], 'subval2')
# Subkey with explicit index
self.assertEqual(vsetm['var3.0.subvar2'], 'subval2')
# Multiple subkeys
self.assertEqual(vsetm['var4.0.subvar1'], 'subval0_1')
# Implicit index
self.assertEqual(vsetm['var4.subvar1'], 'subval0_1')
self.assertEqual(vsetm['var4.1.subvar1'], 'subval1_1')
self.assertEqual(vsetm['var4.0.subvar2'], 'subval0_2')
self.assertEqual(vsetm['var4.1.subvar2'], 'subval1_2')
# Explicit access to conflicting variable
self.assertEqual(vsetm['sys.var1'], 'sys.val1')
def test_deferred_errors(self):
"""Using deferred variables in inappropriate places should raise
errors."""
test = {
'permute_on': ['sys.def'],
'variables': {},
}
var_man = variables.VariableSetManager()
var_man.add_var_set('sys', {'def': variables.DeferredVariable()})
with self.assertRaises(TestConfigError):
self.resolver.resolve_permutations(test, var_man)
test = {'permute_on': ['foo.1'], 'variables': {'foo': 'bleh'}}
with self.assertRaises(TestConfigError):
self.resolver.resolve_permutations(test, var_man)
test = {
'permute_on': ['no_exist'],
'variables': {},
}
with self.assertRaises(TestConfigError):
self.resolver.resolve_permutations(test, var_man)
def test_finalize(self):
cfg = self._quick_test_cfg()
cfg['run']['cmds'] = ['echo "{{sys.host_name}}"']
cfg['result_parse'] = {
'regex': {
'foo': {
'regex': '{{sys.host_name}}'
}
}
}
test = self._quick_test(cfg,
'finalize_test',
build=False,
finalize=False)
test.build()
undefered_sys_vars = system_variables.SysVarDict(unique=True, )
fin_var_man = variables.VariableSetManager()
fin_var_man.add_var_set('sys', undefered_sys_vars)
resolver.TestConfigResolver.finalize(test, fin_var_man)
results = test.gather_results(test.run())
test.save_results(results)
def test_unacceptable_queries(self):
"""Make sure all invalid variable lookups break."""
data = {
'var1':
'val1',
'var2': ['0', '1', '2'],
'var3': {
'subvar1': 'subval1',
'subvar2': 'subval2'
},
'var4': [{
'subvar1': 'subval0_1',
'subvar2': 'subval0_2'
}, {
'subvar1': 'subval1_1',
'subvar2': 'subval1_2'
}]
}
vsetm = variables.VariableSetManager()
vsetm.add_var_set('var', data)
# Missing var
self.assertRaises(KeyError, lambda: vsetm['var99'])
# Too many parts (no vset)
self.assertRaises(KeyError, lambda: vsetm['var1.0.a.b'])
# Too many parts (w/ vset)
self.assertRaises(KeyError, lambda: vsetm['var.var1.0.a.b'])
# Empty key
self.assertRaises(KeyError, lambda: vsetm[''])
# vset only
self.assertRaises(KeyError, lambda: vsetm['var'])
# empty vset
self.assertRaises(KeyError, lambda: vsetm['.var1'])
# empty index/subvar
self.assertRaises(KeyError, lambda: vsetm['var1.'])
# empty index
self.assertRaises(KeyError, lambda: vsetm['var3..subvar1'])
# Out of range index
self.assertRaises(KeyError, lambda: vsetm['var2.1000'])
# Out of range negative index
self.assertRaises(KeyError, lambda: vsetm['var2.-4'])
# Empty subvar
self.assertRaises(KeyError, lambda: vsetm['var1.0.'])
# Unknown subvar
self.assertRaises(KeyError, lambda: vsetm['var1.0.bleh'])
# Out of range
self.assertRaises(KeyError, lambda: vsetm['var1.1'])
# Missing subvar
self.assertRaises(KeyError, lambda: vsetm['var3.0.nope'])
# Has subvar but none referenced
self.assertRaises(KeyError, lambda: vsetm['var3.0'])
# Len of invalid key
self.assertRaises(KeyError, lambda: vsetm.len('var', 'var99'))
# Keys must be unicode or a list/tuple
self.assertRaises(TypeError, lambda: vsetm[1])
def __init__(self, pav_cfg):
self.pav_cfg = pav_cfg
self.base_var_man = variables.VariableSetManager()
try:
self.base_var_man.add_var_set(
'sys', system_variables.get_vars(defer=True))
except system_variables.SystemPluginError as err:
raise TestConfigError("Error in system variables: {}".format(err))
self.base_var_man.add_var_set('pav', pavilion_variables.PavVars())
self.logger = logging.getLogger(__file__)
def test_resolve_template(self):
tmpl_path = os.path.join(self.TEST_DATA_ROOT,
'resolve_template_good.tmpl')
var_man = variables.VariableSetManager()
var_man.add_var_set('sched', {'num_nodes': '3', 'partition': 'test'})
var_man.add_var_set('sys', {
'hostname': 'test.host.com',
'complicated': {
'a': 'yes',
'b': 'no'
}
})
script_path = tempfile.mktemp()
PavTest.resolve_template(tmpl_path, script_path, var_man)
good_path = os.path.join(self.TEST_DATA_ROOT,
'resolve_template_good.sh')
with open(script_path) as gen_script,\
open(good_path) as ver_script:
self.assertEqual(gen_script.read(), ver_script.read())
os.unlink(script_path)
for bad_tmpl in ('resolve_template_keyerror.tmpl',
'resolve_template_bad_key.tmpl'):
script_path = tempfile.mktemp()
tmpl_path = os.path.join(self.TEST_DATA_ROOT, bad_tmpl)
with self.assertRaises(
KeyError,
msg="Error not raised on bad file '{}'".format(bad_tmpl)):
PavTest.resolve_template(tmpl_path, script_path, var_man)
if os.path.exists(script_path):
os.unlink(script_path)
script_path = tempfile.mktemp()
tmpl_path = os.path.join(self.TEST_DATA_ROOT,
'resolve_template_extra_escape.tmpl')
with self.assertRaises(
PavTestError,
msg="Error not raised on bad file '{}'".format(bad_tmpl)):
PavTest.resolve_template(tmpl_path, script_path, var_man)
if os.path.exists(script_path):
os.unlink(script_path)
def setUp(self) -> None:
plugins.initialize_plugins(self.pav_cfg)
self.var_man = variables.VariableSetManager()
self.var_man.add_var_set(
'var', {
'int1':
"1",
'int2':
"2",
'float1':
'1.1',
'str1':
'hello',
'ints': ['0', '1', '2', '3', '4', '5'],
'floats': ['0.1', '2.3'],
'more_ints': ['0', '1'],
'struct': {
'cpus': '200',
'flops': '2.1',
'name': 'earth_chicken',
},
'structs': [
{
'type': 'cat',
'bites': '3',
'evil_rating': '5.2'
},
{
'type': 'dog',
'bites': '0',
'evil_rating': '0.2'
},
{
'type': 'fish',
'bites': '1',
'evil_rating': '9.7'
},
]
})
self.var_man.add_var_set('sys', system_variables.get_vars(defer=True))
def test_deferred(self):
"""Test deferred variables."""
data = {
'var1': 'val1',
'var3': {'subvar1': 'subval1',
'subvar2': 'subval2'},
}
sys_data = {
'var1': variables.DeferredVariable('var1'),
'var3': variables.DeferredVariable('var3',
sub_keys=['subvar1', 'subvar2']),
}
with self.assertRaises(ValueError):
variables.DeferredVariable('test', var_set='var')
slurm_data = {
'num_nodes': '45'
}
vsetm = variables.VariableSetManager()
vsetm.add_var_set('var', data)
vsetm.add_var_set('sys', sys_data)
vsetm.add_var_set('sched', slurm_data)
self.assertEqual(vsetm.len('sys', 'var1'), 1)
self.assertEqual(vsetm['sys.var1'], '[\x1esys.var1\x1e]')
self.assertEqual(vsetm['sys.var3.subvar1'],
'[\x1esys.var3.subvar1\x1e]')
for key in (
'sys.var1.3',
'sys.var1.1.subvar1',
'sys.var3.noexist',
'sys.var1.noexist',
'sys.var3'):
with self.assertRaises(KeyError):
_ = vsetm[key]
def test_finalize(self):
plugins.initialize_plugins(self.pav_cfg)
cfg = self._quick_test_cfg()
cfg['run']['cmds'] = ['echo "{{sys.host_name}}"']
cfg['results'] = {
'regex': [{
'key': 'foo',
'regex': '{{sys.host_name}}',
}]
}
test = self._quick_test(cfg,
'finalize_test',
build=False,
finalize=False)
test.build()
undefered_sys_vars = system_variables.SysVarDict(
defer=False,
unique=True,
)
fin_var_man = variables.VariableSetManager()
fin_var_man.add_var_set('sys', undefered_sys_vars)
test.finalize(fin_var_man)
results = test.gather_results(test.run())
test.save_results(results)
plugins._reset_plugins()
def __init__(self, pav_cfg, config,
build_tracker=None, var_man=None, _id=None,
rebuild=False, build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.load_ok = True
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir/'test_runs'
self.config = config
self.id = None # pylint: disable=invalid-name
self._attrs = {}
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
# If a test access group was given, make sure it exists and the
# current user is a member.
self.group = config.get('group')
if self.group is not None:
try:
group_data = grp.getgrnam(self.group)
user = utils.get_login()
if self.group != user and user not in group_data.gr_mem:
raise TestConfigError(
"Test specified group '{}', but the current user '{}' "
"is not a member of that group."
.format(self.group, user))
except KeyError as err:
raise TestConfigError(
"Test specified group '{}', but that group does not "
"exist on this system. {}"
.format(self.group, err))
self.umask = config.get('umask')
if self.umask is not None:
try:
self.umask = int(self.umask, 8)
except ValueError:
raise RuntimeError(
"Invalid umask. This should have been enforced by the "
"by the config format.")
self.build_only = build_only
self.rebuild = rebuild
self.suite_path = None
if self.config.get('suite_path') is not None:
try:
self.suite_path = Path(self.config['suite_path'])
except ValueError:
pass
# Get an id for the test, if we weren't given one.
if _id is None:
self.id, self.path = self.create_id_dir(tests_path)
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self._variables_path = self.path / 'variables'
self.var_man.save(self._variables_path)
self.save_attributes()
else:
self.id = _id
self.path = utils.make_id_path(tests_path, self.id)
self._variables_path = self.path / 'variables'
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path
)
except RuntimeError as err:
raise TestRunError(*err.args)
self.load_attributes()
name_parts = [
self.config.get('suite', '<unknown>'),
self.config.get('name', '<unnamed>'),
]
subtitle = self.config.get('subtitle')
# Don't add undefined or empty subtitles.
if subtitle:
name_parts.append(subtitle)
self.name = '.'.join(name_parts)
# Set a logger more specific to this test.
self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id))
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path/'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path/'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run', config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build', config.get('build', {}).get('timeout'))
self._attributes = {}
self.build_name = None
self.run_log = self.path/'run.log'
self.results_path = self.path/'results.json'
self.build_origin_path = self.path/'build_origin'
build_config = self.config.get('build', {})
if (build_config.get('source_path') is None and
build_config.get('source_url') is not None):
raise TestConfigError(
"Build source_url specified, but not a source_path.")
self.build_script_path = self.path/'build.sh' # type: Path
self.build_path = self.path/'build'
if _id is None:
self._write_script(
'build',
path=self.build_script_path,
config=build_config)
build_name = None
self._build_name_fn = self.path / 'build_name'
if _id is not None:
build_name = self._load_build_name()
try:
self.builder = builder.TestBuilder(
pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=build_name
)
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err)
)
self.save_build_name()
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path/'run.tmpl'
self.run_script_path = self.path/'run.sh'
if _id is None:
self._write_script(
'run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self._created = None
self.skipped = self._get_skipped()
def __init__(self,
pav_cfg,
config,
build_tracker=None,
var_man=None,
_id=None,
rebuild=False,
build_only=False):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir / 'test_runs'
self.config = config
group, umask = self.get_permissions(pav_cfg, config)
# Get an id for the test, if we weren't given one.
if _id is None:
id_tmp, run_path = dir_db.create_id_dir(tests_path, group, umask)
super().__init__(path=run_path, group=group, umask=umask)
# Set basic attributes
self.id = id_tmp
self.build_only = build_only
self.complete = False
self.created = dt.datetime.now()
self.name = self.make_name(config)
self.rebuild = rebuild
self.suite_path = Path(config.get('suite_path', '.'))
self.user = utils.get_login()
self.uuid = str(uuid.uuid4())
else:
# Load the test info from the given id path.
super().__init__(path=dir_db.make_id_path(tests_path, _id),
group=group,
umask=umask)
self.load_attributes()
self.test_version = config.get('test_version')
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
self._variables_path = self.path / 'variables'
if _id is None:
with PermissionsManager(self.path, self.group, self.umask):
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self.var_man.save(self._variables_path)
self.sys_name = self.var_man.get('sys_name', '<unknown>')
else:
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path)
except RuntimeError as err:
raise TestRunError(*err.args)
# This will be set by the scheduler
self._job_id = None
with PermissionsManager(self.path / 'status', self.group, self.umask):
# Setup the initial status file.
self.status = StatusFile(self.path / 'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run',
config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build',
config.get('build', {}).get('timeout'))
self.run_log = self.path / 'run.log'
self.build_log = self.path / 'build.log'
self.results_log = self.path / 'results.log'
self.results_path = self.path / 'results.json'
self.build_origin_path = self.path / 'build_origin'
self.build_timeout_file = config.get('build', {}).get('timeout_file')
# Use run.log as the default run timeout file
self.timeout_file = self.run_log
run_timeout_file = config.get('run', {}).get('timeout_file')
if run_timeout_file is not None:
self.timeout_file = self.path / run_timeout_file
build_config = self.config.get('build', {})
self.build_script_path = self.path / 'build.sh' # type: Path
self.build_path = self.path / 'build'
if _id is None:
self._write_script('build',
path=self.build_script_path,
config=build_config)
try:
self.builder = builder.TestBuilder(pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=self.build_name)
self.build_name = self.builder.name
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err))
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path / 'run.tmpl'
self.run_script_path = self.path / 'run.sh'
if _id is None:
self._write_script('run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.save_attributes()
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self.skipped = self._get_skipped() # eval skip.
def run(self, sched_vars, sys_vars):
"""Run the test, returning True on success, False otherwise.
:param dict sched_vars: The scheduler variables for resolving the build
template.
:param dict sys_vars: The system variables."""
self.status.set(STATES.PREPPING_RUN, "Resolving final run script.")
if self.run_tmpl_path is not None:
# Convert the run script template into the final run script.
try:
var_man = variables.VariableSetManager()
var_man.add_var_set('sched', sched_vars)
var_man.add_var_set('sys', sys_vars)
self.resolve_template(self.run_tmpl_path, self.run_script_path,
var_man)
except KeyError as err:
msg = ("Error converting run template '{}' into the final "
"script: {}".format(self.run_tmpl_path, err))
self.logger.error(msg)
self.status.set(STATES.RUN_ERROR, msg)
return STATES.RUN_ERROR
except PavTestError as err:
self.logger.error(err)
self.status.set(STATES.RUN_ERROR, err)
return STATES.RUN_ERROR
with self.run_log.open('wb') as run_log:
self.status.set(STATES.RUNNING, "Starting the run script.")
local_tz = tzlocal.get_localzone()
self._started = local_tz.localize(datetime.datetime.now())
# TODO: There should always be a build directory, even if there
# isn't a build.
# Set the working directory to the build path, if there is one.
run_wd = None
if self.build_path is not None:
run_wd = self.build_path.as_posix()
# Run scripts take the test id as a first argument.
cmd = [self.run_script_path.as_posix(), str(self.id)]
proc = subprocess.Popen(cmd,
cwd=run_wd,
stdout=run_log,
stderr=subprocess.STDOUT)
# Run the test, but timeout if it doesn't produce any output every
# RUN_SILENT_TIMEOUT seconds
timeout = self.RUN_SILENT_TIMEOUT
result = None
while result is None:
try:
result = proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
out_stat = self.run_log.stat()
quiet_time = time.time() - out_stat.st_mtime
# Has the output file changed recently?
if self.RUN_SILENT_TIMEOUT < quiet_time:
# Give up on the build, and call it a failure.
proc.kill()
self.status.set(
STATES.RUN_FAILED,
"Run timed out after {} seconds.".format(
self.RUN_SILENT_TIMEOUT))
self._finished = local_tz.localize(
datetime.datetime.now())
return STATES.RUN_TIMEOUT
else:
# Only wait a max of BUILD_SILENT_TIMEOUT next 'wait'
timeout = self.RUN_SILENT_TIMEOUT - quiet_time
self._finished = local_tz.localize(datetime.datetime.now())
if result != 0:
self.status.set(STATES.RUN_FAILED, "Test run failed.")
return STATES.RUN_FAILED
else:
self.status.set(STATES.RUN_DONE,
"Test run has completed successfully.")
return STATES.RUN_DONE
class TestStringParser(PavTestCase):
var_data = {
'var1':
'val1',
'sep':
'-',
'var2': ['0', '1', '2'],
'var3': {
'subvar1': 'subval1',
'subvar2': 'subval2'
},
'var4': [{
'subvar1': 'subval0_1',
'subvar2': 'subval0_2'
}, {
'subvar1': 'subval1_1',
'subvar2': 'subval1_2'
}]
}
pav_data = {
'var1':
'pval1',
'var2': ['p0', 'p1'],
'var3': {
'psubvar1': 'psubval1',
'psubvar2': 'psubval2'
},
'var4': [{
'psubvar1': 'psubval0_1',
'psubvar2': 'psubval0_2'
}, {
'psubvar1': 'psubval1_1',
'psubvar2': 'psubval1_2'
}]
}
var_set_manager = variables.VariableSetManager()
var_set_manager.add_var_set('var', var_data)
var_set_manager.add_var_set('pav', pav_data)
def test_parser(self):
"""Check string parsing and variable substitution."""
# Strings to test (unparsed string, expected result)
test_strings = [
# The empty string is valid.
('', ''),
# So are randomly generic strings.
('Hello you punks.', 'Hello you punks.'),
# Checking that escapes are escaped.
(r'\\\{\}\[\]\ \a', r'\{}[] a'),
# Basic variable substitution
('Hello {{var1}} World', 'Hello val1 World'),
# Variable with var_set. We'll rely on the variable tests for the full combinations,
# as they use the same functions under the hood.
('Hello {{pav.var1}} World', 'Hello pval1 World'),
# Substring substitution with spaces.
('Hello [~{{var2}}~ ] World.', 'Hello 0 1 2 World.'),
# Substring substitution as last item.
('Hello [~{{var2}}~]', 'Hello 012'),
# Substring substitution with multiple loop vars and a non-loop var.
('Hello [~{{var2}}{{sep}}{{pav.var2}}~ ] World.',
'Hello 0-p0 0-p1 1-p0 1-p1 2-p0 2-p1 World.'),
# Substring substitution without spaces.
('Hello [~{{var2}}~]World.', 'Hello 012World.'),
# Sub-strings with repeated usage
('Hello [~{{var4.subvar1}}-{{var4.subvar2}}~ ] World.',
'Hello subval0_1-subval0_2 subval1_1-subval1_2 World.'),
# sub-sub strings
('Hello [~{{var2}}-[~{{var4.subvar1}}~-]~ ] World.',
'Hello 0-subval0_1-subval1_1 1-subval0_1-subval1_1 '
'2-subval0_1-subval1_1 World.'),
# Default values
('Hello {{nope|World}}', 'Hello World'),
('No {{world|}} for you.', 'No for you.'),
]
for test_str, answer_str in test_strings:
self.assertEqual(
string_parser.parse(test_str).resolve(self.var_set_manager),
answer_str)
def test_parser_errors(self):
test_strings = [
# Missing close bracket on variable reference.
('Hello {{bleh World.', string_parser.ScanError),
# Bad variable name
('Hello {{;;dasd}} World.', string_parser.ScanError),
# Bad var_set name (raised by VariableSetManager,
# re-caught in the tokenizer)
('Hello {{;.foo.bar}} World.', string_parser.ScanError),
# Bad sub_var name
('Hello {{pav.bar.;-}} World.', string_parser.ScanError),
# Extra close bracket
('Hello {{hello}}}} World.', string_parser.ScanError),
# Strings cannot end with the escape character.for
('Hello \\', string_parser.ScanError),
# The 'Unknown scanning error' exception shouldn't be reachable.
# Missing close square bracket.
('Hello [~foo World', string_parser.ParseError),
# The 'Unknown token of type' exception shouldn't be reachable.
# Neither should the two RuntimeError's in Substring start and end.
]
show_errors = False
for test_str, error in test_strings:
with self.assertRaises(error):
string_parser.parse(test_str).resolve(self.var_set_manager)
if show_errors:
try:
string_parser.parse(test_str).resolve(self.var_set_manager)
except error:
traceback.print_exc()
def test_resolve_all_vars(self):
"""Most of the variable resolution stuff is tested elsewhere,
but it's good to have it all put together in one final test."""
test = {
'build': {
'cmds': [
"echo {{foo}} {{bar.p}}", "echo {{var.foo}}",
"echo {{bar.q}}"
],
'env': [{
'baz': '{{baz}}'
}, {
'oof': '{{var.blarg.l}}-{{var.blarg.r}}'
}, {
'pav': '{{pav.nope}}'
}, {
'sys': '{{nope}}'
}]
},
'variables': {
'baz': ['6'],
'blarg': [{
'l': '7',
'r': '8'
}],
'foo': ['1', '2'],
'bar': [
{
'p': '4',
'q': '4a'
},
],
},
'permute_on': ['foo', 'bar'],
'subtitle': None,
}
answer1 = {
'permute_on': ['foo', 'bar'],
'subtitle': '1-_bar_',
'build': {
'cmds': ["echo 1 4", "echo 1", "echo 4a"],
'env': [{
'baz': '6'
}, {
'oof': '7-8'
}, {
'pav': '9'
}, {
'sys': '10'
}]
}
}
# This is all that changes between the two.
answer2 = copy.deepcopy(answer1)
answer2['build']['cmds'] = ["echo 2 4", "echo 2", "echo 4a"]
answer2['subtitle'] = '2-_bar_'
answers = [answer1, answer2]
var_man = variables.VariableSetManager()
var_man.add_var_set('pav', {'nope': '9'})
var_man.add_var_set('sys', {'nope': '10'})
var_man.add_var_set('var', test['variables'])
del test['variables']
test, permuted = self.resolver.resolve_permutations(test, var_man)
self.assertEqual(len(permuted), 2)
# Make sure each of our permuted results is in the list of answers.
for var_man in permuted:
out_test = self.resolver.resolve_test_vars(test, var_man)
self.assertIn(out_test, answers)
# Make sure we can successfully disallow deferred variables in a
# section.
test = {
'build': {
'cmds': ['echo {{foo}}']
},
'permute_on': [],
'variables': {}
}
var_man = variables.VariableSetManager()
var_man.add_var_set('sys', {'foo': variables.DeferredVariable()})
var_man.add_var_set('var', {})
test, permuted = self.resolver.resolve_permutations(test, var_man)
with self.assertRaises(resolver.TestConfigError):
# No deferred variables in the build section.
self.resolver.resolve_test_vars(test, permuted[0])
def __init__(self,
pav_cfg,
config,
build_tracker=None,
var_man=None,
_id=None,
**options):
"""Create an new TestRun object. If loading an existing test
instance, use the ``TestRun.from_id()`` method.
:param pav_cfg: The pavilion configuration.
:param dict config: The test configuration dictionary.
:param builder.MultiBuildTracker build_tracker: Tracker for watching
and managing the status of multiple builds.
:param variables.VariableSetManager var_man: The variable set manager for this
test.
:param bool build_only: Only build this test run, do not run it.
:param bool rebuild: After determining the build name, deprecate it and select
a new, non-deprecated build.
:param int _id: The test id of an existing test. (You should be using
TestRun.load).
"""
# Just about every method needs this
self._pav_cfg = pav_cfg
self.load_ok = True
# Compute the actual name of test, using the subtitle config parameter.
self.name = '.'.join([
config.get('suite', '<unknown>'),
config.get('name', '<unnamed>')
])
if 'subtitle' in config and config['subtitle']:
self.name = self.name + '.' + config['subtitle']
self.scheduler = config['scheduler']
# Create the tests directory if it doesn't already exist.
tests_path = pav_cfg.working_dir / 'test_runs'
self.config = config
self.id = None # pylint: disable=invalid-name
# Mark the run to build locally.
self.build_local = config.get('build', {}) \
.get('on_nodes', 'false').lower() != 'true'
# Get an id for the test, if we weren't given one.
if _id is None:
self.id, self.path = self.create_id_dir(tests_path)
self._save_config()
if var_man is None:
var_man = variables.VariableSetManager()
self.var_man = var_man
self._variables_path = self.path / 'variables'
self.var_man.save(self._variables_path)
self.opts = TestRunOptions(**options)
self.opts.save(self)
else:
self.id = _id
self.path = utils.make_id_path(tests_path, self.id)
self._variables_path = self.path / 'variables'
if not self.path.is_dir():
raise TestRunNotFoundError(
"No test with id '{}' could be found.".format(self.id))
try:
self.var_man = variables.VariableSetManager.load(
self._variables_path)
except RuntimeError as err:
raise TestRunError(*err.args)
self.opts = TestRunOptions.load(self)
# Set a logger more specific to this test.
self.logger = logging.getLogger('pav.TestRun.{}'.format(self.id))
# This will be set by the scheduler
self._job_id = None
# Setup the initial status file.
self.status = StatusFile(self.path / 'status')
if _id is None:
self.status.set(STATES.CREATED,
"Test directory and status file created.")
self.run_timeout = self.parse_timeout(
'run',
config.get('run', {}).get('timeout'))
self.build_timeout = self.parse_timeout(
'build',
config.get('build', {}).get('timeout'))
self._started = None
self._finished = None
self.build_name = None
self.run_log = self.path / 'run.log'
self.results_path = self.path / 'results.json'
self.build_origin_path = self.path / 'build_origin'
build_config = self.config.get('build', {})
# make sure build source_download_name is not set without
# source_location
try:
if build_config['source_download_name'] is not None:
if build_config['source_location'] is None:
msg = "Test could not be built. Need 'source_location'."
self.status.set(
STATES.BUILD_ERROR,
"'source_download_name is set without a "
"'source_location'")
raise TestConfigError(msg)
except KeyError:
# this is mostly for unit tests that create test configs without a
# build section at all
pass
self.build_script_path = self.path / 'build.sh' # type: Path
self.build_path = self.path / 'build'
if _id is None:
self._write_script('build',
path=self.build_script_path,
config=build_config)
build_name = None
self._build_name_fn = self.path / 'build_name'
if _id is not None:
build_name = self._load_build_name()
try:
self.builder = builder.TestBuilder(pav_cfg=pav_cfg,
test=self,
mb_tracker=build_tracker,
build_name=build_name)
except builder.TestBuilderError as err:
raise TestRunError(
"Could not create builder for test {s.name} (run {s.id}): {err}"
.format(s=self, err=err))
self.save_build_name()
run_config = self.config.get('run', {})
self.run_tmpl_path = self.path / 'run.tmpl'
self.run_script_path = self.path / 'run.sh'
if _id is None:
self._write_script('run',
path=self.run_tmpl_path,
config=run_config)
if _id is None:
self.status.set(STATES.CREATED, "Test directory setup complete.")
self._results = None
self._created = None
