def test_bad_plugins(self):
"""Make sure bad plugins don't kill Pavilion and print appropriate
errors."""
error_strs = [
'Plugin candidate rejected:',
'Unable to create plugin object:',
'Unable to import plugin:',
]
yapsy_logger = logging.getLogger('yapsy')
stream = io.StringIO()
hndlr = logging.StreamHandler(stream)
yapsy_logger.addHandler(hndlr)
pav_cfg = self.pav_cfg.copy()
cfg_dirs = list(pav_cfg.config_dirs)
cfg_dirs.append(self.TEST_DATA_ROOT/'bad_plugins')
pav_cfg.config_dirs = cfg_dirs
# A bunch of plugins should fail to load, but this should be fine
# anyway.
plugins.initialize_plugins(pav_cfg)
yapsy_logger.removeHandler(hndlr)
stream.seek(0)
logs = stream.read()
for error_str in error_strs:
self.assertIn(error_str, logs)
plugins._reset_plugins()
def test_command_plugins(self):
"""Make sure command plugin loading is sane."""
# Get an empty pavilion config and set some config dirs on it.
pav_cfg = config.PavilionConfigLoader().load_empty()
# We're loading multiple directories of plugins - AT THE SAME TIME!
pav_cfg.config_dirs = [
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir'),
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir2')
]
plugins.initialize_plugins(pav_cfg)
commands.get_command('poof').run(pav_cfg, [])
commands.get_command('blarg').run(pav_cfg, [])
# Clean up our plugin initializations.
plugins._reset_plugins()
pav_cfg.config_dirs.append(
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir_conflicts'))
self.assertRaises(plugins.PluginError,
lambda: plugins.initialize_plugins(pav_cfg))
# Clean up our plugin initializations.
plugins._reset_plugins()
def test_function_plugins(self):
"""Make sure each of the core function plugins work."""
plugins.initialize_plugins(self.pav_cfg)
tests = {
'int': ('0x123', 16),
'floor': (1.2,),
'ceil': (1.3,),
'round': (1.4,),
'sum': ([1, 2, 3, 2.4],),
'avg': ([1, 2, 3, 2.4],),
'len': ([1, 2, 3, 2.4],),
'random': tuple(),
}
for func_name, args in tests.items():
func = expression_functions.get_plugin(func_name)
# Make sure this doesn't throw errors
func(*args)
# Make sure we have a test for all core plugins.
for plugin in plugins.list_plugins()['function']:
if plugin.priority == expression_functions.FunctionPlugin.PRIO_CORE:
self.assertIn(plugin.name, tests)
plugins._reset_plugins()
def test_module_wrappers(self):
"""Make sure module wrapper loading is sane too."""
# Get an empty pavilion config and set some config dirs on it.
pav_cfg = config.PavilionConfigLoader().load_empty()
# We're loading multiple directories of plugins - AT THE SAME TIME!
pav_cfg.config_dirs = [
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir'),
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir2')
]
plugins.initialize_plugins(pav_cfg)
module_wrapper.get_module_wrapper('foo', '1.0')
bar1 = module_wrapper.get_module_wrapper('bar', '1.3')
bar2 = module_wrapper.get_module_wrapper('bar', '1.2.0')
self.assertIsNot(bar1, bar2)
self.assertEqual('1.3', bar1.get_version('1.3'))
self.assertEqual('1.2.0', bar2.get_version('1.2.0'))
self.assertRaises(module_wrapper.ModuleWrapperError,
lambda: bar2.get_version('1.3.0'))
bar1.load({})
plugins._reset_plugins()
def test_files_create_errors(self):
"""Ensure runtime file creation expected errors occur."""
plugins.initialize_plugins(self.pav_cfg)
# Ensure a file can't be written outside the build context.
files_to_fail = ['../file', '../../file', 'wild/../../file']
for file in files_to_fail:
file_arg = {file: []}
config = self._quick_test_cfg()
config['build']['source_path'] = 'file_tests.tgz'
config['build']['create_files'] = file_arg
with self.assertRaises(TestRunError) as context:
self._quick_test(config)
self.assertTrue('outside build context' in str(context.exception))
# Ensure a file can't overwrite existing directories.
files_to_fail = ['wild', 'rec']
for file in files_to_fail:
file_arg = {file: []}
config = self._quick_test_cfg()
config['build']['source_path'] = 'file_tests.tgz'
config['build']['create_files'] = file_arg
test = TestRun(self.pav_cfg, config)
self.assertFalse(test.build())
def test_create_file(self):
"""Check that build time file creation is working correctly."""
plugins.initialize_plugins(self.pav_cfg)
files_to_create = {
'file1': ['line_0', 'line_1'],
'wild/file2': ['line_0', 'line_1'], # wild dir exists
'wild/dir2/file3': ['line_0', 'line_1'], # dir2 does not exist
'real.txt': ['line1', 'line4'] # file exists
}
config = self._quick_test_cfg()
config['build']['source_location'] = 'file_tests.tgz'
config['build']['create_files'] = files_to_create
test = self._quick_test(config)
for file, lines in files_to_create.items():
file_path = test.path / 'build' / file
self.assertTrue(file_path.exists())
# Stage file contents for comparison.
original = io.StringIO()
for line in lines:
original.write("{}\n".format(line))
created_file = open(str(file_path), 'r', encoding='utf-8')
# Compare contents.
self.assertEquals(original.getvalue(), created_file.read())
original.close()
created_file.close()
def test_loading_tests(self):
"""Make sure get_tests can find tests and resolve inheritance."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world'])
self.assertEqual(sorted(['narf', 'hello', 'world']),
sorted([test['name'] for test in tests]))
tests = load_test_configs(self.pav_cfg, 'this', [],
['hello_world.hello'])
hello = tests.pop()
# There should have only been 1
self.assertFalse(tests)
# Check some basic test attributes.
self.assertEqual(hello['scheduler'], 'raw')
self.assertEqual(hello['suite'], 'hello_world')
# Make sure the variable from the host config got propagated.
self.assertIn('hosty', hello['variables'])
tests = load_test_configs(self.pav_cfg, 'this', [],
['hello_world.narf'])
narf = tests.pop()
# Make sure this got overridden from 'world'
self.assertEqual(narf['scheduler'], 'dummy')
# Make sure this didn't get lost.
self.assertEqual(narf['run']['cmds'], ['echo "Running World"'])
plugins._reset_plugins()
def test_plugin_loading(self):
"""Check to make sure the plugin system initializes correctly. Separate
tests will check the internal initialization of each plugin
sub-system."""
# Get an empty pavilion config and set some config dirs on it.
pav_cfg = config.PavilionConfigLoader().load_empty()
# We're loading multiple directories of plugins - AT THE SAME TIME!
pav_cfg.config_dirs = [
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir'),
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir2')
]
for path in pav_cfg.config_dirs:
self.assertTrue(os.path.exists(path))
plugins.initialize_plugins(pav_cfg)
created_manager = plugins._PLUGIN_MANAGER
# Make sure this can run multiple times,
plugins.initialize_plugins(pav_cfg)
# Make sure only one of these is ever created.
self.assertIs(created_manager, plugins._PLUGIN_MANAGER)
# Clean up our plugin initializations.
plugins._reset_plugins()
def test_result_parser_plugins(self):
"""Check basic result parser structure."""
plugins.initialize_plugins(self.pav_cfg)
regex = result_parsers.get_plugin('regex')
plugins._reset_plugins()
def test_create_file(self):
"""Ensure runtime file creation is working correctly."""
plugins.initialize_plugins(self.pav_cfg)
files_to_create = {
'runtime_0': ['line_0', 'line_1'],
'wild/runtime_1': ['line_0', 'line_1'], # dir exists
'wild/dir2/runtime_2': ['line_0', 'line_1'], # dir2 does not exist
'real.txt': ['line_0', 'line_1'], # file exists; overwrite
'runtime_variable': [
'{{var1}}', '{{var2.0}}', '{{var2.1}}', '{{var2.2}}',
'{{var3.subvar_1}}', '{{var3.subvar_2}}',
'{{var4.0.subvar_3}}', '{{var4.0.subvar_4}}',
'{{var4.1.subvar_3}}', '{{var4.1.subvar_4}}'
]
}
variables = {
'var1':
'val_1',
'var2': ['val_2', 'val_3', 'val_4'],
'var3': {
'subvar_1': 'val_5',
'subvar_2': 'val_6'
},
'var4': [{
'subvar_3': 'val_7',
'subvar_4': 'val_8'
}, {
'subvar_3': 'val_9',
'subvar_4': 'val_10'
}]
}
config = self._quick_test_cfg()
config['variables'] = variables
config['build']['source_path'] = 'file_tests.tgz'
config['run']['create_files'] = files_to_create
test = self._quick_test(config)
for file, lines in files_to_create.items():
file_path = test.path / 'build' / file
self.assertTrue(file_path.exists())
# Stage file contents for comparison.
original = io.StringIO()
created_file = open(str(file_path), 'r', encoding='utf-8')
if file == 'runtime_variable':
original.write('val_1\nval_2\nval_3\nval_4\nval_5\nval_6'
'\nval_7\nval_8\nval_9\nval_10\n')
else:
for line in lines:
original.write("{}\n".format(line))
self.assertEquals(original.getvalue(), created_file.read())
original.close()
created_file.close()
def test_regex_expected_sanity(self):
"""Sanity check for the expected parser."""
plugins.initialize_plugins(self.pav_cfg)
test_cfg = {
'scheduler': 'raw',
'run': {
# This will result in 4 output files.
# run.log, other.log, other2.log, other3.log
'cmds': [
"echo I am a test's output.",
"echo Short and stout.",
"echo My min speed is 438.5",
"echo and my max is 968.3",
"echo What do you think of that, punk?",
"echo Also, here's another number to confuse you. 3.7. Take that."
]
},
'results': {
'regex': [
{
# A test using the 'result' key is required.
'key': 'result',
'regex': r'max is',
},
{
'key': 'key',
'regex': r'max is (.*)$'
},
{
# A test using the 'result' key is required.
'key': 'key1',
'regex': r'max is (.*)$',
'expected': ['900-1000'],
'action': result_parsers.ACTION_TRUE,
},
]
}
}
test = self._quick_test(test_cfg, 'result_parser_test')
test.build()
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
self.assertEqual(results['key'], '968.3')
self.assertTrue(results['result'])
def test_show_cmds(self):
plugins.initialize_plugins(self.pav_cfg)
arg_lists = [
('show', 'config'),
('show', 'config', '--template'),
('show', 'functions'),
('show', 'functions', '--detail', 'int'),
('show', 'hosts'),
('show', 'hosts', '--verbose'),
('show', 'hosts', '--vars', 'this'),
('show', 'hosts', '--config', 'this'),
('show', 'modes'),
('show', 'modes', '--verbose'),
('show', 'modes', '--vars', 'defaulted'),
('show', 'modes', '--config', 'defaulted'),
('show', 'module_wrappers'),
('show', 'module_wrappers', '--verbose'),
('show', 'pav_vars'),
('show', 'result_parsers'),
('show', 'result_parsers', '--doc=regex'),
('show', 'result_parsers', '--verbose'),
('show', 'sched'),
('show', 'sched', '--config=slurm'),
('show', 'sched', '--vars=slurm'),
('show', 'states'),
('show', 'suites'),
('show', 'suites', '--err'),
('show', 'suites', '--supersedes'),
('show', 'suites', '--verbose'),
('show', 'system_variables'),
('show', 'system_variables', '--verbose'),
('show', 'test_config'),
('show', 'tests'),
('show', 'tests', '--err'),
('show', 'tests', '--doc', 'hello_world.narf'),
('show', 'tests', '--hidden'),
('show', 'tests', '--verbose'),
]
parser = arguments.get_parser()
show_cmd = commands.get_command('show')
show_cmd.silence()
for arg_list in arg_lists:
args = parser.parse_args(arg_list)
show_cmd.run(self.pav_cfg, args)
plugins._reset_plugins()
def test_loading_hidden(self):
"""Make sure we only get hidden tests when specifically requested."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg, 'this', [], ['hidden'])
names = sorted([t['name'] for t in tests])
self.assertEqual(names, ['hello', 'narf'])
tests = load_test_configs(self.pav_cfg, 'this', [], ['hidden._hidden'])
names = sorted([t['name'] for t in tests])
self.assertEqual(names, ['_hidden'])
plugins._reset_plugins()
def test_copy_build(self):
"""Check that builds are copied correctly."""
plugins.initialize_plugins(self.pav_cfg)
config = self._quick_test_cfg()
# The copy_test source file contains several files to copy
# for real and several to symlink.
config['build']['source_location'] = 'file_tests.tgz'
config['build']['copy_files'] = [
'real.*',
'wild/real_?i*[0-9].dat',
'rec/**/real*',
]
test = self._quick_test(config)
# Make sure the following exist and are regular files.
real_files = [
'real.txt', 'wild/real_wild1.dat', 'wild/real_wild2.dat',
'rec/real_r1.txt', 'rec/rec2/real_r2.txt'
]
for real in real_files:
real = test.path / 'build' / real
self.assertTrue(real.exists(), msg="Missing {}".format(real))
self.assertTrue(real.is_file(),
msg="{} is not a regular file.".format(real))
# Make sure the following exist, but are symlinks.
sym_files = [
'pav_build_log',
'.built_by',
'sym.txt',
'wild/sym.dat',
'rec/sym_r1.txt',
'rec/rec2/sym_r2.txt',
]
for sym in sym_files:
sym = test.path / 'build' / sym
self.assertTrue(sym.exists(), msg="Missing {}".format(sym))
self.assertTrue(sym.is_symlink(),
msg="{} is not a symlink".format(sym))
plugins._reset_plugins()
def test_show_cmds(self):
plugins.initialize_plugins(self.pav_cfg)
arg_lists = [
('show', 'sched'),
('show', 'sched', '--config=slurm'),
('show', 'sched', '--vars=slurm'),
('show', 'result_parsers'),
('show', 'result_parsers', '--verbose'),
('show', 'result_parsers', '--config=regex'),
('show', 'states'),
('show', 'config'),
('show', 'config', '--template'),
('show', 'test_config'),
('show', 'module_wrappers'),
('show', 'module_wrappers', '--verbose'),
('show', 'system_variables'),
('show', 'system_variables', '--verbose'),
('show', 'pav_vars'),
('show', 'suites'),
('show', 'suites', '--verbose'),
('show', 'suites', '--err'),
('show', 'suites', '--supersedes'),
('show', 'tests'),
('show', 'tests', '--verbose'),
('show', 'tests', '--err'),
('show', 'tests', '--hidden'),
('show', 'hosts'),
('show', 'hosts', '--verbose'),
('show', 'modes'),
('show', 'modes', '--verbose'),
]
parser = arguments.get_parser()
show_cmd = commands.get_command('show')
show_cmd.outfile = io.StringIO()
show_cmd.errfile = io.StringIO()
for arg_list in arg_lists:
args = parser.parse_args(arg_list)
show_cmd.run(self.pav_cfg, args)
plugins._reset_plugins()
def test_result_parser_plugins(self):
"""Check basic result parser structure."""
# Get an empty pavilion config and set some config dirs on it.
pav_cfg = config.PavilionConfigLoader().load_empty()
plugins.initialize_plugins(pav_cfg)
# We're loading multiple directories of plugins - AT THE SAME TIME!
pav_cfg.config_dirs = [
os.path.join(self.TEST_DATA_ROOT, 'pav_config_dir')
]
# We should have exactly one test plugin.
self.assertEqual(len(result_parsers.list_plugins()), 1)
regex = result_parsers.get_plugin('regex')
plugins._reset_plugins()
def setUp(self) -> None:
if self.alt_group is None:
self.fail("Your user must be in at least two groups (other than "
"the user's group) to run this test.")
with self.PAV_CONFIG_PATH.open() as pav_cfg_file:
raw_cfg = yaml.load(pav_cfg_file)
self.working_dir = self.PAV_ROOT_DIR/'test'/'working_dir' / \
'wd-spec_perms'
if self.working_dir.exists():
shutil.rmtree(self.working_dir.as_posix())
self.working_dir.mkdir()
raw_cfg['umask'] = self.umask
raw_cfg['working_dir'] = self.working_dir.as_posix()
raw_cfg['config_dirs'] = [self.TEST_DATA_ROOT/'configs-spec_perms']
(self.working_dir/'test_runs').mkdir()
(self.working_dir/'series').mkdir()
(self.working_dir/'builds').mkdir()
(self.working_dir/'users').mkdir()
self.config_dir = self.TEST_DATA_ROOT/'configs-spec_perms'
with (self.config_dir/'pavilion.yaml').open('w') as pav_cfg_file:
yaml.dump(raw_cfg, stream=pav_cfg_file)
tmpl_path = self.config_dir/'tests/perm.yaml.tmpl'
test_path = self.config_dir/'tests/perm.yaml'
with tmpl_path.open() as tmpl, test_path.open('w') as test:
test_yaml = yaml.load(tmpl)
test_yaml['spec_perms1']['group'] = self.alt_group.gr_name
test_yaml['spec_perms2']['group'] = self.alt_group2.gr_name
yaml.dump(test_yaml, test)
self.pav_cfg = config.find(target=self.config_dir/'pavilion.yaml')
plugins.initialize_plugins(self.pav_cfg)
def setUp(self) -> None:
plugins.initialize_plugins(self.pav_cfg)
self.var_man = variables.VariableSetManager()
self.var_man.add_var_set(
'var', {
'int1':
"1",
'int2':
"2",
'float1':
'1.1',
'str1':
'hello',
'ints': ['0', '1', '2', '3', '4', '5'],
'floats': ['0.1', '2.3'],
'more_ints': ['0', '1'],
'struct': {
'cpus': '200',
'flops': '2.1',
'name': 'earth_chicken',
},
'structs': [
{
'type': 'cat',
'bites': '3',
'evil_rating': '5.2'
},
{
'type': 'dog',
'bites': '0',
'evil_rating': '0.2'
},
{
'type': 'fish',
'bites': '1',
'evil_rating': '9.7'
},
]
})
self.var_man.add_var_set('sys', system_variables.get_vars(defer=True))
def test_apply_overrides(self):
"""Make sure overrides get applied to test configs correctly."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg, 'this', [], ['hello_world'])
overrides = {'run': {'env': {'foo': 'bar'}}, 'scheduler': 'fuzzy'}
for test in tests:
otest = test.copy()
apply_overrides(test, overrides)
# Make sure the overrides were applied
self.assertEqual(test['scheduler'], 'fuzzy')
self.assertEqual(test['run']['env']['foo'], 'bar')
# Make sure other stuff wasn't changed.
self.assertEqual(test['build'], otest['build'])
self.assertEqual(test['run']['cmds'], otest['run']['cmds'])
plugins._reset_plugins()
def test_extended_variables(self):
"""Make sure default variables work as expected."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg,
host='extended',
modes=['extended'],
tests=['extended.test'])
test = tests[0]
self.assertEqual(test['variables']['long_base'],
['what', 'are', 'you', 'up', 'to', 'punk?'])
self.assertEqual(test['variables']['single_base'],
['host', 'mode', 'test'])
self.assertEqual(test['variables']['no_base_mode'], ['mode'])
self.assertEqual(test['variables']['no_base'], ['test'])
self.assertEqual(test['variables']['null_base_mode'], ['mode'])
self.assertEqual(test['variables']['null_base'], ['test'])
plugins._reset_plugins()
def test_env_order(self):
"""Make sure environment variables keep their order from the test
config to the final run scripts."""
plugins.initialize_plugins(self.pav_cfg)
test_conf = load_test_configs(self.pav_cfg, 'this', [], ['order'])[0]
test = self._quick_test(test_conf, "order")
# Each exported variable in this config has a value that denotes its
# expected place in the order. The variable names are random letter
# sequences in a random order; hash order should be decidedly different
# than the listed order.
exports = []
with (test.path / 'build.sh').open() as build_script:
for line in build_script:
if line.startswith('export'):
if 'TEST_ID' in line or 'PAV_CONFIG_FILE' in line:
continue
_, var_val = line.split()
var, val = line.split('=')
try:
val = int(val)
except ValueError:
raise
exports.append((var, val))
# The values should be the numbers for 0..99, in that order.
self.assertEqual(list(range(len(exports))),
[val for var, val in exports],
msg="Environment variable order not preserved. \n"
"Got the following instead: \n{}".format(''.join(
["{}: {}\n".format(*v) for v in exports])))
plugins._reset_plugins()
def test_defaulted_variables(self):
"""Make sure default variables work as expected."""
plugins.initialize_plugins(self.pav_cfg)
tests = load_test_configs(self.pav_cfg,
host='defaulted',
modes=['defaulted'],
tests=['defaulted.test'])
test = tests[0]
self.assertEqual(test['variables']['host_def'], ['host'])
self.assertEqual(test['variables']['mode_def'], ['mode'])
self.assertEqual(test['variables']['test_def'], ['test'])
self.assertNotIn('no_val', test['variables'])
with self.assertRaises(TestConfigError):
tests = load_test_configs(self.pav_cfg,
host='defaulted',
modes=['defaulted'],
tests=['defaulted_error.error'])
plugins._reset_plugins()
def test_layering(self):
"""Make sure test config layering works as expected."""
plugins.initialize_plugins(self.pav_cfg)
for host in ('this', 'layer_host'):
for modes in ([], ['layer_mode']):
for test in ('layer_tests.layer_test',
'layer_tests.layer_test_part'):
answer = 'standard'
if host == 'layer_host':
answer = 'host'
if modes:
answer = 'mode'
if test.endswith('part'):
answer = 'test'
tests = load_test_configs(pav_cfg=self.pav_cfg,
host=host,
modes=modes,
tests=[test])
self.assertEqual(tests[0]['slurm']['partition'], answer)
plugins._reset_plugins()
def test_finalize(self):
plugins.initialize_plugins(self.pav_cfg)
cfg = self._quick_test_cfg()
cfg['run']['cmds'] = ['echo "{{sys.host_name}}"']
cfg['results'] = {
'regex': [{
'key': 'foo',
'regex': '{{sys.host_name}}',
}]
}
test = self._quick_test(cfg,
'finalize_test',
build=False,
finalize=False)
test.build()
undefered_sys_vars = system_variables.SysVarDict(
defer=False,
unique=True,
)
fin_var_man = variables.VariableSetManager()
fin_var_man.add_var_set('sys', undefered_sys_vars)
test.finalize(fin_var_man)
results = test.gather_results(test.run())
test.save_results(results)
plugins._reset_plugins()
def test_parse_results(self):
"""Check all the different ways in which we handle parsed results."""
plugins.initialize_plugins(self.pav_cfg)
test_cfg = {
'scheduler': 'raw',
'run': {
# This will result in 4 output files.
# run.log, other.log, other2.log, other3.log
'cmds': [
'echo "Hello World."',
'echo "Goodbye Cruel World."',
'echo "In a World where..." >> other.log',
'echo "something happens..." >> other2.log',
'echo "and someone saves the World." >> other3.log',
'echo "I\'m here to cause Worldwide issues." >> other.txt'
]
},
'results': {
'regex': [
{
# Look at the default output file. (run.log)
'key': 'basic',
'regex': r'.* World',
},
{
# Look all the log files, and save 'True' on match.
'key': 'true',
'files': ['../run.log'],
'regex': r'.* World',
'action': result_parsers.ACTION_TRUE,
},
{
# As before, but false. Also, with lists of data.
'key': 'false',
# By multiple globs.
'files': ['../run.log', 'other.*'],
'regex': r'.* World',
'match_type': result_parsers.MATCH_ALL,
'action': result_parsers.ACTION_FALSE,
},
{
# As before, but keep match counts.
'key': 'count',
'files': ['../run.log', '*.log'],
'regex': r'.* World',
'match_type': result_parsers.MATCH_ALL,
'action': result_parsers.ACTION_COUNT,
'per_file': result_parsers.PER_FULLNAME,
},
{
# Store matches by fullname
'key': 'fullname',
'files': ['../run.log', '*.log'],
'regex': r'.* World',
'per_file': result_parsers.PER_FULLNAME,
},
{
# Store matches by name stub
# Note there is a name conflict here between other.txt
# and other.log.
'key': 'name',
'files': ['other.*'],
'regex': r'.* World',
'per_file': result_parsers.PER_NAME,
},
{
# Store matches by name stub
# Note there is a name conflict here between other.txt
# and other.log.
'key': 'name_list',
'files': ['*.log'],
'regex': r'World',
'per_file': result_parsers.PER_NAME_LIST,
},
{
# Store matches by name stub
# Note there is a name conflict here between other.txt
# and other.log.
'key': 'fullname_list',
'files': ['*.log'],
'regex': r'World',
'per_file': result_parsers.PER_FULLNAME_LIST,
},
{
'key': 'lists',
'files': ['other*'],
'regex': r'.* World',
'match_type': result_parsers.MATCH_ALL,
'per_file': result_parsers.PER_LIST,
},
{
'key': 'all',
'files': ['other*'],
'regex': r'.* World',
'action': result_parsers.ACTION_TRUE,
'per_file': result_parsers.PER_ALL
},
{
'key': 'result',
'files': ['other*'],
'regex': r'.* World',
'action': result_parsers.ACTION_TRUE,
'per_file': result_parsers.PER_ANY
},
]
}
}
test = self._quick_test(test_cfg, 'result_parser_test')
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
# Check all the different results to make sure they're what we expect.
self.assertEqual(
results['basic'],
'Hello World')
self.assertEqual(
results['true'],
True,
)
self.assertEqual(
results['false'],
False,
)
self.assertEqual(results['fn']['run.log']['count'], 2)
self.assertEqual(results['fn']['other.log']['count'], 1)
self.assertEqual(results['fn']['other.log']['fullname'],
'In a World')
self.assertEqual(results['name_list'],
['other', 'other3'])
self.assertEqual(results['fullname_list'],
['other.log', 'other3.log'])
self.assertIn(results['n']['other']['name'],
['In a World', "I'm here to cause World"])
self.assertIn("Duplicate file key 'other' matched by name",
[e['msg'] for e in results['pav_result_errors']])
self.assertEqual(sorted(results['lists']),
sorted(['and someone saves the World',
'In a World',
"I'm here to cause World"]))
self.assertEqual(results['all'], False)
self.assertEqual(results['result'], result_parsers.PASS)
plugins._reset_plugins()
def test_table_result_parser(self):
"""
Makes sure Table Result Parser Works
:return:
"""
plugins.initialize_plugins(self.pav_cfg)
# line+space delimiter
table_test1 = {
'scheduler': 'raw',
'run': {
'cmds': [
'echo "SAMPLE TABLE"',
'echo "Col1 | Col2 | Col3"',
'echo "------------------"',
'echo "data1 | 3 | data2"',
'echo "data3 | 8 | data4"',
'echo "data5 | | data6"',
'echo "some other text that doesnt matter"'
]
},
'results': {
'table': [
{
'key': 'table1',
'delimiter': r'\\|',
'col_num': '3'
}
],
'constant': [
{
'key': 'result',
'const': 'table1'
}
]
}
}
test = self._quick_test(table_test1, 'result_parser_test')
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
self.assertEqual(['data1', 'data3', 'data5'], results['table1']['Col1'])
self.assertEqual(['3', '8', ' '], results['table1']['Col2'])
self.assertEqual(['data2', 'data4', 'data6'], results['table1']['Col3'])
# space delimiter
table_test2 = {
'scheduler': 'raw',
'run': {
'cmds': [
'echo "SAMPLE TABLE"',
'echo "d1 d2 d3"',
'echo "d4 d5 d6"',
'echo "d7 d9"',
'echo "some other text that doesnt matter"'
]
},
'results': {
'table': [
{
'key': 'table2',
'delimiter': ' ',
'col_num': '3'
}
],
'constant': [
{
'key': 'result',
'const': 'table2'
}
]
}
}
test = self._quick_test(table_test2, 'result_parser_test')
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
self.assertEqual(['d4', 'd7'], results['table2']['d1'])
self.assertEqual(['d5', ' '], results['table2']['d2'])
self.assertEqual(['d6', 'd9'], results['table2']['d3'])
# comma delimiter
table_test3 = {
'scheduler': 'raw',
'run': {
'cmds': [
'echo "----------- Comma-delimited summary ---------"',
'echo "./clomp_hwloc 4 -1 256 10 32 1 100, calc_deposit, OMP Barrier, Scaled Serial Ref, Bestcase OMP, Static OMP, Dynamic OMP, Manual OMP"',
'echo "Runtime, 0.000, 0.919, 2.641, 0.517, 2.345, 16.392, 2.324"',
'echo "us/Loop, 0.00, 9.41, 27.04, 5.29, 24.01, 167.85, 23.79"',
'echo "Speedup, N/A, N/A, 1.00, 5.1, 1.1, 0.2, 1.1"',
'echo "Efficacy, N/A, N/A, N/A, 100%, 22.0%, 3.2%, 22.2%"',
'echo "Overhead, N/A, N/A, N/A, 0.00, 18.72, 162.56, 18.50"',
'echo "CORAL2 RFP, 4 -1 256 10 32 1 100, 1.00, 27.04, 27.04, 9.41, 5.1, 18.72, 1.1, 162.56, 0.2, 18.50, 1.1"'
]
},
'results': {
'table': [
{
'key': 'table3',
'delimiter': ',',
'col_num': '8',
'has_header': 'True',
'by_column': 'True',
'col_names': [' ', 'calc_deposit', 'OMP Barrier',
'Scaled Serial Ref', 'Bestcase OMP',
'Static OMP', 'Dynamic OMP', 'Manual OMP']
}
],
'constant': [
{
'key': 'result',
'const': 'table3'
}
]
}
}
test = self._quick_test(table_test3, 'result_parser_test')
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
self.assertEqual('0.000', results['table3']['calc_deposit']['Runtime'])
self.assertEqual('9.41', results['table3']['OMP Barrier']['us/Loop'])
self.assertEqual('1.00',
results['table3']['Scaled Serial Ref']['Speedup'])
self.assertEqual('100%', results['table3']['Bestcase OMP']['Efficacy'])
self.assertEqual('18.72', results['table3']['Static OMP']['Overhead'])
self.assertEqual('16.392', results['table3']['Dynamic OMP']['Runtime'])
self.assertEqual('23.79', results['table3']['Manual OMP']['us/Loop'])
def test_regex_threshold(self):
"""Ensure the match_count parser works appropriately."""
plugins.initialize_plugins(self.pav_cfg)
test_cfg = {
'scheduler': 'raw',
'run': {
# This will result in 4 output files.
# run.log, other.log, other2.log, other3.log
'cmds': [
'echo "Test Name: FakeTest\n"',
'echo "User: Some-gal-or-guy\n"',
'echo "result1=19\n"',
'echo "result3=test\n"',
'echo "result9=-12\n"',
'echo "result12=9.9\n"',
'echo "result13=-22.2\n"',
'echo "result98=\n"',
'echo "result50=18.2,result51=18.3\n"',
'echo "overall=whatevs"'
]
},
'results': {
'regex': [
{
# Look at the default output file. (run.log)
# Test for finding greater than the threshold present
'key': 'key0',
'regex': r'result',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '7',
},
{
# Test for finding equal to the threshold present
'key': 'key1',
'regex': r'result',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '8',
},
{
# Test for finding fewer than the threshold present
'key': 'key2',
'regex': r'result',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '9',
},
{
# Test for finding equal to of a more specific search
'key': 'key3',
'regex': r'result1',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '3',
},
{
# Test for finding fewer than of a more specific search
'key': 'key4',
'regex': r'result1',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '4',
},
{
# Test for a threshold of zero
'key': 'key5',
'regex': r'overall=whatevs',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '0',
},
{
# Test for a more complex search
'key': 'key6',
'regex': r'overall=whatevs',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '1',
},
{
# Test for a more complex search that fails
'key': 'key7',
'regex': r'overall=whatevs',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '2',
},
{
# Test for a more complex search that fails
'key': 'key8',
'regex': r'totallynotthere',
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
'threshold': '0',
},
{
# A test using the 'result' key is required.
'key': 'result',
'regex': r'^overall=(.*)$',
'action': result_parsers.ACTION_TRUE,
}
]
}
}
test = self._quick_test(test_cfg, 'result_parser_test')
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
self.assertTrue(results['key0'])
self.assertTrue(results['key1'])
self.assertFalse(results['key2'])
self.assertTrue(results['key3'])
self.assertFalse(results['key4'])
self.assertTrue(results['key5'])
self.assertTrue(results['key6'])
self.assertFalse(results['key7'])
self.assertFalse(results['key8'])
self.assertTrue(results['result'])
def test_regex_expected(self):
"""Ensure the regex-value parser works appropriately."""
plugins.initialize_plugins(self.pav_cfg)
test_cfg = {
'scheduler': 'raw',
'run': {
# This will result in 4 output files.
# run.log, other.log, other2.log, other3.log
'cmds': [
'echo "Test Name: FakeTest\n"',
'echo "User: Some-gal-or-guy\n"',
'echo "result1=19\n"',
'echo "result3=test\n"',
'echo "result9=-12\n"',
'echo "result12=9.9\n"',
'echo "result13=-22.2\n"',
'echo "result98=\n"',
'echo "result50=18.2,result51=18.3\n"',
'echo "overall=whatevs"'
]
},
'results': {
'regex': [
{
# Look at the default output file. (run.log)
# Test for storing the whole line
'key': 'key0',
'regex': r'^User:.*$',
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for storing a single value
'key': 'key1',
'regex': r'^result1=(.*)$',
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range of negative integers
'key': 'key2',
'regex': r'^result9=(.*)$',
'expected': ['-13:-9'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range of floats where the value
# is equal to the bottom of the range
'key': 'key3',
'regex': r'^result12=(.*)$',
'expected': ['9.0:9.9'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range of floats that has zero
# span
'key': 'key4',
'regex': r'^result12=(.*)$',
'expected': ['9.9:9.9'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range of floats where the value
# is equal to the top of the range
'key': 'key5',
'regex': r'^result12=(.*)$',
'expected': ['9.9:10.0'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range of floats from negative to
# positive
'key': 'key6',
'regex': r'^result12=(.*)$',
'expected': ['-9.9:10.0'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range of negative integers
'key': 'key7',
'regex': r'^result13=(.*)$',
'expected': ['-32:-22'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range from a negative float to a
# positive integer
'key': 'key8',
'regex': r'^result13=(.*)$',
'expected': ['-32.0:22'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for expecting a range from a very large negative
# float to zero
'key': 'key9',
'regex': r'^result13=(.*)$',
'expected': ['-10000000000.0:0'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for checking a set of results that are NOT in a
# list of integer values
'key': 'key10',
'regex': r'^result.*=(.*)$',
'expected': ['100', '101'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for a list of results in a range of floats
'key': 'key11',
'regex': r'result5.=([0-9.]*)',
'expected': ['18.0:18.5'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for a list of results where one value is inside
# the expected range and the other is not
'key': 'key12',
'regex': r'^result50=(.*),result51=(.*)$',
'expected': ['18.0:18.2'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for a list of results in a one-side bounded
# range of floats
'key': 'key13',
'regex': r'result5.=([0-9.]*)',
'expected': [':18.5'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for a list of results in a one-side bounded
# range of floats
'key': 'key14',
'regex': r'result5.=([0-9.]*)',
'expected': ['18.0:'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for a list of results where one value is inside
# the expected range and the other is not
'key': 'key15',
'regex': r'^result50=(.*),result51=(.*)$',
'expected': [':18.2'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# Test for a list of results where one value is inside
# the expected range and the other is not
'key': 'key16',
'regex': r'^result50=(.*),result51=(.*)$',
'expected': ['18.0:'],
'action': result_parsers.ACTION_TRUE,
'match_type': result_parsers.MATCH_ALL,
},
{
# A test using the 'result' key is required.
'key': 'result',
'regex': r'^overall=(.*)$',
'action': result_parsers.ACTION_TRUE,
}
]
}
}
test = self._quick_test(test_cfg, 'result_parser_test')
test.run()
results = result_parsers.parse_results(
test=test,
results={}
)
self.assertEqual(results['key0'], ['User: Some-gal-or-guy'])
self.assertEqual(results['key1'], ['19'])
self.assertTrue(results['key2'])
self.assertTrue(results['key3'])
self.assertTrue(results['key4'])
self.assertTrue(results['key5'])
self.assertTrue(results['key6'])
self.assertTrue(results['key7'])
self.assertTrue(results['key8'])
self.assertTrue(results['key9'])
self.assertFalse(results['key10'])
self.assertTrue(results['key11'])
self.assertFalse(results['key12'])
def setUp(self):
plugins.initialize_plugins(self.pav_cfg)
def test_check_args(self):
plugins.initialize_plugins(self.pav_cfg)
# Make sure we can check arguments.
test_cfg = {
'results': {
'regex': [
{'key': 'ok', 'regex': r'foo'},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
result_parsers.check_args(test.config['results'])
# Make sure duplicate keys aren't allowed.
test_cfg = {
'results': {
'regex': [
{'key': 'repeated', 'regex': r'foo'},
{'key': 'repeated', 'regex': r'foo'},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
# Make sure we handle bad key names.
test_cfg = {
'results': {
'regex': [
{'key': '#!@123948aa', 'regex': r'foo'},
]
}
}
with self.assertRaises(ValueError):
self._quick_test(test_cfg, 'check_args_test')
# Make sure we handle missing the 'key' attribute as expected.
test_cfg = {
'results': {
'regex': [
{'regex': r'foo'},
]
}
}
with self.assertRaises(ValueError):
self._quick_test(test_cfg, 'check_args_test')
# Make sure reserved keys aren't allowed.
test_cfg = {
'results': {
'regex': [
{'key': 'started', 'regex': r'foo'},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
# Missing a key for the parser plugin
test_cfg = {
'results': {
'regex': [
{'key': 'nope'},
]
}
}
with self.assertRaises(yc.RequiredError):
self._quick_test(test_cfg, 'check_args_test')
test_cfg = {
'results': {
'regex': [
{'key': 'test', 'regex': '[[['},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': '^User:(.*)$',
'expected': ['12:11']
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': '^User:(.*)$',
'expected': ['-11:-12']
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': '^User:(.*)$',
'expected': ['11:12:13']
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': '^User:(.*)$',
'expected': ['11:words']
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': 'res',
'threshold': '-5',
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': 'res',
'threshold': 'A',
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
test_cfg = {
'results': {
'regex': [
{
'key': 'test',
'regex': 'res',
'threshold': 'A',
'expected': '10:12',
},
]
}
}
test = self._quick_test(test_cfg, 'check_args_test')
with self.assertRaises(result_parsers.ResultParserError):
result_parsers.check_args(test.config['results'])
plugins._reset_plugins()
