def get_conf(self):
"""Set up the Slurm configuration attributes."""
return yc.KeyedElem(
self.name,
help_text="Configuration for the Slurm scheduler.",
elements=[
yc.StrElem(
'num_nodes',
default="1",
help_text="Number of nodes requested for this test. "
"This can be a range (e.g. 12-24)."),
yc.StrElem('tasks_per_node',
default="1",
help_text="Number of tasks to run per node."),
yc.StrElem(
'mem_per_node',
help_text="The minimum amount of memory required in GB. "
"This can be a range (e.g. 64-128)."),
yc.StrElem(
'partition',
default="standard",
help_text="The partition that the test should be run "
"on."),
yc.StrElem(
'immediate',
choices=['true', 'false', 'True', 'False'],
default='false',
help_text="Only consider nodes not currently running jobs"
"when determining job size"),
yc.StrElem('qos',
help_text="The QOS that this test should use."),
yc.StrElem('account',
help_text="The account that this test should run "
"under."),
yc.StrElem('reservation',
help_text="The reservation that this test should "
"run under."),
yc.RegexElem(
'time_limit',
regex=r'^(\d+-)?(\d+:)?\d+(:\d+)?$',
help_text="The time limit to specify for the slurm job in"
"the formats accepted by slurm "
"(<hours>:<minutes> is typical)"),
yc.ListElem(name='avail_states',
sub_elem=yc.StrElem(),
defaults=['IDLE', 'MAINT'],
help_text="When looking for immediately available "
"nodes, they must be in one of these "
"states."),
yc.ListElem(
name='up_states',
sub_elem=yc.StrElem(),
defaults=['ALLOCATED', 'COMPLETING', 'IDLE', 'MAINT'],
help_text="When looking for nodes that could be "
"allocated, they must be in one of these "
"states."),
])
def get_conf(self):
return yc.KeyedElem(
self.name,
help_text="Configuration for the Slurm scheduler.",
elements=[
yc.StrElem(
'num_nodes',
default="1",
help_text="Number of nodes requested for this test. "
"This can be a range (e.g. 12-24)."),
yc.StrElem('tasks_per_node',
default="1",
help_text="Number of tasks to run per node."),
yc.StrElem(
'mem_per_node',
help_text="The minimum amount of memory required in GB. "
"This can be a range (e.g. 64-128)."),
yc.StrElem(
'partition',
default="standard",
help_text="The partition that the test should be run "
"on."),
yc.StrElem('qos',
help_text="The QOS that this test should use."),
yc.StrElem('account',
help_text="The account that this test should run "
"under."),
yc.StrElem('reservation',
help_text="The reservation that this test should "
"run under."),
yc.StrElem('time_limit',
help_text="The time limit to specify for the slurm "
"job. This can be a range (e.g. "
"00:02:00-01:00:00)."),
yc.StrElem(
name='immediate',
choices=['true', 'false'],
default='false',
help_text="If set to immediate, this test will fail "
"to kick off if the expected resources "
"aren't immediately available."),
yc.ListElem(name='avail_states',
sub_elem=yc.StrElem(),
defaults=['IDLE', 'MAINT'],
help_text="When looking for immediately available "
"nodes, they must be in one of these "
"states."),
yc.ListElem(
name='up_states',
sub_elem=yc.StrElem(),
defaults=['ALLOCATED', 'COMPLETING', 'IDLE', 'MAINT'],
help_text="When looking for nodes that could be "
"allocated, they must be in one of these "
"states."),
])
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem(
'regex',
required=True,
help_text="The python regex to use to search the given file. "
"See: 'https://docs.python.org/3/library/re.html' "
"You can use single quotes in YAML to have the "
"string interpreted literally. IE '\\n' is a '\\' "
"and an 'n'."),
# Use the built-in matches element.
result_parsers.MATCHES_ELEM,
yc.StrElem(
'threshold',
default='0',
help_text="If a threshold is defined, 'True' will be returned "
"if greater than or equal to that many instances "
"of the specified word are found. If fewer "
"instances are found, 'False' is returned. The "
"value must be an integer greater than zero."),
yc.ListElem(
'expected',
sub_elem=yc.StrElem(),
help_text="Expected value(s) and/or range(s). If provided, "
"the result will be 'True' if all of the found "
"values (determined by the 'results' value) are "
"within the expected range(s) or value(s). "
"Otherwise, the result is 'False'. Supports "
"integers and floats.")
])
return config_items
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem(
'delimiter', default=' ',
help_text="Delimiter that splits the data."
),
yc.StrElem(
'col_num', required=True,
help_text="Number of columns in table, including row names, "
"if there is such a column."
),
yc.StrElem(
'has_header', default='False', choices=['True', 'False'],
help_text="Set True if there is a column for row names. Will "
"create dictionary of dictionaries."
),
yc.ListElem(
'col_names', required=False, sub_elem=yc.StrElem(),
help_text="Column names if the user knows what they are."
),
yc.StrElem(
'by_column', choices=['True', 'False'], default='True',
help_text="Set to True if the user wants to organize the "
"nested dictionaries by columns. Default False. "
"Only set if `has_header` is True. "
"Otherwise, Pavilion will ignore."
)
])
return config_items
class TestConfig(yc.YamlConfigLoader):
ELEMENTS = [
yc.StrElem("pet",
default="squirrel",
required=True,
choices=["squirrel", "cat", "dog"],
help_text="The kind of pet."),
yc.IntElem("quantity", required=True, choices=[1, 2, 3]),
yc.FloatRangeElem("quality", vmin=0, vmax=1.0),
yc.ListElem("potential_names",
help_text="What you could name this pet.",
sub_elem=yc.StrElem(help_text="Such as Fido.")),
yc.KeyedElem(
"properties",
help_text="Pet properties",
elements=[
yc.StrElem("description",
help_text="General pet description."),
yc.RegexElem("greeting",
regex=r'hello \w+$',
help_text="A regex of some sort."),
yc.IntRangeElem("legs", vmin=0)
]),
yc.CodeElem("behavior_code",
help_text="Program for pet behavior.")
]
class Config2(yc.YamlConfigLoader):
ELEMENTS = [
yc.ListElem('cars',
sub_elem=yc.KeyedElem(elements=[
yc.StrElem('color'),
yc.StrElem('make'),
yc.ListElem('extras', sub_elem=yc.StrElem())
]))
]
def __init__(self, default_color):
super(Config2, self).__init__()
# The Config init is a good place to do this.
# Set all the default color for all cars in the 'cars' list to red.
self.set_default('cars.*.color', default_color)
self.set_default('cars.*.extras',
['rhoomba', 'heated sunshades'])
def get_conf(self):
"""Add necessary MPI attributes to those of Slurm."""
elems = self._get_config_elems()
elems.extend([
yc.StrElem('rank_by',
help_text="Value for `--rank-by`. Default is slot."),
yc.StrElem('bind_to',
help_text="Value for `--bind-to`. Default is core."),
yc.ListElem('mca',
help_text="Key-Value for pair(s) for --mca.",
sub_elem=yc.RegexElem(
'kv_pair', regex=r'^[a-z1-9]+\s[a-z1-9,]+$'))
])
return yc.KeyedElem(
self.name,
help_text="Configuration for the Slurm scheduler using mpirun.",
elements=elems)
def add_result_parser_config(cls, name, config_items):
"""Add the given list of config items as a result parser
configuration named 'name'. Throws errors for invalid configuraitons.
"""
# Validate the config.
required_keys = {
'key': False,
'files': False,
'action': False,
'per_file': False,
}
for item in config_items:
for req_key in required_keys.keys():
if item.name == req_key:
required_keys[req_key] = True
for req_key, found in required_keys.items():
if not found:
raise TestConfigError(
"Result parser '{}' must have a required config "
"element named '{}'".format(name, req_key))
config = yc.KeyedElem(
'result_parser_{}'.format(name),
elements=config_items
)
list_elem = yc.ListElem(name, sub_elem=config)
if name in [e.name for e in cls._RESULT_PARSERS.config_elems.values()]:
raise ValueError("Tried to add result parser with name '{}'"
"to the config, but one already exists."
.format(name))
try:
cls.check_leaves(config)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based."
.format(name, err.args[0]))
cls._RESULT_PARSERS.config_elems[name] = list_elem
def __init__(self):
super().__init__(
name='table',
description="Parses tables",
config_elems=[
yc.StrElem(
'delimiter',
help_text="Delimiter that splits the data."
),
yc.StrElem(
'col_num', required=True,
help_text="Number of columns in table, including row "
"names, if there is such a column."
),
yc.StrElem(
'has_header',
help_text="Set True if there is a column for row names. "
"Will create dictionary of dictionaries."
),
yc.ListElem(
'col_names', required=False, sub_elem=yc.StrElem(),
help_text="Column names if the user knows what they are."
),
yc.StrElem(
'by_column',
help_text="Set to True if the user wants to organize the "
"nested dictionaries by columns. Default False. "
"Only set if `has_header` is True. "
"Otherwise, Pavilion will ignore."
)
],
defaults={
'delimiter': ' ',
'has_header': 'False',
'by_column': 'True',
},
validators={
'has_header': ('True', 'False'),
'by_column': ('True', 'False'),
'col_num': int,
}
)
class SpackEnvConfig(yc.YamlConfigLoader):
ELEMENTS = [
yc.KeyedElem('spack',
elements=[
yc.KeyedElem('config',
elements=[
yc.StrElem('install_tree'),
yc.StrElem('build_jobs', default=6),
yc.StrElem('install_path_scheme')
]),
yc.CategoryElem('mirrors', sub_elem=yc.StrElem()),
yc.ListElem('repos', sub_elem=yc.StrElem()),
yc.CategoryElem(
'upstreams',
sub_elem=yc.KeyedElem(
elements=[yc.StrElem('install_tree')])),
],
help_text='Spack environment configuration file.')
]
def add_result_parser_config(cls, name, config_items):
"""Add the given list of config items as a result parser
configuration named 'name'. Throws errors for invalid configuraitons.
"""
config = yc.KeyedElem('result_parser_{}'.format(name),
elements=config_items)
list_elem = yc.ListElem(name, sub_elem=config)
if name in [e.name for e in cls._RESULT_PARSERS.config_elems.values()]:
raise ValueError(
"Tried to add result parser with name '{}'"
"to the config, but one already exists.".format(name))
try:
cls.check_leaves(config)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls._RESULT_PARSERS.config_elems[name] = list_elem
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem('delimiter',
default=' ',
help_text="Delimiter that splits the data."),
yc.StrElem(
'col_num',
required=True,
help_text="Number of columns in table, including row names, "
"if there is such a column."),
yc.StrElem(
'has_header',
default='False',
help_text="Set True if there is a column for row names."),
yc.ListElem(
'col_names',
required=False,
sub_elem=yc.StrElem(),
help_text="Column names if the user knows what they are.")
])
return config_items
class SeriesConfigLoader(yc.YamlConfigLoader):
"""This class describes a series file."""
ELEMENTS = [
TestCatElem(
'series', sub_elem=yc.KeyedElem(
elements=[
yc.ListElem('tests', sub_elem=yc.StrElem()),
yc.StrElem('depends_pass',
choices=['True', 'true', 'False', 'false'],
default='False'),
yc.ListElem('depends_on', sub_elem=yc.StrElem()),
yc.ListElem('modes', sub_elem=yc.StrElem()),
CondCategoryElem(
'only_if', sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
key_case=EnvCatElem.KC_MIXED
),
CondCategoryElem(
'not_if', sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
key_case=EnvCatElem.KC_MIXED
),
]
),
),
yc.ListElem(
'modes', sub_elem=yc.StrElem()
),
yc.IntElem(
'simultaneous',
),
yc.StrElem(
'ordered', choices=['True', 'true', 'False', 'false'],
default='False'
),
yc.StrElem(
'restart', choices=['True', 'true', 'False', 'false'],
default='False'
)
]
"""Describes elements in Series Config Loader."""
def get_config_items():
"""Get the config for this result parser. This should be a list of
yaml_config.ConfigElement instances that will be added to the test
config format at plugin activation time. The simplest format is a
list of yaml_config.StrElem objects, but any structure is allowed
as long as the leaf elements are StrElem type.
The config values will be passed as the keyword arguments to the
result parser when it's run and when its arguments are checked. The base
implementation provides several arguments that must be present for every result
parser. See the implementation of this method in result_parser.py for more
info on those arguments and what they do.
Example: ::
config_items = super().get_config_items()
config_items.append(
yaml_config.StrElem('token', default='PASSED',
help="The token to search for in the file."
)
return config_items
"""
return [
yc.StrElem(
"action",
required=True, default="store",
choices=[
ACTION_STORE,
ACTION_TRUE,
ACTION_FALSE,
ACTION_COUNT
],
help_text=(
"What to do with parsed results.\n"
" {STORE} - Just store the result (default).\n"
" {TRUE} - Store True if there was a result.\n"
" {FALSE} - Store True for no result.\n"
" {COUNT} - Count the number of results.\n"
.format(
STORE=ACTION_STORE,
TRUE=ACTION_TRUE,
FALSE=ACTION_FALSE,
COUNT=ACTION_COUNT))
),
# The default for the file is handled by the test object.
yc.ListElem(
"files",
required=True,
sub_elem=yc.StrElem(),
defaults=['../run.log'],
help_text="Path to the file/s that this result parser "
"will examine. Each may be a file glob,"
"such as '*.log'"),
yc.StrElem(
"per_file",
default=PER_FIRST,
required=True,
choices=[
PER_FIRST,
PER_LAST,
PER_FULLNAME,
PER_NAME,
PER_NAME_LIST,
PER_FULLNAME_LIST,
PER_LIST,
PER_ANY,
PER_ALL,
],
help_text=(
"How to save results for multiple file matches.\n"
" {FIRST} - The result from the first file with a "
"non-empty result. (default)\n"
" {LAST} - As '{FIRST}', but last result.\n"
" {FULLNAME} - Store the results on a per file "
"basis under results['fn'][<filename>][<key>]\n"
" {NAME} - As '{FULLNAME}', except use the "
"filename minus extension (foo.bar.log -> foo.bar), and"
"store under the results['n'][<name>][<key>]\n"
" {FULLNAME_LIST} - Save the matching file names, rather\n"
"than the parsed values, in a list.\n"
" {NAME_LIST} - As '{FULLNAME_LIST}' except store the \n"
"filenames minus the extension.\n"
" {LIST} - Merge all each result and result list "
"into a single list.\n"
" {ALL} - Use only with the 'store_true' or "
"'store_false' action. Set true if all files had a "
"true result. Note that 0 is a false result.\n"
" {ANY} - As '{ALL}', but set true if any file had"
"a true result.\n"
.format(
FIRST=PER_FIRST,
LAST=PER_LAST,
FULLNAME=PER_FULLNAME,
NAME=PER_NAME,
NAME_LIST=PER_NAME_LIST,
FULLNAME_LIST=PER_FULLNAME_LIST,
LIST=PER_LIST,
ALL=PER_ALL,
ANY=PER_ANY))
),
]
class TestConfigLoader(yc.YamlConfigLoader):
"""This class describes a test section in a Pavilion config file. It is
expected to be added to by various plugins.
:cvar list(yc.YamlConfig) ELEMENTS: Each YamlConfig instance in this
list defines a key for the test config.
- Each element must result in a string (which is why you see a lot of StrElem
below), or a structure that contains only strings at the lowest layer.
- So lists of dicts of strings are fine, etc.
- yc.RegexElem also produces a string.
- Everything should have a sensible default.
- An empty config should be a valid test.
- For bool values, accept ['true', 'false', 'True', 'False'].
- They should be checked with val.lower() == 'true', etc.
- Every element must have a useful 'help_text'.
"""
ELEMENTS = [
yc.StrElem(
'name',
hidden=True,
default='<unnamed>',
help_text="The base name of the test. Value added automatically."),
yc.StrElem(
'suite',
hidden=True,
default='<no_suite>',
help_text="The name of the suite. Value added automatically."),
yc.StrElem(
'suite_path',
hidden=True,
default='<no_suite>',
help_text="Path to the suite file. Value added automatically."),
yc.StrElem(
'host',
hidden=True,
default='<unknown>',
help_text="Host (typically sys.sys_name) for which this test was "
"created. Value added automatically."),
yc.ListElem(
'modes',
hidden=True,
sub_elem=yc.StrElem(),
help_text="Modes used in the creation of this test. Value is added "
"automatically."),
yc.RegexElem(
'inherits_from',
regex=TEST_NAME_RE_STR,
help_text="Inherit from the given test section, and override "
"parameters those specified in this one. Lists are "
"overridden entirely"),
yc.StrElem('subtitle',
help_text="An extended title for this test. Required for "
"permuted tests."),
yc.StrElem('group',
default=None,
help_text="The group under which to build and run tests. "
"Defaults to the group specified in pavilion.yaml."),
yc.RegexElem(
'umask',
regex=r'[0-7]{3}',
default=None,
help_text="The octal umask to apply to files created during the "
"build and run processes. Defaults to the umask in "
"pavilion.yaml."),
yc.KeyedElem(
'maintainer',
help_text="Information about who maintains this test.",
elements=[
yc.StrElem(
'name',
default='unknown',
help_text="Name or organization of the maintainer."),
yc.StrElem('email',
help_text="Email address of the test maintainer."),
]),
yc.StrElem('summary',
default='',
help_text="Summary of the purpose of this test."),
yc.StrElem('doc',
default='',
help_text="Detailed documentation string for this test."),
yc.ListElem(
'permute_on',
sub_elem=yc.StrElem(),
help_text="List of permuted variables. For every permutation of "
"the values of these variables, a new virtual test will "
"be generated."),
VarCatElem('variables',
sub_elem=yc.ListElem(sub_elem=VariableElem()),
help_text="Variables for this test section. These can be "
"inserted strings anywhere else in the config through "
"the string syntax. They keys 'var', 'per', 'pav', "
"'sys' and 'sched' reserved. Each value may be a "
"single or list of strings key/string pairs."),
yc.RegexElem('scheduler',
regex=r'\w+',
default="raw",
help_text="The scheduler class to use to run this test."),
CondCategoryElem(
'only_if',
sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
key_case=EnvCatElem.KC_MIXED,
help_text="Only run this test if each of the clauses in this "
"section evaluate to true. Each clause consists of "
"a mapping key (that can contain Pavilion variable "
"references, like '{{pav.user}}' or '{{sys.sys_arch}}'"
") and one or more regex values"
"(that much match the whole key). A clause is true "
"if the value of the Pavilion variable matches one or"
" more of the values. "),
CondCategoryElem(
'not_if',
sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
key_case=EnvCatElem.KC_MIXED,
help_text="Will NOT run this test if at least one of the "
"clauses evaluates to true. Each clause consists of "
"a mapping key (that can contain Pavilion variable "
"references, like '{{pav.user}}' or "
"'{{sys.sys_arch}}') and one or more "
"regex values (that much match the whole key)."
"A clause is true if the value of "
"the Pavilion variable matches one or more of the "
" values."),
yc.KeyedElem(
'build',
elements=[
yc.ListElem(
'cmds',
sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to perform '
'the build.'),
yc.ListElem(
'copy_files',
sub_elem=yc.StrElem(),
help_text="When attaching the build to a test run, copy "
"these files instead of creating a symlink."
"They may include path glob wildcards, "
"including the recursive '**'."),
PathCategoryElem(
'create_files',
key_case=PathCategoryElem.KC_MIXED,
sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
help_text="File(s) to create at path relative to the test's"
"test source directory"),
EnvCatElem(
'env',
sub_elem=yc.StrElem(),
key_case=EnvCatElem.KC_MIXED,
help_text="Environment variables to set in the build "
"environment."),
yc.ListElem(
'extra_files',
sub_elem=yc.StrElem(),
help_text='File(s) to copy into the build environment. '
'Relative paths searched for in ~/.pavilion, '
'$PAV_CONFIG. Absolute paths are ok, '
'but not recommended.'),
yc.ListElem(
'modules',
sub_elem=yc.StrElem(),
help_text="Modules to load into the build environment."),
yc.StrElem('on_nodes',
default='False',
choices=['true', 'false', 'True', 'False'],
help_text="Whether to build on or off of the test "
"allocation."),
yc.ListElem(
'preamble',
sub_elem=yc.StrElem(),
help_text="Setup commands for the beginning of the build "
"script. Added to the beginning of the run "
"script. These are generally expected to "
"be host rather than test specific."),
yc.StrElem(
'source_path',
help_text="Path to the test source. It may be a directory, "
"compressed file, compressed or "
"uncompressed archive (zip/tar), and is handled "
"according to the internal (file-magic) type. "
"For relative paths Pavilion looks in the "
"test_src directory "
"within all known config directories. If this"
"is left blank, Pavilion will always assume "
"there is no source to build."),
yc.StrElem(
'source_url',
help_text='Where to find the source on the internet. By '
'default, Pavilion will try to download the '
'source from the given URL if the source file '
'can\'t otherwise be found. You must give a '
'source path so Pavilion knows where to store '
'the file (relative paths will be stored '
'relative to the local test_src directory.'),
yc.StrElem(
'source_download',
choices=['never', 'missing', 'latest'],
default='missing',
help_text="When to attempt to download the test source.\n"
" never - The url is for reference only.\n"
" missing - (default) Download if the source "
"can't be found.\n"
" latest - Always try to fetch the latest "
"source, tracking changes by "
"file size/timestamp/hash."),
yc.StrElem(
'specificity',
default='',
help_text="Use this string, along with variables, to "
"differentiate builds. A common example would be "
"to make per-host specific by using the "
"sys.sys_name variable. Note _deferred_ system "
"variables aren't a good idea hereas configs are "
"compiled on the host that launches the test."),
yc.StrElem(
'timeout',
default='30',
help_text="Time (in seconds) that a build can continue "
"without generating new output before it is "
"cancelled. Can be left empty for no timeout."),
yc.StrElem(
'verbose',
choices=['true', 'True', 'False', 'false'],
default='False',
help_text="Echo commands (including sourced files) in the"
" build log, and print the modules loaded and "
"environment before the cmds run."),
],
help_text="The test build configuration. This will be "
"used to dynamically generate a build script for "
"building the test."),
yc.KeyedElem(
'run',
elements=[
yc.ListElem('cmds',
sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to run '
'the test.'),
PathCategoryElem(
'create_files',
key_case=PathCategoryElem.KC_MIXED,
sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
help_text="File(s) to create at path relative to the test's"
"test source directory"),
EnvCatElem('env',
sub_elem=yc.StrElem(),
key_case=EnvCatElem.KC_MIXED,
help_text="Environment variables to set in the run "
"environment."),
yc.ListElem(
'modules',
sub_elem=yc.StrElem(),
help_text="Modules to load into the run environment."),
yc.ListElem(
'preamble',
sub_elem=yc.StrElem(),
help_text="Setup commands for the beginning of the build "
"script. Added to the beginning of the run "
"script. These are generally expected to "
"be host rather than test specific."),
yc.StrElem('timeout',
default='300',
help_text="Time that a build can continue without "
"generating new output before it is cancelled. "
"Can be left empty for no timeout."),
yc.StrElem(
'verbose',
choices=['true', 'True', 'False', 'false'],
default='False',
help_text="Echo commands (including sourced files) in the "
"build log, and print the modules loaded and "
"environment before the cmds run."),
],
help_text="The test run configuration. This will be used "
"to dynamically generate a run script for the "
"test."),
yc.CategoryElem(
'result_evaluate',
sub_elem=yc.StrElem(),
help_text="The keys and values in this section will also "
"be added to the result json. The values are "
"expressions (like in {{<expr>}} in normal Pavilion "
"strings). Other result values (including those "
"from result parsers and other evaluations are "
"available to reference as variables."),
]
# We'll append the result parsers separately, to have an easy way to
# access it.
_RESULT_PARSERS = yc.KeyedElem(
'result_parse',
elements=[],
help_text="Result parser configurations go here. Each parser config "
"can occur by itself or as a list of configs, in which "
"case the parser will run once for each config given. The "
"output of these parsers will be added to the final "
"result json data.")
ELEMENTS.append(_RESULT_PARSERS)
@classmethod
def add_subsection(cls, subsection):
"""Use this method to add additional sub-sections to the config.
:param yc.ConfigElem subsection: A yaml config element to add. Keyed
elements are expected, though any ConfigElem based instance
(whose leave elements are StrElems) should work.
"""
if not isinstance(subsection, yc.ConfigElement):
raise ValueError("Tried to add a subsection to the config, but it "
"wasn't a yaml_config ConfigElement instance (or "
"an instance of a ConfigElement child "
"class).")
name = subsection.name
names = [el.name for el in cls.ELEMENTS]
if name in names:
raise ValueError("Tried to add a subsection to the config called "
"{0}, but one already exists.".format(name))
try:
cls.check_leaves(subsection)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls.ELEMENTS.append(subsection)
@classmethod
def remove_subsection(cls, subsection_name):
"""Remove a subsection from the config. This is really only for use
in plugin deactivate methods."""
for section in list(cls.ELEMENTS):
if subsection_name == section.name:
cls.ELEMENTS.remove(section)
return
@classmethod
def add_result_parser_config(cls, name, config_items):
"""Add the given list of config items as a result parser
configuration named 'name'. Throws errors for invalid configuraitons.
"""
# Validate the config.
required_keys = {
'files': False,
'action': False,
'per_file': False,
}
for item in config_items:
for req_key in required_keys.keys():
if item.name == req_key:
required_keys[req_key] = True
for req_key, found in required_keys.items():
if not found:
raise TestConfigError(
"Result parser '{}' must have a required config "
"element named '{}'".format(name, req_key))
config = yc.KeyedElem('result_parser_{}'.format(name),
elements=config_items)
list_elem = yc.CategoryElem(name, sub_elem=config)
if name in [e.name for e in cls._RESULT_PARSERS.config_elems.values()]:
raise ValueError(
"Tried to add result parser with name '{}'"
"to the config, but one already exists.".format(name))
try:
cls.check_leaves(config)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls._RESULT_PARSERS.config_elems[name] = list_elem
@classmethod
def remove_result_parser_config(cls, name):
"""Remove the given result parser from the result parser configuration
section.
:param str name: The name of the parser to remove.
"""
for section in list(cls._RESULT_PARSERS.config_elems.values()):
if section.name == name:
del cls._RESULT_PARSERS.config_elems[section.name]
return
@classmethod
def check_leaves(cls, elem):
"""Make sure all of the config elements have a string element or
equivalent as the final node.
:param yc.ConfigElement elem:
"""
# pylint: disable=protected-access
if hasattr(elem, 'config_elems'):
for sub_elem in elem.config_elems.values():
cls.check_leaves(sub_elem)
elif hasattr(elem, '_sub_elem') and elem._sub_elem is not None:
cls.check_leaves(elem._sub_elem)
elif issubclass(elem.type, str):
return
else:
raise ValueError(elem)
def __init__(self):
super().__init__(
name='table',
description="Parses tables of data, creating a mapping of "
"row to a mapping of data by column.",
config_elems=[
yc.StrElem(
'delimiter_re',
help_text="Delimiter that splits the data in each row. "
"Defaults to whitespace one or more "
"characters ('\\s+'). Regardless of "
"the delimiter, whitespace is always stripped "
"from each either end of extracted data. "
"Whitespace delimited tables with missing "
"values are not supported."
),
yc.StrElem(
'row_ignore_re',
help_text="Optional. Regex of lines to ignore when "
"parsing the table. Mainly for skipping "
"divider lines and similar. Make sure your "
"regex matches the whole line with '^' and "
"'$'. Defaults to ignore rows composed only of "
"'|', '-', or '+' (plus whitespace)."
),
yc.StrElem(
'table_end_re',
help_text="The regular expression that denotes the end "
"of the table. Defaults to a '^\\s*$' "
"(a line of nothing but whitespace)."
),
yc.ListElem(
'col_names', required=False, sub_elem=yc.StrElem(),
help_text="Optional. The column names. By default, "
"the first line of the table is considered to be "
"the column names. Data for columns with an "
"empty name (ie '') are not included in the "
"results."
),
yc.StrElem(
'has_row_labels',
help_text="Optional. The first column will be used as the "
"row label. If this is False or the first column "
"is empty, the row will labeled 'row_n' starting "
"from 1. Row labels will be normalized and "
"altered for uniqueness."
),
yc.StrElem(
'by_column',
help_text="Set to True if the user wants to organize the "
"nested dictionaries by columns. Default False. "
"Only set if `has_header` is True. "
"Otherwise, Pavilion will ignore."
),
],
defaults={
'delimiter_re': r'\s+',
'row_ignore_re': r'^(\s*(\||\+|-|=)+)+\s*$',
'table_end_re': r'^\s*$',
'col_names': [],
'has_row_labels': 'True',
'by_column': 'False',
},
validators={
'has_header': ('True', 'False'),
'by_column': ('True', 'False'),
'delimiter_re': re.compile,
'table_end_re': re.compile,
}
)
class PavilionConfigLoader(yc.YamlConfigLoader):
# Each and every configuration element needs to either not be required,
# or have a sensible default. Essentially, Pavilion needs to work if no
# config is given.
ELEMENTS = [
yc.ListElem(
"config_dirs",
defaults=PAV_CONFIG_SEARCH_DIRS,
sub_elem=yc.StrElem(),
help_text="Paths to search for Pavilion config files. Pavilion "
"configs (other than this core config) are searched for "
"in the given order. In the case of identically named "
"files, directories listed earlier take precedent."),
yc.StrElem(
'working_dir', default=USER_HOME_PAV,
help_text="Where pavilion puts it's run files, downloads, etc."),
yc.ListElem(
"disable_plugins", sub_elem=yc.StrElem(),
help_text="Allows you to disable plugins by '<type>.<name>'. For "
"example, 'module.gcc' would disable the gcc module "
"wrapper."),
yc.StrElem(
"shared_group", post_validator=group_validate,
help_text="Pavilion can automatically set group permissions on all "
"created files, so that users can share relevant "
"results, etc."),
yc.StrElem(
"umask", default="0002",
help_text="The umask to apply to all files created by pavilion. "
"This should be in the format needed by the umask shell "
"command."),
yc.StrElem(
"log_format",
default="%{asctime}, ${levelname}, ${name}: ${message}",
help_text="The log format to use for the pavilion logger. See: "
"https://docs.python.org/3/library/logging.html#"
"logrecord-attributes"),
yc.StrElem(
"log_level", default="info", post_validator=log_level_validate,
help_text="The minimum log level for messages sent to the pavilion "
"logfile."),
yc.IntElem(
"wget_timeout", default=5,
help_text="How long to wait on web requests before timing out. On"
"networks without internet access, zero will allow you"
"to spot issues faster."
),
yc.CategoryElem(
"proxies", sub_elem=yc.StrElem(),
help_text="Proxies, by protocol, to use when accessing the "
"internet. Eg: http: 'http://myproxy.myorg.org:8000'"),
yc.ListElem(
"no_proxy", sub_elem=yc.StrElem(),
help_text="A list of DNS suffixes to ignore for proxy purposes. "
"For example: 'blah.com' would match 'www.blah.com', but "
"not 'myblah.com'."),
# The following configuration items are for internal use and provide a
# convenient way to pass around core pavilion components or data.
# They are not intended to be set by the user, and will generally be
# overwritten without even checking for user provided values.
yc.StrElem(
'pav_root', default=pav_root, hidden=True,
help_text="The root directory of the pavilion install. This "
"shouldn't be set by the user."),
yc.KeyedElem(
'sys_vars', elements=[], hidden=True, default={},
help_text="This will contain the system variable dictionary."),
yc.KeyedElem(
'pav_vars', elements=[], hidden=True, default={},
help_text="This will contain the pavilion variable dictionary."),
]
def _get_config_elems():
return [
yc.StrElem('num_nodes',
default="1",
help_text="Number of nodes requested for this test. "
"This can be a range (e.g. 12-24)."),
yc.StrElem('tasks_per_node',
default="1",
help_text="Number of tasks to run per node."),
yc.StrElem(
'mem_per_node',
help_text="The minimum amount of memory required in GB. "
"This can be a range (e.g. 64-128)."),
yc.StrElem('partition',
default="standard",
help_text="The partition that the test should be run "
"on."),
yc.StrElem(
'immediate',
choices=['true', 'false', 'True', 'False'],
default='false',
help_text="Only consider nodes not currently running jobs"
"when determining job size. Will set the minimum"
"number of nodes "),
yc.StrElem('qos', help_text="The QOS that this test should use."),
yc.StrElem('account',
help_text="The account that this test should run "
"under."),
yc.StrElem('reservation',
help_text="The reservation that this test should "
"run under."),
yc.RegexElem(
'time_limit',
regex=r'^(\d+-)?(\d+:)?\d+(:\d+)?$',
help_text="The time limit to specify for the slurm job in"
"the formats accepted by slurm "
"(<hours>:<minutes> is typical)"),
yc.RegexElem('include_nodes',
regex=Slurm.NODE_LIST_RE,
help_text="The nodes to include, in the same format "
"that Slurm expects with the -w or -x option. "
"This will automatically increase num_nodes to "
"at least this node count."),
yc.RegexElem(
'exclude_nodes',
regex=Slurm.NODE_LIST_RE,
help_text="A list of nodes to exclude, in the same format "
"that Slurm expects with the -w or -x option."),
yc.ListElem(name='avail_states',
sub_elem=yc.StrElem(),
defaults=['IDLE', 'MAINT'],
help_text="When looking for immediately available "
"nodes, they must be in one of these "
"states."),
yc.ListElem(name='up_states',
sub_elem=yc.StrElem(),
defaults=['ALLOCATED', 'COMPLETING', 'IDLE', 'MAINT'],
help_text="When looking for nodes that could be "
"allocated, they must be in one of these "
"states."),
yc.StrElem('job_name',
default="pav",
help_text="The job name for this test."),
]
class PavilionConfigLoader(yc.YamlConfigLoader):
"""This object uses YamlConfig to define Pavilion's base configuration
format and options. If you're looking to add an option to the general
pavilion.yaml format, this is the place to do it."""
# Each and every configuration element needs to either not be required,
# or have a sensible default. Essentially, Pavilion needs to work if no
# config is given.
ELEMENTS = [
yc.ListElem(
"config_dirs",
sub_elem=ExPathElem(),
post_validator=config_dirs_validator,
help_text="Additional Paths to search for Pavilion config files. "
"Pavilion configs (other than this core config) are "
"searched for in the given order. In the case of "
"identically named files, directories listed earlier "
"take precedence."),
yc.BoolElem(
"user_config",
default=False,
help_text="Whether to automatically add the user's config "
"directory at ~/.pavilion to the config_dirs. Configs "
"in this directory always take precedence."
),
ExPathElem(
'working_dir', default=USER_HOME_PAV/'working_dir', required=True,
help_text="Where pavilion puts it's run files, downloads, etc."),
yc.ListElem(
"disable_plugins", sub_elem=yc.StrElem(),
help_text="Allows you to disable plugins by '<type>.<name>'. For "
"example, 'module.gcc' would disable the gcc module "
"wrapper."),
yc.StrElem(
"shared_group", post_validator=_group_validate,
help_text="Pavilion can automatically set group permissions on all "
"created files, so that users can share relevant "
"results, etc."),
yc.StrElem(
"umask", default="2",
help_text="The umask to apply to all files created by pavilion. "
"This should be in the format needed by the umask shell "
"command."),
yc.IntRangeElem(
"build_threads", default=4, vmin=1,
help_text="Maximum simultaneous builds. Note that each build may "
"itself spawn off threads/processes, so it's probably "
"reasonable to keep this at just a few."),
yc.StrElem(
"log_format",
default="{asctime}, {levelname}, {hostname}, {name}: {message}",
help_text="The log format to use for the pavilion logger. "
"Uses the modern '{' format style. See: "
"https://docs.python.org/3/library/logging.html#"
"logrecord-attributes"),
yc.StrElem(
"log_level", default="info",
choices=['debug', 'info', 'warning', 'error', 'critical'],
help_text="The minimum log level for messages sent to the pavilion "
"logfile."),
ExPathElem(
"result_log",
# Derive the default from the working directory, if a value isn't
# given.
post_validator=(lambda d, v: v if v is not None else
d['working_dir']/'results.log'),
help_text="Results are put in both the general log and a specific "
"results log. This defaults to 'results.log' in the "
"working directory."),
yc.BoolElem(
"flatten_results", default=True,
help_text="Flatten results with multiple 'per_file' values into "
"multiple result log lines, one for each 'per_file' "
"value. Each flattened result will have a 'file' key, "
"and the contents of its 'per_file' data will be added "
"to the base results mapping."),
ExPathElem(
'exception_log',
# Derive the default from the working directory, if a value isn't
# given.
post_validator=(lambda d, v: v if v is not None else
d['working_dir']/'exceptions.log'),
help_text="Full exception tracebacks and related debugging "
"information is logged here."
),
yc.IntElem(
"wget_timeout", default=5,
help_text="How long to wait on web requests before timing out. On "
"networks without internet access, zero will allow you "
"to spot issues faster."
),
yc.CategoryElem(
"proxies", sub_elem=yc.StrElem(),
help_text="Proxies, by protocol, to use when accessing the "
"internet. Eg: http: 'http://myproxy.myorg.org:8000'"),
yc.ListElem(
"no_proxy", sub_elem=yc.StrElem(),
help_text="A list of DNS suffixes to ignore for proxy purposes. "
"For example: 'blah.com' would match 'www.blah.com', but "
"not 'myblah.com'."),
yc.ListElem(
"env_setup", sub_elem=yc.StrElem(),
help_text="A list of commands to be executed at the beginning of "
"every kickoff script."),
yc.CategoryElem(
"default_results", sub_elem=yc.StrElem(),
help_text="Each of these will be added as a constant result "
"parser with the corresponding key and constant value. "
"Generally, the values should contain a pavilion "
"variable of some sort to resolve."),
# The following configuration items are for internal use and provide a
# convenient way to pass around core pavilion components or data.
# They are not intended to be set by the user, and will generally be
# overwritten without even checking for user provided values.
ExPathElem(
'pav_cfg_file', hidden=True,
help_text="The location of the loaded pav config file."
),
ExPathElem(
'pav_root', default=PAV_ROOT, hidden=True,
help_text="The root directory of the pavilion install. This "
"shouldn't be set by the user."),
yc.KeyedElem(
'pav_vars', elements=[], hidden=True, default={},
help_text="This will contain the pavilion variable dictionary."),
]
class TestConfigLoader(yc.YamlConfigLoader):
"""This class describes a test section in a Pavilion config file. It is
expected to be added to by various plugins."""
ELEMENTS = [
yc.RegexElem(
'inherits_from',
regex=r'\w+',
help_text="Inherit from the given test section, and override "
"parameters those specified in this one. Lists are "
"overridden entirely"),
yc.StrElem(
'subtitle',
help_text="An extended title for this test. This is useful for "
"assigning unique name to virtual tests through "
"variable insertion. example, if a test has a single "
"permutation variable 'subtest', then '{subtest}' "
"would give a useful descriptor."),
VarCatElem('variables',
sub_elem=yc.ListElem(sub_elem=VariableElem()),
help_text="Variables for this test section. These can be "
"inserted strings anywhere else in the config through "
"the string syntax. They keys 'var', 'per', 'pav', "
"'sys' and 'sched' reserved. Each value may be a "
"single or list of strings key/string pairs."),
VarCatElem(
'permutations',
sub_elem=yc.ListElem(sub_elem=VariableElem()),
help_text="Permutation variables for this test section. These are "
"just like normal variables, but they if a list of "
"values (whether a single string or key/string pairs) "
"is given, then a virtual test is created for each "
"combination across all variables in each section. The "
"resulting virtual test is thus given a single "
"permutation of these values."),
yc.RegexElem('scheduler',
regex=r'\w+',
help_text="The scheduler class to use to run this test."),
yc.KeyedElem(
'build',
elements=[
yc.StrElem(
'source_location',
help_text="Path to the test source. It may be a directory, "
"a tar file, or a URI. If it's a directory or "
"file, the path is to '$PAV_CONFIG/test_src' by "
"default. For url's, the is automatically checked "
"for updates every time the test run. Downloaded "
"files are placed in a 'downloads' under the "
"pavilion working directory. (set in pavilion.yaml)"),
yc.StrElem(
'source_download_name',
help_text='When downloading source, we by default use the '
'last of the url path as the filename, or a hash '
'of the url if is no suitable name. Use this '
'parameter to override behavior with a pre-defined '
'filename.'),
yc.ListElem(
'modules',
sub_elem=yc.StrElem(),
help_text="Modules to load into the build environment."),
yc.CategoryElem(
'env',
sub_elem=yc.StrElem(),
help_text="Environment variables to set in the build "
"environment."),
yc.ListElem(
'extra_files',
sub_elem=yc.StrElem(),
help_text='Files to copy into the build environment. '
'Relative paths searched for in ~/.pavilion, '
'$PAV_CONFIG. Absolute paths are ok, '
'but not recommended.'),
yc.StrElem(
'specificity',
help_text="Use this string, along with variables, to "
"differentiate builds. A common example would be "
"to make per-host specific by using the "
"sys.sys_name variable. Note _deferred_ system "
"variables aren't a good idea hereas configs are "
"compiled on the host that launches the test."),
yc.ListElem(
'cmds',
sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to perform '
'the build.')
],
help_text="The test build configuration. This will be used to "
"dynamically generate a build script for building "
"the test."),
yc.KeyedElem(
'run',
elements=[
yc.ListElem(
'modules',
sub_elem=yc.StrElem(),
help_text="Modules to load into the run environment."),
yc.CategoryElem(
'env',
sub_elem=yc.StrElem(),
help_text="Environment variables to set in the run "
"environment."),
yc.ListElem(
'cmds',
sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to run the '
'test.')
],
help_text="The test run configuration. This will be used "
"to dynamically generate a run script for the "
"test."),
]
# We'll append the result parsers separately, to have an easy way to
# access it.
_RESULT_PARSERS = yc.KeyedElem(
'result',
elements=[],
help_text="Result parser configurations go here. Each parser config "
"can occur by itself or as a list of configs, in which "
"case the parser will run once for each config given. The "
"output of these parsers will be combined into the final "
"result json data.")
ELEMENTS.append(_RESULT_PARSERS)
@classmethod
def add_subsection(cls, subsection):
"""Use this method to add additional sub-sections to the config.
:param yc.ConfigElem subsection: A yaml config element to add. Keyed
elements are expected, though any ConfigElem based instance
(whose leave elements are StrElems) should work.
"""
if not isinstance(subsection, yc.ConfigElement):
raise ValueError("Tried to add a subsection to the config, but it "
"wasn't a yaml_config ConfigElement instance (or "
"an instance of a ConfigElement child "
"class).")
name = subsection.name
names = [el.name for el in cls.ELEMENTS]
if name in names:
raise ValueError("Tried to add a subsection to the config called "
"{0}, but one already exists.".format(name))
try:
cls.check_leaves(subsection)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls.ELEMENTS.append(subsection)
@classmethod
def remove_subsection(cls, subsection_name):
"""Remove a subsection from the config. This is really only for use
in plugin deactivate methods."""
for section in list(cls.ELEMENTS):
if subsection_name == section.name:
cls.ELEMENTS.remove(section)
return
@classmethod
def add_result_parser_config(cls, name, config_items):
"""Add the given list of config items as a result parser
configuration named 'name'. Throws errors for invalid configuraitons.
"""
config = yc.KeyedElem('result_parser_{}'.format(name),
elements=config_items)
list_elem = yc.ListElem(name, sub_elem=config)
if name in [e.name for e in cls._RESULT_PARSERS.config_elems.values()]:
raise ValueError(
"Tried to add result parser with name '{}'"
"to the config, but one already exists.".format(name))
try:
cls.check_leaves(config)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls._RESULT_PARSERS.config_elems[name] = list_elem
@classmethod
def remove_result_parser_config(cls, name):
"""Remove the given result parser from the result parser configuration
section.
:param str name: The name of the parser to remove.
:return:
"""
for section in list(cls._RESULT_PARSERS.config_elems.values()):
if section.name == name:
del cls._RESULT_PARSERS.config_elems[section.name]
return
@classmethod
def check_leaves(cls, elem):
"""
:param yc.ConfigElement elem:
:return:
"""
if hasattr(elem, 'config_elems'):
for sub_elem in elem.config_elems.values():
cls.check_leaves(sub_elem)
elif hasattr(elem, '_sub_elem') and elem._sub_elem is not None:
cls.check_leaves(elem._sub_elem)
elif issubclass(elem.type, str):
return
else:
raise ValueError(elem)
class TestConfigLoader(yc.YamlConfigLoader):
"""This class describes a test section in a Pavilion config file. It is
expected to be added to by various plugins.
:cvar list(yc.YamlConfig) ELEMENTS: Each YamlConfig instance in this
list defines a key for the test config.
- Each element must result in a string (which is why you see a lot of StrElem
below), or a structure that contains only strings at the lowest layer.
- So lists of dicts of strings are fine, etc.
- yc.RegexElem also produces a string.
- Everything should have a sensible default.
- An empty config should be a valid test.
- For bool values, accept ['true', 'false', 'True', 'False'].
- They should be checked with val.lower() == 'true', etc.
- Every element must have a useful 'help_text'.
"""
ELEMENTS = [
yc.RegexElem(
'inherits_from', regex=TEST_NAME_RE_STR,
help_text="Inherit from the given test section, and override "
"parameters those specified in this one. Lists are "
"overridden entirely"),
yc.StrElem(
'subtitle',
help_text="An extended title for this test. This is useful for "
"assigning unique name to virtual tests through "
"variable insertion. example, if a test has a single "
"permutation variable 'subtest', then '{subtest}' "
"would give a useful descriptor."),
yc.StrElem(
'summary', default='',
help_text="Summary of the purpose of this test."
),
yc.StrElem(
'doc', default='',
help_text="Detailed documentation string for this test."
),
yc.ListElem(
'permute_on', sub_elem=yc.StrElem(),
help_text="List of permuted variables. For every permutation of "
"the values of these variables, a new virtual test will "
"be generated."
),
VarCatElem(
'variables', sub_elem=yc.ListElem(sub_elem=VariableElem()),
help_text="Variables for this test section. These can be "
"inserted strings anywhere else in the config through "
"the string syntax. They keys 'var', 'per', 'pav', "
"'sys' and 'sched' reserved. Each value may be a "
"single or list of strings key/string pairs."),
yc.RegexElem('scheduler', regex=r'\w+', default="raw",
help_text="The scheduler class to use to run this test."),
yc.KeyedElem(
'build', elements=[
yc.StrElem(
'on_nodes', default='False',
choices=['true', 'false', 'True', 'False'],
help_text="Whether to build on or off of the test "
"allocation."
),
yc.StrElem(
'source_location',
help_text="Path to the test source. It may be a directory, "
"a tar file, or a URI. If it's a directory or "
"file, the path is to '$PAV_CONFIG/test_src' by "
"default. For url's, the is automatically "
"checked for updates every time the test run. "
"Downloaded files are placed in a 'downloads' "
"under the pavilion working directory. (set in "
"pavilion.yaml)"),
yc.StrElem(
'source_download_name',
help_text='When downloading source, we by default use the '
'last of the url path as the filename, or a hash '
'of the url if is no suitable name. Use this '
'parameter to override behavior with a '
'pre-defined filename.'),
yc.ListElem(
'modules', sub_elem=yc.StrElem(),
help_text="Modules to load into the build environment."),
EnvCatElem(
'env', sub_elem=yc.StrElem(), key_case=EnvCatElem.KC_MIXED,
help_text="Environment variables to set in the build "
"environment."),
yc.ListElem(
'extra_files', sub_elem=yc.StrElem(),
help_text='Files to copy into the build environment. '
'Relative paths searched for in ~/.pavilion, '
'$PAV_CONFIG. Absolute paths are ok, '
'but not recommended.'),
yc.StrElem(
'specificity',
default='',
help_text="Use this string, along with variables, to "
"differentiate builds. A common example would be "
"to make per-host specific by using the "
"sys.sys_name variable. Note _deferred_ system "
"variables aren't a good idea hereas configs are "
"compiled on the host that launches the test."),
yc.StrElem(
'timeout',
default='30',
help_text="Time (in seconds) that a build can continue "
"without generating new output before it is "
"cancelled. Can be left empty for no timeout."),
yc.ListElem(
'cmds', sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to perform '
'the build.'),
yc.ListElem(
'preamble', sub_elem=yc.StrElem(),
help_text="Setup commands for the beginning of the build "
"script. Added to the beginning of the run "
"script. These are generally expected to "
"be host rather than test specific."),
yc.StrElem(
'verbose', choices=['true', 'True', 'False', 'false'],
default='False',
help_text="Echo commands (including sourced files) in the"
" build log, and print the modules loaded and "
"environment before the cmds run."),
],
help_text="The test build configuration. This will be "
"used to dynamically generate a build script for "
"building the test."),
yc.KeyedElem(
'run', elements=[
yc.ListElem(
'modules', sub_elem=yc.StrElem(),
help_text="Modules to load into the run environment."),
EnvCatElem(
'env', sub_elem=yc.StrElem(), key_case=EnvCatElem.KC_MIXED,
help_text="Environment variables to set in the run "
"environment."),
yc.ListElem('cmds', sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to run '
'the test.'),
yc.ListElem(
'preamble', sub_elem=yc.StrElem(),
help_text="Setup commands for the beginning of the build "
"script. Added to the beginning of the run "
"script. These are generally expected to "
"be host rather than test specific."),
yc.StrElem(
'verbose', choices=['true', 'True', 'False', 'false'],
default='False',
help_text="Echo commands (including sourced files) in the "
"build log, and print the modules loaded and "
"environment before the cmds run."),
yc.StrElem(
'timeout', default='300',
help_text="Time that a build can continue without "
"generating new output before it is cancelled. "
"Can be left empty for no timeout.")
],
help_text="The test run configuration. This will be used "
"to dynamically generate a run script for the "
"test."),
]
# We'll append the result parsers separately, to have an easy way to
# access it.
_RESULT_PARSERS = yc.KeyedElem(
'results', elements=[],
help_text="Result parser configurations go here. Each parser config "
"can occur by itself or as a list of configs, in which "
"case the parser will run once for each config given. The "
"output of these parsers will be combined into the final "
"result json data.")
ELEMENTS.append(_RESULT_PARSERS)
@classmethod
def add_subsection(cls, subsection):
"""Use this method to add additional sub-sections to the config.
:param yc.ConfigElem subsection: A yaml config element to add. Keyed
elements are expected, though any ConfigElem based instance
(whose leave elements are StrElems) should work.
"""
if not isinstance(subsection, yc.ConfigElement):
raise ValueError("Tried to add a subsection to the config, but it "
"wasn't a yaml_config ConfigElement instance (or "
"an instance of a ConfigElement child "
"class).")
name = subsection.name
names = [el.name for el in cls.ELEMENTS]
if name in names:
raise ValueError("Tried to add a subsection to the config called "
"{0}, but one already exists.".format(name))
try:
cls.check_leaves(subsection)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based."
.format(name, err.args[0]))
cls.ELEMENTS.append(subsection)
@classmethod
def remove_subsection(cls, subsection_name):
"""Remove a subsection from the config. This is really only for use
in plugin deactivate methods."""
for section in list(cls.ELEMENTS):
if subsection_name == section.name:
cls.ELEMENTS.remove(section)
return
@classmethod
def add_result_parser_config(cls, name, config_items):
"""Add the given list of config items as a result parser
configuration named 'name'. Throws errors for invalid configuraitons.
"""
# Validate the config.
required_keys = {
'key': False,
'files': False,
'action': False,
'per_file': False,
}
for item in config_items:
for req_key in required_keys.keys():
if item.name == req_key:
required_keys[req_key] = True
for req_key, found in required_keys.items():
if not found:
raise TestConfigError(
"Result parser '{}' must have a required config "
"element named '{}'".format(name, req_key))
config = yc.KeyedElem(
'result_parser_{}'.format(name),
elements=config_items
)
list_elem = yc.ListElem(name, sub_elem=config)
if name in [e.name for e in cls._RESULT_PARSERS.config_elems.values()]:
raise ValueError("Tried to add result parser with name '{}'"
"to the config, but one already exists."
.format(name))
try:
cls.check_leaves(config)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based."
.format(name, err.args[0]))
cls._RESULT_PARSERS.config_elems[name] = list_elem
@classmethod
def remove_result_parser_config(cls, name):
"""Remove the given result parser from the result parser configuration
section.
:param str name: The name of the parser to remove.
"""
for section in list(cls._RESULT_PARSERS.config_elems.values()):
if section.name == name:
del cls._RESULT_PARSERS.config_elems[section.name]
return
@classmethod
def check_leaves(cls, elem):
"""Make sure all of the config elements have a string element or
equivalent as the final node.
:param yc.ConfigElement elem:
"""
# pylint: disable=protected-access
if hasattr(elem, 'config_elems'):
for sub_elem in elem.config_elems.values():
cls.check_leaves(sub_elem)
elif hasattr(elem, '_sub_elem') and elem._sub_elem is not None:
cls.check_leaves(elem._sub_elem)
elif issubclass(elem.type, str):
return
else:
raise ValueError(elem)
class ResultParser(IPlugin.IPlugin):
"""Base class for creating a result parser plugin. These are essentially
a callable that implements operations on the test or test files. The
arguments for the callable are provided automatically via the test
config. The doc string of result parser classes is used as the user help
text for that class, along with the help from the config items."""
PRIO_CORE = 0
PRIO_COMMON = 10
PRIO_USER = 20
FORCE_DEFAULTS = []
"""Let the user know they can't set these config keys for this result
parser, effectively forcing the value to the default."""
def __init__(self,
name,
description,
defaults=None,
config_elems=None,
validators=None,
priority=PRIO_COMMON):
"""Initialize the plugin object
:param str name: The name of this plugin.
:param str description: A short description of this result parser.
:param List[yaml_config.ConfigElement] config_elems: A list of configuration
elements (from the yaml_config library) to use to define the
config section for this result parser. These will be passed as arguments
to the parser function. Only StrElem and ListElems are accepted. Any type
conversions should be done with validators. Use the defaults and validators
argument to set defaults, rather than the YamlConfig options.
:param dict defaults: A dictionary of defaults for the result parser's
arguments.
:param dict validators: A dictionary of auto-validators. These can
take several forms:
- A tuple - The value must be one of the items in the tuple.
- a function - The function should accept a single argument. The
returned value is used. ValueError or ResultError should be raised
if there are issues. Typically this will be a type conversion function.
For list arguments this is applied to each of the list values.
:param int priority: The priority of this plugin, compared to plugins
of the same name. Higher priority plugins will supersede others.
"""
self.name = name
self.description = description
self.defaults = {} if defaults is None else defaults
self.validators = BASE_VALIDATORS.copy()
if validators is not None:
for key, validator in validators.items():
if not (isinstance(validator, tuple) or callable(validator)):
raise RuntimeError(
"Validator for key {} in result parser {} at {} must "
"be a tuple or a function.".format(
key, name, self.path))
validators[key] = validator
config_elems = config_elems if config_elems is not None else []
for elem in config_elems:
if not (isinstance(elem, yc.StrElem) or
(isinstance(elem, yc.ListElem)
and isinstance(elem._sub_elem, yc.StrElem))):
raise RuntimeError(
"Config elements for result parsers must be strings"
"or lists of strings. Got elem {} in parser at {}".format(
elem, self.path))
if elem.default not in [None, []] or elem._choices is not None:
raise RuntimeError(
"Config elements for result parsers shouldn't set "
"a default or choices (use the defaults or validators"
"argument for the result_parser init). Problem found "
"in {} in result parser at {}".format(elem, self.path))
self.config_elems = config_elems
self.priority = priority
super().__init__()
def __call__(self, file, **kwargs):
"""This is where the result parser is actually implemented.
:param pavilion.test_run.TestRun test: The test run object.
:param file: This will be a file object advanced to a position that fits
the criteria of the 'preceded_by' and 'for_lines_matching' options.
:param dict kwargs: The arguments are the config values from the the
test's result config section for this parser. These should be
explicitly defined in your result parser class.
:raises ResultParserError: When something goes wrong.
"""
raise NotImplementedError("A result parser plugin must implement"
"the __call__ method.")
def _check_args(self, **kwargs) -> dict:
"""Override this to add custom checking of the arguments at test
kickoff time. This prevents errors in your arguments from causing
a problem in the middle of a test run. The yaml_config module handles
structural checking (and can handle more). This should raise a
descriptive ResultParserError if any issues are found.
:param kwargs: Child result parsers should override these with
specific kwargs for their arguments. They should all default to
and rely on the config parser to set their defaults.
:raises ResultParserError: If there are bad arguments.
"""
_ = self
return kwargs
def check_args(self, **kwargs) -> dict:
"""Check the arguments for any errors at test kickoff time, if they
don't contain deferred variables. We can't check tests with
deferred args. On error, should raise a ResultParserError.
:param dict kwargs: The arguments from the config.
:raises ResultError: When bad arguments are given.
"""
# The presence or absence of needed args should be enforced by
# setting 'required' in the yaml_config config items.
args = self.defaults.copy()
for key, val in kwargs.items():
if val is None or (key in args and val == []):
continue
args[key] = kwargs[key]
kwargs = args
base_keys = ('action', 'per_file', 'files', 'match_select',
'for_lines_matching', 'preceded_by')
for key in base_keys:
if key not in kwargs:
raise RuntimeError(
"Result parser '{}' missing required attribute '{}'. These "
"are validated at the config level, so something is "
"probably wrong with plugin.".format(self.name, key))
match_select = MATCH_CHOICES.get(kwargs['match_select'],
kwargs['match_select'])
if match_select is not None:
try:
int(match_select)
except ValueError:
raise ResultError(
"Invalid value for 'match_select'. Must be one of "
"{} or an integer.")
for arg in self.FORCE_DEFAULTS:
if kwargs[arg] != self._DEFAULTS[arg]:
raise ResultError(
"This parser requires that you not set the '{}' key, as "
"the default value is the only valid option.".format(arg))
if kwargs['for_lines_matching'] is None and not kwargs['preceded_by']:
raise ResultError(
"At least one of 'for_lines_matching' or 'preceded_by' "
"must be set. You should only see this if the default for "
"'for_lines_matching' was explicitly set to null.")
for key, validator in self.validators.items():
if isinstance(validator, tuple):
if kwargs[key] not in validator:
raise ResultError("Invalid value for option '{}'.\n"
"Expected one of {}, got '{}'.".format(
key, validator, kwargs[key]))
else:
# Must be a validator function.
value = kwargs[key]
try:
if isinstance(value, list):
kwargs[key] = [validator(val) for val in value]
else:
kwargs[key] = validator(value)
except ValueError as err:
raise ResultError("Validation error for option '{}' with "
"value '{}'.\n{}".format(
key, kwargs[key], err.args[0]))
for key in base_keys:
# The parser plugins don't know about these keys, as they're
# handled at a higher level.
del kwargs[key]
return self._check_args(**kwargs)
GLOBAL_CONFIG_ELEMS = [
yc.StrElem(
"action",
help_text=(
"What to do with parsed results.\n"
"{STORE} - Just store the result automatically converting \n"
" the value's type(default).\n"
"{STORE_STR} - Just store the value, with no type "
"conversion.\n"
"{TRUE} - Store True if there was a result.\n"
"{FALSE} - Store True for no result.\n"
"{COUNT} - Count the number of results.\n".format(
STORE=ACTION_STORE,
STORE_STR=ACTION_STORE_STR,
TRUE=ACTION_TRUE,
FALSE=ACTION_FALSE,
COUNT=ACTION_COUNT))),
# The default for the file is handled by the test object.
yc.ListElem("files",
sub_elem=yc.StrElem(),
help_text="Path to the file/s that this result parser "
"will examine. Each may be a file glob,"
"such as '*.log'"),
yc.StrElem(
"per_file",
help_text=(
"How to save results for multiple file matches.\n"
"{FIRST} - (default) The result from the first file with a \n"
" non-empty result. If no files were found, this is \n"
" considerd an error. (default)\n"
"{LAST} - As '{FIRST}', but last result.\n"
"{NAME} - Store the results on a per file basis under \n"
" results['per_name'][<filename>][<key>]. The \n"
" filename only includes the parts before any '.', and \n"
" is normalized to replace non-alphanum characters with '_'\n"
"{NAME_LIST} - Save the matching (normalized) file \n"
" names, rather than the parsed values, in a list.\n"
"{LIST} - Merge all each result and result list \n"
" into a single list.\n"
"{ALL} - Set 'True' if all files found at least one match.\n"
"{ANY} - Set 'True' if any file found at lest one match.\n".
format(
FIRST=PER_FIRST,
LAST=PER_LAST,
NAME=PER_NAME,
NAME_LIST=PER_NAME_LIST,
LIST=PER_LIST,
ANY=PER_ANY,
ALL=PER_ALL,
))),
yc.StrElem(
"for_lines_matching",
help_text=(
"A regular expression used to identify the line where "
"result parsing should start. The result parser will "
"see the file starting at this line. Defaults to matching "
"every line.")),
yc.ListElem(
"preceded_by",
sub_elem=yc.StrElem(),
help_text=(
"A list of regular expressions that must match lines that "
"precede the lines to be parsed. Empty items"
"in the sequence will match anything. The result parser "
"will see the file starting from start of the line after "
"these match.")),
yc.StrElem(
"match_select",
help_text=(
"In cases where multiple matches are possible, how to"
"handle them. By default, find the first '{FIRST}' "
"match and use it. '{LAST}' returns the final match, and "
"'{ALL}' will return a list of all matches. You may also "
"give an integer to get the Nth match (starting at 0). "
"Negative integers (starting at -1)count in reverse.".format(
FIRST=MATCH_FIRST, LAST=MATCH_LAST, ALL=MATCH_ALL))),
]
def get_config_items(self):
"""Get the config for this result parser. This should be a list of
yaml_config.ConfigElement instances that will be added to the test
config format at plugin activation time. The simplest format is a
list of yaml_config.StrElem objects, but any structure is allowed
as long as the leaf elements are StrElem type.
The config values will be passed as the keyword arguments to the
result parser when it's run and when its arguments are checked. The base
implementation provides several arguments that must be present for every result
parser. See the implementation of this method in result_parser.py for more
info on those arguments and what they do.
Example: ::
config_items = super().get_config_items()
config_items.append(
yaml_config.StrElem('token', default='PASSED',
help="The token to search for in the file."
)
return config_items
"""
config_items = self.GLOBAL_CONFIG_ELEMS.copy()
config_items.extend(self.config_elems)
return config_items
@property
def path(self):
"""The path to the file containing this result parser plugin."""
return inspect.getfile(self.__class__)
def doc(self):
"""Return documentation on this result parser."""
def wrap(text: str, indent=0):
"""Wrap a multi-line string."""
indent = ' ' * indent
out_lines = []
for line in text.splitlines():
out_lines.extend(
textwrap.wrap(line,
initial_indent=indent,
subsequent_indent=indent))
return out_lines
doc = [self.name, '-' * len(self.name)]
doc.extend(wrap(self.description))
doc.append('\n{} Parser Specific Arguments'.format(self.name))
doc.append('-' * len(doc[-1]))
args = self.get_config_items()
specific_args = [arg for arg in args if arg.name not in self._DEFAULTS]
def add_arg_doc(arg):
"""Add an arg to the documentation."""
doc.append(' ' + arg.name)
doc.extend(wrap(arg.help_text, indent=4))
if arg.name in self.defaults:
doc.extend(
wrap("default: '{}'".format(self.defaults[arg.name]),
indent=4))
if arg.name in self.validators:
validator = self.validators[arg.name]
if isinstance(validator, tuple):
doc.extend(wrap('choices: ' + str(validator), indent=4))
doc.append('')
for arg in specific_args:
if arg.name not in self._DEFAULTS:
add_arg_doc(arg)
doc.append('Universal Parser Arguments')
doc.append('-' * len(doc[-1]))
for arg in args:
if arg.name in self._DEFAULTS:
add_arg_doc(arg)
return '\n'.join(doc)
_DEFAULTS = {
'per_file': PER_FIRST,
'action': ACTION_STORE,
'files': ['../run.log'],
'match_select': MATCH_FIRST,
'for_lines_matching': '',
'preceded_by': [],
}
"""Defaults for the common parser arguments. This in not meant to be
changed by subclasses."""
def set_parser_defaults(self, rconf: dict, def_conf: dict):
"""Set the default values for each result parser. The default conf
can hold defaults that apply across an entire result parser."""
base = self._DEFAULTS.copy()
for key, val in def_conf.items():
if val not in (None, []):
base[key] = val
for key, val in rconf.items():
if val is None:
continue
if key in base and val == []:
continue
base[key] = val
return base
def check_config(self, rconf: dict, keys: List[str]) -> None:
"""Validate the parser configuration.
:param rconf: The results parser configuration.
:param keys: The keys (generally one) under which the parsed results
will be stored.
:raises: ResultError on failure.
"""
action = rconf.get('action')
per_file = rconf.get('per_file')
found_deferred = False
# Don't check args if they have deferred values.
for option, values in rconf.items():
if not isinstance(values, list):
values = [values]
for value in values:
if resolver.TestConfigResolver.was_deferred(value):
found_deferred = True
if found_deferred:
# We can't continue checking from this point if anything
# was deferred.
return
if ('result' in keys and action not in (ACTION_TRUE, ACTION_FALSE)
and per_file not in (PER_FIRST, PER_LAST)):
raise ResultError(
"Result parser has key 'result', but must store a "
"boolean. Use action '{}' or '{}', along with a "
"per_file setting of 'first', 'last', 'any', or 'all'".format(
ACTION_TRUE, ACTION_FALSE))
try:
self.check_args(**rconf)
except ResultError as err:
raise ResultError("Key '{}': {}".format(keys, err.args[0]))
def activate(self):
"""Yapsy runs this when adding the plugin.
In this case it:
- Adds the config section (from get_config_items()) to the test config
format.
- Adds the result parser to the list of known result parsers.
"""
config_items = self.get_config_items()
file_format.TestConfigLoader.add_result_parser_config(
self.name, config_items)
if self.name in _RESULT_PARSERS:
other = _RESULT_PARSERS[self.name]
if self.priority > other.priority:
LOGGER.info("Result parser '%s' at %s is superseded by %s.",
self.name, other.path, self.path)
_RESULT_PARSERS[self.name] = self
elif self.priority < other.priority:
LOGGER.info(
"Result parser '%s' at %s is ignored in lieu of %s.",
self.name, self.path, other.path)
else:
raise RuntimeError("Result parser conflict. Parser '{}' at {}"
"has the same priority as {}".format(
self.name, other.path, self.path))
else:
_RESULT_PARSERS[self.name] = self
def deactivate(self):
"""Yapsy calls this to remove this plugin. We only ever
do this in unittests."""
# Remove the section from the config.
file_format.TestConfigLoader.remove_result_parser_config(self.name)
# Remove from list of available result parsers.
del _RESULT_PARSERS[self.name]
