def __init__(self):
super().__init__(
name='command',
description="Runs a command, and uses it's output or return "
"values as a result value.",
config_elems=[
yc.StrElem(
'command',
required=True,
help_text="Run this command in a sub-shell and collect "
"its return value or stdout."),
yc.StrElem(
'output_type',
help_text="Whether to return the return value or stdout."),
yc.StrElem('stderr_dest',
help_text="Where to redirect stderr.")
],
validators={
'output_type': ('return_value', 'stdout'),
'stderr_dest': ('null', 'stdout'),
},
defaults={
'output_type': 'return_value',
'stderr_dest': 'stdout',
})
def get_config_items(self):
"""Get the config for this result parser. This should be a list of
yaml_config.ConfigElement instances that will be added to the test
config format at plugin activation time. The simplest format is a
list of yaml_config.StrElem objects, but any structure is allowed
as long as the leaf elements are StrElem type.
Every result parser is expected to take the following arguments:
'file' - The path to the file to examine (relative to the test
build directory), defaults to the test run's log.
Some parsers may ignore (or change the meaning of)
this argument.
'key' - The name to give the result in the result json. (No
default)
Example:
config_items = super().get_config_items()
config_items.append(
yaml_config.StrElem('token', default='PASSED',
help="The token to search for in the file."
)
return config_items
"""
return [
yc.StrElem("key",
required=True,
help_text="The key value in the result json for this"
"result component."),
# The default for the file is handled by the test object.
yc.StrElem("file",
default=None,
help_text="Path to the file that this result parser "
"will examine.")
]
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem(
'regex',
required=True,
help_text="The python regex to use to search the given file. "
"See: 'https://docs.python.org/3/library/re.html' "
"You can use single quotes in YAML to have the "
"string interpreted literally. IE '\\n' is a '\\' "
"and an 'n'."),
# Use the built-in matches element.
result_parsers.MATCHES_ELEM,
yc.StrElem(
'threshold',
default='0',
help_text="If a threshold is defined, 'True' will be returned "
"if greater than or equal to that many instances "
"of the specified word are found. If fewer "
"instances are found, 'False' is returned. The "
"value must be an integer greater than zero."),
yc.ListElem(
'expected',
sub_elem=yc.StrElem(),
help_text="Expected value(s) and/or range(s). If provided, "
"the result will be 'True' if all of the found "
"values (determined by the 'results' value) are "
"within the expected range(s) or value(s). "
"Otherwise, the result is 'False'. Supports "
"integers and floats.")
])
return config_items
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem(
'delimiter', default=' ',
help_text="Delimiter that splits the data."
),
yc.StrElem(
'col_num', required=True,
help_text="Number of columns in table, including row names, "
"if there is such a column."
),
yc.StrElem(
'has_header', default='False', choices=['True', 'False'],
help_text="Set True if there is a column for row names. Will "
"create dictionary of dictionaries."
),
yc.ListElem(
'col_names', required=False, sub_elem=yc.StrElem(),
help_text="Column names if the user knows what they are."
),
yc.StrElem(
'by_column', choices=['True', 'False'], default='True',
help_text="Set to True if the user wants to organize the "
"nested dictionaries by columns. Default False. "
"Only set if `has_header` is True. "
"Otherwise, Pavilion will ignore."
)
])
return config_items
class TestConfig(yc.YamlConfigLoader):
ELEMENTS = [
yc.StrElem("pet",
default="squirrel",
required=True,
choices=["squirrel", "cat", "dog"],
help_text="The kind of pet."),
yc.IntElem("quantity", required=True, choices=[1, 2, 3]),
yc.FloatRangeElem("quality", vmin=0, vmax=1.0),
yc.ListElem("potential_names",
help_text="What you could name this pet.",
sub_elem=yc.StrElem(help_text="Such as Fido.")),
yc.KeyedElem(
"properties",
help_text="Pet properties",
elements=[
yc.StrElem("description",
help_text="General pet description."),
yc.RegexElem("greeting",
regex=r'hello \w+$',
help_text="A regex of some sort."),
yc.IntRangeElem("legs", vmin=0)
]),
yc.CodeElem("behavior_code",
help_text="Program for pet behavior.")
]
def get_conf(self):
"""Set up the Slurm configuration attributes."""
return yc.KeyedElem(
self.name,
help_text="Configuration for the Slurm scheduler.",
elements=[
yc.StrElem(
'num_nodes',
default="1",
help_text="Number of nodes requested for this test. "
"This can be a range (e.g. 12-24)."),
yc.StrElem('tasks_per_node',
default="1",
help_text="Number of tasks to run per node."),
yc.StrElem(
'mem_per_node',
help_text="The minimum amount of memory required in GB. "
"This can be a range (e.g. 64-128)."),
yc.StrElem(
'partition',
default="standard",
help_text="The partition that the test should be run "
"on."),
yc.StrElem(
'immediate',
choices=['true', 'false', 'True', 'False'],
default='false',
help_text="Only consider nodes not currently running jobs"
"when determining job size"),
yc.StrElem('qos',
help_text="The QOS that this test should use."),
yc.StrElem('account',
help_text="The account that this test should run "
"under."),
yc.StrElem('reservation',
help_text="The reservation that this test should "
"run under."),
yc.RegexElem(
'time_limit',
regex=r'^(\d+-)?(\d+:)?\d+(:\d+)?$',
help_text="The time limit to specify for the slurm job in"
"the formats accepted by slurm "
"(<hours>:<minutes> is typical)"),
yc.ListElem(name='avail_states',
sub_elem=yc.StrElem(),
defaults=['IDLE', 'MAINT'],
help_text="When looking for immediately available "
"nodes, they must be in one of these "
"states."),
yc.ListElem(
name='up_states',
sub_elem=yc.StrElem(),
defaults=['ALLOCATED', 'COMPLETING', 'IDLE', 'MAINT'],
help_text="When looking for nodes that could be "
"allocated, they must be in one of these "
"states."),
])
def __init__(self):
super().__init__(
name='regex',
description="Find data using a basic regular expressions.",
config_elems=[
yc.StrElem(
'regex', required=True,
help_text="The python regex to use to search the given "
"file. See: 'https://docs.python.org/3/"
"library/re.html'.\n"
"You can use single quotes "
"in YAML to have the string interpreted "
"literally. IE '\\n' is a '\\' "
"and an 'n'.\n"
"If you include no matching groups, "
"(ie '^my regex.*') all matched text will be "
"the result. \n"
"With a single matching group, "
"(ie '^my field: (\\d+)') the "
"result will be the value in that group.\n"
"With multiple matching groups, "
"(ie '^(\\d+) \\d+ (\\d+)') the result value "
"will be a list of all matched values. You "
"can use a complex key like 'speed, flops' "
"to store each value in a different result field "
"if desired."
)]
)
def __init__(self, name=None, **kwargs):
"""Just like a CategoryElem, but the sub_elem must be a StrElem
and it can't have defaults."""
super(VariableElem, self).__init__(name=name,
sub_elem=yc.StrElem(),
defaults=None,
**kwargs)
def get_conf(self):
"""Add necessary MPI attributes to those of Slurm."""
elems = self._get_config_elems()
elems.extend([
yc.StrElem('rank_by',
help_text="Value for `--rank-by`. Default is slot."),
yc.StrElem('bind_to',
help_text="Value for `--bind-to`. Default is core."),
yc.ListElem('mca',
help_text="Key-Value for pair(s) for --mca.",
sub_elem=yc.RegexElem(
'kv_pair', regex=r'^[a-z1-9]+\s[a-z1-9,]+$'))
])
return yc.KeyedElem(
self.name,
help_text="Configuration for the Slurm scheduler using mpirun.",
elements=elems)
def get_config_items(self):
# Result parser consists of 1 string elem: filename.
config_items = super().get_config_items()
config_items.extend([
yc.StrElem('filename',
required=True,
help_text="Filename to find in working directory.")
])
return config_items
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem('command',
required=True,
help_text="Command that will be run."),
yc.StrElem('success',
default='return_value',
choices=['return_value', 'output'],
help_text="needs to be either return_value or output"),
yc.StrElem('stderr_out',
choices=['null', 'stdout'],
default='stdout',
help_text="where to redirect stderr")
])
return config_items
class Config2(yc.YamlConfigLoader):
ELEMENTS = [
yc.ListElem('cars',
sub_elem=yc.KeyedElem(elements=[
yc.StrElem('color'),
yc.StrElem('make'),
yc.ListElem('extras', sub_elem=yc.StrElem())
]))
]
def __init__(self, default_color):
super(Config2, self).__init__()
# The Config init is a good place to do this.
# Set all the default color for all cars in the 'cars' list to red.
self.set_default('cars.*.color', default_color)
self.set_default('cars.*.extras',
['rhoomba', 'heated sunshades'])
def __init__(self):
super().__init__(
name='table',
description="Parses tables",
config_elems=[
yc.StrElem(
'delimiter',
help_text="Delimiter that splits the data."
),
yc.StrElem(
'col_num', required=True,
help_text="Number of columns in table, including row "
"names, if there is such a column."
),
yc.StrElem(
'has_header',
help_text="Set True if there is a column for row names. "
"Will create dictionary of dictionaries."
),
yc.ListElem(
'col_names', required=False, sub_elem=yc.StrElem(),
help_text="Column names if the user knows what they are."
),
yc.StrElem(
'by_column',
help_text="Set to True if the user wants to organize the "
"nested dictionaries by columns. Default False. "
"Only set if `has_header` is True. "
"Otherwise, Pavilion will ignore."
)
],
defaults={
'delimiter': ' ',
'has_header': 'False',
'by_column': 'True',
},
validators={
'has_header': ('True', 'False'),
'by_column': ('True', 'False'),
'col_num': int,
}
)
def __init__(self):
super().__init__(
name='constant',
description="Insert a constant (can contain Pavilion variables) "
"into the results.",
config_elems=[
yc.StrElem('const',
required=True,
help_text="Constant that will be placed in result.")
])
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem('const',
required=True,
help_text="Constant that will be placed in result.")
])
return config_items
def __init__(self):
super().__init__(
name='split',
description="Split by a substring, are return the whitespace "
"stripped parts.",
config_elems=[
yc.StrElem('sep',
help_text="The substring to split by. Default is "
"to split by whitespace.")
])
def get_conf(self):
return yc.KeyedElem(
'raw',
elements=[
yc.StrElem(
'concurrent',
choices=['true', 'false', 'True', 'False'],
default='False',
help_text="Allow this test to run concurrently with other"
"concurrent tests under the 'raw' scheduler.")
])
def __init__(self):
super().__init__(
name='filecheck',
description="Checks working directory for a given file. Globs are"
"accepted.",
config_elems=[
yc.StrElem(
'filename', required=True,
help_text="Filename to find in working directory."
)
]
)
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem(
'regex',
default=None,
help_text="The python regex to use to search the given file. "
"See: 'https://docs.python.org/3/library/re.html' "
"You can use single quotes in YAML to have the string"
"interpreted literally. IE '\\n' is a '\\' and an 'n'"),
yc.StrElem(
'results',
default='first',
choices=['first', 'all', 'last'],
help_text="This can return the first, last, or all matches. "
"If there are no matches the result will be null"
"or an empty list.")
])
return config_items
def __init__(self):
super().__init__(
name='regex',
description="Find data using a basic regular expressions.",
config_elems=[
yc.StrElem(
'regex',
required=True,
help_text="The python regex to use to search the given "
"file. See: 'https://docs.python.org/3/"
"library/re.html'. You can use single quotes "
"in YAML to have the string interpreted "
"literally. IE '\\n' is a '\\' "
"and an 'n'. "), parsers.MATCHES_ELEM
])
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem('delimiter',
default=' ',
help_text="Delimiter that splits the data."),
yc.StrElem(
'col_num',
required=True,
help_text="Number of columns in table, including row names, "
"if there is such a column."),
yc.StrElem(
'has_header',
default='False',
help_text="Set True if there is a column for row names."),
yc.ListElem(
'col_names',
required=False,
sub_elem=yc.StrElem(),
help_text="Column names if the user knows what they are.")
])
return config_items
def get_config_items(self):
config_items = super().get_config_items()
config_items.extend([
yc.StrElem(
'regex',
required=True,
help_text="The python regex to use to search the given file. "
"See: 'https://docs.python.org/3/library/re.html' "
"You can use single quotes in YAML to have the "
"string interpreted literally. IE '\\n' is a '\\' "
"and an 'n'."),
parsers.MATCHES_ELEM,
])
return config_items
class SeriesConfigLoader(yc.YamlConfigLoader):
"""This class describes a series file."""
ELEMENTS = [
TestCatElem(
'series', sub_elem=yc.KeyedElem(
elements=[
yc.ListElem('tests', sub_elem=yc.StrElem()),
yc.StrElem('depends_pass',
choices=['True', 'true', 'False', 'false'],
default='False'),
yc.ListElem('depends_on', sub_elem=yc.StrElem()),
yc.ListElem('modes', sub_elem=yc.StrElem()),
CondCategoryElem(
'only_if', sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
key_case=EnvCatElem.KC_MIXED
),
CondCategoryElem(
'not_if', sub_elem=yc.ListElem(sub_elem=yc.StrElem()),
key_case=EnvCatElem.KC_MIXED
),
]
),
),
yc.ListElem(
'modes', sub_elem=yc.StrElem()
),
yc.IntElem(
'simultaneous',
),
yc.StrElem(
'ordered', choices=['True', 'true', 'False', 'false'],
default='False'
),
yc.StrElem(
'restart', choices=['True', 'true', 'False', 'false'],
default='False'
)
]
"""Describes elements in Series Config Loader."""
class SpackEnvConfig(yc.YamlConfigLoader):
ELEMENTS = [
yc.KeyedElem('spack',
elements=[
yc.KeyedElem('config',
elements=[
yc.StrElem('install_tree'),
yc.StrElem('build_jobs', default=6),
yc.StrElem('install_path_scheme')
]),
yc.CategoryElem('mirrors', sub_elem=yc.StrElem()),
yc.ListElem('repos', sub_elem=yc.StrElem()),
yc.CategoryElem(
'upstreams',
sub_elem=yc.KeyedElem(
elements=[yc.StrElem('install_tree')])),
],
help_text='Spack environment configuration file.')
]
def get_config_items():
"""Get the config for this result parser. This should be a list of
yaml_config.ConfigElement instances that will be added to the test
config format at plugin activation time. The simplest format is a
list of yaml_config.StrElem objects, but any structure is allowed
as long as the leaf elements are StrElem type.
The config values will be passed as the keyword arguments to the
result parser when it's run and when its arguments are checked. The base
implementation provides several arguments that must be present for every result
parser. See the implementation of this method in result_parser.py for more
info on those arguments and what they do.
Example: ::
config_items = super().get_config_items()
config_items.append(
yaml_config.StrElem('token', default='PASSED',
help="The token to search for in the file."
)
return config_items
"""
return [
yc.StrElem(
"action",
required=True, default="store",
choices=[
ACTION_STORE,
ACTION_TRUE,
ACTION_FALSE,
ACTION_COUNT
],
help_text=(
"What to do with parsed results.\n"
" {STORE} - Just store the result (default).\n"
" {TRUE} - Store True if there was a result.\n"
" {FALSE} - Store True for no result.\n"
" {COUNT} - Count the number of results.\n"
.format(
STORE=ACTION_STORE,
TRUE=ACTION_TRUE,
FALSE=ACTION_FALSE,
COUNT=ACTION_COUNT))
),
# The default for the file is handled by the test object.
yc.ListElem(
"files",
required=True,
sub_elem=yc.StrElem(),
defaults=['../run.log'],
help_text="Path to the file/s that this result parser "
"will examine. Each may be a file glob,"
"such as '*.log'"),
yc.StrElem(
"per_file",
default=PER_FIRST,
required=True,
choices=[
PER_FIRST,
PER_LAST,
PER_FULLNAME,
PER_NAME,
PER_NAME_LIST,
PER_FULLNAME_LIST,
PER_LIST,
PER_ANY,
PER_ALL,
],
help_text=(
"How to save results for multiple file matches.\n"
" {FIRST} - The result from the first file with a "
"non-empty result. (default)\n"
" {LAST} - As '{FIRST}', but last result.\n"
" {FULLNAME} - Store the results on a per file "
"basis under results['fn'][<filename>][<key>]\n"
" {NAME} - As '{FULLNAME}', except use the "
"filename minus extension (foo.bar.log -> foo.bar), and"
"store under the results['n'][<name>][<key>]\n"
" {FULLNAME_LIST} - Save the matching file names, rather\n"
"than the parsed values, in a list.\n"
" {NAME_LIST} - As '{FULLNAME_LIST}' except store the \n"
"filenames minus the extension.\n"
" {LIST} - Merge all each result and result list "
"into a single list.\n"
" {ALL} - Use only with the 'store_true' or "
"'store_false' action. Set true if all files had a "
"true result. Note that 0 is a false result.\n"
" {ANY} - As '{ALL}', but set true if any file had"
"a true result.\n"
.format(
FIRST=PER_FIRST,
LAST=PER_LAST,
FULLNAME=PER_FULLNAME,
NAME=PER_NAME,
NAME_LIST=PER_NAME_LIST,
FULLNAME_LIST=PER_FULLNAME_LIST,
LIST=PER_LIST,
ALL=PER_ALL,
ANY=PER_ANY))
),
]
class ResultParser(IPlugin.IPlugin):
"""Base class for creating a result parser plugin. These are essentially
a callable that implements operations on the test or test files. The
arguments for the callable are provided automatically via the test
config. The doc string of result parser classes is used as the user help
text for that class, along with the help from the config items."""
PRIO_CORE = 0
PRIO_COMMON = 10
PRIO_USER = 20
FORCE_DEFAULTS = []
"""Let the user know they can't set these config keys for this result
parser, effectively forcing the value to the default."""
def __init__(self,
name,
description,
defaults=None,
config_elems=None,
validators=None,
priority=PRIO_COMMON):
"""Initialize the plugin object
:param str name: The name of this plugin.
:param str description: A short description of this result parser.
:param List[yaml_config.ConfigElement] config_elems: A list of configuration
elements (from the yaml_config library) to use to define the
config section for this result parser. These will be passed as arguments
to the parser function. Only StrElem and ListElems are accepted. Any type
conversions should be done with validators. Use the defaults and validators
argument to set defaults, rather than the YamlConfig options.
:param dict defaults: A dictionary of defaults for the result parser's
arguments.
:param dict validators: A dictionary of auto-validators. These can
take several forms:
- A tuple - The value must be one of the items in the tuple.
- a function - The function should accept a single argument. The
returned value is used. ValueError or ResultError should be raised
if there are issues. Typically this will be a type conversion function.
For list arguments this is applied to each of the list values.
:param int priority: The priority of this plugin, compared to plugins
of the same name. Higher priority plugins will supersede others.
"""
self.name = name
self.description = description
self.defaults = {} if defaults is None else defaults
self.validators = BASE_VALIDATORS.copy()
if validators is not None:
for key, validator in validators.items():
if not (isinstance(validator, tuple) or callable(validator)):
raise RuntimeError(
"Validator for key {} in result parser {} at {} must "
"be a tuple or a function.".format(
key, name, self.path))
validators[key] = validator
config_elems = config_elems if config_elems is not None else []
for elem in config_elems:
if not (isinstance(elem, yc.StrElem) or
(isinstance(elem, yc.ListElem)
and isinstance(elem._sub_elem, yc.StrElem))):
raise RuntimeError(
"Config elements for result parsers must be strings"
"or lists of strings. Got elem {} in parser at {}".format(
elem, self.path))
if elem.default not in [None, []] or elem._choices is not None:
raise RuntimeError(
"Config elements for result parsers shouldn't set "
"a default or choices (use the defaults or validators"
"argument for the result_parser init). Problem found "
"in {} in result parser at {}".format(elem, self.path))
self.config_elems = config_elems
self.priority = priority
super().__init__()
def __call__(self, file, **kwargs):
"""This is where the result parser is actually implemented.
:param pavilion.test_run.TestRun test: The test run object.
:param file: This will be a file object advanced to a position that fits
the criteria of the 'preceded_by' and 'for_lines_matching' options.
:param dict kwargs: The arguments are the config values from the the
test's result config section for this parser. These should be
explicitly defined in your result parser class.
:raises ResultParserError: When something goes wrong.
"""
raise NotImplementedError("A result parser plugin must implement"
"the __call__ method.")
def _check_args(self, **kwargs) -> dict:
"""Override this to add custom checking of the arguments at test
kickoff time. This prevents errors in your arguments from causing
a problem in the middle of a test run. The yaml_config module handles
structural checking (and can handle more). This should raise a
descriptive ResultParserError if any issues are found.
:param kwargs: Child result parsers should override these with
specific kwargs for their arguments. They should all default to
and rely on the config parser to set their defaults.
:raises ResultParserError: If there are bad arguments.
"""
_ = self
return kwargs
def check_args(self, **kwargs) -> dict:
"""Check the arguments for any errors at test kickoff time, if they
don't contain deferred variables. We can't check tests with
deferred args. On error, should raise a ResultParserError.
:param dict kwargs: The arguments from the config.
:raises ResultError: When bad arguments are given.
"""
# The presence or absence of needed args should be enforced by
# setting 'required' in the yaml_config config items.
args = self.defaults.copy()
for key, val in kwargs.items():
if val is None or (key in args and val == []):
continue
args[key] = kwargs[key]
kwargs = args
base_keys = ('action', 'per_file', 'files', 'match_select',
'for_lines_matching', 'preceded_by')
for key in base_keys:
if key not in kwargs:
raise RuntimeError(
"Result parser '{}' missing required attribute '{}'. These "
"are validated at the config level, so something is "
"probably wrong with plugin.".format(self.name, key))
match_select = MATCH_CHOICES.get(kwargs['match_select'],
kwargs['match_select'])
if match_select is not None:
try:
int(match_select)
except ValueError:
raise ResultError(
"Invalid value for 'match_select'. Must be one of "
"{} or an integer.")
for arg in self.FORCE_DEFAULTS:
if kwargs[arg] != self._DEFAULTS[arg]:
raise ResultError(
"This parser requires that you not set the '{}' key, as "
"the default value is the only valid option.".format(arg))
if kwargs['for_lines_matching'] is None and not kwargs['preceded_by']:
raise ResultError(
"At least one of 'for_lines_matching' or 'preceded_by' "
"must be set. You should only see this if the default for "
"'for_lines_matching' was explicitly set to null.")
for key, validator in self.validators.items():
if isinstance(validator, tuple):
if kwargs[key] not in validator:
raise ResultError("Invalid value for option '{}'.\n"
"Expected one of {}, got '{}'.".format(
key, validator, kwargs[key]))
else:
# Must be a validator function.
value = kwargs[key]
try:
if isinstance(value, list):
kwargs[key] = [validator(val) for val in value]
else:
kwargs[key] = validator(value)
except ValueError as err:
raise ResultError("Validation error for option '{}' with "
"value '{}'.\n{}".format(
key, kwargs[key], err.args[0]))
for key in base_keys:
# The parser plugins don't know about these keys, as they're
# handled at a higher level.
del kwargs[key]
return self._check_args(**kwargs)
GLOBAL_CONFIG_ELEMS = [
yc.StrElem(
"action",
help_text=(
"What to do with parsed results.\n"
"{STORE} - Just store the result automatically converting \n"
" the value's type(default).\n"
"{STORE_STR} - Just store the value, with no type "
"conversion.\n"
"{TRUE} - Store True if there was a result.\n"
"{FALSE} - Store True for no result.\n"
"{COUNT} - Count the number of results.\n".format(
STORE=ACTION_STORE,
STORE_STR=ACTION_STORE_STR,
TRUE=ACTION_TRUE,
FALSE=ACTION_FALSE,
COUNT=ACTION_COUNT))),
# The default for the file is handled by the test object.
yc.ListElem("files",
sub_elem=yc.StrElem(),
help_text="Path to the file/s that this result parser "
"will examine. Each may be a file glob,"
"such as '*.log'"),
yc.StrElem(
"per_file",
help_text=(
"How to save results for multiple file matches.\n"
"{FIRST} - (default) The result from the first file with a \n"
" non-empty result. If no files were found, this is \n"
" considerd an error. (default)\n"
"{LAST} - As '{FIRST}', but last result.\n"
"{NAME} - Store the results on a per file basis under \n"
" results['per_name'][<filename>][<key>]. The \n"
" filename only includes the parts before any '.', and \n"
" is normalized to replace non-alphanum characters with '_'\n"
"{NAME_LIST} - Save the matching (normalized) file \n"
" names, rather than the parsed values, in a list.\n"
"{LIST} - Merge all each result and result list \n"
" into a single list.\n"
"{ALL} - Set 'True' if all files found at least one match.\n"
"{ANY} - Set 'True' if any file found at lest one match.\n".
format(
FIRST=PER_FIRST,
LAST=PER_LAST,
NAME=PER_NAME,
NAME_LIST=PER_NAME_LIST,
LIST=PER_LIST,
ANY=PER_ANY,
ALL=PER_ALL,
))),
yc.StrElem(
"for_lines_matching",
help_text=(
"A regular expression used to identify the line where "
"result parsing should start. The result parser will "
"see the file starting at this line. Defaults to matching "
"every line.")),
yc.ListElem(
"preceded_by",
sub_elem=yc.StrElem(),
help_text=(
"A list of regular expressions that must match lines that "
"precede the lines to be parsed. Empty items"
"in the sequence will match anything. The result parser "
"will see the file starting from start of the line after "
"these match.")),
yc.StrElem(
"match_select",
help_text=(
"In cases where multiple matches are possible, how to"
"handle them. By default, find the first '{FIRST}' "
"match and use it. '{LAST}' returns the final match, and "
"'{ALL}' will return a list of all matches. You may also "
"give an integer to get the Nth match (starting at 0). "
"Negative integers (starting at -1)count in reverse.".format(
FIRST=MATCH_FIRST, LAST=MATCH_LAST, ALL=MATCH_ALL))),
]
def get_config_items(self):
"""Get the config for this result parser. This should be a list of
yaml_config.ConfigElement instances that will be added to the test
config format at plugin activation time. The simplest format is a
list of yaml_config.StrElem objects, but any structure is allowed
as long as the leaf elements are StrElem type.
The config values will be passed as the keyword arguments to the
result parser when it's run and when its arguments are checked. The base
implementation provides several arguments that must be present for every result
parser. See the implementation of this method in result_parser.py for more
info on those arguments and what they do.
Example: ::
config_items = super().get_config_items()
config_items.append(
yaml_config.StrElem('token', default='PASSED',
help="The token to search for in the file."
)
return config_items
"""
config_items = self.GLOBAL_CONFIG_ELEMS.copy()
config_items.extend(self.config_elems)
return config_items
@property
def path(self):
"""The path to the file containing this result parser plugin."""
return inspect.getfile(self.__class__)
def doc(self):
"""Return documentation on this result parser."""
def wrap(text: str, indent=0):
"""Wrap a multi-line string."""
indent = ' ' * indent
out_lines = []
for line in text.splitlines():
out_lines.extend(
textwrap.wrap(line,
initial_indent=indent,
subsequent_indent=indent))
return out_lines
doc = [self.name, '-' * len(self.name)]
doc.extend(wrap(self.description))
doc.append('\n{} Parser Specific Arguments'.format(self.name))
doc.append('-' * len(doc[-1]))
args = self.get_config_items()
specific_args = [arg for arg in args if arg.name not in self._DEFAULTS]
def add_arg_doc(arg):
"""Add an arg to the documentation."""
doc.append(' ' + arg.name)
doc.extend(wrap(arg.help_text, indent=4))
if arg.name in self.defaults:
doc.extend(
wrap("default: '{}'".format(self.defaults[arg.name]),
indent=4))
if arg.name in self.validators:
validator = self.validators[arg.name]
if isinstance(validator, tuple):
doc.extend(wrap('choices: ' + str(validator), indent=4))
doc.append('')
for arg in specific_args:
if arg.name not in self._DEFAULTS:
add_arg_doc(arg)
doc.append('Universal Parser Arguments')
doc.append('-' * len(doc[-1]))
for arg in args:
if arg.name in self._DEFAULTS:
add_arg_doc(arg)
return '\n'.join(doc)
_DEFAULTS = {
'per_file': PER_FIRST,
'action': ACTION_STORE,
'files': ['../run.log'],
'match_select': MATCH_FIRST,
'for_lines_matching': '',
'preceded_by': [],
}
"""Defaults for the common parser arguments. This in not meant to be
changed by subclasses."""
def set_parser_defaults(self, rconf: dict, def_conf: dict):
"""Set the default values for each result parser. The default conf
can hold defaults that apply across an entire result parser."""
base = self._DEFAULTS.copy()
for key, val in def_conf.items():
if val not in (None, []):
base[key] = val
for key, val in rconf.items():
if val is None:
continue
if key in base and val == []:
continue
base[key] = val
return base
def check_config(self, rconf: dict, keys: List[str]) -> None:
"""Validate the parser configuration.
:param rconf: The results parser configuration.
:param keys: The keys (generally one) under which the parsed results
will be stored.
:raises: ResultError on failure.
"""
action = rconf.get('action')
per_file = rconf.get('per_file')
found_deferred = False
# Don't check args if they have deferred values.
for option, values in rconf.items():
if not isinstance(values, list):
values = [values]
for value in values:
if resolver.TestConfigResolver.was_deferred(value):
found_deferred = True
if found_deferred:
# We can't continue checking from this point if anything
# was deferred.
return
if ('result' in keys and action not in (ACTION_TRUE, ACTION_FALSE)
and per_file not in (PER_FIRST, PER_LAST)):
raise ResultError(
"Result parser has key 'result', but must store a "
"boolean. Use action '{}' or '{}', along with a "
"per_file setting of 'first', 'last', 'any', or 'all'".format(
ACTION_TRUE, ACTION_FALSE))
try:
self.check_args(**rconf)
except ResultError as err:
raise ResultError("Key '{}': {}".format(keys, err.args[0]))
def activate(self):
"""Yapsy runs this when adding the plugin.
In this case it:
- Adds the config section (from get_config_items()) to the test config
format.
- Adds the result parser to the list of known result parsers.
"""
config_items = self.get_config_items()
file_format.TestConfigLoader.add_result_parser_config(
self.name, config_items)
if self.name in _RESULT_PARSERS:
other = _RESULT_PARSERS[self.name]
if self.priority > other.priority:
LOGGER.info("Result parser '%s' at %s is superseded by %s.",
self.name, other.path, self.path)
_RESULT_PARSERS[self.name] = self
elif self.priority < other.priority:
LOGGER.info(
"Result parser '%s' at %s is ignored in lieu of %s.",
self.name, self.path, other.path)
else:
raise RuntimeError("Result parser conflict. Parser '{}' at {}"
"has the same priority as {}".format(
self.name, other.path, self.path))
else:
_RESULT_PARSERS[self.name] = self
def deactivate(self):
"""Yapsy calls this to remove this plugin. We only ever
do this in unittests."""
# Remove the section from the config.
file_format.TestConfigLoader.remove_result_parser_config(self.name)
# Remove from list of available result parsers.
del _RESULT_PARSERS[self.name]
def __init__(self, name=None, **kwargs):
"""Just like a CategoryElem, but we force some of the params."""
super(VariableElem, self).__init__(name=name,
sub_elem=yc.StrElem(),
defaults=None,
**kwargs)
class DessertConfig(yc.YamlConfigLoader):
ELEMENTS = [yc.KeyedElem('pie', elements=[yc.StrElem('fruit')])]
class TestConfigLoader(yc.YamlConfigLoader):
"""This class describes a test section in a Pavilion config file. It is
expected to be added to by various plugins."""
ELEMENTS = [
yc.RegexElem(
'inherits_from',
regex=r'\w+',
help_text="Inherit from the given test section, and override "
"parameters those specified in this one. Lists are "
"overridden entirely"),
yc.StrElem(
'subtitle',
help_text="An extended title for this test. This is useful for "
"assigning unique name to virtual tests through "
"variable insertion. example, if a test has a single "
"permutation variable 'subtest', then '{subtest}' "
"would give a useful descriptor."),
VarCatElem('variables',
sub_elem=yc.ListElem(sub_elem=VariableElem()),
help_text="Variables for this test section. These can be "
"inserted strings anywhere else in the config through "
"the string syntax. They keys 'var', 'per', 'pav', "
"'sys' and 'sched' reserved. Each value may be a "
"single or list of strings key/string pairs."),
VarCatElem(
'permutations',
sub_elem=yc.ListElem(sub_elem=VariableElem()),
help_text="Permutation variables for this test section. These are "
"just like normal variables, but they if a list of "
"values (whether a single string or key/string pairs) "
"is given, then a virtual test is created for each "
"combination across all variables in each section. The "
"resulting virtual test is thus given a single "
"permutation of these values."),
yc.RegexElem('scheduler',
regex=r'\w+',
help_text="The scheduler class to use to run this test."),
yc.KeyedElem(
'build',
elements=[
yc.StrElem(
'source_location',
help_text="Path to the test source. It may be a directory, "
"a tar file, or a URI. If it's a directory or "
"file, the path is to '$PAV_CONFIG/test_src' by "
"default. For url's, the is automatically checked "
"for updates every time the test run. Downloaded "
"files are placed in a 'downloads' under the "
"pavilion working directory. (set in pavilion.yaml)"),
yc.StrElem(
'source_download_name',
help_text='When downloading source, we by default use the '
'last of the url path as the filename, or a hash '
'of the url if is no suitable name. Use this '
'parameter to override behavior with a pre-defined '
'filename.'),
yc.ListElem(
'modules',
sub_elem=yc.StrElem(),
help_text="Modules to load into the build environment."),
yc.CategoryElem(
'env',
sub_elem=yc.StrElem(),
help_text="Environment variables to set in the build "
"environment."),
yc.ListElem(
'extra_files',
sub_elem=yc.StrElem(),
help_text='Files to copy into the build environment. '
'Relative paths searched for in ~/.pavilion, '
'$PAV_CONFIG. Absolute paths are ok, '
'but not recommended.'),
yc.StrElem(
'specificity',
help_text="Use this string, along with variables, to "
"differentiate builds. A common example would be "
"to make per-host specific by using the "
"sys.sys_name variable. Note _deferred_ system "
"variables aren't a good idea hereas configs are "
"compiled on the host that launches the test."),
yc.ListElem(
'cmds',
sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to perform '
'the build.')
],
help_text="The test build configuration. This will be used to "
"dynamically generate a build script for building "
"the test."),
yc.KeyedElem(
'run',
elements=[
yc.ListElem(
'modules',
sub_elem=yc.StrElem(),
help_text="Modules to load into the run environment."),
yc.CategoryElem(
'env',
sub_elem=yc.StrElem(),
help_text="Environment variables to set in the run "
"environment."),
yc.ListElem(
'cmds',
sub_elem=yc.StrElem(),
help_text='The sequence of commands to run to run the '
'test.')
],
help_text="The test run configuration. This will be used "
"to dynamically generate a run script for the "
"test."),
]
# We'll append the result parsers separately, to have an easy way to
# access it.
_RESULT_PARSERS = yc.KeyedElem(
'result',
elements=[],
help_text="Result parser configurations go here. Each parser config "
"can occur by itself or as a list of configs, in which "
"case the parser will run once for each config given. The "
"output of these parsers will be combined into the final "
"result json data.")
ELEMENTS.append(_RESULT_PARSERS)
@classmethod
def add_subsection(cls, subsection):
"""Use this method to add additional sub-sections to the config.
:param yc.ConfigElem subsection: A yaml config element to add. Keyed
elements are expected, though any ConfigElem based instance
(whose leave elements are StrElems) should work.
"""
if not isinstance(subsection, yc.ConfigElement):
raise ValueError("Tried to add a subsection to the config, but it "
"wasn't a yaml_config ConfigElement instance (or "
"an instance of a ConfigElement child "
"class).")
name = subsection.name
names = [el.name for el in cls.ELEMENTS]
if name in names:
raise ValueError("Tried to add a subsection to the config called "
"{0}, but one already exists.".format(name))
try:
cls.check_leaves(subsection)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls.ELEMENTS.append(subsection)
@classmethod
def remove_subsection(cls, subsection_name):
"""Remove a subsection from the config. This is really only for use
in plugin deactivate methods."""
for section in list(cls.ELEMENTS):
if subsection_name == section.name:
cls.ELEMENTS.remove(section)
return
@classmethod
def add_result_parser_config(cls, name, config_items):
"""Add the given list of config items as a result parser
configuration named 'name'. Throws errors for invalid configuraitons.
"""
config = yc.KeyedElem('result_parser_{}'.format(name),
elements=config_items)
list_elem = yc.ListElem(name, sub_elem=config)
if name in [e.name for e in cls._RESULT_PARSERS.config_elems.values()]:
raise ValueError(
"Tried to add result parser with name '{}'"
"to the config, but one already exists.".format(name))
try:
cls.check_leaves(config)
except ValueError as err:
raise ValueError("Tried to add result parser named '{}', but "
"leaf element '{}' was not string based.".format(
name, err.args[0]))
cls._RESULT_PARSERS.config_elems[name] = list_elem
@classmethod
def remove_result_parser_config(cls, name):
"""Remove the given result parser from the result parser configuration
section.
:param str name: The name of the parser to remove.
:return:
"""
for section in list(cls._RESULT_PARSERS.config_elems.values()):
if section.name == name:
del cls._RESULT_PARSERS.config_elems[section.name]
return
@classmethod
def check_leaves(cls, elem):
"""
:param yc.ConfigElement elem:
:return:
"""
if hasattr(elem, 'config_elems'):
for sub_elem in elem.config_elems.values():
cls.check_leaves(sub_elem)
elif hasattr(elem, '_sub_elem') and elem._sub_elem is not None:
cls.check_leaves(elem._sub_elem)
elif issubclass(elem.type, str):
return
else:
raise ValueError(elem)
def _get_config_elems():
return [
yc.StrElem('num_nodes',
default="1",
help_text="Number of nodes requested for this test. "
"This can be a range (e.g. 12-24)."),
yc.StrElem('tasks_per_node',
default="1",
help_text="Number of tasks to run per node."),
yc.StrElem(
'mem_per_node',
help_text="The minimum amount of memory required in GB. "
"This can be a range (e.g. 64-128)."),
yc.StrElem('partition',
default="standard",
help_text="The partition that the test should be run "
"on."),
yc.StrElem(
'immediate',
choices=['true', 'false', 'True', 'False'],
default='false',
help_text="Only consider nodes not currently running jobs"
"when determining job size. Will set the minimum"
"number of nodes "),
yc.StrElem('qos', help_text="The QOS that this test should use."),
yc.StrElem('account',
help_text="The account that this test should run "
"under."),
yc.StrElem('reservation',
help_text="The reservation that this test should "
"run under."),
yc.RegexElem(
'time_limit',
regex=r'^(\d+-)?(\d+:)?\d+(:\d+)?$',
help_text="The time limit to specify for the slurm job in"
"the formats accepted by slurm "
"(<hours>:<minutes> is typical)"),
yc.RegexElem('include_nodes',
regex=Slurm.NODE_LIST_RE,
help_text="The nodes to include, in the same format "
"that Slurm expects with the -w or -x option. "
"This will automatically increase num_nodes to "
"at least this node count."),
yc.RegexElem(
'exclude_nodes',
regex=Slurm.NODE_LIST_RE,
help_text="A list of nodes to exclude, in the same format "
"that Slurm expects with the -w or -x option."),
yc.ListElem(name='avail_states',
sub_elem=yc.StrElem(),
defaults=['IDLE', 'MAINT'],
help_text="When looking for immediately available "
"nodes, they must be in one of these "
"states."),
yc.ListElem(name='up_states',
sub_elem=yc.StrElem(),
defaults=['ALLOCATED', 'COMPLETING', 'IDLE', 'MAINT'],
help_text="When looking for nodes that could be "
"allocated, they must be in one of these "
"states."),
yc.StrElem('job_name',
default="pav",
help_text="The job name for this test."),
]
