def test_parse_matchers__without_provider(self):
unit = TestPlanUnit({}, provider=None)
self.assertEqual(
list(unit.parse_matchers("foo")),
[(0, 'id', OperatorMatcher(operator.eq, 'foo'), None)])
self.assertEqual(
list(unit.parse_matchers("other::bar")),
[(0, 'id', OperatorMatcher(operator.eq, "other::bar"), None)])
self.assertEqual(list(unit.parse_matchers("sd[a-z]")),
[(0, 'id', PatternMatcher("^sd[a-z]$"), None)])
self.assertEqual(list(unit.parse_matchers("sd[a-z]$")),
[(0, 'id', PatternMatcher("^sd[a-z]$"), None)])
self.assertEqual(list(unit.parse_matchers("^sd[a-z]")),
[(0, 'id', PatternMatcher("^sd[a-z]$"), None)])
self.assertEqual(list(unit.parse_matchers("^sd[a-z]$")),
[(0, 'id', PatternMatcher("^sd[a-z]$"), None)])
def visit_IncludeStmt_node(self, node: IncludeStmt):
if isinstance(node.pattern, ReErr):
matcher = None
error = node.pattern.exc
elif isinstance(node.pattern, ReFixed):
target_id = outer_self.qualify_id(node.pattern.text)
matcher = OperatorMatcher(operator.eq, target_id)
error = None
elif isinstance(node.pattern, RePattern):
text = node.pattern.text
# Ensure that pattern is surrounded by ^ and $
if text.startswith('^') and text.endswith('$'):
target_id_pattern = '^{}$'.format(
outer_self.qualify_id(text[1:-1]))
elif text.startswith('^'):
target_id_pattern = '^{}$'.format(
outer_self.qualify_id(text[1:]))
elif text.endswith('$'):
target_id_pattern = '^{}$'.format(
outer_self.qualify_id(text[:-1]))
else:
target_id_pattern = '^{}$'.format(
outer_self.qualify_id(text))
matcher = PatternMatcher(target_id_pattern)
error = None
result = (node.lineno, 'id', matcher, error)
self.results.append(result)
def visit_IncludeStmt_node(self, node: IncludeStmt):
if isinstance(node.pattern, ReFixed):
target_id = testplan.qualify_id(node.pattern.text)
matcher = OperatorMatcher(operator.eq, target_id)
elif isinstance(node.pattern, RePattern):
pattern = testplan.qualify_pattern(node.pattern.text)
matcher = PatternMatcher(pattern)
result = (node.lineno, 'id', matcher)
results.append(result)
def get_bootstrap_qualifier(self, excluding=False):
"""
Convert this test plan to an equivalent qualifier for job selection
"""
qual_list = []
if self.bootstrap_include is not None:
field_origin = self.origin.just_line().with_offset(
self.field_offset_map['bootstrap_include'])
qual_list = [FieldQualifier(
'id', OperatorMatcher(operator.eq, target_id), field_origin,
not excluding) for target_id in self.get_bootstrap_job_ids()]
for tp_unit in self.get_nested_part():
qual_list.extend([tp_unit.get_bootstrap_qualifier(excluding)])
return CompositeQualifier(qual_list)
def select_local_jobs(self):
print(self.C.header(_("Selecting Job Generators")))
# Create a qualifier list that will pick all local jobs out of the
# subset of jobs also enumerated by the whitelists we've already
# picked.
#
# Since each whitelist is a qualifier that selects jobs enumerated
# within, we only need to and an exclusive qualifier that deselects
# non-local jobs and we're done.
qualifier_list = []
qualifier_list.extend(self._qualifier_list)
origin = Origin.get_caller_origin()
qualifier_list.append(FieldQualifier(
'plugin', OperatorMatcher(operator.ne, 'local'), origin,
inclusive=False))
local_job_list = select_jobs(
self.manager.state.job_list, qualifier_list)
self._update_desired_job_list(local_job_list)
def test_repr(self):
self.assertEqual(repr(OperatorMatcher(operator.eq, "foo")),
"OperatorMatcher(<built-in function eq>, 'foo')")
def test_match(self):
matcher = OperatorMatcher(operator.eq, "foo")
self.assertTrue(matcher.match("foo"))
self.assertFalse(matcher.match("bar"))
class TestPlanUnitSupport:
"""
Helper class that distills test plan data into more usable form
This class serves to offload some of the code from :class:`TestPlanUnit`
branch. It takes a single test plan unit and extracts all the interesting
information out of it. Subsequently it exposes that data so that some
methods on the test plan unit class itself can be implemented in an easier
way.
The key data to handle are obviously the ``include`` and ``exclude``
fields. Those are used to come up with a qualifier object suitable for
selecting jobs.
The second key piece of data is obtained from the ``include`` field and
from the ``category-overrides`` and ``certification-status-overrides``
fields. From those fields we come up with a data structure that can be
applied to a list of jobs to compute their override values.
Some examples of how that works, given this test plan:
>>> testplan = TestPlanUnit({
... 'include': '''
... job-a certification_status=blocker, category-id=example
... job-b certification_status=non-blocker
... job-c
... ''',
... 'exclude': '''
... job-[x-z]
... ''',
... 'category_overrides': '''
... apply other-example to job-[bc]
... ''',
... 'certification_status_overrides': '''
... apply not-part-of-certification to job-c
... ''',
... })
>>> support = TestPlanUnitSupport(testplan)
We can look at the override list:
>>> support.override_list
... # doctest: +NORMALIZE_WHITESPACE
[('^job-[bc]$', [('category_id', 'other-example')]),
('^job-a$', [('certification_status', 'blocker'),
('category_id', 'example')]),
('^job-b$', [('certification_status', 'non-blocker')]),
('^job-c$', [('certification_status', 'not-part-of-certification')])]
And the qualifiers:
>>> support.qualifier # doctest: +NORMALIZE_WHITESPACE
CompositeQualifier(qualifier_list=[FieldQualifier('id', \
OperatorMatcher(<built-in function eq>, 'job-a'), inclusive=True),
FieldQualifier('id', \
OperatorMatcher(<built-in function eq>, 'job-b'), inclusive=True),
FieldQualifier('id', \
OperatorMatcher(<built-in function eq>, 'job-c'), inclusive=True),
FieldQualifier('id', \
PatternMatcher('^job-[x-z]$'), inclusive=False)])
"""
def __init__(self, testplan):
self.override_list = self._get_override_list(testplan)
self.qualifier = self._get_qualifier(testplan)
def _get_qualifier(self, testplan):
qual_list = []
qual_list.extend(
self._get_qualifier_for(testplan, 'include', True))
qual_list.extend(
self._get_qualifier_for(testplan, 'exclude', False))
return CompositeQualifier(qual_list)
def _get_qualifier_for(self, testplan, field_name, inclusive):
field_value = getattr(testplan, field_name)
if field_value is None:
return []
field_origin = testplan.origin.just_line().with_offset(
testplan.field_offset_map[field_name])
matchers_gen = self._get_matchers(testplan, field_value)
results = []
for lineno_offset, matcher_field, matcher in matchers_gen:
offset = field_origin.with_offset(lineno_offset)
results.append(
FieldQualifier(matcher_field, matcher, offset, inclusive))
return results
def _get_matchers(self, testplan, text):
"""
Parse the specified text and create a list of matchers
:param text:
string of text, including newlines and comments, to parse
:returns:
A generator returning quads (lineno_offset, field, matcher, error)
where ``lineno_offset`` is the offset of a line number from the
start of the text, ``field`` is the name of the field in a job
definition unit that the matcher should be applied,
``matcher`` can be None (then ``error`` is relevant) or one of
the ``IMatcher`` subclasses discussed below.
Supported matcher objects include:
PatternMatcher:
This matcher is created for lines of text that **are** regular
expressions. The pattern is automatically expanded to include
^...$ (if missing) so that it cannot silently match a portion of
a job definition
OperatorMatcher:
This matcher is created for lines of text that **are not** regular
expressions. The matcher uses the operator.eq operator (equality)
and stores the expected job identifier as the right-hand-side value
"""
results = []
class V(Visitor):
def visit_IncludeStmt_node(self, node: IncludeStmt):
if isinstance(node.pattern, ReFixed):
target_id = testplan.qualify_id(node.pattern.text)
matcher = OperatorMatcher(operator.eq, target_id)
elif isinstance(node.pattern, RePattern):
pattern = testplan.qualify_pattern(node.pattern.text)
matcher = PatternMatcher(pattern)
result = (node.lineno, 'id', matcher)
results.append(result)
V().visit(IncludeStmtList.parse(text, 0))
return results
def _get_override_list(
self, testplan: TestPlanUnit
) -> "List[Tuple[str, List[Tuple[str, str]]]]":
"""
Look at a test plan and compute the full (overall) override list. The
list contains information about each job selection pattern (fully
qualified pattern) to a list of pairs ``(field, value)`` that ought to
be applied to a :class:`JobState` object.
The code below ensures that each ``field`` is an existing attribute of
the job state object.
.. note::
The code below in *not* resilient to errors so make sure to
validate the unit before starting with the helper.
"""
override_map = collections.defaultdict(list)
# ^^ Dict[str, Tuple[str, str]]
for pattern, field_value_list in self._get_inline_overrides(testplan):
override_map[pattern].extend(field_value_list)
for pattern, field, value in self._get_category_overrides(testplan):
override_map[pattern].append((field, value))
for pattern, field, value in self._get_blocker_status_overrides(
testplan):
override_map[pattern].append((field, value))
return sorted((key, field_value_list)
for key, field_value_list in override_map.items())
def _get_category_overrides(
self, testplan: TestPlanUnit
) -> "List[Tuple[str, str, str]]]":
"""
Look at the category overrides and collect refined data about what
overrides to apply. The result is represented as a list of tuples
``(pattern, field, value)`` where ``pattern`` is the string that
describes the pattern, ``field`` is the field to which an override must
be applied (but without the ``effective_`` prefix) and ``value`` is the
overridden value.
"""
override_list = []
if testplan.category_overrides is None:
return override_list
class V(Visitor):
def visit_FieldOverride_node(self, node: FieldOverride):
category_id = testplan.qualify_id(node.value.text)
pattern = r"^{}$".format(
testplan.qualify_id(node.pattern.text))
override_list.append((pattern, 'category_id', category_id))
V().visit(OverrideFieldList.parse(testplan.category_overrides))
for tp_unit in testplan.get_nested_part():
override_list.extend(self._get_category_overrides(tp_unit))
return override_list
def _get_blocker_status_overrides(
self, testplan: TestPlanUnit
) -> "List[Tuple[str, str, str]]]":
"""
Look at the certification blocker status overrides and collect refined
data about what overrides to apply. The result is represented as a list
of tuples ``(pattern, field, value)`` where ``pattern`` is the string
that describes the pattern, ``field`` is the field to which an override
must be applied (but without the ``effective_`` prefix) and ``value``
is the overridden value.
"""
override_list = []
if testplan.certification_status_overrides is not None:
class V(Visitor):
def visit_FieldOverride_node(self, node: FieldOverride):
blocker_status = node.value.text
pattern = r"^{}$".format(
testplan.qualify_id(node.pattern.text))
override_list.append(
(pattern, 'certification_status', blocker_status))
V().visit(OverrideFieldList.parse(
testplan.certification_status_overrides))
for tp_unit in testplan.get_nested_part():
override_list.extend(self._get_blocker_status_overrides(tp_unit))
return override_list
def _get_inline_overrides(
self, testplan: TestPlanUnit
) -> "List[Tuple[str, List[Tuple[str, str]]]]":
"""
Look at the include field of a test plan and collect all of the in-line
overrides. For an include statement that has any overrides they are
collected into a list of tuples ``(field, value)`` and this list is
subsequently packed into a tuple ``(pattern, field_value_list)``.
"""
override_list = []
if testplan.include is not None:
class V(Visitor):
def visit_IncludeStmt_node(self, node: IncludeStmt):
if not node.overrides:
return
pattern = r"^{}$".format(
testplan.qualify_id(node.pattern.text))
field_value_list = [
(override_exp.field.text.replace('-', '_'),
override_exp.value.text)
for override_exp in node.overrides]
override_list.append((pattern, field_value_list))
V().visit(IncludeStmtList.parse(testplan.include))
for tp_unit in testplan.get_nested_part():
override_list.extend(self._get_inline_overrides(tp_unit))
return override_list
def test_match(self):
matcher = OperatorMatcher(operator.eq, "foo")
self.assertTrue(matcher.match("foo"))
self.assertFalse(matcher.match("bar"))
