def test_parse_matchers__without_provider(self): unit = TestPlanUnit({}, provider=None) self.assertEqual( list(unit.parse_matchers("foo")), [(0, 'id', OperatorMatcher(operator.eq, 'foo'), None)]) self.assertEqual( list(unit.parse_matchers("other::bar")), [(0, 'id', OperatorMatcher(operator.eq, "other::bar"), None)]) self.assertEqual(list(unit.parse_matchers("sd[a-z]")), [(0, 'id', PatternMatcher("^sd[a-z]$"), None)]) self.assertEqual(list(unit.parse_matchers("sd[a-z]$")), [(0, 'id', PatternMatcher("^sd[a-z]$"), None)]) self.assertEqual(list(unit.parse_matchers("^sd[a-z]")), [(0, 'id', PatternMatcher("^sd[a-z]$"), None)]) self.assertEqual(list(unit.parse_matchers("^sd[a-z]$")), [(0, 'id', PatternMatcher("^sd[a-z]$"), None)])
def visit_IncludeStmt_node(self, node: IncludeStmt): if isinstance(node.pattern, ReErr): matcher = None error = node.pattern.exc elif isinstance(node.pattern, ReFixed): target_id = outer_self.qualify_id(node.pattern.text) matcher = OperatorMatcher(operator.eq, target_id) error = None elif isinstance(node.pattern, RePattern): text = node.pattern.text # Ensure that pattern is surrounded by ^ and $ if text.startswith('^') and text.endswith('$'): target_id_pattern = '^{}$'.format( outer_self.qualify_id(text[1:-1])) elif text.startswith('^'): target_id_pattern = '^{}$'.format( outer_self.qualify_id(text[1:])) elif text.endswith('$'): target_id_pattern = '^{}$'.format( outer_self.qualify_id(text[:-1])) else: target_id_pattern = '^{}$'.format( outer_self.qualify_id(text)) matcher = PatternMatcher(target_id_pattern) error = None result = (node.lineno, 'id', matcher, error) self.results.append(result)
def visit_IncludeStmt_node(self, node: IncludeStmt): if isinstance(node.pattern, ReFixed): target_id = testplan.qualify_id(node.pattern.text) matcher = OperatorMatcher(operator.eq, target_id) elif isinstance(node.pattern, RePattern): pattern = testplan.qualify_pattern(node.pattern.text) matcher = PatternMatcher(pattern) result = (node.lineno, 'id', matcher) results.append(result)
def get_bootstrap_qualifier(self, excluding=False): """ Convert this test plan to an equivalent qualifier for job selection """ qual_list = [] if self.bootstrap_include is not None: field_origin = self.origin.just_line().with_offset( self.field_offset_map['bootstrap_include']) qual_list = [FieldQualifier( 'id', OperatorMatcher(operator.eq, target_id), field_origin, not excluding) for target_id in self.get_bootstrap_job_ids()] for tp_unit in self.get_nested_part(): qual_list.extend([tp_unit.get_bootstrap_qualifier(excluding)]) return CompositeQualifier(qual_list)
def select_local_jobs(self): print(self.C.header(_("Selecting Job Generators"))) # Create a qualifier list that will pick all local jobs out of the # subset of jobs also enumerated by the whitelists we've already # picked. # # Since each whitelist is a qualifier that selects jobs enumerated # within, we only need to and an exclusive qualifier that deselects # non-local jobs and we're done. qualifier_list = [] qualifier_list.extend(self._qualifier_list) origin = Origin.get_caller_origin() qualifier_list.append(FieldQualifier( 'plugin', OperatorMatcher(operator.ne, 'local'), origin, inclusive=False)) local_job_list = select_jobs( self.manager.state.job_list, qualifier_list) self._update_desired_job_list(local_job_list)
def test_repr(self): self.assertEqual(repr(OperatorMatcher(operator.eq, "foo")), "OperatorMatcher(<built-in function eq>, 'foo')")
def test_match(self): matcher = OperatorMatcher(operator.eq, "foo") self.assertTrue(matcher.match("foo")) self.assertFalse(matcher.match("bar"))
class TestPlanUnitSupport: """ Helper class that distills test plan data into more usable form This class serves to offload some of the code from :class:`TestPlanUnit` branch. It takes a single test plan unit and extracts all the interesting information out of it. Subsequently it exposes that data so that some methods on the test plan unit class itself can be implemented in an easier way. The key data to handle are obviously the ``include`` and ``exclude`` fields. Those are used to come up with a qualifier object suitable for selecting jobs. The second key piece of data is obtained from the ``include`` field and from the ``category-overrides`` and ``certification-status-overrides`` fields. From those fields we come up with a data structure that can be applied to a list of jobs to compute their override values. Some examples of how that works, given this test plan: >>> testplan = TestPlanUnit({ ... 'include': ''' ... job-a certification_status=blocker, category-id=example ... job-b certification_status=non-blocker ... job-c ... ''', ... 'exclude': ''' ... job-[x-z] ... ''', ... 'category_overrides': ''' ... apply other-example to job-[bc] ... ''', ... 'certification_status_overrides': ''' ... apply not-part-of-certification to job-c ... ''', ... }) >>> support = TestPlanUnitSupport(testplan) We can look at the override list: >>> support.override_list ... # doctest: +NORMALIZE_WHITESPACE [('^job-[bc]$', [('category_id', 'other-example')]), ('^job-a$', [('certification_status', 'blocker'), ('category_id', 'example')]), ('^job-b$', [('certification_status', 'non-blocker')]), ('^job-c$', [('certification_status', 'not-part-of-certification')])] And the qualifiers: >>> support.qualifier # doctest: +NORMALIZE_WHITESPACE CompositeQualifier(qualifier_list=[FieldQualifier('id', \ OperatorMatcher(<built-in function eq>, 'job-a'), inclusive=True), FieldQualifier('id', \ OperatorMatcher(<built-in function eq>, 'job-b'), inclusive=True), FieldQualifier('id', \ OperatorMatcher(<built-in function eq>, 'job-c'), inclusive=True), FieldQualifier('id', \ PatternMatcher('^job-[x-z]$'), inclusive=False)]) """ def __init__(self, testplan): self.override_list = self._get_override_list(testplan) self.qualifier = self._get_qualifier(testplan) def _get_qualifier(self, testplan): qual_list = [] qual_list.extend( self._get_qualifier_for(testplan, 'include', True)) qual_list.extend( self._get_qualifier_for(testplan, 'exclude', False)) return CompositeQualifier(qual_list) def _get_qualifier_for(self, testplan, field_name, inclusive): field_value = getattr(testplan, field_name) if field_value is None: return [] field_origin = testplan.origin.just_line().with_offset( testplan.field_offset_map[field_name]) matchers_gen = self._get_matchers(testplan, field_value) results = [] for lineno_offset, matcher_field, matcher in matchers_gen: offset = field_origin.with_offset(lineno_offset) results.append( FieldQualifier(matcher_field, matcher, offset, inclusive)) return results def _get_matchers(self, testplan, text): """ Parse the specified text and create a list of matchers :param text: string of text, including newlines and comments, to parse :returns: A generator returning quads (lineno_offset, field, matcher, error) where ``lineno_offset`` is the offset of a line number from the start of the text, ``field`` is the name of the field in a job definition unit that the matcher should be applied, ``matcher`` can be None (then ``error`` is relevant) or one of the ``IMatcher`` subclasses discussed below. Supported matcher objects include: PatternMatcher: This matcher is created for lines of text that **are** regular expressions. The pattern is automatically expanded to include ^...$ (if missing) so that it cannot silently match a portion of a job definition OperatorMatcher: This matcher is created for lines of text that **are not** regular expressions. The matcher uses the operator.eq operator (equality) and stores the expected job identifier as the right-hand-side value """ results = [] class V(Visitor): def visit_IncludeStmt_node(self, node: IncludeStmt): if isinstance(node.pattern, ReFixed): target_id = testplan.qualify_id(node.pattern.text) matcher = OperatorMatcher(operator.eq, target_id) elif isinstance(node.pattern, RePattern): pattern = testplan.qualify_pattern(node.pattern.text) matcher = PatternMatcher(pattern) result = (node.lineno, 'id', matcher) results.append(result) V().visit(IncludeStmtList.parse(text, 0)) return results def _get_override_list( self, testplan: TestPlanUnit ) -> "List[Tuple[str, List[Tuple[str, str]]]]": """ Look at a test plan and compute the full (overall) override list. The list contains information about each job selection pattern (fully qualified pattern) to a list of pairs ``(field, value)`` that ought to be applied to a :class:`JobState` object. The code below ensures that each ``field`` is an existing attribute of the job state object. .. note:: The code below in *not* resilient to errors so make sure to validate the unit before starting with the helper. """ override_map = collections.defaultdict(list) # ^^ Dict[str, Tuple[str, str]] for pattern, field_value_list in self._get_inline_overrides(testplan): override_map[pattern].extend(field_value_list) for pattern, field, value in self._get_category_overrides(testplan): override_map[pattern].append((field, value)) for pattern, field, value in self._get_blocker_status_overrides( testplan): override_map[pattern].append((field, value)) return sorted((key, field_value_list) for key, field_value_list in override_map.items()) def _get_category_overrides( self, testplan: TestPlanUnit ) -> "List[Tuple[str, str, str]]]": """ Look at the category overrides and collect refined data about what overrides to apply. The result is represented as a list of tuples ``(pattern, field, value)`` where ``pattern`` is the string that describes the pattern, ``field`` is the field to which an override must be applied (but without the ``effective_`` prefix) and ``value`` is the overridden value. """ override_list = [] if testplan.category_overrides is None: return override_list class V(Visitor): def visit_FieldOverride_node(self, node: FieldOverride): category_id = testplan.qualify_id(node.value.text) pattern = r"^{}$".format( testplan.qualify_id(node.pattern.text)) override_list.append((pattern, 'category_id', category_id)) V().visit(OverrideFieldList.parse(testplan.category_overrides)) for tp_unit in testplan.get_nested_part(): override_list.extend(self._get_category_overrides(tp_unit)) return override_list def _get_blocker_status_overrides( self, testplan: TestPlanUnit ) -> "List[Tuple[str, str, str]]]": """ Look at the certification blocker status overrides and collect refined data about what overrides to apply. The result is represented as a list of tuples ``(pattern, field, value)`` where ``pattern`` is the string that describes the pattern, ``field`` is the field to which an override must be applied (but without the ``effective_`` prefix) and ``value`` is the overridden value. """ override_list = [] if testplan.certification_status_overrides is not None: class V(Visitor): def visit_FieldOverride_node(self, node: FieldOverride): blocker_status = node.value.text pattern = r"^{}$".format( testplan.qualify_id(node.pattern.text)) override_list.append( (pattern, 'certification_status', blocker_status)) V().visit(OverrideFieldList.parse( testplan.certification_status_overrides)) for tp_unit in testplan.get_nested_part(): override_list.extend(self._get_blocker_status_overrides(tp_unit)) return override_list def _get_inline_overrides( self, testplan: TestPlanUnit ) -> "List[Tuple[str, List[Tuple[str, str]]]]": """ Look at the include field of a test plan and collect all of the in-line overrides. For an include statement that has any overrides they are collected into a list of tuples ``(field, value)`` and this list is subsequently packed into a tuple ``(pattern, field_value_list)``. """ override_list = [] if testplan.include is not None: class V(Visitor): def visit_IncludeStmt_node(self, node: IncludeStmt): if not node.overrides: return pattern = r"^{}$".format( testplan.qualify_id(node.pattern.text)) field_value_list = [ (override_exp.field.text.replace('-', '_'), override_exp.value.text) for override_exp in node.overrides] override_list.append((pattern, field_value_list)) V().visit(IncludeStmtList.parse(testplan.include)) for tp_unit in testplan.get_nested_part(): override_list.extend(self._get_inline_overrides(tp_unit)) return override_list