def test_macro_expansion(self): converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS) configs_to_match = set( [TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release")] ) self.assertEqual(converter.to_config_set(set(["win", "release"])), configs_to_match) configs_to_match = set( [ TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("trusty", "x86_64", "release"), ] ) self.assertEqual(converter.to_config_set(set(["win", "trusty", "release"])), configs_to_match) configs_to_match = set( [ TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("snowleopard", "x86", "release"), ] ) self.assertEqual(converter.to_config_set(set(["win", "mac", "release"])), configs_to_match)
def test_symmetric_difference(self): self.assertEqual( TestConfigurationConverter.symmetric_difference([set(["a", "b"]), set(["b", "c"])]), set(["a", "c"]) ) self.assertEqual( TestConfigurationConverter.symmetric_difference([set(["a", "b"]), set(["b", "c"]), set(["b", "d"])]), set(["a", "c", "d"]), )
def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter( set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier
def test_macro_expansion(self): converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), ]) self.assertEqual(converter.to_config_set(set(['win', 'release'])), configs_to_match) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('linux32', 'x86', 'release'), TestConfiguration('trusty', 'x86_64', 'release'), ]) self.assertEqual(converter.to_config_set(set(['win', 'linux32', 'trusty', 'release'])), configs_to_match) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'release'), ]) self.assertEqual(converter.to_config_set(set(['win', 'mac', 'release'])), configs_to_match)
def test_macro_collapsing(self): macros = {'foo': ['bar', 'baz'], 'people': ['bob', 'alice', 'john']} specifiers_list = [set(['john', 'godzilla', 'bob', 'alice'])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(['people', 'godzilla'])]) specifiers_list = [set(['john', 'godzilla', 'alice'])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(['john', 'godzilla', 'alice', 'godzilla'])]) specifiers_list = [set(['bar', 'godzilla', 'baz', 'bob', 'alice', 'john'])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(['foo', 'godzilla', 'people'])]) specifiers_list = [set(['bar', 'godzilla', 'baz', 'bob']), set(['bar', 'baz']), set(['people', 'alice', 'bob', 'john'])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(['bob', 'foo', 'godzilla']), set(['foo']), set(['people'])])
def test_macro_collapsing(self): macros = {"foo": ["bar", "baz"], "people": ["bob", "alice", "john"]} specifiers_list = [set(["john", "godzilla", "bob", "alice"])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(["people", "godzilla"])]) specifiers_list = [set(["john", "godzilla", "alice"])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(["john", "godzilla", "alice", "godzilla"])]) specifiers_list = [set(["bar", "godzilla", "baz", "bob", "alice", "john"])] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(["foo", "godzilla", "people"])]) specifiers_list = [ set(["bar", "godzilla", "baz", "bob"]), set(["bar", "baz"]), set(["people", "alice", "bob", "john"]), ] TestConfigurationConverter.collapse_macros(macros, specifiers_list) self.assertEqual(specifiers_list, [set(["bob", "foo", "godzilla"]), set(["foo"]), set(["people"])])
def test_converter_macro_collapsing(self): converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS) configs_to_match = set( [TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release")] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "release"])]) configs_to_match = set( [ TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("precise", "x86_64", "release"), TestConfiguration("trusty", "x86_64", "release"), ] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "linux", "release"])]) configs_to_match = set( [ TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("snowleopard", "x86", "release"), ] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "mac", "release"])]) configs_to_match = set( [ TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("snowleopard", "x86", "release"), ] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "mac", "release"])]) configs_to_match = set( [TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release")] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "release"])])
def test_converter_macro_collapsing(self): converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'release'])]) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('linux32', 'x86', 'release'), TestConfiguration('precise', 'x86_64', 'release'), TestConfiguration('trusty', 'x86_64', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'linux', 'release'])]) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'mac', 'release'])]) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'mac', 'release'])]) configs_to_match = set([ TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'release'])])
class TestExpectationParser(object): """Provides parsing facilities for lines in the test_expectation.txt file.""" DUMMY_BUG_MODIFIER = "bug_dummy" BUG_MODIFIER_PREFIX = "bug" BUG_MODIFIER_REGEX = "bug\d+" REBASELINE_MODIFIER = "rebaseline" PASS_EXPECTATION = "pass" SKIP_MODIFIER = "skip" SLOW_MODIFIER = "slow" WONTFIX_MODIFIER = "wontfix" TIMEOUT_EXPECTATION = "timeout" MISSING_BUG_WARNING = "Test lacks BUG modifier." def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter( set(port.all_test_configurations()), port.configuration_specifier_macros() ) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier def parse(self, filename, expectations_string): expectation_lines = [] line_number = 0 for line in expectations_string.split("\n"): line_number += 1 test_expectation = self._tokenize_line(filename, line, line_number) self._parse_line(test_expectation) expectation_lines.append(test_expectation) return expectation_lines def expectation_for_skipped_test(self, test_name): if not self._port.test_exists(test_name): _log.warning("The following test %s from the Skipped list doesn't exist" % test_name) expectation_line = TestExpectationLine() expectation_line.original_string = test_name expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER] # FIXME: It's not clear what the expectations for a skipped test should be; the expectations # might be different for different entries in a Skipped file, or from the command line, or from # only running parts of the tests. It's also not clear if it matters much. expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER) expectation_line.name = test_name # FIXME: we should pass in a more descriptive string here. expectation_line.filename = "<Skipped file>" expectation_line.line_number = 0 expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION] self._parse_line(expectation_line) return expectation_line def _parse_line(self, expectation_line): if not expectation_line.name: return if not self._check_test_exists(expectation_line): return expectation_line.is_file = self._port.test_isfile(expectation_line.name) if expectation_line.is_file: expectation_line.path = expectation_line.name else: expectation_line.path = self._port.normalize_test_name(expectation_line.name) self._collect_matching_tests(expectation_line) self._parse_modifiers(expectation_line) self._parse_expectations(expectation_line) def _parse_modifiers(self, expectation_line): has_wontfix = False has_bugid = False parsed_specifiers = set() modifiers = [modifier.lower() for modifier in expectation_line.modifiers] expectations = [expectation.lower() for expectation in expectation_line.expectations] if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations: expectation_line.warnings.append( "A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT." ) for modifier in modifiers: if modifier in TestExpectations.MODIFIERS: expectation_line.parsed_modifiers.append(modifier) if modifier == self.WONTFIX_MODIFIER: has_wontfix = True elif modifier.startswith(self.BUG_MODIFIER_PREFIX): has_bugid = True if re.match(self.BUG_MODIFIER_REGEX, modifier): expectation_line.warnings.append( "BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier." ) else: expectation_line.parsed_bug_modifiers.append(modifier) else: parsed_specifiers.add(modifier) if ( not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations() ): expectation_line.warnings.append(self.MISSING_BUG_WARNING) if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers: expectation_line.warnings.append( "REBASELINE should only be used for running rebaseline.py. Cannot be checked in." ) expectation_line.matching_configurations = self._test_configuration_converter.to_config_set( parsed_specifiers, expectation_line.warnings ) def _parse_expectations(self, expectation_line): result = set() for part in expectation_line.expectations: expectation = TestExpectations.expectation_from_string(part) if expectation is None: # Careful, PASS is currently 0. expectation_line.warnings.append("Unsupported expectation: %s" % part) continue result.add(expectation) expectation_line.parsed_expectations = result def _check_test_exists(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. if not self._port.test_exists(expectation_line.name) and not self._port.test_exists( expectation_line.name + "-disabled" ): # Log a warning here since you hit this case any # time you update TestExpectations without syncing # the LayoutTests directory expectation_line.warnings.append("Path does not exist.") return False return True def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized path and make sure directories end with the OS path separator.""" # FIXME: full_test_list can quickly contain a big amount of # elements. We should consider at some point to use a more # efficient structure instead of a list. Maybe a dictionary of # lists to represent the tree of tests, leaves being test # files and nodes being categories. if not self._full_test_list: expectation_line.matching_tests = [expectation_line.path] return if not expectation_line.is_file: # this is a test category, return all the tests of the category. expectation_line.matching_tests = [ test for test in self._full_test_list if test.startswith(expectation_line.path) ] return # this is a test file, do a quick check if it's in the # full test suite. if expectation_line.path in self._full_test_list: expectation_line.matching_tests.append(expectation_line.path) # FIXME: Update the original modifiers and remove this once the old syntax is gone. _configuration_tokens_list = [ "Mac", "SnowLeopard", "Lion", "MountainLion", "Win", "XP", "Vista", "Win7", "Linux", "Android", "Release", "Debug", ] _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list) _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems()) # FIXME: Update the original modifiers list and remove this once the old syntax is gone. _expectation_tokens = { "Crash": "CRASH", "Failure": "FAIL", "ImageOnlyFailure": "IMAGE", "Missing": "MISSING", "Pass": "******", "Rebaseline": "REBASELINE", "Skip": "SKIP", "Slow": "SLOW", "Timeout": "TIMEOUT", "WontFix": "WONTFIX", } _inverted_expectation_tokens = dict( [(value, name) for name, value in _expectation_tokens.iteritems()] + [("TEXT", "Failure"), ("IMAGE+TEXT", "Failure"), ("AUDIO", "Failure")] ) # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser. @classmethod def _tokenize_line(cls, filename, expectation_string, line_number): """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format. The new format for a test expectation line is: [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] Any errant whitespace is not preserved. """ expectation_line = TestExpectationLine() expectation_line.original_string = expectation_string expectation_line.filename = filename expectation_line.line_number = line_number comment_index = expectation_string.find("#") if comment_index == -1: comment_index = len(expectation_string) else: expectation_line.comment = expectation_string[comment_index + 1 :] remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) if len(remaining_string) == 0: return expectation_line # special-case parsing this so that we fail immediately instead of treating this as a test name if remaining_string.startswith("//"): expectation_line.warnings = ['use "#" instead of "//" for comments'] return expectation_line bugs = [] modifiers = [] name = None expectations = [] warnings = [] WEBKIT_BUG_PREFIX = "webkit.org/b/" CHROMIUM_BUG_PREFIX = "crbug.com/" V8_BUG_PREFIX = "code.google.com/p/v8/issues/detail?id=" tokens = remaining_string.split() state = "start" for token in tokens: if ( token.startswith(WEBKIT_BUG_PREFIX) or token.startswith(CHROMIUM_BUG_PREFIX) or token.startswith(V8_BUG_PREFIX) or token.startswith("Bug(") ): if state != "start": warnings.append('"%s" is not at the start of the line.' % token) break if token.startswith(WEBKIT_BUG_PREFIX): bugs.append(token.replace(WEBKIT_BUG_PREFIX, "BUGWK")) elif token.startswith(CHROMIUM_BUG_PREFIX): bugs.append(token.replace(CHROMIUM_BUG_PREFIX, "BUGCR")) elif token.startswith(V8_BUG_PREFIX): bugs.append(token.replace(V8_BUG_PREFIX, "BUGV8_")) else: match = re.match("Bug\((\w+)\)$", token) if not match: warnings.append('unrecognized bug identifier "%s"' % token) break else: bugs.append("BUG" + match.group(1).upper()) elif token.startswith("BUG"): warnings.append('unrecognized old-style bug identifier "%s"' % token) break elif token == "[": if state == "start": state = "configuration" elif state == "name_found": state = "expectations" else: warnings.append('unexpected "["') break elif token == "]": if state == "configuration": state = "name" elif state == "expectations": state = "done" else: warnings.append('unexpected "]"') break elif token in ("//", ":", "="): warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token) break elif state == "configuration": modifiers.append(cls._configuration_tokens.get(token, token)) elif state == "expectations": if token in ("Rebaseline", "Skip", "Slow", "WontFix"): modifiers.append(token.upper()) elif token not in cls._expectation_tokens: warnings.append('Unrecognized expectation "%s"' % token) else: expectations.append(cls._expectation_tokens.get(token, token)) elif state == "name_found": warnings.append('expecting "[", "#", or end of line instead of "%s"' % token) break else: name = token state = "name_found" if not warnings: if not name: warnings.append("Did not find a test name.") elif state not in ("name_found", "done"): warnings.append('Missing a "]"') if "WONTFIX" in modifiers and "SKIP" not in modifiers and not expectations: modifiers.append("SKIP") if "SKIP" in modifiers and expectations: # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax. warnings.append("A test marked Skip must not have other expectations.") elif not expectations: if "SKIP" not in modifiers and "REBASELINE" not in modifiers and "SLOW" not in modifiers: modifiers.append("SKIP") expectations = ["PASS"] # FIXME: expectation line should just store bugs and modifiers separately. expectation_line.modifiers = bugs + modifiers expectation_line.expectations = expectations expectation_line.name = name expectation_line.warnings = warnings return expectation_line @classmethod def _split_space_separated(cls, space_separated_string): """Splits a space-separated string into an array.""" return [part.strip() for part in space_separated_string.strip().split(" ")]
def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier
def test_to_specifier_lists(self): converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS) self.assertEqual(converter.to_specifiers_list(set(self._all_test_configurations)), [[]]) self.assertEqual(converter.to_specifiers_list(set()), []) configs_to_match = set([TestConfiguration("win7", "x86", "release")]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["release", "win7"])]) configs_to_match = set([TestConfiguration("win7", "x86", "release"), TestConfiguration("win7", "x86", "debug")]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win7"])]) configs_to_match = set( [ TestConfiguration("precise", "x86_64", "debug"), TestConfiguration("trusty", "x86_64", "debug"), TestConfiguration("win7", "x86", "release"), ] ) self.assertEqual( converter.to_specifiers_list(configs_to_match), [set(["release", "win7"]), set(["debug", "linux"])] ) configs_to_match = set( [ TestConfiguration("win7", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("trusty", "x86_64", "debug"), TestConfiguration("precise", "x86_64", "debug"), TestConfiguration("trusty", "x86_64", "debug"), TestConfiguration("precise", "x86_64", "debug"), ] ) self.assertEqual( converter.to_specifiers_list(configs_to_match), [set(["release", "win7"]), set(["debug", "linux"])] ) configs_to_match = set( [ TestConfiguration("win7", "x86", "release"), TestConfiguration("snowleopard", "x86", "release"), TestConfiguration("vista", "x86", "release"), TestConfiguration("precise", "x86_64", "release"), TestConfiguration("trusty", "x86_64", "release"), ] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["release"])]) configs_to_match = set( [TestConfiguration("win7", "x86", "release"), TestConfiguration("snowleopard", "x86", "release")] ) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win7", "mac", "release"])]) configs_to_match = set( [ TestConfiguration("snowleopard", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("win7", "x86", "debug"), TestConfiguration("trusty", "x86_64", "release"), ] ) self.assertEqual( converter.to_specifiers_list(configs_to_match), [set(["win7"]), set(["release", "mac", "trusty"])] )
def execute(self, options, args, tool): port = tool.port_factory.get("chromium-win-win7") # FIXME: This should be selectable. parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False) expectation_lines = parser.parse(port.test_expectations()) converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros()) tool.filesystem.write_text_file(port.path_to_test_expectations_file(), TestExpectationSerializer.list_to_string(expectation_lines, converter))
def test_to_config_set(self): converter = TestConfigurationConverter(self._all_test_configurations) self.assertEqual(converter.to_config_set(set()), self._all_test_configurations) self.assertEqual(converter.to_config_set(set(["foo"])), set()) self.assertEqual(converter.to_config_set(set(["win7", "foo"])), set()) errors = [] self.assertEqual(converter.to_config_set(set(["win7", "foo"]), errors), set()) self.assertEqual(errors, ["Unrecognized specifier 'foo'"]) self.assertEqual(converter.to_config_set(set(["win7", "x86_64"])), set()) configs_to_match = set([TestConfiguration("win7", "x86", "release")]) self.assertEqual(converter.to_config_set(set(["win7", "release"])), configs_to_match) configs_to_match = set( [ TestConfiguration("snowleopard", "x86", "release"), TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release"), TestConfiguration("precise", "x86_64", "release"), TestConfiguration("trusty", "x86_64", "release"), ] ) self.assertEqual(converter.to_config_set(set(["release"])), configs_to_match) configs_to_match = set( [ TestConfiguration("precise", "x86_64", "release"), TestConfiguration("precise", "x86_64", "debug"), TestConfiguration("trusty", "x86_64", "release"), TestConfiguration("trusty", "x86_64", "debug"), ] ) self.assertEqual(converter.to_config_set(set(["x86_64"])), configs_to_match) configs_to_match = set( [ TestConfiguration("trusty", "x86_64", "release"), TestConfiguration("trusty", "x86_64", "debug"), TestConfiguration("precise", "x86_64", "release"), TestConfiguration("precise", "x86_64", "debug"), TestConfiguration("snowleopard", "x86", "release"), TestConfiguration("snowleopard", "x86", "debug"), ] ) self.assertEqual(converter.to_config_set(set(["trusty", "precise", "snowleopard"])), configs_to_match) configs_to_match = set( [TestConfiguration("snowleopard", "x86", "release"), TestConfiguration("snowleopard", "x86", "debug")] ) self.assertEqual(converter.to_config_set(set(["snowleopard", "x86"])), configs_to_match) configs_to_match = set( [TestConfiguration("trusty", "x86_64", "release"), TestConfiguration("snowleopard", "x86", "release")] ) self.assertEqual(converter.to_config_set(set(["trusty", "snowleopard", "release"])), configs_to_match)
def test_symmetric_difference(self): self.assertEqual(TestConfigurationConverter.symmetric_difference([set(['a', 'b']), set(['b', 'c'])]), set(['a', 'c'])) self.assertEqual(TestConfigurationConverter.symmetric_difference( [set(['a', 'b']), set(['b', 'c']), set(['b', 'd'])]), set(['a', 'c', 'd']))
def __init__(self, port, full_test_list, is_lint_mode): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._is_lint_mode = is_lint_mode
class TestExpectationParser(object): """Provides parsing facilities for lines in the test_expectation.txt file.""" # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the case studdly-caps to match the actual file contents. REBASELINE_MODIFIER = 'rebaseline' NEEDS_REBASELINE_MODIFIER = 'needsrebaseline' NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline' PASS_EXPECTATION = 'pass' SKIP_MODIFIER = 'skip' SLOW_MODIFIER = 'slow' WONTFIX_MODIFIER = 'wontfix' TIMEOUT_EXPECTATION = 'timeout' MISSING_BUG_WARNING = 'Test lacks BUG specifier.' def __init__(self, port, full_test_list, is_lint_mode): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._is_lint_mode = is_lint_mode def parse(self, filename, expectations_string): expectation_lines = [] line_number = 0 for line in expectations_string.split("\n"): line_number += 1 test_expectation = self._tokenize_line(filename, line, line_number) self._parse_line(test_expectation) expectation_lines.append(test_expectation) return expectation_lines def _create_expectation_line(self, test_name, expectations, file_name): expectation_line = TestExpectationLine() expectation_line.original_string = test_name expectation_line.name = test_name expectation_line.filename = file_name expectation_line.expectations = expectations return expectation_line def expectation_line_for_test(self, test_name, expectations): expectation_line = self._create_expectation_line(test_name, expectations, '<Bot TestExpectations>') self._parse_line(expectation_line) return expectation_line def expectation_for_skipped_test(self, test_name): if not self._port.test_exists(test_name): _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name) expectation_line = self._create_expectation_line(test_name, [TestExpectationParser.PASS_EXPECTATION], '<Skipped file>') expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER] expectation_line.is_skipped_outside_expectations_file = True self._parse_line(expectation_line) return expectation_line def _parse_line(self, expectation_line): if not expectation_line.name: return if not self._check_test_exists(expectation_line): return expectation_line.is_file = self._port.test_isfile(expectation_line.name) if expectation_line.is_file: expectation_line.path = expectation_line.name else: expectation_line.path = self._port.normalize_test_name(expectation_line.name) self._collect_matching_tests(expectation_line) self._parse_specifiers(expectation_line) self._parse_expectations(expectation_line) def _parse_specifiers(self, expectation_line): if self._is_lint_mode: self._lint_line(expectation_line) parsed_specifiers = set([specifier.lower() for specifier in expectation_line.specifiers]) expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) def _lint_line(self, expectation_line): expectations = [expectation.lower() for expectation in expectation_line.expectations] if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectations: expectation_line.warnings.append(self.MISSING_BUG_WARNING) if self.REBASELINE_MODIFIER in expectations: expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') def _parse_expectations(self, expectation_line): result = set() for part in expectation_line.expectations: expectation = TestExpectations.expectation_from_string(part) if expectation is None: # Careful, PASS is currently 0. expectation_line.warnings.append('Unsupported expectation: %s' % part) continue result.add(expectation) expectation_line.parsed_expectations = result def _check_test_exists(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'): # Log a warning here since you hit this case any # time you update TestExpectations without syncing # the LayoutTests directory expectation_line.warnings.append('Path does not exist.') return False return True def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized path and make sure directories end with the OS path separator.""" # FIXME: full_test_list can quickly contain a big amount of # elements. We should consider at some point to use a more # efficient structure instead of a list. Maybe a dictionary of # lists to represent the tree of tests, leaves being test # files and nodes being categories. if not self._full_test_list: expectation_line.matching_tests = [expectation_line.path] return if not expectation_line.is_file: # this is a test category, return all the tests of the category. expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] return # this is a test file, do a quick check if it's in the # full test suite. if expectation_line.path in self._full_test_list: expectation_line.matching_tests.append(expectation_line.path) # FIXME: Update the original specifiers and remove this once the old syntax is gone. _configuration_tokens_list = [ 'Mac', 'SnowLeopard', 'Lion', 'Retina', 'MountainLion', 'Win', 'XP', 'Win7', 'Linux', 'Android', 'Release', 'Debug', ] _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list) _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems()) # FIXME: Update the original specifiers list and remove this once the old syntax is gone. _expectation_tokens = { 'Crash': 'CRASH', 'Failure': 'FAIL', 'ImageOnlyFailure': 'IMAGE', MISSING_KEYWORD: 'MISSING', 'Pass': '******', 'Rebaseline': 'REBASELINE', NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE', NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE', 'Skip': 'SKIP', 'Slow': 'SLOW', 'Timeout': 'TIMEOUT', 'WontFix': 'WONTFIX', } _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] + [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser. @classmethod def _tokenize_line(cls, filename, expectation_string, line_number): """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format. The new format for a test expectation line is: [[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] Any errant whitespace is not preserved. """ expectation_line = TestExpectationLine() expectation_line.original_string = expectation_string expectation_line.filename = filename expectation_line.line_numbers = str(line_number) comment_index = expectation_string.find("#") if comment_index == -1: comment_index = len(expectation_string) else: expectation_line.comment = expectation_string[comment_index + 1:] remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) if len(remaining_string) == 0: return expectation_line # special-case parsing this so that we fail immediately instead of treating this as a test name if remaining_string.startswith('//'): expectation_line.warnings = ['use "#" instead of "//" for comments'] return expectation_line bugs = [] specifiers = [] name = None expectations = [] warnings = [] has_unrecognized_expectation = False tokens = remaining_string.split() state = 'start' for token in tokens: if (token.startswith(WEBKIT_BUG_PREFIX) or token.startswith(CHROMIUM_BUG_PREFIX) or token.startswith(V8_BUG_PREFIX) or token.startswith(NAMED_BUG_PREFIX)): if state != 'start': warnings.append('"%s" is not at the start of the line.' % token) break if token.startswith(WEBKIT_BUG_PREFIX): bugs.append(token) elif token.startswith(CHROMIUM_BUG_PREFIX): bugs.append(token) elif token.startswith(V8_BUG_PREFIX): bugs.append(token) else: match = re.match('Bug\((\w+)\)$', token) if not match: warnings.append('unrecognized bug identifier "%s"' % token) break else: bugs.append(token) elif token == '[': if state == 'start': state = 'configuration' elif state == 'name_found': state = 'expectations' else: warnings.append('unexpected "["') break elif token == ']': if state == 'configuration': state = 'name' elif state == 'expectations': state = 'done' else: warnings.append('unexpected "]"') break elif token in ('//', ':', '='): warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token) break elif state == 'configuration': specifiers.append(cls._configuration_tokens.get(token, token)) elif state == 'expectations': if token not in cls._expectation_tokens: has_unrecognized_expectation = True warnings.append('Unrecognized expectation "%s"' % token) else: expectations.append(cls._expectation_tokens.get(token, token)) elif state == 'name_found': warnings.append('expecting "[", "#", or end of line instead of "%s"' % token) break else: name = token state = 'name_found' if not warnings: if not name: warnings.append('Did not find a test name.') elif state not in ('name_found', 'done'): warnings.append('Missing a "]"') if 'WONTFIX' in expectations and 'SKIP' not in expectations: expectations.append('SKIP') if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(expectations) - set(['SKIP', 'WONTFIX'])): warnings.append('A test marked Skip or WontFix must not have other expectations.') if not expectations and not has_unrecognized_expectation: warnings.append('Missing expectations.') expectation_line.bugs = bugs expectation_line.specifiers = specifiers expectation_line.expectations = expectations expectation_line.name = name expectation_line.warnings = warnings return expectation_line @classmethod def _split_space_separated(cls, space_separated_string): """Splits a space-separated string into an array.""" return [part.strip() for part in space_separated_string.strip().split(' ')]
class TestExpectationParser(object): """Provides parsing facilities for lines in the test_expectation.txt file.""" DUMMY_BUG_MODIFIER = "bug_dummy" BUG_MODIFIER_PREFIX = 'bug' BUG_MODIFIER_REGEX = 'bug\d+' REBASELINE_MODIFIER = 'rebaseline' PASS_EXPECTATION = 'pass' SKIP_MODIFIER = 'skip' SLOW_MODIFIER = 'slow' WONTFIX_MODIFIER = 'wontfix' TIMEOUT_EXPECTATION = 'timeout' def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier def parse(self, expectations_string): expectations = TestExpectationParser._tokenize_list(expectations_string) for expectation_line in expectations: self._parse_line(expectation_line) return expectations def expectation_for_skipped_test(self, test_name): expectation_line = TestExpectationLine() expectation_line.original_string = test_name expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER] expectation_line.name = test_name expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION] self._parse_line(expectation_line) return expectation_line def _parse_line(self, expectation_line): if not expectation_line.name: return self._check_modifiers_against_expectations(expectation_line) expectation_line.is_file = self._port.test_isfile(expectation_line.name) if not expectation_line.is_file and self._check_path_does_not_exist(expectation_line): return if expectation_line.is_file: expectation_line.path = expectation_line.name else: expectation_line.path = self._port.normalize_test_name(expectation_line.name) self._collect_matching_tests(expectation_line) self._parse_modifiers(expectation_line) self._parse_expectations(expectation_line) def _parse_modifiers(self, expectation_line): has_wontfix = False has_bugid = False parsed_specifiers = set() for modifier in expectation_line.modifiers: if modifier in TestExpectations.MODIFIERS: expectation_line.parsed_modifiers.append(modifier) if modifier == self.WONTFIX_MODIFIER: has_wontfix = True elif modifier.startswith(self.BUG_MODIFIER_PREFIX): has_bugid = True if re.match(self.BUG_MODIFIER_REGEX, modifier): expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.') else: expectation_line.parsed_bug_modifiers.append(modifier) else: parsed_specifiers.add(modifier) if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid: expectation_line.warnings.append('Test lacks BUG modifier.') if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers: expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) def _parse_expectations(self, expectation_line): result = set() for part in expectation_line.expectations: expectation = TestExpectations.expectation_from_string(part) if expectation is None: # Careful, PASS is currently 0. expectation_line.warnings.append('Unsupported expectation: %s' % part) continue result.add(expectation) expectation_line.parsed_expectations = result def _check_modifiers_against_expectations(self, expectation_line): if self.SLOW_MODIFIER in expectation_line.modifiers and self.TIMEOUT_EXPECTATION in expectation_line.expectations: expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.') def _check_path_does_not_exist(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. if (not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled')): # Log a warning here since you hit this case any # time you update test_expectations.txt without syncing # the LayoutTests directory expectation_line.warnings.append('Path does not exist.') return True return False def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized path and make sure directories end with the OS path separator.""" # FIXME: full_test_list can quickly contain a big amount of # elements. We should consider at some point to use a more # efficient structure instead of a list. Maybe a dictionary of # lists to represent the tree of tests, leaves being test # files and nodes being categories. if not self._full_test_list: expectation_line.matching_tests = [expectation_line.path] return if not expectation_line.is_file: # this is a test category, return all the tests of the category. expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] return # this is a test file, do a quick check if it's in the # full test suite. if expectation_line.path in self._full_test_list: expectation_line.matching_tests.append(expectation_line.path) @classmethod def _tokenize(cls, expectation_string, line_number=None): """Tokenizes a line from test_expectations.txt and returns an unparsed TestExpectationLine instance. The format of a test expectation line is: [[<modifiers>] : <name> = <expectations>][ //<comment>] Any errant whitespace is not preserved. """ expectation_line = TestExpectationLine() expectation_line.original_string = expectation_string expectation_line.line_number = line_number comment_index = expectation_string.find("//") if comment_index == -1: comment_index = len(expectation_string) else: expectation_line.comment = expectation_string[comment_index + 2:] remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) if len(remaining_string) == 0: return expectation_line parts = remaining_string.split(':') if len(parts) != 2: expectation_line.warnings.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'") else: test_and_expectation = parts[1].split('=') if len(test_and_expectation) != 2: expectation_line.warnings.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='") if not expectation_line.is_invalid(): expectation_line.modifiers = cls._split_space_separated(parts[0]) expectation_line.name = test_and_expectation[0].strip() expectation_line.expectations = cls._split_space_separated(test_and_expectation[1]) return expectation_line @classmethod def _tokenize_list(cls, expectations_string): """Returns a list of TestExpectationLines, one for each line in expectations_string.""" expectation_lines = [] line_number = 0 for line in expectations_string.split("\n"): line_number += 1 expectation_lines.append(cls._tokenize(line, line_number)) return expectation_lines @classmethod def _split_space_separated(cls, space_separated_string): """Splits a space-separated string into an array.""" # FIXME: Lower-casing is necessary to support legacy code. Need to eliminate. return [part.strip().lower() for part in space_separated_string.strip().split(' ')]
class TestExpectationParser(object): """Provides parsing facilities for lines in the test_expectation.txt file.""" DUMMY_BUG_MODIFIER = "bug_dummy" BUG_MODIFIER_PREFIX = 'bug' BUG_MODIFIER_REGEX = 'bug\d+' REBASELINE_MODIFIER = 'rebaseline' PASS_EXPECTATION = 'pass' SKIP_MODIFIER = 'skip' SLOW_MODIFIER = 'slow' WONTFIX_MODIFIER = 'wontfix' TIMEOUT_EXPECTATION = 'timeout' MISSING_BUG_WARNING = 'Test lacks BUG modifier.' def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier def parse(self, filename, expectations_string): expectation_lines = [] line_number = 0 for line in expectations_string.split("\n"): line_number += 1 test_expectation = self._tokenize_line(filename, line, line_number) self._parse_line(test_expectation) expectation_lines.append(test_expectation) return expectation_lines def expectation_for_skipped_test(self, test_name): if not self._port.test_exists(test_name): _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name) expectation_line = TestExpectationLine() expectation_line.original_string = test_name expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER] # FIXME: It's not clear what the expectations for a skipped test should be; the expectations # might be different for different entries in a Skipped file, or from the command line, or from # only running parts of the tests. It's also not clear if it matters much. expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER) expectation_line.name = test_name # FIXME: we should pass in a more descriptive string here. expectation_line.filename = '<Skipped file>' expectation_line.line_number = 0 expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION] self._parse_line(expectation_line) return expectation_line def _parse_line(self, expectation_line): if not expectation_line.name: return if not self._check_test_exists(expectation_line): return expectation_line.is_file = self._port.test_isfile(expectation_line.name) if expectation_line.is_file: expectation_line.path = expectation_line.name else: expectation_line.path = self._port.normalize_test_name(expectation_line.name) self._collect_matching_tests(expectation_line) self._parse_modifiers(expectation_line) self._parse_expectations(expectation_line) def _parse_modifiers(self, expectation_line): has_wontfix = False has_bugid = False parsed_specifiers = set() modifiers = [modifier.lower() for modifier in expectation_line.modifiers] expectations = [expectation.lower() for expectation in expectation_line.expectations] if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations: expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.') for modifier in modifiers: if modifier in TestExpectations.MODIFIERS: expectation_line.parsed_modifiers.append(modifier) if modifier == self.WONTFIX_MODIFIER: has_wontfix = True elif modifier.startswith(self.BUG_MODIFIER_PREFIX): has_bugid = True if re.match(self.BUG_MODIFIER_REGEX, modifier): expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.') else: expectation_line.parsed_bug_modifiers.append(modifier) else: parsed_specifiers.add(modifier) if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations(): expectation_line.warnings.append(self.MISSING_BUG_WARNING) if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers: expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) def _parse_expectations(self, expectation_line): result = set() for part in expectation_line.expectations: expectation = TestExpectations.expectation_from_string(part) if expectation is None: # Careful, PASS is currently 0. expectation_line.warnings.append('Unsupported expectation: %s' % part) continue result.add(expectation) expectation_line.parsed_expectations = result def _check_test_exists(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'): # Log a warning here since you hit this case any # time you update TestExpectations without syncing # the LayoutTests directory expectation_line.warnings.append('Path does not exist.') return False return True def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized path and make sure directories end with the OS path separator.""" # FIXME: full_test_list can quickly contain a big amount of # elements. We should consider at some point to use a more # efficient structure instead of a list. Maybe a dictionary of # lists to represent the tree of tests, leaves being test # files and nodes being categories. if not self._full_test_list: expectation_line.matching_tests = [expectation_line.path] return if not expectation_line.is_file: # this is a test category, return all the tests of the category. expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] return # this is a test file, do a quick check if it's in the # full test suite. if expectation_line.path in self._full_test_list: expectation_line.matching_tests.append(expectation_line.path) # FIXME: Update the original modifiers and remove this once the old syntax is gone. _configuration_tokens_list = [ 'Mac', 'SnowLeopard', 'Lion', 'MountainLion', 'Win', 'XP', 'Vista', 'Win7', 'Linux', 'Android', 'Release', 'Debug', ] _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list) _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems()) # FIXME: Update the original modifiers list and remove this once the old syntax is gone. _expectation_tokens = { 'Crash': 'CRASH', 'Failure': 'FAIL', 'ImageOnlyFailure': 'IMAGE', 'Missing': 'MISSING', 'Pass': '******', 'Rebaseline': 'REBASELINE', 'Skip': 'SKIP', 'Slow': 'SLOW', 'Timeout': 'TIMEOUT', 'WontFix': 'WONTFIX', } _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] + [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser. @classmethod def _tokenize_line(cls, filename, expectation_string, line_number): """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format. The new format for a test expectation line is: [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] Any errant whitespace is not preserved. """ expectation_line = TestExpectationLine() expectation_line.original_string = expectation_string expectation_line.filename = filename expectation_line.line_number = line_number comment_index = expectation_string.find("#") if comment_index == -1: comment_index = len(expectation_string) else: expectation_line.comment = expectation_string[comment_index + 1:] remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) if len(remaining_string) == 0: return expectation_line # special-case parsing this so that we fail immediately instead of treating this as a test name if remaining_string.startswith('//'): expectation_line.warnings = ['use "#" instead of "//" for comments'] return expectation_line bugs = [] modifiers = [] name = None expectations = [] warnings = [] WEBKIT_BUG_PREFIX = 'webkit.org/b/' CHROMIUM_BUG_PREFIX = 'crbug.com/' V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id=' tokens = remaining_string.split() state = 'start' for token in tokens: if (token.startswith(WEBKIT_BUG_PREFIX) or token.startswith(CHROMIUM_BUG_PREFIX) or token.startswith(V8_BUG_PREFIX) or token.startswith('Bug(')): if state != 'start': warnings.append('"%s" is not at the start of the line.' % token) break if token.startswith(WEBKIT_BUG_PREFIX): bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK')) elif token.startswith(CHROMIUM_BUG_PREFIX): bugs.append(token.replace(CHROMIUM_BUG_PREFIX, 'BUGCR')) elif token.startswith(V8_BUG_PREFIX): bugs.append(token.replace(V8_BUG_PREFIX, 'BUGV8_')) else: match = re.match('Bug\((\w+)\)$', token) if not match: warnings.append('unrecognized bug identifier "%s"' % token) break else: bugs.append('BUG' + match.group(1).upper()) elif token.startswith('BUG'): warnings.append('unrecognized old-style bug identifier "%s"' % token) break elif token == '[': if state == 'start': state = 'configuration' elif state == 'name_found': state = 'expectations' else: warnings.append('unexpected "["') break elif token == ']': if state == 'configuration': state = 'name' elif state == 'expectations': state = 'done' else: warnings.append('unexpected "]"') break elif token in ('//', ':', '='): warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token) break elif state == 'configuration': modifiers.append(cls._configuration_tokens.get(token, token)) elif state == 'expectations': if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix'): modifiers.append(token.upper()) elif token not in cls._expectation_tokens: warnings.append('Unrecognized expectation "%s"' % token) else: expectations.append(cls._expectation_tokens.get(token, token)) elif state == 'name_found': warnings.append('expecting "[", "#", or end of line instead of "%s"' % token) break else: name = token state = 'name_found' if not warnings: if not name: warnings.append('Did not find a test name.') elif state not in ('name_found', 'done'): warnings.append('Missing a "]"') if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations: modifiers.append('SKIP') if 'SKIP' in modifiers and expectations: # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax. warnings.append('A test marked Skip must not have other expectations.') elif not expectations: if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers: modifiers.append('SKIP') expectations = ['PASS'] # FIXME: expectation line should just store bugs and modifiers separately. expectation_line.modifiers = bugs + modifiers expectation_line.expectations = expectations expectation_line.name = name expectation_line.warnings = warnings return expectation_line @classmethod def _split_space_separated(cls, space_separated_string): """Splits a space-separated string into an array.""" return [part.strip() for part in space_separated_string.strip().split(' ')]
def test_specifier_converter_access(self): specifier_sorter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS).specifier_sorter() self.assertEqual(specifier_sorter.category_for_specifier("snowleopard"), "version") self.assertEqual(specifier_sorter.category_for_specifier("mac"), "version")
def test_to_config_set(self): converter = TestConfigurationConverter(self._all_test_configurations) self.assertEqual(converter.to_config_set(set()), self._all_test_configurations) self.assertEqual(converter.to_config_set(set(['foo'])), set()) self.assertEqual(converter.to_config_set(set(['win7', 'foo'])), set()) errors = [] self.assertEqual(converter.to_config_set(set(['win7', 'foo']), errors), set()) self.assertEqual(errors, ["Unrecognized specifier 'foo'"]) self.assertEqual(converter.to_config_set(set(['win7', 'x86_64'])), set()) configs_to_match = set([ TestConfiguration('win7', 'x86', 'release'), ]) self.assertEqual(converter.to_config_set(set(['win7', 'release'])), configs_to_match) configs_to_match = set([ TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('vista', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('linux32', 'x86', 'release'), TestConfiguration('precise', 'x86_64', 'release'), TestConfiguration('trusty', 'x86_64', 'release'), ]) self.assertEqual(converter.to_config_set(set(['release'])), configs_to_match) configs_to_match = set([ TestConfiguration('precise', 'x86_64', 'release'), TestConfiguration('precise', 'x86_64', 'debug'), TestConfiguration('trusty', 'x86_64', 'release'), TestConfiguration('trusty', 'x86_64', 'debug'), ]) self.assertEqual(converter.to_config_set(set(['x86_64'])), configs_to_match) configs_to_match = set([ TestConfiguration('trusty', 'x86_64', 'release'), TestConfiguration('trusty', 'x86_64', 'debug'), TestConfiguration('precise', 'x86_64', 'release'), TestConfiguration('precise', 'x86_64', 'debug'), TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'debug'), ]) self.assertEqual(converter.to_config_set(set(['trusty', 'precise', 'snowleopard'])), configs_to_match) configs_to_match = set([ TestConfiguration('linux32', 'x86', 'release'), TestConfiguration('linux32', 'x86', 'debug'), TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'debug'), ]) self.assertEqual(converter.to_config_set(set(['linux32', 'snowleopard', 'x86'])), configs_to_match) configs_to_match = set([ TestConfiguration('trusty', 'x86_64', 'release'), TestConfiguration('linux32', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'release'), ]) self.assertEqual( converter.to_config_set(set(['trusty', 'linux32', 'snowleopard', 'release'])), configs_to_match)
def test_to_specifier_lists(self): converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS) self.assertEqual(converter.to_specifiers_list(set(self._all_test_configurations)), [[]]) self.assertEqual(converter.to_specifiers_list(set()), []) configs_to_match = set([ TestConfiguration('win7', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['release', 'win7'])]) configs_to_match = set([ TestConfiguration('win7', 'x86', 'release'), TestConfiguration('win7', 'x86', 'debug'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win7'])]) configs_to_match = set([ TestConfiguration('linux32', 'x86', 'debug'), TestConfiguration('precise', 'x86_64', 'debug'), TestConfiguration('trusty', 'x86_64', 'debug'), TestConfiguration('win7', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['release', 'win7']), set(['debug', 'linux'])]) configs_to_match = set([ TestConfiguration('win7', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('trusty', 'x86_64', 'debug'), TestConfiguration('precise', 'x86_64', 'debug'), TestConfiguration('linux32', 'x86', 'debug'), TestConfiguration('trusty', 'x86_64', 'debug'), TestConfiguration('precise', 'x86_64', 'debug'), TestConfiguration('linux32', 'x86', 'debug'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['release', 'win7']), set(['debug', 'linux'])]) configs_to_match = set([ TestConfiguration('win7', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('vista', 'x86', 'release'), TestConfiguration('linux32', 'x86', 'release'), TestConfiguration('precise', 'x86_64', 'release'), TestConfiguration('trusty', 'x86_64', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['release'])]) configs_to_match = set([ TestConfiguration('win7', 'x86', 'release'), TestConfiguration('snowleopard', 'x86', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win7', 'mac', 'release'])]) configs_to_match = set([ TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('win7', 'x86', 'release'), TestConfiguration('win7', 'x86', 'debug'), TestConfiguration('trusty', 'x86_64', 'release'), ]) self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win7']), set(['release', 'mac', 'trusty'])])
class TestExpectationParser(object): """Provides parsing facilities for lines in the test_expectation.txt file.""" DUMMY_BUG_MODIFIER = "bug_dummy" BUG_MODIFIER_PREFIX = 'bug' BUG_MODIFIER_REGEX = 'bug\d+' REBASELINE_MODIFIER = 'rebaseline' PASS_EXPECTATION = 'pass' SKIP_MODIFIER = 'skip' SLOW_MODIFIER = 'slow' WONTFIX_MODIFIER = 'wontfix' TIMEOUT_EXPECTATION = 'timeout' MISSING_BUG_WARNING = 'Test lacks BUG modifier.' def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier def parse(self, filename, expectations_string): expectation_lines = [] line_number = 0 for line in expectations_string.split("\n"): line_number += 1 test_expectation = self._tokenize_line(filename, line, line_number) self._parse_line(test_expectation) expectation_lines.append(test_expectation) return expectation_lines def expectation_for_skipped_test(self, test_name): if not self._port.test_exists(test_name): _log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name) expectation_line = TestExpectationLine() expectation_line.original_string = test_name expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER] # FIXME: It's not clear what the expectations for a skipped test should be; the expectations # might be different for different entries in a Skipped file, or from the command line, or from # only running parts of the tests. It's also not clear if it matters much. expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER) expectation_line.name = test_name # FIXME: we should pass in a more descriptive string here. expectation_line.filename = '<Skipped file>' expectation_line.line_number = 0 expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION] self._parse_line(expectation_line) return expectation_line def _parse_line(self, expectation_line): if not expectation_line.name: return if not self._check_test_exists(expectation_line): return expectation_line.is_file = self._port.test_isfile(expectation_line.name) if expectation_line.is_file: expectation_line.path = expectation_line.name else: expectation_line.path = self._port.normalize_test_name(expectation_line.name) self._collect_matching_tests(expectation_line) self._parse_modifiers(expectation_line) self._parse_expectations(expectation_line) def _parse_modifiers(self, expectation_line): has_wontfix = False has_bugid = False parsed_specifiers = set() modifiers = [modifier.lower() for modifier in expectation_line.modifiers] expectations = [expectation.lower() for expectation in expectation_line.expectations] if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations: expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.') for modifier in modifiers: if modifier in TestExpectations.MODIFIERS: expectation_line.parsed_modifiers.append(modifier) if modifier == self.WONTFIX_MODIFIER: has_wontfix = True elif modifier.startswith(self.BUG_MODIFIER_PREFIX): has_bugid = True if re.match(self.BUG_MODIFIER_REGEX, modifier): expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.') else: expectation_line.parsed_bug_modifiers.append(modifier) else: parsed_specifiers.add(modifier) if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations(): expectation_line.warnings.append(self.MISSING_BUG_WARNING) if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers: expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) def _parse_expectations(self, expectation_line): result = set() for part in expectation_line.expectations: expectation = TestExpectations.expectation_from_string(part) if expectation is None: # Careful, PASS is currently 0. expectation_line.warnings.append('Unsupported expectation: %s' % part) continue result.add(expectation) expectation_line.parsed_expectations = result def _check_test_exists(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'): # Log a warning here since you hit this case any # time you update TestExpectations without syncing # the LayoutTests directory expectation_line.warnings.append('Path does not exist.') return False return True def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized path and make sure directories end with the OS path separator.""" # FIXME: full_test_list can quickly contain a big amount of # elements. We should consider at some point to use a more # efficient structure instead of a list. Maybe a dictionary of # lists to represent the tree of tests, leaves being test # files and nodes being categories. if not self._full_test_list: expectation_line.matching_tests = [expectation_line.path] return if not expectation_line.is_file: # this is a test category, return all the tests of the category. expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] return # this is a test file, do a quick check if it's in the # full test suite. if expectation_line.path in self._full_test_list: expectation_line.matching_tests.append(expectation_line.path) # FIXME: Update the original modifiers and remove this once the old syntax is gone. _configuration_tokens_list = [ 'Mac', 'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks', 'Win', 'XP', 'Vista', 'Win7', 'Linux', 'Android', 'Release', 'Debug', ] _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list) _inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems()) # FIXME: Update the original modifiers list and remove this once the old syntax is gone. _expectation_tokens = { 'Crash': 'CRASH', 'Failure': 'FAIL', 'ImageOnlyFailure': 'IMAGE', 'Missing': 'MISSING', 'Pass': '******', 'Rebaseline': 'REBASELINE', 'Skip': 'SKIP', 'Slow': 'SLOW', 'Timeout': 'TIMEOUT', 'WontFix': 'WONTFIX', } _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] + [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')]) # FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser. @classmethod def _tokenize_line(cls, filename, expectation_string, line_number): """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format. The new format for a test expectation line is: [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>] Any errant whitespace is not preserved. """ expectation_line = TestExpectationLine() expectation_line.original_string = expectation_string expectation_line.filename = filename expectation_line.line_number = line_number comment_index = expectation_string.find("#") if comment_index == -1: comment_index = len(expectation_string) else: expectation_line.comment = expectation_string[comment_index + 1:] remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) if len(remaining_string) == 0: return expectation_line # special-case parsing this so that we fail immediately instead of treating this as a test name if remaining_string.startswith('//'): expectation_line.warnings = ['use "#" instead of "//" for comments'] return expectation_line bugs = [] modifiers = [] name = None expectations = [] warnings = [] WEBKIT_BUG_PREFIX = 'webkit.org/b/' tokens = remaining_string.split() state = 'start' for token in tokens: if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('): if state != 'start': warnings.append('"%s" is not at the start of the line.' % token) break if token.startswith(WEBKIT_BUG_PREFIX): bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK')) else: match = re.match('Bug\((\w+)\)$', token) if not match: warnings.append('unrecognized bug identifier "%s"' % token) break else: bugs.append('BUG' + match.group(1).upper()) elif token.startswith('BUG'): warnings.append('unrecognized old-style bug identifier "%s"' % token) break elif token == '[': if state == 'start': state = 'configuration' elif state == 'name_found': state = 'expectations' else: warnings.append('unexpected "["') break elif token == ']': if state == 'configuration': state = 'name' elif state == 'expectations': state = 'done' else: warnings.append('unexpected "]"') break elif token in ('//', ':', '='): warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token) break elif state == 'configuration': modifiers.append(cls._configuration_tokens.get(token, token)) elif state == 'expectations': if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix'): modifiers.append(token.upper()) elif token not in cls._expectation_tokens: warnings.append('Unrecognized expectation "%s"' % token) else: expectations.append(cls._expectation_tokens.get(token, token)) elif state == 'name_found': warnings.append('expecting "[", "#", or end of line instead of "%s"' % token) break else: name = token state = 'name_found' if not warnings: if not name: warnings.append('Did not find a test name.') elif state not in ('name_found', 'done'): warnings.append('Missing a "]"') if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations: modifiers.append('SKIP') if 'SKIP' in modifiers and expectations: # FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax. warnings.append('A test marked Skip must not have other expectations.') elif not expectations: if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers: modifiers.append('SKIP') expectations = ['PASS'] # FIXME: expectation line should just store bugs and modifiers separately. expectation_line.modifiers = bugs + modifiers expectation_line.expectations = expectations expectation_line.name = name expectation_line.warnings = warnings return expectation_line @classmethod def _split_space_separated(cls, space_separated_string): """Splits a space-separated string into an array.""" return [part.strip() for part in space_separated_string.strip().split(' ')]
class TestExpectationParser(object): """Provides parsing facilities for lines in the test_expectation.txt file.""" DUMMY_BUG_MODIFIER = "bug_dummy" BUG_MODIFIER_PREFIX = 'bug' BUG_MODIFIER_REGEX = 'bug\d+' REBASELINE_MODIFIER = 'rebaseline' PASS_EXPECTATION = 'pass' SKIP_MODIFIER = 'skip' SLOW_MODIFIER = 'slow' WONTFIX_MODIFIER = 'wontfix' TIMEOUT_EXPECTATION = 'timeout' def __init__(self, port, full_test_list, allow_rebaseline_modifier): self._port = port self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros()) self._full_test_list = full_test_list self._allow_rebaseline_modifier = allow_rebaseline_modifier def parse(self, filename, expectations_string): expectations = TestExpectationParser._tokenize_list(filename, expectations_string) for expectation_line in expectations: self._parse_line(expectation_line) return expectations def expectation_for_skipped_test(self, test_name): expectation_line = TestExpectationLine() expectation_line.original_string = test_name expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER] expectation_line.name = test_name # FIXME: we should pass in a more descriptive string here. expectation_line.filename = '<Skipped file>' expectation_line.line_number = 0 expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION] self._parse_line(expectation_line) return expectation_line def _parse_line(self, expectation_line): if not expectation_line.name: return self._check_modifiers_against_expectations(expectation_line) expectation_line.is_file = self._port.test_isfile(expectation_line.name) if not expectation_line.is_file and self._check_path_does_not_exist(expectation_line): return if expectation_line.is_file: expectation_line.path = expectation_line.name else: expectation_line.path = self._port.normalize_test_name(expectation_line.name) self._collect_matching_tests(expectation_line) self._parse_modifiers(expectation_line) self._parse_expectations(expectation_line) def _parse_modifiers(self, expectation_line): has_wontfix = False has_bugid = False parsed_specifiers = set() for modifier in expectation_line.modifiers: if modifier in TestExpectations.MODIFIERS: expectation_line.parsed_modifiers.append(modifier) if modifier == self.WONTFIX_MODIFIER: has_wontfix = True elif modifier.startswith(self.BUG_MODIFIER_PREFIX): has_bugid = True if re.match(self.BUG_MODIFIER_REGEX, modifier): expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.') else: expectation_line.parsed_bug_modifiers.append(modifier) else: parsed_specifiers.add(modifier) if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid: expectation_line.warnings.append('Test lacks BUG modifier.') if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers: expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.') expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings) def _parse_expectations(self, expectation_line): result = set() for part in expectation_line.expectations: expectation = TestExpectations.expectation_from_string(part) if expectation is None: # Careful, PASS is currently 0. expectation_line.warnings.append('Unsupported expectation: %s' % part) continue result.add(expectation) expectation_line.parsed_expectations = result def _check_modifiers_against_expectations(self, expectation_line): if self.SLOW_MODIFIER in expectation_line.modifiers and self.TIMEOUT_EXPECTATION in expectation_line.expectations: expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.') def _check_path_does_not_exist(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. if (not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled')): # Log a warning here since you hit this case any # time you update TestExpectations without syncing # the LayoutTests directory expectation_line.warnings.append('Path does not exist.') return True return False def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized path and make sure directories end with the OS path separator.""" # FIXME: full_test_list can quickly contain a big amount of # elements. We should consider at some point to use a more # efficient structure instead of a list. Maybe a dictionary of # lists to represent the tree of tests, leaves being test # files and nodes being categories. if not self._full_test_list: expectation_line.matching_tests = [expectation_line.path] return if not expectation_line.is_file: # this is a test category, return all the tests of the category. expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)] return # this is a test file, do a quick check if it's in the # full test suite. if expectation_line.path in self._full_test_list: expectation_line.matching_tests.append(expectation_line.path) @classmethod def _tokenize(cls, filename, expectation_string, line_number): """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance. The format of a test expectation line is: [[<modifiers>] : <name> = <expectations>][ //<comment>] Any errant whitespace is not preserved. """ expectation_line = TestExpectationLine() expectation_line.original_string = expectation_string expectation_line.line_number = line_number expectation_line.filename = filename comment_index = expectation_string.find("//") if comment_index == -1: comment_index = len(expectation_string) else: expectation_line.comment = expectation_string[comment_index + 2:] remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip()) if len(remaining_string) == 0: return expectation_line parts = remaining_string.split(':') if len(parts) != 2: expectation_line.warnings.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'") else: test_and_expectation = parts[1].split('=') if len(test_and_expectation) != 2: expectation_line.warnings.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='") if not expectation_line.is_invalid(): expectation_line.modifiers = cls._split_space_separated(parts[0]) expectation_line.name = test_and_expectation[0].strip() expectation_line.expectations = cls._split_space_separated(test_and_expectation[1]) return expectation_line @classmethod def _tokenize_list(cls, filename, expectations_string): """Returns a list of TestExpectationLines, one for each line in expectations_string.""" expectation_lines = [] line_number = 0 for line in expectations_string.split("\n"): line_number += 1 expectation_lines.append(cls._tokenize(filename, line, line_number)) return expectation_lines @classmethod def _split_space_separated(cls, space_separated_string): """Splits a space-separated string into an array.""" # FIXME: Lower-casing is necessary to support legacy code. Need to eliminate. return [part.strip().lower() for part in space_separated_string.strip().split(' ')]