def test_macro_expansion(self):
converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS)
configs_to_match = set(
[TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release")]
)
self.assertEqual(converter.to_config_set(set(["win", "release"])), configs_to_match)
configs_to_match = set(
[
TestConfiguration("vista", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("trusty", "x86_64", "release"),
]
)
self.assertEqual(converter.to_config_set(set(["win", "trusty", "release"])), configs_to_match)
configs_to_match = set(
[
TestConfiguration("vista", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("snowleopard", "x86", "release"),
]
)
self.assertEqual(converter.to_config_set(set(["win", "mac", "release"])), configs_to_match)
def test_symmetric_difference(self):
self.assertEqual(
TestConfigurationConverter.symmetric_difference([set(["a", "b"]), set(["b", "c"])]), set(["a", "c"])
)
self.assertEqual(
TestConfigurationConverter.symmetric_difference([set(["a", "b"]), set(["b", "c"]), set(["b", "d"])]),
set(["a", "c", "d"]),
)
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(
set(port.all_test_configurations()),
port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def test_macro_expansion(self):
converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS)
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
])
self.assertEqual(converter.to_config_set(set(['win', 'release'])), configs_to_match)
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('linux32', 'x86', 'release'),
TestConfiguration('trusty', 'x86_64', 'release'),
])
self.assertEqual(converter.to_config_set(set(['win', 'linux32', 'trusty', 'release'])),
configs_to_match)
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'release'),
])
self.assertEqual(converter.to_config_set(set(['win', 'mac', 'release'])), configs_to_match)
def test_macro_collapsing(self):
macros = {'foo': ['bar', 'baz'], 'people': ['bob', 'alice', 'john']}
specifiers_list = [set(['john', 'godzilla', 'bob', 'alice'])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(['people', 'godzilla'])])
specifiers_list = [set(['john', 'godzilla', 'alice'])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(['john', 'godzilla', 'alice', 'godzilla'])])
specifiers_list = [set(['bar', 'godzilla', 'baz', 'bob', 'alice', 'john'])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(['foo', 'godzilla', 'people'])])
specifiers_list = [set(['bar', 'godzilla', 'baz', 'bob']), set(['bar', 'baz']), set(['people', 'alice', 'bob', 'john'])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(['bob', 'foo', 'godzilla']), set(['foo']), set(['people'])])
def test_macro_collapsing(self):
macros = {"foo": ["bar", "baz"], "people": ["bob", "alice", "john"]}
specifiers_list = [set(["john", "godzilla", "bob", "alice"])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(["people", "godzilla"])])
specifiers_list = [set(["john", "godzilla", "alice"])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(["john", "godzilla", "alice", "godzilla"])])
specifiers_list = [set(["bar", "godzilla", "baz", "bob", "alice", "john"])]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(["foo", "godzilla", "people"])])
specifiers_list = [
set(["bar", "godzilla", "baz", "bob"]),
set(["bar", "baz"]),
set(["people", "alice", "bob", "john"]),
]
TestConfigurationConverter.collapse_macros(macros, specifiers_list)
self.assertEqual(specifiers_list, [set(["bob", "foo", "godzilla"]), set(["foo"]), set(["people"])])
def test_converter_macro_collapsing(self):
converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS)
configs_to_match = set(
[TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release")]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "release"])])
configs_to_match = set(
[
TestConfiguration("vista", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("precise", "x86_64", "release"),
TestConfiguration("trusty", "x86_64", "release"),
]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "linux", "release"])])
configs_to_match = set(
[
TestConfiguration("vista", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("snowleopard", "x86", "release"),
]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "mac", "release"])])
configs_to_match = set(
[
TestConfiguration("vista", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("snowleopard", "x86", "release"),
]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "mac", "release"])])
configs_to_match = set(
[TestConfiguration("vista", "x86", "release"), TestConfiguration("win7", "x86", "release")]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win", "release"])])
def test_converter_macro_collapsing(self):
converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS)
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'release'])])
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('linux32', 'x86', 'release'),
TestConfiguration('precise', 'x86_64', 'release'),
TestConfiguration('trusty', 'x86_64', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'linux', 'release'])])
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'mac', 'release'])])
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'mac', 'release'])])
configs_to_match = set([
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win', 'release'])])
class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
DUMMY_BUG_MODIFIER = "bug_dummy"
BUG_MODIFIER_PREFIX = "bug"
BUG_MODIFIER_REGEX = "bug\d+"
REBASELINE_MODIFIER = "rebaseline"
PASS_EXPECTATION = "pass"
SKIP_MODIFIER = "skip"
SLOW_MODIFIER = "slow"
WONTFIX_MODIFIER = "wontfix"
TIMEOUT_EXPECTATION = "timeout"
MISSING_BUG_WARNING = "Test lacks BUG modifier."
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(
set(port.all_test_configurations()), port.configuration_specifier_macros()
)
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def parse(self, filename, expectations_string):
expectation_lines = []
line_number = 0
for line in expectations_string.split("\n"):
line_number += 1
test_expectation = self._tokenize_line(filename, line, line_number)
self._parse_line(test_expectation)
expectation_lines.append(test_expectation)
return expectation_lines
def expectation_for_skipped_test(self, test_name):
if not self._port.test_exists(test_name):
_log.warning("The following test %s from the Skipped list doesn't exist" % test_name)
expectation_line = TestExpectationLine()
expectation_line.original_string = test_name
expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
# FIXME: It's not clear what the expectations for a skipped test should be; the expectations
# might be different for different entries in a Skipped file, or from the command line, or from
# only running parts of the tests. It's also not clear if it matters much.
expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
expectation_line.name = test_name
# FIXME: we should pass in a more descriptive string here.
expectation_line.filename = "<Skipped file>"
expectation_line.line_number = 0
expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
self._parse_line(expectation_line)
return expectation_line
def _parse_line(self, expectation_line):
if not expectation_line.name:
return
if not self._check_test_exists(expectation_line):
return
expectation_line.is_file = self._port.test_isfile(expectation_line.name)
if expectation_line.is_file:
expectation_line.path = expectation_line.name
else:
expectation_line.path = self._port.normalize_test_name(expectation_line.name)
self._collect_matching_tests(expectation_line)
self._parse_modifiers(expectation_line)
self._parse_expectations(expectation_line)
def _parse_modifiers(self, expectation_line):
has_wontfix = False
has_bugid = False
parsed_specifiers = set()
modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
expectations = [expectation.lower() for expectation in expectation_line.expectations]
if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
expectation_line.warnings.append(
"A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT."
)
for modifier in modifiers:
if modifier in TestExpectations.MODIFIERS:
expectation_line.parsed_modifiers.append(modifier)
if modifier == self.WONTFIX_MODIFIER:
has_wontfix = True
elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
has_bugid = True
if re.match(self.BUG_MODIFIER_REGEX, modifier):
expectation_line.warnings.append(
"BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier."
)
else:
expectation_line.parsed_bug_modifiers.append(modifier)
else:
parsed_specifiers.add(modifier)
if (
not expectation_line.parsed_bug_modifiers
and not has_wontfix
and not has_bugid
and self._port.warn_if_bug_missing_in_test_expectations()
):
expectation_line.warnings.append(self.MISSING_BUG_WARNING)
if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
expectation_line.warnings.append(
"REBASELINE should only be used for running rebaseline.py. Cannot be checked in."
)
expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(
parsed_specifiers, expectation_line.warnings
)
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
expectation_line.warnings.append("Unsupported expectation: %s" % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_test_exists(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(
expectation_line.name + "-disabled"
):
# Log a warning here since you hit this case any
# time you update TestExpectations without syncing
# the LayoutTests directory
expectation_line.warnings.append("Path does not exist.")
return False
return True
def _collect_matching_tests(self, expectation_line):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
# FIXME: full_test_list can quickly contain a big amount of
# elements. We should consider at some point to use a more
# efficient structure instead of a list. Maybe a dictionary of
# lists to represent the tree of tests, leaves being test
# files and nodes being categories.
if not self._full_test_list:
expectation_line.matching_tests = [expectation_line.path]
return
if not expectation_line.is_file:
# this is a test category, return all the tests of the category.
expectation_line.matching_tests = [
test for test in self._full_test_list if test.startswith(expectation_line.path)
]
return
# this is a test file, do a quick check if it's in the
# full test suite.
if expectation_line.path in self._full_test_list:
expectation_line.matching_tests.append(expectation_line.path)
# FIXME: Update the original modifiers and remove this once the old syntax is gone.
_configuration_tokens_list = [
"Mac",
"SnowLeopard",
"Lion",
"MountainLion",
"Win",
"XP",
"Vista",
"Win7",
"Linux",
"Android",
"Release",
"Debug",
]
_configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
_inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
# FIXME: Update the original modifiers list and remove this once the old syntax is gone.
_expectation_tokens = {
"Crash": "CRASH",
"Failure": "FAIL",
"ImageOnlyFailure": "IMAGE",
"Missing": "MISSING",
"Pass": "******",
"Rebaseline": "REBASELINE",
"Skip": "SKIP",
"Slow": "SLOW",
"Timeout": "TIMEOUT",
"WontFix": "WONTFIX",
}
_inverted_expectation_tokens = dict(
[(value, name) for name, value in _expectation_tokens.iteritems()]
+ [("TEXT", "Failure"), ("IMAGE+TEXT", "Failure"), ("AUDIO", "Failure")]
)
# FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
@classmethod
def _tokenize_line(cls, filename, expectation_string, line_number):
"""Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
The new format for a test expectation line is:
[[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
Any errant whitespace is not preserved.
"""
expectation_line = TestExpectationLine()
expectation_line.original_string = expectation_string
expectation_line.filename = filename
expectation_line.line_number = line_number
comment_index = expectation_string.find("#")
if comment_index == -1:
comment_index = len(expectation_string)
else:
expectation_line.comment = expectation_string[comment_index + 1 :]
remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
if len(remaining_string) == 0:
return expectation_line
# special-case parsing this so that we fail immediately instead of treating this as a test name
if remaining_string.startswith("//"):
expectation_line.warnings = ['use "#" instead of "//" for comments']
return expectation_line
bugs = []
modifiers = []
name = None
expectations = []
warnings = []
WEBKIT_BUG_PREFIX = "webkit.org/b/"
CHROMIUM_BUG_PREFIX = "crbug.com/"
V8_BUG_PREFIX = "code.google.com/p/v8/issues/detail?id="
tokens = remaining_string.split()
state = "start"
for token in tokens:
if (
token.startswith(WEBKIT_BUG_PREFIX)
or token.startswith(CHROMIUM_BUG_PREFIX)
or token.startswith(V8_BUG_PREFIX)
or token.startswith("Bug(")
):
if state != "start":
warnings.append('"%s" is not at the start of the line.' % token)
break
if token.startswith(WEBKIT_BUG_PREFIX):
bugs.append(token.replace(WEBKIT_BUG_PREFIX, "BUGWK"))
elif token.startswith(CHROMIUM_BUG_PREFIX):
bugs.append(token.replace(CHROMIUM_BUG_PREFIX, "BUGCR"))
elif token.startswith(V8_BUG_PREFIX):
bugs.append(token.replace(V8_BUG_PREFIX, "BUGV8_"))
else:
match = re.match("Bug\((\w+)\)$", token)
if not match:
warnings.append('unrecognized bug identifier "%s"' % token)
break
else:
bugs.append("BUG" + match.group(1).upper())
elif token.startswith("BUG"):
warnings.append('unrecognized old-style bug identifier "%s"' % token)
break
elif token == "[":
if state == "start":
state = "configuration"
elif state == "name_found":
state = "expectations"
else:
warnings.append('unexpected "["')
break
elif token == "]":
if state == "configuration":
state = "name"
elif state == "expectations":
state = "done"
else:
warnings.append('unexpected "]"')
break
elif token in ("//", ":", "="):
warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
break
elif state == "configuration":
modifiers.append(cls._configuration_tokens.get(token, token))
elif state == "expectations":
if token in ("Rebaseline", "Skip", "Slow", "WontFix"):
modifiers.append(token.upper())
elif token not in cls._expectation_tokens:
warnings.append('Unrecognized expectation "%s"' % token)
else:
expectations.append(cls._expectation_tokens.get(token, token))
elif state == "name_found":
warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
break
else:
name = token
state = "name_found"
if not warnings:
if not name:
warnings.append("Did not find a test name.")
elif state not in ("name_found", "done"):
warnings.append('Missing a "]"')
if "WONTFIX" in modifiers and "SKIP" not in modifiers and not expectations:
modifiers.append("SKIP")
if "SKIP" in modifiers and expectations:
# FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
warnings.append("A test marked Skip must not have other expectations.")
elif not expectations:
if "SKIP" not in modifiers and "REBASELINE" not in modifiers and "SLOW" not in modifiers:
modifiers.append("SKIP")
expectations = ["PASS"]
# FIXME: expectation line should just store bugs and modifiers separately.
expectation_line.modifiers = bugs + modifiers
expectation_line.expectations = expectations
expectation_line.name = name
expectation_line.warnings = warnings
return expectation_line
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
return [part.strip() for part in space_separated_string.strip().split(" ")]
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def test_to_specifier_lists(self):
converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS)
self.assertEqual(converter.to_specifiers_list(set(self._all_test_configurations)), [[]])
self.assertEqual(converter.to_specifiers_list(set()), [])
configs_to_match = set([TestConfiguration("win7", "x86", "release")])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["release", "win7"])])
configs_to_match = set([TestConfiguration("win7", "x86", "release"), TestConfiguration("win7", "x86", "debug")])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win7"])])
configs_to_match = set(
[
TestConfiguration("precise", "x86_64", "debug"),
TestConfiguration("trusty", "x86_64", "debug"),
TestConfiguration("win7", "x86", "release"),
]
)
self.assertEqual(
converter.to_specifiers_list(configs_to_match), [set(["release", "win7"]), set(["debug", "linux"])]
)
configs_to_match = set(
[
TestConfiguration("win7", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("trusty", "x86_64", "debug"),
TestConfiguration("precise", "x86_64", "debug"),
TestConfiguration("trusty", "x86_64", "debug"),
TestConfiguration("precise", "x86_64", "debug"),
]
)
self.assertEqual(
converter.to_specifiers_list(configs_to_match), [set(["release", "win7"]), set(["debug", "linux"])]
)
configs_to_match = set(
[
TestConfiguration("win7", "x86", "release"),
TestConfiguration("snowleopard", "x86", "release"),
TestConfiguration("vista", "x86", "release"),
TestConfiguration("precise", "x86_64", "release"),
TestConfiguration("trusty", "x86_64", "release"),
]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["release"])])
configs_to_match = set(
[TestConfiguration("win7", "x86", "release"), TestConfiguration("snowleopard", "x86", "release")]
)
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(["win7", "mac", "release"])])
configs_to_match = set(
[
TestConfiguration("snowleopard", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("win7", "x86", "debug"),
TestConfiguration("trusty", "x86_64", "release"),
]
)
self.assertEqual(
converter.to_specifiers_list(configs_to_match), [set(["win7"]), set(["release", "mac", "trusty"])]
)
def execute(self, options, args, tool):
port = tool.port_factory.get("chromium-win-win7") # FIXME: This should be selectable.
parser = TestExpectationParser(port, [], allow_rebaseline_modifier=False)
expectation_lines = parser.parse(port.test_expectations())
converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
tool.filesystem.write_text_file(port.path_to_test_expectations_file(), TestExpectationSerializer.list_to_string(expectation_lines, converter))
def test_to_config_set(self):
converter = TestConfigurationConverter(self._all_test_configurations)
self.assertEqual(converter.to_config_set(set()), self._all_test_configurations)
self.assertEqual(converter.to_config_set(set(["foo"])), set())
self.assertEqual(converter.to_config_set(set(["win7", "foo"])), set())
errors = []
self.assertEqual(converter.to_config_set(set(["win7", "foo"]), errors), set())
self.assertEqual(errors, ["Unrecognized specifier 'foo'"])
self.assertEqual(converter.to_config_set(set(["win7", "x86_64"])), set())
configs_to_match = set([TestConfiguration("win7", "x86", "release")])
self.assertEqual(converter.to_config_set(set(["win7", "release"])), configs_to_match)
configs_to_match = set(
[
TestConfiguration("snowleopard", "x86", "release"),
TestConfiguration("vista", "x86", "release"),
TestConfiguration("win7", "x86", "release"),
TestConfiguration("precise", "x86_64", "release"),
TestConfiguration("trusty", "x86_64", "release"),
]
)
self.assertEqual(converter.to_config_set(set(["release"])), configs_to_match)
configs_to_match = set(
[
TestConfiguration("precise", "x86_64", "release"),
TestConfiguration("precise", "x86_64", "debug"),
TestConfiguration("trusty", "x86_64", "release"),
TestConfiguration("trusty", "x86_64", "debug"),
]
)
self.assertEqual(converter.to_config_set(set(["x86_64"])), configs_to_match)
configs_to_match = set(
[
TestConfiguration("trusty", "x86_64", "release"),
TestConfiguration("trusty", "x86_64", "debug"),
TestConfiguration("precise", "x86_64", "release"),
TestConfiguration("precise", "x86_64", "debug"),
TestConfiguration("snowleopard", "x86", "release"),
TestConfiguration("snowleopard", "x86", "debug"),
]
)
self.assertEqual(converter.to_config_set(set(["trusty", "precise", "snowleopard"])), configs_to_match)
configs_to_match = set(
[TestConfiguration("snowleopard", "x86", "release"), TestConfiguration("snowleopard", "x86", "debug")]
)
self.assertEqual(converter.to_config_set(set(["snowleopard", "x86"])), configs_to_match)
configs_to_match = set(
[TestConfiguration("trusty", "x86_64", "release"), TestConfiguration("snowleopard", "x86", "release")]
)
self.assertEqual(converter.to_config_set(set(["trusty", "snowleopard", "release"])), configs_to_match)
def test_symmetric_difference(self):
self.assertEqual(TestConfigurationConverter.symmetric_difference([set(['a', 'b']), set(['b', 'c'])]), set(['a', 'c']))
self.assertEqual(TestConfigurationConverter.symmetric_difference(
[set(['a', 'b']), set(['b', 'c']), set(['b', 'd'])]), set(['a', 'c', 'd']))
def __init__(self, port, full_test_list, is_lint_mode):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._is_lint_mode = is_lint_mode
class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
# FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make the case studdly-caps to match the actual file contents.
REBASELINE_MODIFIER = 'rebaseline'
NEEDS_REBASELINE_MODIFIER = 'needsrebaseline'
NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline'
PASS_EXPECTATION = 'pass'
SKIP_MODIFIER = 'skip'
SLOW_MODIFIER = 'slow'
WONTFIX_MODIFIER = 'wontfix'
TIMEOUT_EXPECTATION = 'timeout'
MISSING_BUG_WARNING = 'Test lacks BUG specifier.'
def __init__(self, port, full_test_list, is_lint_mode):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._is_lint_mode = is_lint_mode
def parse(self, filename, expectations_string):
expectation_lines = []
line_number = 0
for line in expectations_string.split("\n"):
line_number += 1
test_expectation = self._tokenize_line(filename, line, line_number)
self._parse_line(test_expectation)
expectation_lines.append(test_expectation)
return expectation_lines
def _create_expectation_line(self, test_name, expectations, file_name):
expectation_line = TestExpectationLine()
expectation_line.original_string = test_name
expectation_line.name = test_name
expectation_line.filename = file_name
expectation_line.expectations = expectations
return expectation_line
def expectation_line_for_test(self, test_name, expectations):
expectation_line = self._create_expectation_line(test_name, expectations, '<Bot TestExpectations>')
self._parse_line(expectation_line)
return expectation_line
def expectation_for_skipped_test(self, test_name):
if not self._port.test_exists(test_name):
_log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
expectation_line = self._create_expectation_line(test_name, [TestExpectationParser.PASS_EXPECTATION], '<Skipped file>')
expectation_line.expectations = [TestExpectationParser.SKIP_MODIFIER, TestExpectationParser.WONTFIX_MODIFIER]
expectation_line.is_skipped_outside_expectations_file = True
self._parse_line(expectation_line)
return expectation_line
def _parse_line(self, expectation_line):
if not expectation_line.name:
return
if not self._check_test_exists(expectation_line):
return
expectation_line.is_file = self._port.test_isfile(expectation_line.name)
if expectation_line.is_file:
expectation_line.path = expectation_line.name
else:
expectation_line.path = self._port.normalize_test_name(expectation_line.name)
self._collect_matching_tests(expectation_line)
self._parse_specifiers(expectation_line)
self._parse_expectations(expectation_line)
def _parse_specifiers(self, expectation_line):
if self._is_lint_mode:
self._lint_line(expectation_line)
parsed_specifiers = set([specifier.lower() for specifier in expectation_line.specifiers])
expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
def _lint_line(self, expectation_line):
expectations = [expectation.lower() for expectation in expectation_line.expectations]
if not expectation_line.bugs and self.WONTFIX_MODIFIER not in expectations:
expectation_line.warnings.append(self.MISSING_BUG_WARNING)
if self.REBASELINE_MODIFIER in expectations:
expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
expectation_line.warnings.append('Unsupported expectation: %s' % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_test_exists(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
# Log a warning here since you hit this case any
# time you update TestExpectations without syncing
# the LayoutTests directory
expectation_line.warnings.append('Path does not exist.')
return False
return True
def _collect_matching_tests(self, expectation_line):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
# FIXME: full_test_list can quickly contain a big amount of
# elements. We should consider at some point to use a more
# efficient structure instead of a list. Maybe a dictionary of
# lists to represent the tree of tests, leaves being test
# files and nodes being categories.
if not self._full_test_list:
expectation_line.matching_tests = [expectation_line.path]
return
if not expectation_line.is_file:
# this is a test category, return all the tests of the category.
expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
return
# this is a test file, do a quick check if it's in the
# full test suite.
if expectation_line.path in self._full_test_list:
expectation_line.matching_tests.append(expectation_line.path)
# FIXME: Update the original specifiers and remove this once the old syntax is gone.
_configuration_tokens_list = [
'Mac', 'SnowLeopard', 'Lion', 'Retina', 'MountainLion',
'Win', 'XP', 'Win7',
'Linux',
'Android',
'Release',
'Debug',
]
_configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
_inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
# FIXME: Update the original specifiers list and remove this once the old syntax is gone.
_expectation_tokens = {
'Crash': 'CRASH',
'Failure': 'FAIL',
'ImageOnlyFailure': 'IMAGE',
MISSING_KEYWORD: 'MISSING',
'Pass': '******',
'Rebaseline': 'REBASELINE',
NEEDS_REBASELINE_KEYWORD: 'NEEDSREBASELINE',
NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE',
'Skip': 'SKIP',
'Slow': 'SLOW',
'Timeout': 'TIMEOUT',
'WontFix': 'WONTFIX',
}
_inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
[('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
# FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
@classmethod
def _tokenize_line(cls, filename, expectation_string, line_number):
"""Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
The new format for a test expectation line is:
[[bugs] [ "[" <configuration specifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
Any errant whitespace is not preserved.
"""
expectation_line = TestExpectationLine()
expectation_line.original_string = expectation_string
expectation_line.filename = filename
expectation_line.line_numbers = str(line_number)
comment_index = expectation_string.find("#")
if comment_index == -1:
comment_index = len(expectation_string)
else:
expectation_line.comment = expectation_string[comment_index + 1:]
remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
if len(remaining_string) == 0:
return expectation_line
# special-case parsing this so that we fail immediately instead of treating this as a test name
if remaining_string.startswith('//'):
expectation_line.warnings = ['use "#" instead of "//" for comments']
return expectation_line
bugs = []
specifiers = []
name = None
expectations = []
warnings = []
has_unrecognized_expectation = False
tokens = remaining_string.split()
state = 'start'
for token in tokens:
if (token.startswith(WEBKIT_BUG_PREFIX) or
token.startswith(CHROMIUM_BUG_PREFIX) or
token.startswith(V8_BUG_PREFIX) or
token.startswith(NAMED_BUG_PREFIX)):
if state != 'start':
warnings.append('"%s" is not at the start of the line.' % token)
break
if token.startswith(WEBKIT_BUG_PREFIX):
bugs.append(token)
elif token.startswith(CHROMIUM_BUG_PREFIX):
bugs.append(token)
elif token.startswith(V8_BUG_PREFIX):
bugs.append(token)
else:
match = re.match('Bug\((\w+)\)$', token)
if not match:
warnings.append('unrecognized bug identifier "%s"' % token)
break
else:
bugs.append(token)
elif token == '[':
if state == 'start':
state = 'configuration'
elif state == 'name_found':
state = 'expectations'
else:
warnings.append('unexpected "["')
break
elif token == ']':
if state == 'configuration':
state = 'name'
elif state == 'expectations':
state = 'done'
else:
warnings.append('unexpected "]"')
break
elif token in ('//', ':', '='):
warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
break
elif state == 'configuration':
specifiers.append(cls._configuration_tokens.get(token, token))
elif state == 'expectations':
if token not in cls._expectation_tokens:
has_unrecognized_expectation = True
warnings.append('Unrecognized expectation "%s"' % token)
else:
expectations.append(cls._expectation_tokens.get(token, token))
elif state == 'name_found':
warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
break
else:
name = token
state = 'name_found'
if not warnings:
if not name:
warnings.append('Did not find a test name.')
elif state not in ('name_found', 'done'):
warnings.append('Missing a "]"')
if 'WONTFIX' in expectations and 'SKIP' not in expectations:
expectations.append('SKIP')
if ('SKIP' in expectations or 'WONTFIX' in expectations) and len(set(expectations) - set(['SKIP', 'WONTFIX'])):
warnings.append('A test marked Skip or WontFix must not have other expectations.')
if not expectations and not has_unrecognized_expectation:
warnings.append('Missing expectations.')
expectation_line.bugs = bugs
expectation_line.specifiers = specifiers
expectation_line.expectations = expectations
expectation_line.name = name
expectation_line.warnings = warnings
return expectation_line
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
return [part.strip() for part in space_separated_string.strip().split(' ')]
class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
DUMMY_BUG_MODIFIER = "bug_dummy"
BUG_MODIFIER_PREFIX = 'bug'
BUG_MODIFIER_REGEX = 'bug\d+'
REBASELINE_MODIFIER = 'rebaseline'
PASS_EXPECTATION = 'pass'
SKIP_MODIFIER = 'skip'
SLOW_MODIFIER = 'slow'
WONTFIX_MODIFIER = 'wontfix'
TIMEOUT_EXPECTATION = 'timeout'
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def parse(self, expectations_string):
expectations = TestExpectationParser._tokenize_list(expectations_string)
for expectation_line in expectations:
self._parse_line(expectation_line)
return expectations
def expectation_for_skipped_test(self, test_name):
expectation_line = TestExpectationLine()
expectation_line.original_string = test_name
expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
expectation_line.name = test_name
expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
self._parse_line(expectation_line)
return expectation_line
def _parse_line(self, expectation_line):
if not expectation_line.name:
return
self._check_modifiers_against_expectations(expectation_line)
expectation_line.is_file = self._port.test_isfile(expectation_line.name)
if not expectation_line.is_file and self._check_path_does_not_exist(expectation_line):
return
if expectation_line.is_file:
expectation_line.path = expectation_line.name
else:
expectation_line.path = self._port.normalize_test_name(expectation_line.name)
self._collect_matching_tests(expectation_line)
self._parse_modifiers(expectation_line)
self._parse_expectations(expectation_line)
def _parse_modifiers(self, expectation_line):
has_wontfix = False
has_bugid = False
parsed_specifiers = set()
for modifier in expectation_line.modifiers:
if modifier in TestExpectations.MODIFIERS:
expectation_line.parsed_modifiers.append(modifier)
if modifier == self.WONTFIX_MODIFIER:
has_wontfix = True
elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
has_bugid = True
if re.match(self.BUG_MODIFIER_REGEX, modifier):
expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
else:
expectation_line.parsed_bug_modifiers.append(modifier)
else:
parsed_specifiers.add(modifier)
if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid:
expectation_line.warnings.append('Test lacks BUG modifier.')
if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers:
expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
expectation_line.warnings.append('Unsupported expectation: %s' % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_modifiers_against_expectations(self, expectation_line):
if self.SLOW_MODIFIER in expectation_line.modifiers and self.TIMEOUT_EXPECTATION in expectation_line.expectations:
expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
def _check_path_does_not_exist(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
if (not self._port.test_exists(expectation_line.name)
and not self._port.test_exists(expectation_line.name + '-disabled')):
# Log a warning here since you hit this case any
# time you update test_expectations.txt without syncing
# the LayoutTests directory
expectation_line.warnings.append('Path does not exist.')
return True
return False
def _collect_matching_tests(self, expectation_line):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
# FIXME: full_test_list can quickly contain a big amount of
# elements. We should consider at some point to use a more
# efficient structure instead of a list. Maybe a dictionary of
# lists to represent the tree of tests, leaves being test
# files and nodes being categories.
if not self._full_test_list:
expectation_line.matching_tests = [expectation_line.path]
return
if not expectation_line.is_file:
# this is a test category, return all the tests of the category.
expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
return
# this is a test file, do a quick check if it's in the
# full test suite.
if expectation_line.path in self._full_test_list:
expectation_line.matching_tests.append(expectation_line.path)
@classmethod
def _tokenize(cls, expectation_string, line_number=None):
"""Tokenizes a line from test_expectations.txt and returns an unparsed TestExpectationLine instance.
The format of a test expectation line is:
[[<modifiers>] : <name> = <expectations>][ //<comment>]
Any errant whitespace is not preserved.
"""
expectation_line = TestExpectationLine()
expectation_line.original_string = expectation_string
expectation_line.line_number = line_number
comment_index = expectation_string.find("//")
if comment_index == -1:
comment_index = len(expectation_string)
else:
expectation_line.comment = expectation_string[comment_index + 2:]
remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
if len(remaining_string) == 0:
return expectation_line
parts = remaining_string.split(':')
if len(parts) != 2:
expectation_line.warnings.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'")
else:
test_and_expectation = parts[1].split('=')
if len(test_and_expectation) != 2:
expectation_line.warnings.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='")
if not expectation_line.is_invalid():
expectation_line.modifiers = cls._split_space_separated(parts[0])
expectation_line.name = test_and_expectation[0].strip()
expectation_line.expectations = cls._split_space_separated(test_and_expectation[1])
return expectation_line
@classmethod
def _tokenize_list(cls, expectations_string):
"""Returns a list of TestExpectationLines, one for each line in expectations_string."""
expectation_lines = []
line_number = 0
for line in expectations_string.split("\n"):
line_number += 1
expectation_lines.append(cls._tokenize(line, line_number))
return expectation_lines
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
# FIXME: Lower-casing is necessary to support legacy code. Need to eliminate.
return [part.strip().lower() for part in space_separated_string.strip().split(' ')]
class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
DUMMY_BUG_MODIFIER = "bug_dummy"
BUG_MODIFIER_PREFIX = 'bug'
BUG_MODIFIER_REGEX = 'bug\d+'
REBASELINE_MODIFIER = 'rebaseline'
PASS_EXPECTATION = 'pass'
SKIP_MODIFIER = 'skip'
SLOW_MODIFIER = 'slow'
WONTFIX_MODIFIER = 'wontfix'
TIMEOUT_EXPECTATION = 'timeout'
MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def parse(self, filename, expectations_string):
expectation_lines = []
line_number = 0
for line in expectations_string.split("\n"):
line_number += 1
test_expectation = self._tokenize_line(filename, line, line_number)
self._parse_line(test_expectation)
expectation_lines.append(test_expectation)
return expectation_lines
def expectation_for_skipped_test(self, test_name):
if not self._port.test_exists(test_name):
_log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
expectation_line = TestExpectationLine()
expectation_line.original_string = test_name
expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
# FIXME: It's not clear what the expectations for a skipped test should be; the expectations
# might be different for different entries in a Skipped file, or from the command line, or from
# only running parts of the tests. It's also not clear if it matters much.
expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
expectation_line.name = test_name
# FIXME: we should pass in a more descriptive string here.
expectation_line.filename = '<Skipped file>'
expectation_line.line_number = 0
expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
self._parse_line(expectation_line)
return expectation_line
def _parse_line(self, expectation_line):
if not expectation_line.name:
return
if not self._check_test_exists(expectation_line):
return
expectation_line.is_file = self._port.test_isfile(expectation_line.name)
if expectation_line.is_file:
expectation_line.path = expectation_line.name
else:
expectation_line.path = self._port.normalize_test_name(expectation_line.name)
self._collect_matching_tests(expectation_line)
self._parse_modifiers(expectation_line)
self._parse_expectations(expectation_line)
def _parse_modifiers(self, expectation_line):
has_wontfix = False
has_bugid = False
parsed_specifiers = set()
modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
expectations = [expectation.lower() for expectation in expectation_line.expectations]
if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
for modifier in modifiers:
if modifier in TestExpectations.MODIFIERS:
expectation_line.parsed_modifiers.append(modifier)
if modifier == self.WONTFIX_MODIFIER:
has_wontfix = True
elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
has_bugid = True
if re.match(self.BUG_MODIFIER_REGEX, modifier):
expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
else:
expectation_line.parsed_bug_modifiers.append(modifier)
else:
parsed_specifiers.add(modifier)
if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
expectation_line.warnings.append(self.MISSING_BUG_WARNING)
if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
expectation_line.warnings.append('Unsupported expectation: %s' % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_test_exists(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
# Log a warning here since you hit this case any
# time you update TestExpectations without syncing
# the LayoutTests directory
expectation_line.warnings.append('Path does not exist.')
return False
return True
def _collect_matching_tests(self, expectation_line):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
# FIXME: full_test_list can quickly contain a big amount of
# elements. We should consider at some point to use a more
# efficient structure instead of a list. Maybe a dictionary of
# lists to represent the tree of tests, leaves being test
# files and nodes being categories.
if not self._full_test_list:
expectation_line.matching_tests = [expectation_line.path]
return
if not expectation_line.is_file:
# this is a test category, return all the tests of the category.
expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
return
# this is a test file, do a quick check if it's in the
# full test suite.
if expectation_line.path in self._full_test_list:
expectation_line.matching_tests.append(expectation_line.path)
# FIXME: Update the original modifiers and remove this once the old syntax is gone.
_configuration_tokens_list = [
'Mac', 'SnowLeopard', 'Lion', 'MountainLion',
'Win', 'XP', 'Vista', 'Win7',
'Linux',
'Android',
'Release',
'Debug',
]
_configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
_inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
# FIXME: Update the original modifiers list and remove this once the old syntax is gone.
_expectation_tokens = {
'Crash': 'CRASH',
'Failure': 'FAIL',
'ImageOnlyFailure': 'IMAGE',
'Missing': 'MISSING',
'Pass': '******',
'Rebaseline': 'REBASELINE',
'Skip': 'SKIP',
'Slow': 'SLOW',
'Timeout': 'TIMEOUT',
'WontFix': 'WONTFIX',
}
_inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
[('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
# FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
@classmethod
def _tokenize_line(cls, filename, expectation_string, line_number):
"""Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
The new format for a test expectation line is:
[[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
Any errant whitespace is not preserved.
"""
expectation_line = TestExpectationLine()
expectation_line.original_string = expectation_string
expectation_line.filename = filename
expectation_line.line_number = line_number
comment_index = expectation_string.find("#")
if comment_index == -1:
comment_index = len(expectation_string)
else:
expectation_line.comment = expectation_string[comment_index + 1:]
remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
if len(remaining_string) == 0:
return expectation_line
# special-case parsing this so that we fail immediately instead of treating this as a test name
if remaining_string.startswith('//'):
expectation_line.warnings = ['use "#" instead of "//" for comments']
return expectation_line
bugs = []
modifiers = []
name = None
expectations = []
warnings = []
WEBKIT_BUG_PREFIX = 'webkit.org/b/'
CHROMIUM_BUG_PREFIX = 'crbug.com/'
V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id='
tokens = remaining_string.split()
state = 'start'
for token in tokens:
if (token.startswith(WEBKIT_BUG_PREFIX) or
token.startswith(CHROMIUM_BUG_PREFIX) or
token.startswith(V8_BUG_PREFIX) or
token.startswith('Bug(')):
if state != 'start':
warnings.append('"%s" is not at the start of the line.' % token)
break
if token.startswith(WEBKIT_BUG_PREFIX):
bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
elif token.startswith(CHROMIUM_BUG_PREFIX):
bugs.append(token.replace(CHROMIUM_BUG_PREFIX, 'BUGCR'))
elif token.startswith(V8_BUG_PREFIX):
bugs.append(token.replace(V8_BUG_PREFIX, 'BUGV8_'))
else:
match = re.match('Bug\((\w+)\)$', token)
if not match:
warnings.append('unrecognized bug identifier "%s"' % token)
break
else:
bugs.append('BUG' + match.group(1).upper())
elif token.startswith('BUG'):
warnings.append('unrecognized old-style bug identifier "%s"' % token)
break
elif token == '[':
if state == 'start':
state = 'configuration'
elif state == 'name_found':
state = 'expectations'
else:
warnings.append('unexpected "["')
break
elif token == ']':
if state == 'configuration':
state = 'name'
elif state == 'expectations':
state = 'done'
else:
warnings.append('unexpected "]"')
break
elif token in ('//', ':', '='):
warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
break
elif state == 'configuration':
modifiers.append(cls._configuration_tokens.get(token, token))
elif state == 'expectations':
if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix'):
modifiers.append(token.upper())
elif token not in cls._expectation_tokens:
warnings.append('Unrecognized expectation "%s"' % token)
else:
expectations.append(cls._expectation_tokens.get(token, token))
elif state == 'name_found':
warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
break
else:
name = token
state = 'name_found'
if not warnings:
if not name:
warnings.append('Did not find a test name.')
elif state not in ('name_found', 'done'):
warnings.append('Missing a "]"')
if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
modifiers.append('SKIP')
if 'SKIP' in modifiers and expectations:
# FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
warnings.append('A test marked Skip must not have other expectations.')
elif not expectations:
if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers:
modifiers.append('SKIP')
expectations = ['PASS']
# FIXME: expectation line should just store bugs and modifiers separately.
expectation_line.modifiers = bugs + modifiers
expectation_line.expectations = expectations
expectation_line.name = name
expectation_line.warnings = warnings
return expectation_line
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
return [part.strip() for part in space_separated_string.strip().split(' ')]
def test_specifier_converter_access(self):
specifier_sorter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS).specifier_sorter()
self.assertEqual(specifier_sorter.category_for_specifier("snowleopard"), "version")
self.assertEqual(specifier_sorter.category_for_specifier("mac"), "version")
def test_to_config_set(self):
converter = TestConfigurationConverter(self._all_test_configurations)
self.assertEqual(converter.to_config_set(set()), self._all_test_configurations)
self.assertEqual(converter.to_config_set(set(['foo'])), set())
self.assertEqual(converter.to_config_set(set(['win7', 'foo'])), set())
errors = []
self.assertEqual(converter.to_config_set(set(['win7', 'foo']), errors), set())
self.assertEqual(errors, ["Unrecognized specifier 'foo'"])
self.assertEqual(converter.to_config_set(set(['win7', 'x86_64'])), set())
configs_to_match = set([
TestConfiguration('win7', 'x86', 'release'),
])
self.assertEqual(converter.to_config_set(set(['win7', 'release'])), configs_to_match)
configs_to_match = set([
TestConfiguration('snowleopard', 'x86', 'release'),
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('linux32', 'x86', 'release'),
TestConfiguration('precise', 'x86_64', 'release'),
TestConfiguration('trusty', 'x86_64', 'release'),
])
self.assertEqual(converter.to_config_set(set(['release'])), configs_to_match)
configs_to_match = set([
TestConfiguration('precise', 'x86_64', 'release'),
TestConfiguration('precise', 'x86_64', 'debug'),
TestConfiguration('trusty', 'x86_64', 'release'),
TestConfiguration('trusty', 'x86_64', 'debug'),
])
self.assertEqual(converter.to_config_set(set(['x86_64'])), configs_to_match)
configs_to_match = set([
TestConfiguration('trusty', 'x86_64', 'release'),
TestConfiguration('trusty', 'x86_64', 'debug'),
TestConfiguration('precise', 'x86_64', 'release'),
TestConfiguration('precise', 'x86_64', 'debug'),
TestConfiguration('snowleopard', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'debug'),
])
self.assertEqual(converter.to_config_set(set(['trusty', 'precise', 'snowleopard'])),
configs_to_match)
configs_to_match = set([
TestConfiguration('linux32', 'x86', 'release'),
TestConfiguration('linux32', 'x86', 'debug'),
TestConfiguration('snowleopard', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'debug'),
])
self.assertEqual(converter.to_config_set(set(['linux32', 'snowleopard', 'x86'])),
configs_to_match)
configs_to_match = set([
TestConfiguration('trusty', 'x86_64', 'release'),
TestConfiguration('linux32', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'release'),
])
self.assertEqual(
converter.to_config_set(set(['trusty', 'linux32', 'snowleopard', 'release'])),
configs_to_match)
def test_to_specifier_lists(self):
converter = TestConfigurationConverter(self._all_test_configurations, MOCK_MACROS)
self.assertEqual(converter.to_specifiers_list(set(self._all_test_configurations)), [[]])
self.assertEqual(converter.to_specifiers_list(set()), [])
configs_to_match = set([
TestConfiguration('win7', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['release', 'win7'])])
configs_to_match = set([
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'debug'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win7'])])
configs_to_match = set([
TestConfiguration('linux32', 'x86', 'debug'),
TestConfiguration('precise', 'x86_64', 'debug'),
TestConfiguration('trusty', 'x86_64', 'debug'),
TestConfiguration('win7', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match),
[set(['release', 'win7']), set(['debug', 'linux'])])
configs_to_match = set([
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('trusty', 'x86_64', 'debug'),
TestConfiguration('precise', 'x86_64', 'debug'),
TestConfiguration('linux32', 'x86', 'debug'),
TestConfiguration('trusty', 'x86_64', 'debug'),
TestConfiguration('precise', 'x86_64', 'debug'),
TestConfiguration('linux32', 'x86', 'debug'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match),
[set(['release', 'win7']), set(['debug', 'linux'])])
configs_to_match = set([
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'release'),
TestConfiguration('vista', 'x86', 'release'),
TestConfiguration('linux32', 'x86', 'release'),
TestConfiguration('precise', 'x86_64', 'release'),
TestConfiguration('trusty', 'x86_64', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['release'])])
configs_to_match = set([
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('snowleopard', 'x86', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match), [set(['win7', 'mac', 'release'])])
configs_to_match = set([
TestConfiguration('snowleopard', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'release'),
TestConfiguration('win7', 'x86', 'debug'),
TestConfiguration('trusty', 'x86_64', 'release'),
])
self.assertEqual(converter.to_specifiers_list(configs_to_match),
[set(['win7']), set(['release', 'mac', 'trusty'])])
class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
DUMMY_BUG_MODIFIER = "bug_dummy"
BUG_MODIFIER_PREFIX = 'bug'
BUG_MODIFIER_REGEX = 'bug\d+'
REBASELINE_MODIFIER = 'rebaseline'
PASS_EXPECTATION = 'pass'
SKIP_MODIFIER = 'skip'
SLOW_MODIFIER = 'slow'
WONTFIX_MODIFIER = 'wontfix'
TIMEOUT_EXPECTATION = 'timeout'
MISSING_BUG_WARNING = 'Test lacks BUG modifier.'
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def parse(self, filename, expectations_string):
expectation_lines = []
line_number = 0
for line in expectations_string.split("\n"):
line_number += 1
test_expectation = self._tokenize_line(filename, line, line_number)
self._parse_line(test_expectation)
expectation_lines.append(test_expectation)
return expectation_lines
def expectation_for_skipped_test(self, test_name):
if not self._port.test_exists(test_name):
_log.warning('The following test %s from the Skipped list doesn\'t exist' % test_name)
expectation_line = TestExpectationLine()
expectation_line.original_string = test_name
expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
# FIXME: It's not clear what the expectations for a skipped test should be; the expectations
# might be different for different entries in a Skipped file, or from the command line, or from
# only running parts of the tests. It's also not clear if it matters much.
expectation_line.modifiers.append(TestExpectationParser.WONTFIX_MODIFIER)
expectation_line.name = test_name
# FIXME: we should pass in a more descriptive string here.
expectation_line.filename = '<Skipped file>'
expectation_line.line_number = 0
expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
self._parse_line(expectation_line)
return expectation_line
def _parse_line(self, expectation_line):
if not expectation_line.name:
return
if not self._check_test_exists(expectation_line):
return
expectation_line.is_file = self._port.test_isfile(expectation_line.name)
if expectation_line.is_file:
expectation_line.path = expectation_line.name
else:
expectation_line.path = self._port.normalize_test_name(expectation_line.name)
self._collect_matching_tests(expectation_line)
self._parse_modifiers(expectation_line)
self._parse_expectations(expectation_line)
def _parse_modifiers(self, expectation_line):
has_wontfix = False
has_bugid = False
parsed_specifiers = set()
modifiers = [modifier.lower() for modifier in expectation_line.modifiers]
expectations = [expectation.lower() for expectation in expectation_line.expectations]
if self.SLOW_MODIFIER in modifiers and self.TIMEOUT_EXPECTATION in expectations:
expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
for modifier in modifiers:
if modifier in TestExpectations.MODIFIERS:
expectation_line.parsed_modifiers.append(modifier)
if modifier == self.WONTFIX_MODIFIER:
has_wontfix = True
elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
has_bugid = True
if re.match(self.BUG_MODIFIER_REGEX, modifier):
expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
else:
expectation_line.parsed_bug_modifiers.append(modifier)
else:
parsed_specifiers.add(modifier)
if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations():
expectation_line.warnings.append(self.MISSING_BUG_WARNING)
if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers:
expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
expectation_line.warnings.append('Unsupported expectation: %s' % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_test_exists(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'):
# Log a warning here since you hit this case any
# time you update TestExpectations without syncing
# the LayoutTests directory
expectation_line.warnings.append('Path does not exist.')
return False
return True
def _collect_matching_tests(self, expectation_line):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
# FIXME: full_test_list can quickly contain a big amount of
# elements. We should consider at some point to use a more
# efficient structure instead of a list. Maybe a dictionary of
# lists to represent the tree of tests, leaves being test
# files and nodes being categories.
if not self._full_test_list:
expectation_line.matching_tests = [expectation_line.path]
return
if not expectation_line.is_file:
# this is a test category, return all the tests of the category.
expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
return
# this is a test file, do a quick check if it's in the
# full test suite.
if expectation_line.path in self._full_test_list:
expectation_line.matching_tests.append(expectation_line.path)
# FIXME: Update the original modifiers and remove this once the old syntax is gone.
_configuration_tokens_list = [
'Mac', 'SnowLeopard', 'Lion', 'MountainLion', 'Mavericks',
'Win', 'XP', 'Vista', 'Win7',
'Linux',
'Android',
'Release',
'Debug',
]
_configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
_inverted_configuration_tokens = dict((value, name) for name, value in _configuration_tokens.iteritems())
# FIXME: Update the original modifiers list and remove this once the old syntax is gone.
_expectation_tokens = {
'Crash': 'CRASH',
'Failure': 'FAIL',
'ImageOnlyFailure': 'IMAGE',
'Missing': 'MISSING',
'Pass': '******',
'Rebaseline': 'REBASELINE',
'Skip': 'SKIP',
'Slow': 'SLOW',
'Timeout': 'TIMEOUT',
'WontFix': 'WONTFIX',
}
_inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
[('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
# FIXME: Seems like these should be classmethods on TestExpectationLine instead of TestExpectationParser.
@classmethod
def _tokenize_line(cls, filename, expectation_string, line_number):
"""Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
The new format for a test expectation line is:
[[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
Any errant whitespace is not preserved.
"""
expectation_line = TestExpectationLine()
expectation_line.original_string = expectation_string
expectation_line.filename = filename
expectation_line.line_number = line_number
comment_index = expectation_string.find("#")
if comment_index == -1:
comment_index = len(expectation_string)
else:
expectation_line.comment = expectation_string[comment_index + 1:]
remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
if len(remaining_string) == 0:
return expectation_line
# special-case parsing this so that we fail immediately instead of treating this as a test name
if remaining_string.startswith('//'):
expectation_line.warnings = ['use "#" instead of "//" for comments']
return expectation_line
bugs = []
modifiers = []
name = None
expectations = []
warnings = []
WEBKIT_BUG_PREFIX = 'webkit.org/b/'
tokens = remaining_string.split()
state = 'start'
for token in tokens:
if token.startswith(WEBKIT_BUG_PREFIX) or token.startswith('Bug('):
if state != 'start':
warnings.append('"%s" is not at the start of the line.' % token)
break
if token.startswith(WEBKIT_BUG_PREFIX):
bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
else:
match = re.match('Bug\((\w+)\)$', token)
if not match:
warnings.append('unrecognized bug identifier "%s"' % token)
break
else:
bugs.append('BUG' + match.group(1).upper())
elif token.startswith('BUG'):
warnings.append('unrecognized old-style bug identifier "%s"' % token)
break
elif token == '[':
if state == 'start':
state = 'configuration'
elif state == 'name_found':
state = 'expectations'
else:
warnings.append('unexpected "["')
break
elif token == ']':
if state == 'configuration':
state = 'name'
elif state == 'expectations':
state = 'done'
else:
warnings.append('unexpected "]"')
break
elif token in ('//', ':', '='):
warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
break
elif state == 'configuration':
modifiers.append(cls._configuration_tokens.get(token, token))
elif state == 'expectations':
if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix'):
modifiers.append(token.upper())
elif token not in cls._expectation_tokens:
warnings.append('Unrecognized expectation "%s"' % token)
else:
expectations.append(cls._expectation_tokens.get(token, token))
elif state == 'name_found':
warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
break
else:
name = token
state = 'name_found'
if not warnings:
if not name:
warnings.append('Did not find a test name.')
elif state not in ('name_found', 'done'):
warnings.append('Missing a "]"')
if 'WONTFIX' in modifiers and 'SKIP' not in modifiers and not expectations:
modifiers.append('SKIP')
if 'SKIP' in modifiers and expectations:
# FIXME: This is really a semantic warning and shouldn't be here. Remove when we drop the old syntax.
warnings.append('A test marked Skip must not have other expectations.')
elif not expectations:
if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers:
modifiers.append('SKIP')
expectations = ['PASS']
# FIXME: expectation line should just store bugs and modifiers separately.
expectation_line.modifiers = bugs + modifiers
expectation_line.expectations = expectations
expectation_line.name = name
expectation_line.warnings = warnings
return expectation_line
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
return [part.strip() for part in space_separated_string.strip().split(' ')]
class TestExpectationParser(object):
"""Provides parsing facilities for lines in the test_expectation.txt file."""
DUMMY_BUG_MODIFIER = "bug_dummy"
BUG_MODIFIER_PREFIX = 'bug'
BUG_MODIFIER_REGEX = 'bug\d+'
REBASELINE_MODIFIER = 'rebaseline'
PASS_EXPECTATION = 'pass'
SKIP_MODIFIER = 'skip'
SLOW_MODIFIER = 'slow'
WONTFIX_MODIFIER = 'wontfix'
TIMEOUT_EXPECTATION = 'timeout'
def __init__(self, port, full_test_list, allow_rebaseline_modifier):
self._port = port
self._test_configuration_converter = TestConfigurationConverter(set(port.all_test_configurations()), port.configuration_specifier_macros())
self._full_test_list = full_test_list
self._allow_rebaseline_modifier = allow_rebaseline_modifier
def parse(self, filename, expectations_string):
expectations = TestExpectationParser._tokenize_list(filename, expectations_string)
for expectation_line in expectations:
self._parse_line(expectation_line)
return expectations
def expectation_for_skipped_test(self, test_name):
expectation_line = TestExpectationLine()
expectation_line.original_string = test_name
expectation_line.modifiers = [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER]
expectation_line.name = test_name
# FIXME: we should pass in a more descriptive string here.
expectation_line.filename = '<Skipped file>'
expectation_line.line_number = 0
expectation_line.expectations = [TestExpectationParser.PASS_EXPECTATION]
self._parse_line(expectation_line)
return expectation_line
def _parse_line(self, expectation_line):
if not expectation_line.name:
return
self._check_modifiers_against_expectations(expectation_line)
expectation_line.is_file = self._port.test_isfile(expectation_line.name)
if not expectation_line.is_file and self._check_path_does_not_exist(expectation_line):
return
if expectation_line.is_file:
expectation_line.path = expectation_line.name
else:
expectation_line.path = self._port.normalize_test_name(expectation_line.name)
self._collect_matching_tests(expectation_line)
self._parse_modifiers(expectation_line)
self._parse_expectations(expectation_line)
def _parse_modifiers(self, expectation_line):
has_wontfix = False
has_bugid = False
parsed_specifiers = set()
for modifier in expectation_line.modifiers:
if modifier in TestExpectations.MODIFIERS:
expectation_line.parsed_modifiers.append(modifier)
if modifier == self.WONTFIX_MODIFIER:
has_wontfix = True
elif modifier.startswith(self.BUG_MODIFIER_PREFIX):
has_bugid = True
if re.match(self.BUG_MODIFIER_REGEX, modifier):
expectation_line.warnings.append('BUG\d+ is not allowed, must be one of BUGCR\d+, BUGWK\d+, BUGV8_\d+, or a non-numeric bug identifier.')
else:
expectation_line.parsed_bug_modifiers.append(modifier)
else:
parsed_specifiers.add(modifier)
if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid:
expectation_line.warnings.append('Test lacks BUG modifier.')
if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in expectation_line.modifiers:
expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
expectation_line.matching_configurations = self._test_configuration_converter.to_config_set(parsed_specifiers, expectation_line.warnings)
def _parse_expectations(self, expectation_line):
result = set()
for part in expectation_line.expectations:
expectation = TestExpectations.expectation_from_string(part)
if expectation is None: # Careful, PASS is currently 0.
expectation_line.warnings.append('Unsupported expectation: %s' % part)
continue
result.add(expectation)
expectation_line.parsed_expectations = result
def _check_modifiers_against_expectations(self, expectation_line):
if self.SLOW_MODIFIER in expectation_line.modifiers and self.TIMEOUT_EXPECTATION in expectation_line.expectations:
expectation_line.warnings.append('A test can not be both SLOW and TIMEOUT. If it times out indefinitely, then it should be just TIMEOUT.')
def _check_path_does_not_exist(self, expectation_line):
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
if (not self._port.test_exists(expectation_line.name)
and not self._port.test_exists(expectation_line.name + '-disabled')):
# Log a warning here since you hit this case any
# time you update TestExpectations without syncing
# the LayoutTests directory
expectation_line.warnings.append('Path does not exist.')
return True
return False
def _collect_matching_tests(self, expectation_line):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
# FIXME: full_test_list can quickly contain a big amount of
# elements. We should consider at some point to use a more
# efficient structure instead of a list. Maybe a dictionary of
# lists to represent the tree of tests, leaves being test
# files and nodes being categories.
if not self._full_test_list:
expectation_line.matching_tests = [expectation_line.path]
return
if not expectation_line.is_file:
# this is a test category, return all the tests of the category.
expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
return
# this is a test file, do a quick check if it's in the
# full test suite.
if expectation_line.path in self._full_test_list:
expectation_line.matching_tests.append(expectation_line.path)
@classmethod
def _tokenize(cls, filename, expectation_string, line_number):
"""Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance.
The format of a test expectation line is:
[[<modifiers>] : <name> = <expectations>][ //<comment>]
Any errant whitespace is not preserved.
"""
expectation_line = TestExpectationLine()
expectation_line.original_string = expectation_string
expectation_line.line_number = line_number
expectation_line.filename = filename
comment_index = expectation_string.find("//")
if comment_index == -1:
comment_index = len(expectation_string)
else:
expectation_line.comment = expectation_string[comment_index + 2:]
remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
if len(remaining_string) == 0:
return expectation_line
parts = remaining_string.split(':')
if len(parts) != 2:
expectation_line.warnings.append("Missing a ':'" if len(parts) < 2 else "Extraneous ':'")
else:
test_and_expectation = parts[1].split('=')
if len(test_and_expectation) != 2:
expectation_line.warnings.append("Missing expectations" if len(test_and_expectation) < 2 else "Extraneous '='")
if not expectation_line.is_invalid():
expectation_line.modifiers = cls._split_space_separated(parts[0])
expectation_line.name = test_and_expectation[0].strip()
expectation_line.expectations = cls._split_space_separated(test_and_expectation[1])
return expectation_line
@classmethod
def _tokenize_list(cls, filename, expectations_string):
"""Returns a list of TestExpectationLines, one for each line in expectations_string."""
expectation_lines = []
line_number = 0
for line in expectations_string.split("\n"):
line_number += 1
expectation_lines.append(cls._tokenize(filename, line, line_number))
return expectation_lines
@classmethod
def _split_space_separated(cls, space_separated_string):
"""Splits a space-separated string into an array."""
# FIXME: Lower-casing is necessary to support legacy code. Need to eliminate.
return [part.strip().lower() for part in space_separated_string.strip().split(' ')]
