def test_lint_globs(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
test_expectations = ('# tags: [ mac mac10.10 ]\n'
'# results: [ Failure Pass ]\n'
'[ mac ] test1 [ Failure ]\n'
'[ mac10.10 ] test2 [ Pass ]\n')
port.expectations_dict = lambda: {
'testexpectations': test_expectations
}
host.filesystem.maybe_make_directory(
host.filesystem.join(port.web_tests_dir(), 'test2'))
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertTrue(failures)
self.assertEqual(warnings, [])
all_logs = ''.join(self.logMessages())
self.assertIn('directory', all_logs)
def test_lint_flag_specific_expectation_errors(self):
options = optparse.Values({
'platform': 'test',
'debug_rwt_logging': False,
'additional_expectations': []
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {
'flag-specific': 'does/not/exist',
'noproblem': ''
}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertTrue(failures)
self.assertEqual(warnings, [])
all_logs = ''.join(self.logMessages())
self.assertIn('flag-specific', all_logs)
self.assertIn('does/not/exist', all_logs)
self.assertNotIn('noproblem', all_logs)
def test_lint_conflicts_in_test_expectations_between_os_and_os_version(
self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
test_expectations = ('# tags: [ mac mac10.10 ]\n'
'# results: [ Failure Pass ]\n'
'[ mac ] test1 [ Failure ]\n'
'[ mac10.10 ] test1 [ Pass ]\n')
port.expectations_dict = lambda: {
'testexpectations': test_expectations
}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertTrue(failures)
self.assertEqual(warnings, [])
all_logs = ''.join(self.logMessages())
self.assertIn('conflict', all_logs)
def test_virtual_test_redundant_expectation(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
port.virtual_test_suites = lambda: [
VirtualTestSuite(prefix='foo', bases=['test'], args=['--foo'])
]
test_expectations = (
'# tags: [ mac win ]\n'
'# tags: [ debug release ]\n'
'# results: [ Failure Pass ]\n'
'[ mac ] test/test1.html [ Failure ]\n'
'[ mac debug ] virtual/foo/test/test1.html [ Failure ]\n'
'[ win ] virtual/foo/test/test1.html [ Failure ]\n'
'[ mac release ] virtual/foo/test/test1.html [ Pass ]\n'
'test/test2.html [ Failure ]\n'
'crbug.com/1234 virtual/foo/test/test2.html [ Failure ]')
port.expectations_dict = lambda: {
'testexpectations': test_expectations
}
port.test_exists = lambda test: True
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertEqual(failures, [])
self.assertEquals(len(warnings), 1)
self.assertRegexpMatches(warnings[0], ':5 .*redundant with.* line 4$')
def test_lint_test_files(self):
options = optparse.Values({'platform': 'test-mac-mac10.10'})
host = MockHost()
host.port_factory.all_port_names = lambda platform=None: [platform]
res = lint_test_expectations.lint(host, options)
self.assertEqual(res, [])
def test_lint_exsistence(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
test_expectations = ('# results: [ Pass Failure ]\n'
'test1/* [ Failure ]\n'
'test2/* [ Failure ]\n'
'test2/foo.html [ Failure ]\n'
'test2/bar.html [ Failure ]\n'
'test3/foo.html [ Failure ]\n'
'virtual/foo/* [ Failure ]\n'
'virtual/foo/test2/* [ Pass ]\n'
'virtual/foo/test2/foo.html [ Pass ]\n'
'virtual/foo/test2/bar.html [ Pass ]\n'
'virtual/foo/test3/foo.html [ Pass ]\n'
'virtual/bar/* [ Pass ]\n'
'virtual/bar/test2/foo.html [ Pass ]\n'
'external/wpt/abc/def [ Failure ]\n')
port.expectations_dict = lambda: {
'testexpectations': test_expectations
}
port.virtual_test_suites = lambda: [
VirtualTestSuite(prefix='foo', bases=['test2'], args=['--foo'])
]
host.filesystem.write_text_file(
host.filesystem.join(port.web_tests_dir(), 'test2', 'foo.html'),
'foo')
host.filesystem.write_text_file(
host.filesystem.join(port.web_tests_dir(), 'test3', 'foo.html'),
'foo')
host.filesystem.write_text_file(
host.filesystem.join(port.web_tests_dir(), 'virtual', 'foo',
'README.md'), 'foo')
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertTrue(failures)
self.assertEqual(warnings, [])
self.assertEquals(len(failures), 6)
expected_non_existence = [
'test1/*',
'test2/bar.html',
'virtual/foo/test2/bar.html',
'virtual/foo/test3/foo.html',
'virtual/bar/*',
'virtual/bar/test2/foo.html',
]
for i in range(len(failures)):
self.assertIn('Test does not exist', failures[i])
self.assertIn(expected_non_existence[i], failures[i])
def test_lint_test_files(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test-mac-mac10.10'
})
host = MockHost()
host.port_factory.all_port_names = lambda platform=None: [platform]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertEqual(failures, [])
self.assertEqual(warnings, [])
def test_all_configurations(self):
host = MockHost()
host.ports_parsed = []
host.port_factory = FakeFactory(
host,
(FakePort(host, 'a', 'path-to-a'), FakePort(
host, 'b', 'path-to-b'), FakePort(host, 'b-win', 'path-to-b')))
options = optparse.Values({'platform': None})
res = lint_test_expectations.lint(host, options)
self.assertEqual(res, [])
self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
def test_only_wpt_in_android_override_files(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
raw_expectations = ('# results: [ Failure ]\n'
'external/wpt/test.html [ Failure ]\n'
'non-wpt/test.html [ Failure ]\n')
for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values():
host.filesystem.write_text_file(path, raw_expectations)
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
port.test_exists = lambda _: True
port.tests = lambda _: {'external/wpt/test.html', 'non-wpt/test.html'}
failures, _ = lint_test_expectations.lint(host, options)
self.assertTrue(all('is for a non WPT test' in f for f in failures))
def test_extra_files_errors(self):
options = optparse.Values({
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
host.filesystem.write_text_file(
'/test.checkout/wtests/LeakExpectations', '-- syntax error')
res = lint_test_expectations.lint(host, options)
self.assertTrue(res)
all_logs = ''.join(self.logMessages())
self.assertIn('LeakExpectations:1', all_logs)
def test_lint_test_files_errors(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {'foo': '-- syntax error1', 'bar': '-- syntax error2'}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertTrue(failures)
self.assertEqual(warnings, [])
all_logs = ''.join(self.logMessages())
self.assertIn('foo', all_logs)
self.assertIn('bar', all_logs)
def test_extra_files_errors(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
port.expectations_dict = lambda: {}
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
host.filesystem.write_text_file(WEB_TEST_DIR + '/LeakExpectations',
'-- syntax error')
failures, warnings = lint_test_expectations.lint(host, options)
self.assertTrue(failures)
self.assertEqual(warnings, [])
all_logs = ''.join(self.logMessages())
self.assertIn('LeakExpectations', all_logs)
def test_never_fix_tests(self):
options = optparse.Values({
'additional_expectations': [],
'platform': 'test',
'debug_rwt_logging': False
})
host = MockHost()
port = host.port_factory.get(options.platform, options=options)
port.virtual_test_suites = lambda: [
VirtualTestSuite(
prefix='foo', bases=['test', 'test1'], args=['--foo'])
]
test_expectations = ('# tags: [ mac win ]\n'
'# results: [ Skip Pass ]\n'
'test/* [ Skip ]\n'
'[ mac ] test1/* [ Skip ]\n'
'test/sub/* [ Pass ]\n'
'test/test1.html [ Pass ]\n'
'test1/foo/* [ Pass ]\n'
'test2/* [ Pass ]\n'
'test2.html [ Skip Pass ]\n'
'virtual/foo/test/* [ Pass ]\n'
'virtual/foo/test1/* [ Pass ]\n')
port.expectations_dict = lambda: {'NeverFixTests': test_expectations}
port.test_exists = lambda test: True
host.port_factory.get = lambda platform, options=None: port
host.port_factory.all_port_names = lambda platform=None: [port.name()]
failures, warnings = lint_test_expectations.lint(host, options)
self.assertEqual(warnings, [])
self.assertEquals(len(failures), 4)
self.assertRegexpMatches(failures[0], ':7 .*must override')
self.assertRegexpMatches(failures[1], ':8 .*must override')
self.assertRegexpMatches(failures[2], ':9 Only one of')
self.assertRegexpMatches(failures[3], ':11 .*must override')