def test_lint_globs(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) test_expectations = ('# tags: [ mac mac10.10 ]\n' '# results: [ Failure Pass ]\n' '[ mac ] test1 [ Failure ]\n' '[ mac10.10 ] test2 [ Pass ]\n') port.expectations_dict = lambda: { 'testexpectations': test_expectations } host.filesystem.maybe_make_directory( host.filesystem.join(port.web_tests_dir(), 'test2')) host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertTrue(failures) self.assertEqual(warnings, []) all_logs = ''.join(self.logMessages()) self.assertIn('directory', all_logs)
def test_lint_flag_specific_expectation_errors(self): options = optparse.Values({ 'platform': 'test', 'debug_rwt_logging': False, 'additional_expectations': [] }) host = MockHost() port = host.port_factory.get(options.platform, options=options) port.expectations_dict = lambda: { 'flag-specific': 'does/not/exist', 'noproblem': '' } host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertTrue(failures) self.assertEqual(warnings, []) all_logs = ''.join(self.logMessages()) self.assertIn('flag-specific', all_logs) self.assertIn('does/not/exist', all_logs) self.assertNotIn('noproblem', all_logs)
def test_lint_conflicts_in_test_expectations_between_os_and_os_version( self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) test_expectations = ('# tags: [ mac mac10.10 ]\n' '# results: [ Failure Pass ]\n' '[ mac ] test1 [ Failure ]\n' '[ mac10.10 ] test1 [ Pass ]\n') port.expectations_dict = lambda: { 'testexpectations': test_expectations } host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertTrue(failures) self.assertEqual(warnings, []) all_logs = ''.join(self.logMessages()) self.assertIn('conflict', all_logs)
def test_virtual_test_redundant_expectation(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) port.virtual_test_suites = lambda: [ VirtualTestSuite(prefix='foo', bases=['test'], args=['--foo']) ] test_expectations = ( '# tags: [ mac win ]\n' '# tags: [ debug release ]\n' '# results: [ Failure Pass ]\n' '[ mac ] test/test1.html [ Failure ]\n' '[ mac debug ] virtual/foo/test/test1.html [ Failure ]\n' '[ win ] virtual/foo/test/test1.html [ Failure ]\n' '[ mac release ] virtual/foo/test/test1.html [ Pass ]\n' 'test/test2.html [ Failure ]\n' 'crbug.com/1234 virtual/foo/test/test2.html [ Failure ]') port.expectations_dict = lambda: { 'testexpectations': test_expectations } port.test_exists = lambda test: True host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertEqual(failures, []) self.assertEquals(len(warnings), 1) self.assertRegexpMatches(warnings[0], ':5 .*redundant with.* line 4$')
def test_lint_test_files(self): options = optparse.Values({'platform': 'test-mac-mac10.10'}) host = MockHost() host.port_factory.all_port_names = lambda platform=None: [platform] res = lint_test_expectations.lint(host, options) self.assertEqual(res, [])
def test_lint_exsistence(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) test_expectations = ('# results: [ Pass Failure ]\n' 'test1/* [ Failure ]\n' 'test2/* [ Failure ]\n' 'test2/foo.html [ Failure ]\n' 'test2/bar.html [ Failure ]\n' 'test3/foo.html [ Failure ]\n' 'virtual/foo/* [ Failure ]\n' 'virtual/foo/test2/* [ Pass ]\n' 'virtual/foo/test2/foo.html [ Pass ]\n' 'virtual/foo/test2/bar.html [ Pass ]\n' 'virtual/foo/test3/foo.html [ Pass ]\n' 'virtual/bar/* [ Pass ]\n' 'virtual/bar/test2/foo.html [ Pass ]\n' 'external/wpt/abc/def [ Failure ]\n') port.expectations_dict = lambda: { 'testexpectations': test_expectations } port.virtual_test_suites = lambda: [ VirtualTestSuite(prefix='foo', bases=['test2'], args=['--foo']) ] host.filesystem.write_text_file( host.filesystem.join(port.web_tests_dir(), 'test2', 'foo.html'), 'foo') host.filesystem.write_text_file( host.filesystem.join(port.web_tests_dir(), 'test3', 'foo.html'), 'foo') host.filesystem.write_text_file( host.filesystem.join(port.web_tests_dir(), 'virtual', 'foo', 'README.md'), 'foo') host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertTrue(failures) self.assertEqual(warnings, []) self.assertEquals(len(failures), 6) expected_non_existence = [ 'test1/*', 'test2/bar.html', 'virtual/foo/test2/bar.html', 'virtual/foo/test3/foo.html', 'virtual/bar/*', 'virtual/bar/test2/foo.html', ] for i in range(len(failures)): self.assertIn('Test does not exist', failures[i]) self.assertIn(expected_non_existence[i], failures[i])
def test_lint_test_files(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test-mac-mac10.10' }) host = MockHost() host.port_factory.all_port_names = lambda platform=None: [platform] failures, warnings = lint_test_expectations.lint(host, options) self.assertEqual(failures, []) self.assertEqual(warnings, [])
def test_all_configurations(self): host = MockHost() host.ports_parsed = [] host.port_factory = FakeFactory( host, (FakePort(host, 'a', 'path-to-a'), FakePort( host, 'b', 'path-to-b'), FakePort(host, 'b-win', 'path-to-b'))) options = optparse.Values({'platform': None}) res = lint_test_expectations.lint(host, options) self.assertEqual(res, []) self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
def test_only_wpt_in_android_override_files(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) raw_expectations = ('# results: [ Failure ]\n' 'external/wpt/test.html [ Failure ]\n' 'non-wpt/test.html [ Failure ]\n') for path in PRODUCTS_TO_EXPECTATION_FILE_PATHS.values(): host.filesystem.write_text_file(path, raw_expectations) host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] port.test_exists = lambda _: True port.tests = lambda _: {'external/wpt/test.html', 'non-wpt/test.html'} failures, _ = lint_test_expectations.lint(host, options) self.assertTrue(all('is for a non WPT test' in f for f in failures))
def test_extra_files_errors(self): options = optparse.Values({ 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) port.expectations_dict = lambda: {} host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] host.filesystem.write_text_file( '/test.checkout/wtests/LeakExpectations', '-- syntax error') res = lint_test_expectations.lint(host, options) self.assertTrue(res) all_logs = ''.join(self.logMessages()) self.assertIn('LeakExpectations:1', all_logs)
def test_lint_test_files_errors(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) port.expectations_dict = lambda: {'foo': '-- syntax error1', 'bar': '-- syntax error2'} host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertTrue(failures) self.assertEqual(warnings, []) all_logs = ''.join(self.logMessages()) self.assertIn('foo', all_logs) self.assertIn('bar', all_logs)
def test_extra_files_errors(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) port.expectations_dict = lambda: {} host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] host.filesystem.write_text_file(WEB_TEST_DIR + '/LeakExpectations', '-- syntax error') failures, warnings = lint_test_expectations.lint(host, options) self.assertTrue(failures) self.assertEqual(warnings, []) all_logs = ''.join(self.logMessages()) self.assertIn('LeakExpectations', all_logs)
def test_never_fix_tests(self): options = optparse.Values({ 'additional_expectations': [], 'platform': 'test', 'debug_rwt_logging': False }) host = MockHost() port = host.port_factory.get(options.platform, options=options) port.virtual_test_suites = lambda: [ VirtualTestSuite( prefix='foo', bases=['test', 'test1'], args=['--foo']) ] test_expectations = ('# tags: [ mac win ]\n' '# results: [ Skip Pass ]\n' 'test/* [ Skip ]\n' '[ mac ] test1/* [ Skip ]\n' 'test/sub/* [ Pass ]\n' 'test/test1.html [ Pass ]\n' 'test1/foo/* [ Pass ]\n' 'test2/* [ Pass ]\n' 'test2.html [ Skip Pass ]\n' 'virtual/foo/test/* [ Pass ]\n' 'virtual/foo/test1/* [ Pass ]\n') port.expectations_dict = lambda: {'NeverFixTests': test_expectations} port.test_exists = lambda test: True host.port_factory.get = lambda platform, options=None: port host.port_factory.all_port_names = lambda platform=None: [port.name()] failures, warnings = lint_test_expectations.lint(host, options) self.assertEqual(warnings, []) self.assertEquals(len(failures), 4) self.assertRegexpMatches(failures[0], ':7 .*must override') self.assertRegexpMatches(failures[1], ':8 .*must override') self.assertRegexpMatches(failures[2], ':9 Only one of') self.assertRegexpMatches(failures[3], ':11 .*must override')