def test_create_line_dict_old_tests(self): # In this example, there are two failures that are not in wpt. updater = WPTExpectationsUpdater(self.mock_host()) results = { 'fake/test/path.html': { 'one': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), 'two': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), } } self.assertEqual(updater.create_line_dict(results), {})
def test_specifiers_can_extend_to_all_platforms(self): host = self.mock_host() expectations_path = '/test.checkout/LayoutTests/NeverFixTests' host.filesystem.write_text_file( expectations_path, 'crbug.com/111 [ Linux ] external/wpt/test.html [ WontFix ]\n') host.filesystem.write_text_file( '/test.checkout/LayoutTests/external/wpt/test.html', '') updater = WPTExpectationsUpdater(host) self.assertTrue( updater.specifiers_can_extend_to_all_platforms( ['Mac10.10', 'Mac10.11', 'Win7', 'Win10'], 'external/wpt/test.html')) self.assertFalse( updater.specifiers_can_extend_to_all_platforms( ['Mac10.10', 'Win7', 'Win10'], 'external/wpt/test.html'))
def test_get_failing_results_dict_non_wpt_test(self): host = self.mock_host() host.results_fetcher.set_results(Build('MOCK Try Mac10.10', 123), WebTestResults({ 'tests': { 'x': { 'failing-test.html': { 'expected': 'PASS', 'actual': 'IMAGE', 'is_unexpected': True, }, }, }, })) updater = WPTExpectationsUpdater(host) results_dict = updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)) self.assertEqual(results_dict, {})
def test_write_to_test_expectations_with_marker_comment(self): host = self.mock_host() expectations_path = host.port_factory.get( ).path_to_generic_test_expectations_file() host.filesystem.write_text_file(expectations_path, MARKER_COMMENT + '\n') updater = WPTExpectationsUpdater(host) line_dict = { 'fake/file/path.html': ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'] } updater.write_to_test_expectations(line_dict) value = host.filesystem.read_text_file(expectations_path) self.assertMultiLineEqual( value, (MARKER_COMMENT + '\n' 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n'))
def test_merge_same_valued_keys_all_match(self): updater = WPTExpectationsUpdater(self.mock_host()) self.assertEqual( updater.merge_same_valued_keys({ 'one': { 'expected': 'FAIL', 'actual': 'PASS' }, 'two': { 'expected': 'FAIL', 'actual': 'PASS' }, }), {('two', 'one'): { 'expected': 'FAIL', 'actual': 'PASS' }})
def get_updater(host=None, args=None): host = host or Host() args = args or [] if ('--update-android-expectations-only' in args or any(arg.startswith('--android-product') for arg in args)): return AndroidWPTExpectationsUpdater(host, args) else: return WPTExpectationsUpdater(host, args)
def test_get_test_to_rebaseline_returns_only_tests_with_failures(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) two = { 'external/wpt/test/path.html': { 'one': SimpleTestResult(expected='FAIL', actual='PASS', bug='bug'), 'two': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='bug'), 'three': SimpleTestResult(expected='FAIL', actual='PASS', bug='bug'), } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) # external/wpt/test/zzzz.html is another possible candidate, but it # is not listed in the results dict, so it shall not be rebaselined. self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
def test_merge_dicts_with_conflict_raise_exception(self): updater = WPTExpectationsUpdater(self.mock_host()) # Both dicts here have the key "one", and the value is not equal. with self.assertRaises(ValueError): updater.merge_dicts( { 'external/wpt/test/path.html': { 'one': {'expected': 'FAIL', 'actual': 'PASS'}, 'two': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, 'three': {'expected': 'FAIL', 'actual': 'PASS'}, }, }, { 'external/wpt/test/path.html': { 'one': {'expected': 'FAIL', 'actual': 'TIMEOUT'}, } })
def test_simplify_specifiers_uses_specifiers_in_builder_list(self): # Even if there are extra specifiers in the macro dictionary, we can simplify specifier # lists if they contain all of the specifiers the are represented in the builder list. # This way specifier simplification can still be done while a new platform is being added. host = self.mock_host() updater = WPTExpectationsUpdater(host) macros = { 'mac': ['Mac10.10', 'mac10.11', 'mac10.14'], 'win': ['Win7', 'win10'], 'Linux': ['Trusty'], } self.assertEqual( updater.simplify_specifiers(['mac10.10', 'mac10.11'], macros), ['Mac']) self.assertEqual( updater.simplify_specifiers( ['Mac10.10', 'Mac10.11', 'Trusty', 'Win7', 'Win10'], macros), [])
def test_create_line_dict_with_manual_tests(self): # In this example, there are three unexpected results for wpt tests. # The new test expectation lines are sorted by test, and then specifier. updater = WPTExpectationsUpdater(self.mock_host()) results = { 'virtual/foo/external/wpt/test/aa-manual.html': { 'test-linux-trusty': SimpleTestResult(expected='PASS', actual='TIMEOUT', bug='crbug.com/test'), 'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'), }, } self.assertEqual( updater.create_line_dict(results), { 'virtual/foo/external/wpt/test/aa-manual.html': [ 'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]', 'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]', ], })
def test_merge_dicts_merges_second_dict_into_first(self): updater = WPTExpectationsUpdater(self.mock_host()) one = { 'fake/test/path.html': { 'one': { 'expected': 'FAIL', 'actual': 'PASS' }, 'two': { 'expected': 'FAIL', 'actual': 'PASS' }, } } two = { 'external/wpt/test/path.html': { 'one': { 'expected': 'FAIL', 'actual': 'PASS' }, 'two': { 'expected': 'FAIL', 'actual': 'TIMEOUT' }, 'three': { 'expected': 'FAIL', 'actual': 'PASS' }, } } three = { 'external/wpt/test/path.html': { 'four': { 'expected': 'FAIL', 'actual': 'PASS' }, } } output = updater.merge_dicts(one, three) self.assertEqual(output, one) output = updater.merge_dicts(two, three) self.assertEqual(output, two)
def test_write_to_test_expectations_with_marker_and_no_lines(self): host = self.mock_host() expectations_path = host.port_factory.get( ).path_to_generic_test_expectations_file() host.filesystem.write_text_file( expectations_path, MARKER_COMMENT + '\n' + 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') skip_path = host.port_factory.get().path_to_never_fix_tests_file() skip_value_origin = host.filesystem.read_text_file(skip_path) updater = WPTExpectationsUpdater(host) updater.write_to_test_expectations({}) value = host.filesystem.read_text_file(expectations_path) self.assertMultiLineEqual( value, MARKER_COMMENT + '\n' + 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n') skip_value = host.filesystem.read_text_file(skip_path) self.assertMultiLineEqual(skip_value, skip_value_origin)
def test_get_tests_to_rebaseline_handles_retries(self): host = self.mock_host() results = { 'external/wpt/test/foo.html': { 'bot': SimpleTestResult(expected='PASS', actual='TEXT TEXT', bug='bug'), }, 'external/wpt/test/bar.html': { 'bot': SimpleTestResult(expected='PASS', actual='TIMEOUT TIMEOUT', bug='bug'), }, } updater = WPTExpectationsUpdater(host) tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(results) self.assertEqual(tests_to_rebaseline, ['external/wpt/test/foo.html']) self.assertEqual(modified_test_results, { 'external/wpt/test/foo.html': {}, 'external/wpt/test/bar.html': { 'bot': SimpleTestResult(expected='PASS', actual='TIMEOUT TIMEOUT', bug='bug'), }, })
def test_create_line_dict_with_asterisks(self): # Literal asterisks in test names need to be escaped in expectations. updater = WPTExpectationsUpdater(self.mock_host()) results = { 'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)': { 'test-linux-trusty': SimpleTestResult(expected='PASS', actual='FAIL', bug='crbug.com/test'), }, } self.assertEqual( updater.create_line_dict(results), { 'external/wpt/html/dom/interfaces.https.html?exclude=(Document.*|HTML.*)': [ 'crbug.com/test [ Trusty ] external/wpt/html/dom/interfaces.https.html?exclude=(Document.\*|HTML.\*) [ Failure ]', ], })
def test_get_failing_results_dict_only_passing_results(self): host = self.mock_host() host.results_fetcher.set_results(Build('MOCK Try Mac10.10', 123), WebTestResults({ 'tests': { 'external': { 'wpt': { 'x': { 'passing-test.html': { 'expected': 'PASS', 'actual': 'PASS', }, }, }, }, }, })) updater = WPTExpectationsUpdater(host) self.assertEqual( updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
def test_get_failing_results_dict_unexpected_pass(self): host = self.mock_host() host.buildbot.set_results(Build('MOCK Try Mac10.10', 123), WebTestResults({ 'tests': { 'external': { 'wpt': { 'x': { 'passing-test.html': { 'expected': 'FAIL TIMEOUT', 'actual': 'PASS', 'is_unexpected': True, }, }, }, }, }, })) updater = WPTExpectationsUpdater(host) self.assertEqual( updater.get_failing_results_dict(Build('MOCK Try Mac10.10', 123)), {})
def test_generate_results_dict(self): updater = WPTExpectationsUpdater(self.mock_host()) web_test_list = [ WebTestResult( 'external/wpt/test/name.html', { 'expected': 'bar', 'actual': 'foo', 'is_unexpected': True, 'has_stderr': True, }) ] self.assertEqual(updater.generate_results_dict('test-mac-mac10.10', web_test_list), { 'external/wpt/test/name.html': { 'test-mac-mac10.10': SimpleTestResult( expected='bar', actual='foo', bug='crbug.com/626703', ) } })
def test_write_to_test_expectations_with_webdriver_lines(self): host = self.mock_host() webdriver_expectations_path = \ host.port_factory.get().path_to_webdriver_expectations_file() host.filesystem.write_text_file( webdriver_expectations_path, WPTExpectationsUpdater.MARKER_COMMENT + '\n') updater = WPTExpectationsUpdater(host) test_expectations = { 'external/wpt/webdriver/fake/file/path.html': { tuple([DesktopConfig(port_name='test-linux-trusty')]): SimpleTestResult(actual='PASS', expected='', bug='crbug.com/123') } } expectations_path = \ host.port_factory.get().path_to_generic_test_expectations_file() expectations_value_origin = host.filesystem.read_text_file( expectations_path) skip_path = host.port_factory.get().path_to_never_fix_tests_file() skip_value_origin = host.filesystem.read_text_file(skip_path) updater.write_to_test_expectations(test_expectations) value = host.filesystem.read_text_file(webdriver_expectations_path) self.assertMultiLineEqual(value, ( WPTExpectationsUpdater.MARKER_COMMENT + '\n' 'crbug.com/123 [ Trusty ] external/wpt/webdriver/fake/file/path.html [ Pass ]\n' )) skip_value = host.filesystem.read_text_file(skip_path) self.assertMultiLineEqual(skip_value, skip_value_origin) expectations_value = host.filesystem.read_text_file(expectations_path) self.assertMultiLineEqual(expectations_value, expectations_value_origin)
def test_new_manual_tests_get_skip_expectation(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) results = { 'external/wpt/x-manual.html': { ( 'test-linux-precise', 'test-linux-trusty', 'test-mac-mac10.10', 'test-mac-mac10.11', 'test-win-win7', 'test-win-win10', ): SimpleTestResult(expected='PASS', actual='MISSING', bug='crbug.com/test') } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(results) self.assertEqual(tests_to_rebaseline, []) self.assertEqual( updater.create_line_dict(results), {'external/wpt/x-manual.html': ['crbug.com/test external/wpt/x-manual.html [ Skip ]']} )
def test_get_tests_to_rebaseline_also_returns_slow_tests(self): host = self.mock_host() results = { 'external/wpt/test/path.html': { 'one': SimpleTestResult(expected='SLOW', actual='TEXT', bug='bug'), 'two': SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'), }, } results_copy = copy.deepcopy(results) updater = WPTExpectationsUpdater(host) tests_to_rebaseline, modified_test_results = updater.get_tests_to_rebaseline(results) self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html']) # The record for the builder with a timeout is kept, but not with a text mismatch, # since that should be covered by downloading a new baseline. self.assertEqual(modified_test_results, { 'external/wpt/test/path.html': { 'two': SimpleTestResult(expected='SLOW', actual='TIMEOUT', bug='bug'), }, }) # The original dict isn't modified. self.assertEqual(results, results_copy)
def test_get_test_to_rebaseline_does_not_return_ref_tests(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) two = { 'external/wpt/reftest.html': { 'one': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), 'two': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'), 'three': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), } } tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two) self.assertEqual(tests_to_rebaseline, [])
def test_one_platform_has_no_results(self): # In this example, there is a failure that has been observed on # Linux and one Mac port, but the other Mac port has no results at all. # The specifiers are "filled in" and the failure is assumed to apply # to all Mac platforms. host = self.mock_host() updater = WPTExpectationsUpdater(host) results = { 'external/wpt/x.html': { ( 'test-linux-precise', 'test-linux-trusty', 'test-mac-mac10.11', ): SimpleTestResult(expected='PASS', actual='TEXT', bug='crbug.com/test') } } updater.ports_with_no_results = {'test-mac-mac10.10'} self.assertEqual( updater.create_line_dict(results), {'external/wpt/x.html': ['crbug.com/test [ Linux Mac ] external/wpt/x.html [ Failure ]']} )
def test_specifier_part_with_skipped_test(self): host = self.mock_host() expectations_path = '/test.checkout/wtests/NeverFixTests' host.filesystem.write_text_file( expectations_path, 'crbug.com/111 [ Linux Mac10.11 ] external/wpt/test.html [ WontFix ]\n' ) host.filesystem.write_text_file( '/test.checkout/wtests/external/wpt/test.html', '') updater = WPTExpectationsUpdater(host) self.assertEqual( updater.specifier_part( 'external/wpt/test.html', ['test-mac-mac10.10', 'test-win-win7', 'test-win-win10']), '') self.assertEqual( updater.specifier_part('external/wpt/test.html', ['test-win-win7', 'test-win-win10']), '[ Win ]') self.assertEqual( updater.specifier_part('external/wpt/another.html', ['test-win-win7', 'test-win-win10']), '[ Win ]')
def test_create_line_dict_with_manual_tests(self): # In this example, there are two manual tests that should be skipped. updater = WPTExpectationsUpdater(self.mock_host()) results = { 'virtual/foo/external/wpt/test/aa-manual.html': { 'test-linux-trusty': SimpleTestResult(expected='PASS', actual='TIMEOUT', bug='crbug.com/test'), 'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'), }, } self.assertEqual( updater.create_line_dict(results), { 'virtual/foo/external/wpt/test/aa-manual.html': [ 'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]', 'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ Skip ]', ], })
def test_create_line_dict_new_tests(self): # In this example, there are three unexpected results for wpt tests. # The new test expectation lines are sorted by test, and then specifier. updater = WPTExpectationsUpdater(self.mock_host()) results = { 'external/wpt/test/zzzz.html': { 'test-mac-mac10.10': SimpleTestResult(expected='PASS', actual='TEXT', bug='crbug.com/test'), }, 'virtual/foo/external/wpt/test/zzzz.html': { 'test-linux-trusty': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), 'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'), }, 'unrelated/test.html': { 'test-linux-trusty': SimpleTestResult(expected='FAIL', actual='PASS', bug='crbug.com/test'), }, } self.assertEqual( updater.create_line_dict(results), { 'external/wpt/test/zzzz.html': [ 'crbug.com/test [ Mac10.10 ] external/wpt/test/zzzz.html [ Failure ]' ], 'virtual/foo/external/wpt/test/zzzz.html': [ 'crbug.com/test [ Trusty ] virtual/foo/external/wpt/test/zzzz.html [ Pass ]', 'crbug.com/test [ Mac10.11 ] virtual/foo/external/wpt/test/zzzz.html [ Timeout ]', ], })
def test_write_to_test_expectations_with_no_marker_comment(self): host = self.mock_host() expectations_path = host.port_factory.get().path_to_generic_test_expectations_file() host.filesystem.write_text_file( expectations_path, 'crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n') updater = WPTExpectationsUpdater(host) line_dict = {'fake/file/path.html': ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']} skip_path = host.port_factory.get().path_to_never_fix_tests_file() skip_value_origin = host.filesystem.read_text_file(skip_path) updater.write_to_test_expectations(line_dict) value = host.filesystem.read_text_file(expectations_path) self.assertMultiLineEqual( value, ('crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n' '\n' + MARKER_COMMENT + '\n' 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]')) skip_value = host.filesystem.read_text_file(skip_path) self.assertMultiLineEqual( skip_value, skip_value_origin)
def test_write_to_test_expectations_without_newline(self): host = self.mock_host() expectations_path = host.port_factory.get( ).path_to_generic_test_expectations_file() skip_path = host.port_factory.get().path_to_never_fix_tests_file() line_dict = { 'fake/file/path-manual.html': ['[ Trusty ] fake/file/path-manual.html [ Skip ]'] } host.filesystem.write_text_file(expectations_path, MARKER_COMMENT + '\n') host.filesystem.write_text_file( skip_path, '[ Trusty ] fake/file/path-manual.html [ Skip ]') updater = WPTExpectationsUpdater(host) updater.write_to_test_expectations(line_dict) expectations_value = host.filesystem.read_text_file(expectations_path) skip_value = host.filesystem.read_text_file(skip_path) self.assertMultiLineEqual(expectations_value, MARKER_COMMENT + '\n') self.assertMultiLineEqual( skip_value, '[ Trusty ] fake/file/path-manual.html [ Skip ]\n' '[ Trusty ] fake/file/path-manual.html [ Skip ]\n')
def test_merging_platforms_if_possible(self): host = self.mock_host() updater = WPTExpectationsUpdater(host) results = { 'external/wpt/x.html': { ( 'test-linux-precise', 'test-linux-trusty', 'test-mac-mac10.10', 'test-mac-mac10.11', 'test-win-win7', ): SimpleTestResult(expected='PASS', actual='TEXT', bug='crbug.com/test') } } self.assertEqual( updater.create_line_dict(results), {'external/wpt/x.html': [ 'crbug.com/test [ Linux ] external/wpt/x.html [ Failure ]', 'crbug.com/test [ Mac ] external/wpt/x.html [ Failure ]', 'crbug.com/test [ Win7 ] external/wpt/x.html [ Failure ]', ] } )
def __init__(self, host, wpt_github=None, wpt_manifests=None): self.host = host self.wpt_github = wpt_github self.executive = host.executive self.fs = host.filesystem self.finder = PathFinder(self.fs) self.chromium_git = self.host.git(self.finder.chromium_base()) self.dest_path = self.finder.path_from_web_tests('external', 'wpt') # A common.net.git_cl.GitCL instance. self.git_cl = None # Another Git instance with local WPT as CWD, which can only be # instantiated after the working directory is created. self.wpt_git = None # The WPT revision we are importing and the one imported last time. self.wpt_revision = None self.last_wpt_revision = None # A set of rebaselined tests and a dictionary of new test expectations # mapping failing tests to platforms to # wpt_expectations_updater.SimpleTestResult. self.rebaselined_tests = set() self.new_test_expectations = {} # New override expectations for Android targets. A dictionary mapping # products to dictionary that maps test names to test expectation lines self.new_override_expectations = {} self.verbose = False args = [ '--clean-up-affected-tests-only', '--clean-up-test-expectations' ] self._expectations_updater = WPTExpectationsUpdater( self.host, args, wpt_manifests) args = ['--android-product', 'android_weblayer'] self._android_expectations_updater = AndroidWPTExpectationsUpdater( self.host, args, wpt_manifests)
def test_write_to_test_expectations_with_webdriver_lines(self): host = self.mock_host() webdriver_expectations_path = host.port_factory.get( ).path_to_webdriver_expectations_file() host.filesystem.write_text_file(webdriver_expectations_path, MARKER_COMMENT + '\n') updater = WPTExpectationsUpdater(host) line_dict = { 'external/wpt/webdriver/fake/file/path.html': [ 'crbug.com/123 [ Trusty ] external/wpt/webdriver/fake/file/path.html [ Pass ]' ] } expectations_path = host.port_factory.get( ).path_to_generic_test_expectations_file() expectations_value_origin = host.filesystem.read_text_file( expectations_path) skip_path = host.port_factory.get().path_to_never_fix_tests_file() skip_value_origin = host.filesystem.read_text_file(skip_path) updater.write_to_test_expectations(line_dict) value = host.filesystem.read_text_file(webdriver_expectations_path) self.assertMultiLineEqual(value, ( MARKER_COMMENT + '\n' 'crbug.com/123 [ Trusty ] external/wpt/webdriver/fake/file/path.html [ Pass ]\n' )) skip_value = host.filesystem.read_text_file(skip_path) self.assertMultiLineEqual(skip_value, skip_value_origin) expectations_value = host.filesystem.read_text_file(expectations_path) self.assertMultiLineEqual(expectations_value, expectations_value_origin)