def testGenerateContextLostExampleTagsForNoAsan(self): args = gpu_helper.GetMockArgs() tag_set = self._TestTagGenerationForMockPlatform( context_lost_integration_test.ContextLostIntegrationTest, args) self.assertIn('no-asan', tag_set) self.assertNotIn('asan', tag_set)
def testFoundBrokenExpectations(self): test_expectations = ('# tags: [ mac ]\n' '# results: [ Failure ]\n' '[ mac ] a/b/d [ Failure ]\n' 'a/c/* [ Failure ]\n') options = gpu_helper.GetMockArgs() test_class = gpu_integration_test.GpuIntegrationTest with tempfile_ext.NamedTemporaryFile() as expectations_file, \ mock.patch.object( test_class, 'GenerateGpuTests', return_value=[('a/b/c', ())]), \ mock.patch.object( test_class, 'ExpectationsFiles', return_value=[expectations_file.name]): expectations_file.write(test_expectations) expectations_file.close() with self.assertRaises(AssertionError) as context: CheckTestExpectationsAreForExistingTests( self, test_class, options) self.assertIn( 'The following expectations were found to not apply' ' to any tests in the GpuIntegrationTest test suite', str(context.exception)) self.assertIn( '4: Expectation with pattern \'a/c/*\' does not match' ' any tests in the GpuIntegrationTest test suite', str(context.exception)) self.assertIn( '3: Expectation with pattern \'a/b/d\' does not match' ' any tests in the GpuIntegrationTest test suite', str(context.exception))
def testForBrokenWebglExtensionExpectations(self): webgl_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for webgl_version in xrange(1, 3): tests = [ test[0] for test in webgl_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version='%d.0.0' % webgl_version)) ] with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f: expectations = expectations_parser.TestExpectations() expectations.parse_tagged_list(f.read()) patterns_to_exps = expectations.individual_exps.copy() patterns_to_exps.update(expectations.glob_exps) patterns_to_exps = { k: v for k, v in patterns_to_exps.items() if k.lower().startswith('webglextension') } broken_expectations = expectations.get_broken_expectations( patterns_to_exps, tests) msg = '' for ununsed_pattern in set( [e.test for e in broken_expectations]): msg += ( "Expectations with pattern '{0}' in {1} do not apply to any " "webgl version {2} extension tests\n".format( ununsed_pattern, os.path.basename(f.name), webgl_version)) assert not msg, msg
def testWebglTestExpectationsForDriverTags(self): webgl_conformance_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) expectations_driver_tags = set() for i in range(1, 3): _ = list( webgl_conformance_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % i)))) with open(webgl_conformance_test_class.ExpectationsFiles()[0], 'r') as f: parser = expectations_parser.TestExpectations() parser.parse_tagged_list(f.read(), f.name) driver_tag_set = set() for tag_set in parser.tag_sets: if gpu_helper.MatchDriverTag(list(tag_set)[0]): for tag in tag_set: assert gpu_helper.MatchDriverTag(tag) assert not driver_tag_set driver_tag_set = tag_set else: for tag in tag_set: assert not gpu_helper.MatchDriverTag(tag) expectations_driver_tags |= driver_tag_set self.assertEqual(gpu_helper.ExpectationsDriverTags(), expectations_driver_tags)
def testGenerateWebglConformanceExampleTagsForWebglVersion2andNoAsan(self): args = gpu_helper.GetMockArgs(is_asan=False, webgl_version='2.0.0') tag_set = self._TestTagGenerationForMockPlatform( webgl_conformance_integration_test.WebGLConformanceIntegrationTest, args) self.assertTrue(set(['no-asan', 'webgl-version-2']).issubset(tag_set)) self.assertFalse(set(['asan', 'webgl-version-1']) & tag_set)
def testWebglTestExpectationsForDriverTags(self): webgl_conformance_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) expectations_driver_tags = set() for webgl_version in range(1, 3): _ = list( webgl_conformance_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % webgl_version)))) with open(webgl_conformance_test_class.ExpectationsFiles()[0], 'r') as f: parser = expectations_parser.TestExpectations() parser.parse_tagged_list(f.read(), f.name) driver_tag_set = set() for tag_set in parser.tag_sets: if gpu_helper.MatchDriverTag(list(tag_set)[0]): for tag in tag_set: match = gpu_helper.MatchDriverTag(tag) assert match if match.group(1) == 'intel': if not check_intel_driver_version( match.group(3)): assert False, INTEL_DRIVER_VERSION_SCHEMA assert not driver_tag_set driver_tag_set = tag_set else: for tag in tag_set: assert not gpu_helper.MatchDriverTag(tag) expectations_driver_tags |= driver_tag_set self.assertEqual(gpu_helper.ExpectationsDriverTags(), expectations_driver_tags)
def testForBrokenWebglExtensionExpectations(self): webgl_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for webgl_version in xrange(1, 3): tests = [ test[0] for test in webgl_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version='%d.0.0' % webgl_version)) ] with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f: expectations = expectations_parser.TestExpectations() expectations.parse_tagged_list(f.read()) # remove non webgl extension expectations for test in expectations.individual_exps.keys(): if not test.lower().startswith('webglextension'): expectations.individual_exps.pop(test) for test in expectations.glob_exps.keys(): if not test.lower().startswith('webglextension'): expectations.glob_exps.pop(test) broken_expectations = expectations.check_for_broken_expectations( tests) msg = '' for ununsed_pattern in set( [e.test for e in broken_expectations]): msg += ( "Expectations with pattern '{0}' in {1} do not apply to any " "webgl version {2} extension tests\n".format( ununsed_pattern, os.path.basename(f.name), webgl_version)) self.assertEqual(msg, '')
def testWebglTestExpectationsForDriverTags(self): webgl_conformance_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) expectations_driver_tags = set() for webgl_version in range(1, 3): _ = list( webgl_conformance_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % webgl_version)))) with open(webgl_conformance_test_class.ExpectationsFiles()[0], 'r') as f: parser = expectations_parser.TestExpectations() parser.parse_tagged_list(f.read(), f.name) driver_tag_set = set() for tag_set in parser.tag_sets: if gpu_helper.MatchDriverTag(list(tag_set)[0]): for tag in tag_set: match = gpu_helper.MatchDriverTag(tag) self.assertIsNotNone(match) if match.group(1) == 'intel': self.assertTrue( check_intel_driver_version(match.group(3))) self.assertSetEqual(driver_tag_set, set()) driver_tag_set = tag_set else: for tag in tag_set: self.assertIsNone(gpu_helper.MatchDriverTag(tag)) expectations_driver_tags |= driver_tag_set self.assertEqual(gpu_helper.ExpectationsDriverTags(), expectations_driver_tags)
def testForBrokenGpuTestExpectations(self): options = gpu_helper.GetMockArgs() for test_case in _FindTestCases(): if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: if (test_case.Name() not in ('pixel', 'webgl_conformance') and test_case.ExpectationsFiles()): CheckTestExpectationsAreForExistingTests(self, test_case, options)
def testWithoutExpectationsFilesGenerateTagsReturnsEmptyList(self): # we need to make sure that GenerateTags() returns an empty list if # there are no expectations files returned from ExpectationsFiles() or # else Typ will raise an exception args = gpu_helper.GetMockArgs() possible_browser = mock.MagicMock() self.assertFalse(gpu_integration_test.GpuIntegrationTest.GenerateTags( args, possible_browser))
def _GetTagsToTest(browser, test_class=None, args=None): test_class = test_class or gpu_integration_test.GpuIntegrationTest with mock.patch.object( test_class, 'ExpectationsFiles', return_value=['exp.txt']): possible_browser = fakes.FakePossibleBrowser() possible_browser._returned_browser = browser args = args or gpu_helper.GetMockArgs() return set(test_class.GenerateTags(args, possible_browser))
def testForBrokenPixelTestExpectations(self): pixel_test_names = [] for _, method in inspect.getmembers( pixel_test_pages.PixelTestPages, predicate=inspect.isfunction): pixel_test_names.extend( [p.name for p in method( pixel_integration_test.PixelIntegrationTest.test_base_name)]) CheckTestExpectationsAreForExistingTests( pixel_integration_test.PixelIntegrationTest, gpu_helper.GetMockArgs(), pixel_test_names)
def testWebglTestPathsExist(self): webgl_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for webgl_version in xrange(1, 3): _ = list( webgl_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version='%d.0.0' % webgl_version))) with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f: expectations = expectations_parser.TestExpectations() expectations.parse_tagged_list(f.read()) for pattern, _ in expectations.individual_exps.items(): _CheckWebglConformanceTestPathIsValid(pattern)
def _TestCheckTestExpectationsAreForExistingTests(expectations): options = gpu_helper.GetMockArgs() expectations_file = tempfile.NamedTemporaryFile(delete=False) expectations_file.write(expectations) expectations_file.close() gpu_tests = gpu_integration_test.GpuIntegrationTest with mock.patch.object(gpu_tests, 'GenerateGpuTests', return_value=[('a/b/c', ())]): with mock.patch.object(gpu_tests, 'ExpectationsFiles', return_value=[expectations_file.name]): CheckTestExpectationsAreForExistingTests(gpu_tests, options)
def _GetTagsToTest(browser, test_class=None, args=None): gpu_tests = test_class or gpu_integration_test.GpuIntegrationTest expectations_fn = gpu_tests.ExpectationsFiles gpu_tests.ExpectationsFiles = mock.MagicMock(return_value=['exp.txt']) ret = None try: possible_browser = fakes.FakePossibleBrowser() possible_browser._returned_browser = browser args = args or gpu_helper.GetMockArgs() ret = set(gpu_tests.GenerateTags(args, possible_browser)) finally: gpu_tests.ExpectationsFiles = expectations_fn return ret
def testWebglTestPathsExist(self): webgl_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for test_case in _FindTestCases(): if test_case == webgl_test_class: for i in xrange(1, 3): _ = list( test_case.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version='%d.0.0' % i))) with open(test_case.ExpectationsFiles()[0], 'r') as f: expectations = expectations_parser.TaggedTestListParser( f.read()) for exp in expectations.expectations: _CheckPatternIsValid(exp.test)
def testNoCollisionsWithGlobsInGpuTestExpectations(self): webgl_conformance_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for test_case in _FindTestCases(): if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: for i in xrange( 1, 2 + (test_case == webgl_conformance_test_class)): _ = list( test_case.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % i)))) if test_case.ExpectationsFiles(): with open(test_case.ExpectationsFiles()[0]) as f: CheckTestExpectationGlobsForCollision( f.read(), os.path.basename(f.name))
def testNoConflictsInGpuTestExpectations(self): webgl_conformance_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) errors = '' for test_case in _FindTestCases(): if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: for webgl_version in xrange( 1, 2 + (test_case == webgl_conformance_test_class)): _ = list(test_case.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % webgl_version)))) if test_case.ExpectationsFiles(): with open(test_case.ExpectationsFiles()[0]) as f: errors += CheckTestExpectationPatternsForConflicts(f.read(), os.path.basename(f.name)) assert not errors, errors
def testExpectationsFilesCanBeParsed(self): webgl_conformance_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for test_case in _FindTestCases(): if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__: for webgl_version in xrange( 1, 2 + (test_case == webgl_conformance_test_class)): _ = list( test_case.GenerateGpuTests( gpu_helper.GetMockArgs( webgl_version=('%d.0.0' % webgl_version)))) if test_case.ExpectationsFiles(): with open(test_case.ExpectationsFiles()[0]) as f: test_expectations = expectations_parser.TestExpectations() ret, err = test_expectations.parse_tagged_list(f.read(), f.name) self.assertEqual( ret, 0, 'Error parsing %s:\n\t%s' % (os.path.basename(f.name), err))
def testWebglTestPathsExist(self): def _CheckWebglConformanceTestPathIsValid(pattern): if not 'WebglExtension_' in pattern: full_path = os.path.normpath( os.path.join(webgl_test_util.conformance_path, pattern)) self.assertTrue(os.path.exists(full_path)) webgl_test_class = ( webgl_conformance_integration_test.WebGLConformanceIntegrationTest) for webgl_version in xrange(1, 3): _ = list( webgl_test_class.GenerateGpuTests( gpu_helper.GetMockArgs(webgl_version='%d.0.0' % webgl_version))) with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f: expectations = expectations_parser.TestExpectations() expectations.parse_tagged_list(f.read()) for pattern, _ in expectations.individual_exps.items(): _CheckWebglConformanceTestPathIsValid(pattern)