def testGenerateContextLostExampleTagsForNoAsan(self):
args = gpu_helper.GetMockArgs()
tag_set = self._TestTagGenerationForMockPlatform(
context_lost_integration_test.ContextLostIntegrationTest,
args)
self.assertIn('no-asan', tag_set)
self.assertNotIn('asan', tag_set)
def testFoundBrokenExpectations(self):
test_expectations = ('# tags: [ mac ]\n'
'# results: [ Failure ]\n'
'[ mac ] a/b/d [ Failure ]\n'
'a/c/* [ Failure ]\n')
options = gpu_helper.GetMockArgs()
test_class = gpu_integration_test.GpuIntegrationTest
with tempfile_ext.NamedTemporaryFile() as expectations_file, \
mock.patch.object(
test_class, 'GenerateGpuTests', return_value=[('a/b/c', ())]), \
mock.patch.object(
test_class,
'ExpectationsFiles', return_value=[expectations_file.name]):
expectations_file.write(test_expectations)
expectations_file.close()
with self.assertRaises(AssertionError) as context:
CheckTestExpectationsAreForExistingTests(
self, test_class, options)
self.assertIn(
'The following expectations were found to not apply'
' to any tests in the GpuIntegrationTest test suite',
str(context.exception))
self.assertIn(
'4: Expectation with pattern \'a/c/*\' does not match'
' any tests in the GpuIntegrationTest test suite',
str(context.exception))
self.assertIn(
'3: Expectation with pattern \'a/b/d\' does not match'
' any tests in the GpuIntegrationTest test suite',
str(context.exception))
def testForBrokenWebglExtensionExpectations(self):
webgl_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for webgl_version in xrange(1, 3):
tests = [
test[0] for test in webgl_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version='%d.0.0' %
webgl_version))
]
with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f:
expectations = expectations_parser.TestExpectations()
expectations.parse_tagged_list(f.read())
patterns_to_exps = expectations.individual_exps.copy()
patterns_to_exps.update(expectations.glob_exps)
patterns_to_exps = {
k: v
for k, v in patterns_to_exps.items()
if k.lower().startswith('webglextension')
}
broken_expectations = expectations.get_broken_expectations(
patterns_to_exps, tests)
msg = ''
for ununsed_pattern in set(
[e.test for e in broken_expectations]):
msg += (
"Expectations with pattern '{0}' in {1} do not apply to any "
"webgl version {2} extension tests\n".format(
ununsed_pattern, os.path.basename(f.name),
webgl_version))
assert not msg, msg
def testWebglTestExpectationsForDriverTags(self):
webgl_conformance_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
expectations_driver_tags = set()
for i in range(1, 3):
_ = list(
webgl_conformance_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % i))))
with open(webgl_conformance_test_class.ExpectationsFiles()[0],
'r') as f:
parser = expectations_parser.TestExpectations()
parser.parse_tagged_list(f.read(), f.name)
driver_tag_set = set()
for tag_set in parser.tag_sets:
if gpu_helper.MatchDriverTag(list(tag_set)[0]):
for tag in tag_set:
assert gpu_helper.MatchDriverTag(tag)
assert not driver_tag_set
driver_tag_set = tag_set
else:
for tag in tag_set:
assert not gpu_helper.MatchDriverTag(tag)
expectations_driver_tags |= driver_tag_set
self.assertEqual(gpu_helper.ExpectationsDriverTags(),
expectations_driver_tags)
def testGenerateWebglConformanceExampleTagsForWebglVersion2andNoAsan(self):
args = gpu_helper.GetMockArgs(is_asan=False, webgl_version='2.0.0')
tag_set = self._TestTagGenerationForMockPlatform(
webgl_conformance_integration_test.WebGLConformanceIntegrationTest,
args)
self.assertTrue(set(['no-asan', 'webgl-version-2']).issubset(tag_set))
self.assertFalse(set(['asan', 'webgl-version-1']) & tag_set)
def testWebglTestExpectationsForDriverTags(self):
webgl_conformance_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
expectations_driver_tags = set()
for webgl_version in range(1, 3):
_ = list(
webgl_conformance_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version=('%d.0.0' %
webgl_version))))
with open(webgl_conformance_test_class.ExpectationsFiles()[0],
'r') as f:
parser = expectations_parser.TestExpectations()
parser.parse_tagged_list(f.read(), f.name)
driver_tag_set = set()
for tag_set in parser.tag_sets:
if gpu_helper.MatchDriverTag(list(tag_set)[0]):
for tag in tag_set:
match = gpu_helper.MatchDriverTag(tag)
assert match
if match.group(1) == 'intel':
if not check_intel_driver_version(
match.group(3)):
assert False, INTEL_DRIVER_VERSION_SCHEMA
assert not driver_tag_set
driver_tag_set = tag_set
else:
for tag in tag_set:
assert not gpu_helper.MatchDriverTag(tag)
expectations_driver_tags |= driver_tag_set
self.assertEqual(gpu_helper.ExpectationsDriverTags(),
expectations_driver_tags)
def testForBrokenWebglExtensionExpectations(self):
webgl_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for webgl_version in xrange(1, 3):
tests = [
test[0] for test in webgl_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version='%d.0.0' %
webgl_version))
]
with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f:
expectations = expectations_parser.TestExpectations()
expectations.parse_tagged_list(f.read())
# remove non webgl extension expectations
for test in expectations.individual_exps.keys():
if not test.lower().startswith('webglextension'):
expectations.individual_exps.pop(test)
for test in expectations.glob_exps.keys():
if not test.lower().startswith('webglextension'):
expectations.glob_exps.pop(test)
broken_expectations = expectations.check_for_broken_expectations(
tests)
msg = ''
for ununsed_pattern in set(
[e.test for e in broken_expectations]):
msg += (
"Expectations with pattern '{0}' in {1} do not apply to any "
"webgl version {2} extension tests\n".format(
ununsed_pattern, os.path.basename(f.name),
webgl_version))
self.assertEqual(msg, '')
def testWebglTestExpectationsForDriverTags(self):
webgl_conformance_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
expectations_driver_tags = set()
for webgl_version in range(1, 3):
_ = list(
webgl_conformance_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version=('%d.0.0' %
webgl_version))))
with open(webgl_conformance_test_class.ExpectationsFiles()[0],
'r') as f:
parser = expectations_parser.TestExpectations()
parser.parse_tagged_list(f.read(), f.name)
driver_tag_set = set()
for tag_set in parser.tag_sets:
if gpu_helper.MatchDriverTag(list(tag_set)[0]):
for tag in tag_set:
match = gpu_helper.MatchDriverTag(tag)
self.assertIsNotNone(match)
if match.group(1) == 'intel':
self.assertTrue(
check_intel_driver_version(match.group(3)))
self.assertSetEqual(driver_tag_set, set())
driver_tag_set = tag_set
else:
for tag in tag_set:
self.assertIsNone(gpu_helper.MatchDriverTag(tag))
expectations_driver_tags |= driver_tag_set
self.assertEqual(gpu_helper.ExpectationsDriverTags(),
expectations_driver_tags)
def testForBrokenGpuTestExpectations(self):
options = gpu_helper.GetMockArgs()
for test_case in _FindTestCases():
if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__:
if (test_case.Name() not in ('pixel', 'webgl_conformance')
and test_case.ExpectationsFiles()):
CheckTestExpectationsAreForExistingTests(self, test_case, options)
def testWithoutExpectationsFilesGenerateTagsReturnsEmptyList(self):
# we need to make sure that GenerateTags() returns an empty list if
# there are no expectations files returned from ExpectationsFiles() or
# else Typ will raise an exception
args = gpu_helper.GetMockArgs()
possible_browser = mock.MagicMock()
self.assertFalse(gpu_integration_test.GpuIntegrationTest.GenerateTags(
args, possible_browser))
def _GetTagsToTest(browser, test_class=None, args=None):
test_class = test_class or gpu_integration_test.GpuIntegrationTest
with mock.patch.object(
test_class, 'ExpectationsFiles', return_value=['exp.txt']):
possible_browser = fakes.FakePossibleBrowser()
possible_browser._returned_browser = browser
args = args or gpu_helper.GetMockArgs()
return set(test_class.GenerateTags(args, possible_browser))
def testForBrokenPixelTestExpectations(self):
pixel_test_names = []
for _, method in inspect.getmembers(
pixel_test_pages.PixelTestPages, predicate=inspect.isfunction):
pixel_test_names.extend(
[p.name for p in method(
pixel_integration_test.PixelIntegrationTest.test_base_name)])
CheckTestExpectationsAreForExistingTests(
pixel_integration_test.PixelIntegrationTest,
gpu_helper.GetMockArgs(), pixel_test_names)
def testWebglTestPathsExist(self):
webgl_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for webgl_version in xrange(1, 3):
_ = list(
webgl_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version='%d.0.0' % webgl_version)))
with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f:
expectations = expectations_parser.TestExpectations()
expectations.parse_tagged_list(f.read())
for pattern, _ in expectations.individual_exps.items():
_CheckWebglConformanceTestPathIsValid(pattern)
def _TestCheckTestExpectationsAreForExistingTests(expectations):
options = gpu_helper.GetMockArgs()
expectations_file = tempfile.NamedTemporaryFile(delete=False)
expectations_file.write(expectations)
expectations_file.close()
gpu_tests = gpu_integration_test.GpuIntegrationTest
with mock.patch.object(gpu_tests,
'GenerateGpuTests',
return_value=[('a/b/c', ())]):
with mock.patch.object(gpu_tests,
'ExpectationsFiles',
return_value=[expectations_file.name]):
CheckTestExpectationsAreForExistingTests(gpu_tests, options)
def _GetTagsToTest(browser, test_class=None, args=None):
gpu_tests = test_class or gpu_integration_test.GpuIntegrationTest
expectations_fn = gpu_tests.ExpectationsFiles
gpu_tests.ExpectationsFiles = mock.MagicMock(return_value=['exp.txt'])
ret = None
try:
possible_browser = fakes.FakePossibleBrowser()
possible_browser._returned_browser = browser
args = args or gpu_helper.GetMockArgs()
ret = set(gpu_tests.GenerateTags(args, possible_browser))
finally:
gpu_tests.ExpectationsFiles = expectations_fn
return ret
def testWebglTestPathsExist(self):
webgl_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for test_case in _FindTestCases():
if test_case == webgl_test_class:
for i in xrange(1, 3):
_ = list(
test_case.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version='%d.0.0' %
i)))
with open(test_case.ExpectationsFiles()[0], 'r') as f:
expectations = expectations_parser.TaggedTestListParser(
f.read())
for exp in expectations.expectations:
_CheckPatternIsValid(exp.test)
def testNoCollisionsWithGlobsInGpuTestExpectations(self):
webgl_conformance_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for test_case in _FindTestCases():
if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__:
for i in xrange(
1, 2 + (test_case == webgl_conformance_test_class)):
_ = list(
test_case.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version=('%d.0.0' %
i))))
if test_case.ExpectationsFiles():
with open(test_case.ExpectationsFiles()[0]) as f:
CheckTestExpectationGlobsForCollision(
f.read(), os.path.basename(f.name))
def testNoConflictsInGpuTestExpectations(self):
webgl_conformance_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
errors = ''
for test_case in _FindTestCases():
if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__:
for webgl_version in xrange(
1, 2 + (test_case == webgl_conformance_test_class)):
_ = list(test_case.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version=('%d.0.0' % webgl_version))))
if test_case.ExpectationsFiles():
with open(test_case.ExpectationsFiles()[0]) as f:
errors += CheckTestExpectationPatternsForConflicts(f.read(),
os.path.basename(f.name))
assert not errors, errors
def testExpectationsFilesCanBeParsed(self):
webgl_conformance_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for test_case in _FindTestCases():
if 'gpu_tests.gpu_integration_test_unittest' not in test_case.__module__:
for webgl_version in xrange(
1, 2 + (test_case == webgl_conformance_test_class)):
_ = list(
test_case.GenerateGpuTests(
gpu_helper.GetMockArgs(
webgl_version=('%d.0.0' % webgl_version))))
if test_case.ExpectationsFiles():
with open(test_case.ExpectationsFiles()[0]) as f:
test_expectations = expectations_parser.TestExpectations()
ret, err = test_expectations.parse_tagged_list(f.read(), f.name)
self.assertEqual(
ret, 0,
'Error parsing %s:\n\t%s' % (os.path.basename(f.name), err))
def testWebglTestPathsExist(self):
def _CheckWebglConformanceTestPathIsValid(pattern):
if not 'WebglExtension_' in pattern:
full_path = os.path.normpath(
os.path.join(webgl_test_util.conformance_path, pattern))
self.assertTrue(os.path.exists(full_path))
webgl_test_class = (
webgl_conformance_integration_test.WebGLConformanceIntegrationTest)
for webgl_version in xrange(1, 3):
_ = list(
webgl_test_class.GenerateGpuTests(
gpu_helper.GetMockArgs(webgl_version='%d.0.0' % webgl_version)))
with open(webgl_test_class.ExpectationsFiles()[0], 'r') as f:
expectations = expectations_parser.TestExpectations()
expectations.parse_tagged_list(f.read())
for pattern, _ in expectations.individual_exps.items():
_CheckWebglConformanceTestPathIsValid(pattern)
