def testFramesFromMp4(self):
host_platform = platform.GetHostPlatform()
try:
host_platform.InstallApplication('avconv')
finally:
if not host_platform.CanLaunchApplication('avconv'):
logging.warning('Test not supported on this platform')
return # pylint: disable=W0150
vid = os.path.join(util.GetUnittestDataDir(), 'vid.mp4')
expected_timestamps = [
0,
763,
783,
940,
1715,
1732,
1842,
1926,
]
video_obj = video.Video(vid)
# Calling _FramesFromMp4 should return all frames.
# pylint: disable=W0212
for i, timestamp_bitmap in enumerate(video_obj._FramesFromMp4(vid)):
timestamp, bmp = timestamp_bitmap
self.assertEquals(timestamp, expected_timestamps[i])
expected_bitmap = image_util.FromPngFile(
os.path.join(util.GetUnittestDataDir(), 'frame%d.png' % i))
self.assertTrue(image_util.AreEqual(expected_bitmap, bmp))
def testWriteCroppedBmpToPngFile(self):
pixels = [255, 0, 0, 255, 255, 0, 0, 0, 0,
255, 255, 0, 0, 255, 0, 0, 0, 0]
orig = image_util.FromRGBPixels(3, 2, pixels)
orig = image_util.Crop(orig, 0, 0, 2, 2)
temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
image_util.WritePngFile(orig, temp_file)
new_file = image_util.FromPngFile(temp_file)
self.assertTrue(image_util.AreEqual(orig, new_file, likely_equal=True))
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version', story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.output_formats = ['json-test-results']
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
self.options.output_dir = tempdir
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
# This ensures the output stream used by the json test results object is
# closed. On windows, we can't delete the temp directory if a file in that
# directory is still in use.
with results:
possible_browser = browser_finder.FindBrowser(self.options)
story_runner.RunStorySet(
test=DummyTest(),
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=self.options.browser_options,
finder_options=self.options,
results=results,
max_failures=2,
)
self.assertTrue(results.had_failures)
artifacts = results._artifact_results.GetTestArtifacts(
failing_page.name)
self.assertIsNotNone(artifacts)
self.assertIn('screenshot', artifacts)
screenshot_file_path = os.path.join(tempdir, artifacts['screenshot'][0])
actual_screenshot_img = image_util.FromPngFile(screenshot_file_path)
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
story_runner.Run(DummyTest(),
story_set,
self.options,
results,
max_failures=2,
metadata=EmptyMetadataForTest())
self.assertEquals(1, len(results.failures))
self.assertEquals(1, len(results.pages_to_profiling_files))
self.assertIn(failing_page, results.pages_to_profiling_files)
screenshot_file_path = (
results.pages_to_profiling_files[failing_page][0].GetAbsPath())
try:
actual_screenshot_img = image_util.FromPngFile(
screenshot_file_path)
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
finally: # Must clean up screenshot file if exists.
os.remove(screenshot_file_path)
def testScreenShotTakenSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
fh = screenshot.TryCaptureScreenShot(fake_platform, None)
screenshot_file_path = fh.GetAbsPath()
try:
actual_screenshot_img = image_util.FromPngFile(screenshot_file_path)
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
finally: # Must clean up screenshot file if exists.
os.remove(screenshot_file_path)
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
expected_png_base64 = ('iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91'
'JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A'
'T8tgwbJAAAAABJRU5ErkJggg==')
self.fake_platform.screenshot_png_data = expected_png_base64
# After setting up some fake data, now the platform supports screenshots.
self.assertTrue(self.fake_platform.CanTakeScreenshot())
self.options.browser_options.take_screenshot_for_failed_page = True
story_set = test_stories.SinglePageStorySet(
story_run_side_effect=exceptions.AppCrashException(msg='fake crash'))
results = self.RunStorySetAndGetResults(story_set)
self.assertEqual(results['status'], 'FAIL')
self.assertIn('screenshot.png', results['outputArtifacts'])
actual_screenshot_img = image_util.FromPngFile(
results['outputArtifacts']['screenshot.png']['filePath'])
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.browser_options.take_screenshot_for_failed_page = True
self.options.output_formats = ['json-test-results']
results = RunStorySet(DummyTest(),
story_set,
self.options,
max_failures=2)
self.assertTrue(results.had_failures)
failed_run = next(run for run in results.IterStoryRuns()
if run.story.name == failing_page.name)
screenshot_file_path = failed_run.GetArtifact(
'screenshot.png').local_path
actual_screenshot_img = image_util.FromPngFile(screenshot_file_path)
self.assertTrue(
image_util.AreEqual(image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
def RunSkiaGoldBasedPixelTest(self, test_path, do_page_action, args):
page = args[0]
tab = self.tab
try:
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(
screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr), int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
# This is required by Gold whether this is a tryjob or not.
build_id_args = [
'--commit',
self.GetParsedCommandLineOptions().build_revision,
]
parsed_options = self.GetParsedCommandLineOptions()
# Trybots conceptually download their reference images from
# cloud storage; they don't produce them. (TODO(kbr): remove
# this once fully switched over to Gold.)
if parsed_options.download_refimg_from_cloud_storage:
# Tryjobs in Gold need:
# CL issue number (in Gerrit)
# Patchset number
# Buildbucket ID
build_id_args += [
'--issue',
self.GetParsedCommandLineOptions().review_patch_issue,
'--patchset',
self.GetParsedCommandLineOptions().review_patch_set,
'--jobid',
self.GetParsedCommandLineOptions().buildbucket_build_id
]
if page.expected_colors:
# Use expected colors instead of ref images for validation.
self._ValidateScreenshotSamplesWithSkiaGold(
tab, page, screenshot, page.expected_colors, page.tolerance,
dpr, build_id_args)
return
image_name = self._UrlToImageName(page.name)
is_local_run = not (parsed_options.upload_refimg_to_cloud_storage or
parsed_options.download_refimg_from_cloud_storage)
if is_local_run:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(
self.GetParsedCommandLineOptions().reference_dir,
image_name, page.revision, screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(ref_png, screenshot,
tolerance=page.tolerance):
if parsed_options.test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot,
ref_png)
else:
self._WriteErrorImages(
parsed_options.generated_dir, image_name, screenshot, ref_png)
self.fail('Reference image did not match captured screen')
else:
try:
self._UploadTestResultToSkiaGold(
image_name, screenshot,
tab, page,
is_check_mode=True,
build_id_args=build_id_args)
except CalledProcessError:
self.fail('Gold said the test failed, so fail.')
finally:
if do_page_action:
# Assume that page actions might have killed the GPU process.
self._RestartBrowser('Must restart after page actions')
def RunLegacyPixelTest(self, test_path, do_page_action, args):
page = args[0]
tab = self.tab
try:
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(
screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr), int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
if page.expected_colors:
# Use expected colors instead of ref images for validation.
self._ValidateScreenshotSamples(
tab, page.name, screenshot, page.expected_colors, page.tolerance,
dpr)
return
image_name = self._UrlToImageName(page.name)
if self.GetParsedCommandLineOptions().upload_refimg_to_cloud_storage:
if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
screenshot):
# This is the new reference image; there's nothing to compare against.
ref_png = screenshot
else:
# There was a preexisting reference image, so we might as well
# compare against it.
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
elif self.GetParsedCommandLineOptions().\
download_refimg_from_cloud_storage:
# This bot doesn't have the ability to properly generate a
# reference image, so download it from cloud storage.
try:
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
except cloud_storage.NotFoundError:
# There is no reference image yet in cloud storage. This
# happens when the revision of the test is incremented or when
# a new test is added, because the trybots are not allowed to
# produce reference images, only the bots on the main
# waterfalls. Report this as a failure so the developer has to
# take action by explicitly suppressing the failure and
# removing the suppression once the reference images have been
# generated. Otherwise silent failures could happen for long
# periods of time.
self.fail('Could not find image %s in cloud storage' % image_name)
else:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(
self.GetParsedCommandLineOptions().reference_dir,
image_name, page.revision, screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(ref_png, screenshot, tolerance=page.tolerance):
if self.GetParsedCommandLineOptions().test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
else:
self._WriteErrorImages(
self.GetParsedCommandLineOptions().generated_dir, image_name,
screenshot, ref_png)
self.fail('Reference image did not match captured screen')
finally:
if do_page_action:
# Assume that page actions might have killed the GPU process.
self._RestartBrowser('Must restart after page actions')
def RunActualGpuTest(self, test_path, *args):
page = args[0]
# Some pixel tests require non-standard browser arguments. Need to
# check before running each page that it can run in the current
# browser instance.
self.RestartBrowserIfNecessaryWithArgs(page.browser_args)
url = self.UrlOfStaticFilePath(test_path)
# This property actually comes off the class, not 'self'.
tab = self.tab
tab.Navigate(url, script_to_evaluate_on_commit=test_harness_script)
tab.action_runner.WaitForJavaScriptCondition(
'domAutomationController._finished', timeout_in_seconds=300)
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(screenshot, page.test_rect[0] * dpr,
page.test_rect[1] * dpr,
page.test_rect[2] * dpr,
page.test_rect[3] * dpr)
if page.expected_colors:
# Use expected colors instead of ref images for validation.
self._ValidateScreenshotSamples(tab, page.name, screenshot,
page.expected_colors, dpr)
return
image_name = self._UrlToImageName(page.name)
if self.GetParsedCommandLineOptions().upload_refimg_to_cloud_storage:
if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
screenshot):
# This is the new reference image; there's nothing to compare against.
ref_png = screenshot
else:
# There was a preexisting reference image, so we might as well
# compare against it.
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
elif self.GetParsedCommandLineOptions(
).download_refimg_from_cloud_storage:
# This bot doesn't have the ability to properly generate a
# reference image, so download it from cloud storage.
try:
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
except cloud_storage.NotFoundError:
# There is no reference image yet in cloud storage. This
# happens when the revision of the test is incremented or when
# a new test is added, because the trybots are not allowed to
# produce reference images, only the bots on the main
# waterfalls. Report this as a failure so the developer has to
# take action by explicitly suppressing the failure and
# removing the suppression once the reference images have been
# generated. Otherwise silent failures could happen for long
# periods of time.
self.fail('Could not find image %s in cloud storage' %
image_name)
else:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(
self.GetParsedCommandLineOptions().reference_dir, image_name,
page.revision, screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(
ref_png, screenshot, tolerance=page.tolerance):
if self.GetParsedCommandLineOptions().test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot,
ref_png)
else:
self._WriteErrorImages(
self.GetParsedCommandLineOptions().generated_dir,
image_name, screenshot, ref_png)
self.fail('Reference image did not match captured screen')
def ValidateAndMeasurePageInner(self, page, tab, results):
if not _DidTestSucceed(tab):
raise page_test.Failure('Page indicated a failure')
if not tab.screenshot_supported:
raise page_test.Failure(
'Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
raise page_test.Failure('Could not capture screenshot')
if hasattr(page, 'test_rect'):
screenshot = image_util.Crop(screenshot, page.test_rect[0],
page.test_rect[1], page.test_rect[2],
page.test_rect[3])
image_name = self._UrlToImageName(page.display_name)
if self.options.upload_refimg_to_cloud_storage:
if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
screenshot):
# This is the new reference image; there's nothing to compare against.
ref_png = screenshot
else:
# There was a preexisting reference image, so we might as well
# compare against it.
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
elif self.options.download_refimg_from_cloud_storage:
# This bot doesn't have the ability to properly generate a
# reference image, so download it from cloud storage.
try:
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
except cloud_storage.NotFoundError as e:
# There is no reference image yet in cloud storage. This
# happens when the revision of the test is incremented or when
# a new test is added, because the trybots are not allowed to
# produce reference images, only the bots on the main
# waterfalls. Report this as a failure so the developer has to
# take action by explicitly suppressing the failure and
# removing the suppression once the reference images have been
# generated. Otherwise silent failures could happen for long
# periods of time.
raise page_test.Failure(
'Could not find image %s in cloud storage' % image_name)
else:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(self.options.reference_dir,
image_name, page.revision,
screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(ref_png, screenshot, tolerance=2):
if self.options.test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot,
ref_png)
else:
self._WriteErrorImages(self.options.generated_dir, image_name,
screenshot, ref_png)
raise page_test.Failure(
'Reference image did not match captured screen')
def testIsEqual(self): bmp = image_util.FromBase64Png(test_png) file_bmp = image_util.FromPngFile(test_png_path) self.assertTrue(image_util.AreEqual(bmp, file_bmp, likely_equal=True))
def testWritePngToPngFile(self): orig = image_util.FromPngFile(test_png_path) temp_file = tempfile.NamedTemporaryFile(suffix='.png').name image_util.WritePngFile(orig, temp_file) new_file = image_util.FromPngFile(temp_file) self.assertTrue(image_util.AreEqual(orig, new_file, likely_equal=True))
