def RunActualGpuTest(self, url, *_):
tab = self.tab
action_runner = tab.action_runner
action_runner.Navigate(url)
action_runner.WaitForJavaScriptCondition(
'window.startTest != undefined')
action_runner.EvaluateJavaScript('window.startTest()')
action_runner.WaitForJavaScriptCondition('window.testDone',
timeout=320)
# Wait for the page to process immediate work and load tiles.
action_runner.EvaluateJavaScript('''
window.testCompleted = false;
requestIdleCallback(
() => window.testCompleted = true,
{ timeout : 10000 })''')
action_runner.WaitForJavaScriptCondition('window.testCompleted',
timeout=30)
expected = _ReadPixelExpectations('maps_pixel_expectations.json')
page = _GetMapsPageForUrl(url, expected)
# Special case some tests on Fuchsia that need to grab the entire contents
# in the screenshot instead of just the visible portion due to small screen
# sizes.
if (MapsIntegrationTest.browser.platform.GetOSName() == 'fuchsia'
and page.name in pixel_test_pages.PROBLEMATIC_FUCHSIA_TESTS):
screenshot = tab.FullScreenshot(5)
else:
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
print('Maps\' devicePixelRatio is ' + str(dpr))
# The bottom corners of Mac screenshots have black triangles due to the
# rounded corners of Mac windows. So, crop the bottom few rows off now to
# get rid of those. The triangles appear to be 5 pixels wide and tall
# regardless of DPI, so 10 pixels should be sufficient. However, when
# running under Python 3, 10 isn't quite enough for some reason, so use
# 20 instead.
if self.browser.platform.GetOSName() == 'mac':
img_height = image_util.Height(screenshot)
img_width = image_util.Width(screenshot)
screenshot = image_util.Crop(screenshot, 0, 0, img_width,
img_height - 20)
x1, y1, x2, y2 = _GetCropBoundaries(screenshot)
screenshot = image_util.Crop(screenshot, x1, y1, x2 - x1, y2 - y1)
self._ValidateScreenshotSamplesWithSkiaGold(tab, page, screenshot, dpr)
def ValidateAndMeasurePage(self, page, tab, results):
if not _DidTestSucceed(tab):
raise page_test.Failure('Page indicated a failure')
if not hasattr(page, 'expectations') or not page.expectations:
raise page_test.Failure('Expectations not specified')
if not tab.screenshot_supported:
raise page_test.Failure('Browser does not support screenshot capture')
screenshot = tab.Screenshot()
if screenshot is None:
raise page_test.Failure('Could not capture screenshot')
device_pixel_ratio = tab.EvaluateJavaScript('window.devicePixelRatio')
if hasattr(page, 'test_rect'):
test_rect = [int(x * device_pixel_ratio) for x in page.test_rect]
screenshot = image_util.Crop(screenshot, test_rect[0], test_rect[1],
test_rect[2], test_rect[3])
self._ValidateScreenshotSamples(
tab,
page.display_name,
screenshot,
page.expectations,
device_pixel_ratio)
def _RunSkiaGoldBasedPixelTest(self, page):
"""Captures and compares a test image using Skia Gold.
Raises an Exception if the comparison fails.
Args:
page: the GPU PixelTestPage object for the test.
"""
tab = self.tab
# Actually run the test and capture the screenshot.
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr),
int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
build_id_args = self._GetBuildIdArgs()
# Compare images against approved images/colors.
if page.expected_colors:
# Use expected colors instead of hash comparison for validation.
self._ValidateScreenshotSamplesWithSkiaGold(tab, page, screenshot, dpr,
build_id_args)
return
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(
image_name, screenshot, page, build_id_args=build_id_args)
def _RunSkiaGoldBasedPixelTest(self, page):
"""Captures and compares a test image using Skia Gold.
Raises an Exception if the comparison fails.
Args:
page: the GPU PixelTestPage object for the test.
"""
tab = self.tab
# Actually run the test and capture the screenshot.
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
# Special case some tests on Fuchsia that need to grab the entire contents
# in the screenshot instead of just the visible portion due to small screen
# sizes.
if (PixelIntegrationTest.browser.platform.GetOSName() == 'fuchsia'
and page.name in pixel_test_pages.PROBLEMATIC_FUCHSIA_TESTS):
screenshot = tab.FullScreenshot(5)
else:
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
# When actually clamping the value, it's possible we'll catch the
# scrollbar, so account for its width in the clamp.
end_x = min(int(page.test_rect[2] * dpr),
image_util.Width(screenshot) - SCROLLBAR_WIDTH)
end_y = min(int(page.test_rect[3] * dpr), image_util.Height(screenshot))
screenshot = image_util.Crop(screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr), end_x, end_y)
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(image_name, screenshot, page)
def _RunSkiaGoldBasedPixelTest(self, do_page_action, page):
"""Captures and compares a test image using Skia Gold.
Raises an Exception if the comparison fails.
Args:
do_page_action: a bool indicating if an action was run on the page.
page: the GPU PixelTestPage object for the test.
"""
tab = self.tab
try:
# Actually run the test and capture the screenshot.
if not tab.EvaluateJavaScript(
'domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(screenshot,
int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr),
int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
# Get all the information that goldctl requires.
parsed_options = self.GetParsedCommandLineOptions()
build_id_args = [
'--commit',
parsed_options.build_revision,
]
# If --review-patch-issue is passed, then we assume we're running on a
# trybot.
if parsed_options.review_patch_issue:
build_id_args += [
'--issue', parsed_options.review_patch_issue, '--patchset',
parsed_options.review_patch_set, '--jobid',
parsed_options.buildbucket_build_id
]
# Compare images against approved images/colors.
if page.expected_colors:
# Use expected colors instead of hash comparison for validation.
self._ValidateScreenshotSamplesWithSkiaGold(
tab, page, screenshot, dpr, build_id_args)
return
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(image_name,
screenshot,
tab,
page,
build_id_args=build_id_args)
finally:
if do_page_action:
# Assume that page actions might have killed the GPU process.
self._RestartBrowser('Must restart after page actions')
def testWriteCroppedBmpToPngFile(self):
pixels = [255, 0, 0, 255, 255, 0, 0, 0, 0,
255, 255, 0, 0, 255, 0, 0, 0, 0]
orig = image_util.FromRGBPixels(3, 2, pixels)
orig = image_util.Crop(orig, 0, 0, 2, 2)
temp_file = tempfile.NamedTemporaryFile(suffix='.png').name
image_util.WritePngFile(orig, temp_file)
new_file = image_util.FromPngFile(temp_file)
self.assertTrue(image_util.AreEqual(orig, new_file, likely_equal=True))
def testCrop(self):
pixels = [0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0,
0, 1, 0, 1, 1, 0, 2, 1, 0, 3, 1, 0,
0, 2, 0, 1, 2, 0, 2, 2, 0, 3, 2, 0]
bmp = image_util.FromRGBPixels(4, 3, pixels)
bmp = image_util.Crop(bmp, 1, 2, 2, 1)
self.assertEquals(2, image_util.Width(bmp))
self.assertEquals(1, image_util.Height(bmp))
image_util.GetPixelColor(bmp, 0, 0).AssertIsRGB(1, 2, 0)
image_util.GetPixelColor(bmp, 1, 0).AssertIsRGB(2, 2, 0)
self.assertEquals(image_util.Pixels(bmp), bytearray([1, 2, 0, 2, 2, 0]))
def RunActualGpuTest(self, url, *_):
tab = self.tab
action_runner = tab.action_runner
action_runner.Navigate(url)
action_runner.WaitForJavaScriptCondition(
'window.startTest != undefined')
action_runner.EvaluateJavaScript('window.startTest()')
action_runner.WaitForJavaScriptCondition('window.testDone',
timeout=320)
# Wait for the page to process immediate work and load tiles.
action_runner.EvaluateJavaScript('''
window.testCompleted = false;
requestIdleCallback(
() => window.testCompleted = true,
{ timeout : 10000 })''')
action_runner.WaitForJavaScriptCondition('window.testCompleted',
timeout=30)
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
print 'Maps\' devicePixelRatio is ' + str(dpr)
expected = _ReadPixelExpectations('maps_pixel_expectations.json')
page = _GetMapsPageForUrl(url, expected)
# The bottom corners of Mac screenshots have black triangles due to the
# rounded corners of Mac windows. So, crop the bottom few rows off now to
# get rid of those. The triangles appear to be 5 pixels wide and tall
# regardless of DPI, so 10 pixels should be sufficient.
if self.browser.platform.GetOSName() == 'mac':
img_height, img_width = screenshot.shape[:2]
screenshot = image_util.Crop(screenshot, 0, 0, img_width,
img_height - 10)
x1, y1, x2, y2 = _GetCropBoundaries(screenshot)
screenshot = image_util.Crop(screenshot, x1, y1, x2 - x1, y2 - y1)
self._ValidateScreenshotSamplesWithSkiaGold(tab, page, screenshot, dpr)
def GetVideoFrameIter(self):
"""Returns the iteration for processing the video capture.
This looks for the initial color flash in the first frame to establish the
tab content boundaries and then omits all frames displaying the flash.
Yields:
(time_ms, image) tuples representing each video keyframe. Only the first
frame is a run of sequential duplicate bitmaps is typically included.
time_ms is milliseconds since navigationStart.
image may be a telemetry.core.Bitmap, or a numpy array depending on
whether numpy is installed.
"""
frame_generator = self._FramesFromMp4(self._video_file_obj.name)
# Flip through frames until we find the initial tab contents flash.
content_box = None
for _, bmp in frame_generator:
content_box = self._FindHighlightBoundingBox(
bmp, HIGHLIGHT_ORANGE_FRAME)
if content_box:
break
if not content_box:
raise BoundingBoxNotFoundException(
'Failed to identify tab contents in video capture.')
# Flip through frames until the flash goes away and emit that as frame 0.
timestamp = 0
for timestamp, bmp in frame_generator:
if not self._FindHighlightBoundingBox(bmp, HIGHLIGHT_ORANGE_FRAME):
yield 0, image_util.Crop(bmp, *content_box)
break
start_time = timestamp
for timestamp, bmp in frame_generator:
yield timestamp - start_time, image_util.Crop(bmp, *content_box)
def _RunSkiaGoldBasedPixelTest(self, do_page_action, page):
"""Captures and compares a test image using Skia Gold.
Raises an Exception if the comparison fails.
Args:
do_page_action: a bool indicating if an action was run on the page.
page: the GPU PixelTestPage object for the test.
"""
tab = self.tab
try:
# Actually run the test and capture the screenshot.
if not tab.EvaluateJavaScript(
'domAutomationController._succeeded'):
self.fail('page indicated test failure')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(screenshot,
int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr),
int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
build_id_args = self._GetBuildIdArgs()
# Compare images against approved images/colors.
if page.expected_colors:
# Use expected colors instead of hash comparison for validation.
self._ValidateScreenshotSamplesWithSkiaGold(
tab, page, screenshot, dpr, build_id_args)
return
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(image_name,
screenshot,
tab,
page,
build_id_args=build_id_args)
finally:
test_messages = self._TestHarnessMessages(tab)
if test_messages:
logging.info('Logging messages from the test:\n' +
test_messages)
if do_page_action or page.restart_browser_after_test:
self._RestartBrowser(
'Must restart after page actions or if required by test')
def testHistogram(self):
pixels = [
1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 8, 7, 6, 5, 4, 6, 1,
2, 3, 1, 2, 3, 8, 7, 6, 5, 4, 6, 1, 2, 3
]
bmp = image_util.FromRGBPixels(4, 3, pixels)
bmp = image_util.Crop(bmp, 1, 1, 2, 2)
hist = image_util.GetColorHistogram(bmp)
for i in xrange(3):
self.assertEquals(sum(hist[i]),
image_util.Width(bmp) * image_util.Height(bmp))
self.assertEquals(hist.r[1], 0)
self.assertEquals(hist.r[5], 2)
self.assertEquals(hist.r[8], 2)
self.assertEquals(hist.g[2], 0)
self.assertEquals(hist.g[4], 2)
self.assertEquals(hist.g[7], 2)
self.assertEquals(hist.b[3], 0)
self.assertEquals(hist.b[6], 4)
def _RunSkiaGoldBasedPixelTest(self, page):
"""Captures and compares a test image using Skia Gold.
Raises an Exception if the comparison fails.
Args:
page: the GPU PixelTestPage object for the test.
"""
tab = self.tab
# Actually run the test and capture the screenshot.
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
# Special case some tests on Fuchsia that need to grab the entire contents
# in the screenshot instead of just the visible portion due to small screen
# sizes.
if (PixelIntegrationTest.browser.platform.GetOSName() == 'fuchsia'
and page.name in pixel_test_pages.PROBLEMATIC_FUCHSIA_TESTS):
screenshot = tab.FullScreenshot(5)
else:
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(screenshot,
int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr),
int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
# Compare images against approved images/colors.
if page.expected_colors:
# Use expected colors instead of hash comparison for validation.
self._ValidateScreenshotSamplesWithSkiaGold(
tab, page, screenshot, dpr)
return
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(image_name, screenshot, page)
def RunActualGpuTest(self, url, *_):
tab = self.tab
action_runner = tab.action_runner
action_runner.Navigate(url)
action_runner.WaitForJavaScriptCondition(
'window.startTest != undefined')
action_runner.EvaluateJavaScript('window.startTest()')
action_runner.WaitForJavaScriptCondition('window.testDone',
timeout=320)
# Wait for the page to process immediate work and load tiles.
action_runner.EvaluateJavaScript('''
window.testCompleted = false;
requestIdleCallback(
() => window.testCompleted = true,
{ timeout : 10000 })''')
action_runner.WaitForJavaScriptCondition('window.testCompleted',
timeout=30)
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
print 'Maps\' devicePixelRatio is ' + str(dpr)
page = _GetMapsPageForUrl(url)
if page.test_rect:
img_height, img_width = screenshot.shape[:2]
screenshot = image_util.Crop(
screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr),
min(int(page.test_rect[2] * dpr), img_width),
min(int(page.test_rect[3] * dpr), img_height))
self._UploadTestResultToSkiaGold(_TEST_NAME, screenshot, page,
self._GetBuildIdArgs())
def RunSkiaGoldBasedPixelTest(self, test_path, do_page_action, args):
page = args[0]
tab = self.tab
try:
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(
screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr), int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
# This is required by Gold whether this is a tryjob or not.
build_id_args = [
'--commit',
self.GetParsedCommandLineOptions().build_revision,
]
parsed_options = self.GetParsedCommandLineOptions()
# Trybots conceptually download their reference images from
# cloud storage; they don't produce them. (TODO(kbr): remove
# this once fully switched over to Gold.)
if parsed_options.download_refimg_from_cloud_storage:
# Tryjobs in Gold need:
# CL issue number (in Gerrit)
# Patchset number
# Buildbucket ID
build_id_args += [
'--issue',
self.GetParsedCommandLineOptions().review_patch_issue,
'--patchset',
self.GetParsedCommandLineOptions().review_patch_set,
'--jobid',
self.GetParsedCommandLineOptions().buildbucket_build_id
]
if page.expected_colors:
# Use expected colors instead of ref images for validation.
self._ValidateScreenshotSamplesWithSkiaGold(
tab, page, screenshot, page.expected_colors, page.tolerance,
dpr, build_id_args)
return
image_name = self._UrlToImageName(page.name)
is_local_run = not (parsed_options.upload_refimg_to_cloud_storage or
parsed_options.download_refimg_from_cloud_storage)
if is_local_run:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(
self.GetParsedCommandLineOptions().reference_dir,
image_name, page.revision, screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(ref_png, screenshot,
tolerance=page.tolerance):
if parsed_options.test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot,
ref_png)
else:
self._WriteErrorImages(
parsed_options.generated_dir, image_name, screenshot, ref_png)
self.fail('Reference image did not match captured screen')
else:
try:
self._UploadTestResultToSkiaGold(
image_name, screenshot,
tab, page,
is_check_mode=True,
build_id_args=build_id_args)
except CalledProcessError:
self.fail('Gold said the test failed, so fail.')
finally:
if do_page_action:
# Assume that page actions might have killed the GPU process.
self._RestartBrowser('Must restart after page actions')
def RunLegacyPixelTest(self, test_path, do_page_action, args):
page = args[0]
tab = self.tab
try:
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(
screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr), int(page.test_rect[2] * dpr),
int(page.test_rect[3] * dpr))
if page.expected_colors:
# Use expected colors instead of ref images for validation.
self._ValidateScreenshotSamples(
tab, page.name, screenshot, page.expected_colors, page.tolerance,
dpr)
return
image_name = self._UrlToImageName(page.name)
if self.GetParsedCommandLineOptions().upload_refimg_to_cloud_storage:
if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
screenshot):
# This is the new reference image; there's nothing to compare against.
ref_png = screenshot
else:
# There was a preexisting reference image, so we might as well
# compare against it.
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
elif self.GetParsedCommandLineOptions().\
download_refimg_from_cloud_storage:
# This bot doesn't have the ability to properly generate a
# reference image, so download it from cloud storage.
try:
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
except cloud_storage.NotFoundError:
# There is no reference image yet in cloud storage. This
# happens when the revision of the test is incremented or when
# a new test is added, because the trybots are not allowed to
# produce reference images, only the bots on the main
# waterfalls. Report this as a failure so the developer has to
# take action by explicitly suppressing the failure and
# removing the suppression once the reference images have been
# generated. Otherwise silent failures could happen for long
# periods of time.
self.fail('Could not find image %s in cloud storage' % image_name)
else:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(
self.GetParsedCommandLineOptions().reference_dir,
image_name, page.revision, screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(ref_png, screenshot, tolerance=page.tolerance):
if self.GetParsedCommandLineOptions().test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png)
else:
self._WriteErrorImages(
self.GetParsedCommandLineOptions().generated_dir, image_name,
screenshot, ref_png)
self.fail('Reference image did not match captured screen')
finally:
if do_page_action:
# Assume that page actions might have killed the GPU process.
self._RestartBrowser('Must restart after page actions')
def RunActualGpuTest(self, test_path, *args):
page = args[0]
# Some pixel tests require non-standard browser arguments. Need to
# check before running each page that it can run in the current
# browser instance.
self.RestartBrowserIfNecessaryWithArgs(page.browser_args)
url = self.UrlOfStaticFilePath(test_path)
# This property actually comes off the class, not 'self'.
tab = self.tab
tab.Navigate(url, script_to_evaluate_on_commit=test_harness_script)
tab.action_runner.WaitForJavaScriptCondition(
'domAutomationController._finished', timeout_in_seconds=300)
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
if not tab.screenshot_supported:
self.fail('Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
screenshot = image_util.Crop(screenshot, page.test_rect[0] * dpr,
page.test_rect[1] * dpr,
page.test_rect[2] * dpr,
page.test_rect[3] * dpr)
if page.expected_colors:
# Use expected colors instead of ref images for validation.
self._ValidateScreenshotSamples(tab, page.name, screenshot,
page.expected_colors, dpr)
return
image_name = self._UrlToImageName(page.name)
if self.GetParsedCommandLineOptions().upload_refimg_to_cloud_storage:
if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
screenshot):
# This is the new reference image; there's nothing to compare against.
ref_png = screenshot
else:
# There was a preexisting reference image, so we might as well
# compare against it.
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
elif self.GetParsedCommandLineOptions(
).download_refimg_from_cloud_storage:
# This bot doesn't have the ability to properly generate a
# reference image, so download it from cloud storage.
try:
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
except cloud_storage.NotFoundError:
# There is no reference image yet in cloud storage. This
# happens when the revision of the test is incremented or when
# a new test is added, because the trybots are not allowed to
# produce reference images, only the bots on the main
# waterfalls. Report this as a failure so the developer has to
# take action by explicitly suppressing the failure and
# removing the suppression once the reference images have been
# generated. Otherwise silent failures could happen for long
# periods of time.
self.fail('Could not find image %s in cloud storage' %
image_name)
else:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(
self.GetParsedCommandLineOptions().reference_dir, image_name,
page.revision, screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(
ref_png, screenshot, tolerance=page.tolerance):
if self.GetParsedCommandLineOptions().test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot,
ref_png)
else:
self._WriteErrorImages(
self.GetParsedCommandLineOptions().generated_dir,
image_name, screenshot, ref_png)
self.fail('Reference image did not match captured screen')
def ValidateAndMeasurePageInner(self, page, tab, results):
if not _DidTestSucceed(tab):
raise page_test.Failure('Page indicated a failure')
if not tab.screenshot_supported:
raise page_test.Failure(
'Browser does not support screenshot capture')
screenshot = tab.Screenshot(5)
if screenshot is None:
raise page_test.Failure('Could not capture screenshot')
if hasattr(page, 'test_rect'):
screenshot = image_util.Crop(screenshot, page.test_rect[0],
page.test_rect[1], page.test_rect[2],
page.test_rect[3])
image_name = self._UrlToImageName(page.display_name)
if self.options.upload_refimg_to_cloud_storage:
if self._ConditionallyUploadToCloudStorage(image_name, page, tab,
screenshot):
# This is the new reference image; there's nothing to compare against.
ref_png = screenshot
else:
# There was a preexisting reference image, so we might as well
# compare against it.
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
elif self.options.download_refimg_from_cloud_storage:
# This bot doesn't have the ability to properly generate a
# reference image, so download it from cloud storage.
try:
ref_png = self._DownloadFromCloudStorage(image_name, page, tab)
except cloud_storage.NotFoundError as e:
# There is no reference image yet in cloud storage. This
# happens when the revision of the test is incremented or when
# a new test is added, because the trybots are not allowed to
# produce reference images, only the bots on the main
# waterfalls. Report this as a failure so the developer has to
# take action by explicitly suppressing the failure and
# removing the suppression once the reference images have been
# generated. Otherwise silent failures could happen for long
# periods of time.
raise page_test.Failure(
'Could not find image %s in cloud storage' % image_name)
else:
# Legacy path using on-disk results.
ref_png = self._GetReferenceImage(self.options.reference_dir,
image_name, page.revision,
screenshot)
# Test new snapshot against existing reference image
if not image_util.AreEqual(ref_png, screenshot, tolerance=2):
if self.options.test_machine_name:
self._UploadErrorImagesToCloudStorage(image_name, screenshot,
ref_png)
else:
self._WriteErrorImages(self.options.generated_dir, image_name,
screenshot, ref_png)
raise page_test.Failure(
'Reference image did not match captured screen')
