def _CompareScreenshotWithExpectation(expectation):
"""Compares a portion of the screenshot to the given expectation.
Fails the test if a the screenshot does not match within the tolerance.
Args:
expectation: A dict defining an expected color region. It must contain
'location', 'size', and 'color' keys. See pixel_test_pages.py for
examples.
"""
location = expectation["location"]
size = expectation["size"]
x0 = int(location[0] * device_pixel_ratio)
x1 = int((location[0] + size[0]) * device_pixel_ratio)
y0 = int(location[1] * device_pixel_ratio)
y1 = int((location[1] + size[1]) * device_pixel_ratio)
for x in range(x0, x1):
for y in range(y0, y1):
if (x < 0 or y < 0 or x >= image_util.Width(screenshot)
or y >= image_util.Height(screenshot)):
self.fail(('Expected pixel location [%d, %d] is out of range on ' +
'[%d, %d] image') % (x, y, image_util.Width(screenshot),
image_util.Height(screenshot)))
actual_color = image_util.GetPixelColor(screenshot, x, y)
expected_color = rgba_color.RgbaColor(
expectation["color"][0], expectation["color"][1],
expectation["color"][2],
expectation["color"][3] if len(expectation["color"]) > 3 else 255)
if not actual_color.IsEqual(expected_color, tolerance):
self.fail('Expected pixel at ' + str(location) +
' (actual pixel (' + str(x) + ', ' + str(y) + ')) ' +
' to be ' + str(expectation["color"]) + " but got [" +
str(actual_color.r) + ", " + str(actual_color.g) + ", " +
str(actual_color.b) + ", " + str(actual_color.a) + "]")
def testDiff(self):
file_bmp = image_util.FromPngFile(test_png_path)
file_bmp_2 = image_util.FromPngFile(test_png_2_path)
diff_bmp = image_util.Diff(file_bmp, file_bmp)
self.assertEquals(2, image_util.Width(diff_bmp))
self.assertEquals(2, image_util.Height(diff_bmp))
image_util.GetPixelColor(diff_bmp, 0, 0).AssertIsRGB(0, 0, 0)
image_util.GetPixelColor(diff_bmp, 1, 1).AssertIsRGB(0, 0, 0)
image_util.GetPixelColor(diff_bmp, 0, 1).AssertIsRGB(0, 0, 0)
image_util.GetPixelColor(diff_bmp, 1, 0).AssertIsRGB(0, 0, 0)
diff_bmp = image_util.Diff(file_bmp, file_bmp_2)
self.assertEquals(3, image_util.Width(diff_bmp))
self.assertEquals(3, image_util.Height(diff_bmp))
image_util.GetPixelColor(diff_bmp, 0, 0).AssertIsRGB(0, 255, 255)
image_util.GetPixelColor(diff_bmp, 1, 1).AssertIsRGB(255, 0, 255)
image_util.GetPixelColor(diff_bmp, 0, 1).AssertIsRGB(255, 255, 0)
image_util.GetPixelColor(diff_bmp, 1, 0).AssertIsRGB(0, 0, 255)
image_util.GetPixelColor(diff_bmp, 0, 2).AssertIsRGB(255, 255, 255)
image_util.GetPixelColor(diff_bmp, 1, 2).AssertIsRGB(255, 255, 255)
image_util.GetPixelColor(diff_bmp, 2, 0).AssertIsRGB(255, 255, 255)
image_util.GetPixelColor(diff_bmp, 2, 1).AssertIsRGB(255, 255, 255)
image_util.GetPixelColor(diff_bmp, 2, 2).AssertIsRGB(255, 255, 255)
def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio):
for expectation in expectations:
location = expectation["location"]
size = expectation["size"]
x0 = int(location[0] * device_pixel_ratio)
x1 = int((location[0] + size[0]) * device_pixel_ratio)
y0 = int(location[1] * device_pixel_ratio)
y1 = int((location[1] + size[1]) * device_pixel_ratio)
for x in range(x0, x1):
for y in range(y0, y1):
if (x < 0 or y < 0 or x >= image_util.Width(screenshot)
or y >= image_util.Height(screenshot)):
raise page_test.Failure(
('Expected pixel location [%d, %d] is out of range on '
+ '[%d, %d] image') %
(x, y, image_util.Width(screenshot),
image_util.Height(screenshot)))
actual_color = image_util.GetPixelColor(screenshot, x, y)
expected_color = rgba_color.RgbaColor(expectation["color"][0],
expectation["color"][1],
expectation["color"][2])
if not actual_color.IsEqual(expected_color,
expectation["tolerance"]):
raise page_test.Failure('Expected pixel at ' +
str(location) + ' to be ' +
str(expectation["color"]) +
" but got [" +
str(actual_color.r) + ", " +
str(actual_color.g) + ", " +
str(actual_color.b) + "]")
def _RunSkiaGoldBasedPixelTest(self, page):
"""Captures and compares a test image using Skia Gold.
Raises an Exception if the comparison fails.
Args:
page: the GPU PixelTestPage object for the test.
"""
tab = self.tab
# Actually run the test and capture the screenshot.
if not tab.EvaluateJavaScript('domAutomationController._succeeded'):
self.fail('page indicated test failure')
# Special case some tests on Fuchsia that need to grab the entire contents
# in the screenshot instead of just the visible portion due to small screen
# sizes.
if (PixelIntegrationTest.browser.platform.GetOSName() == 'fuchsia'
and page.name in pixel_test_pages.PROBLEMATIC_FUCHSIA_TESTS):
screenshot = tab.FullScreenshot(5)
else:
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
if page.test_rect:
# When actually clamping the value, it's possible we'll catch the
# scrollbar, so account for its width in the clamp.
end_x = min(int(page.test_rect[2] * dpr),
image_util.Width(screenshot) - SCROLLBAR_WIDTH)
end_y = min(int(page.test_rect[3] * dpr), image_util.Height(screenshot))
screenshot = image_util.Crop(screenshot, int(page.test_rect[0] * dpr),
int(page.test_rect[1] * dpr), end_x, end_y)
image_name = self._UrlToImageName(page.name)
self._UploadTestResultToSkiaGold(image_name, screenshot, page)
def testScreenShotTakenForFailedPage(self):
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.browser_options.take_screenshot_for_failed_page = True
results = self.RunStorySet(DummyTest(), story_set, max_failures=2)
self.assertTrue(results.had_failures)
# Check that we can find the artifact with a non-empty PNG screenshot.
failing_run = next(run for run in self.ReadTestResults()
if run['testPath'].endswith('/failing'))
screenshot = image_util.FromPngFile(
failing_run['outputArtifacts']['screenshot.png']['filePath'])
self.assertGreater(image_util.Width(screenshot), 0)
self.assertGreater(image_util.Height(screenshot), 0)
def ValidateAndMeasurePage(self, page, tab, results):
try:
tab.WaitForDocumentReadyStateToBeComplete()
except py_utils.TimeoutException:
logging.warning("WaitForDocumentReadyStateToBeComplete() timeout, "
"page: %s", page.name)
return
time.sleep(self._wait_time)
if not os.path.exists(self._png_outdir):
logging.info("Creating directory %s", self._png_outdir)
try:
os.makedirs(self._png_outdir)
except OSError:
logging.warning("Directory %s could not be created", self._png_outdir)
raise
outpath = os.path.abspath(
os.path.join(self._png_outdir, page.file_safe_name)) + '.png'
# Replace win32 path separator char '\' with '\\'.
outpath = outpath.replace('\\', '\\\\')
screenshot = tab.Screenshot()
image_width = image_util.Width(screenshot)
image_height = image_util.Height(screenshot)
num_total_pixels = image_width * image_height
content_pixels = image_util.Pixels(screenshot)
# Dynamic content flag.
if self._dc_detect:
for i in range(self._dc_extra_screenshots):
logging.info("Sleeping for %f seconds.", self._dc_wait_time)
time.sleep(self._dc_wait_time)
# After the specified wait time, take another screenshot of the page.
logging.info("Taking extra screenshot %d of %d.", i+1,
self._dc_extra_screenshots)
next_screenshot = tab.Screenshot()
# Compare this screenshot to the original to mark inconsistent pixels,
# and check the percentage of dynamic content against the threshold.
if not IsScreenshotWithinDynamicContentThreshold(
screenshot, next_screenshot, content_pixels, num_total_pixels,
self._dc_threshold):
raise legacy_page_test.MeasurementFailure("Percentage of pixels "
"with dynamic content is greater than threshold.")
# Convert the pixel bytearray back into an image.
image = image_util.FromRGBPixels(image_width, image_height,
content_pixels)
# TODO(lchoi): Add logging to image_util.py and/or augment error handling of
# image_util.WritePngFile
logging.info("Writing PNG file to %s. This may take awhile.", outpath)
start = time.time()
image_util.WritePngFile(image, outpath)
logging.info("PNG file written successfully. (Took %f seconds)",
time.time()-start)
def testReadFromPngFile(self):
file_bmp = image_util.FromPngFile(test_png_path)
self.assertEquals(2, image_util.Width(file_bmp))
self.assertEquals(2, image_util.Height(file_bmp))
image_util.GetPixelColor(file_bmp, 0, 0).AssertIsRGB(255, 0, 0)
image_util.GetPixelColor(file_bmp, 1, 1).AssertIsRGB(0, 255, 0)
image_util.GetPixelColor(file_bmp, 0, 1).AssertIsRGB(0, 0, 255)
image_util.GetPixelColor(file_bmp, 1, 0).AssertIsRGB(255, 255, 0)
def testReadFromBase64Png(self):
bmp = image_util.FromBase64Png(test_png)
self.assertEquals(2, image_util.Width(bmp))
self.assertEquals(2, image_util.Height(bmp))
image_util.GetPixelColor(bmp, 0, 0).AssertIsRGB(255, 0, 0)
image_util.GetPixelColor(bmp, 1, 1).AssertIsRGB(0, 255, 0)
image_util.GetPixelColor(bmp, 0, 1).AssertIsRGB(0, 0, 255)
image_util.GetPixelColor(bmp, 1, 0).AssertIsRGB(255, 255, 0)
def testCrop(self):
pixels = [0, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 0,
0, 1, 0, 1, 1, 0, 2, 1, 0, 3, 1, 0,
0, 2, 0, 1, 2, 0, 2, 2, 0, 3, 2, 0]
bmp = image_util.FromRGBPixels(4, 3, pixels)
bmp = image_util.Crop(bmp, 1, 2, 2, 1)
self.assertEquals(2, image_util.Width(bmp))
self.assertEquals(1, image_util.Height(bmp))
image_util.GetPixelColor(bmp, 0, 0).AssertIsRGB(1, 2, 0)
image_util.GetPixelColor(bmp, 1, 0).AssertIsRGB(2, 2, 0)
self.assertEquals(image_util.Pixels(bmp), bytearray([1, 2, 0, 2, 2, 0]))
def _GetCropBoundaries(screenshot):
"""Returns the boundaries to crop the screenshot to.
Specifically, we look for the boundaries where the white background
transitions into the (non-white) content we care about.
Args:
screenshot: A screenshot returned by Tab.Screenshot() (numpy ndarray?)
Returns:
A 4-tuple (x1, y1, x2, y2) denoting the top left and bottom right
coordinates to crop to.
"""
img_height = image_util.Height(screenshot)
img_width = image_util.Width(screenshot)
def RowIsWhite(row):
for col in range(img_width):
pixel = image_util.GetPixelColor(screenshot, col, row)
if pixel.r != 255 or pixel.g != 255 or pixel.b != 255:
return False
return True
def ColumnIsWhite(column):
for row in range(img_height):
pixel = image_util.GetPixelColor(screenshot, column, row)
if pixel.r != 255 or pixel.g != 255 or pixel.b != 255:
return False
return True
x1 = y1 = 0
x2 = img_width
y2 = img_height
for column in range(img_width):
if not ColumnIsWhite(column):
x1 = column
break
for row in range(img_height):
if not RowIsWhite(row):
y1 = row
break
for column in range(x1 + 1, img_width):
if ColumnIsWhite(column):
x2 = column
break
for row in range(y1 + 1, img_height):
if RowIsWhite(row):
y2 = row
break
return x1, y1, x2, y2
def RunActualGpuTest(self, url, *_):
tab = self.tab
action_runner = tab.action_runner
action_runner.Navigate(url)
action_runner.WaitForJavaScriptCondition(
'window.startTest != undefined')
action_runner.EvaluateJavaScript('window.startTest()')
action_runner.WaitForJavaScriptCondition('window.testDone',
timeout=320)
# Wait for the page to process immediate work and load tiles.
action_runner.EvaluateJavaScript('''
window.testCompleted = false;
requestIdleCallback(
() => window.testCompleted = true,
{ timeout : 10000 })''')
action_runner.WaitForJavaScriptCondition('window.testCompleted',
timeout=30)
expected = _ReadPixelExpectations('maps_pixel_expectations.json')
page = _GetMapsPageForUrl(url, expected)
# Special case some tests on Fuchsia that need to grab the entire contents
# in the screenshot instead of just the visible portion due to small screen
# sizes.
if (MapsIntegrationTest.browser.platform.GetOSName() == 'fuchsia'
and page.name in pixel_test_pages.PROBLEMATIC_FUCHSIA_TESTS):
screenshot = tab.FullScreenshot(5)
else:
screenshot = tab.Screenshot(5)
if screenshot is None:
self.fail('Could not capture screenshot')
dpr = tab.EvaluateJavaScript('window.devicePixelRatio')
print('Maps\' devicePixelRatio is ' + str(dpr))
# The bottom corners of Mac screenshots have black triangles due to the
# rounded corners of Mac windows. So, crop the bottom few rows off now to
# get rid of those. The triangles appear to be 5 pixels wide and tall
# regardless of DPI, so 10 pixels should be sufficient. However, when
# running under Python 3, 10 isn't quite enough for some reason, so use
# 20 instead.
if self.browser.platform.GetOSName() == 'mac':
img_height = image_util.Height(screenshot)
img_width = image_util.Width(screenshot)
screenshot = image_util.Crop(screenshot, 0, 0, img_width,
img_height - 20)
x1, y1, x2, y2 = _GetCropBoundaries(screenshot)
screenshot = image_util.Crop(screenshot, x1, y1, x2 - x1, y2 - y1)
self._ValidateScreenshotSamplesWithSkiaGold(tab, page, screenshot, dpr)
def testHistogram(self):
pixels = [
1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 8, 7, 6, 5, 4, 6, 1,
2, 3, 1, 2, 3, 8, 7, 6, 5, 4, 6, 1, 2, 3
]
bmp = image_util.FromRGBPixels(4, 3, pixels)
bmp = image_util.Crop(bmp, 1, 1, 2, 2)
hist = image_util.GetColorHistogram(bmp)
for i in xrange(3):
self.assertEquals(sum(hist[i]),
image_util.Width(bmp) * image_util.Height(bmp))
self.assertEquals(hist.r[1], 0)
self.assertEquals(hist.r[5], 2)
self.assertEquals(hist.r[8], 2)
self.assertEquals(hist.g[2], 0)
self.assertEquals(hist.g[4], 2)
self.assertEquals(hist.g[7], 2)
self.assertEquals(hist.b[3], 0)
self.assertEquals(hist.b[6], 4)
def _CompareScreenshotSamples(self, tab, screenshot, expected_colors,
tolerance, device_pixel_ratio,
test_machine_name):
# First scan through the expected_colors and see if there are any scale
# factor overrides that would preempt the device pixel ratio. This
# is mainly a workaround for complex tests like the Maps test.
for expectation in expected_colors:
if 'scale_factor_overrides' in expectation:
for override in expectation['scale_factor_overrides']:
# Require exact matches to avoid confusion, because some
# machine models and names might be subsets of others
# (e.g. Nexus 5 vs Nexus 5X).
if ('device_type' in override
and (tab.browser.platform.GetDeviceTypeName()
== override['device_type'])):
logging.warning('Overriding device_pixel_ratio ' +
str(device_pixel_ratio) +
' with scale factor ' +
str(override['scale_factor']) +
' for device type ' +
override['device_type'])
device_pixel_ratio = override['scale_factor']
break
if (test_machine_name and 'machine_name' in override
and override["machine_name"] == test_machine_name):
logging.warning('Overriding device_pixel_ratio ' +
str(device_pixel_ratio) +
' with scale factor ' +
str(override['scale_factor']) +
' for machine name ' +
test_machine_name)
device_pixel_ratio = override['scale_factor']
break
# Only support one "scale_factor_overrides" in the expectation format.
break
for expectation in expected_colors:
if "scale_factor_overrides" in expectation:
continue
location = expectation["location"]
size = expectation["size"]
x0 = int(location[0] * device_pixel_ratio)
x1 = int((location[0] + size[0]) * device_pixel_ratio)
y0 = int(location[1] * device_pixel_ratio)
y1 = int((location[1] + size[1]) * device_pixel_ratio)
for x in range(x0, x1):
for y in range(y0, y1):
if (x < 0 or y < 0 or x >= image_util.Width(screenshot)
or y >= image_util.Height(screenshot)):
self.fail((
'Expected pixel location [%d, %d] is out of range on '
+ '[%d, %d] image') %
(x, y, image_util.Width(screenshot),
image_util.Height(screenshot)))
actual_color = image_util.GetPixelColor(screenshot, x, y)
expected_color = rgba_color.RgbaColor(
expectation["color"][0], expectation["color"][1],
expectation["color"][2])
if not actual_color.IsEqual(expected_color, tolerance):
self.fail('Expected pixel at ' + str(location) +
' (actual pixel (' + str(x) + ', ' + str(y) +
')) ' + ' to be ' +
str(expectation["color"]) + " but got [" +
str(actual_color.r) + ", " +
str(actual_color.g) + ", " +
str(actual_color.b) + "]")
