def ValidateAndMeasurePage(self, page, tab, results):
try:
tab.WaitForDocumentReadyStateToBeComplete()
except py_utils.TimeoutException:
logging.warning("WaitForDocumentReadyStateToBeComplete() timeout, "
"page: %s", page.name)
return
time.sleep(self._wait_time)
if not os.path.exists(self._png_outdir):
logging.info("Creating directory %s", self._png_outdir)
try:
os.makedirs(self._png_outdir)
except OSError:
logging.warning("Directory %s could not be created", self._png_outdir)
raise
outpath = os.path.abspath(
os.path.join(self._png_outdir, page.file_safe_name)) + '.png'
# Replace win32 path separator char '\' with '\\'.
outpath = outpath.replace('\\', '\\\\')
screenshot = tab.Screenshot()
image_width = image_util.Width(screenshot)
image_height = image_util.Height(screenshot)
num_total_pixels = image_width * image_height
content_pixels = image_util.Pixels(screenshot)
# Dynamic content flag.
if self._dc_detect:
for i in range(self._dc_extra_screenshots):
logging.info("Sleeping for %f seconds.", self._dc_wait_time)
time.sleep(self._dc_wait_time)
# After the specified wait time, take another screenshot of the page.
logging.info("Taking extra screenshot %d of %d.", i+1,
self._dc_extra_screenshots)
next_screenshot = tab.Screenshot()
# Compare this screenshot to the original to mark inconsistent pixels,
# and check the percentage of dynamic content against the threshold.
if not IsScreenshotWithinDynamicContentThreshold(
screenshot, next_screenshot, content_pixels, num_total_pixels,
self._dc_threshold):
raise legacy_page_test.MeasurementFailure("Percentage of pixels "
"with dynamic content is greater than threshold.")
# Convert the pixel bytearray back into an image.
image = image_util.FromRGBPixels(image_width, image_height,
content_pixels)
# TODO(lchoi): Add logging to image_util.py and/or augment error handling of
# image_util.WritePngFile
logging.info("Writing PNG file to %s. This may take awhile.", outpath)
start = time.time()
image_util.WritePngFile(image, outpath)
logging.info("PNG file written successfully. (Took %f seconds)",
time.time()-start)
def ValidateAndMeasurePage(self, page, tab, results):
if tab.browser.platform.GetOSName() in ['android', 'chromeos']:
raise legacy_page_test.MeasurementFailure(
'Multipage SkPicture printing not supported on this platform')
# Replace win32 path separator char '\' with '\\'.
outpath = os.path.abspath(
os.path.join(self._mskp_outdir, page.file_safe_name + '.mskp'))
js = _JS.format(outpath.replace('\\', '\\\\'))
tab.EvaluateJavaScript(js)
def ValidateAndMeasurePage(self, page, tab, results):
tab.WaitForJavaScriptCondition(
'testRunner.isDone || testRunner.isWaitingForTracingStart',
timeout=600)
trace_cpu_time_metrics = {}
if tab.EvaluateJavaScript('testRunner.isWaitingForTracingStart'):
trace_data = self._ContinueTestRunWithTracing(tab)
# TODO(#763375): Rely on results.telemetry_info.trace_local_path/etc.
kwargs = {}
if hasattr(results.telemetry_info, 'trace_local_path'):
kwargs['file_path'] = results.telemetry_info.trace_local_path
kwargs[
'remote_path'] = results.telemetry_info.trace_remote_path
kwargs['upload_bucket'] = results.telemetry_info.upload_bucket
kwargs['cloud_url'] = results.telemetry_info.trace_remote_url
trace_value = trace.TraceValue(page, trace_data, **kwargs)
results.AddValue(trace_value)
trace_events_to_measure = tab.EvaluateJavaScript(
'window.testRunner.traceEventsToMeasure')
model = model_module.TimelineModel(trace_data)
renderer_thread = model.GetRendererThreadFromTabId(tab.id)
trace_cpu_time_metrics = _ComputeTraceEventsThreadTimeForBlinkPerf(
model, renderer_thread, trace_events_to_measure)
log = tab.EvaluateJavaScript(
'document.getElementById("log").innerHTML')
for line in log.splitlines():
if line.startswith("FATAL: "):
print line
continue
if not line.startswith('values '):
continue
parts = line.split()
values = [float(v.replace(',', '')) for v in parts[1:-1]]
units = parts[-1]
metric = page.name.split('.')[0].replace('/', '_')
if values:
results.AddValue(
list_of_scalar_values.ListOfScalarValues(
results.current_page, metric, units, values))
else:
raise legacy_page_test.MeasurementFailure('Empty test results')
break
print log
self.PrintAndCollectTraceEventMetrics(trace_cpu_time_metrics, results)
def ValidateAndMeasurePage(self, page, tab, results):
if tab.browser.platform.GetOSName() in ['android', 'chromeos']:
raise legacy_page_test.MeasurementFailure(
'SkPicture printing not supported on this platform')
outpath = os.path.abspath(
os.path.join(self._skp_outdir, page.file_safe_name))
# Replace win32 path separator char '\' with '\\'.
outpath = outpath.replace('\\', '\\\\')
tab.EvaluateJavaScript(
'chrome.gpuBenchmarking.printToSkPicture({{ outpath }});',
outpath=outpath)
pictures = glob.glob(os.path.join(outpath, '*.skp'))
results.AddMeasurement('saved_picture_count', 'count', len(pictures))
def ValidateAndMeasurePage(self, page, tab, results):
if tab.browser.platform.GetOSName() in ['android', 'chromeos']:
raise legacy_page_test.MeasurementFailure(
'SkPicture printing not supported on this platform')
# Replace win32 path separator char '\' with '\\'.
outpath = os.path.abspath(
os.path.join(self._skp_outdir, page.file_safe_name))
js = _JS.format(outpath.replace('\\', '\\\\'))
tab.EvaluateJavaScript(js)
pictures = glob.glob(os.path.join(outpath, '*.skp'))
results.AddValue(
scalar.ScalarValue(results.current_page, 'saved_picture_count',
'count', len(pictures)))
def RunPageInteractions(self, action_runner):
action_runner.WaitForJavaScriptCondition(
'document.readyState == "complete"', timeout=30)
action_runner.ExecuteJavaScript(
'chrome.gpuBenchmarking.setRasterizeOnlyVisibleContent();')
mode = 'viewport'
width = None
height = None
args = {}
args['mode'] = mode
if width:
args['width'] = width
if height:
args['height'] = height
# Enqueue benchmark
action_runner.ExecuteJavaScript("""
window.benchmark_results = {};
window.benchmark_results.id =
chrome.gpuBenchmarking.runMicroBenchmark(
"invalidation_benchmark",
function(value) {},
{{ args }}
);
""",
args=args)
micro_benchmark_id = action_runner.EvaluateJavaScript(
'window.benchmark_results.id')
if not micro_benchmark_id:
raise legacy_page_test.MeasurementFailure(
'Failed to schedule invalidation_benchmark.')
with action_runner.CreateInteraction('Repaint'):
action_runner.RepaintContinuously(seconds=5)
action_runner.ExecuteJavaScript("""
window.benchmark_results.message_handled =
chrome.gpuBenchmarking.sendMessageToMicroBenchmark(
{{ micro_benchmark_id }}, {
"notify_done": true
});
""",
micro_benchmark_id=micro_benchmark_id)
def Repaint(action_runner, mode='viewport', width=None, height=None):
action_runner.WaitForJavaScriptCondition(
'document.readyState == "complete"', 90)
# Rasterize only what's visible.
action_runner.ExecuteJavaScript(
'chrome.gpuBenchmarking.setRasterizeOnlyVisibleContent();')
args = {}
args['mode'] = mode
if width:
args['width'] = width
if height:
args['height'] = height
# Enqueue benchmark
action_runner.ExecuteJavaScript("""
window.benchmark_results = {};
window.benchmark_results.id =
chrome.gpuBenchmarking.runMicroBenchmark(
"invalidation_benchmark",
function(value) {},
""" + str(args) + """
);
""")
micro_benchmark_id = action_runner.EvaluateJavaScript(
'window.benchmark_results.id')
if not micro_benchmark_id:
raise legacy_page_test.MeasurementFailure(
'Failed to schedule invalidation_benchmark.')
with action_runner.CreateInteraction('Repaint'):
action_runner.RepaintContinuously(seconds=5)
# TODO(catapult:#3028): Fix interpolation of JavaScript values.
action_runner.ExecuteJavaScript("""
window.benchmark_results.message_handled =
chrome.gpuBenchmarking.sendMessageToMicroBenchmark(
""" + str(micro_benchmark_id) + """, {
"notify_done": true
});
""")
def ValidateAndMeasurePage(self, page, tab, results):
trace_cpu_time_metrics = {}
if self._is_tracing:
trace_data = tab.browser.platform.tracing_controller.StopTracing()
results.AddTraces(trace_data)
self._is_tracing = False
trace_events_to_measure = tab.EvaluateJavaScript(
'window.testRunner.traceEventsToMeasure')
if trace_events_to_measure:
model = model_module.TimelineModel(trace_data)
renderer_thread = model.GetFirstRendererThread(tab.id)
trace_cpu_time_metrics = _ComputeTraceEventsThreadTimeForBlinkPerf(
model, renderer_thread, trace_events_to_measure)
log = tab.EvaluateJavaScript(
'document.getElementById("log").innerHTML')
for line in log.splitlines():
if line.startswith("FATAL: "):
print(line)
continue
if not line.startswith('values '):
continue
parts = line.split()
values = [float(v.replace(',', '')) for v in parts[1:-1]]
units = parts[-1]
metric = page.name.split('.')[0].replace('/', '_')
if values:
results.AddMeasurement(metric, units, values)
else:
raise legacy_page_test.MeasurementFailure('Empty test results')
break
print(log)
self.PrintAndCollectTraceEventMetrics(trace_cpu_time_metrics, results)
def ValidateAndMeasurePage(self, page, tab, results):
if not tab.screenshot_supported:
raise legacy_page_test.MeasurementFailure(
'Screenshotting not supported on this platform')
try:
tab.WaitForDocumentReadyStateToBeComplete()
except py_utils.TimeoutException:
logging.warning(
"WaitForDocumentReadyStateToBeComplete() timeout, " +
"page: %s", page.display_name)
return
time.sleep(self._wait_time)
if not os.path.exists(self._png_outdir):
logging.info("Creating directory %s", self._png_outdir)
try:
os.makedirs(self._png_outdir)
except OSError:
logging.warning("Directory %s could not be created",
self._png_outdir)
raise
outpath = os.path.abspath(
os.path.join(self._png_outdir, page.file_safe_name)) + '.png'
# Replace win32 path separator char '\' with '\\'.
outpath = outpath.replace('\\', '\\\\')
screenshot = tab.Screenshot()
# TODO(lchoi): Add logging to image_util.py and/or augment error handling of
# image_util.WritePngFile
logging.info("Writing PNG file to %s. This may take awhile.", outpath)
start = time.time()
image_util.WritePngFile(screenshot, outpath)
logging.info("PNG file written successfully. (Took %f seconds)",
time.time() - start)
def ValidateAndMeasurePage(self, page, tab, results):
del page # unused
try:
tab.WaitForDocumentReadyStateToBeComplete()
except exceptions.TimeoutException:
pass
time.sleep(self._start_wait_time)
# Enqueue benchmark
tab.ExecuteJavaScript("""
window.benchmark_results = {};
window.benchmark_results.done = false;
window.benchmark_results.id =
chrome.gpuBenchmarking.runMicroBenchmark(
"rasterize_and_record_benchmark",
function(value) {
window.benchmark_results.done = true;
window.benchmark_results.results = value;
}, {
"record_repeat_count": %i,
"rasterize_repeat_count": %i
});
""" % (self._record_repeat, self._rasterize_repeat))
benchmark_id = tab.EvaluateJavaScript('window.benchmark_results.id')
if not benchmark_id:
raise legacy_page_test.MeasurementFailure(
'Failed to schedule rasterize_and_record_micro')
tab.WaitForJavaScriptExpression('window.benchmark_results.done',
self._timeout)
data = tab.EvaluateJavaScript('window.benchmark_results.results')
pixels_recorded = data['pixels_recorded']
record_time = data['record_time_ms']
pixels_rasterized = data['pixels_rasterized']
rasterize_time = data['rasterize_time_ms']
picture_memory_usage = data['picture_memory_usage']
results.AddValue(
scalar.ScalarValue(results.current_page, 'pixels_recorded',
'pixels', pixels_recorded))
results.AddValue(
scalar.ScalarValue(results.current_page, 'pixels_rasterized',
'pixels', pixels_rasterized))
results.AddValue(
scalar.ScalarValue(results.current_page, 'rasterize_time', 'ms',
rasterize_time))
results.AddValue(
scalar.ScalarValue(results.current_page, 'viewport_picture_size',
'bytes', picture_memory_usage))
results.AddValue(
scalar.ScalarValue(results.current_page, 'record_time', 'ms',
record_time))
record_time_painting_disabled = data[
'record_time_painting_disabled_ms']
record_time_caching_disabled = data['record_time_caching_disabled_ms']
record_time_construction_disabled = \
data['record_time_construction_disabled_ms']
# TODO(wangxianzhu): Remove this workaround when reference builds get past
# r367465.
record_time_subsequence_caching_disabled = \
data.get('record_time_subsequence_caching_disabled_ms', 0)
# TODO(wkorman): Remove the default-to-zero workaround below when
# reference builds get past the change that adds this comment.
record_time_partial_invalidation = \
data.get('record_time_partial_invalidation_ms', 0)
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_painting_disabled', 'ms',
record_time_painting_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_caching_disabled', 'ms',
record_time_caching_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_construction_disabled', 'ms',
record_time_construction_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_subsequence_caching_disabled',
'ms', record_time_subsequence_caching_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_partial_invalidation_ms', 'ms',
record_time_partial_invalidation))
if self._report_detailed_results:
pixels_rasterized_with_non_solid_color = \
data['pixels_rasterized_with_non_solid_color']
pixels_rasterized_as_opaque = \
data['pixels_rasterized_as_opaque']
total_layers = data['total_layers']
total_picture_layers = data['total_picture_layers']
total_picture_layers_with_no_content = \
data['total_picture_layers_with_no_content']
total_picture_layers_off_screen = \
data['total_picture_layers_off_screen']
# TODO(wkorman): Why are we storing rasterize_results_.total_memory_usage
# in a field called |total_pictures_in_pile_size|? Did we just repurpose
# that field to avoid having to rename/create another?
total_pictures_in_pile_size = data['total_pictures_in_pile_size']
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_size_of_pictures_in_piles', 'bytes',
total_pictures_in_pile_size))
results.AddValue(
scalar.ScalarValue(results.current_page,
'pixels_rasterized_with_non_solid_color',
'pixels',
pixels_rasterized_with_non_solid_color))
results.AddValue(
scalar.ScalarValue(results.current_page,
'pixels_rasterized_as_opaque', 'pixels',
pixels_rasterized_as_opaque))
results.AddValue(
scalar.ScalarValue(results.current_page, 'total_layers',
'count', total_layers))
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_picture_layers', 'count',
total_picture_layers))
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_picture_layers_with_no_content',
'count',
total_picture_layers_with_no_content))
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_picture_layers_off_screen', 'count',
total_picture_layers_off_screen))
def ValidateAndMeasurePage(self, page, tab, results):
query = tab.EvaluateJavaScript('window.location.search')
expected = '?foo=1'
if query.strip() != expected:
raise legacy_page_test.MeasurementFailure('query was %s, not %s.' %
(query, expected))
def ValidateAndMeasurePage(self, page, tab, results):
# Web Page Replay returns '404 Not found' if a page is not in the archive.
contents = tab.EvaluateJavaScript('document.body.textContent')
if '404 Not Found' in contents.strip():
raise legacy_page_test.MeasurementFailure('Page not in archive.')
def ValidateAndMeasurePage(self, page, tab, results):
contents = tab.EvaluateJavaScript('document.body.textContent')
if contents.strip() != 'Hello world':
raise legacy_page_test.MeasurementFailure('Page contents were: ' +
contents)
def ValidateAndMeasurePage(self, page, tab, results):
del page # unused
try:
tab.WaitForDocumentReadyStateToBeComplete()
except py_utils.TimeoutException:
pass
time.sleep(self._start_wait_time)
# Enqueue benchmark
tab.ExecuteJavaScript("""
window.benchmark_results = {};
window.benchmark_results.done = false;
window.benchmark_results.id =
chrome.gpuBenchmarking.runMicroBenchmark(
"rasterize_and_record_benchmark",
function(value) {
window.benchmark_results.done = true;
window.benchmark_results.results = value;
}, {
"record_repeat_count": {{ record_repeat_count }},
"rasterize_repeat_count": {{ rasterize_repeat_count }}
});
""",
record_repeat_count=self._record_repeat,
rasterize_repeat_count=self._rasterize_repeat)
# Evaluating this expression usually takes between 60 and 90 seconds.
benchmark_id = tab.EvaluateJavaScript('window.benchmark_results.id',
timeout=self._timeout)
if not benchmark_id:
raise legacy_page_test.MeasurementFailure(
'Failed to schedule rasterize_and_record_micro')
tab.WaitForJavaScriptCondition('window.benchmark_results.done',
timeout=self._timeout)
data = tab.EvaluateJavaScript('window.benchmark_results.results')
pixels_recorded = data['pixels_recorded']
record_time = data['record_time_ms']
pixels_rasterized = data['pixels_rasterized']
rasterize_time = data['rasterize_time_ms']
painter_memory_usage = data.get('painter_memory_usage', 0)
paint_op_memory_usage = data.get('paint_op_memory_usage', 0)
paint_op_count = data.get('paint_op_count', 0)
results.AddValue(
scalar.ScalarValue(results.current_page, 'pixels_recorded',
'pixels', pixels_recorded))
results.AddValue(
scalar.ScalarValue(results.current_page, 'pixels_rasterized',
'pixels', pixels_rasterized))
results.AddValue(
scalar.ScalarValue(results.current_page, 'rasterize_time', 'ms',
rasterize_time))
results.AddValue(
scalar.ScalarValue(results.current_page, 'record_time', 'ms',
record_time))
results.AddValue(
scalar.ScalarValue(results.current_page, 'painter_memory_usage',
'bytes', painter_memory_usage))
results.AddValue(
scalar.ScalarValue(results.current_page, 'paint_op_memory_usage',
'bytes', paint_op_memory_usage))
results.AddValue(
scalar.ScalarValue(results.current_page, 'paint_op_count', 'count',
paint_op_count))
record_time_painting_disabled = data[
'record_time_painting_disabled_ms']
record_time_caching_disabled = data['record_time_caching_disabled_ms']
record_time_construction_disabled = \
data['record_time_construction_disabled_ms']
record_time_subsequence_caching_disabled = \
data['record_time_subsequence_caching_disabled_ms']
record_time_partial_invalidation = \
data['record_time_partial_invalidation_ms']
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_painting_disabled', 'ms',
record_time_painting_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_caching_disabled', 'ms',
record_time_caching_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_construction_disabled', 'ms',
record_time_construction_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_subsequence_caching_disabled',
'ms', record_time_subsequence_caching_disabled))
results.AddValue(
scalar.ScalarValue(results.current_page,
'record_time_partial_invalidation_ms', 'ms',
record_time_partial_invalidation))
if self._report_detailed_results:
pixels_rasterized_with_non_solid_color = \
data['pixels_rasterized_with_non_solid_color']
pixels_rasterized_as_opaque = data['pixels_rasterized_as_opaque']
total_layers = data['total_layers']
total_picture_layers = data['total_picture_layers']
total_picture_layers_with_no_content = \
data['total_picture_layers_with_no_content']
total_picture_layers_off_screen = data[
'total_picture_layers_off_screen']
results.AddValue(
scalar.ScalarValue(results.current_page,
'pixels_rasterized_with_non_solid_color',
'pixels',
pixels_rasterized_with_non_solid_color))
results.AddValue(
scalar.ScalarValue(results.current_page,
'pixels_rasterized_as_opaque', 'pixels',
pixels_rasterized_as_opaque))
results.AddValue(
scalar.ScalarValue(results.current_page, 'total_layers',
'count', total_layers))
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_picture_layers', 'count',
total_picture_layers))
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_picture_layers_with_no_content',
'count',
total_picture_layers_with_no_content))
results.AddValue(
scalar.ScalarValue(results.current_page,
'total_picture_layers_off_screen', 'count',
total_picture_layers_off_screen))
def ValidateAndMeasurePage(self, page, tab, results):
del page # unused
try:
tab.WaitForDocumentReadyStateToBeComplete()
except py_utils.TimeoutException:
pass
time.sleep(self._start_wait_time)
# Enqueue benchmark
tab.ExecuteJavaScript("""
window.benchmark_results = {};
window.benchmark_results.done = false;
window.benchmark_results.id =
chrome.gpuBenchmarking.runMicroBenchmark(
"rasterize_and_record_benchmark",
function(value) {
window.benchmark_results.done = true;
window.benchmark_results.results = value;
}, {
"record_repeat_count": {{ record_repeat_count }},
"rasterize_repeat_count": {{ rasterize_repeat_count }}
});
""",
record_repeat_count=self._record_repeat,
rasterize_repeat_count=self._rasterize_repeat)
# Evaluating this expression usually takes between 60 and 90 seconds.
benchmark_id = tab.EvaluateJavaScript(
'window.benchmark_results.id', timeout=self._timeout)
if not benchmark_id:
raise legacy_page_test.MeasurementFailure(
'Failed to schedule rasterize_and_record_micro')
tab.WaitForJavaScriptCondition(
'window.benchmark_results.done', timeout=self._timeout)
data = tab.EvaluateJavaScript('window.benchmark_results.results')
pixels_recorded = data['pixels_recorded']
pixels_rasterized = data['pixels_rasterized']
painter_memory_usage = data.get('painter_memory_usage', 0)
paint_op_memory_usage = data.get('paint_op_memory_usage', 0)
paint_op_count = data.get('paint_op_count', 0)
results.AddMeasurement('pixels_recorded', 'count', pixels_recorded)
results.AddMeasurement('pixels_rasterized', 'count', pixels_rasterized)
results.AddMeasurement('painter_memory_usage', 'bytes',
painter_memory_usage)
results.AddMeasurement('paint_op_memory_usage', 'bytes',
paint_op_memory_usage)
results.AddMeasurement('paint_op_count', 'count', paint_op_count)
for metric in ('rasterize_time', 'record_time',
'record_time_caching_disabled',
'record_time_subsequence_caching_disabled',
'record_time_partial_invalidation',
'raster_invalidation_and_convert_time',
'paint_artifact_compositor_update_time'):
results.AddMeasurement(metric, 'ms', data.get(metric + '_ms', 0))
if self._report_detailed_results:
for metric in ('pixels_rasterized_with_non_solid_color',
'pixels_rasterized_as_opaque', 'total_layers',
'total_picture_layers',
'total_picture_layers_with_no_content',
'total_picture_layers_off_screen'):
results.AddMeasurement(metric, 'count', data[metric])
lcd_text_pixels = data['visible_pixels_by_lcd_text_disallowed_reason']
for reason in lcd_text_pixels:
if reason == 'none':
name = 'visible_pixels_lcd_text'
else:
name = 'visible_pixels_non_lcd_text:' + reason
results.AddMeasurement(name, 'count', lcd_text_pixels[reason])
