def testNoLeakedTempFileWhenUploadingTrace(self):
v = trace.TraceValue(
None, trace_data.CreateTraceDataFromRawData([{
'test': 1
}]))
v.CleanUp()
self.assertTrue(_IsEmptyDir(self.temp_test_dir))
def testMultiplePagesAndValues(self, cs_insert_mock):
cs_insert_mock.return_value = 'https://cloud_storage_url/foo'
trace_value = trace.TraceValue(None, trace_data.TraceData('{"events": 0}'))
trace_value.UploadToCloud(bucket='foo')
self.SimulateBenchmarkRun({
self._story_set[0]: [
scalar.ScalarValue(
None, 'foo', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)],
self._story_set[1]: [
scalar.ScalarValue(
None, 'foo', 'seconds', 3.4,
improvement_direction=improvement_direction.DOWN),
trace_value,
scalar.ScalarValue(
None, 'bar', 'km', 10,
improvement_direction=improvement_direction.DOWN),
scalar.ScalarValue(
None, 'baz', 'count', 5,
improvement_direction=improvement_direction.DOWN)]})
# Parse CSV output into list of lists.
csv_string = self.Format()
lines = csv_string.split(self._LINE_SEPARATOR)
values = [s.split(',') for s in lines[1:-1]]
self.assertEquals(len(values), 5) # We expect 5 value in total.
self.assertEquals(len(set((v[1] for v in values))), 2) # 2 pages.
self.assertEquals(len(set((v[2] for v in values))), 4) # 4 value names.
self.assertEquals(values[2],
['story_set', 'http://www.bar.com/', 'trace',
'https://cloud_storage_url/foo', '', '1'])
def Measure(self, platform, results):
"""Collect all possible metrics and added them to results."""
platform.tracing_controller.SetTelemetryInfo(results.telemetry_info)
trace_result = platform.tracing_controller.StopTracing()
trace_value = trace.TraceValue(
results.current_page,
trace_result,
file_path=results.telemetry_info.trace_local_path,
remote_path=results.telemetry_info.trace_remote_path,
upload_bucket=results.telemetry_info.upload_bucket,
cloud_url=results.telemetry_info.trace_remote_url,
trace_url=results.telemetry_info.trace_url)
results.AddValue(trace_value)
if self._tbm_options.GetTimelineBasedMetrics():
assert not self._tbm_options.GetLegacyTimelineBasedMetrics(), (
'Specifying both TBMv1 and TBMv2 metrics is not allowed.')
# The metrics computation happens later asynchronously in
# results.ComputeTimelineBasedMetrics().
trace_value.SetTimelineBasedMetrics(
self._tbm_options.GetTimelineBasedMetrics())
else:
# Run all TBMv1 metrics if no other metric is specified
# (legacy behavior)
if not self._tbm_options.GetLegacyTimelineBasedMetrics():
raise Exception(
'Please specify the TBMv1 metrics you are interested in '
'explicitly.')
trace_value.SerializeTraceData()
self._ComputeLegacyTimelineBasedMetrics(results,
trace_result.AsData())
def AddTraces(self, traces, tbm_metrics=None):
"""Associate some recorded traces with the current story run.
Args:
traces: A TraceDataBuilder object with traces recorded from all
tracing agents.
tbm_metrics: Optional list of TBMv2 metrics to be computed from the
input traces.
"""
assert self._current_page_run, 'Not currently running test.'
trace_value = trace.TraceValue(
self.current_page,
traces,
file_path=self.telemetry_info.trace_local_path,
remote_path=self.telemetry_info.trace_remote_path,
upload_bucket=self.telemetry_info.upload_bucket,
cloud_url=self.telemetry_info.trace_remote_url,
trace_url=self.telemetry_info.trace_url)
self.AddValue(trace_value)
if tbm_metrics:
# Both trace serialization and metric computation will happen later
# asynchronously during ComputeTimelineBasedMetrics.
self._current_page_run.SetTbmMetrics(tbm_metrics)
else:
# Otherwise we immediately serialize the trace data.
trace_value.SerializeTraceData()
def testRepr(self):
with trace.TraceValue(page.Page('http://www.bar.com/',
name='load:story:bar'),
trace_data.CreateTestTrace(),
important=True,
description='desc') as v:
self.assertEquals("TraceValue('load:story:bar', 'trace')", str(v))
def ValidateAndMeasurePage(self, _, tab, results):
self._results = results
trace_result = tab.browser.platform.tracing_controller.StopTracing()
# TODO(charliea): This is part of a three-sided Chromium/Telemetry patch
# where we're changing the return type of StopTracing from a TraceValue to a
# (TraceValue, nonfatal_exception_list) tuple. Once the tuple return value
# lands in Chromium, the non-tuple logic should be deleted.
if isinstance(trace_result, tuple):
trace_result = trace_result[0]
trace_value = trace.TraceValue(
results.current_page,
trace_result,
file_path=results.telemetry_info.trace_local_path,
remote_path=results.telemetry_info.trace_remote_path,
upload_bucket=results.telemetry_info.upload_bucket,
cloud_url=results.telemetry_info.trace_remote_url)
results.AddValue(trace_value)
model = model_module.TimelineModel(trace_result)
renderer_thread = model.GetRendererThreadFromTabId(tab.id)
records = _CollectRecordsFromRendererThreads(model, renderer_thread)
metric = smoothness.SmoothnessMetric()
metric.AddResults(model, renderer_thread, records, results)
def testNoLeakedTempFileOnImplicitCleanUp(self):
with trace.TraceValue(
None, trace_data.CreateTraceDataFromRawData([{
'test': 1
}])):
pass
self.assertTrue(_IsEmptyDir(self.temp_test_dir))
def testTraceValue(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(
trace.TraceValue(None, trace_data.TraceData({'test': 1})))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(
trace.TraceValue(None, trace_data.TraceData({'test': 2})))
results.DidRunPage(self.pages[1])
results.PrintSummary()
values = results.FindAllTraceValues()
self.assertEquals(2, len(values))
def testCleanUpCleansUpTraceValues(self):
results = page_test_results.PageTestResults()
v0 = trace.TraceValue(None, trace_data.TraceData({'test': 1}))
v1 = trace.TraceValue(None, trace_data.TraceData({'test': 2}))
results.WillRunPage(self.pages[0])
results.AddValue(v0)
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(v1)
results.DidRunPage(self.pages[1])
results.CleanUp()
self.assertTrue(v0.cleaned_up)
self.assertTrue(v1.cleaned_up)
def testFindTraceParts(self):
raw_data = {
'powerTraceAsString': 'BattOr Data',
'traceEvents': 'Chrome Data',
'tabIds': 'Tab Data',
}
data = trace_data.TraceData(raw_data)
v = trace.TraceValue(None, data)
tempdir = tempfile.mkdtemp()
temp_path = os.path.join(tempdir, 'test.json')
battor_seen = False
chrome_seen = False
tabs_seen = False
try:
with codecs.open(v.filename, mode='r', encoding='utf-8') as f:
trace_files = html2trace.CopyTraceDataFromHTMLFilePath(
f, temp_path)
for f in trace_files:
with open(f, 'r') as trace_file:
d = trace_file.read()
if d == raw_data['powerTraceAsString']:
self.assertFalse(battor_seen)
battor_seen = True
elif d == raw_data['traceEvents']:
self.assertFalse(chrome_seen)
chrome_seen = True
elif d == raw_data['tabIds']:
self.assertFalse(tabs_seen)
tabs_seen = True
self.assertTrue(battor_seen)
self.assertTrue(chrome_seen)
self.assertTrue(tabs_seen)
finally:
shutil.rmtree(tempdir)
os.remove(v.filename)
def testRepr(self):
v = trace.TraceValue(self.pages[0],
trace_data.TraceData({'test': 1}),
important=True,
description='desc')
self.assertEquals('TraceValue(http://www.bar.com/, trace)', str(v))
def testFindTraceParts(self):
raw_data = {
'traceEvents': [{'trace': 1}],
'telemetry': 'Telemetry Data',
}
data = trace_data.CreateTraceDataFromRawData(raw_data)
v = trace.TraceValue(None, data)
v.SerializeTraceData()
tempdir = tempfile.mkdtemp()
temp_path = os.path.join(tempdir, 'test.json')
chrome_seen = False
telemetry_seen = False
try:
with codecs.open(v.filename, mode='r', encoding='utf-8') as f:
trace_files = html2trace.CopyTraceDataFromHTMLFilePath(f, temp_path)
for f in trace_files:
with open(f, 'r') as trace_file:
d = trace_file.read()
if d == json.dumps({'traceEvents': raw_data['traceEvents']}):
self.assertFalse(chrome_seen)
chrome_seen = True
elif d == raw_data['telemetry']:
self.assertFalse(telemetry_seen)
telemetry_seen = True
self.assertTrue(chrome_seen)
self.assertTrue(telemetry_seen)
finally:
shutil.rmtree(tempdir)
os.remove(v.filename)
def testAsDictWithTraceValue(self):
tempdir = tempfile.mkdtemp()
try:
results = page_test_results.PageTestResults()
results.WillRunPage(self._story_set[0])
v0 = trace.TraceValue(results.current_page,
trace_data.CreateTraceDataFromRawData([{
'event':
'test'
}]),
file_path=os.path.join(
tempdir,
'testAsDictWithTraceValue.html'))
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results._SerializeTracesToDirPath()
d = json_output_formatter.ResultsAsDict(
results, self._formatter.benchmark_metadata)
self.assertTrue(_HasPage(d['pages'], self._story_set[0]))
self.assertTrue(_HasValueNamed(d['per_page_values'], 'trace'))
self.assertEquals(len(d['files']), 1)
output_trace_path = d['files'].values()[0]
self.assertTrue(output_trace_path.startswith(tempdir))
self.assertTrue(os.path.exists(output_trace_path))
finally:
shutil.rmtree(tempdir)
def Measure(self, platform, results):
"""Collect all possible metrics and added them to results."""
platform.tracing_controller.telemetry_info = results.telemetry_info
# TODO(charliea): Add all nonfatal errors to results as FailureValues.
trace_result, _ = platform.tracing_controller.StopTracing()
trace_value = trace.TraceValue(
results.current_page,
trace_result,
file_path=results.telemetry_info.trace_local_path,
remote_path=results.telemetry_info.trace_remote_path,
upload_bucket=results.telemetry_info.upload_bucket,
cloud_url=results.telemetry_info.trace_remote_url)
results.AddValue(trace_value)
try:
if self._tbm_options.GetTimelineBasedMetrics():
assert not self._tbm_options.GetLegacyTimelineBasedMetrics(), (
'Specifying both TBMv1 and TBMv2 metrics is not allowed.')
self._ComputeTimelineBasedMetrics(results, trace_value)
else:
# Run all TBMv1 metrics if no other metric is specified
# (legacy behavior)
if not self._tbm_options.GetLegacyTimelineBasedMetrics():
raise Exception(
'Please specify the TBMv1 metrics you are interested in '
'explicitly.')
self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
finally:
trace_result.CleanUpAllTraces()
def Measure(self, platform, results):
"""Collect all possible metrics and added them to results."""
platform.tracing_controller.telemetry_info = results.telemetry_info
trace_result = platform.tracing_controller.StopTracing()
trace_value = trace.TraceValue(results.current_page, trace_result)
results.AddValue(trace_value)
try:
if self._tbm_options.GetTimelineBasedMetrics():
assert not self._tbm_options.GetLegacyTimelineBasedMetrics(), (
'Specifying both TBMv1 and TBMv2 metrics is not allowed.')
self._ComputeTimelineBasedMetrics(results, trace_value)
else:
# Run all TBMv1 metrics if no other metric is specified
# (legacy behavior)
if not self._tbm_options.GetLegacyTimelineBasedMetrics():
logging.warn(
'Please specify the TBMv1 metrics you are interested in '
'explicitly. This implicit functionality will be removed'
' on July 17, 2016.')
self._tbm_options.SetLegacyTimelineBasedMetrics(
_GetAllLegacyTimelineBasedMetrics())
self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
finally:
trace_result.CleanUpAllTraces()
def testMultiplePagesAndValues(self, cs_insert_mock):
cs_insert_mock.return_value = 'https://cloud_storage_url/foo'
trace_value = trace.TraceValue(
None, trace_data.CreateTraceDataFromRawData('{"traceEvents": []}'))
trace_value.UploadToCloud(bucket='foo')
self.SimulateBenchmarkRun([
(self._story_set[0], [
scalar.ScalarValue(
None, 'foo', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)]),
(self._story_set[1], [
scalar.ScalarValue(
None, 'foo', 'seconds', 3.4,
improvement_direction=improvement_direction.DOWN),
trace_value,
scalar.ScalarValue(
None, 'bar', 'km', 10,
improvement_direction=improvement_direction.DOWN),
scalar.ScalarValue(
None, 'baz', 'count', 5,
improvement_direction=improvement_direction.DOWN)])])
# Parse CSV output into list of lists.
csv_string = self.Format()
lines = csv_string.split('\r\n')
values = [s.split(',') for s in lines[1:-1]]
values.sort()
self.assertEquals(len(values), 4)
self.assertEquals(len(set((v[1] for v in values))), 2) # 2 pages.
self.assertEquals(len(set((v[2] for v in values))), 4) # 4 value names.
self.assertEquals(values[2], [
'foo', 'ms', '3400', '1', '3400', '3400', '0', '3400', '', 'benchmark',
'2017-07-14 02:40:00', '', '', 'benchmark 2017-07-14 02:40:00', '', '',
'', '', '', 'http://www.bar.com/', '', ''])
def Stop(self, tab, results):
# End the smooth marker for all actions.
self._interaction.End()
# Stop tracing.
timeline_data = tab.browser.platform.tracing_controller.Stop()
results.AddValue(trace.TraceValue(
results.current_page, timeline_data))
self._model = TimelineModel(timeline_data)
self._renderer_process = self._model.GetRendererProcessFromTabId(tab.id)
renderer_thread = self.model.GetRendererThreadFromTabId(tab.id)
run_smooth_actions_record = None
self._smooth_records = []
for event in renderer_thread.async_slices:
if not tir_module.IsTimelineInteractionRecord(event.name):
continue
r = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
if r.label == RUN_SMOOTH_ACTIONS:
assert run_smooth_actions_record is None, (
'TimelineController cannot issue more than 1 %s record' %
RUN_SMOOTH_ACTIONS)
run_smooth_actions_record = r
elif r.is_smooth:
self._smooth_records.append(
smooth_gesture_util.GetAdjustedInteractionIfContainGesture(
self.model, r))
# If there is no other smooth records, we make measurements on time range
# marked by timeline_controller itself.
# TODO(nednguyen): when crbug.com/239179 is marked fixed, makes sure that
# page sets are responsible for issueing the markers themselves.
if len(self._smooth_records) == 0 and run_smooth_actions_record:
self._smooth_records = [run_smooth_actions_record]
def testNoLeakedTempFiles(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
with mock.patch('tempfile.tempdir', new=tempdir):
with trace.TraceValue(None, trace_data.CreateTestTrace()) as v:
v.SerializeTraceData()
self.assertTrue(os.path.exists(tempdir))
self.assertFalse(os.listdir(tempdir))
def testNoTracesLeftAfterCleanUp(self):
results = page_test_results.PageTestResults()
v0 = trace.TraceValue(None,
trace_data.CreateTraceDataFromRawData([{'test': 1}]))
v1 = trace.TraceValue(None,
trace_data.CreateTraceDataFromRawData([{'test': 2}]))
results.WillRunPage(self.pages[0])
results.AddValue(v0)
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(v1)
results.DidRunPage(self.pages[1])
results.CleanUp()
self.assertFalse(results.FindAllTraceValues())
def testNoTracesLeftAfterCleanUp(self):
results = self.getPageTestResults()
try:
v0 = trace.TraceValue(None, trace_data.CreateTestTrace(1))
v1 = trace.TraceValue(None, trace_data.CreateTestTrace(2))
results.WillRunPage(self.pages[0])
results.AddValue(v0)
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(v1)
results.DidRunPage(self.pages[1])
finally:
results.CleanUp()
self.assertFalse(results.FindAllTraceValues())
def testTraceValue(self):
results = self.getPageTestResults()
try:
results.WillRunPage(self.pages[0])
results.AddValue(trace.TraceValue(None, trace_data.CreateTestTrace(1)))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(trace.TraceValue(None, trace_data.CreateTestTrace(2)))
results.DidRunPage(self.pages[1])
results.PrintSummary()
values = results.FindAllTraceValues()
self.assertEquals(2, len(values))
finally:
results.CleanUp()
def testNoLeakedTempFileWhenUploadingTrace(self):
v = trace.TraceValue(
None, tracing_timeline_data.TracingTimelineData({'test': 1}))
trace.cloud_storage.SetCalculatedHashesForTesting(
TestDefaultDict(123))
bucket = trace.cloud_storage.PUBLIC_BUCKET
v.UploadToCloud(bucket)
self.assertTrue(_IsEmptyDir(self.temp_test_dir))
def testCleanUpCleansUpTraceValues(self):
results = self.getPageTestResults()
try:
v0 = trace.TraceValue(None, trace_data.CreateTestTrace(1))
v1 = trace.TraceValue(None, trace_data.CreateTestTrace(2))
results.WillRunPage(self.pages[0])
results.AddValue(v0)
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(v1)
results.DidRunPage(self.pages[1])
finally:
results.CleanUp()
self.assertTrue(v0.cleaned_up)
self.assertTrue(v1.cleaned_up)
def ValidateAndMeasurePage(self, page, tab, results):
""" Collect all possible metrics and added them to results. """
trace_result = tab.browser.platform.tracing_controller.Stop()
results.AddValue(trace.TraceValue(results.current_page, trace_result))
model = model_module.TimelineModel(trace_result)
renderer_thread = model.GetRendererThreadFromTabId(tab.id)
meta_metrics = _TimelineBasedMetrics(model, renderer_thread,
_GetMetricFromMetricType)
meta_metrics.AddResults(results)
def testTraceSerializationContainStoryName(self):
tempdir = tempfile.mkdtemp()
try:
v = trace.TraceValue(self.pages[0], trace_data.TraceData({'test': 1}))
fh = v.Serialize(tempdir)
self.assertTrue(os.path.basename(fh.GetAbsPath()).startswith(
'http___www_bar_com'))
finally:
shutil.rmtree(tempdir)
def testNoLeakedTempFileWhenTraceSerialize(self):
tempdir = tempfile.mkdtemp()
v = trace.TraceValue(None, trace_data.TraceData({'test': 1}))
fh = v.Serialize(tempdir)
try:
shutil.rmtree(fh.GetAbsPath(), ignore_errors=True)
self.assertTrue(tempdir)
finally:
shutil.rmtree(tempdir)
self.assertTrue(_IsEmptyDir(self.temp_test_dir))
def Stop(self, tab, results):
# End the smooth marker for all actions.
if self._enable_auto_issuing_record:
self._interaction.End()
# Stop tracing.
timeline_data = tab.browser.platform.tracing_controller.StopTracing()
# TODO(charliea): This is part of a three-sided Chromium/Telemetry patch
# where we're changing the return type of StopTracing from a TraceValue to a
# (TraceValue, nonfatal_exception_list) tuple. Once the tuple return value
# lands in Chromium, the non-tuple logic should be deleted.
if isinstance(timeline_data, tuple):
timeline_data = timeline_data[0]
# TODO(#763375): Rely on results.telemetry_info.trace_local_path/etc.
kwargs = {}
if hasattr(results.telemetry_info, 'trace_local_path'):
kwargs['file_path'] = results.telemetry_info.trace_local_path
kwargs['remote_path'] = results.telemetry_info.trace_remote_path
kwargs['upload_bucket'] = results.telemetry_info.upload_bucket
kwargs['cloud_url'] = results.telemetry_info.trace_remote_url
results.AddValue(
trace.TraceValue(results.current_page, timeline_data, **kwargs))
self._model = TimelineModel(timeline_data)
self._renderer_process = self._model.GetRendererProcessFromTabId(
tab.id)
renderer_thread = self.model.GetRendererThreadFromTabId(tab.id)
run_smooth_actions_record = None
self._smooth_records = []
for event in renderer_thread.async_slices:
if not tir_module.IsTimelineInteractionRecord(event.name):
continue
r = tir_module.TimelineInteractionRecord.FromAsyncEvent(event)
if r.label == RUN_SMOOTH_ACTIONS:
assert run_smooth_actions_record is None, (
'TimelineController cannot issue more than 1 %s record' %
RUN_SMOOTH_ACTIONS)
run_smooth_actions_record = r
else:
self._smooth_records.append(
smooth_gesture_util.GetAdjustedInteractionIfContainGesture(
self.model, r))
# If there is no other smooth records, we make measurements on time range
# marked by timeline_controller itself.
# TODO(nednguyen): when crbug.com/239179 is marked fixed, makes sure that
# page sets are responsible for issueing the markers themselves.
if len(self._smooth_records) == 0 and run_smooth_actions_record:
self._smooth_records = [run_smooth_actions_record]
if len(self._smooth_records) == 0:
raise legacy_page_test.Failure(
'No interaction record was created.')
def Measure(self, platform, results):
"""Collect all possible metrics and added them to results."""
trace_result = platform.tracing_controller.StopTracing()
trace_value = trace.TraceValue(results.current_page, trace_result)
results.AddValue(trace_value)
if self._tbm_options.GetTimelineBasedMetrics():
self._ComputeTimelineBasedMetrics(results, trace_value)
else:
assert self._tbm_options.GetLegacyTimelineBasedMetrics()
self._ComputeLegacyTimelineBasedMetrics(results, trace_result)
def DidRunStory(self, platform, results):
"""Clean up after running the story."""
if platform.tracing_controller.is_tracing_running:
trace_result = platform.tracing_controller.StopTracing()
trace_value = trace.TraceValue(
results.current_page, trace_result,
file_path=results.telemetry_info.trace_local_path,
remote_path=results.telemetry_info.trace_remote_path,
upload_bucket=results.telemetry_info.upload_bucket,
cloud_url=results.telemetry_info.trace_remote_url)
results.AddValue(trace_value)
def Measure(self, tracing_controller, results):
"""Collect all possible metrics and added them to results."""
trace_result = tracing_controller.Stop()
results.AddValue(trace.TraceValue(results.current_page, trace_result))
model = model_module.TimelineModel(trace_result)
threads_to_records_map = _GetRendererThreadsToInteractionRecordsMap(model)
for renderer_thread, interaction_records in (
threads_to_records_map.iteritems()):
meta_metrics = _TimelineBasedMetrics(
model, renderer_thread, interaction_records, _GetMetricFromMetricType)
meta_metrics.AddResults(results)
