def testUploadArtifactsToCloud(self, cs_insert_mock):
cs_path_name = 'https://cs_foo'
cs_insert_mock.return_value = cs_path_name
with self.CreateResults(upload_bucket='abc') as results:
results.WillRunPage(self.pages[0])
with results.CreateArtifact('screenshot.png') as screenshot1:
pass
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
with results.CreateArtifact('log.txt') as log2:
pass
results.DidRunPage(self.pages[1])
results_processor.UploadArtifactsToCloud(results)
cs_insert_mock.assert_has_calls([
mock.call('abc', mock.ANY, screenshot1.name),
mock.call('abc', mock.ANY, log2.name)
],
any_order=True)
# Assert that the path is now the cloud storage path
for run in results.IterStoryRuns():
for artifact in run.IterArtifacts():
self.assertEqual(cs_path_name, artifact.url)
def testUploadArtifactsToCloud(self, cloud_storage_insert_patch):
cs_path_name = 'https://cs_foo'
cloud_storage_insert_patch.return_value = cs_path_name
with tempfile_ext.NamedTemporaryDirectory(
prefix='artifact_tests') as tempdir:
results = page_test_results.PageTestResults(upload_bucket='abc',
output_dir=tempdir)
results.WillRunPage(self.pages[0])
with results.CreateArtifact('screenshot.png') as screenshot1:
pass
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
with results.CreateArtifact('log.txt') as log2:
pass
results.DidRunPage(self.pages[1])
results_processor.UploadArtifactsToCloud(results)
cloud_storage_insert_patch.assert_has_calls([
mock.call('abc', mock.ANY, screenshot1.name),
mock.call('abc', mock.ANY, log2.name)
],
any_order=True)
# Assert that the path is now the cloud storage path
for run in results.IterStoryRuns():
for artifact in run.IterArtifacts():
self.assertEquals(cs_path_name, artifact.url)
def testUploadArtifactsToCloud_withNoOpArtifact(self, _):
with self.CreateResults(upload_bucket='abc',
output_dir=None) as results:
results.WillRunPage(self.pages[0])
with results.CreateArtifact('screenshot.png'):
pass
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
with results.CreateArtifact('log.txt'):
pass
results.DidRunPage(self.pages[1])
# Just make sure that this does not crash
results_processor.UploadArtifactsToCloud(results)
def testUploadArtifactsToCloud_withNoOpArtifact(
self, cloud_storage_insert_patch):
del cloud_storage_insert_patch # unused
results = page_test_results.PageTestResults(upload_bucket='abc',
output_dir=None)
results.WillRunPage(self.pages[0])
with results.CreateArtifact('screenshot.png'):
pass
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
with results.CreateArtifact('log.txt'):
pass
results.DidRunPage(self.pages[1])
# Just make sure that this does not crash
results_processor.UploadArtifactsToCloud(results)
def RunBenchmark(benchmark, finder_options):
"""Run this test with the given options.
Returns:
-1 if the benchmark was skipped,
0 for success
1 if there was a failure
2 if there was an uncaught exception.
"""
benchmark.CustomizeOptions(finder_options)
with results_options.CreateResults(
finder_options,
benchmark_name=benchmark.Name(),
benchmark_description=benchmark.Description(),
report_progress=not finder_options.suppress_gtest_report
) as results:
possible_browser = browser_finder.FindBrowser(finder_options)
if not possible_browser:
print('No browser of type "%s" found for running benchmark "%s".' %
(finder_options.browser_options.browser_type,
benchmark.Name()))
return -1
typ_expectation_tags = possible_browser.GetTypExpectationsTags()
logging.info(
'The following expectations condition tags were generated %s',
str(typ_expectation_tags))
try:
benchmark.expectations.SetTags(
typ_expectation_tags,
not finder_options.skip_typ_expectations_tags_validation)
except ValueError as e: # pylint: disable=broad-except
traceback.print_exc(file=sys.stdout)
logging.error(
str(e) +
'\nYou can use the --skip-typ-expectations-tags-validation '
'argument to suppress this exception.')
return -1
if not _ShouldRunBenchmark(benchmark, possible_browser,
finder_options):
return -1
test = benchmark.CreatePageTest(finder_options)
test.__name__ = benchmark.__class__.__name__
story_set = benchmark.CreateStorySet(finder_options)
if isinstance(test, legacy_page_test.LegacyPageTest):
if any(not isinstance(p, page.Page) for p in story_set.stories):
raise Exception(
'PageTest must be used with StorySet containing only '
'telemetry.page.Page stories.')
try:
RunStorySet(test,
story_set,
finder_options,
results,
benchmark.max_failures,
expectations=benchmark.expectations,
max_num_values=benchmark.MAX_NUM_VALUES)
if results.benchmark_interrupted:
return_code = 2
elif results.had_failures:
return_code = 1
elif results.had_successes:
return_code = 0
else:
return_code = -1 # All stories were skipped.
except Exception as exc: # pylint: disable=broad-except
interruption = 'Benchmark execution interrupted: %r' % exc
results.InterruptBenchmark(interruption)
exception_formatter.PrintFormattedException()
return_code = 2
# TODO(crbug.com/981349): merge two calls to AddSharedDiagnostics
# (see RunStorySet() method for the second one).
results.AddSharedDiagnostics(
owners=benchmark.GetOwners(),
bug_components=benchmark.GetBugComponents(),
documentation_urls=benchmark.GetDocumentationLinks(),
)
if finder_options.upload_results:
results_processor.UploadArtifactsToCloud(results)
return return_code
def RunBenchmark(benchmark, finder_options):
"""Run this test with the given options.
Returns:
-1 if the benchmark was skipped,
0 for success
1 if there was a failure
2 if there was an uncaught exception.
"""
benchmark.CustomizeOptions(finder_options)
with results_options.CreateResults(
finder_options,
benchmark_name=benchmark.Name(),
benchmark_description=benchmark.Description(),
report_progress=not finder_options.suppress_gtest_report,
should_add_value=benchmark.ShouldAddValue) as results:
possible_browser = browser_finder.FindBrowser(finder_options)
if not possible_browser:
print ('No browser of type "%s" found for running benchmark "%s".' % (
finder_options.browser_options.browser_type, benchmark.Name()))
return -1
benchmark.expectations.SetTags(
possible_browser.GetTypExpectationsTags())
if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options):
return -1
pt = benchmark.CreatePageTest(finder_options)
pt.__name__ = benchmark.__class__.__name__
story_set = benchmark.CreateStorySet(finder_options)
if isinstance(pt, legacy_page_test.LegacyPageTest):
if any(not isinstance(p, page.Page) for p in story_set.stories):
raise Exception(
'PageTest must be used with StorySet containing only '
'telemetry.page.Page stories.')
try:
Run(pt, story_set, finder_options, results, benchmark.max_failures,
expectations=benchmark.expectations,
max_num_values=benchmark.MAX_NUM_VALUES)
if results.benchmark_interrupted:
return_code = 2
elif results.had_failures:
return_code = 1
elif results.had_successes:
return_code = 0
else:
return_code = -1 # All stories were skipped.
# We want to make sure that all expectations are linked to real stories,
# this will log error messages if names do not match what is in the set.
benchmark.GetBrokenExpectations(story_set)
except Exception as exc: # pylint: disable=broad-except
interruption = 'Benchmark execution interrupted: %r' % exc
results.InterruptBenchmark(interruption)
exception_formatter.PrintFormattedException()
return_code = 2
benchmark_owners = benchmark.GetOwners()
benchmark_component = benchmark.GetBugComponents()
benchmark_documentation_url = benchmark.GetDocumentationLink()
if benchmark_owners:
results.AddSharedDiagnosticToAllHistograms(
reserved_infos.OWNERS.name, benchmark_owners)
if benchmark_component:
results.AddSharedDiagnosticToAllHistograms(
reserved_infos.BUG_COMPONENTS.name, benchmark_component)
if benchmark_documentation_url:
results.AddSharedDiagnosticToAllHistograms(
reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url)
if finder_options.upload_results:
results_processor.UploadArtifactsToCloud(results)
return return_code
def RunBenchmark(benchmark, finder_options):
"""Run this test with the given options.
Returns:
-1 if the benchmark was skipped,
0 for success
1 if there was a failure
2 if there was an uncaught exception.
"""
benchmark.CustomizeOptions(finder_options)
possible_browser = browser_finder.FindBrowser(finder_options)
if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options):
return -1
pt = benchmark.CreatePageTest(finder_options)
pt.__name__ = benchmark.__class__.__name__
story_set = benchmark.CreateStorySet(finder_options)
if isinstance(pt, legacy_page_test.LegacyPageTest):
if any(not isinstance(p, page.Page) for p in story_set.stories):
raise Exception(
'PageTest must be used with StorySet containing only '
'telemetry.page.Page stories.')
with results_options.CreateResults(
finder_options,
benchmark_name=benchmark.Name(),
benchmark_description=benchmark.Description(),
benchmark_enabled=True,
should_add_value=benchmark.ShouldAddValue) as results:
try:
Run(pt, story_set, finder_options, results, benchmark.max_failures,
expectations=benchmark.expectations,
max_num_values=benchmark.MAX_NUM_VALUES)
if results.had_failures:
return_code = 1
elif results.had_successes:
return_code = 0
else:
return_code = -1 # All stories were skipped.
# We want to make sure that all expectations are linked to real stories,
# this will log error messages if names do not match what is in the set.
benchmark.GetBrokenExpectations(story_set)
except Exception as e: # pylint: disable=broad-except
logging.fatal(
'Benchmark execution interrupted by a fatal exception: %s(%s)' %
(type(e), e))
filtered_stories = story_module.StoryFilter.FilterStories(
story_set.stories)
# TODO(crbug.com/980781): This appears to mark expected skipped stories
# as unexpectedly skipped stories.
results.InterruptBenchmark(
filtered_stories, finder_options.pageset_repeat)
exception_formatter.PrintFormattedException()
return_code = 2
benchmark_owners = benchmark.GetOwners()
benchmark_component = benchmark.GetBugComponents()
benchmark_documentation_url = benchmark.GetDocumentationLink()
if benchmark_owners:
results.AddSharedDiagnosticToAllHistograms(
reserved_infos.OWNERS.name, benchmark_owners)
if benchmark_component:
results.AddSharedDiagnosticToAllHistograms(
reserved_infos.BUG_COMPONENTS.name, benchmark_component)
if benchmark_documentation_url:
results.AddSharedDiagnosticToAllHistograms(
reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url)
try:
if finder_options.upload_results:
results_processor.UploadArtifactsToCloud(results)
finally:
memory_debug.LogHostMemoryUsage()
results.PrintSummary()
return return_code
