def testUploadArtifactsToCloud(self, cloud_storage_insert_patch):
cs_path_name = 'https://cs_foo'
cloud_storage_insert_patch.return_value = cs_path_name
with tempfile_ext.NamedTemporaryDirectory(
prefix='artifact_tests') as tempdir:
ar = artifact_results.ArtifactResults(tempdir)
results = page_test_results.PageTestResults(upload_bucket='abc',
artifact_results=ar)
with results.CreateArtifact('story1', 'screenshot') as screenshot1:
pass
with results.CreateArtifact('story2', 'log') as log2:
pass
results.UploadArtifactsToCloud()
cloud_storage_insert_patch.assert_has_calls([
mock.call('abc', mock.ANY, screenshot1.name),
mock.call('abc', mock.ANY, log2.name)
],
any_order=True)
# Assert that the path is now the cloud storage path
for _, artifacts in ar.IterTestAndArtifacts():
for artifact_type in artifacts:
for i, _ in enumerate(artifacts[artifact_type]):
self.assertEquals(cs_path_name,
artifacts[artifact_type][i])
def testAddBasic(self, make_patch, move_patch):
ar = artifact_results.ArtifactResults(_abs_join('foo'))
ar.AddArtifact('test', 'artifact_name',
_abs_join('foo', 'artifacts', 'bar.log'))
move_patch.assert_not_called()
make_patch.assert_called_with(_abs_join('foo', 'artifacts'))
self.assertEqual({k: dict(v)
for k, v in ar._test_artifacts.items()},
{'test': {
'artifact_name': ['bar.log'],
}})
def testCreateDuplicateStoryName(self):
with tempfile_ext.NamedTemporaryDirectory(
prefix='artifact_tests') as tempdir:
ar = artifact_results.ArtifactResults(tempdir)
filenames = []
with ar.CreateArtifact('story_name', 'logs') as log_file:
filenames.append(log_file.name)
log_file.write('hi\n')
with ar.CreateArtifact('story_name', 'logs') as log_file:
filenames.append(log_file.name)
log_file.write('hi\n')
for filename in filenames:
with open(filename) as f:
self.assertEqual(f.read(), 'hi\n')
def testIterTestAndArtifacts(self, make_patch, move_patch):
del make_patch, move_patch # unused
ar = artifact_results.ArtifactResults(_abs_join('foo'))
ar.AddArtifact('foo', 'log', _abs_join(
'artifacts', 'foo.log'))
ar.AddArtifact('bar', 'screenshot', _abs_join(
'artifacts', 'bar.jpg'))
test_artifacts = {}
for test_name, artifacts in ar.IterTestAndArtifacts():
test_artifacts[test_name] = artifacts
self.assertEqual({
'foo': {'log': ['artifacts/foo.log']},
'bar': {'screenshot': ['artifacts/bar.jpg']}
}, test_artifacts)
def testUploadArtifactsToCloud(self, cloud_storage_insert_patch):
with tempfile_ext.NamedTemporaryDirectory(
prefix='artifact_tests') as tempdir:
ar = artifact_results.ArtifactResults(tempdir)
results = page_test_results.PageTestResults(
upload_bucket='abc', artifact_results=ar)
with results.CreateArtifact('story1', 'screenshot') as screenshot1:
pass
with results.CreateArtifact('story2', 'log') as log2:
pass
results.UploadArtifactsToCloud()
cloud_storage_insert_patch.assert_has_calls(
[mock.call('abc', mock.ANY, screenshot1.name),
mock.call('abc', mock.ANY, log2.name)],
any_order=True)
def CreateResults(benchmark_metadata,
options,
value_can_be_added_predicate=lambda v, is_first: True,
benchmark_enabled=True):
"""
Args:
options: Contains the options specified in AddResultsOptions.
"""
if not options.output_formats:
options.output_formats = [_DEFAULT_OUTPUT_FORMAT]
artifacts = artifact_results.ArtifactResults(options.output_dir)
upload_bucket = None
if options.upload_results:
upload_bucket = options.upload_bucket
if upload_bucket in cloud_storage.BUCKET_ALIASES:
upload_bucket = cloud_storage.BUCKET_ALIASES[upload_bucket]
output_formatters = []
for output_format in options.output_formats:
if output_format == 'none' or output_format == "gtest":
continue
output_stream = _GetOutputStream(output_format, options.output_dir)
if output_format == 'html':
output_formatters.append(
html_output_formatter.HtmlOutputFormatter(
output_stream, benchmark_metadata, options.reset_results,
upload_bucket))
elif output_format == 'json-test-results':
output_formatters.append(
json_3_output_formatter.JsonOutputFormatter(
output_stream, artifacts))
elif output_format == 'chartjson':
output_formatters.append(
chart_json_output_formatter.ChartJsonOutputFormatter(
output_stream, benchmark_metadata))
elif output_format == 'csv':
output_formatters.append(
csv_output_formatter.CsvOutputFormatter(
output_stream, options.reset_results))
elif output_format == 'histograms':
output_formatters.append(
histogram_set_json_output_formatter.
HistogramSetJsonOutputFormatter(output_stream,
benchmark_metadata,
options.reset_results))
else:
# Should never be reached. The parser enforces the choices.
raise Exception(
'Invalid --output-format "%s". Valid choices are: %s' %
(output_format, ', '.join(_OUTPUT_FORMAT_CHOICES)))
# TODO(chrishenry): This is here to not change the output of
# gtest. Let's try enabling skipped tests summary for gtest test
# results too (in a separate patch), and see if we break anything.
output_skipped_tests_summary = 'gtest' in options.output_formats
reporter = _GetProgressReporter(output_skipped_tests_summary,
options.suppress_gtest_report)
results = page_test_results.PageTestResults(
output_formatters=output_formatters,
progress_reporter=reporter,
output_dir=options.output_dir,
value_can_be_added_predicate=value_can_be_added_predicate,
benchmark_enabled=benchmark_enabled,
upload_bucket=upload_bucket,
artifact_results=artifacts)
results.telemetry_info.benchmark_name = benchmark_metadata.name
results.telemetry_info.benchmark_start_epoch = time.time()
if options.results_label:
results.telemetry_info.label = options.results_label
return results
def CreateResults(benchmark_metadata,
options,
should_add_value=None,
benchmark_enabled=True):
"""
Args:
options: Contains the options specified in AddResultsOptions.
"""
if not options.output_formats:
options.output_formats = [_DEFAULT_OUTPUT_FORMAT]
artifacts = artifact_results.NoopArtifactResults(options.output_dir)
upload_bucket = None
if options.upload_results:
upload_bucket = options.upload_bucket
if upload_bucket in cloud_storage.BUCKET_ALIASES:
upload_bucket = cloud_storage.BUCKET_ALIASES[upload_bucket]
output_formatters = []
for output_format in options.output_formats:
if output_format == 'none' or output_format == "gtest":
continue
# pylint: disable=redefined-variable-type
if isinstance(artifacts, artifact_results.NoopArtifactResults):
artifacts = artifact_results.ArtifactResults(options.output_dir)
output_stream = _GetOutputStream(output_format, options.output_dir)
if output_format == 'html':
output_formatters.append(
html_output_formatter.HtmlOutputFormatter(
output_stream, benchmark_metadata, options.reset_results,
upload_bucket))
elif output_format == 'json-test-results':
# Only create artifact results if we're going to actually output them
# through an output format.
artifacts = artifact_results.ArtifactResults(options.output_dir)
output_formatters.append(
json_3_output_formatter.JsonOutputFormatter(
output_stream, artifacts))
elif output_format == 'chartjson':
output_formatters.append(
chart_json_output_formatter.ChartJsonOutputFormatter(
output_stream, benchmark_metadata))
elif output_format == 'csv':
output_formatters.append(
csv_output_formatter.CsvOutputFormatter(
output_stream, options.reset_results))
elif output_format == 'histograms':
output_formatters.append(
histogram_set_json_output_formatter.
HistogramSetJsonOutputFormatter(output_stream,
benchmark_metadata,
options.reset_results))
else:
# Should never be reached. The parser enforces the choices.
raise Exception(
'Invalid --output-format "%s". Valid choices are: %s' %
(output_format, ', '.join(_OUTPUT_FORMAT_CHOICES)))
reporter = _GetProgressReporter(options.suppress_gtest_report)
results = page_test_results.PageTestResults(
output_formatters=output_formatters,
progress_reporter=reporter,
output_dir=options.output_dir,
should_add_value=should_add_value,
benchmark_enabled=benchmark_enabled,
upload_bucket=upload_bucket,
artifact_results=artifacts,
benchmark_metadata=benchmark_metadata)
results.telemetry_info.benchmark_name = benchmark_metadata.name
results.telemetry_info.benchmark_descriptions = benchmark_metadata.description
results.telemetry_info.benchmark_start_us = time.time() * 1e6
if options.results_label:
results.telemetry_info.label = options.results_label
return results
