def BenchmarkSmokeTest(self):
# Some benchmarks are running multiple iterations
# which is not needed for a smoke test
if hasattr(benchmark_class, 'enable_smoke_test_mode'):
benchmark_class.enable_smoke_test_mode = True
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
options = testing.GetRunOptions(
output_dir=temp_dir,
benchmark_cls=benchmark_class,
overrides={'story_shard_end_index': num_pages},
environment=chromium_config.GetDefaultChromiumConfig())
options.pageset_repeat = 1 # For smoke testing only run the page once.
options.output_formats = ['histograms']
options.max_values_per_test_case = MAX_VALUES_PER_TEST_CASE
results_processor.ProcessOptions(options)
return_code = benchmark_class().Run(options)
# TODO(crbug.com/1019139): Make 111 be the exit code that means
# "no stories were run.".
if return_code in (-1, 111):
self.skipTest('The benchmark was not run.')
self.assertEqual(return_code,
0,
msg='Benchmark run failed: %s' %
benchmark_class.Name())
return_code = results_processor.ProcessResults(options)
self.assertEqual(return_code,
0,
msg='Result processing failed: %s' %
benchmark_class.Name())
def main(config, args=None):
options = command_line.ParseArgs(
environment=config, args=args,
results_arg_parser=results_processor.ArgumentParser())
results_processor.ProcessOptions(options)
run_return_code = command_line.RunCommand(options)
process_return_code = results_processor.ProcessResults(options)
return max(run_return_code, process_return_code)
def ProcessResults(options):
"""Collect generated results and call results_processor to compute results."""
_MergeResultsJson(_GetTraceDir(options),
os.path.join(options.intermediate_dir, MERGED_RESULTS))
process_return_code = results_processor.ProcessResults(options)
if process_return_code != 0:
return process_return_code
expected_perf_filename = os.path.join(options.output_dir,
'histograms.json')
output_perf_results = os.path.join(options.output_dir, 'perf_results.json')
shutil.move(expected_perf_filename, output_perf_results)
return process_return_code
def main(config, args=None):
results_arg_parser = results_processor.ArgumentParser()
options = command_line.ParseArgs(environment=config,
args=args,
results_arg_parser=results_arg_parser)
results_processor.ProcessOptions(options)
run_return_code = command_line.RunCommand(options)
process_return_code = results_processor.ProcessResults(options)
if process_return_code != 0:
return process_return_code
else:
return run_return_code
def RunTest(self):
class SinglePageBenchmark(benchmark_class): # pylint: disable=no-init
def CreateStorySet(self, options):
# pylint: disable=super-on-old-class
story_set = super(SinglePageBenchmark,
self).CreateStorySet(options)
stories_to_remove = [
s for s in story_set.stories if s != story_to_smoke_test
]
for s in stories_to_remove:
story_set.RemoveStory(s)
assert story_set.stories
return story_set
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
# Set the benchmark's default arguments.
options = GenerateBenchmarkOptions(
output_dir=temp_dir, benchmark_cls=SinglePageBenchmark)
simplified_test_name = self.id().replace(
'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.',
'')
# Sanity check to ensure that that substring removal was effective.
assert len(simplified_test_name) < len(self.id())
if (simplified_test_name in _DISABLED_TESTS
and not options.run_disabled_tests):
self.skipTest('Test is explicitly disabled')
single_page_benchmark = SinglePageBenchmark()
return_code = single_page_benchmark.Run(options)
# TODO(crbug.com/1019139): Make 111 be the exit code that means
# "no stories were run.".
if return_code in (-1, 111):
self.skipTest('The benchmark was not run.')
self.assertEqual(return_code,
0,
msg='Benchmark run failed: %s' %
benchmark_class.Name())
return_code = results_processor.ProcessResults(options)
self.assertEqual(return_code,
0,
msg='Result processing failed: %s' %
benchmark_class.Name())
def RunBenchmark(self, benchmark_class):
"""Run a benchmark, process results, and return generated histograms."""
# TODO(crbug.com/985712): Ideally we should be able to just call
# telemetry.command_line.RunCommand(self.options) with the right set
# of options chosen. However, argument parsing and command running are
# currently tangled in Telemetry. In particular the class property
# Run._benchmark is not set when we skip argument parsing, and the Run.Run
# method call fails. Simplify this when argument parsing and command
# running are no longer intertwined like this.
run_return_code = benchmark_class().Run(self.options)
self.assertEqual(run_return_code, 0)
process_return_code = results_processor.ProcessResults(self.options)
self.assertEqual(process_return_code, 0)
histograms_file = os.path.join(self.options.output_dir, 'histograms.json')
self.assertTrue(os.path.exists(histograms_file))
with open(histograms_file) as f:
dicts = json.load(f)
histograms = histogram_set.HistogramSet()
histograms.ImportDicts(dicts)
return histograms
