def BenchmarkSmokeTest(self): # Some benchmarks are running multiple iterations # which is not needed for a smoke test if hasattr(benchmark_class, 'enable_smoke_test_mode'): benchmark_class.enable_smoke_test_mode = True with tempfile_ext.NamedTemporaryDirectory() as temp_dir: options = testing.GetRunOptions( output_dir=temp_dir, benchmark_cls=benchmark_class, overrides={'story_shard_end_index': num_pages}, environment=chromium_config.GetDefaultChromiumConfig()) options.pageset_repeat = 1 # For smoke testing only run the page once. options.output_formats = ['histograms'] options.max_values_per_test_case = MAX_VALUES_PER_TEST_CASE results_processor.ProcessOptions(options) return_code = benchmark_class().Run(options) # TODO(crbug.com/1019139): Make 111 be the exit code that means # "no stories were run.". if return_code in (-1, 111): self.skipTest('The benchmark was not run.') self.assertEqual(return_code, 0, msg='Benchmark run failed: %s' % benchmark_class.Name()) return_code = results_processor.ProcessResults(options) self.assertEqual(return_code, 0, msg='Result processing failed: %s' % benchmark_class.Name())
def BenchmarkSmokeTest(self): class SinglePageBenchmark(benchmark): # pylint: disable=no-init def CreateStorySet(self, options): # pylint: disable=super-on-old-class story_set = super(SinglePageBenchmark, self).CreateStorySet(options) # We want to prevent benchmarks from accidentally trying to upload too # much data to the chrome perf dashboard. So this tests tries to # estimate the amount of values that the benchmark _would_ create when # running on the waterfall, and fails if too many values are produced. # As we run a single story and not the whole benchmark, the number of # max values allowed is scaled proportionally. # TODO(crbug.com/981349): This logic is only really valid for legacy # values, and does not take histograms into account. An alternative # should be implemented when using the results processor. type(self).MAX_NUM_VALUES = MAX_NUM_VALUES / len(story_set) # Only smoke test the first story since smoke testing everything takes # too long. for s in story_set.stories[num_pages:]: story_set.RemoveStory(s) return story_set # Some benchmarks are running multiple iterations # which is not needed for a smoke test if hasattr(SinglePageBenchmark, 'enable_smoke_test_mode'): SinglePageBenchmark.enable_smoke_test_mode = True with tempfile_ext.NamedTemporaryDirectory() as temp_dir: # Set the benchmark's default arguments. options = options_for_unittests.GetRunOptions( output_dir=temp_dir, benchmark_cls=SinglePageBenchmark, environment=chromium_config.GetDefaultChromiumConfig()) options.pageset_repeat = 1 # For smoke testing only run the page once. single_page_benchmark = SinglePageBenchmark() # TODO(crbug.com/985103): Remove this code once # AugmentExpectationsWithFile is deleted and replaced with functionality # in story_filter.py. if hasattr(single_page_benchmark, 'AugmentExpectationsWithFile'): with open(path_util.GetExpectationsPath()) as fp: single_page_benchmark.AugmentExpectationsWithFile( fp.read()) return_code = single_page_benchmark.Run(options) if return_code == -1: self.skipTest('The benchmark was not run.') self.assertEqual(0, return_code, msg='Failed: %s' % benchmark)
def GenerateBenchmarkOptions(output_dir, benchmark_cls): options = options_for_unittests.GetRunOptions( output_dir=output_dir, benchmark_cls=benchmark_cls, environment=chromium_config.GetDefaultChromiumConfig()) options.pageset_repeat = 1 # For smoke testing only run each page once. # Enable browser logging in the smoke test only. Hopefully, this will detect # all crashes and hence remove the need to enable logging in actual perf # benchmarks. options.browser_options.logging_verbosity = 'non-verbose' options.target_platforms = benchmark_cls.GetSupportedPlatformNames( benchmark_cls.SUPPORTED_PLATFORMS) return options
def GenerateBenchmarkOptions(output_dir, benchmark_cls): options = testing.GetRunOptions( output_dir=output_dir, benchmark_cls=benchmark_cls, overrides={'run_full_story_set': True}, environment=chromium_config.GetDefaultChromiumConfig()) options.pageset_repeat = 1 # For smoke testing only run each page once. options.output_formats = ['histograms'] options.max_values_per_test_case = MAX_VALUES_PER_TEST_CASE # Enable browser logging in the smoke test only. Hopefully, this will detect # all crashes and hence remove the need to enable logging in actual perf # benchmarks. options.browser_options.logging_verbosity = 'non-verbose' options.target_platforms = benchmark_cls.GetSupportedPlatformNames( benchmark_cls.SUPPORTED_PLATFORMS) return options