def testPageTestWithIncompatibleStory(self):
options = options_for_unittests.GetCopy()
b = TestBenchmark(
story_module.Story(
name='test story',
shared_state_class=shared_page_state.SharedPageState))
with self.assertRaisesRegexp(
Exception, 'containing only telemetry.page.Page stories'):
story_runner.RunBenchmark(b, b.CreateStorySet(options), None,
options.browser_options, options)
state_class = story_module.SharedState
b = TestBenchmark(
story_module.Story(name='test benchmark',
shared_state_class=state_class))
with self.assertRaisesRegexp(
Exception, 'containing only telemetry.page.Page stories'):
story_runner.RunBenchmark(b, b.CreateStorySet(options), None,
options.browser_options, options)
b = TestBenchmark(
android.AndroidStory(name='test benchmark', start_intent=None))
with self.assertRaisesRegexp(
Exception, 'containing only telemetry.page.Page stories'):
story_runner.RunBenchmark(b, b.CreateStorySet(options), None,
options.browser_options, options)
def Run(self, finder_options):
"""We shouldn't be overriding this according to
telemetry.benchmark.Benchmark"""
assert 'chartjson' in finder_options.output_formats, (
'loading.desktop.network_service requires --output-format=chartjson. '
'Please contact owner to rewrite the benchmark if chartjson is going '
'away.')
assert finder_options.output_dir
output_dir = finder_options.output_dir
temp_file_path = os.path.join(output_dir, 'results-chart.json')
# Run test with feature disabled.
self.enable_feature = False
control_return_code = story_runner.RunBenchmark(self, finder_options)
if control_return_code != 0:
return control_return_code
control_chart_json = json.load(open(temp_file_path))
# Run test again with feature enabled.
self.enable_feature = True
enabled_return_code = story_runner.RunBenchmark(self, finder_options)
if enabled_return_code != 0:
return enabled_return_code
enabled_chart_json = json.load(open(temp_file_path))
logging.info(
'Starting to merge control chartjson into enabled chartjson')
try:
# Merge the result and compute the difference.
_MergeControlChartJsonIntoEnabled(enabled_chart_json,
control_chart_json)
except Exception as e:
logging.error('exception merging two chart json: %s', repr(e))
traceback.print_exc()
with open(temp_file_path, 'w') as f:
json.dump(
{
'control_chart_json': control_chart_json,
'enabled_chart_json': enabled_chart_json
},
f,
indent=2,
separators=(',', ': '))
f.write('\n')
return 1
else:
logging.info('Finished merging chartjsons, writing back to disk')
with open(temp_file_path, 'w') as f:
json.dump(enabled_chart_json,
f,
indent=2,
separators=(',', ': '))
f.write('\n')
return 0
def testPageTestWithCompatibleStory(self):
original_run_fn = story_runner.RunStorySet
was_run = [False]
def RunStub(*arg, **kwargs):
del arg, kwargs
was_run[0] = True
story_runner.RunStorySet = RunStub
try:
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.output_dir = ''
options.upload_results = False
options.suppress_gtest_report = True
options.results_label = ''
b = TestBenchmark(page.Page(url='about:blank', name='about:blank'))
story_runner.RunBenchmark(b, b.CreateStorySet(options), None,
options.browser_options, options)
finally:
story_runner.RunStorySet = original_run_fn
self.assertTrue(was_run[0])
def testRunBenchmarkTimeDuration(self):
fake_benchmark = FakeBenchmark()
options = fakes.CreateBrowserFinderOptions()
options.upload_results = None
options.suppress_gtest_report = False
options.results_label = None
options.use_live_sites = False
options.max_failures = 100
options.pageset_repeat = 1
options.output_formats = ['chartjson']
with mock.patch(
'telemetry.internal.story_runner.time.time') as time_patch:
# 3, because telemetry code asks for the time at some point
time_patch.side_effect = [1, 0, 61]
tmp_path = tempfile.mkdtemp()
try:
options.output_dir = tmp_path
story_runner.RunBenchmark(fake_benchmark, options)
with open(os.path.join(tmp_path, 'results-chart.json')) as f:
data = json.load(f)
self.assertEqual(len(data['charts']), 1)
charts = data['charts']
self.assertIn('benchmark_duration', charts)
duration = charts['benchmark_duration']
self.assertIn("summary", duration)
summary = duration['summary']
duration = summary['value']
self.assertAlmostEqual(duration, 1)
finally:
shutil.rmtree(tmp_path)
def testRunBenchmarkTimeDuration(self):
fake_benchmark = FakeBenchmark()
options = self._GenerateBaseBrowserFinderOptions()
with mock.patch('telemetry.internal.story_runner.time.time') as time_patch:
# 3, because telemetry code asks for the time at some point
time_patch.side_effect = [1, 0, 61]
tmp_path = tempfile.mkdtemp()
try:
options.output_dir = tmp_path
story_runner.RunBenchmark(fake_benchmark, options)
with open(os.path.join(tmp_path, 'results-chart.json')) as f:
data = json.load(f)
self.assertEqual(len(data['charts']), 1)
charts = data['charts']
self.assertIn('benchmark_duration', charts)
duration = charts['benchmark_duration']
self.assertIn("summary", duration)
summary = duration['summary']
duration = summary['value']
self.assertAlmostEqual(duration, 1)
finally:
shutil.rmtree(tmp_path)
def testReturnCodeCaughtException(self):
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory(
'story', run_side_effect=exceptions.AppCrashException())])
options = self.GetFakeBrowserOptions()
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, exit_codes.TEST_FAILURE)
def testReturnCodeUnhandleableError(self):
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory(
'story', run_side_effect=MemoryError('Unhandleable'))])
options = self.GetFakeBrowserOptions()
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, exit_codes.FATAL_ERROR)
def testRunBenchmark_AddsOwners_NoComponent(self):
@benchmark.Owner(emails=['*****@*****.**'])
class FakeBenchmarkWithOwner(FakeBenchmark):
def __init__(self):
super(FakeBenchmark, self).__init__()
self._disabled = False
self._story_disabled = False
fake_benchmark = FakeBenchmarkWithOwner()
options = self._GenerateBaseBrowserFinderOptions()
options.output_formats = ['histograms']
temp_path = tempfile.mkdtemp()
try:
options.output_dir = temp_path
story_runner.RunBenchmark(fake_benchmark, options)
with open(os.path.join(temp_path, 'histograms.json')) as f:
data = json.load(f)
hs = histogram_set.HistogramSet()
hs.ImportDicts(data)
generic_diagnostics = hs.GetSharedDiagnosticsOfType(
histogram_module.GenericSet)
self.assertGreater(len(generic_diagnostics), 0)
generic_diagnostics_values = [
list(diagnostic) for diagnostic in generic_diagnostics]
self.assertIn(['*****@*****.**'], generic_diagnostics_values)
finally:
shutil.rmtree(temp_path)
def testRunStoryWithMissingArchiveFile(self):
fake_benchmark = FakeBenchmark(
archive_data_file='data/does-not-exist.json')
options = self.GetFakeBrowserOptions()
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, 2) # Benchmark was interrupted.
self.assertIn('ArchiveError', sys.stderr.getvalue())
def testUnexpectedSkipsWithFiltering(self):
# We prepare side effects for 50 stories, the first 30 run fine, the
# remaining 20 fail with a fatal error.
fatal_error = MemoryError('this is an unexpected exception')
side_effects = [None] * 30 + [fatal_error] * 20
fake_benchmark = FakeBenchmark(stories=(
test_stories.DummyStory('story_%i' % i, run_side_effect=effect)
for i, effect in enumerate(side_effects)))
# Set the filtering to only run from story_10 --> story_40
options = self.GetFakeBrowserOptions({
'story_shard_begin_index': 10,
'story_shard_end_index': 41
})
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEquals(exit_codes.FATAL_ERROR, return_code)
# The results should contain entries of story 10 --> story 40. Of those
# entries, story 31's actual result is 'FAIL' and
# stories from 31 to 40 will shows 'SKIP'.
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 31)
expected = []
expected.extend(('story_%i' % i, 'PASS') for i in xrange(10, 30))
expected.append(('story_30', 'FAIL'))
expected.extend(('story_%i' % i, 'SKIP') for i in xrange(31, 41))
for (story, status), result in zip(expected, test_results):
self.assertEqual(result['testPath'], 'fake_benchmark/%s' % story)
self.assertEqual(result['status'], status)
def testDisabledBenchmarkViaCanRunOnPlatform(self):
fake_benchmark = FakeBenchmark()
fake_benchmark.SUPPORTED_PLATFORMS = []
options = self.GetFakeBrowserOptions()
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertFalse(test_results) # No tests ran at all.
def testArtifactLogsContainHandleableException(self):
def failed_run():
logging.warning('This will fail gracefully')
raise exceptions.TimeoutException('karma!')
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory('story1', run_side_effect=failed_run),
test_stories.DummyStory('story2')
])
options = self.GetFakeBrowserOptions()
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, exit_codes.TEST_FAILURE)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 2)
# First story failed.
self.assertEqual(test_results[0]['testPath'], 'fake_benchmark/story1')
self.assertEqual(test_results[0]['status'], 'FAIL')
self.assertIn('logs.txt', test_results[0]['outputArtifacts'])
with open(test_results[0]['outputArtifacts']['logs.txt']
['filePath']) as f:
test_log = f.read()
# Ensure that the log contains warning messages and python stack.
self.assertIn('Handleable error', test_log)
self.assertIn('This will fail gracefully', test_log)
self.assertIn("raise exceptions.TimeoutException('karma!')", test_log)
# Second story ran fine.
self.assertEqual(test_results[1]['testPath'], 'fake_benchmark/story2')
self.assertEqual(test_results[1]['status'], 'PASS')
def testArtifactLogsContainUnhandleableException(self):
def failed_run():
logging.warning('This will fail badly')
raise MemoryError('this is a fatal exception')
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory('story1', run_side_effect=failed_run),
test_stories.DummyStory('story2')
])
options = self.GetFakeBrowserOptions()
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, exit_codes.FATAL_ERROR)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 2)
# First story failed.
self.assertEqual(test_results[0]['testPath'], 'fake_benchmark/story1')
self.assertEqual(test_results[0]['status'], 'FAIL')
self.assertIn('logs.txt', test_results[0]['outputArtifacts'])
with open(test_results[0]['outputArtifacts']['logs.txt']
['filePath']) as f:
test_log = f.read()
# Ensure that the log contains warning messages and python stack.
self.assertIn('Unhandleable error', test_log)
self.assertIn('This will fail badly', test_log)
self.assertIn("raise MemoryError('this is a fatal exception')",
test_log)
# Second story was skipped.
self.assertEqual(test_results[1]['testPath'], 'fake_benchmark/story2')
self.assertEqual(test_results[1]['status'], 'SKIP')
def testReturnCodeDisabledStory(self):
fake_benchmark = FakeBenchmark(stories=['fake_story'])
fake_story_filter = FakeStoryFilter(stories_to_skip=['fake_story'])
options = self.GetFakeBrowserOptions()
with mock.patch(
'telemetry.story.story_filter.StoryFilterFactory.BuildStoryFilter',
return_value=fake_story_filter):
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, exit_codes.ALL_TESTS_SKIPPED)
def testRangeIndexRanges(self):
fake_benchmark = FakeBenchmark(
stories=(test_stories.DummyStory('story_%i' % i)
for i in range(100)))
options = self.GetFakeBrowserOptions(
{'story_shard_indexes': "-10, 20-30, 90-"})
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 30)
def testValidateBenchmarkName(self):
class FakeBenchmarkWithBadName(FakeBenchmark):
NAME = 'bad/benchmark (name)'
fake_benchmark = FakeBenchmarkWithBadName()
options = self.GetFakeBrowserOptions()
return_code = story_runner.RunBenchmark(fake_benchmark, options)
self.assertEqual(return_code, 2)
self.assertIn('Invalid benchmark name', sys.stderr.getvalue())
def testFullRun(self):
options = self.GetFakeBrowserOptions()
story_filter.StoryFilterFactory.ProcessCommandLineArgs(
parser=None, args=options)
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory('story1', tags=['important']),
test_stories.DummyStory('story2', tags=['other']),
], abridging_tag='important')
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 2)
def testSkippedWithStoryFilter(self):
fake_benchmark = FakeBenchmark(stories=['fake_story'])
options = self.GetFakeBrowserOptions()
fake_story_filter = FakeStoryFilter(stories_to_skip=['fake_story'])
with mock.patch(
'telemetry.story.story_filter.StoryFilterFactory.BuildStoryFilter',
return_value=fake_story_filter):
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertTrue(test_results) # Some tests ran, but all skipped.
self.assertTrue(all(t['status'] == 'SKIP' for t in test_results))
def testDeviceInfo(self):
fake_benchmark = FakeBenchmark(stories=['fake_story'])
options = self.GetFakeBrowserOptions()
options.fake_possible_browser = fakes.FakePossibleBrowser(
arch_name='abc', os_name='win', os_version_name='win10')
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
diagnostics = ReadDiagnostics(test_results[0])
self.assertEqual(diagnostics['architectures'], ['abc'])
self.assertEqual(diagnostics['osNames'], ['win'])
self.assertEqual(diagnostics['osVersions'], ['win10'])
def testStoryFlag(self):
options = self.GetFakeBrowserOptions()
args = fakes.FakeParsedArgsForStoryFilter(stories=['story1', 'story3'])
story_filter.StoryFilterFactory.ProcessCommandLineArgs(parser=None,
args=args)
fake_benchmark = FakeBenchmark(stories=['story1', 'story2', 'story3'])
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 2)
self.assertTrue(test_results[0]['testPath'].endswith('/story1'))
self.assertTrue(test_results[1]['testPath'].endswith('/story3'))
def testRunBenchmarkDisabledStoryWithBadName(self):
fake_benchmark = FakeBenchmark()
fake_benchmark.story_disabled = True
options = self._GenerateBaseBrowserFinderOptions()
tmp_path = tempfile.mkdtemp()
try:
options.output_dir = tmp_path
rc = story_runner.RunBenchmark(fake_benchmark, options)
# Test should return 0 since only error messages are logged.
self.assertEqual(rc, 0)
finally:
shutil.rmtree(tmp_path)
def testWithOwnerInfoButNoUrl(self):
@benchmark.Owner(emails=['*****@*****.**'])
class FakeBenchmarkWithOwner(FakeBenchmark):
pass
fake_benchmark = FakeBenchmarkWithOwner()
options = self.GetFakeBrowserOptions()
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
diagnostics = ReadDiagnostics(test_results[0])
self.assertEqual(diagnostics['owners'], ['*****@*****.**'])
self.assertNotIn('documentationLinks', diagnostics)
def testOneStoryFilteredOneNot(self):
fake_story_filter = FakeStoryFilter(stories_to_filter_out=['story1'])
fake_benchmark = FakeBenchmark(stories=['story1', 'story2'])
options = self.GetFakeBrowserOptions()
with mock.patch(
'telemetry.story.story_filter.StoryFilterFactory.BuildStoryFilter',
return_value=fake_story_filter):
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 1)
self.assertEqual(test_results[0]['status'], 'PASS')
self.assertTrue(test_results[0]['testPath'].endswith('/story2'))
def testAbridged(self):
options = self.GetFakeBrowserOptions()
options.run_abridged_story_set = True
story_filter.StoryFilterFactory.ProcessCommandLineArgs(
parser=None, args=options)
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory('story1', tags=['important']),
test_stories.DummyStory('story2', tags=['other']),
], abridging_tag='important')
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 1)
self.assertTrue(test_results[0]['testPath'].endswith('/story1'))
def testRunBenchmarkDisabledBenchmark(self):
fake_benchmark = FakeBenchmark()
fake_benchmark.disabled = True
options = self._GenerateBaseBrowserFinderOptions()
tmp_path = tempfile.mkdtemp()
try:
options.output_dir = tmp_path
story_runner.RunBenchmark(fake_benchmark, options)
with open(os.path.join(tmp_path, 'results-chart.json')) as f:
data = json.load(f)
self.assertFalse(data['enabled'])
finally:
shutil.rmtree(tmp_path)
def testOneStorySkippedOneNot(self):
fake_story_filter = FakeStoryFilter(stories_to_skip=['story1'])
fake_benchmark = FakeBenchmark(stories=['story1', 'story2'])
options = self.GetFakeBrowserOptions()
with mock.patch(
'telemetry.story.story_filter.StoryFilterFactory.BuildStoryFilter',
return_value=fake_story_filter):
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
status = [t['status'] for t in test_results]
self.assertEqual(len(status), 2)
self.assertIn('SKIP', status)
self.assertIn('PASS', status)
def testDownloadMinimalServingDirs(self):
fake_benchmark = FakeBenchmark(stories=[
test_stories.DummyStory(
'story_foo', serving_dir='/files/foo', tags=['foo']),
test_stories.DummyStory(
'story_bar', serving_dir='/files/bar', tags=['bar']),
], cloud_bucket=cloud_storage.PUBLIC_BUCKET)
options = self.GetFakeBrowserOptions(overrides={'story_tag_filter': 'foo'})
with mock.patch(
'py_utils.cloud_storage.GetFilesInDirectoryIfChanged') as get_files:
story_runner.RunBenchmark(fake_benchmark, options)
# Foo is the only included story serving dir.
self.assertEqual(get_files.call_count, 1)
get_files.assert_called_once_with('/files/foo', cloud_storage.PUBLIC_BUCKET)
def testWithOwnerInfo(self):
@benchmark.Owner(emails=['*****@*****.**', '*****@*****.**'],
component='fooBar',
documentation_url='https://example.com/')
class FakeBenchmarkWithOwner(FakeBenchmark):
pass
fake_benchmark = FakeBenchmarkWithOwner()
options = self.GetFakeBrowserOptions()
story_runner.RunBenchmark(fake_benchmark, options)
test_results = self.ReadTestResults()
diagnostics = ReadDiagnostics(test_results[0])
self.assertEqual(diagnostics['owners'],
['*****@*****.**', '*****@*****.**'])
self.assertEqual(diagnostics['bugComponents'], ['fooBar'])
self.assertEqual(
diagnostics['documentationLinks'],
[['Benchmark documentation link', 'https://example.com/']])
def Run(self, finder_options):
"""Do not override this method."""
finder_options.target_platforms = self.GetSupportedPlatformNames(
self.SUPPORTED_PLATFORMS)
return story_runner.RunBenchmark(self, finder_options)
def Run(self, finder_options):
"""Do not override this method."""
return story_runner.RunBenchmark(self, finder_options)
