def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
assert benchmark_classes, 'This list should never be empty'
for benchmark_class in benchmark_classes:
# HACK: these options should be derived from options_for_unittests which are
# the resolved options from run_tests' arguments. However, options is only
# parsed during test time which happens after load_tests are called.
# Since none of our system health benchmarks creates stories based on
# command line options, it should be ok to pass options=None to
# CreateStorySet.
stories_set = benchmark_class().CreateStorySet(options=None)
# Prefetch WPR archive needed by the stories set to avoid race condition
# when feching them when tests are run in parallel.
# See crbug.com/700426 for more details.
stories_set.wpr_archive_info.DownloadArchivesIfNeeded()
for story_to_smoke_test in stories_set.stories:
suite.addTest(
_GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
return suite
def testTestRunner(self):
suite = progress_reporter.TestSuite()
suite.addTest(TestFoo(methodName='RunPassingTest'))
suite.addTest(TestFoo(methodName='RunFailingTest'))
reporter = LoggingProgressReporter()
runner = progress_reporter.TestRunner()
progress_reporters = (reporter, )
result = runner.run(suite, progress_reporters, 1, None)
self.assertEqual(len(result.successes), 1)
self.assertEqual(len(result.failures), 1)
self.assertEqual(len(result.failures_and_errors), 1)
expected = (
'StartTestRun',
'StartTestSuite',
'StartTest',
'Success',
'StopTest',
'StartTest',
'Failure',
'StopTest',
'StopTestSuite',
'StopTestRun',
)
self.assertEqual(reporter.call_log, expected)
def load_tests(loader, standard_tests, pattern):
del loader, pattern # unused
suite = progress_reporter.TestSuite()
for t in standard_tests:
suite.addTests(t)
_AddBenchmarkOptionsTests(suite)
return suite
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmarks_dir = os.path.dirname(__file__)
top_level_dir = os.path.dirname(benchmarks_dir)
# Using the default of |index_by_class_name=False| means that if a module
# has multiple benchmarks, only the last one is returned.
all_benchmarks = discover.DiscoverClasses(
benchmarks_dir,
top_level_dir,
benchmark_module.Benchmark,
index_by_class_name=False).values()
for benchmark in all_benchmarks:
if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES:
continue
# TODO(tonyg): Smoke doesn't work with session_restore yet.
if (benchmark.Name().startswith('session_restore')
or benchmark.Name().startswith('skpicture_printer')):
continue
if hasattr(benchmark, 'generated_profile_archive'):
# We'd like to test these, but don't know how yet.
continue
class BenchmarkSmokeTest(unittest.TestCase):
pass
method = SmokeTestGenerator(benchmark)
# Make sure any decorators are propagated from the original declaration.
# (access to protected members) pylint: disable=W0212
# TODO(dpranke): Since we only pick the first test from every class
# (above), if that test is disabled, we'll end up not running *any*
# test from the class. We should probably discover all of the tests
# in a class, and then throw the ones we don't need away instead.
# Merge decorators.
for attribute in ['_enabled_strings', '_disabled_strings']:
# Do set union of attributes to eliminate duplicates.
merged_attributes = getattr(method, attribute, set()).union(
getattr(benchmark, attribute, set()))
if merged_attributes:
setattr(method, attribute, merged_attributes)
# Disable some tests on android platform only.
if sys.modules[benchmark.__module__] in _ANDROID_BLACK_LIST_MODULES:
method._disabled_strings.add('android')
# TODO(bashi): Remove once crrev.com/1266833004 is landed.
if benchmark.Name() == 'memory.blink_memory_mobile':
method._disabled_strings.add('android')
setattr(BenchmarkSmokeTest, benchmark.Name(), method)
suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
return suite
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmarks_dir = os.path.dirname(__file__)
top_level_dir = os.path.dirname(benchmarks_dir)
# Using the default of |index_by_class_name=False| means that if a module
# has multiple benchmarks, only the last one is returned.
all_benchmarks = discover.DiscoverClasses(
benchmarks_dir,
top_level_dir,
benchmark_module.Benchmark,
index_by_class_name=False).values()
for benchmark in all_benchmarks:
if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES:
continue
# TODO(tonyg): Smoke doesn't work with session_restore yet.
if (benchmark.Name().startswith('session_restore')
or benchmark.Name().startswith('skpicture_printer')):
continue
if hasattr(benchmark, 'generated_profile_archive'):
# We'd like to test these, but don't know how yet.
continue
class BenchmarkSmokeTest(unittest.TestCase):
pass
# tab_switching needs more than one page to test correctly.
if 'tab_switching' in benchmark.Name():
method = SmokeTestGenerator(benchmark, num_pages=2)
else:
method = SmokeTestGenerator(benchmark)
# Make sure any decorators are propagated from the original declaration.
# (access to protected members) pylint: disable=protected-access
# TODO(dpranke): Since we only pick the first test from every class
# (above), if that test is disabled, we'll end up not running *any*
# test from the class. We should probably discover all of the tests
# in a class, and then throw the ones we don't need away instead.
disabled_benchmark_attr = decorators.DisabledAttributeName(benchmark)
disabled_method_attr = decorators.DisabledAttributeName(method)
enabled_benchmark_attr = decorators.EnabledAttributeName(benchmark)
enabled_method_attr = decorators.EnabledAttributeName(method)
MergeDecorators(method, disabled_method_attr, benchmark,
disabled_benchmark_attr)
MergeDecorators(method, enabled_method_attr, benchmark,
enabled_benchmark_attr)
setattr(BenchmarkSmokeTest, benchmark.Name(), method)
suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
return suite
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
assert benchmark_classes, 'This list should never be empty'
names_stories_to_smoke_tests = []
for benchmark_class in benchmark_classes:
# HACK: these options should be derived from GetRunOptions which are
# the resolved options from run_tests' arguments. However, options is only
# parsed during test time which happens after load_tests are called.
# Since none of our system health benchmarks creates stories based on
# command line options, it should be ok to pass options=None to
# CreateStorySet.
stories_set = benchmark_class().CreateStorySet(options=None)
# Prefetch WPR archive needed by the stories set to avoid race condition
# when feching them when tests are run in parallel.
# See crbug.com/700426 for more details.
story_names = [s.name for s in stories_set if not s.is_local]
stories_set.wpr_archive_info.DownloadArchivesIfNeeded(
story_names=story_names)
for story_to_smoke_test in stories_set.stories:
# Per crbug.com/1019383 we don't have many device cycles to work with on
# Android, so let's just run the most important stories.
if (benchmark_class.Name() == 'system_health.memory_mobile'
and 'health_check' not in story_to_smoke_test.tags):
continue
suite.addTest(
_GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
names_stories_to_smoke_tests.append(benchmark_class.Name() + '/' +
story_to_smoke_test.name)
# The full story name should follow this convention: story_name[:version],
# where version is a year. Please refer to the link below for details:
# https://docs.google.com/document/d/134u_j_Lk2hLiDHYxK3NVdZM_sOtExrsExU-hiNFa1uw
# Raise exception for stories which have more than one version enabled.
multi_version_stories = find_multi_version_stories(
names_stories_to_smoke_tests, _DISABLED_TESTS)
if len(multi_version_stories):
msg = ''
for prefix, stories in multi_version_stories.items():
msg += prefix + ' : ' + ','.join(stories) + '\n'
raise ValueError(
'The stories below has multiple versions.'
'In order to save CQ capacity, we should only run the latest '
'version on CQ. Please put the legacy stories in _DISABLED_TESTS '
'list or remove them to save CQ capacity (see crbug.com/893615)). '
'You can use crbug.com/878390 for the disabling reference.'
'[StoryName] : [StoryVersion1],[StoryVersion2]...\n%s' % (msg))
return suite
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
assert benchmark_classes, 'This list should never be empty'
names_stories_to_smoke_tests = []
for benchmark_class in benchmark_classes:
# HACK: these options should be derived from options_for_unittests which are
# the resolved options from run_tests' arguments. However, options is only
# parsed during test time which happens after load_tests are called.
# Since none of our system health benchmarks creates stories based on
# command line options, it should be ok to pass options=None to
# CreateStorySet.
stories_set = benchmark_class().CreateStorySet(options=None)
# Prefetch WPR archive needed by the stories set to avoid race condition
# when feching them when tests are run in parallel.
# See crbug.com/700426 for more details.
story_names = [s.name for s in stories_set if not s.is_local]
stories_set.wpr_archive_info.DownloadArchivesIfNeeded(
story_names=story_names)
for story_to_smoke_test in stories_set.stories:
suite.addTest(
_GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
names_stories_to_smoke_tests.append(benchmark_class.Name() + '/' +
story_to_smoke_test.name)
for i, story_name in enumerate(names_stories_to_smoke_tests):
for j in xrange(i + 1, len(names_stories_to_smoke_tests)):
other_story_name = names_stories_to_smoke_tests[j]
if (other_story_name.startswith(story_name + ':')
and story_name not in _DISABLED_TESTS):
raise ValueError(
'Story %s is to be replaced by %s. Please put %s in '
'_DISABLED_TESTS list to save CQ capacity (see crbug.com/893615)). '
'You can use crbug.com/878390 for the disabling reference.'
% (repr(story_name), repr(other_story_name),
repr(story_name)))
return suite
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
assert benchmark_classes, 'This list should never be empty'
for benchmark_class in benchmark_classes:
# HACK: these options should be derived from options_for_unittests which are
# the resolved options from run_tests' arguments. However, options is only
# parsed during test time which happens after load_tests are called.
# Since none of our system health benchmarks creates stories based on
# command line options, it should be ok to pass options=None to
# CreateStorySet.
for story_to_smoke_test in (benchmark_class().CreateStorySet(
options=None).stories):
suite.addTest(
_GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
return suite
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
assert benchmark_classes, 'This list should never be empty'
validate_smoke_test_name_versions()
for benchmark_class in benchmark_classes:
stories_set = _create_story_set(benchmark_class)
remote_story_names = []
for story_to_smoke_test in stories_set:
if _should_skip_story(benchmark_class, story_to_smoke_test):
continue
if not story_to_smoke_test.is_local:
remote_story_names.append(story_to_smoke_test)
suite.addTest(
_GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
# Prefetch WPR archive needed by the stories set to avoid race condition
# when fetching them when tests are run in parallel.
# See crbug.com/700426 for more details.
stories_set.wpr_archive_info.DownloadArchivesIfNeeded(
story_names=remote_story_names)
return suite
