def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmark_classes = GetSystemHealthBenchmarksToSmokeTest() assert benchmark_classes, 'This list should never be empty' for benchmark_class in benchmark_classes: # HACK: these options should be derived from options_for_unittests which are # the resolved options from run_tests' arguments. However, options is only # parsed during test time which happens after load_tests are called. # Since none of our system health benchmarks creates stories based on # command line options, it should be ok to pass options=None to # CreateStorySet. stories_set = benchmark_class().CreateStorySet(options=None) # Prefetch WPR archive needed by the stories set to avoid race condition # when feching them when tests are run in parallel. # See crbug.com/700426 for more details. stories_set.wpr_archive_info.DownloadArchivesIfNeeded() for story_to_smoke_test in stories_set.stories: suite.addTest( _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test)) return suite
def testTestRunner(self): suite = progress_reporter.TestSuite() suite.addTest(TestFoo(methodName='RunPassingTest')) suite.addTest(TestFoo(methodName='RunFailingTest')) reporter = LoggingProgressReporter() runner = progress_reporter.TestRunner() progress_reporters = (reporter, ) result = runner.run(suite, progress_reporters, 1, None) self.assertEqual(len(result.successes), 1) self.assertEqual(len(result.failures), 1) self.assertEqual(len(result.failures_and_errors), 1) expected = ( 'StartTestRun', 'StartTestSuite', 'StartTest', 'Success', 'StopTest', 'StartTest', 'Failure', 'StopTest', 'StopTestSuite', 'StopTestRun', ) self.assertEqual(reporter.call_log, expected)
def load_tests(loader, standard_tests, pattern): del loader, pattern # unused suite = progress_reporter.TestSuite() for t in standard_tests: suite.addTests(t) _AddBenchmarkOptionsTests(suite) return suite
def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmarks_dir = os.path.dirname(__file__) top_level_dir = os.path.dirname(benchmarks_dir) # Using the default of |index_by_class_name=False| means that if a module # has multiple benchmarks, only the last one is returned. all_benchmarks = discover.DiscoverClasses( benchmarks_dir, top_level_dir, benchmark_module.Benchmark, index_by_class_name=False).values() for benchmark in all_benchmarks: if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES: continue # TODO(tonyg): Smoke doesn't work with session_restore yet. if (benchmark.Name().startswith('session_restore') or benchmark.Name().startswith('skpicture_printer')): continue if hasattr(benchmark, 'generated_profile_archive'): # We'd like to test these, but don't know how yet. continue class BenchmarkSmokeTest(unittest.TestCase): pass method = SmokeTestGenerator(benchmark) # Make sure any decorators are propagated from the original declaration. # (access to protected members) pylint: disable=W0212 # TODO(dpranke): Since we only pick the first test from every class # (above), if that test is disabled, we'll end up not running *any* # test from the class. We should probably discover all of the tests # in a class, and then throw the ones we don't need away instead. # Merge decorators. for attribute in ['_enabled_strings', '_disabled_strings']: # Do set union of attributes to eliminate duplicates. merged_attributes = getattr(method, attribute, set()).union( getattr(benchmark, attribute, set())) if merged_attributes: setattr(method, attribute, merged_attributes) # Disable some tests on android platform only. if sys.modules[benchmark.__module__] in _ANDROID_BLACK_LIST_MODULES: method._disabled_strings.add('android') # TODO(bashi): Remove once crrev.com/1266833004 is landed. if benchmark.Name() == 'memory.blink_memory_mobile': method._disabled_strings.add('android') setattr(BenchmarkSmokeTest, benchmark.Name(), method) suite.addTest(BenchmarkSmokeTest(benchmark.Name())) return suite
def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmarks_dir = os.path.dirname(__file__) top_level_dir = os.path.dirname(benchmarks_dir) # Using the default of |index_by_class_name=False| means that if a module # has multiple benchmarks, only the last one is returned. all_benchmarks = discover.DiscoverClasses( benchmarks_dir, top_level_dir, benchmark_module.Benchmark, index_by_class_name=False).values() for benchmark in all_benchmarks: if sys.modules[benchmark.__module__] in _BLACK_LIST_TEST_MODULES: continue # TODO(tonyg): Smoke doesn't work with session_restore yet. if (benchmark.Name().startswith('session_restore') or benchmark.Name().startswith('skpicture_printer')): continue if hasattr(benchmark, 'generated_profile_archive'): # We'd like to test these, but don't know how yet. continue class BenchmarkSmokeTest(unittest.TestCase): pass # tab_switching needs more than one page to test correctly. if 'tab_switching' in benchmark.Name(): method = SmokeTestGenerator(benchmark, num_pages=2) else: method = SmokeTestGenerator(benchmark) # Make sure any decorators are propagated from the original declaration. # (access to protected members) pylint: disable=protected-access # TODO(dpranke): Since we only pick the first test from every class # (above), if that test is disabled, we'll end up not running *any* # test from the class. We should probably discover all of the tests # in a class, and then throw the ones we don't need away instead. disabled_benchmark_attr = decorators.DisabledAttributeName(benchmark) disabled_method_attr = decorators.DisabledAttributeName(method) enabled_benchmark_attr = decorators.EnabledAttributeName(benchmark) enabled_method_attr = decorators.EnabledAttributeName(method) MergeDecorators(method, disabled_method_attr, benchmark, disabled_benchmark_attr) MergeDecorators(method, enabled_method_attr, benchmark, enabled_benchmark_attr) setattr(BenchmarkSmokeTest, benchmark.Name(), method) suite.addTest(BenchmarkSmokeTest(benchmark.Name())) return suite
def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmark_classes = GetSystemHealthBenchmarksToSmokeTest() assert benchmark_classes, 'This list should never be empty' names_stories_to_smoke_tests = [] for benchmark_class in benchmark_classes: # HACK: these options should be derived from GetRunOptions which are # the resolved options from run_tests' arguments. However, options is only # parsed during test time which happens after load_tests are called. # Since none of our system health benchmarks creates stories based on # command line options, it should be ok to pass options=None to # CreateStorySet. stories_set = benchmark_class().CreateStorySet(options=None) # Prefetch WPR archive needed by the stories set to avoid race condition # when feching them when tests are run in parallel. # See crbug.com/700426 for more details. story_names = [s.name for s in stories_set if not s.is_local] stories_set.wpr_archive_info.DownloadArchivesIfNeeded( story_names=story_names) for story_to_smoke_test in stories_set.stories: # Per crbug.com/1019383 we don't have many device cycles to work with on # Android, so let's just run the most important stories. if (benchmark_class.Name() == 'system_health.memory_mobile' and 'health_check' not in story_to_smoke_test.tags): continue suite.addTest( _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test)) names_stories_to_smoke_tests.append(benchmark_class.Name() + '/' + story_to_smoke_test.name) # The full story name should follow this convention: story_name[:version], # where version is a year. Please refer to the link below for details: # https://docs.google.com/document/d/134u_j_Lk2hLiDHYxK3NVdZM_sOtExrsExU-hiNFa1uw # Raise exception for stories which have more than one version enabled. multi_version_stories = find_multi_version_stories( names_stories_to_smoke_tests, _DISABLED_TESTS) if len(multi_version_stories): msg = '' for prefix, stories in multi_version_stories.items(): msg += prefix + ' : ' + ','.join(stories) + '\n' raise ValueError( 'The stories below has multiple versions.' 'In order to save CQ capacity, we should only run the latest ' 'version on CQ. Please put the legacy stories in _DISABLED_TESTS ' 'list or remove them to save CQ capacity (see crbug.com/893615)). ' 'You can use crbug.com/878390 for the disabling reference.' '[StoryName] : [StoryVersion1],[StoryVersion2]...\n%s' % (msg)) return suite
def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmark_classes = GetSystemHealthBenchmarksToSmokeTest() assert benchmark_classes, 'This list should never be empty' names_stories_to_smoke_tests = [] for benchmark_class in benchmark_classes: # HACK: these options should be derived from options_for_unittests which are # the resolved options from run_tests' arguments. However, options is only # parsed during test time which happens after load_tests are called. # Since none of our system health benchmarks creates stories based on # command line options, it should be ok to pass options=None to # CreateStorySet. stories_set = benchmark_class().CreateStorySet(options=None) # Prefetch WPR archive needed by the stories set to avoid race condition # when feching them when tests are run in parallel. # See crbug.com/700426 for more details. story_names = [s.name for s in stories_set if not s.is_local] stories_set.wpr_archive_info.DownloadArchivesIfNeeded( story_names=story_names) for story_to_smoke_test in stories_set.stories: suite.addTest( _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test)) names_stories_to_smoke_tests.append(benchmark_class.Name() + '/' + story_to_smoke_test.name) for i, story_name in enumerate(names_stories_to_smoke_tests): for j in xrange(i + 1, len(names_stories_to_smoke_tests)): other_story_name = names_stories_to_smoke_tests[j] if (other_story_name.startswith(story_name + ':') and story_name not in _DISABLED_TESTS): raise ValueError( 'Story %s is to be replaced by %s. Please put %s in ' '_DISABLED_TESTS list to save CQ capacity (see crbug.com/893615)). ' 'You can use crbug.com/878390 for the disabling reference.' % (repr(story_name), repr(other_story_name), repr(story_name))) return suite
def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmark_classes = GetSystemHealthBenchmarksToSmokeTest() assert benchmark_classes, 'This list should never be empty' for benchmark_class in benchmark_classes: # HACK: these options should be derived from options_for_unittests which are # the resolved options from run_tests' arguments. However, options is only # parsed during test time which happens after load_tests are called. # Since none of our system health benchmarks creates stories based on # command line options, it should be ok to pass options=None to # CreateStorySet. for story_to_smoke_test in (benchmark_class().CreateStorySet( options=None).stories): suite.addTest( _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test)) return suite
def load_tests(loader, standard_tests, pattern): del loader, standard_tests, pattern # unused suite = progress_reporter.TestSuite() benchmark_classes = GetSystemHealthBenchmarksToSmokeTest() assert benchmark_classes, 'This list should never be empty' validate_smoke_test_name_versions() for benchmark_class in benchmark_classes: stories_set = _create_story_set(benchmark_class) remote_story_names = [] for story_to_smoke_test in stories_set: if _should_skip_story(benchmark_class, story_to_smoke_test): continue if not story_to_smoke_test.is_local: remote_story_names.append(story_to_smoke_test) suite.addTest( _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test)) # Prefetch WPR archive needed by the stories set to avoid race condition # when fetching them when tests are run in parallel. # See crbug.com/700426 for more details. stories_set.wpr_archive_info.DownloadArchivesIfNeeded( story_names=remote_story_names) return suite