def testLoadAllTestsInModule(self):
context = browser_test_context.TypTestContext()
context.finder_options = options_for_unittests.GetCopy()
context.test_class = Algebra
context.test_case_ids_to_run.add(
'telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber')
context.test_case_ids_to_run.add(
'telemetry.testing.browser_test_runner_unittest.Algebra.testOne')
context.Freeze()
browser_test_context._global_test_context = context
try:
# This should not invoke GenerateTestCases of ErrorneousGeometric class,
# otherwise that would throw Exception.
tests = serially_executed_browser_test_case.LoadAllTestsInModule(
sys.modules[__name__])
self.assertEquals(sorted([t.id() for t in tests]),
['telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber',
'telemetry.testing.browser_test_runner_unittest.Algebra.testOne'])
finally:
browser_test_context._global_test_context = None
def RunTests(args):
parser = _CreateTestArgParsers()
try:
options, extra_args = parser.parse_known_args(args)
except arg_parser._Bailout:
PrintTelemetryHelp()
return parser.exit_status
binary_manager.InitDependencyManager(options.client_configs)
for start_dir in options.start_dirs:
modules_to_classes = discover.DiscoverClasses(
start_dir,
options.top_level_dir,
base_class=serially_executed_browser_test_case.
SeriallyExecutedBrowserTestCase)
browser_test_classes = modules_to_classes.values()
_ValidateDistinctNames(browser_test_classes)
test_class = None
for cl in browser_test_classes:
if cl.Name() == options.test:
test_class = cl
break
if not test_class:
print('Cannot find test class with name matching %s' % options.test)
print('Available tests: %s' % '\n'.join(
cl.Name() for cl in browser_test_classes))
return 1
test_class._typ_runner = typ_runner = typ.Runner()
# Create test context.
typ_runner.context = browser_test_context.TypTestContext()
for c in options.client_configs:
typ_runner.context.client_configs.append(c)
typ_runner.context.finder_options = ProcessCommandLineOptions(
test_class, options, extra_args)
typ_runner.context.test_class = test_class
typ_runner.context.expectations_files = options.expectations_files
test_times = None
if options.read_abbreviated_json_results_from:
with open(options.read_abbreviated_json_results_from, 'r') as f:
abbr_results = json.load(f)
test_times = abbr_results.get('times')
# Setup typ.Runner instance.
typ_runner.args.all = options.all
typ_runner.args.expectations_files = options.expectations_files
typ_runner.args.jobs = options.jobs
typ_runner.args.list_only = options.list_only
typ_runner.args.metadata = options.metadata
typ_runner.args.passthrough = options.passthrough
typ_runner.args.path = options.path
typ_runner.args.quiet = options.quiet
typ_runner.args.repeat = options.repeat
typ_runner.args.repository_absolute_path = options.repository_absolute_path
typ_runner.args.retry_limit = options.retry_limit
typ_runner.args.retry_only_retry_on_failure_tests = (
options.retry_only_retry_on_failure_tests)
typ_runner.args.skip = options.skip
typ_runner.args.suffixes = TEST_SUFFIXES
typ_runner.args.tags = options.tags
typ_runner.args.test_name_prefix = options.test_name_prefix
typ_runner.args.test_filter = options.test_filter
typ_runner.args.test_results_server = options.test_results_server
typ_runner.args.test_type = options.test_type
typ_runner.args.top_level_dir = options.top_level_dir
typ_runner.args.write_full_results_to = options.write_full_results_to
typ_runner.args.write_trace_to = options.write_trace_to
typ_runner.args.disable_resultsink = options.disable_resultsink
typ_runner.classifier = _GetClassifier(typ_runner)
typ_runner.path_delimiter = test_class.GetJSONResultsDelimiter()
typ_runner.setup_fn = _SetUpProcess
typ_runner.teardown_fn = _TearDownProcess
tests_to_run = LoadTestCasesToBeRun(
test_class=test_class, finder_options=typ_runner.context.finder_options,
filter_tests_after_sharding=options.filter_tests_after_sharding,
total_shards=options.total_shards, shard_index=options.shard_index,
test_times=test_times,
debug_shard_distributions=options.debug_shard_distributions,
typ_runner=typ_runner)
for t in tests_to_run:
typ_runner.context.test_case_ids_to_run.add(t.id())
typ_runner.context.Freeze()
browser_test_context._global_test_context = typ_runner.context
# several class level variables are set for GPU tests when
# LoadTestCasesToBeRun is called. Functions line ExpectationsFiles and
# GenerateTags which use these variables should be called after
# LoadTestCasesToBeRun
test_class_expectations_files = test_class.ExpectationsFiles()
# all file paths in test_class_expectations-files must be absolute
assert all(os.path.isabs(path) for path in test_class_expectations_files)
typ_runner.args.expectations_files.extend(
test_class_expectations_files)
typ_runner.args.ignored_tags.extend(test_class.IgnoredTags())
# Since sharding logic is handled by browser_test_runner harness by passing
# browser_test_context.test_case_ids_to_run to subprocess to indicate test
# cases to be run, we explicitly disable sharding logic in typ.
typ_runner.args.total_shards = 1
typ_runner.args.shard_index = 0
typ_runner.args.timing = True
typ_runner.args.verbose = options.verbose
typ_runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = typ_runner.run()
except KeyboardInterrupt:
print("interrupted, exiting", file=sys.stderr)
ret = 130
return ret
def RunTests(args):
parser = _CreateTestArgParsers()
try:
options, extra_args = parser.parse_known_args(args)
except arg_parser._Bailout:
return parser.exit_status
binary_manager.InitDependencyManager(options.client_configs)
for start_dir in options.start_dirs:
modules_to_classes = discover.DiscoverClasses(
start_dir,
options.top_level_dir,
base_class=serially_executed_browser_test_case.
SeriallyExecutedBrowserTestCase)
browser_test_classes = modules_to_classes.values()
_ValidateDistinctNames(browser_test_classes)
test_class = None
for cl in browser_test_classes:
if cl.Name() == options.test:
test_class = cl
break
if not test_class:
print 'Cannot find test class with name matching %s' % options.test
print 'Available tests: %s' % '\n'.join(cl.Name()
for cl in browser_test_classes)
return 1
# Create test context.
context = browser_test_context.TypTestContext()
for c in options.client_configs:
context.client_configs.append(c)
context.finder_options = ProcessCommandLineOptions(test_class, options,
extra_args)
context.test_class = test_class
test_times = None
if options.read_abbreviated_json_results_from:
with open(options.read_abbreviated_json_results_from, 'r') as f:
abbr_results = json.load(f)
test_times = abbr_results.get('times')
tests_to_run = LoadTestCasesToBeRun(
test_class=test_class,
finder_options=context.finder_options,
filter_regex_str=options.test_filter,
filter_tests_after_sharding=options.filter_tests_after_sharding,
total_shards=options.total_shards,
shard_index=options.shard_index,
test_times=test_times,
debug_shard_distributions=options.debug_shard_distributions)
for t in tests_to_run:
context.test_case_ids_to_run.add(t.id())
context.Freeze()
browser_test_context._global_test_context = context
# Setup typ runner.
runner = typ.Runner()
runner.context = context
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.args.jobs = options.jobs
runner.args.metadata = options.metadata
runner.args.passthrough = options.passthrough
runner.args.path = options.path
runner.args.retry_limit = options.retry_limit
runner.args.test_results_server = options.test_results_server
runner.args.test_type = options.test_type
runner.args.top_level_dir = options.top_level_dir
runner.args.write_full_results_to = options.write_full_results_to
runner.args.write_trace_to = options.write_trace_to
runner.args.list_only = options.list_only
runner.classifier = _GetClassifier(options)
runner.args.suffixes = TEST_SUFFIXES
# Since sharding logic is handled by browser_test_runner harness by passing
# browser_test_context.test_case_ids_to_run to subprocess to indicate test
# cases to be run, we explicitly disable sharding logic in typ.
runner.args.total_shards = 1
runner.args.shard_index = 0
runner.args.timing = True
runner.args.verbose = options.verbose
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
return ret
def RunTests(args):
parser = _CreateTestArgParsers()
try:
options, extra_args = parser.parse_known_args(args)
except arg_parser._Bailout:
PrintTelemetryHelp()
return parser.exit_status
binary_manager.InitDependencyManager(options.client_configs)
not_using_typ_expectation = False
if options.expectations_files:
parser.error('--expectation-files flag is not supported yet.')
else:
not_using_typ_expectation = True
for start_dir in options.start_dirs:
modules_to_classes = discover.DiscoverClasses(
start_dir,
options.top_level_dir,
base_class=serially_executed_browser_test_case.
SeriallyExecutedBrowserTestCase)
browser_test_classes = modules_to_classes.values()
_ValidateDistinctNames(browser_test_classes)
test_class = None
for cl in browser_test_classes:
if cl.Name() == options.test:
test_class = cl
break
if not test_class:
print 'Cannot find test class with name matching %s' % options.test
print 'Available tests: %s' % '\n'.join(
cl.Name() for cl in browser_test_classes)
return 1
# Create test context.
context = browser_test_context.TypTestContext()
for c in options.client_configs:
context.client_configs.append(c)
context.finder_options = ProcessCommandLineOptions(
test_class, options, extra_args)
context.test_class = test_class
test_times = None
if options.read_abbreviated_json_results_from:
with open(options.read_abbreviated_json_results_from, 'r') as f:
abbr_results = json.load(f)
test_times = abbr_results.get('times')
tests_to_run = LoadTestCasesToBeRun(
test_class=test_class, finder_options=context.finder_options,
filter_regex_str=options.test_filter,
filter_tests_after_sharding=options.filter_tests_after_sharding,
total_shards=options.total_shards, shard_index=options.shard_index,
test_times=test_times,
debug_shard_distributions=options.debug_shard_distributions)
for t in tests_to_run:
context.test_case_ids_to_run.add(t.id())
context.Freeze()
browser_test_context._global_test_context = context
# Setup typ runner.
runner = typ.Runner()
runner.context = context
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.args.jobs = options.jobs
runner.args.metadata = options.metadata
runner.args.passthrough = options.passthrough
runner.args.path = options.path
runner.args.repeat = options.repeat
runner.args.retry_limit = options.retry_limit
runner.args.test_results_server = options.test_results_server
runner.args.test_type = options.test_type
runner.args.top_level_dir = options.top_level_dir
runner.args.write_full_results_to = options.write_full_results_to
runner.args.write_trace_to = options.write_trace_to
runner.args.list_only = options.list_only
runner.classifier = _GetClassifier(options)
runner.args.suffixes = TEST_SUFFIXES
# Since sharding logic is handled by browser_test_runner harness by passing
# browser_test_context.test_case_ids_to_run to subprocess to indicate test
# cases to be run, we explicitly disable sharding logic in typ.
runner.args.total_shards = 1
runner.args.shard_index = 0
runner.args.timing = True
runner.args.verbose = options.verbose
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
finally:
if (options.write_full_results_to and
os.path.exists(options.write_full_results_to) and
not_using_typ_expectation):
# Set expectation of all skipped tests to skip to keep the test behavior
# the same as when typ doesn't support test expectation.
# (also see crbug.com/904019) for why this work around is needed)
# TODO(crbug.com/698902): remove this once gpu tests are converted to use
# typ's expectation.
_SetSkippedTestExpectationsToSkip(options.write_full_results_to)
return ret
