def main(args):
host = typ.Host()
runner = typ.Runner(host)
parser = ArgumentParser(host)
parser.prog = os.path.basename(sys.argv[0])
parser.description = __doc__
parser.formatter_class = argparse.RawDescriptionHelpFormatter
runner.parse_args(
parser=parser,
argv=args,
isolate=['installer_test.*'], # InstallerTest must be serialized.
top_level_dir=CUR_DIR,
retry_limit=3, # Retry failures by default since the tests are flaky.
)
if parser.exit_status is not None:
return parser.exit_status
# Stuff args into environment vars for use by child procs.
_prepare_env_for_subprocesses(parser, runner.args)
try:
return runner.run()[0]
except KeyboardInterrupt:
self.print_("interrupted, exiting", stream=self.host.stderr)
return 130
def main(argv):
"""Runs Blink bindings IDL compiler on test IDL files and compares the
results with reference files.
Please execute the script whenever changes are made to the compiler
(this is automatically done as a presubmit script),
and submit changes to the test results in the same patch.
This makes it easier to track and review changes in generated code.
"""
argument_parser = create_argument_parser()
# First, run bindings unit tests.
runner = typ.Runner()
runner.parse_args(argument_parser, argv[1:])
if argument_parser.exit_status is not None:
return argument_parser.exit_status
args = runner.args
args.top_level_dir = path_finder.get_bindings_scripts_dir()
if not args.skip_unit_tests:
return_code, _, _ = runner.run()
if return_code != 0:
return return_code
# Now run the bindings end-to-end tests.
if args.skip_reference_tests:
return 0
return run_bindings_tests(args.reset_results, args.verbose,
args.suppress_diff)
def Run(self, args):
runner = typ.Runner()
if self.stream:
runner.host.stdout = self.stream
if args.no_browser:
possible_browser = None
platform = platform_module.GetHostPlatform()
else:
possible_browser = browser_finder.FindBrowser(args)
platform = possible_browser.platform
# Telemetry seems to overload the system if we run one test per core,
# so we scale things back a fair amount. Many of the telemetry tests
# are long-running, so there's a limit to how much parallelism we
# can effectively use for now anyway.
#
# It should be possible to handle multiple devices if we adjust the
# browser_finder code properly, but for now we only handle one on ChromeOS.
if platform.GetOSName() == 'chromeos':
runner.args.jobs = 1
elif platform.GetOSName() == 'android':
runner.args.jobs = len(device_finder.GetDevicesMatchingOptions(args))
print 'Running tests with %d Android device(s).' % runner.args.jobs
elif platform.GetOSVersionName() == 'xp':
# For an undiagnosed reason, XP falls over with more parallelism.
# See crbug.com/388256
runner.args.jobs = max(int(args.jobs) // 4, 1)
else:
runner.args.jobs = max(int(args.jobs) // 2, 1)
runner.args.metadata = args.metadata
runner.args.passthrough = args.passthrough
runner.args.path = args.path
runner.args.retry_limit = args.retry_limit
runner.args.test_results_server = args.test_results_server
runner.args.test_type = args.test_type
runner.args.top_level_dir = args.top_level_dir
runner.args.write_full_results_to = args.write_full_results_to
runner.args.write_trace_to = args.write_trace_to
runner.args.list_only = args.list_only
runner.args.path.append(util.GetUnittestDataDir())
# Always print out these info for the ease of debugging.
runner.args.timing = True
runner.args.verbose = 3
runner.classifier = GetClassifier(args, possible_browser)
runner.context = args
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
return ret
def main():
parser = typ.ArgumentParser()
parser.add_argument('--build-dir',
help='Specifies chromium build directory.')
parser.add_argument('--target-gen-dir')
runner = typ.Runner()
# Set this when using context that will be passed to tests.
runner.win_multiprocessing = typ.WinMultiprocessing.importable
runner.parse_args(parser,
argv=None,
tests=[path_finder.get_integration_tests_dir()])
# setup logging level
if runner.args.verbose > 1:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(level=level)
# copy dynamically generated updater version_info.py from
# target gen directory to
# //chrome/updater/test/integration_tests/updater so that
# it can be imported as a module during test runs.
target_gen_dir_abs_path = os.path.abspath(runner.args.target_gen_dir)
version_file_path = os.path.join(target_gen_dir_abs_path, 'gen', 'chrome',
'updater', 'version_info.py')
if os.path.exists(version_file_path):
dest = os.path.join(path_finder.get_integration_tests_dir(), 'updater',
'version_info.py')
copy_file(version_file_path, dest)
else:
logging.info('File not found: %s' % version_file_path)
return -1
# copy dynamically generated updater branding_info.py from
# target gen directory to
# //chrome/updater/test/integration_tests/updater so that
# it can be imported as a module during test runs.
branding_file_path = os.path.join(target_gen_dir_abs_path, 'gen', 'chrome',
'updater', 'branding_info.py')
if os.path.exists(branding_file_path):
dest = os.path.join(path_finder.get_integration_tests_dir(), 'updater',
'branding_info.py')
copy_file(branding_file_path, dest)
else:
logging.info('File not found: %s' % branding_file_path)
return -2
runner.context = Context(runner.args.build_dir)
return runner.run()[0]
def main(argv):
argument_parser = create_argument_parser()
runner = typ.Runner()
runner.parse_args(argument_parser, argv[1:])
runner.args.top_level_dirs = [os.path.dirname(__file__)]
runner.context = runner.args
# Needs to be set to enable customizing runner.context
runner.win_multiprocessing = typ.WinMultiprocessing.importable
return_code, _, _ = runner.run()
return return_code
def main(argv):
devil_chromium.Initialize()
argument_parser = create_argument_parser()
runner = typ.Runner()
runner.parse_args(argument_parser, argv[1:])
if argument_parser.exit_status is not None:
return argument_parser.exit_status
runner.args.top_level_dirs = [os.path.dirname(__file__)]
runner.context = runner.args
# Needs to be set to enable customizing runner.context
runner.win_multiprocessing = typ.WinMultiprocessing.importable
return_code, _, _ = runner.run()
return return_code
def Run(self, args):
possible_browser = browser_finder.FindBrowser(args)
runner = typ.Runner()
if self.stream:
runner.host.stdout = self.stream
# Telemetry seems to overload the system if we run one test per core,
# so we scale things back a fair amount. Many of the telemetry tests
# are long-running, so there's a limit to how much parallelism we
# can effectively use for now anyway.
#
# It should be possible to handle multiple devices if we adjust
# the browser_finder code properly, but for now we only handle the one
# on Android and ChromeOS.
if possible_browser.platform.GetOSName() in ('android', 'chromeos'):
runner.args.jobs = 1
else:
runner.args.jobs = max(int(args.jobs) // 4, 1)
runner.args.metadata = args.metadata
runner.args.passthrough = args.passthrough
runner.args.path = args.path
runner.args.retry_limit = args.retry_limit
runner.args.test_results_server = args.test_results_server
runner.args.test_type = args.test_type
runner.args.timing = args.timing
runner.args.top_level_dir = args.top_level_dir
runner.args.verbose = args.verbosity
runner.args.write_full_results_to = args.write_full_results_to
runner.args.write_trace_to = args.write_trace_to
runner.args.path.append(util.GetUnittestDataDir())
runner.classifier = GetClassifier(args, possible_browser)
runner.context = args
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
return ret
def main(argv):
argument_parser = create_argument_parser()
# Run bindings unit tests.
runner = typ.Runner()
runner.parse_args(argument_parser, argv[1:])
if argument_parser.exit_status is not None:
return argument_parser.exit_status
args = runner.args
args.top_level_dirs = [
path_finder.get_bindings_scripts_dir(),
path_finder.get_build_scripts_dir(),
]
if not args.skip_unit_tests:
return_code, _, _ = runner.run()
if return_code != 0:
return return_code
return 0
def RunTests(args):
parser = _CreateTestArgParsers()
try:
options, extra_args = parser.parse_known_args(args)
except arg_parser._Bailout:
PrintTelemetryHelp()
return parser.exit_status
binary_manager.InitDependencyManager(options.client_configs)
for start_dir in options.start_dirs:
modules_to_classes = discover.DiscoverClasses(
start_dir,
options.top_level_dir,
base_class=serially_executed_browser_test_case.
SeriallyExecutedBrowserTestCase)
browser_test_classes = modules_to_classes.values()
_ValidateDistinctNames(browser_test_classes)
test_class = None
for cl in browser_test_classes:
if cl.Name() == options.test:
test_class = cl
break
if not test_class:
print('Cannot find test class with name matching %s' % options.test)
print('Available tests: %s' % '\n'.join(
cl.Name() for cl in browser_test_classes))
return 1
test_class._typ_runner = typ_runner = typ.Runner()
# Create test context.
typ_runner.context = browser_test_context.TypTestContext()
for c in options.client_configs:
typ_runner.context.client_configs.append(c)
typ_runner.context.finder_options = ProcessCommandLineOptions(
test_class, options, extra_args)
typ_runner.context.test_class = test_class
typ_runner.context.expectations_files = options.expectations_files
test_times = None
if options.read_abbreviated_json_results_from:
with open(options.read_abbreviated_json_results_from, 'r') as f:
abbr_results = json.load(f)
test_times = abbr_results.get('times')
# Setup typ.Runner instance.
typ_runner.args.all = options.all
typ_runner.args.expectations_files = options.expectations_files
typ_runner.args.jobs = options.jobs
typ_runner.args.list_only = options.list_only
typ_runner.args.metadata = options.metadata
typ_runner.args.passthrough = options.passthrough
typ_runner.args.path = options.path
typ_runner.args.quiet = options.quiet
typ_runner.args.repeat = options.repeat
typ_runner.args.repository_absolute_path = options.repository_absolute_path
typ_runner.args.retry_limit = options.retry_limit
typ_runner.args.retry_only_retry_on_failure_tests = (
options.retry_only_retry_on_failure_tests)
typ_runner.args.skip = options.skip
typ_runner.args.suffixes = TEST_SUFFIXES
typ_runner.args.tags = options.tags
typ_runner.args.test_name_prefix = options.test_name_prefix
typ_runner.args.test_filter = options.test_filter
typ_runner.args.test_results_server = options.test_results_server
typ_runner.args.test_type = options.test_type
typ_runner.args.top_level_dir = options.top_level_dir
typ_runner.args.write_full_results_to = options.write_full_results_to
typ_runner.args.write_trace_to = options.write_trace_to
typ_runner.args.disable_resultsink = options.disable_resultsink
typ_runner.classifier = _GetClassifier(typ_runner)
typ_runner.path_delimiter = test_class.GetJSONResultsDelimiter()
typ_runner.setup_fn = _SetUpProcess
typ_runner.teardown_fn = _TearDownProcess
tests_to_run = LoadTestCasesToBeRun(
test_class=test_class, finder_options=typ_runner.context.finder_options,
filter_tests_after_sharding=options.filter_tests_after_sharding,
total_shards=options.total_shards, shard_index=options.shard_index,
test_times=test_times,
debug_shard_distributions=options.debug_shard_distributions,
typ_runner=typ_runner)
for t in tests_to_run:
typ_runner.context.test_case_ids_to_run.add(t.id())
typ_runner.context.Freeze()
browser_test_context._global_test_context = typ_runner.context
# several class level variables are set for GPU tests when
# LoadTestCasesToBeRun is called. Functions line ExpectationsFiles and
# GenerateTags which use these variables should be called after
# LoadTestCasesToBeRun
test_class_expectations_files = test_class.ExpectationsFiles()
# all file paths in test_class_expectations-files must be absolute
assert all(os.path.isabs(path) for path in test_class_expectations_files)
typ_runner.args.expectations_files.extend(
test_class_expectations_files)
typ_runner.args.ignored_tags.extend(test_class.IgnoredTags())
# Since sharding logic is handled by browser_test_runner harness by passing
# browser_test_context.test_case_ids_to_run to subprocess to indicate test
# cases to be run, we explicitly disable sharding logic in typ.
typ_runner.args.total_shards = 1
typ_runner.args.shard_index = 0
typ_runner.args.timing = True
typ_runner.args.verbose = options.verbose
typ_runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = typ_runner.run()
except KeyboardInterrupt:
print("interrupted, exiting", file=sys.stderr)
ret = 130
return ret
def Run(self, args):
runner = typ.Runner()
if self.stream:
runner.host.stdout = self.stream
if args.no_browser:
possible_browser = None
platform = platform_module.GetHostPlatform()
else:
possible_browser = browser_finder.FindBrowser(args)
platform = possible_browser.platform
fetch_reference_chrome_binary = False
# Fetch all binaries needed by telemetry before we run the benchmark.
if possible_browser and possible_browser.browser_type == 'reference':
fetch_reference_chrome_binary = True
binary_manager.FetchBinaryDependencies(
platform, args.client_configs, fetch_reference_chrome_binary)
# Telemetry seems to overload the system if we run one test per core,
# so we scale things back a fair amount. Many of the telemetry tests
# are long-running, so there's a limit to how much parallelism we
# can effectively use for now anyway.
#
# It should be possible to handle multiple devices if we adjust the
# browser_finder code properly, but for now we only handle one on ChromeOS.
if platform.GetOSName() == 'chromeos':
runner.args.jobs = 1
elif platform.GetOSName() == 'android':
android_devs = android_device.FindAllAvailableDevices(args)
runner.args.jobs = len(android_devs)
if runner.args.jobs == 0:
raise RuntimeError("No Android device found")
print 'Running tests with %d Android device(s).' % runner.args.jobs
elif platform.GetOSVersionName() == 'xp':
# For an undiagnosed reason, XP falls over with more parallelism.
# See crbug.com/388256
runner.args.jobs = max(int(args.jobs) // 4, 1)
else:
runner.args.jobs = max(int(args.jobs) // 2, 1)
runner.args.skip = args.skip
runner.args.metadata = args.metadata
runner.args.passthrough = args.passthrough
runner.args.path = args.path
runner.args.retry_limit = args.retry_limit
runner.args.test_results_server = args.test_results_server
runner.args.test_type = args.test_type
runner.args.top_level_dirs = args.top_level_dirs
runner.args.write_full_results_to = args.write_full_results_to
runner.args.write_trace_to = args.write_trace_to
runner.args.repeat = args.repeat
runner.args.list_only = args.list_only
runner.args.shard_index = args.shard_index
runner.args.total_shards = args.total_shards
runner.args.path.append(util.GetUnittestDataDir())
# Standard verbosity will only emit output on test failure. Higher verbosity
# levels spam the output with logging, making it very difficult to figure
# out what's going on when digging into test failures.
runner.args.timing = True
runner.args.verbose = 1
runner.classifier = GetClassifier(args, possible_browser)
runner.context = args
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
return ret
def RunTests(args):
parser = _CreateTestArgParsers()
try:
options, extra_args = parser.parse_known_args(args)
except arg_parser._Bailout:
return parser.exit_status
binary_manager.InitDependencyManager(options.client_configs)
for start_dir in options.start_dirs:
modules_to_classes = discover.DiscoverClasses(
start_dir,
options.top_level_dir,
base_class=serially_executed_browser_test_case.
SeriallyExecutedBrowserTestCase)
browser_test_classes = modules_to_classes.values()
_ValidateDistinctNames(browser_test_classes)
test_class = None
for cl in browser_test_classes:
if cl.Name() == options.test:
test_class = cl
break
if not test_class:
print 'Cannot find test class with name matching %s' % options.test
print 'Available tests: %s' % '\n'.join(cl.Name()
for cl in browser_test_classes)
return 1
# Create test context.
context = browser_test_context.TypTestContext()
for c in options.client_configs:
context.client_configs.append(c)
context.finder_options = ProcessCommandLineOptions(test_class, options,
extra_args)
context.test_class = test_class
test_times = None
if options.read_abbreviated_json_results_from:
with open(options.read_abbreviated_json_results_from, 'r') as f:
abbr_results = json.load(f)
test_times = abbr_results.get('times')
tests_to_run = LoadTestCasesToBeRun(
test_class=test_class,
finder_options=context.finder_options,
filter_regex_str=options.test_filter,
filter_tests_after_sharding=options.filter_tests_after_sharding,
total_shards=options.total_shards,
shard_index=options.shard_index,
test_times=test_times,
debug_shard_distributions=options.debug_shard_distributions)
for t in tests_to_run:
context.test_case_ids_to_run.add(t.id())
context.Freeze()
browser_test_context._global_test_context = context
# Setup typ runner.
runner = typ.Runner()
runner.context = context
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.args.jobs = options.jobs
runner.args.metadata = options.metadata
runner.args.passthrough = options.passthrough
runner.args.path = options.path
runner.args.retry_limit = options.retry_limit
runner.args.test_results_server = options.test_results_server
runner.args.test_type = options.test_type
runner.args.top_level_dir = options.top_level_dir
runner.args.write_full_results_to = options.write_full_results_to
runner.args.write_trace_to = options.write_trace_to
runner.args.list_only = options.list_only
runner.classifier = _GetClassifier(options)
runner.args.suffixes = TEST_SUFFIXES
# Since sharding logic is handled by browser_test_runner harness by passing
# browser_test_context.test_case_ids_to_run to subprocess to indicate test
# cases to be run, we explicitly disable sharding logic in typ.
runner.args.total_shards = 1
runner.args.shard_index = 0
runner.args.timing = True
runner.args.verbose = options.verbose
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
return ret
def RunTests(args):
parser = _CreateTestArgParsers()
try:
options, extra_args = parser.parse_known_args(args)
except arg_parser._Bailout:
PrintTelemetryHelp()
return parser.exit_status
binary_manager.InitDependencyManager(options.client_configs)
not_using_typ_expectation = False
if options.expectations_files:
parser.error('--expectation-files flag is not supported yet.')
else:
not_using_typ_expectation = True
for start_dir in options.start_dirs:
modules_to_classes = discover.DiscoverClasses(
start_dir,
options.top_level_dir,
base_class=serially_executed_browser_test_case.
SeriallyExecutedBrowserTestCase)
browser_test_classes = modules_to_classes.values()
_ValidateDistinctNames(browser_test_classes)
test_class = None
for cl in browser_test_classes:
if cl.Name() == options.test:
test_class = cl
break
if not test_class:
print 'Cannot find test class with name matching %s' % options.test
print 'Available tests: %s' % '\n'.join(
cl.Name() for cl in browser_test_classes)
return 1
# Create test context.
context = browser_test_context.TypTestContext()
for c in options.client_configs:
context.client_configs.append(c)
context.finder_options = ProcessCommandLineOptions(
test_class, options, extra_args)
context.test_class = test_class
test_times = None
if options.read_abbreviated_json_results_from:
with open(options.read_abbreviated_json_results_from, 'r') as f:
abbr_results = json.load(f)
test_times = abbr_results.get('times')
tests_to_run = LoadTestCasesToBeRun(
test_class=test_class, finder_options=context.finder_options,
filter_regex_str=options.test_filter,
filter_tests_after_sharding=options.filter_tests_after_sharding,
total_shards=options.total_shards, shard_index=options.shard_index,
test_times=test_times,
debug_shard_distributions=options.debug_shard_distributions)
for t in tests_to_run:
context.test_case_ids_to_run.add(t.id())
context.Freeze()
browser_test_context._global_test_context = context
# Setup typ runner.
runner = typ.Runner()
runner.context = context
runner.setup_fn = _SetUpProcess
runner.teardown_fn = _TearDownProcess
runner.args.jobs = options.jobs
runner.args.metadata = options.metadata
runner.args.passthrough = options.passthrough
runner.args.path = options.path
runner.args.repeat = options.repeat
runner.args.retry_limit = options.retry_limit
runner.args.test_results_server = options.test_results_server
runner.args.test_type = options.test_type
runner.args.top_level_dir = options.top_level_dir
runner.args.write_full_results_to = options.write_full_results_to
runner.args.write_trace_to = options.write_trace_to
runner.args.list_only = options.list_only
runner.classifier = _GetClassifier(options)
runner.args.suffixes = TEST_SUFFIXES
# Since sharding logic is handled by browser_test_runner harness by passing
# browser_test_context.test_case_ids_to_run to subprocess to indicate test
# cases to be run, we explicitly disable sharding logic in typ.
runner.args.total_shards = 1
runner.args.shard_index = 0
runner.args.timing = True
runner.args.verbose = options.verbose
runner.win_multiprocessing = typ.WinMultiprocessing.importable
try:
ret, _, _ = runner.run()
except KeyboardInterrupt:
print >> sys.stderr, "interrupted, exiting"
ret = 130
finally:
if (options.write_full_results_to and
os.path.exists(options.write_full_results_to) and
not_using_typ_expectation):
# Set expectation of all skipped tests to skip to keep the test behavior
# the same as when typ doesn't support test expectation.
# (also see crbug.com/904019) for why this work around is needed)
# TODO(crbug.com/698902): remove this once gpu tests are converted to use
# typ's expectation.
_SetSkippedTestExpectationsToSkip(options.write_full_results_to)
return ret
