def ProcessJavaTestOptions(options, error_func):
"""Processes options/arguments and populates |options| with defaults."""
if options.java_only and options.python_only:
error_func('Options java_only (-j) and python_only (-p) '
'are mutually exclusive.')
options.run_java_tests = True
options.run_python_tests = True
if options.java_only:
options.run_python_tests = False
elif options.python_only:
options.run_java_tests = False
if not options.python_test_root:
options.run_python_tests = False
if options.annotation_str:
options.annotations = options.annotation_str.split(',')
elif options.test_filter:
options.annotations = []
else:
options.annotations = [
'Smoke', 'SmallTest', 'MediumTest', 'LargeTest', 'EnormousTest'
]
if options.exclude_annotation_str:
options.exclude_annotations = options.exclude_annotation_str.split(',')
else:
options.exclude_annotations = []
if not options.keep_test_server_ports:
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
def main(argv):
parser = optparse.OptionParser()
parser.add_option('-s',
'--steps',
help='A JSON file containing all the steps to be '
'sharded.')
parser.add_option('-p',
'--print_results',
help='Only prints the results for the previously '
'executed step, do not run it again.')
options, urls = parser.parse_args(argv)
if options.print_results:
return _PrintStepOutput(options.print_results)
# At this point, we should kill everything that may have been left over from
# previous runs.
_KillPendingServers()
# Reset the test port allocation. It's important to do it before starting
# to dispatch any step.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
# Sort the devices so that we'll try to always run a step in the same device.
devices = sorted(android_commands.GetAttachedDevices())
if not devices:
print 'You must attach a device'
return 1
with file(options.steps, 'r') as f:
steps = json.load(f)
return _RunShardedSteps(steps, devices)
def DispatchInstrumentationTests(options):
"""Dispatches the Java and Python instrumentation tests, sharding if possible.
Uses the logging module to print the combined final results and
summary of the Java and Python tests. If the java_only option is set, only
the Java tests run. If the python_only option is set, only the python tests
run. If neither are set, run both Java and Python tests.
Args:
options: command-line options for running the Java and Python tests.
Returns:
An integer representing the number of failing tests.
"""
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
start_date = int(time.time() * 1000)
java_results = TestResults()
python_results = TestResults()
if options.run_java_tests:
java_results = run_java_tests.DispatchJavaTests(
options, [
apk_info.ApkInfo(options.test_apk_path,
options.test_apk_jar_path)
])
if options.run_python_tests:
python_results = run_python_tests.DispatchPythonTests(options)
all_results, summary_string, num_failing = SummarizeResults(
java_results, python_results, options.annotation, options.build_type)
return num_failing
def _KillPendingServers():
for retry in range(5):
for server in ['lighttpd', 'web-page-replay']:
pids = [p.pid for p in psutil.process_iter() if server in p.name]
for pid in pids:
try:
logging.warning('Killing %s %s', server, pid)
os.kill(pid, signal.SIGQUIT)
except Exception as e:
logging.warning('Failed killing %s %s %s', server, pid, e)
# Restart the adb server with taskset to set a single CPU affinity.
cmd_helper.RunCmd(['adb', 'kill-server'])
cmd_helper.RunCmd(['taskset', '-c', '0', 'adb', 'start-server'])
cmd_helper.RunCmd(['taskset', '-c', '0', 'adb', 'root'])
i = 1
while not android_commands.GetAttachedDevices():
time.sleep(i)
i *= 2
if i > 10:
break
# Reset the test port allocation. It's important to do it before starting
# to dispatch any step.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
forwarder.Forwarder.UseMultiprocessing()
def _RunATestSuite(options, suite_name):
"""Run a single test suite.
Helper for Dispatch() to allow stop/restart of the emulator across
test bundles. If using the emulator, we start it on entry and stop
it on exit.
Args:
options: options for running the tests.
suite_name: name of the test suite being run.
Returns:
0 if successful, number of failing tests otherwise.
"""
step_name = os.path.basename(options.test_suite).replace('-debug.apk', '')
attached_devices = []
buildbot_emulators = []
if options.use_emulator:
buildbot_emulators = emulator.LaunchEmulators(options.emulator_count,
wait_for_boot=True)
attached_devices = [e.device for e in buildbot_emulators]
elif options.test_device:
attached_devices = [options.test_device]
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
if options.gtest_filter:
logging.warning('Sharding is not possible with these configurations.')
attached_devices = [attached_devices[0]]
sharder = test_sharder.TestSharder(attached_devices, options.test_suite,
options.gtest_filter,
options.test_arguments, options.timeout,
options.cleanup_test_files,
options.tool, options.build_type,
options.webkit)
test_results = sharder.RunShardedTests()
test_results.LogFull(test_type='Unit test',
test_package=suite_name,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
test_results.PrintAnnotation()
for buildbot_emulator in buildbot_emulators:
buildbot_emulator.Shutdown()
return len(test_results.GetAllBroken())
def Dispatch(options):
attached_devices = []
if options.test_device:
attached_devices = [options.test_device]
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(),
options.build_type)
options.test_suite = os.path.join(
test_suite_dir, 'apks', constants.BROWSERTEST_SUITE_NAME + '.apk')
# Constructs a new TestRunner with the current options.
def RunnerFactory(device, shard_index):
return test_runner.TestRunner(device, options.test_suite,
options.test_arguments, options.timeout,
options.cleanup_test_files, options.tool,
options.build_type, options.webkit,
constants.BROWSERTEST_TEST_PACKAGE_NAME,
constants.BROWSERTEST_TEST_ACTIVITY_NAME,
constants.BROWSERTEST_COMMAND_LINE_FILE)
# Get tests and split them up based on the number of devices.
if options.gtest_filter:
all_tests = [t for t in options.gtest_filter.split(':') if t]
else:
all_enabled = gtest_dispatch.GetAllEnabledTests(
RunnerFactory, attached_devices)
all_tests = _FilterTests(all_enabled)
# Run tests.
# TODO(nileshagrawal): remove this abnormally long setup timeout once fewer
# files are pushed to the devices for content_browsertests: crbug.com/138275
setup_timeout = 20 * 60 # 20 minutes
test_results = shard.ShardAndRunTests(RunnerFactory,
attached_devices,
all_tests,
options.build_type,
setup_timeout=setup_timeout,
test_timeout=None,
num_retries=options.num_retries)
report_results.LogFull(results=test_results,
test_type='Unit test',
test_package=constants.BROWSERTEST_SUITE_NAME,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
report_results.PrintAnnotation(test_results)
def RunTestsCommand(command, options, args, option_parser):
"""Checks test type and dispatches to the appropriate function.
Args:
command: String indicating the command that was received to trigger
this function.
options: optparse options dictionary.
args: List of extra args from optparse.
option_parser: optparse.OptionParser object.
Returns:
Integer indicated exit code.
Raises:
Exception: Unknown command name passed in, or an exception from an
individual test runner.
"""
# Check for extra arguments
if len(args) > 2 and command != 'perf':
option_parser.error('Unrecognized arguments: %s' %
(' '.join(args[2:])))
return constants.ERROR_EXIT_CODE
if command == 'perf':
if ((options.single_step and len(args) <= 2)
or (not options.single_step and len(args) > 2)):
option_parser.error('Unrecognized arguments: %s' %
(' '.join(args)))
return constants.ERROR_EXIT_CODE
ProcessCommonOptions(options)
devices = _GetAttachedDevices(options.test_device)
forwarder.Forwarder.RemoveHostLog()
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
if command == 'gtest':
return _RunGTests(options, devices)
elif command == 'linker':
return _RunLinkerTests(options, devices)
elif command == 'instrumentation':
return _RunInstrumentationTests(options, option_parser.error, devices)
elif command == 'uiautomator':
return _RunUIAutomatorTests(options, option_parser.error, devices)
elif command == 'junit':
return _RunJUnitTests(options, option_parser.error)
elif command == 'monkey':
return _RunMonkeyTests(options, option_parser.error, devices)
elif command == 'perf':
return _RunPerfTests(options, args, option_parser.error)
else:
raise Exception('Unknown test type.')
def RunTestsCommand(args, parser):
"""Checks test type and dispatches to the appropriate function.
Args:
args: argparse.Namespace object.
parser: argparse.ArgumentParser object.
Returns:
Integer indicated exit code.
Raises:
Exception: Unknown command name passed in, or an exception from an
individual test runner.
"""
command = args.command
ProcessCommonOptions(args)
if args.enable_platform_mode:
return RunTestsInPlatformMode(args, parser)
if command in constants.LOCAL_MACHINE_TESTS:
devices = []
else:
devices = _GetAttachedDevices(args.test_device)
forwarder.Forwarder.RemoveHostLog()
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
if command == 'gtest':
if args.suite_name[0] in gtest_test_instance.BROWSER_TEST_SUITES:
return RunTestsInPlatformMode(args, parser)
return _RunGTests(args, devices)
elif command == 'linker':
return _RunLinkerTests(args, devices)
elif command == 'instrumentation':
return _RunInstrumentationTests(args, devices)
elif command == 'uiautomator':
return _RunUIAutomatorTests(args, devices)
elif command == 'junit':
return _RunJUnitTests(args)
elif command == 'monkey':
return _RunMonkeyTests(args, devices)
elif command == 'perf':
return _RunPerfTests(args)
elif command == 'python':
return _RunPythonTests(args)
else:
raise Exception('Unknown test type.')
def Dispatch(options):
attached_devices = []
if options.test_device:
attached_devices = [options.test_device]
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
test_suite_dir = os.path.join(cmd_helper.OutDirectory.get(),
options.build_type)
options.test_suite = os.path.join(test_suite_dir, 'apks',
CONTENT_BROWSERTEST_SUITENAME + '.apk')
options.test_arguments = '--single_process %s' % options.test_arguments
# Constructs a new TestRunner with the current options.
def RunnerFactory(device):
return test_runner.TestRunner(device, options.test_suite,
options.test_arguments, options.timeout,
options.cleanup_test_files, options.tool,
options.build_type, options.webkit,
constants.BROWSERTEST_TEST_PACKAGE_NAME,
constants.BROWSERTEST_TEST_ACTIVITY_NAME,
constants.BROWSERTEST_COMMAND_LINE_FILE)
# Get tests and split them up based on the number of devices.
if options.gtest_filter:
all_tests = [t for t in options.gtest_filter.split(':') if t]
else:
all_tests = gtest_dispatch.GetAllEnabledTests(RunnerFactory,
attached_devices)
# Run tests.
test_results = shard.ShardAndRunTests(RunnerFactory, attached_devices,
all_tests, options.build_type)
test_results.LogFull(test_type='Unit test',
test_package=CONTENT_BROWSERTEST_SUITENAME,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
test_results.PrintAnnotation()
def Setup(test_options, devices):
"""Create the test runner factory and tests.
Args:
test_options: A GTestOptions object.
devices: A list of attached devices.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
test_package = test_package_apk.TestPackageApk(test_options.suite_name)
if not os.path.exists(test_package.suite_path):
test_package = test_package_exe.TestPackageExecutable(
test_options.suite_name)
if not os.path.exists(test_package.suite_path):
raise Exception(
'Did not find %s target. Ensure it has been built.' %
test_options.suite_name)
logging.warning('Found target %s', test_package.suite_path)
_GenerateDepsDirUsingIsolate(test_options.suite_name)
# Constructs a new TestRunner with the current options.
def TestRunnerFactory(device, shard_index):
return test_runner.TestRunner(test_options, device, test_package)
tests = _GetTestsFromDevice(TestRunnerFactory, devices)
if test_options.run_disabled:
test_options = test_options._replace(
test_arguments=('%s --gtest_also_run_disabled_tests' %
test_options.test_arguments))
else:
tests = _FilterDisabledTests(tests, test_options.suite_name,
bool(test_options.gtest_filter))
if test_options.gtest_filter:
tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
# Coalesce unit tests into a single test per device
if test_options.suite_name != 'content_browsertests':
num_devices = len(devices)
tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
tests = [t for t in tests if t]
return (TestRunnerFactory, tests)
def Setup(test_options):
"""Create the test runner factory and tests.
Args:
test_options: A GTestOptions object.
Returns:
A tuple of (TestRunnerFactory, tests).
"""
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
test_package = test_package_apk.TestPackageApk(test_options.suite_name,
test_options.build_type)
if not os.path.exists(test_package.suite_path):
test_package = test_package_exe.TestPackageExecutable(
test_options.suite_name, test_options.build_type)
if not os.path.exists(test_package.suite_path):
raise Exception(
'Did not find %s target. Ensure it has been built.' %
test_options.suite_name)
logging.warning('Found target %s', test_package.suite_path)
_GenerateDepsDirUsingIsolate(test_options.suite_name,
test_options.build_type)
# Constructs a new TestRunner with the current options.
def TestRunnerFactory(device, shard_index):
return test_runner.TestRunner(test_options, device, test_package)
attached_devices = android_commands.GetAttachedDevices()
tests = _GetTestsFiltered(test_options.suite_name,
test_options.gtest_filter, TestRunnerFactory,
attached_devices)
# Coalesce unit tests into a single test per device
if test_options.suite_name != 'content_browsertests':
num_devices = len(attached_devices)
tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
tests = [t for t in tests if t]
return (TestRunnerFactory, tests)
def DispatchInstrumentationTests(options):
"""Dispatches the Java and Python instrumentation tests, sharding if possible.
Uses the logging module to print the combined final results and
summary of the Java and Python tests. If the java_only option is set, only
the Java tests run. If the python_only option is set, only the python tests
run. If neither are set, run both Java and Python tests.
Args:
options: command-line options for running the Java and Python tests.
Returns:
An integer representing the number of broken tests.
"""
if not options.keep_test_server_ports:
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
start_date = int(time.time() * 1000)
java_results = TestResults()
python_results = TestResults()
if options.run_java_tests:
java_results = run_java_tests.DispatchJavaTests(
options, [
apk_info.ApkInfo(options.test_apk_path,
options.test_apk_jar_path)
])
if options.run_python_tests:
python_results = run_python_tests.DispatchPythonTests(options)
all_results = TestResults.FromTestResults([java_results, python_results])
all_results.LogFull(test_type='Instrumentation',
test_package=options.test_apk,
annotation=options.annotation,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
return len(all_results.GetAllBroken())
def ProcessJavaTestOptions(options, error_func):
"""Processes options/arguments and populates |options| with defaults."""
if options.annotation_str:
options.annotations = options.annotation_str.split(',')
elif options.test_filter:
options.annotations = []
else:
options.annotations = [
'Smoke', 'SmallTest', 'MediumTest', 'LargeTest', 'EnormousTest'
]
if options.exclude_annotation_str:
options.exclude_annotations = options.exclude_annotation_str.split(',')
else:
options.exclude_annotations = []
if not options.keep_test_server_ports:
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
def DispatchUIAutomatorTests(options):
"""Dispatches the UIAutomator tests, sharding if possible.
Uses the logging module to print the combined final results and
summary of the Java and Python tests. If the java_only option is set, only
the Java tests run. If the python_only option is set, only the python tests
run. If neither are set, run both Java and Python tests.
Args:
options: command-line options for running the Java and Python tests.
Returns:
An integer representing the number of broken tests.
"""
if not options.keep_test_server_ports:
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
all_results = base_test_result.TestRunResults()
if options.run_java_tests:
all_results.AddTestRunResults(dispatch.Dispatch(options))
if options.run_python_tests:
all_results.AddTestRunResults(
run_python_tests.DispatchPythonTests(options))
report_results.LogFull(results=all_results,
test_type='UIAutomator',
test_package=os.path.basename(options.test_jar),
annotation=options.annotation,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
return len(all_results.GetNotPass())
def _RunATestSuite(options):
"""Run a single test suite.
Helper for Dispatch() to allow stop/restart of the emulator across
test bundles. If using the emulator, we start it on entry and stop
it on exit.
Args:
options: options for running the tests.
Returns:
0 if successful, number of failing tests otherwise.
"""
step_name = os.path.basename(options.test_suite).replace('-debug.apk', '')
buildbot_report.PrintNamedStep(step_name)
attached_devices = []
buildbot_emulators = []
if options.use_emulator:
for n in range(options.emulator_count):
t = TimeProfile('Emulator launch %d' % n)
avd_name = None
if n > 0:
# Creates a temporary AVD for the extra emulators.
avd_name = 'run_tests_avd_%d' % n
buildbot_emulator = emulator.Emulator(avd_name,
options.fast_and_loose)
buildbot_emulator.Launch(kill_all_emulators=n == 0)
t.Stop()
buildbot_emulators.append(buildbot_emulator)
attached_devices.append(buildbot_emulator.device)
# Wait for all emulators to boot completed.
map(lambda buildbot_emulator: buildbot_emulator.ConfirmLaunch(True),
buildbot_emulators)
elif options.test_device:
attached_devices = [options.test_device]
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
buildbot_report.PrintError()
return 1
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
if options.performance_test or options.gtest_filter:
# These configuration can't be split in multiple devices.
attached_devices = [attached_devices[0]]
sharder = TestSharder(attached_devices, options.test_suite,
options.gtest_filter, options.test_arguments,
options.timeout, options.rebaseline,
options.performance_test, options.cleanup_test_files,
options.tool, options.log_dump,
options.fast_and_loose, options.build_type)
test_results = sharder.RunShardedTests()
for buildbot_emulator in buildbot_emulators:
buildbot_emulator.Shutdown()
# Another chance if we timed out? At this point It is safe(r) to
# run fast and loose since we just uploaded all the test data and
# binary.
if test_results.timed_out and options.repeat:
logging.critical('Timed out; repeating in fast_and_loose mode.')
options.fast_and_loose = True
options.repeat -= 1
logging.critical('Repeats left: ' + str(options.repeat))
return _RunATestSuite(options)
return len(test_results.failed)
def _RunATestSuite(options):
"""Run a single test suite.
Helper for Dispatch() to allow stop/restart of the emulator across
test bundles. If using the emulator, we start it on entry and stop
it on exit.
Args:
options: options for running the tests.
Returns:
0 if successful, number of failing tests otherwise.
"""
step_name = os.path.basename(options.test_suite).replace('-debug.apk', '')
buildbot_report.PrintNamedStep(step_name)
attached_devices = []
buildbot_emulators = []
if options.use_emulator:
for n in range(options.emulator_count):
t = time_profile.TimeProfile('Emulator launch %d' % n)
avd_name = None
if n > 0:
# Creates a temporary AVD for the extra emulators.
avd_name = 'run_tests_avd_%d' % n
buildbot_emulator = emulator.Emulator(avd_name, options.fast_and_loose)
buildbot_emulator.Launch(kill_all_emulators=n == 0)
t.Stop()
buildbot_emulators.append(buildbot_emulator)
attached_devices.append(buildbot_emulator.device)
# Wait for all emulators to boot completed.
map(lambda buildbot_emulator: buildbot_emulator.ConfirmLaunch(True),
buildbot_emulators)
elif options.test_device:
attached_devices = [options.test_device]
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
buildbot_report.PrintError()
return 1
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
if options.gtest_filter:
logging.warning('Sharding is not possible with these configurations.')
attached_devices = [attached_devices[0]]
sharder = TestSharder(
attached_devices,
options.test_suite,
options.gtest_filter,
options.test_arguments,
options.timeout,
options.cleanup_test_files,
options.tool,
options.log_dump,
options.fast_and_loose,
options.build_type,
options.webkit,
options.flakiness_dashboard_server)
test_results = sharder.RunShardedTests()
for buildbot_emulator in buildbot_emulators:
buildbot_emulator.Shutdown()
return len(test_results.failed)
def ResetTestServerPortAllocation():
return ports.ResetTestServerPortAllocation()
def _RunATestSuite(options, suite_name):
"""Run a single test suite.
Helper for Dispatch() to allow stop/restart of the emulator across
test bundles. If using the emulator, we start it on entry and stop
it on exit.
Args:
options: options for running the tests.
suite_name: name of the test suite being run.
Returns:
0 if successful, number of failing tests otherwise.
"""
step_name = os.path.basename(options.test_suite).replace('-debug.apk', '')
attached_devices = []
buildbot_emulators = []
if options.use_emulator:
buildbot_emulators = emulator.LaunchEmulators(options.emulator_count,
options.abi,
wait_for_boot=True)
attached_devices = [e.device for e in buildbot_emulators]
elif options.test_device:
attached_devices = [options.test_device]
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
# Constructs a new TestRunner with the current options.
def RunnerFactory(device, shard_index):
return test_runner.TestRunner(device, options.test_suite,
options.test_arguments, options.timeout,
options.cleanup_test_files, options.tool,
options.build_type, options.webkit,
constants.GTEST_TEST_PACKAGE_NAME,
constants.GTEST_TEST_ACTIVITY_NAME,
constants.GTEST_COMMAND_LINE_FILE)
# Get tests and split them up based on the number of devices.
if options.gtest_filter:
all_tests = [t for t in options.gtest_filter.split(':') if t]
else:
all_tests = GetAllEnabledTests(RunnerFactory, attached_devices)
num_devices = len(attached_devices)
tests = [':'.join(all_tests[i::num_devices]) for i in xrange(num_devices)]
tests = [t for t in tests if t]
# Run tests.
test_results = shard.ShardAndRunTests(RunnerFactory,
attached_devices,
tests,
options.build_type,
test_timeout=None,
num_retries=options.num_retries)
report_results.LogFull(results=test_results,
test_type='Unit test',
test_package=suite_name,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
report_results.PrintAnnotation(test_results)
for buildbot_emulator in buildbot_emulators:
buildbot_emulator.Shutdown()
return len(test_results.GetNotPass())
