def main(): parser = CommandParser() args, extra_cmd_args = parser.parse_known_args(sys.argv[1:]) logging.basicConfig(level=logging.INFO) with tempfile_ext.NamedTemporaryDirectory() as logcat_output_dir: test_cmd = [ os.path.join('bin', 'run_%s' % args.target), '--logcat-output-file', (args.logcat_output_file if args.logcat_output_file else os.path.join(logcat_output_dir, 'logcats')), '--target-devices-file', args.target_devices_file, '-v' ] with tempfile_ext.NamedTemporaryDirectory( prefix='tmp_android_logdog_wrapper') as temp_directory: if not os.path.exists(args.logdog_bin_cmd): logging.error( 'Logdog binary %s unavailable. Unable to create logdog client', args.logdog_bin_cmd) else: test_cmd += ['--upload-logcats-file'] streamserver_uri = 'unix:%s' % os.path.join( temp_directory, 'butler.sock') prefix = os.path.join('android', 'swarming', 'logcats', os.environ.get('SWARMING_TASK_ID')) # Call test_cmdline through logdog butler subcommand. test_cmd = [ args.logdog_bin_cmd, '-project', PROJECT, '-output', OUTPUT, '-prefix', prefix, '--service-account-json', SERVICE_ACCOUNT_JSON, '-coordinator-host', COORDINATOR_HOST, 'run', '-streamserver-uri', streamserver_uri, '--' ] + test_cmd test_cmd += extra_cmd_args test_proc = subprocess.Popen(test_cmd) with signal_handler.SignalHandler( signal.SIGTERM, CreateStopTestsMethod(test_proc)): result = test_proc.wait() return result
def RunTests(self): tests = self._GetTests() exit_now = threading.Event() @local_device_environment.handle_shard_failures def run_tests_on_device(dev, tests, results): for test in tests: if exit_now.isSet(): thread.exit() result = None rerun = None try: result, rerun = crash_handler.RetryOnSystemCrash( lambda d, t=test: self._RunTest(d, t), device=dev) if isinstance(result, base_test_result.BaseTestResult): results.AddResult(result) elif isinstance(result, list): results.AddResults(result) else: raise Exception('Unexpected result type: %s' % type(result).__name__) except Exception as e: # pylint: disable=broad-except if isinstance(tests, test_collection.TestCollection): rerun = test if (isinstance(e, device_errors.DeviceUnreachableError) or not isinstance(e, base_error.BaseError)): # If we get a device error but believe the device is still # reachable, attempt to continue using it. Otherwise, raise # the exception and terminate this run_tests_on_device call. raise finally: if isinstance(tests, test_collection.TestCollection): if rerun: tests.add(rerun) tests.test_completed() logging.info('Finished running tests on this device.') def stop_tests(_signum, _frame): logging.critical('Received SIGTERM. Stopping test execution.') exit_now.set() raise TestsTerminated() try: with signal_handler.SignalHandler(signal.SIGTERM, stop_tests): tries = 0 results = [] while tries < self._env.max_tries and tests: logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) logging.info('Will run %d tests on %d devices: %s', len(tests), len(self._env.devices), ', '.join(str(d) for d in self._env.devices)) for t in tests: logging.debug(' %s', t) try_results = base_test_result.TestRunResults() test_names = (self._GetUniqueTestName(t) for t in tests) try_results.AddResults( base_test_result.BaseTestResult( t, base_test_result.ResultType.NOTRUN) for t in test_names if not t.endswith('*')) try: if self._ShouldShard(): tc = test_collection.TestCollection( self._CreateShards(tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: self._env.parallel_devices.pMap( run_tests_on_device, tests, try_results).pGet(None) except TestsTerminated: for unknown_result in try_results.GetUnknown(): try_results.AddResult( base_test_result.BaseTestResult( unknown_result.GetName(), base_test_result.ResultType.TIMEOUT, log=_SIGTERM_TEST_LOG)) raise finally: results.append(try_results) tries += 1 tests = self._GetTestsToRetry(tests, try_results) logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries) if tests: logging.info('%d failed tests remain.', len(tests)) else: logging.info('All tests completed.') except TestsTerminated: pass return results
def main(): parser = CommandParser() args, extra_cmd_args = parser.parse_known_args(sys.argv[1:]) logging.basicConfig(level=logging.INFO) if args.target: test_cmd = [os.path.join('bin', 'run_%s' % args.target), '-v'] test_cmd += extra_cmd_args elif args.script: test_cmd = [args.script] test_cmd += extra_cmd_args else: test_cmd = extra_cmd_args test_env = dict(os.environ) logdog_cmd = [] with tempfile_ext.NamedTemporaryDirectory( prefix='tmp_android_logdog_wrapper') as temp_directory: if not os.path.exists(args.logdog_bin_cmd): logging.error( 'Logdog binary %s unavailable. Unable to create logdog client', args.logdog_bin_cmd) else: streamserver_uri = 'unix:%s' % os.path.join( temp_directory, 'butler.sock') prefix = os.path.join('android', 'swarming', 'logcats', os.environ.get('SWARMING_TASK_ID')) logdog_cmd = [ args.logdog_bin_cmd, '-project', PROJECT, '-output', OUTPUT, '-prefix', prefix, '--service-account-json', SERVICE_ACCOUNT_JSON, '-coordinator-host', COORDINATOR_HOST, 'serve', '-streamserver-uri', streamserver_uri ] test_env.update({ 'LOGDOG_STREAM_PROJECT': PROJECT, 'LOGDOG_STREAM_PREFIX': prefix, 'LOGDOG_STREAM_SERVER_PATH': streamserver_uri, 'LOGDOG_COORDINATOR_HOST': COORDINATOR_HOST, }) logdog_proc = None if logdog_cmd: logdog_proc = subprocess.Popen(logdog_cmd) with NoLeakingProcesses(logdog_proc): with NoLeakingProcesses(subprocess.Popen( test_cmd, env=test_env)) as test_proc: with signal_handler.SignalHandler( signal.SIGTERM, CreateStopTestsMethod(test_proc)): result = test_proc.wait() if logdog_proc: def logdog_stopped(): return logdog_proc.poll() is not None logdog_proc.terminate() timeout_retry.WaitFor( logdog_stopped, wait_period=1, max_tries=LOGDOG_TERMINATION_TIMEOUT) # If logdog_proc hasn't finished by this point, allow # NoLeakingProcesses to kill it. return result
def RunTestsInPlatformMode(args): def infra_error(message): logging.fatal(message) sys.exit(constants.INFRA_EXIT_CODE) if args.command not in _SUPPORTED_IN_PLATFORM_MODE: infra_error('%s is not yet supported in platform mode' % args.command) ### Set up sigterm handler. def unexpected_sigterm(_signum, _frame): infra_error('Received SIGTERM. Shutting down.') sigterm_handler = signal_handler.SignalHandler(signal.SIGTERM, unexpected_sigterm) ### Set up results handling. # TODO(jbudorick): Rewrite results handling. # all_raw_results is a list of lists of # base_test_result.TestRunResults objects. Each instance of # TestRunResults contains all test results produced by a single try, # while each list of TestRunResults contains all tries in a single # iteration. all_raw_results = [] # all_iteration_results is a list of base_test_result.TestRunResults # objects. Each instance of TestRunResults contains the last test # result for each test run in that iteration. all_iteration_results = [] @contextlib.contextmanager def noop(): yield json_writer = noop() if args.json_results_file: @contextlib.contextmanager def write_json_file(): try: yield finally: json_results.GenerateJsonResultsFile(all_raw_results, args.json_results_file) json_writer = write_json_file() ### Set up test objects. env = environment_factory.CreateEnvironment(args, infra_error) test_instance = test_instance_factory.CreateTestInstance(args, infra_error) test_run = test_run_factory.CreateTestRun(args, env, test_instance, infra_error) ### Run. with sigterm_handler, json_writer, env, test_instance, test_run: repetitions = (xrange(args.repeat + 1) if args.repeat >= 0 else itertools.count()) result_counts = collections.defaultdict( lambda: collections.defaultdict(int)) iteration_count = 0 for _ in repetitions: raw_results = test_run.RunTests() if not raw_results: continue all_raw_results.append(raw_results) iteration_results = base_test_result.TestRunResults() for r in reversed(raw_results): iteration_results.AddTestRunResults(r) all_iteration_results.append(iteration_results) iteration_count += 1 for r in iteration_results.GetAll(): result_counts[r.GetName()][r.GetType()] += 1 report_results.LogFull(results=iteration_results, test_type=test_instance.TestType(), test_package=test_run.TestPackage(), annotation=getattr(args, 'annotations', None), flakiness_server=getattr( args, 'flakiness_dashboard_server', None)) if args.break_on_failure and not iteration_results.DidRunPass(): break if iteration_count > 1: # display summary results # only display results for a test if at least one test did not pass all_pass = 0 tot_tests = 0 for test_name in result_counts: tot_tests += 1 if any(result_counts[test_name][x] for x in (base_test_result.ResultType.FAIL, base_test_result.ResultType.CRASH, base_test_result.ResultType.TIMEOUT, base_test_result.ResultType.UNKNOWN)): logging.critical( '%s: %s', test_name, ', '.join( '%s %s' % (str(result_counts[test_name][i]), i) for i in base_test_result.ResultType.GetTypes())) else: all_pass += 1 logging.critical('%s of %s tests passed in all %s runs', str(all_pass), str(tot_tests), str(iteration_count)) if args.command == 'perf' and (args.steps or args.single_step): return 0 return (0 if all( r.DidRunPass() for r in all_iteration_results) else constants.ERROR_EXIT_CODE)
def RunTests(self): tests = self._GetTests() exit_now = threading.Event() @local_device_environment.handle_shard_failures def run_tests_on_device(dev, tests, results): for test in tests: if exit_now.isSet(): thread.exit() result = None try: result = self._RunTest(dev, test) if isinstance(result, base_test_result.BaseTestResult): results.AddResult(result) elif isinstance(result, list): results.AddResults(result) else: raise Exception('Unexpected result type: %s' % type(result).__name__) except: if isinstance(tests, test_collection.TestCollection): tests.add(test) raise finally: if isinstance(tests, test_collection.TestCollection): tests.test_completed() logging.info('Finished running tests on this device.') class TestsTerminated(Exception): pass def stop_tests(_signum, _frame): logging.critical('Received SIGTERM. Stopping test execution.') exit_now.set() raise TestsTerminated() try: with signal_handler.SignalHandler(signal.SIGTERM, stop_tests): tries = 0 results = [] while tries < self._env.max_tries and tests: logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) logging.info('Will run %d tests on %d devices: %s', len(tests), len(self._env.devices), ', '.join(str(d) for d in self._env.devices)) for t in tests: logging.debug(' %s', t) try_results = base_test_result.TestRunResults() test_names = (self._GetUniqueTestName(t) for t in tests) try_results.AddResults( base_test_result.BaseTestResult( t, base_test_result.ResultType.UNKNOWN) for t in test_names if not t.endswith('*')) try: if self._ShouldShard(): tc = test_collection.TestCollection( self._CreateShards(tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: self._env.parallel_devices.pMap( run_tests_on_device, tests, try_results).pGet(None) except TestsTerminated: for unknown_result in try_results.GetUnknown(): try_results.AddResult( base_test_result.BaseTestResult( unknown_result.GetName(), base_test_result.ResultType.TIMEOUT, log=_SIGTERM_TEST_LOG)) raise finally: results.append(try_results) tries += 1 tests = self._GetTestsToRetry(tests, try_results) logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries) if tests: logging.info('%d failed tests remain.', len(tests)) else: logging.info('All tests completed.') except TestsTerminated: pass return results