def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: GTestResult instance. """ result = gtest_utils.GTestResult(cmd) if self.webrtc_xctest: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. proc = self.start_proc(cmd) old_handler = self.set_sigterm_handler( lambda _signum, _frame: self.handle_sigterm(proc)) print_process_output(proc, 'xcodebuild', parser) LOGGER.info('Waiting for test process to terminate.') proc.wait() LOGGER.info('Test process terminated.') self.set_sigterm_handler(old_handler) sys.stdout.flush() LOGGER.debug('Stdout flushed after test process.') returncode = proc.returncode if self.webrtc_xctest and parser.SystemAlertPresent(): raise SystemAlertPresentError() LOGGER.debug('Processing test results.') for test in parser.FailedTests(include_flaky=True): # Test cases are named as <test group>.<test case>. If the test case # is prefixed with "FLAKY_", it should be reported as flaked not failed. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) # Only GTest outputs compiled tests in a json file. if not self.webrtc_xctest: result.disabled_tests_from_compiled_tests_file.extend( parser.DisabledTestsFromCompiledTestsFile()) LOGGER.info('%s returned %s\n', cmd[0], returncode) # xcodebuild can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. result.finalize(returncode, parser.CompletedWithoutFailure()) return result
def _run(self, cmd): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: GTestResult instance. """ print ' '.join(cmd) print result = gtest_utils.GTestResult(cmd) if self.xctest_path: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() proc = subprocess.Popen( cmd, env=self.get_launch_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) while True: line = proc.stdout.readline() if not line: break line = line.rstrip() parser.ProcessLine(line) print line sys.stdout.flush() proc.wait() sys.stdout.flush() for test in parser.FailedTests(include_flaky=True): # Test cases are named as <test group>.<test case>. If the test case # is prefixed with "FLAKY_", it should be reported as flaked not failed. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) print '%s returned %s' % (cmd[0], proc.returncode) print # iossim can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. result.finalize(proc.returncode, parser.CompletedWithoutFailure()) return result
def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. Returns: GTestResult instance. """ result = gtest_utils.GTestResult(cmd) if self.xctest_path: parser = xctest_utils.XCTestLogParser() else: parser = gtest_utils.GTestLogParser() if shards > 1: test_shards = shard_xctest( os.path.join(self.app_path, self.app_name), shards, self.test_cases) thread_pool = pool.ThreadPool(processes=shards) for out, name, ret in thread_pool.imap_unordered( self.run_tests, test_shards): print "Simulator %s" % name for line in out: print line parser.ProcessLine(line) returncode = ret if ret else 0 thread_pool.close() thread_pool.join() else: # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. proc = subprocess.Popen( cmd, env=self.get_launch_env(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) while True: line = proc.stdout.readline() if not line: break line = line.rstrip() parser.ProcessLine(line) print line sys.stdout.flush() proc.wait() sys.stdout.flush() returncode = proc.returncode for test in parser.FailedTests(include_flaky=True): # Test cases are named as <test group>.<test case>. If the test case # is prefixed with "FLAKY_", it should be reported as flaked not failed. if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[test] = parser.FailureDescription(test) else: result.failed_tests[test] = parser.FailureDescription(test) result.passed_tests.extend(parser.PassedTests(include_flaky=True)) print '%s returned %s' % (cmd[0], returncode) print # iossim can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. result.finalize(returncode, parser.CompletedWithoutFailure()) return result
def _run(self, cmd, shards=1): """Runs the specified command, parsing GTest output. Args: cmd: List of strings forming the command to run. NOTE: in the case of WprProxySimulatorTestRunner, cmd is a dict forming the configuration for the test (including filter rules), and not indicative of the actual command we build and execute in _run. Returns: GTestResult instance. Raises: ShardingDisabledError: If shards > 1 as currently sharding is not supported. SystemAlertPresentError: If system alert is shown on the device. """ result = gtest_utils.GTestResult(cmd) completed_without_failure = True total_returncode = 0 if shards > 1: # TODO(crbug.com/881096): reimplement sharding in the future raise test_runner.ShardingDisabledError() # TODO(crbug.com/812705): Implement test sharding for unit tests. # TODO(crbug.com/812712): Use thread pool for DeviceTestRunner as well. # Create a simulator for these tests, and prepare it with the # certificate needed for HTTPS proxying. udid = self.getSimulator() self.copy_trusted_certificate() for recipe_path in glob.glob('{}/*.test'.format(self.replay_path)): base_name = os.path.basename(recipe_path) test_name = os.path.splitext(base_name)[0] replay_path = '{}/{}'.format(self.replay_path, test_name) if self.should_run_wpr_test(test_name, cmd['test_filter'], cmd['invert']): parser, returncode = self.run_wpr_test(udid, test_name, recipe_path, replay_path) # If this test fails, immediately rerun it to see if it deflakes. # We simply overwrite the first result with the second. if parser.FailedTests(include_flaky=True): parser, returncode = self.run_wpr_test(udid, test_name, recipe_path, replay_path) for test in parser.FailedTests(include_flaky=True): # All test names will be the same since we re-run the same suite; # therefore, to differentiate the results, we append the recipe # name to the test suite. testWithRecipeName = "{}.{}".format(base_name, test) # Test cases are named as <test group>.<test case>. If the test case # is prefixed w/"FLAKY_", it should be reported as flaked not failed if '.' in test and test.split('.', 1)[1].startswith('FLAKY_'): result.flaked_tests[testWithRecipeName] = parser.FailureDescription( test) else: result.failed_tests[testWithRecipeName] = parser.FailureDescription( test) for test in parser.PassedTests(include_flaky=True): testWithRecipeName = "{}.{}".format(base_name, test) result.passed_tests.extend([testWithRecipeName]) # Check for runtime errors. if self.xctest_path and parser.SystemAlertPresent(): raise test_runner.SystemAlertPresentError() if returncode != 0: total_returncode = returncode if not parser.CompletedWithoutFailure(): completed_without_failure = False LOGGER.info('%s test returned %s\n', recipe_path, returncode) self.deleteSimulator(udid) # xcodebuild can return 5 if it exits noncleanly even if all tests passed. # Therefore we cannot rely on process exit code to determine success. # NOTE: total_returncode is 0 OR the last non-zero return code from a test. result.finalize(total_returncode, completed_without_failure) return result