def testGenerateResultsDict_multipleResults(self): result1 = base_test_result.BaseTestResult( 'test.package.TestName1', base_test_result.ResultType.PASS) result2 = base_test_result.BaseTestResult( 'test.package.TestName2', base_test_result.ResultType.PASS) all_results = base_test_result.TestRunResults() all_results.AddResult(result1) all_results.AddResult(result2) results_dict = json_results.GenerateResultsDict([all_results]) self.assertEquals(['test.package.TestName1', 'test.package.TestName2'], results_dict['all_tests']) self.assertTrue('per_iteration_data' in results_dict) iterations = results_dict['per_iteration_data'] self.assertEquals(1, len(iterations)) expected_tests = set([ 'test.package.TestName1', 'test.package.TestName2', ]) for test_name, iteration_result in six.iteritems(iterations[0]): self.assertTrue(test_name in expected_tests) expected_tests.remove(test_name) self.assertEquals(1, len(iteration_result)) test_iteration_result = iteration_result[0] self.assertTrue('status' in test_iteration_result) self.assertEquals('SUCCESS', test_iteration_result['status'])
def testGenerateResultsDict_globalTags(self): raw_results = [] global_tags = ['UNRELIABLE_RESULTS'] results_dict = json_results.GenerateResultsDict( [raw_results], global_tags=global_tags) self.assertEquals(['UNRELIABLE_RESULTS'], results_dict['global_tags'])
def post_run(self, return_code): # If we don't need to parse the host-side Tast tool's results, fall back to # the parent method's default behavior. if self._llvm_profile_var: return super(TastTest, self).post_run(return_code) tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') if not os.path.exists(tast_results_path): logging.error( 'Tast results not found at %s. Falling back to generic result ' 'reporting.', tast_results_path) return super(TastTest, self).post_run(return_code) # See the link below for the format of the results: # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult with jsonlines.open(tast_results_path) as reader: tast_results = collections.deque(reader) suite_results = base_test_result.TestRunResults() for test in tast_results: errors = test['errors'] start, end = test['start'], test['end'] # Use dateutil to parse the timestamps since datetime can't handle # nanosecond precision. duration = dateutil.parser.parse(end) - dateutil.parser.parse( start) duration_ms = duration.total_seconds() * 1000 if bool(test['skipReason']): result = base_test_result.ResultType.SKIP elif errors: result = base_test_result.ResultType.FAIL else: result = base_test_result.ResultType.PASS error_log = '' if errors: # See the link below for the format of these errors: # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/tast/testing#Error for err in errors: error_log += str(err['stack']) + '\n' base_result = base_test_result.BaseTestResult(test['name'], result, duration=duration_ms, log=error_log) suite_results.AddResult(base_result) if self._test_launcher_summary_output: with open(self._test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([suite_results]), f) if not suite_results.DidRunPass(): return 1 elif return_code: logging.warning( 'No failed tests found, but exit code of %d was returned from ' 'cros_run_test.', return_code) return return_code return 0
def post_run(self, return_code): # Create a simple json results file for a test run. The results will contain # only one test (suite_name), and will either be a PASS or FAIL depending on # return_code. if self._test_launcher_summary_output: result = (base_test_result.ResultType.FAIL if return_code else base_test_result.ResultType.PASS) suite_result = base_test_result.BaseTestResult(self.suite_name, result) run_results = base_test_result.TestRunResults() run_results.AddResult(suite_result) with open(self._test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([run_results]), f)
def post_run(self, return_code): # Create a simple json results file for the sanity test if needed. The # results will contain only one test (SANITY_TEST_TARGET), and will # either be a PASS or FAIL depending on the return code of cros_run_vm_test. if self._test_launcher_summary_output: result = (base_test_result.ResultType.FAIL if return_code else base_test_result.ResultType.PASS) sanity_test_result = base_test_result.BaseTestResult( SANITY_TEST_TARGET, result) run_results = base_test_result.TestRunResults() run_results.AddResult(sanity_test_result) with open(self._test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([run_results]), f)
def testGenerateResultsDict_failedResult(self): result = base_test_result.BaseTestResult( 'test.package.TestName', base_test_result.ResultType.FAIL) all_results = base_test_result.TestRunResults() all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) self.assertEquals(['test.package.TestName'], results_dict['all_tests']) self.assertEquals(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) self.assertEquals(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('status' in test_iteration_result) self.assertEquals('FAILURE', test_iteration_result['status'])
def testGenerateResultsDict_duration(self): result = base_test_result.BaseTestResult( 'test.package.TestName', base_test_result.ResultType.PASS, duration=123) all_results = base_test_result.TestRunResults() all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) self.assertEquals(['test.package.TestName'], results_dict['all_tests']) self.assertEquals(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) self.assertEquals(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('elapsed_time_ms' in test_iteration_result) self.assertEquals(123, test_iteration_result['elapsed_time_ms'])
def testGenerateResultsDict_passOnRetry(self): raw_results = [] result1 = base_test_result.BaseTestResult( 'test.package.TestName1', base_test_result.ResultType.FAIL) run_results1 = base_test_result.TestRunResults() run_results1.AddResult(result1) raw_results.append(run_results1) result2 = base_test_result.BaseTestResult( 'test.package.TestName1', base_test_result.ResultType.PASS) run_results2 = base_test_result.TestRunResults() run_results2.AddResult(result2) raw_results.append(run_results2) results_dict = json_results.GenerateResultsDict([raw_results]) self.assertEquals(['test.package.TestName1'], results_dict['all_tests']) # Check that there's only one iteration. self.assertIn('per_iteration_data', results_dict) iterations = results_dict['per_iteration_data'] self.assertEquals(1, len(iterations)) # Check that test.package.TestName1 is the only test in the iteration. self.assertEquals(1, len(iterations[0])) self.assertIn('test.package.TestName1', iterations[0]) # Check that there are two results for test.package.TestName1. actual_test_results = iterations[0]['test.package.TestName1'] self.assertEquals(2, len(actual_test_results)) # Check that the first result is a failure. self.assertIn('status', actual_test_results[0]) self.assertEquals('FAILURE', actual_test_results[0]['status']) # Check that the second result is a success. self.assertIn('status', actual_test_results[1]) self.assertEquals('SUCCESS', actual_test_results[1]['status'])
def testGenerateResultsDict_loslessSnippet(self): result = base_test_result.BaseTestResult( 'test.package.TestName', base_test_result.ResultType.FAIL) log = 'blah-blah' result.SetLog(log) all_results = base_test_result.TestRunResults() all_results.AddResult(result) results_dict = json_results.GenerateResultsDict([all_results]) self.assertEquals(['test.package.TestName'], results_dict['all_tests']) self.assertEquals(1, len(results_dict['per_iteration_data'])) iteration_result = results_dict['per_iteration_data'][0] self.assertTrue('test.package.TestName' in iteration_result) self.assertEquals(1, len(iteration_result['test.package.TestName'])) test_iteration_result = iteration_result['test.package.TestName'][0] self.assertTrue('losless_snippet' in test_iteration_result) self.assertTrue(test_iteration_result['losless_snippet']) self.assertTrue('output_snippet' in test_iteration_result) self.assertEquals(log, test_iteration_result['output_snippet']) self.assertTrue('output_snippet_base64' in test_iteration_result) self.assertEquals('', test_iteration_result['output_snippet_base64'])
def vm_test(args): is_sanity_test = args.test_exe == 'cros_vm_sanity_test' # To keep things easy for us, ensure both types of output locations are # the same. if args.test_launcher_summary_output and args.vm_logs_dir: json_output_dir = os.path.dirname( args.test_launcher_summary_output) or '.' if os.path.abspath(json_output_dir) != os.path.abspath( args.vm_logs_dir): logging.error( '--test-launcher-summary-output and --vm-logs-dir must point to ' 'the same directory.') return 1 cros_run_vm_test_cmd = [ CROS_RUN_VM_TEST_PATH, '--start', '--board', args.board, '--cache-dir', args.cros_cache, ] # cros_run_vm_test has trouble with relative paths that go up directories, so # cd to src/, which should be the root of all data deps. os.chdir(CHROMIUM_SRC_PATH) runtime_files = read_runtime_files(args.runtime_deps_path, args.path_to_outdir) if args.vpython_dir: # --vpython-dir is relative to the out dir, but --files expects paths # relative to src dir, so fix the path up a bit. runtime_files.append( os.path.relpath( os.path.abspath( os.path.join(args.path_to_outdir, args.vpython_dir)), CHROMIUM_SRC_PATH)) runtime_files.append('.vpython') # If we're pushing files, we need to set the cwd. if runtime_files: cros_run_vm_test_cmd.extend( ['--cwd', os.path.relpath(args.path_to_outdir, CHROMIUM_SRC_PATH)]) for f in runtime_files: cros_run_vm_test_cmd.extend(['--files', f]) if args.vm_logs_dir: cros_run_vm_test_cmd += [ '--results-src', '/var/log/', '--results-dest-dir', args.vm_logs_dir, ] if args.test_launcher_summary_output and not is_sanity_test: result_dir, result_file = os.path.split( args.test_launcher_summary_output) # If args.test_launcher_summary_output is a file in cwd, result_dir will be # an empty string, so replace it with '.' when this is the case so # cros_run_vm_test can correctly handle it. if not result_dir: result_dir = '.' vm_result_file = '/tmp/%s' % result_file cros_run_vm_test_cmd += [ '--results-src', vm_result_file, '--results-dest-dir', result_dir, ] if is_sanity_test: # run_cros_vm_test's default behavior when no cmd is specified is the sanity # test that's baked into the VM image. This test smoke-checks the system # browser, so deploy our locally-built chrome to the VM before testing. cros_run_vm_test_cmd += [ '--deploy', '--build-dir', os.path.relpath(args.path_to_outdir, CHROMIUM_SRC_PATH), ] else: pre_test_cmds = [ # /home is mounted with "noexec" in the VM, but some of our tools # and tests use the home dir as a workspace (eg: vpython downloads # python binaries to ~/.vpython-root). /tmp doesn't have this # restriction, so change the location of the home dir for the # duration of the test. 'export HOME=/tmp', '\;', ] if args.vpython_dir: vpython_spec_path = os.path.relpath( os.path.join(CHROMIUM_SRC_PATH, '.vpython'), args.path_to_outdir) pre_test_cmds += [ # Backslash is needed to prevent $PATH from getting prematurely # executed on the host. 'export PATH=\$PATH:\$PWD/%s' % args.vpython_dir, '\;', # Initialize the vpython cache. This can take 10-20s, and some tests # can't afford to wait that long on the first invocation. 'vpython', '-vpython-spec', vpython_spec_path, '-vpython-tool', 'install', '\;', ] cros_run_vm_test_cmd += [ '--cmd', '--', # Wrap the cmd to run in the VM around quotes (") so that the # interpreter on the host doesn't stop at any ";" or "&&" tokens in the # cmd. '"', ] + pre_test_cmds + [ './' + args.test_exe, '--test-launcher-shard-index=%d' % args.test_launcher_shard_index, '--test-launcher-total-shards=%d' % args.test_launcher_total_shards, '"', ] if args.test_launcher_summary_output and not is_sanity_test: cros_run_vm_test_cmd += [ '--test-launcher-summary-output=%s' % vm_result_file, ] logging.info('Running the following command:') logging.info(' '.join(cros_run_vm_test_cmd)) # deploy_chrome needs a set of GN args used to build chrome to determine if # certain libraries need to be pushed to the VM. It looks for the args via an # env var. To trigger the default deploying behavior, give it a dummy set of # args. # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd-line # args. env_copy = os.environ.copy() if not env_copy.get('GN_ARGS'): env_copy['GN_ARGS'] = 'is_chromeos = true' env_copy['PATH'] = env_copy['PATH'] + ':' + os.path.join( CHROMITE_PATH, 'bin') rc = subprocess.call(cros_run_vm_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=env_copy) # Create a simple json results file for the sanity test if needed. The results # will contain only one test ('cros_vm_sanity_test'), and will either be a # PASS or FAIL depending on the return code of cros_run_vm_test above. if args.test_launcher_summary_output and is_sanity_test: result = (base_test_result.ResultType.FAIL if rc else base_test_result.ResultType.PASS) sanity_test_result = base_test_result.BaseTestResult( 'cros_vm_sanity_test', result) run_results = base_test_result.TestRunResults() run_results.AddResult(sanity_test_result) with open(args.test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([run_results]), f) return rc
def post_run(self, return_code): # If we don't need to parse the host-side Tast tool's results, fall back to # the parent method's default behavior. if self._llvm_profile_var: return super(TastTest, self).post_run(return_code) tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') if not os.path.exists(tast_results_path): logging.error( 'Tast results not found at %s. Falling back to generic result ' 'reporting.', tast_results_path) return super(TastTest, self).post_run(return_code) # See the link below for the format of the results: # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult with jsonlines.open(tast_results_path) as reader: tast_results = collections.deque(reader) suite_results = base_test_result.TestRunResults() for test in tast_results: errors = test['errors'] start, end = test['start'], test['end'] # Use dateutil to parse the timestamps since datetime can't handle # nanosecond precision. duration = dateutil.parser.parse(end) - dateutil.parser.parse( start) duration_ms = duration.total_seconds() * 1000 if bool(test['skipReason']): result = base_test_result.ResultType.SKIP elif errors: result = base_test_result.ResultType.FAIL else: result = base_test_result.ResultType.PASS error_log = '' if errors: # See the link below for the format of these errors: # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/tast/testing#Error for err in errors: error_log += err['stack'].encode('utf-8') + '\n' error_log += ( "\nIf you're unsure why this test failed, consult the steps " 'outlined in\n%s\n' % TAST_DEBUG_DOC) base_result = base_test_result.BaseTestResult(test['name'], result, duration=duration_ms, log=error_log) suite_results.AddResult(base_result) self._maybe_handle_perf_results(test['name']) if self._rdb_client: # Walk the contents of the test's "outDir" and atttach any file found # inside as an RDB 'artifact'. (This could include system logs, screen # shots, etc.) artifacts = self.get_artifacts(test['outDir']) self._rdb_client.Post(test['name'], result, duration_ms, error_log, artifacts) if self._rdb_client and self._logs_dir: # Attach artifacts from the device that don't apply to a single test. artifacts = self.get_artifacts( os.path.join(self._logs_dir, 'system_logs')) artifacts.update( self.get_artifacts(os.path.join(self._logs_dir, 'crashes'))) self._rdb_client.ReportInvocationLevelArtifacts(artifacts) if self._test_launcher_summary_output: with open(self._test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([suite_results]), f) if not suite_results.DidRunPass(): return 1 elif return_code: logging.warning( 'No failed tests found, but exit code of %d was returned from ' 'cros_run_test.', return_code) return return_code return 0
def vm_test(args): is_sanity_test = args.test_exe == 'cros_vm_sanity_test' cros_run_vm_test_cmd = [ CROS_RUN_VM_TEST_PATH, '--start', '--board', args.board, '--cache-dir', args.cros_cache, ] # cros_run_vm_test has trouble with relative paths that go up directories, so # cd to src/, which should be the root of all data deps. os.chdir(CHROMIUM_SRC_PATH) runtime_files = read_runtime_files( args.runtime_deps_path, args.path_to_outdir) # If we're pushing files, we need to set the cwd. if runtime_files: cros_run_vm_test_cmd.extend( ['--cwd', os.path.relpath(args.path_to_outdir, CHROMIUM_SRC_PATH)]) for f in runtime_files: cros_run_vm_test_cmd.extend(['--files', f]) if args.test_launcher_summary_output and not is_sanity_test: result_dir, result_file = os.path.split(args.test_launcher_summary_output) # If args.test_launcher_summary_output is a file in cwd, result_dir will be # an empty string, so replace it with '.' when this is the case so # cros_run_vm_test can correctly handle it. if not result_dir: result_dir = '.' vm_result_file = '/tmp/%s' % result_file cros_run_vm_test_cmd += [ '--results-src', vm_result_file, '--results-dest-dir', result_dir, ] if is_sanity_test: # run_cros_vm_test's default behavior when no cmd is specified is the sanity # test that's baked into the VM image. This test smoke-checks the system # browser, so deploy our locally-built chrome to the VM before testing. cros_run_vm_test_cmd += [ '--deploy', '--build-dir', os.path.relpath(args.path_to_outdir, CHROMIUM_SRC_PATH), ] else: cros_run_vm_test_cmd += [ '--cmd', '--', './' + args.test_exe, '--test-launcher-shard-index=%d' % args.test_launcher_shard_index, '--test-launcher-total-shards=%d' % args.test_launcher_total_shards, ] if args.test_launcher_summary_output and not is_sanity_test: cros_run_vm_test_cmd += [ '--test-launcher-summary-output=%s' % vm_result_file, ] logging.info('Running the following command:') logging.info(' '.join(cros_run_vm_test_cmd)) # deploy_chrome needs a set of GN args used to build chrome to determine if # certain libraries need to be pushed to the VM. It looks for the args via an # env var. To trigger the default deploying behavior, give it a dummy set of # args. # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd-line # args. env_copy = os.environ.copy() if not env_copy.get('GN_ARGS'): env_copy['GN_ARGS'] = 'is_chromeos = true' env_copy['PATH'] = env_copy['PATH'] + ':' + os.path.join(CHROMITE_PATH, 'bin') rc = subprocess.call( cros_run_vm_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=env_copy) # Create a simple json results file for the sanity test if needed. The results # will contain only one test ('cros_vm_sanity_test'), and will either be a # PASS or FAIL depending on the return code of cros_run_vm_test above. if args.test_launcher_summary_output and is_sanity_test: result = (base_test_result.ResultType.FAIL if rc else base_test_result.ResultType.PASS) sanity_test_result = base_test_result.BaseTestResult( 'cros_vm_sanity_test', result) run_results = base_test_result.TestRunResults() run_results.AddResult(sanity_test_result) with open(args.test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([run_results]), f) return rc
def post_run(self, return_code): tast_results_path = os.path.join(self._logs_dir, 'streamed_results.jsonl') if not os.path.exists(tast_results_path): logging.error( 'Tast results not found at %s. Falling back to generic result ' 'reporting.', tast_results_path) return super().post_run(return_code) # See the link below for the format of the results: # https://godoc.org/chromium.googlesource.com/chromiumos/platform/tast.git/src/chromiumos/cmd/tast/run#TestResult with jsonlines.open(tast_results_path) as reader: tast_results = collections.deque(reader) suite_results = base_test_result.TestRunResults() for test in tast_results: errors = test['errors'] start, end = test['start'], test['end'] # Use dateutil to parse the timestamps since datetime can't handle # nanosecond precision. duration = dateutil.parser.parse(end) - dateutil.parser.parse( start) # If the duration is negative, Tast has likely reported an incorrect # duration. See https://issuetracker.google.com/issues/187973541. Round # up to 0 in that case to avoid confusing RDB. duration_ms = max(duration.total_seconds() * 1000, 0) if bool(test['skipReason']): result = base_test_result.ResultType.SKIP elif errors: result = base_test_result.ResultType.FAIL else: result = base_test_result.ResultType.PASS primary_error_message = None error_log = '' if errors: # See the link below for the format of these errors: # https://source.chromium.org/chromiumos/chromiumos/codesearch/+/main:src/platform/tast/src/chromiumos/tast/cmd/tast/internal/run/resultsjson/resultsjson.go primary_error_message = errors[0]['reason'] for err in errors: error_log += err['stack'] + '\n' debug_link = ( "If you're unsure why this test failed, consult the steps " 'outlined <a href="%s">here</a>.' % TAST_DEBUG_DOC) base_result = base_test_result.BaseTestResult(test['name'], result, duration=duration_ms, log=error_log) suite_results.AddResult(base_result) self._maybe_handle_perf_results(test['name']) if self._rdb_client: # Walk the contents of the test's "outDir" and atttach any file found # inside as an RDB 'artifact'. (This could include system logs, screen # shots, etc.) artifacts = self.get_artifacts(test['outDir']) self._rdb_client.Post(test['name'], result, duration_ms, error_log, None, artifacts=artifacts, failure_reason=primary_error_message, html_artifact=debug_link) if self._rdb_client and self._logs_dir: # Attach artifacts from the device that don't apply to a single test. artifacts = self.get_artifacts( os.path.join(self._logs_dir, 'system_logs')) artifacts.update( self.get_artifacts(os.path.join(self._logs_dir, 'crashes'))) self._rdb_client.ReportInvocationLevelArtifacts(artifacts) if self._test_launcher_summary_output: with open(self._test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([suite_results]), f) if not suite_results.DidRunPass(): return 1 if return_code: logging.warning( 'No failed tests found, but exit code of %d was returned from ' 'cros_run_test.', return_code) return return_code return 0
def vm_test(args, unknown_args): is_sanity_test = args.test_exe == 'cros_vm_sanity_test' # To keep things easy for us, ensure both types of output locations are # the same. if args.test_launcher_summary_output and args.vm_logs_dir: json_output_dir = os.path.dirname( args.test_launcher_summary_output) or '.' if os.path.abspath(json_output_dir) != os.path.abspath( args.vm_logs_dir): logging.error( '--test-launcher-summary-output and --vm-logs-dir must point to ' 'the same directory.') return 1 cros_run_vm_test_cmd = [ CROS_RUN_VM_TEST_PATH, '--start', '--board', args.board, '--cache-dir', args.cros_cache, ] # cros_run_vm_test has trouble with relative paths that go up directories, so # cd to src/, which should be the root of all data deps. os.chdir(CHROMIUM_SRC_PATH) runtime_files = read_runtime_files(args.runtime_deps_path, args.path_to_outdir) if args.vpython_dir: # --vpython-dir is relative to the out dir, but --files expects paths # relative to src dir, so fix the path up a bit. runtime_files.append( os.path.relpath( os.path.abspath( os.path.join(args.path_to_outdir, args.vpython_dir)), CHROMIUM_SRC_PATH)) runtime_files.append('.vpython') # If we're pushing files, we need to set the cwd. if runtime_files: cros_run_vm_test_cmd.extend( ['--cwd', os.path.relpath(args.path_to_outdir, CHROMIUM_SRC_PATH)]) for f in runtime_files: cros_run_vm_test_cmd.extend(['--files', f]) if args.vm_logs_dir: cros_run_vm_test_cmd += [ '--results-src', '/var/log/', '--results-dest-dir', args.vm_logs_dir, ] if args.test_launcher_summary_output and not is_sanity_test: result_dir, result_file = os.path.split( args.test_launcher_summary_output) # If args.test_launcher_summary_output is a file in cwd, result_dir will be # an empty string, so replace it with '.' when this is the case so # cros_run_vm_test can correctly handle it. if not result_dir: result_dir = '.' vm_result_file = '/tmp/%s' % result_file cros_run_vm_test_cmd += [ '--results-src', vm_result_file, '--results-dest-dir', result_dir, ] if is_sanity_test: # run_cros_vm_test's default behavior when no cmd is specified is the sanity # test that's baked into the VM image. This test smoke-checks the system # browser, so deploy our locally-built chrome to the VM before testing. cros_run_vm_test_cmd += [ '--deploy', '--build-dir', os.path.relpath(args.path_to_outdir, CHROMIUM_SRC_PATH), ] else: pre_test_cmds = [ # /home is mounted with "noexec" in the VM, but some of our tools # and tests use the home dir as a workspace (eg: vpython downloads # python binaries to ~/.vpython-root). /tmp doesn't have this # restriction, so change the location of the home dir for the # duration of the test. 'export HOME=/tmp', '\;', ] if args.vpython_dir: vpython_spec_path = os.path.relpath( os.path.join(CHROMIUM_SRC_PATH, '.vpython'), args.path_to_outdir) pre_test_cmds += [ # Backslash is needed to prevent $PATH from getting prematurely # executed on the host. 'export PATH=\$PATH:\$PWD/%s' % args.vpython_dir, '\;', # Initialize the vpython cache. This can take 10-20s, and some tests # can't afford to wait that long on the first invocation. 'vpython', '-vpython-spec', vpython_spec_path, '-vpython-tool', 'install', '\;', ] cros_run_vm_test_cmd += [ # Some tests fail as root, so run as the less privileged user 'chronos'. '--as-chronos', '--cmd', '--', # Wrap the cmd to run in the VM around quotes (") so that the # interpreter on the host doesn't stop at any ";" or "&&" tokens in the # cmd. '"', ] + pre_test_cmds + [ './' + args.test_exe, '--test-launcher-shard-index=%d' % args.test_launcher_shard_index, '--test-launcher-total-shards=%d' % args.test_launcher_total_shards, ] + unknown_args + [ '"', ] if args.test_launcher_summary_output and not is_sanity_test: cros_run_vm_test_cmd += [ '--test-launcher-summary-output=%s' % vm_result_file, ] logging.info('Running the following command:') logging.info(' '.join(cros_run_vm_test_cmd)) # deploy_chrome needs a set of GN args used to build chrome to determine if # certain libraries need to be pushed to the VM. It looks for the args via an # env var. To trigger the default deploying behavior, give it a dummy set of # args. # TODO(crbug.com/823996): Make the GN-dependent deps controllable via cmd-line # args. env_copy = os.environ.copy() if not env_copy.get('GN_ARGS'): env_copy['GN_ARGS'] = 'is_chromeos = true' env_copy['PATH'] = env_copy['PATH'] + ':' + os.path.join( CHROMITE_PATH, 'bin') # Traps SIGTERM and kills all child processes of cros_run_vm_test when it's # caught. This will allow us to capture logs from the VM if a test hangs # and gets timeout-killed by swarming. See also: # https://chromium.googlesource.com/infra/luci/luci-py/+/master/appengine/swarming/doc/Bot.md#graceful-termination_aka-the-sigterm-and-sigkill-dance test_proc = None def _kill_child_procs(trapped_signal, _): logging.warning('Received signal %d. Killing child processes of test.', trapped_signal) if not test_proc or not test_proc.pid: # This shouldn't happen? logging.error('Test process not running.') return for child in psutil.Process(test_proc.pid).children(): logging.warning('Killing process %s', child) child.kill() # Standard GTests should handle retries and timeouts themselves. retries, timeout = 0, None if is_sanity_test: # 5 min should be enough time for the sanity test to pass. retries, timeout = 2, 300 signal.signal(signal.SIGTERM, _kill_child_procs) for i in xrange(retries + 1): logging.info('########################################') logging.info('Test attempt #%d', i) logging.info('########################################') test_proc = subprocess42.Popen(cros_run_vm_test_cmd, stdout=sys.stdout, stderr=sys.stderr, env=env_copy) try: test_proc.wait(timeout=timeout) except subprocess42.TimeoutExpired: logging.error('Test timed out. Sending SIGTERM.') # SIGTERM the proc and wait 10s for it to close. test_proc.terminate() try: test_proc.wait(timeout=10) except subprocess42.TimeoutExpired: # If it hasn't closed in 10s, SIGKILL it. logging.error('Test did not exit in time. Sending SIGKILL.') test_proc.kill() test_proc.wait() logging.info('Test exitted with %d.', test_proc.returncode) if test_proc.returncode == 0: break rc = test_proc.returncode # Create a simple json results file for the sanity test if needed. The results # will contain only one test ('cros_vm_sanity_test'), and will either be a # PASS or FAIL depending on the return code of cros_run_vm_test above. if args.test_launcher_summary_output and is_sanity_test: result = (base_test_result.ResultType.FAIL if rc else base_test_result.ResultType.PASS) sanity_test_result = base_test_result.BaseTestResult( 'cros_vm_sanity_test', result) run_results = base_test_result.TestRunResults() run_results.AddResult(sanity_test_result) with open(args.test_launcher_summary_output, 'w') as f: json.dump(json_results.GenerateResultsDict([run_results]), f) return rc