def _RunTest(self, device, test): # Run the test. timeout = (self._test_instance.shard_timeout * self.GetTool(device).GetTimeoutScale()) output = self._delegate.Run(test, device, flags=self._test_instance.test_arguments, timeout=timeout, retries=0) for s in self._servers[str(device)]: s.Reset() if self._test_instance.app_files: self._delegate.PullAppFiles(device, self._test_instance.app_files, self._test_instance.app_file_dir) if not self._env.skip_clear_data: self._delegate.Clear(device) # Parse the output. # TODO(jbudorick): Transition test scripts away from parsing stdout. results = gtest_test_instance.ParseGTestOutput(output) # Check whether there are any crashed testcases. self._crashes.update( r.GetName() for r in results if r.GetType() == base_test_result.ResultType.CRASH) return results
def _RunTest(self, device, test): # Run the test. timeout = (self._test_instance.shard_timeout * self.GetTool(device).GetTimeoutScale()) if self._test_instance.store_tombstones: tombstones.ClearAllTombstones(device) with device_temp_file.DeviceTempFile( adb=device.adb, dir=self._delegate.ResultsDirectory(device), suffix='.xml') as device_tmp_results_file: flags = self._test_instance.test_arguments or '' if self._test_instance.enable_xml_result_parsing: flags += ' --gtest_output=xml:%s' % device_tmp_results_file.name if self._test_instance.gtest_also_run_disabled_tests: flags += ' --gtest_also_run_disabled_tests' output = self._delegate.Run( test, device, flags=flags, timeout=timeout, retries=0) if self._test_instance.enable_xml_result_parsing: gtest_xml = device.ReadFile( device_tmp_results_file.name, as_root=True) for s in self._servers[str(device)]: s.Reset() if self._test_instance.app_files: self._delegate.PullAppFiles(device, self._test_instance.app_files, self._test_instance.app_file_dir) if not self._env.skip_clear_data: self._delegate.Clear(device) # Parse the output. # TODO(jbudorick): Transition test scripts away from parsing stdout. if self._test_instance.enable_xml_result_parsing: results = gtest_test_instance.ParseGTestXML(gtest_xml) else: results = gtest_test_instance.ParseGTestOutput(output) # Check whether there are any crashed testcases. self._crashes.update(r.GetName() for r in results if r.GetType() == base_test_result.ResultType.CRASH) if self._test_instance.store_tombstones: resolved_tombstones = None for result in results: if result.GetType() == base_test_result.ResultType.CRASH: if not resolved_tombstones: resolved_tombstones = '\n'.join(tombstones.ResolveTombstones( device, resolve_all_tombstones=True, include_stack_symbols=False, wipe_tombstones=True)) result.SetTombstones(resolved_tombstones) not_run_tests = set(test).difference(set(r.GetName() for r in results)) return results, not_run_tests
def testParseGTestOutput_unknown(self): raw_output = [ '[ RUN ] FooTest.Bar', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEquals(1, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(0, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.UNKNOWN, actual[0].GetType())
def testParseGTestOutput_pass(self): raw_output = [ '[ RUN ] FooTest.Bar', '[ OK ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEquals(1, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(1, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.PASS, actual[0].GetType())
def testParseGTestOutput_errorCrash(self): raw_output = [ '[ RUN ] FooTest.Bar', '[ERROR:blah] Currently running: FooTest.Bar', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEquals(1, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(0, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType())
def testParseGTestOutput_fail(self): raw_output = [ '[ RUN ] FooTest.Bar', '[ FAILED ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output) self.assertEquals(1, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(1, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType())
def testParseGTestOutput_typeParameterized(self): raw_output = [ '[ RUN ] Baz/FooTest.Bar/0', '[ FAILED ] Baz/FooTest.Bar/0, where TypeParam = (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEquals(1, len(actual)) self.assertEquals('Baz/FooTest.Bar/0', actual[0].GetName()) self.assertEquals(1, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType())
def testParseGTestOutput_skippedTest(self): raw_output = [ '[ RUN ] FooTest.Bar', '[ SKIPPED ] FooTest.Bar (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEqual(1, len(actual)) self.assertEqual('FooTest.Bar', actual[0].GetName()) self.assertEqual(1, actual[0].GetDuration()) self.assertEqual(base_test_result.ResultType.SKIP, actual[0].GetType())
def _ParseTestResults(self): logging.info('Parsing results from stdout.') results = base_test_result.TestRunResults() output = self._results['results']['output'].splitlines() output = (l[len(self._INSTRUMENTATION_STREAM_LEADER):] for l in output if l.startswith(self._INSTRUMENTATION_STREAM_LEADER)) results_list = gtest_test_instance.ParseGTestOutput(output) results.AddResults(results_list) self._DetectPlatformErrors(results) return results
def testParseGTestOutput_fatalDcheck(self): raw_output = [ '[ RUN ] FooTest.Bar', '[0324/183029.116334:FATAL:test_timeouts.cc(103)] Check failed: !init', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEquals(1, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertIsNone(actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType())
def testParseGTestOutput_valueParameterized(self): raw_output = [ '[ RUN ] Baz/FooTest.Bar/0', '[ FAILED ] Baz/FooTest.Bar/0,' + ' where GetParam() = 4-byte object <00-00 00-00> (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEqual(1, len(actual)) self.assertEqual('Baz/FooTest.Bar/0', actual[0].GetName()) self.assertEqual(1, actual[0].GetDuration()) self.assertEqual(base_test_result.ResultType.FAIL, actual[0].GetType())
def _RunTest(self, device, test): # Run the test. timeout = (self._test_instance.shard_timeout * self.GetTool(device).GetTimeoutScale()) with tempfile.NamedTemporaryFile( suffix='.xml') as host_tmp_results_file: with device_temp_file.DeviceTempFile( adb=device.adb, dir=self._delegate.ResultsDirectory(device), suffix='.xml') as device_tmp_results_file: flags = self._test_instance.test_arguments or '' if self._test_instance.enable_xml_result_parsing: flags += ' --gtest_output=xml:%s' % device_tmp_results_file.name output = self._delegate.Run(test, device, flags=flags, timeout=timeout, retries=0) if self._test_instance.enable_xml_result_parsing: device.PullFile(device_tmp_results_file.name, host_tmp_results_file.name) for s in self._servers[str(device)]: s.Reset() if self._test_instance.app_files: self._delegate.PullAppFiles(device, self._test_instance.app_files, self._test_instance.app_file_dir) if not self._env.skip_clear_data: self._delegate.Clear(device) # Parse the output. # TODO(jbudorick): Transition test scripts away from parsing stdout. if self._test_instance.enable_xml_result_parsing: with open(host_tmp_results_file.name) as xml_results_file: results = gtest_test_instance.ParseGTestXML( xml_results_file.read()) else: results = gtest_test_instance.ParseGTestOutput(output) # Check whether there are any crashed testcases. self._crashes.update( r.GetName() for r in results if r.GetType() == base_test_result.ResultType.CRASH) return results
def testParseGTestOutput_nonterminalUnknown(self): raw_output = [ '[ RUN ] FooTest.Bar', '[ RUN ] FooTest.Baz', '[ OK ] FooTest.Baz (1 ms)', ] actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None) self.assertEquals(2, len(actual)) self.assertEquals('FooTest.Bar', actual[0].GetName()) self.assertEquals(0, actual[0].GetDuration()) self.assertEquals(base_test_result.ResultType.UNKNOWN, actual[0].GetType()) self.assertEquals('FooTest.Baz', actual[1].GetName()) self.assertEquals(1, actual[1].GetDuration()) self.assertEquals(base_test_result.ResultType.PASS, actual[1].GetType())
def _RunTest(self, device, test): # Run the test. timeout = (self._test_instance.shard_timeout * self.GetTool(device).GetTimeoutScale()) if self._test_instance.wait_for_java_debugger: timeout = None if self._test_instance.store_tombstones: tombstones.ClearAllTombstones(device) test_perf_output_filename = next(self._test_perf_output_filenames) with device_temp_file.DeviceTempFile( adb=device.adb, dir=self._delegate.ResultsDirectory(device), suffix='.xml') as device_tmp_results_file: with contextlib_ext.Optional( device_temp_file.NamedDeviceTemporaryDirectory( adb=device.adb, dir='/sdcard/'), self._test_instance. gs_test_artifacts_bucket) as test_artifacts_dir: with (contextlib_ext.Optional( device_temp_file.DeviceTempFile( adb=device.adb, dir=self._delegate.ResultsDirectory(device)), test_perf_output_filename) ) as isolated_script_test_perf_output: flags = list(self._test_instance.flags) if self._test_instance.enable_xml_result_parsing: flags.append('--gtest_output=xml:%s' % device_tmp_results_file.name) if self._test_instance.gs_test_artifacts_bucket: flags.append('--test_artifacts_dir=%s' % test_artifacts_dir.name) if test_perf_output_filename: flags.append('--isolated_script_test_perf_output=%s' % isolated_script_test_perf_output.name) logging.info('flags:') for f in flags: logging.info(' %s', f) stream_name = 'logcat_%s_%s_%s' % ( hash(tuple(test)), time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) with self._env.output_manager.ArchivedTempfile( stream_name, 'logcat') as logcat_file: with logcat_monitor.LogcatMonitor( device.adb, filter_specs=local_device_environment. LOGCAT_FILTERS, output_file=logcat_file.name, check_error=False) as logmon: with contextlib_ext.Optional( trace_event.trace(str(test)), self._env.trace_output): output = self._delegate.Run( test, device, flags=' '.join(flags), timeout=timeout, retries=0) logmon.Close() if logcat_file.Link(): logging.info('Logcat saved to %s', logcat_file.Link()) if self._test_instance.enable_xml_result_parsing: try: gtest_xml = device.ReadFile( device_tmp_results_file.name, as_root=True) except device_errors.CommandFailedError as e: logging.warning( 'Failed to pull gtest results XML file %s: %s', device_tmp_results_file.name, str(e)) gtest_xml = None if test_perf_output_filename: try: device.PullFile( isolated_script_test_perf_output.name, test_perf_output_filename) except device_errors.CommandFailedError as e: logging.warning( 'Failed to pull chartjson results %s: %s', isolated_script_test_perf_output.name, str(e)) test_artifacts_url = self._UploadTestArtifacts( device, test_artifacts_dir) for s in self._servers[str(device)]: s.Reset() if self._test_instance.app_files: self._delegate.PullAppFiles(device, self._test_instance.app_files, self._test_instance.app_file_dir) if not self._env.skip_clear_data: self._delegate.Clear(device) for l in output: logging.info(l) # Parse the output. # TODO(jbudorick): Transition test scripts away from parsing stdout. if self._test_instance.enable_xml_result_parsing: results = gtest_test_instance.ParseGTestXML(gtest_xml) else: results = gtest_test_instance.ParseGTestOutput( output, self._test_instance.symbolizer, device.product_cpu_abi) tombstones_url = None for r in results: if logcat_file: r.SetLink('logcat', logcat_file.Link()) if self._test_instance.gs_test_artifacts_bucket: r.SetLink('test_artifacts', test_artifacts_url) if r.GetType() == base_test_result.ResultType.CRASH: self._crashes.add(r.GetName()) if self._test_instance.store_tombstones: if not tombstones_url: resolved_tombstones = tombstones.ResolveTombstones( device, resolve_all_tombstones=True, include_stack_symbols=False, wipe_tombstones=True) stream_name = 'tombstones_%s_%s' % (time.strftime( '%Y%m%dT%H%M%S', time.localtime()), device.serial) tombstones_url = logdog_helper.text( stream_name, '\n'.join(resolved_tombstones)) r.SetLink('tombstones', tombstones_url) tests_stripped_disabled_prefix = set() for t in test: tests_stripped_disabled_prefix.add( gtest_test_instance.TestNameWithoutDisabledPrefix(t)) not_run_tests = tests_stripped_disabled_prefix.difference( set(r.GetName() for r in results)) return results, list(not_run_tests) if results else None
def _RunTest(self, device, test): # Run the test. timeout = (self._test_instance.shard_timeout * self.GetTool(device).GetTimeoutScale()) if self._test_instance.store_tombstones: tombstones.ClearAllTombstones(device) with device_temp_file.DeviceTempFile( adb=device.adb, dir=self._delegate.ResultsDirectory(device), suffix='.xml') as device_tmp_results_file: flags = list(self._test_instance.flags) if self._test_instance.enable_xml_result_parsing: flags.append('--gtest_output=xml:%s' % device_tmp_results_file.name) logging.info('flags:') for f in flags: logging.info(' %s', f) with contextlib_ext.Optional(trace_event.trace(str(test)), self._env.trace_output): output = self._delegate.Run(test, device, flags=' '.join(flags), timeout=timeout, retries=0) if self._test_instance.enable_xml_result_parsing: gtest_xml = device.ReadFile(device_tmp_results_file.name, as_root=True) for s in self._servers[str(device)]: s.Reset() if self._test_instance.app_files: self._delegate.PullAppFiles(device, self._test_instance.app_files, self._test_instance.app_file_dir) if not self._env.skip_clear_data: self._delegate.Clear(device) # Parse the output. # TODO(jbudorick): Transition test scripts away from parsing stdout. if self._test_instance.enable_xml_result_parsing: results = gtest_test_instance.ParseGTestXML(gtest_xml) else: results = gtest_test_instance.ParseGTestOutput(output) # Check whether there are any crashed testcases. self._crashes.update( r.GetName() for r in results if r.GetType() == base_test_result.ResultType.CRASH) if self._test_instance.store_tombstones: tombstones_url = None for result in results: if result.GetType() == base_test_result.ResultType.CRASH: if not tombstones_url: resolved_tombstones = tombstones.ResolveTombstones( device, resolve_all_tombstones=True, include_stack_symbols=False, wipe_tombstones=True) stream_name = 'tombstones_%s_%s' % (time.strftime( '%Y%m%dT%H%M%S', time.localtime()), device.serial) tombstones_url = logdog_helper.text( stream_name, resolved_tombstones) result.SetLink('tombstones', tombstones_url) tests_stripped_disabled_prefix = set() for t in test: tests_stripped_disabled_prefix.add( gtest_test_instance.TestNameWithoutDisabledPrefix(t)) not_run_tests = tests_stripped_disabled_prefix.difference( set(r.GetName() for r in results)) return results, list(not_run_tests) if results else None