def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
output = self._delegate.Run(test,
device,
flags=self._test_instance.test_arguments,
timeout=timeout,
retries=0)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
return results
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout
* self.GetTool(device).GetTimeoutScale())
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = self._test_instance.test_arguments or ''
if self._test_instance.enable_xml_result_parsing:
flags += ' --gtest_output=xml:%s' % device_tmp_results_file.name
if self._test_instance.gtest_also_run_disabled_tests:
flags += ' --gtest_also_run_disabled_tests'
output = self._delegate.Run(
test, device, flags=flags,
timeout=timeout, retries=0)
if self._test_instance.enable_xml_result_parsing:
gtest_xml = device.ReadFile(
device_tmp_results_file.name,
as_root=True)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
if self._test_instance.store_tombstones:
resolved_tombstones = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not resolved_tombstones:
resolved_tombstones = '\n'.join(tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True))
result.SetTombstones(resolved_tombstones)
not_run_tests = set(test).difference(set(r.GetName() for r in results))
return results, not_run_tests
def testParseGTestOutput_unknown(self):
raw_output = [
'[ RUN ] FooTest.Bar',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEquals(1, len(actual))
self.assertEquals('FooTest.Bar', actual[0].GetName())
self.assertEquals(0, actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.UNKNOWN, actual[0].GetType())
def testParseGTestOutput_pass(self):
raw_output = [
'[ RUN ] FooTest.Bar',
'[ OK ] FooTest.Bar (1 ms)',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEquals(1, len(actual))
self.assertEquals('FooTest.Bar', actual[0].GetName())
self.assertEquals(1, actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.PASS, actual[0].GetType())
def testParseGTestOutput_errorCrash(self):
raw_output = [
'[ RUN ] FooTest.Bar',
'[ERROR:blah] Currently running: FooTest.Bar',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEquals(1, len(actual))
self.assertEquals('FooTest.Bar', actual[0].GetName())
self.assertEquals(0, actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.CRASH, actual[0].GetType())
def testParseGTestOutput_fail(self):
raw_output = [
'[ RUN ] FooTest.Bar',
'[ FAILED ] FooTest.Bar (1 ms)',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output)
self.assertEquals(1, len(actual))
self.assertEquals('FooTest.Bar', actual[0].GetName())
self.assertEquals(1, actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType())
def testParseGTestOutput_typeParameterized(self):
raw_output = [
'[ RUN ] Baz/FooTest.Bar/0',
'[ FAILED ] Baz/FooTest.Bar/0, where TypeParam = (1 ms)',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEquals(1, len(actual))
self.assertEquals('Baz/FooTest.Bar/0', actual[0].GetName())
self.assertEquals(1, actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.FAIL, actual[0].GetType())
def testParseGTestOutput_skippedTest(self):
raw_output = [
'[ RUN ] FooTest.Bar',
'[ SKIPPED ] FooTest.Bar (1 ms)',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEqual(1, len(actual))
self.assertEqual('FooTest.Bar', actual[0].GetName())
self.assertEqual(1, actual[0].GetDuration())
self.assertEqual(base_test_result.ResultType.SKIP, actual[0].GetType())
def _ParseTestResults(self):
logging.info('Parsing results from stdout.')
results = base_test_result.TestRunResults()
output = self._results['results']['output'].splitlines()
output = (l[len(self._INSTRUMENTATION_STREAM_LEADER):] for l in output
if l.startswith(self._INSTRUMENTATION_STREAM_LEADER))
results_list = gtest_test_instance.ParseGTestOutput(output)
results.AddResults(results_list)
self._DetectPlatformErrors(results)
return results
def testParseGTestOutput_fatalDcheck(self):
raw_output = [
'[ RUN ] FooTest.Bar',
'[0324/183029.116334:FATAL:test_timeouts.cc(103)] Check failed: !init',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEquals(1, len(actual))
self.assertEquals('FooTest.Bar', actual[0].GetName())
self.assertIsNone(actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.CRASH,
actual[0].GetType())
def testParseGTestOutput_valueParameterized(self):
raw_output = [
'[ RUN ] Baz/FooTest.Bar/0',
'[ FAILED ] Baz/FooTest.Bar/0,' +
' where GetParam() = 4-byte object <00-00 00-00> (1 ms)',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEqual(1, len(actual))
self.assertEqual('Baz/FooTest.Bar/0', actual[0].GetName())
self.assertEqual(1, actual[0].GetDuration())
self.assertEqual(base_test_result.ResultType.FAIL, actual[0].GetType())
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
with tempfile.NamedTemporaryFile(
suffix='.xml') as host_tmp_results_file:
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = self._test_instance.test_arguments or ''
if self._test_instance.enable_xml_result_parsing:
flags += ' --gtest_output=xml:%s' % device_tmp_results_file.name
output = self._delegate.Run(test,
device,
flags=flags,
timeout=timeout,
retries=0)
if self._test_instance.enable_xml_result_parsing:
device.PullFile(device_tmp_results_file.name,
host_tmp_results_file.name)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device,
self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
with open(host_tmp_results_file.name) as xml_results_file:
results = gtest_test_instance.ParseGTestXML(
xml_results_file.read())
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
return results
def testParseGTestOutput_nonterminalUnknown(self):
raw_output = [
'[ RUN ] FooTest.Bar',
'[ RUN ] FooTest.Baz',
'[ OK ] FooTest.Baz (1 ms)',
]
actual = gtest_test_instance.ParseGTestOutput(raw_output, None, None)
self.assertEquals(2, len(actual))
self.assertEquals('FooTest.Bar', actual[0].GetName())
self.assertEquals(0, actual[0].GetDuration())
self.assertEquals(base_test_result.ResultType.UNKNOWN, actual[0].GetType())
self.assertEquals('FooTest.Baz', actual[1].GetName())
self.assertEquals(1, actual[1].GetDuration())
self.assertEquals(base_test_result.ResultType.PASS, actual[1].GetType())
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.wait_for_java_debugger:
timeout = None
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
test_perf_output_filename = next(self._test_perf_output_filenames)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
with contextlib_ext.Optional(
device_temp_file.NamedDeviceTemporaryDirectory(
adb=device.adb, dir='/sdcard/'), self._test_instance.
gs_test_artifacts_bucket) as test_artifacts_dir:
with (contextlib_ext.Optional(
device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device)),
test_perf_output_filename)
) as isolated_script_test_perf_output:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' %
device_tmp_results_file.name)
if self._test_instance.gs_test_artifacts_bucket:
flags.append('--test_artifacts_dir=%s' %
test_artifacts_dir.name)
if test_perf_output_filename:
flags.append('--isolated_script_test_perf_output=%s' %
isolated_script_test_perf_output.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
stream_name = 'logcat_%s_%s_%s' % (
hash(tuple(test)),
time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with self._env.output_manager.ArchivedTempfile(
stream_name, 'logcat') as logcat_file:
with logcat_monitor.LogcatMonitor(
device.adb,
filter_specs=local_device_environment.
LOGCAT_FILTERS,
output_file=logcat_file.name,
check_error=False) as logmon:
with contextlib_ext.Optional(
trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(
test,
device,
flags=' '.join(flags),
timeout=timeout,
retries=0)
logmon.Close()
if logcat_file.Link():
logging.info('Logcat saved to %s', logcat_file.Link())
if self._test_instance.enable_xml_result_parsing:
try:
gtest_xml = device.ReadFile(
device_tmp_results_file.name, as_root=True)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull gtest results XML file %s: %s',
device_tmp_results_file.name, str(e))
gtest_xml = None
if test_perf_output_filename:
try:
device.PullFile(
isolated_script_test_perf_output.name,
test_perf_output_filename)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull chartjson results %s: %s',
isolated_script_test_perf_output.name, str(e))
test_artifacts_url = self._UploadTestArtifacts(
device, test_artifacts_dir)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
for l in output:
logging.info(l)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(
output, self._test_instance.symbolizer, device.product_cpu_abi)
tombstones_url = None
for r in results:
if logcat_file:
r.SetLink('logcat', logcat_file.Link())
if self._test_instance.gs_test_artifacts_bucket:
r.SetLink('test_artifacts', test_artifacts_url)
if r.GetType() == base_test_result.ResultType.CRASH:
self._crashes.add(r.GetName())
if self._test_instance.store_tombstones:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
r.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' %
device_tmp_results_file.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
with contextlib_ext.Optional(trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(test,
device,
flags=' '.join(flags),
timeout=timeout,
retries=0)
if self._test_instance.enable_xml_result_parsing:
gtest_xml = device.ReadFile(device_tmp_results_file.name,
as_root=True)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, resolved_tombstones)
result.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
