def _check_output(self, case_name, output):
"""Inspects the output of the test run and determines if a test passed.
Args:
case_name: The test case name.
output: The raw output string of the test case.
Returns:
A TestMethodResult object.
"""
output_file_path = os.path.join(self._work_dir, 'output.txt')
with open(output_file_path, 'w') as output_file:
output_file.write(_cleanup_output(output))
expected_file_path = os.path.join(self._source_dir, 'expected.txt')
# We diff the output against expected, ignoring whitespace,
# as the output file comes from running adb in ARC and so
# has \r\n instead of just \n.
try:
self.run_subprocess(
['diff', '-b', output_file_path, expected_file_path],
omit_xvfb=True)
except subprocess.CalledProcessError:
result = test_method_result.TestMethodResult(
case_name, test_method_result.TestMethodResult.FAIL)
else:
result = test_method_result.TestMethodResult(
case_name, test_method_result.TestMethodResult.PASS)
return result
def run(self, test_methods_to_run, scoreboard):
for case_name in test_methods_to_run:
output = None
with system_mode.SystemMode(self) as arc:
self._push_test_files(arc)
begin_time = time.time()
output = self._run_test(arc, case_name)
elapsed_time = time.time() - begin_time
# At this point, output can be None because evil SystemMode.__exit__()
# quietly suppress an exception raised from SystemMode.run_adb()!
if output is None:
# In this case, arc.run_adb() should have recorded some logs
# which will be retrieved by arc.get_log() later.
result = test_method_result.TestMethodResult(
case_name, test_method_result.TestMethodResult.FAIL)
elif self._is_benchmark:
self._logger.write(
'Benchmark %s: %d ms\n' % (case_name, elapsed_time * 1000))
result = test_method_result.TestMethodResult(
case_name, test_method_result.TestMethodResult.PASS)
else:
result = self._check_output(case_name, output)
scoreboard.update([result])
def _handle_status_code(self, match):
code = int(match.group(1))
if code == AtfInstrumentationResultParser._IN_PROGRESS:
# Ignore IN_PROGRESS message, which is delivered from
# android.cts.util.DeviceReportLog, as it is not related to test status.
return
assert self._current_class is not None
assert self._current_test is not None
duration = 0
if code == AtfInstrumentationResultParser._START:
self._current_test_start_time = time.time()
else:
duration = time.time() - self._current_test_start_time
fqn = self._current_class + '#' + self._current_test
assert (fqn not in self._test_fqn_to_result_map
or self._test_fqn_to_result_map[fqn].incomplete)
result = test_method_result.TestMethodResult(
fqn, self._METHOD_RESULT_MAP[code],
self._get_current_stream_message(), duration)
self._test_fqn_to_result_map[fqn] = result
self._clear_current_values()
self._update_tests_counts_from_result(result)
self._latest_result = result
def _process_test_end(self, match):
name = _build_test_name(
self._fixture_prefix + match.group('fixture'), match.group('method'))
result = test_method_result.TestMethodResult(
name,
GoogleTestResultParser._STATUS_CODE_MAP[match.group('status')],
duration=_parse_duration(match.group('duration')))
self._result_map[name] = result
self._callback.update([result])
def run_subprocess_test(self, scoreboard, test_name, command, env=None):
"""Runs a test which runs subprocess and sets status appropriately.
- test_name: The name of this test. This should be
'test_fixter#test_method' style.
- command: A list of strings. Command line to run subprocess.
- env: Environment dict for the subprocess. Maybe None.
"""
assert not launch_chrome_util.is_launch_chrome_command(command), (
'For testing with ./launch_chrome, you should use run_subprocess() '
'method instead.: %s' % str(command))
scoreboard.start_test(test_name)
try:
self.run_subprocess(command, env=env)
status = test_method_result.TestMethodResult.PASS
except subprocess.CalledProcessError:
status = test_method_result.TestMethodResult.FAIL
result = test_method_result.TestMethodResult(test_name, status)
scoreboard.update([result])
def _update_tests(self, sb, actuals):
for name, result in actuals.iteritems():
sb.start_test(name)
if result:
sb.update([test_method_result.TestMethodResult(name, result)])