def RunTest(self, test):
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
cmdline_parameters = self._GetTestCmdlineParameters(test)
for flag_modifiers in cmdline_parameters:
start_ms = 0
duration_ms = 0
try:
if self._IsFreTest(test):
flag_modifiers.remove.append('--disable-fre')
self.TestSetup(test, flag_modifiers)
try:
self.device.GoHome()
except device_errors.CommandTimeoutError:
logging.exception('Failed to focus the launcher.')
time_ms = lambda: int(time.time() * 1000)
start_ms = time_ms()
raw_output = self._RunTest(test, timeout)
duration_ms = time_ms() - start_ms
# Parse the test output
result_code, result_bundle, statuses = (
instrumentation_test_instance.ParseAmInstrumentRawOutput(
raw_output))
result = self._GenerateTestResult(
test, result_code, result_bundle, statuses, start_ms, duration_ms)
if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
self.test_pkg.GetPackageName(), self.device):
result.SetType(base_test_result.ResultType.CRASH)
except device_errors.CommandTimeoutError as e:
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms,
log=str(e) or 'No information')
if self.package_info:
self.device.ForceStop(self.package_info.package)
self.device.ForceStop(self.package_info.test_package)
except device_errors.DeviceUnreachableError as e:
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.CRASH, start_ms, duration_ms,
log=str(e) or 'No information')
if len(cmdline_parameters) > 1:
# Specify commandline flag modifications used in the test run
result_name = result.GetName()
if flag_modifiers.add:
result_name = '%s with {%s}' % (
result_name, ' '.join(flag_modifiers.add))
if flag_modifiers.remove:
result_name = '%s without {%s}' % (
result_name, ' '.join(flag_modifiers.remove))
result.SetName(result_name)
results.AddResult(result)
self.TestTeardown(test, results)
return (results, None if results.DidRunPass() else test)
def RunTest(self, test):
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
if (self.device.GetProp('ro.build.version.sdk')
< constants.ANDROID_SDK_VERSION_CODES.JELLY_BEAN):
timeout *= 10
start_ms = 0
duration_ms = 0
try:
self.TestSetup(test)
time_ms = lambda: int(time.time() * 1000)
start_ms = time_ms()
raw_output = self._RunTest(test, timeout)
duration_ms = time_ms() - start_ms
# Parse the test output
_, _, statuses = self._ParseAmInstrumentRawOutput(raw_output)
result = self._GenerateTestResult(test, statuses, start_ms, duration_ms)
results.AddResult(result)
except device_errors.CommandTimeoutError as e:
results.AddResult(test_result.InstrumentationTestResult(
test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms,
log=str(e) or 'No information'))
except device_errors.DeviceUnreachableError as e:
results.AddResult(test_result.InstrumentationTestResult(
test, base_test_result.ResultType.CRASH, start_ms, duration_ms,
log=str(e) or 'No information'))
self.TestTeardown(test, results)
return (results, None if results.DidRunPass() else test)
def RunTest(self, test):
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
start_ms = 0
duration_ms = 0
try:
self.TestSetup(test)
time_ms = lambda: int(time.time() * 1000)
start_ms = time_ms()
raw_output = self._RunTest(test, timeout)
duration_ms = time_ms() - start_ms
# Parse the test output
result_code, result_bundle, statuses = (
instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output))
result = self._GenerateTestResult(
test, result_code, result_bundle, statuses, start_ms, duration_ms)
if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
self.test_pkg.GetPackageName(), self.device):
result.SetType(base_test_result.ResultType.CRASH)
results.AddResult(result)
except device_errors.CommandTimeoutError as e:
results.AddResult(test_result.InstrumentationTestResult(
test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms,
log=str(e) or 'No information'))
except device_errors.DeviceUnreachableError as e:
results.AddResult(test_result.InstrumentationTestResult(
test, base_test_result.ResultType.CRASH, start_ms, duration_ms,
log=str(e) or 'No information'))
self.TestTeardown(test, results)
return (results, None if results.DidRunPass() else test)
def RunTest(self, test):
raw_result = None
start_date_ms = None
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
try:
self.TestSetup(test)
start_date_ms = int(time.time()) * 1000
raw_result = self._RunTest(test, timeout)
duration_ms = int(time.time()) * 1000 - start_date_ms
status_code = raw_result.GetStatusCode()
if status_code:
if self.options.screenshot_failures:
self._TakeScreenshot(test)
log = raw_result.GetFailureReason()
if not log:
log = 'No information.'
result_type = base_test_result.ResultType.FAIL
package = self.adb.DismissCrashDialogIfNeeded()
# Assume test package convention of ".test" suffix
if package and package in self.test_pkg.GetPackageName():
result_type = base_test_result.ResultType.CRASH
result = test_result.InstrumentationTestResult(test,
result_type,
start_date_ms,
duration_ms,
log=log)
else:
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.PASS, start_date_ms,
duration_ms)
results.AddResult(result)
# Catch exceptions thrown by StartInstrumentation().
# See ../../third_party/android/testrunner/adb_interface.py
except (android_commands.errors.WaitForResponseTimedOutError,
android_commands.errors.DeviceUnresponsiveError,
android_commands.errors.InstrumentationError), e:
if start_date_ms:
duration_ms = int(time.time()) * 1000 - start_date_ms
else:
start_date_ms = int(time.time()) * 1000
duration_ms = 0
message = str(e)
if not message:
message = 'No information.'
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.CRASH,
start_date_ms,
duration_ms,
log=message))
raw_result = None
def _RunJavaTestFilters(self, test_filters, additional_flags=None):
"""Calls a list of tests and stops at the first test failure.
This method iterates until either it encounters a non-passing test or it
exhausts the list of tests. Then it returns the appropriate overall result.
Test cases may make use of this method internally to assist in running
instrumentation tests. This function relies on instrumentation_options
being defined.
Args:
test_filters: A list of Java test filters.
additional_flags: A list of addition flags to add to the command line.
Returns:
A TestRunResults object containing an overall result for this set of Java
tests. If any Java tests do not pass, this is a fail overall.
"""
test_type = base_test_result.ResultType.PASS
log = ''
test_pkg = test_package.TestPackage(
self.instrumentation_options.test_apk_path,
self.instrumentation_options.test_apk_jar_path,
self.instrumentation_options.test_support_apk_path)
start_ms = int(time.time()) * 1000
done = False
for test_filter in test_filters:
tests = test_pkg.GetAllMatchingTests(None, None, test_filter,
[self.device])
# Filters should always result in >= 1 test.
if len(tests) == 0:
raise Exception('Java test filter "%s" returned no tests.' %
test_filter)
for test in tests:
# We're only running one test at a time, so this TestRunResults object
# will hold only one result.
java_result = self.__RunJavaTest(test, test_pkg,
additional_flags)
assert len(java_result.GetAll()) == 1
if not java_result.DidRunPass():
result = java_result.GetNotPass().pop()
log = result.GetLog()
log += self.__GetHostForwarderLog()
test_type = result.GetType()
done = True
break
if done:
break
duration_ms = int(time.time()) * 1000 - start_ms
overall_result = base_test_result.TestRunResults()
overall_result.AddResult(
test_result.InstrumentationTestResult(self.tagged_name,
test_type,
start_ms,
duration_ms,
log=log))
return overall_result
def _GenerateTestResult(self, test, instr_statuses, start_ms, duration_ms):
"""Generate the result of |test| from |instr_statuses|.
Args:
instr_statuses: A list of 2-tuples containing:
- the status code as an integer
- the bundle dump as a dict mapping string keys to string values
Note that this is the same as the third item in the 3-tuple returned by
|_ParseAmInstrumentRawOutput|.
start_ms: The start time of the test in milliseconds.
duration_ms: The duration of the test in milliseconds.
Returns:
An InstrumentationTestResult object.
"""
INSTR_STATUS_CODE_START = 1
INSTR_STATUS_CODE_OK = 0
INSTR_STATUS_CODE_ERROR = -1
INSTR_STATUS_CODE_FAIL = -2
log = ''
result_type = base_test_result.ResultType.UNKNOWN
for status_code, bundle in instr_statuses:
if status_code == INSTR_STATUS_CODE_START:
pass
elif status_code == INSTR_STATUS_CODE_OK:
bundle_test = '%s#%s' % (
''.join(bundle.get('class', [''])),
''.join(bundle.get('test', [''])))
skipped = ''.join(bundle.get('test_skipped', ['']))
if (test == bundle_test and
result_type == base_test_result.ResultType.UNKNOWN):
result_type = base_test_result.ResultType.PASS
elif skipped.lower() in ('true', '1', 'yes'):
result_type = base_test_result.ResultType.SKIP
logging.info('Skipped ' + test)
else:
if status_code not in (INSTR_STATUS_CODE_ERROR,
INSTR_STATUS_CODE_FAIL):
logging.info('Unrecognized status code %d. Handling as an error.',
status_code)
result_type = base_test_result.ResultType.FAIL
if 'stack' in bundle:
log = '\n'.join(bundle['stack'])
# Dismiss any error dialogs. Limit the number in case we have an error
# loop or we are failing to dismiss.
for _ in xrange(10):
package = self.device.old_interface.DismissCrashDialogIfNeeded()
if not package:
break
# Assume test package convention of ".test" suffix
if package in self.test_pkg.GetPackageName():
result_type = base_test_result.ResultType.CRASH
break
return test_result.InstrumentationTestResult(
test, result_type, start_ms, duration_ms, log=log)
def _GenerateTestResult(self, test, instr_result_code, instr_result_bundle,
statuses, start_ms, duration_ms):
results = instrumentation_test_instance.GenerateTestResults(
instr_result_code, instr_result_bundle, statuses, start_ms, duration_ms)
for r in results:
if r.GetName() == test:
return r
logging.error('Could not find result for test: %s', test)
return test_result.InstrumentationTestResult(
test, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
def GenerateTestResult(test_name, instr_statuses, start_ms, duration_ms):
"""Generate the result of |test| from |instr_statuses|.
Args:
test_name: The name of the test as "class#method"
instr_statuses: A list of 2-tuples containing:
- the status code as an integer
- the bundle dump as a dict mapping string keys to string values
Note that this is the same as the third item in the 3-tuple returned by
|_ParseAmInstrumentRawOutput|.
start_ms: The start time of the test in milliseconds.
duration_ms: The duration of the test in milliseconds.
Returns:
An InstrumentationTestResult object.
"""
INSTR_STATUS_CODE_START = 1
INSTR_STATUS_CODE_OK = 0
INSTR_STATUS_CODE_ERROR = -1
INSTR_STATUS_CODE_FAIL = -2
log = ''
result_type = base_test_result.ResultType.UNKNOWN
for status_code, bundle in instr_statuses:
if status_code == INSTR_STATUS_CODE_START:
pass
elif status_code == INSTR_STATUS_CODE_OK:
bundle_test = '%s#%s' % (''.join(bundle.get(
'class', [''])), ''.join(bundle.get('test', [''])))
skipped = ''.join(bundle.get('test_skipped', ['']))
if (test_name == bundle_test
and result_type == base_test_result.ResultType.UNKNOWN):
result_type = base_test_result.ResultType.PASS
elif skipped.lower() in ('true', '1', 'yes'):
result_type = base_test_result.ResultType.SKIP
logging.info('Skipped ' + test_name)
else:
if status_code not in (INSTR_STATUS_CODE_ERROR,
INSTR_STATUS_CODE_FAIL):
logging.error(
'Unrecognized status code %d. Handling as an error.',
status_code)
result_type = base_test_result.ResultType.FAIL
if 'stack' in bundle:
log = '\n'.join(bundle['stack'])
return test_result.InstrumentationTestResult(test_name,
result_type,
start_ms,
duration_ms,
log=log)
def _RunJavaTests(self, package_name, tests):
"""Calls a list of tests and stops at the first test failure.
This method iterates until either it encounters a non-passing test or it
exhausts the list of tests. Then it returns the appropriate overall result.
Test cases may make use of this method internally to assist in running
instrumentation tests. This function relies on instrumentation_options
being defined.
Args:
package_name: Package name in which the java tests live
(e.g. foo.bar.baz.tests)
tests: A list of Java test names which will be run
Returns:
A TestRunResults object containing an overall result for this set of Java
tests. If any Java tests do not pass, this is a fail overall.
"""
test_type = base_test_result.ResultType.PASS
log = ''
start_ms = int(time.time()) * 1000
for test in tests:
# We're only running one test at a time, so this TestRunResults object
# will hold only one result.
suite, test_name = test.split('.')
java_result = self.__RunJavaTest(package_name, suite, test_name)
assert len(java_result.GetAll()) == 1
if not java_result.DidRunPass():
result = java_result.GetNotPass().pop()
log = result.GetLog()
test_type = result.GetType()
break
duration_ms = int(time.time()) * 1000 - start_ms
overall_result = base_test_result.TestRunResults()
overall_result.AddResult(
test_result.InstrumentationTestResult(
self.tagged_name, test_type, start_ms, duration_ms, log=log))
return overall_result
def _RunJavaTests(self, fname, tests):
"""Calls a list of tests and stops at the first test failure.
This method iterates until either it encounters a non-passing test or it
exhausts the list of tests. Then it returns the appropriate Python result.
Args:
fname: filename for the Python test
tests: a list of Java test names which will be run
Returns:
A TestRunResults object containing a result for this Python test.
"""
test_type = base_test_result.ResultType.PASS
log = ''
start_ms = int(time.time()) * 1000
for test in tests:
# We're only running one test at a time, so this TestRunResults object
# will hold only one result.
suite, test_name = test.split('.')
java_results = self._RunJavaTest(fname, suite, test_name)
assert len(java_results.GetAll()) == 1
if not java_results.DidRunPass():
result = java_results.GetNotPass().pop()
log = result.GetLog()
test_type = result.GetType()
break
duration_ms = int(time.time()) * 1000 - start_ms
python_results = base_test_result.TestRunResults()
python_results.AddResult(
test_result.InstrumentationTestResult(self.qualified_name,
test_type,
start_ms,
duration_ms,
log=log))
return python_results
def GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms, device_abi,
symbolizer):
"""Generate test results from |statuses|.
Args:
result_code: The overall status code as an integer.
result_bundle: The summary bundle dump as a dict.
statuses: A list of 2-tuples containing:
- the status code as an integer
- the bundle dump as a dict mapping string keys to string values
Note that this is the same as the third item in the 3-tuple returned by
|_ParseAmInstrumentRawOutput|.
start_ms: The start time of the test in milliseconds.
duration_ms: The duration of the test in milliseconds.
device_abi: The device_abi, which is needed for symbolization.
symbolizer: The symbolizer used to symbolize stack.
Returns:
A list containing an instance of InstrumentationTestResult for each test
parsed.
"""
results = []
current_result = None
for status_code, bundle in statuses:
test_class = bundle.get('class', '')
test_method = bundle.get('test', '')
if test_class and test_method:
test_name = '%s#%s' % (test_class, test_method)
else:
continue
if status_code == instrumentation_parser.STATUS_CODE_START:
if current_result:
results.append(current_result)
current_result = test_result.InstrumentationTestResult(
test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
else:
if status_code == instrumentation_parser.STATUS_CODE_OK:
if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'):
current_result.SetType(base_test_result.ResultType.SKIP)
elif current_result.GetType() == base_test_result.ResultType.UNKNOWN:
current_result.SetType(base_test_result.ResultType.PASS)
elif status_code == instrumentation_parser.STATUS_CODE_SKIP:
current_result.SetType(base_test_result.ResultType.SKIP)
elif status_code == instrumentation_parser.STATUS_CODE_ASSUMPTION_FAILURE:
current_result.SetType(base_test_result.ResultType.SKIP)
else:
if status_code not in (instrumentation_parser.STATUS_CODE_ERROR,
instrumentation_parser.STATUS_CODE_FAILURE):
logging.error('Unrecognized status code %d. Handling as an error.',
status_code)
current_result.SetType(base_test_result.ResultType.FAIL)
if 'stack' in bundle:
if symbolizer and device_abi:
current_result.SetLog(
'%s\n%s' % (
bundle['stack'],
'\n'.join(symbolizer.ExtractAndResolveNativeStackTraces(
bundle['stack'], device_abi))))
else:
current_result.SetLog(bundle['stack'])
if current_result:
if current_result.GetType() == base_test_result.ResultType.UNKNOWN:
crashed = (result_code == _ACTIVITY_RESULT_CANCELED
and any(_NATIVE_CRASH_RE.search(l)
for l in result_bundle.itervalues()))
if crashed:
current_result.SetType(base_test_result.ResultType.CRASH)
results.append(current_result)
return results
def GenerateTestResults(result_code, result_bundle, statuses, duration_ms,
device_abi, symbolizer):
"""Generate test results from |statuses|.
Args:
result_code: The overall status code as an integer.
result_bundle: The summary bundle dump as a dict.
statuses: A list of 2-tuples containing:
- the status code as an integer
- the bundle dump as a dict mapping string keys to string values
Note that this is the same as the third item in the 3-tuple returned by
|_ParseAmInstrumentRawOutput|.
duration_ms: The duration of the test in milliseconds.
device_abi: The device_abi, which is needed for symbolization.
symbolizer: The symbolizer used to symbolize stack.
Returns:
A list containing an instance of InstrumentationTestResult for each test
parsed.
"""
results = []
current_result = None
cumulative_duration = 0
for status_code, bundle in statuses:
# If the last test was a failure already, don't override that failure with
# post-test failures that could be caused by the original failure.
if (status_code == instrumentation_parser.STATUS_CODE_BATCH_FAILURE and
current_result.GetType() != base_test_result.ResultType.FAIL):
current_result.SetType(base_test_result.ResultType.FAIL)
_MaybeSetLog(bundle, current_result, symbolizer, device_abi)
continue
if status_code == instrumentation_parser.STATUS_CODE_TEST_DURATION:
# For the first result, duration will be set below to the difference
# between the reported and actual durations to account for overhead like
# starting instrumentation.
if results:
current_duration = int(
bundle.get(_BUNDLE_DURATION_ID, duration_ms))
current_result.SetDuration(current_duration)
cumulative_duration += current_duration
continue
test_class = bundle.get(_BUNDLE_CLASS_ID, '')
test_method = bundle.get(_BUNDLE_TEST_ID, '')
if test_class and test_method:
test_name = '%s#%s' % (test_class, test_method)
else:
continue
if status_code == instrumentation_parser.STATUS_CODE_START:
if current_result:
results.append(current_result)
current_result = test_result.InstrumentationTestResult(
test_name, base_test_result.ResultType.UNKNOWN, duration_ms)
else:
if status_code == instrumentation_parser.STATUS_CODE_OK:
if bundle.get(_BUNDLE_SKIPPED_ID,
'').lower() in ('true', '1', 'yes'):
current_result.SetType(base_test_result.ResultType.SKIP)
elif current_result.GetType(
) == base_test_result.ResultType.UNKNOWN:
current_result.SetType(base_test_result.ResultType.PASS)
elif status_code == instrumentation_parser.STATUS_CODE_SKIP:
current_result.SetType(base_test_result.ResultType.SKIP)
elif status_code == instrumentation_parser.STATUS_CODE_ASSUMPTION_FAILURE:
current_result.SetType(base_test_result.ResultType.SKIP)
else:
if status_code not in (
instrumentation_parser.STATUS_CODE_ERROR,
instrumentation_parser.STATUS_CODE_FAILURE):
logging.error(
'Unrecognized status code %d. Handling as an error.',
status_code)
current_result.SetType(base_test_result.ResultType.FAIL)
_MaybeSetLog(bundle, current_result, symbolizer, device_abi)
if current_result:
if current_result.GetType() == base_test_result.ResultType.UNKNOWN:
crashed = (result_code == _ACTIVITY_RESULT_CANCELED and any(
_NATIVE_CRASH_RE.search(l)
for l in six.itervalues(result_bundle)))
if crashed:
current_result.SetType(base_test_result.ResultType.CRASH)
results.append(current_result)
if results:
logging.info('Adding cumulative overhead to test %s: %dms',
results[0].GetName(), duration_ms - cumulative_duration)
results[0].SetDuration(duration_ms - cumulative_duration)
return results
def RunTest(self, test):
raw_result = None
start_date_ms = None
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
try:
self.TestSetup(test)
start_date_ms = int(time.time()) * 1000
raw_result = self._RunTest(test, timeout)
duration_ms = int(time.time()) * 1000 - start_date_ms
status_code = raw_result.GetStatusCode()
if status_code:
if self.options.screenshot_failures:
self._TakeScreenshot(test)
log = raw_result.GetFailureReason()
if not log:
log = 'No information.'
result_type = base_test_result.ResultType.FAIL
# Dismiss any error dialogs. Limit the number in case we have an error
# loop or we are failing to dismiss.
for _ in xrange(10):
package = self.device.old_interface.DismissCrashDialogIfNeeded(
)
if not package:
break
# Assume test package convention of ".test" suffix
if package in self.test_pkg.GetPackageName():
result_type = base_test_result.ResultType.CRASH
break
result = test_result.InstrumentationTestResult(test,
result_type,
start_date_ms,
duration_ms,
log=log)
else:
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.PASS, start_date_ms,
duration_ms)
results.AddResult(result)
# Catch exceptions thrown by StartInstrumentation().
# See ../../third_party/android/testrunner/adb_interface.py
except (
device_errors.CommandTimeoutError,
device_errors.DeviceUnreachableError,
# TODO(jbudorick) Remove these once the underlying implementations
# for the above are switched or wrapped.
android_commands.errors.WaitForResponseTimedOutError,
android_commands.errors.DeviceUnresponsiveError,
android_commands.errors.InstrumentationError), e:
if start_date_ms:
duration_ms = int(time.time()) * 1000 - start_date_ms
else:
start_date_ms = int(time.time()) * 1000
duration_ms = 0
message = str(e)
if not message:
message = 'No information.'
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.CRASH,
start_date_ms,
duration_ms,
log=message))
raw_result = None
