def run_tests_on_device(dev, tests):
r = base_test_result.TestRunResults()
for test in tests:
result = self._RunTest(dev, test)
if isinstance(result, base_test_result.BaseTestResult):
r.AddResult(result)
elif isinstance(result, list):
r.AddResults(result)
else:
raise Exception('Unexpected result type: %s' %
type(result).__name__)
if isinstance(tests, test_collection.TestCollection):
tests.test_completed()
return r
def post_run(self, return_code):
if self._on_device_script:
os.remove(self._on_device_script)
# Create a simple json results file for a test run. The results will contain
# only one test (suite_name), and will either be a PASS or FAIL depending on
# return_code.
if self._test_launcher_summary_output:
result = (base_test_result.ResultType.FAIL if return_code else
base_test_result.ResultType.PASS)
suite_result = base_test_result.BaseTestResult(self.suite_name, result)
run_results = base_test_result.TestRunResults()
run_results.AddResult(suite_result)
with open(self._test_launcher_summary_output, 'w') as f:
json.dump(json_results.GenerateResultsDict([run_results]), f)
def RunTest(self, test):
raw_result = None
start_date_ms = None
results = base_test_result.TestRunResults()
timeout=(self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
try:
self.TestSetup(test)
start_date_ms = int(time.time()) * 1000
if self.is_uiautomator_test:
raw_result = self._RunUIAutomatorTest(test, timeout)
else:
raw_result = self.adb.RunInstrumentationTest(
test, self.test_pkg.GetPackageName(),
self._GetInstrumentationArgs(), timeout)
duration_ms = int(time.time()) * 1000 - start_date_ms
status_code = raw_result.GetStatusCode()
if status_code:
log = raw_result.GetFailureReason()
if not log:
log = 'No information.'
if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0:
self._TakeScreenshot(test)
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.FAIL, start_date_ms, duration_ms,
log=log)
else:
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.PASS, start_date_ms, duration_ms)
results.AddResult(result)
# Catch exceptions thrown by StartInstrumentation().
# See ../../third_party/android/testrunner/adb_interface.py
except (android_commands.errors.WaitForResponseTimedOutError,
android_commands.errors.DeviceUnresponsiveError,
android_commands.errors.InstrumentationError), e:
if start_date_ms:
duration_ms = int(time.time()) * 1000 - start_date_ms
else:
start_date_ms = int(time.time()) * 1000
duration_ms = 0
message = str(e)
if not message:
message = 'No information.'
results.AddResult(test_result.InstrumentationTestResult(
test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
log=message))
raw_result = None
def testGetTestsToRetry_testUnknown(self):
results = [
base_test_result.BaseTestResult(
'Test2', base_test_result.ResultType.PASS),
]
tests = ['Test1'] + [r.GetName() for r in results]
try_results = base_test_result.TestRunResults()
try_results.AddResults(results)
test_run = TestLocalDeviceTestRun()
tests_to_retry = test_run._GetTestsToRetry(tests, try_results)
self.assertEquals(1, len(tests_to_retry))
self.assertIn('Test1', tests_to_retry)
def RunTest(self, test):
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
start_ms = 0
duration_ms = 0
try:
self.TestSetup(test)
try:
self.device.GoHome()
except device_errors.CommandTimeoutError:
logging.exception('Failed to focus the launcher.')
time_ms = lambda: int(time.time() * 1000)
start_ms = time_ms()
raw_output = self._RunTest(test, timeout)
duration_ms = time_ms() - start_ms
# Parse the test output
result_code, result_bundle, statuses = (
instrumentation_test_instance.ParseAmInstrumentRawOutput(
raw_output))
result = self._GenerateTestResult(test, result_code, result_bundle,
statuses, start_ms, duration_ms)
if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
self.test_pkg.GetPackageName(), self.device):
result.SetType(base_test_result.ResultType.CRASH)
results.AddResult(result)
except device_errors.CommandTimeoutError as e:
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.TIMEOUT,
start_ms,
duration_ms,
log=str(e) or 'No information'))
except device_errors.DeviceUnreachableError as e:
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.CRASH,
start_ms,
duration_ms,
log=str(e) or 'No information'))
self.TestTeardown(test, results)
return (results, None if results.DidRunPass() else test)
def testGetTestsToRetry_wildcardFilter_allPass(self):
results = [
base_test_result.BaseTestResult(
'TestCase.Test1', base_test_result.ResultType.PASS),
base_test_result.BaseTestResult(
'TestCase.Test2', base_test_result.ResultType.PASS),
]
tests = ['TestCase.*']
try_results = base_test_result.TestRunResults()
try_results.AddResults(results)
test_run = TestLocalDeviceTestRun()
tests_to_retry = test_run._GetTestsToRetry(tests, try_results)
self.assertEquals(0, len(tests_to_retry))
def testGenerateJsonTestResultFormatDict_failedResultWithRetry(self):
result_1 = base_test_result.BaseTestResult(
'test.package.TestName', base_test_result.ResultType.FAIL)
run_results_1 = base_test_result.TestRunResults()
run_results_1.AddResult(result_1)
# Simulate a second retry with failure.
result_2 = base_test_result.BaseTestResult(
'test.package.TestName', base_test_result.ResultType.FAIL)
run_results_2 = base_test_result.TestRunResults()
run_results_2.AddResult(result_2)
all_results = [run_results_1, run_results_2]
results_dict = json_results.GenerateJsonTestResultFormatDict(
all_results, False)
self.assertEquals(1, len(results_dict['tests']))
self.assertEquals(1, len(results_dict['tests']['test']))
self.assertEquals(1, len(results_dict['tests']['test']['package']))
self.assertEquals(
'PASS',
results_dict['tests']['test']['package']['TestName']['expected'])
self.assertEquals(
'FAIL FAIL',
results_dict['tests']['test']['package']['TestName']['actual'])
self.assertEquals(
True, results_dict['tests']['test']['package']['TestName']
['is_unexpected'])
# Note: technically a missing entry counts as zero.
self.assertEquals(2, len(results_dict['num_failures_by_type']))
self.assertEquals(0, results_dict['num_failures_by_type']['PASS'])
# According to the spec: If a test was run more than once, only the first
# invocation's result is included in the totals.
self.assertEquals(1, results_dict['num_failures_by_type']['FAIL'])
def RunTestsOnShard(self):
results = base_test_result.TestRunResults()
for test in self._tests:
tries_left = self._retries
result_type = None
while (result_type != base_test_result.ResultType.PASS
and tries_left > 0):
try:
self._TestSetUp(test)
result_type = self._RunSingleTest(test)
finally:
self._TestTearDown()
results.AddResult(
base_test_result.BaseTestResult(test, result_type))
return results
def _RunInstrumentationTests(options, error_func, devices):
"""Subcommand of RunTestsCommands which runs instrumentation tests."""
instrumentation_options = ProcessInstrumentationOptions(options, error_func)
if len(devices) > 1 and options.wait_for_debugger:
logging.warning('Debugger can not be sharded, using first available device')
devices = devices[:1]
results = base_test_result.TestRunResults()
exit_code = 0
if options.run_java_tests:
runner_factory, tests = instrumentation_setup.Setup(instrumentation_options)
test_results, exit_code = test_dispatcher.RunTests(
tests, runner_factory, devices, shard=True, test_timeout=None,
num_retries=options.num_retries)
results.AddTestRunResults(test_results)
if options.run_python_tests:
runner_factory, tests = host_driven_setup.InstrumentationSetup(
options.host_driven_root, options.official_build,
instrumentation_options)
if tests:
test_results, test_exit_code = test_dispatcher.RunTests(
tests, runner_factory, devices, shard=True, test_timeout=None,
num_retries=options.num_retries)
results.AddTestRunResults(test_results)
# Only allow exit code escalation
if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
exit_code = test_exit_code
if options.device_flags:
options.device_flags = os.path.join(constants.DIR_SOURCE_ROOT,
options.device_flags)
report_results.LogFull(
results=results,
test_type='Instrumentation',
test_package=os.path.basename(options.test_apk),
annotation=options.annotations,
flakiness_server=options.flakiness_dashboard_server)
return exit_code
def RunTest(self, test_name):
"""Run a perf test on the device.
Args:
test_name: String to use for logging the test result.
Returns:
A tuple of (TestRunResults, retry).
"""
_, result_type = self._LaunchPerfTest(test_name)
results = base_test_result.TestRunResults()
results.AddResult(base_test_result.BaseTestResult(test_name, result_type))
retry = None
if not results.DidRunPass():
retry = test_name
return results, retry
def _ParseTestResults(self):
logging.info('Parsing results from stdout.')
results = base_test_result.TestRunResults()
output = self._results['results']['output'].splitlines()
output = (l[len(self._INSTRUMENTATION_STREAM_LEADER):] for l in output
if l.startswith(self._INSTRUMENTATION_STREAM_LEADER))
results_list = self._test_instance.ParseGTestOutput(output)
results.AddResults(results_list)
if self._env.only_output_failures:
logging.info('See logcat for more results information.')
if not self._results['results']['pass']:
results.AddResult(
base_test_result.BaseTestResult(
'Remote Service detected error.',
base_test_result.ResultType.FAIL))
return results
def RunTests(self):
with tempfile.NamedTemporaryFile() as json_file:
java_script = os.path.join(constants.GetOutDirectory(), 'bin', 'helper',
self._test_instance.suite)
command = [java_script]
# Add Jar arguments.
jar_args = ['-test-jars', self._test_instance.suite + '.jar',
'-json-results-file', json_file.name]
if self._test_instance.test_filter:
jar_args.extend(['-gtest-filter', self._test_instance.test_filter])
if self._test_instance.package_filter:
jar_args.extend(['-package-filter',
self._test_instance.package_filter])
if self._test_instance.runner_filter:
jar_args.extend(['-runner-filter', self._test_instance.runner_filter])
command.extend(['--jar-args', '"%s"' % ' '.join(jar_args)])
# Add JVM arguments.
jvm_args = []
# TODO(mikecase): Add a --robolectric-dep-dir arg to test runner.
# Have this arg set by GN in the generated test runner scripts.
jvm_args += [
'-Drobolectric.dependency.dir=%s' % os.path.join(
constants.GetOutDirectory(), 'lib.java', 'third_party',
'robolectric'),
'-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT,
]
if self._test_instance.coverage_dir:
if not os.path.exists(self._test_instance.coverage_dir):
os.makedirs(self._test_instance.coverage_dir)
elif not os.path.isdir(self._test_instance.coverage_dir):
raise Exception('--coverage-dir takes a directory, not file path.')
jvm_args.append('-Demma.coverage.out.file=%s' % os.path.join(
self._test_instance.coverage_dir,
'%s.ec' % self._test_instance.suite))
if jvm_args:
command.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)])
cmd_helper.RunCmd(command)
results_list = json_results.ParseResultsFromJson(
json.loads(json_file.read()))
test_run_results = base_test_result.TestRunResults()
test_run_results.AddResults(results_list)
return [test_run_results]
def RunTest(self, test):
raw_result = None
start_date_ms = None
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
try:
self.TestSetup(test)
start_date_ms = int(time.time()) * 1000
raw_result = self._RunTest(test, timeout)
duration_ms = int(time.time()) * 1000 - start_date_ms
status_code = raw_result.GetStatusCode()
if status_code:
if self.options.screenshot_failures:
self._TakeScreenshot(test)
log = raw_result.GetFailureReason()
if not log:
log = 'No information.'
result_type = base_test_result.ResultType.FAIL
package = self.adb.DismissCrashDialogIfNeeded()
# Assume test package convention of ".test" suffix
if package and package in self.test_pkg.GetPackageName():
result_type = base_test_result.ResultType.CRASH
result = test_result.InstrumentationTestResult(
test, result_type, start_date_ms, duration_ms, log=log)
else:
result = test_result.InstrumentationTestResult(
test, base_test_result.ResultType.PASS, start_date_ms, duration_ms)
results.AddResult(result)
# Catch exceptions thrown by StartInstrumentation().
# See ../../third_party/android/testrunner/adb_interface.py
except (android_commands.errors.WaitForResponseTimedOutError,
android_commands.errors.DeviceUnresponsiveError,
android_commands.errors.InstrumentationError), e:
if start_date_ms:
duration_ms = int(time.time()) * 1000 - start_date_ms
else:
start_date_ms = int(time.time()) * 1000
duration_ms = 0
message = str(e)
if not message:
message = 'No information.'
results.AddResult(test_result.InstrumentationTestResult(
test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms,
log=message))
raw_result = None
def Dispatch(options):
"""Dispatches instrumentation tests onto connected device(s).
If possible, this method will attempt to shard the tests to
all connected devices. Otherwise, dispatch and run tests on one device.
Args:
options: Command line options.
Returns:
A TestRunResults object holding the results of the Java tests.
Raises:
Exception: when there are no attached devices.
"""
test_pkg = test_package.TestPackage(options.test_apk_path,
options.test_apk_jar_path)
tests = test_pkg._GetAllMatchingTests(options.annotations,
options.exclude_annotations,
options.test_filter)
if not tests:
logging.warning('No instrumentation tests to run with current args.')
return base_test_result.TestRunResults()
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
raise Exception('There are no devices online.')
if options.device:
assert options.device in attached_devices
attached_devices = [options.device]
if len(attached_devices) > 1 and options.wait_for_debugger:
logging.warning(
'Debugger can not be sharded, using first available device')
attached_devices = attached_devices[:1]
def TestRunnerFactory(device, shard_index):
return test_runner.TestRunner(options, device, shard_index, test_pkg,
[])
return shard.ShardAndRunTests(TestRunnerFactory,
attached_devices,
tests,
options.build_type,
test_timeout=None,
num_retries=options.num_retries)
def RunTest(self, test):
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
if (self.device.build_version_sdk <
constants.ANDROID_SDK_VERSION_CODES.JELLY_BEAN):
timeout *= 10
start_ms = 0
duration_ms = 0
try:
self.TestSetup(test)
time_ms = lambda: int(time.time() * 1000)
start_ms = time_ms()
raw_output = self._RunTest(test, timeout)
duration_ms = time_ms() - start_ms
# Parse the test output
_, _, statuses = (instrumentation_test_instance.
ParseAmInstrumentRawOutput(raw_output))
result = self._GenerateTestResult(test, statuses, start_ms,
duration_ms)
if local_device_instrumentation_test_run.DidPackageCrashOnDevice(
self.test_pkg.GetPackageName(), self.device):
result.SetType(base_test_result.ResultType.CRASH)
results.AddResult(result)
except device_errors.CommandTimeoutError as e:
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.TIMEOUT,
start_ms,
duration_ms,
log=str(e) or 'No information'))
except device_errors.DeviceUnreachableError as e:
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.CRASH,
start_ms,
duration_ms,
log=str(e) or 'No information'))
self.TestTeardown(test, results)
return (results, None if results.DidRunPass() else test)
def testGenerateResultsDict_failedResult(self):
result = base_test_result.BaseTestResult(
'test.package.TestName', base_test_result.ResultType.FAIL)
all_results = base_test_result.TestRunResults()
all_results.AddResult(result)
results_dict = json_results.GenerateResultsDict([all_results])
self.assertEquals(['test.package.TestName'], results_dict['all_tests'])
self.assertEquals(1, len(results_dict['per_iteration_data']))
iteration_result = results_dict['per_iteration_data'][0]
self.assertTrue('test.package.TestName' in iteration_result)
self.assertEquals(1, len(iteration_result['test.package.TestName']))
test_iteration_result = iteration_result['test.package.TestName'][0]
self.assertTrue('status' in test_iteration_result)
self.assertEquals('FAILURE', test_iteration_result['status'])
def _ParseTestResults(self):
logging.info('Parsing results from stdout.')
r = base_test_result.TestRunResults()
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(
self._results['results']['output'].splitlines()))
result = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, 0, 0)
if isinstance(result, base_test_result.BaseTestResult):
r.AddResult(result)
elif isinstance(result, list):
r.AddResults(result)
else:
raise Exception('Unexpected result type: %s' % type(result).__name__)
self._DetectPlatformErrors(r)
return r
def testGenerateJsonTestResultFormatDict_failedResult(self):
result = base_test_result.BaseTestResult(
'test.package.TestName', base_test_result.ResultType.FAIL)
all_results = base_test_result.TestRunResults()
all_results.AddResult(result)
results_dict = json_results.GenerateJsonTestResultFormatDict(
[all_results])
self.assertEquals(1, len(results_dict['tests']))
self.assertEquals(1, len(results_dict['tests']['test']))
self.assertEquals(1, len(results_dict['tests']['test']['package']))
self.assertEquals(
'PASS',
results_dict['tests']['test']['package']['TestName']['expected'])
self.assertEquals(
'FAIL',
results_dict['tests']['test']['package']['TestName']['actual'])
def _RunInstrumentationTests(options, error_func):
"""Subcommand of RunTestsCommands which runs instrumentation tests."""
instrumentation_options = ProcessInstrumentationOptions(
options, error_func)
results = base_test_result.TestRunResults()
exit_code = 0
if options.run_java_tests:
runner_factory, tests = instrumentation_setup.Setup(
instrumentation_options)
test_results, exit_code = test_dispatcher.RunTests(
tests,
runner_factory,
options.wait_for_debugger,
options.test_device,
shard=True,
build_type=options.build_type,
test_timeout=None,
num_retries=options.num_retries)
results.AddTestRunResults(test_results)
if options.run_python_tests:
test_results, test_exit_code = (
python_dispatch.DispatchPythonTests(options))
results.AddTestRunResults(test_results)
# Only allow exit code escalation
if test_exit_code and exit_code != constants.ERROR_EXIT_CODE:
exit_code = test_exit_code
report_results.LogFull(results=results,
test_type='Instrumentation',
test_package=os.path.basename(options.test_apk),
annotation=options.annotations,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
return exit_code
def testGetTestsToRetry_nonStringTests(self):
results = [
base_test_result.BaseTestResult(
'TestCase.Test1', base_test_result.ResultType.PASS),
base_test_result.BaseTestResult(
'TestCase.Test2', base_test_result.ResultType.FAIL),
]
tests = [
{'name': 'TestCase.Test1'},
{'name': 'TestCase.Test2'},
]
try_results = base_test_result.TestRunResults()
try_results.AddResults(results)
test_run = TestLocalDeviceNonStringTestRun()
tests_to_retry = test_run._GetTestsToRetry(tests, try_results)
self.assertEquals(1, len(tests_to_retry))
self.assertIsInstance(tests_to_retry[0], dict)
self.assertEquals(tests[1], tests_to_retry[0])
def testGenerateResultsDict_duration(self):
result = base_test_result.BaseTestResult(
'test.package.TestName',
base_test_result.ResultType.PASS,
duration=123)
all_results = base_test_result.TestRunResults()
all_results.AddResult(result)
results_dict = json_results.GenerateResultsDict([all_results])
self.assertEquals(['test.package.TestName'], results_dict['all_tests'])
self.assertEquals(1, len(results_dict['per_iteration_data']))
iteration_result = results_dict['per_iteration_data'][0]
self.assertTrue('test.package.TestName' in iteration_result)
self.assertEquals(1, len(iteration_result['test.package.TestName']))
test_iteration_result = iteration_result['test.package.TestName'][0]
self.assertTrue('elapsed_time_ms' in test_iteration_result)
self.assertEquals(123, test_iteration_result['elapsed_time_ms'])
def _RunJavaTests(self, package_name, tests):
"""Calls a list of tests and stops at the first test failure.
This method iterates until either it encounters a non-passing test or it
exhausts the list of tests. Then it returns the appropriate overall result.
Test cases may make use of this method internally to assist in running
instrumentation tests. This function relies on instrumentation_options
being defined.
Args:
package_name: Package name in which the java tests live
(e.g. foo.bar.baz.tests)
tests: A list of Java test names which will be run
Returns:
A TestRunResults object containing an overall result for this set of Java
tests. If any Java tests do not pass, this is a fail overall.
"""
test_type = base_test_result.ResultType.PASS
log = ''
start_ms = int(time.time()) * 1000
for test in tests:
# We're only running one test at a time, so this TestRunResults object
# will hold only one result.
suite, test_name = test.split('.')
java_result = self.__RunJavaTest(package_name, suite, test_name)
assert len(java_result.GetAll()) == 1
if not java_result.DidRunPass():
result = java_result.GetNotPass().pop()
log = result.GetLog()
test_type = result.GetType()
break
duration_ms = int(time.time()) * 1000 - start_ms
overall_result = base_test_result.TestRunResults()
overall_result.AddResult(
test_result.InstrumentationTestResult(
self.tagged_name, test_type, start_ms, duration_ms, log=log))
return overall_result
def run_tests_on_device(dev, tests):
r = base_test_result.TestRunResults()
for test in tests:
try:
result = self._RunTest(dev, test)
if isinstance(result, base_test_result.BaseTestResult):
r.AddResult(result)
elif isinstance(result, list):
r.AddResults(result)
else:
raise Exception(
'Unexpected result type: %s' % type(result).__name__)
except:
if isinstance(tests, test_collection.TestCollection):
tests.add(test)
raise
finally:
if isinstance(tests, test_collection.TestCollection):
tests.test_completed()
logging.info('Finished running tests on this device.')
return r
def RunTests(tests, runner_factory):
"""Runs a set of java tests on the host.
Return:
A tuple containing the results & the exit code.
"""
def run(t):
runner = runner_factory(None, None)
runner.SetUp()
results_list, return_code = runner.RunTest(t)
runner.TearDown()
return (results_list, return_code == 0)
test_run_results = base_test_result.TestRunResults()
exit_code = 0
for t in tests:
results_list, passed = run(t)
test_run_results.AddResults(results_list)
if not passed:
exit_code = constants.ERROR_EXIT_CODE
return (test_run_results, exit_code)
def RunTest(self, test):
results = base_test_result.TestRunResults()
timeout = (self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale())
start_ms = 0
duration_ms = 0
try:
self.TestSetup(test)
time_ms = lambda: int(time.time() * 1000)
start_ms = time_ms()
raw_output = self._RunTest(test, timeout)
duration_ms = time_ms() - start_ms
# Parse the test output
_, _, statuses = self._ParseAmInstrumentRawOutput(raw_output)
result = self._GenerateTestResult(test, statuses, start_ms,
duration_ms)
results.AddResult(result)
except device_errors.CommandTimeoutError as e:
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.TIMEOUT,
start_ms,
duration_ms,
log=str(e) or 'No information'))
except device_errors.DeviceUnreachableError as e:
results.AddResult(
test_result.InstrumentationTestResult(
test,
base_test_result.ResultType.CRASH,
start_ms,
duration_ms,
log=str(e) or 'No information'))
self.TestTeardown(test, results)
return (results, None if results.DidRunPass() else test)
def RunTest(self, test):
test_results = base_test_result.TestRunResults()
if not test:
return test_results, None
try:
self.test_package.ClearApplicationState(self.device)
self.test_package.CreateCommandLineFileOnDevice(
self.device, test, self._test_arguments)
test_results = self._ParseTestOutput(
self.test_package.SpawnTestProcess(self.device))
finally:
self.CleanupSpawningServerState()
# Calculate unknown test results.
all_tests = set(test.split(':'))
all_tests_ran = set([t.GetName() for t in test_results.GetAll()])
unknown_tests = all_tests - all_tests_ran
test_results.AddResults(
[base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN)
for t in unknown_tests])
retry = ':'.join([t.GetName() for t in test_results.GetNotPass()])
return test_results, retry
def testGenerateJsonTestResultFormatDict_passedResult(self):
result = base_test_result.BaseTestResult(
'test.package.TestName', base_test_result.ResultType.PASS)
all_results = base_test_result.TestRunResults()
all_results.AddResult(result)
results_dict = json_results.GenerateJsonTestResultFormatDict(
[all_results], False)
self.assertEquals(1, len(results_dict['tests']))
self.assertEquals(1, len(results_dict['tests']['test']))
self.assertEquals(1, len(results_dict['tests']['test']['package']))
self.assertEquals(
'PASS',
results_dict['tests']['test']['package']['TestName']['expected'])
self.assertEquals(
'PASS',
results_dict['tests']['test']['package']['TestName']['actual'])
# Note: technically a missing entry counts as zero.
self.assertEquals(1, results_dict['num_failures_by_type']['PASS'])
self.assertEquals(0, results_dict['num_failures_by_type']['FAIL'])
def RunTest(self, test_name):
"""Run a Monkey test on the device.
Args:
test_name: String to use for logging the test result.
Returns:
A tuple of (TestRunResults, retry).
"""
self.adb.StartActivity(self._package,
self._activity,
wait_for_completion=True,
action='android.intent.action.MAIN',
force_stop=True)
# Chrome crashes are not always caught by Monkey test runner.
# Verify Chrome has the same PID before and after the test.
before_pids = self.adb.ExtractPid(self._package)
# Run the test.
output = ''
if before_pids:
output = '\n'.join(self._LaunchMonkeyTest())
after_pids = self.adb.ExtractPid(self._package)
crashed = (not before_pids or not after_pids
or after_pids[0] != before_pids[0])
results = base_test_result.TestRunResults()
success_pattern = 'Events injected: %d' % self._options.event_count
if success_pattern in output and not crashed:
result = base_test_result.BaseTestResult(
test_name, base_test_result.ResultType.PASS, log=output)
else:
result = base_test_result.BaseTestResult(
test_name, base_test_result.ResultType.FAIL, log=output)
results.AddResult(result)
return results, False
def _RunJavaTests(self, fname, tests):
"""Calls a list of tests and stops at the first test failure.
This method iterates until either it encounters a non-passing test or it
exhausts the list of tests. Then it returns the appropriate Python result.
Args:
fname: filename for the Python test
tests: a list of Java test names which will be run
Returns:
A TestRunResults object containing a result for this Python test.
"""
test_type = base_test_result.ResultType.PASS
log = ''
start_ms = int(time.time()) * 1000
for test in tests:
# We're only running one test at a time, so this TestRunResults object
# will hold only one result.
suite, test_name = test.split('.')
java_results = self._RunJavaTest(fname, suite, test_name)
assert len(java_results.GetAll()) == 1
if not java_results.DidRunPass():
result = java_results.GetNotPass().pop()
log = result.GetLog()
test_type = result.GetType()
break
duration_ms = int(time.time()) * 1000 - start_ms
python_results = base_test_result.TestRunResults()
python_results.AddResult(
test_result.InstrumentationTestResult(self.qualified_name,
test_type,
start_ms,
duration_ms,
log=log))
return python_results
def Run(self, device):
"""Run the test on a given device.
Args:
device: Name of target device where to run the test.
Returns:
A base_test_result.TestRunResult() instance.
"""
margin = 8
print '[ %-*s ] %s' % (margin, 'RUN', self.tagged_name)
logging.info('Running linker test: %s', self.tagged_name)
# Create command-line file on device.
if self.is_modern_linker:
command_line_flags = '--use-linker=modern'
else:
command_line_flags = '--use-linker=legacy'
if self.is_low_memory:
command_line_flags += ' --low-memory-device'
device.WriteFile(_COMMAND_LINE_FILE, command_line_flags)
# Run the test.
status, logs = self._RunTest(device)
result_text = 'OK'
if status == ResultType.FAIL:
result_text = 'FAILED'
elif status == ResultType.TIMEOUT:
result_text = 'TIMEOUT'
print '[ %*s ] %s' % (margin, result_text, self.tagged_name)
results = base_test_result.TestRunResults()
results.AddResult(
base_test_result.BaseTestResult(
self.tagged_name,
status,
log=logs))
return results
