class BaseTestRunner(object):
"""Base class for running tests on a single device.
A subclass should implement RunTests() with no parameter, so that calling
the Run() method will set up tests, run them and tear them down.
"""
def __init__(self, device, tool, build_type):
"""
Args:
device: Tests will run on the device of this ID.
shard_index: Index number of the shard on which the test suite will run.
build_type: 'Release' or 'Debug'.
"""
self.device = device
self.adb = android_commands.AndroidCommands(device=device)
self.tool = CreateTool(tool, self.adb)
self._http_server = None
self._forwarder = None
self._forwarder_device_port = 8000
self.forwarder_base_url = ('http://localhost:%d' %
self._forwarder_device_port)
self.flags = FlagChanger(self.adb)
self.flags.AddFlags(['--disable-fre'])
self._spawning_server = None
self._spawner_forwarder = None
# We will allocate port for test server spawner when calling method
# LaunchChromeTestServerSpawner and allocate port for test server when
# starting it in TestServerThread.
self.test_server_spawner_port = 0
self.test_server_port = 0
self.build_type = build_type
def _PushTestServerPortInfoToDevice(self):
"""Pushes the latest port information to device."""
self.adb.SetFileContents(
self.adb.GetExternalStorage() + '/' +
NET_TEST_SERVER_PORT_INFO_FILE,
'%d:%d' % (self.test_server_spawner_port, self.test_server_port))
def RunTest(self, test):
"""Runs a test. Needs to be overridden.
Args:
test: A test to run.
Returns:
Tuple containing:
(base_test_result.TestRunResults, tests to rerun or None)
"""
raise NotImplementedError
def PushDependencies(self):
"""Push all dependencies to device once before all tests are run."""
pass
def SetUp(self):
"""Run once before all tests are run."""
Forwarder.KillDevice(self.adb, self.tool)
self.PushDependencies()
def TearDown(self):
"""Run once after all tests are run."""
self.ShutdownHelperToolsForTestSuite()
def CopyTestData(self, test_data_paths, dest_dir):
"""Copies |test_data_paths| list of files/directories to |dest_dir|.
Args:
test_data_paths: A list of files or directories relative to |dest_dir|
which should be copied to the device. The paths must exist in
|CHROME_DIR|.
dest_dir: Absolute path to copy to on the device.
"""
for p in test_data_paths:
self.adb.PushIfNeeded(os.path.join(constants.CHROME_DIR, p),
os.path.join(dest_dir, p))
def LaunchTestHttpServer(self,
document_root,
port=None,
extra_config_contents=None):
"""Launches an HTTP server to serve HTTP tests.
Args:
document_root: Document root of the HTTP server.
port: port on which we want to the http server bind.
extra_config_contents: Extra config contents for the HTTP server.
"""
self._http_server = lighttpd_server.LighttpdServer(
document_root,
port=port,
extra_config_contents=extra_config_contents)
if self._http_server.StartupHttpServer():
logging.info('http server started: http://localhost:%s',
self._http_server.port)
else:
logging.critical('Failed to start http server')
self.StartForwarderForHttpServer()
return (self._forwarder_device_port, self._http_server.port)
def _ForwardPort(self, port_pairs):
"""Creates a forwarder instance if needed and forward a port."""
if not self._forwarder:
self._forwarder = Forwarder(self.adb, self.build_type)
self._forwarder.Run(port_pairs, self.tool, '127.0.0.1')
def StartForwarder(self, port_pairs):
"""Starts TCP traffic forwarding for the given |port_pairs|.
Args:
host_port_pairs: A list of (device_port, local_port) tuples to forward.
"""
self._ForwardPort(port_pairs)
def StartForwarderForHttpServer(self):
"""Starts a forwarder for the HTTP server.
The forwarder forwards HTTP requests and responses between host and device.
"""
self._ForwardPort([(self._forwarder_device_port,
self._http_server.port)])
def RestartHttpServerForwarderIfNecessary(self):
"""Restarts the forwarder if it's not open."""
# Checks to see if the http server port is being used. If not forwards the
# request.
# TODO(dtrainor): This is not always reliable because sometimes the port
# will be left open even after the forwarder has been killed.
if not ports.IsDevicePortUsed(self.adb, self._forwarder_device_port):
self.StartForwarderForHttpServer()
def ShutdownHelperToolsForTestSuite(self):
"""Shuts down the server and the forwarder."""
# Forwarders should be killed before the actual servers they're forwarding
# to as they are clients potentially with open connections and to allow for
# proper hand-shake/shutdown.
Forwarder.KillDevice(self.adb, self.tool)
if self._forwarder:
self._forwarder.Close()
if self._http_server:
self._http_server.ShutdownHttpServer()
if self._spawning_server:
self._spawning_server.Stop()
self.flags.Restore()
def CleanupSpawningServerState(self):
"""Tells the spawning server to clean up any state.
If the spawning server is reused for multiple tests, this should be called
after each test to prevent tests affecting each other.
"""
if self._spawning_server:
self._spawning_server.CleanupState()
def LaunchChromeTestServerSpawner(self):
"""Launches test server spawner."""
server_ready = False
error_msgs = []
# TODO(pliard): deflake this function. The for loop should be removed as
# well as IsHttpServerConnectable(). spawning_server.Start() should also
# block until the server is ready.
# Try 3 times to launch test spawner server.
for i in xrange(0, 3):
self.test_server_spawner_port = ports.AllocateTestServerPort()
self._ForwardPort([(self.test_server_spawner_port,
self.test_server_spawner_port)])
self._spawning_server = SpawningServer(
self.test_server_spawner_port, self.adb, self.tool,
self._forwarder, self.build_type)
self._spawning_server.Start()
server_ready, error_msg = ports.IsHttpServerConnectable(
'127.0.0.1',
self.test_server_spawner_port,
path='/ping',
expected_read='ready')
if server_ready:
break
else:
error_msgs.append(error_msg)
self._spawning_server.Stop()
# Wait for 2 seconds then restart.
time.sleep(2)
if not server_ready:
logging.error(';'.join(error_msgs))
raise Exception('Can not start the test spawner server.')
self._PushTestServerPortInfoToDevice()
def _CreateAndRunForwarder(self, adb, port_pairs, tool, host_name,
build_type):
"""Creates and run a forwarder."""
forwarder = Forwarder(adb, build_type)
forwarder.Run(port_pairs, tool, host_name)
return forwarder
class TestRunner(BaseTestRunner):
"""Responsible for running a series of tests connected to a single device."""
_DEVICE_DATA_DIR = 'chrome/test/data'
_EMMA_JAR = os.path.join(os.environ.get('ANDROID_BUILD_TOP', ''),
'external/emma/lib/emma.jar')
_COVERAGE_MERGED_FILENAME = 'unittest_coverage.es'
_COVERAGE_WEB_ROOT_DIR = os.environ.get('EMMA_WEB_ROOTDIR')
_COVERAGE_FILENAME = 'coverage.ec'
_COVERAGE_RESULT_PATH = ('/data/data/com.google.android.apps.chrome/files/' +
_COVERAGE_FILENAME)
_COVERAGE_META_INFO_PATH = os.path.join(os.environ.get('ANDROID_BUILD_TOP',
''),
'out/target/common/obj/APPS',
'Chrome_intermediates/coverage.em')
_HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
_DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
'/chrome-profile*')
_DEVICE_HAS_TEST_FILES = {}
def __init__(self, options, device, tests_iter, coverage, shard_index, apks,
ports_to_forward):
"""Create a new TestRunner.
Args:
options: An options object with the following required attributes:
- build_type: 'Release' or 'Debug'.
- install_apk: Re-installs the apk if opted.
- save_perf_json: Whether or not to save the JSON file from UI perf
tests.
- screenshot_failures: Take a screenshot for a test failure
- tool: Name of the Valgrind tool.
- wait_for_debugger: blocks until the debugger is connected.
- disable_assertions: Whether to disable java assertions on the device.
device: Attached android device.
tests_iter: A list of tests to be run.
coverage: Collects coverage information if opted.
shard_index: shard # for this TestRunner, used to create unique port
numbers.
apks: A list of ApkInfo objects need to be installed. The first element
should be the tests apk, the rests could be the apks used in test.
The default is ChromeTest.apk.
ports_to_forward: A list of port numbers for which to set up forwarders.
Can be optionally requested by a test case.
Raises:
FatalTestException: if coverage metadata is not available.
"""
BaseTestRunner.__init__(
self, device, options.tool, shard_index, options.build_type)
if not apks:
apks = [apk_info.ApkInfo(options.test_apk_path,
options.test_apk_jar_path)]
self.build_type = options.build_type
self.install_apk = options.install_apk
self.test_data = options.test_data
self.save_perf_json = options.save_perf_json
self.screenshot_failures = options.screenshot_failures
self.wait_for_debugger = options.wait_for_debugger
self.disable_assertions = options.disable_assertions
self.tests_iter = tests_iter
self.coverage = coverage
self.apks = apks
self.test_apk = apks[0]
self.instrumentation_class_path = self.test_apk.GetPackageName()
self.ports_to_forward = ports_to_forward
self.test_results = TestResults()
self.forwarder = None
if self.coverage:
if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
os.remove(TestRunner._COVERAGE_MERGED_FILENAME)
if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH):
raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
' : Coverage meta info [' +
TestRunner._COVERAGE_META_INFO_PATH +
'] does not exist.')
if (not TestRunner._COVERAGE_WEB_ROOT_DIR or
not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)):
raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
' : Path specified in $EMMA_WEB_ROOTDIR [' +
TestRunner._COVERAGE_WEB_ROOT_DIR +
'] does not exist.')
def _GetTestsIter(self):
if not self.tests_iter:
# multiprocessing.Queue can't be pickled across processes if we have it as
# a member set during constructor. Grab one here instead.
self.tests_iter = (BaseTestSharder.tests_container)
assert self.tests_iter
return self.tests_iter
def CopyTestFilesOnce(self):
"""Pushes the test data files to the device. Installs the apk if opted."""
if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
logging.warning('Already copied test files to device %s, skipping.',
self.device)
return
for dest_host_pair in self.test_data:
dst_src = dest_host_pair.split(':',1)
dst_layer = dst_src[0]
host_src = dst_src[1]
host_test_files_path = constants.CHROME_DIR + '/' + host_src
if os.path.exists(host_test_files_path):
self.adb.PushIfNeeded(host_test_files_path,
self.adb.GetExternalStorage() + '/' +
TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
if self.install_apk:
for apk in self.apks:
self.adb.ManagedInstall(apk.GetApkPath(),
package_name=apk.GetPackageName())
self.tool.CopyFiles()
TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True
def SaveCoverageData(self, test):
"""Saves the Emma coverage data before it's overwritten by the next test.
Args:
test: the test whose coverage data is collected.
"""
if not self.coverage:
return
if not self.adb.Adb().Pull(TestRunner._COVERAGE_RESULT_PATH,
constants.CHROME_DIR):
logging.error('ERROR: Unable to find file ' +
TestRunner._COVERAGE_RESULT_PATH +
' on the device for test ' + test)
pulled_coverage_file = os.path.join(constants.CHROME_DIR,
TestRunner._COVERAGE_FILENAME)
if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
cmd = ['java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'merge',
'-in', pulled_coverage_file,
'-in', TestRunner._COVERAGE_MERGED_FILENAME,
'-out', TestRunner._COVERAGE_MERGED_FILENAME]
cmd_helper.RunCmd(cmd)
else:
shutil.copy(pulled_coverage_file,
TestRunner._COVERAGE_MERGED_FILENAME)
os.remove(pulled_coverage_file)
def GenerateCoverageReportIfNeeded(self):
"""Uses the Emma to generate a coverage report and a html page."""
if not self.coverage:
return
cmd = ['java', '-classpath', TestRunner._EMMA_JAR,
'emma', 'report', '-r', 'html',
'-in', TestRunner._COVERAGE_MERGED_FILENAME,
'-in', TestRunner._COVERAGE_META_INFO_PATH]
cmd_helper.RunCmd(cmd)
new_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
time.strftime('Coverage_for_%Y_%m_%d_%a_%H:%M'))
shutil.copytree('coverage', new_dir)
latest_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
'Latest_Coverage_Run')
if os.path.exists(latest_dir):
shutil.rmtree(latest_dir)
os.mkdir(latest_dir)
webserver_new_index = os.path.join(new_dir, 'index.html')
webserver_new_files = os.path.join(new_dir, '_files')
webserver_latest_index = os.path.join(latest_dir, 'index.html')
webserver_latest_files = os.path.join(latest_dir, '_files')
# Setup new softlinks to last result.
os.symlink(webserver_new_index, webserver_latest_index)
os.symlink(webserver_new_files, webserver_latest_files)
cmd_helper.RunCmd(['chmod', '755', '-R', latest_dir, new_dir])
def _GetInstrumentationArgs(self):
ret = {}
if self.coverage:
ret['coverage'] = 'true'
if self.wait_for_debugger:
ret['debug'] = 'true'
return ret
def _TakeScreenshot(self, test):
"""Takes a screenshot from the device."""
screenshot_name = os.path.join(constants.SCREENSHOTS_DIR, test + '.png')
logging.info('Taking screenshot named %s', screenshot_name)
self.adb.TakeScreenshot(screenshot_name)
def SetUp(self):
"""Sets up the test harness and device before all tests are run."""
super(TestRunner, self).SetUp()
if not self.adb.IsRootEnabled():
logging.warning('Unable to enable java asserts for %s, non rooted device',
self.device)
else:
if self.adb.SetJavaAssertsEnabled(enable=not self.disable_assertions):
self.adb.Reboot(full_reboot=False)
# We give different default value to launch HTTP server based on shard index
# because it may have race condition when multiple processes are trying to
# launch lighttpd with same port at same time.
http_server_ports = self.LaunchTestHttpServer(
os.path.join(constants.CHROME_DIR),
(constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index))
if self.ports_to_forward:
port_pairs = [(port, port) for port in self.ports_to_forward]
# We need to remember which ports the HTTP server is using, since the
# forwarder will stomp on them otherwise.
port_pairs.append(http_server_ports)
self.forwarder = Forwarder(self.adb, self.build_type)
self.forwarder.Run(port_pairs, self.tool, '127.0.0.1')
self.CopyTestFilesOnce()
self.flags.AddFlags(['--enable-test-intents'])
def TearDown(self):
"""Cleans up the test harness and saves outstanding data from test run."""
if self.forwarder:
self.forwarder.Close()
self.GenerateCoverageReportIfNeeded()
super(TestRunner, self).TearDown()
def TestSetup(self, test):
"""Sets up the test harness for running a particular test.
Args:
test: The name of the test that will be run.
"""
self.SetupPerfMonitoringIfNeeded(test)
self._SetupIndividualTestTimeoutScale(test)
self.tool.SetupEnvironment()
# Make sure the forwarder is still running.
self.RestartHttpServerForwarderIfNecessary()
def _IsPerfTest(self, test):
"""Determines whether a test is a performance test.
Args:
test: The name of the test to be checked.
Returns:
Whether the test is annotated as a performance test.
"""
return _PERF_TEST_ANNOTATION in self.test_apk.GetTestAnnotations(test)
def SetupPerfMonitoringIfNeeded(self, test):
"""Sets up performance monitoring if the specified test requires it.
Args:
test: The name of the test to be run.
"""
if not self._IsPerfTest(test):
return
self.adb.Adb().SendCommand('shell rm ' +
TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX)
self.adb.StartMonitoringLogcat()
def TestTeardown(self, test, test_result):
"""Cleans up the test harness after running a particular test.
Depending on the options of this TestRunner this might handle coverage
tracking or performance tracking. This method will only be called if the
test passed.
Args:
test: The name of the test that was just run.
test_result: result for this test.
"""
self.tool.CleanUpEnvironment()
# The logic below relies on the test passing.
if not test_result or test_result.GetStatusCode():
return
self.TearDownPerfMonitoring(test)
self.SaveCoverageData(test)
def TearDownPerfMonitoring(self, test):
"""Cleans up performance monitoring if the specified test required it.
Args:
test: The name of the test that was just run.
Raises:
FatalTestException: if there's anything wrong with the perf data.
"""
if not self._IsPerfTest(test):
return
raw_test_name = test.split('#')[1]
# Wait and grab annotation data so we can figure out which traces to parse
regex = self.adb.WaitForLogMatch(re.compile('\*\*PERFANNOTATION\(' +
raw_test_name +
'\)\:(.*)'), None)
# If the test is set to run on a specific device type only (IE: only
# tablet or phone) and it is being run on the wrong device, the test
# just quits and does not do anything. The java test harness will still
# print the appropriate annotation for us, but will add --NORUN-- for
# us so we know to ignore the results.
# The --NORUN-- tag is managed by MainActivityTestBase.java
if regex.group(1) != '--NORUN--':
# Obtain the relevant perf data. The data is dumped to a
# JSON formatted file.
json_string = self.adb.GetProtectedFileContents(
'/data/data/com.google.android.apps.chrome/files/PerfTestData.txt')
if json_string:
json_string = '\n'.join(json_string)
else:
raise FatalTestException('Perf file does not exist or is empty')
if self.save_perf_json:
json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
with open(json_local_file, 'w') as f:
f.write(json_string)
logging.info('Saving Perf UI JSON from test ' +
test + ' to ' + json_local_file)
raw_perf_data = regex.group(1).split(';')
for raw_perf_set in raw_perf_data:
if raw_perf_set:
perf_set = raw_perf_set.split(',')
if len(perf_set) != 3:
raise FatalTestException('Unexpected number of tokens in '
'perf annotation string: ' + raw_perf_set)
# Process the performance data
result = GetAverageRunInfoFromJSONString(json_string, perf_set[0])
PrintPerfResult(perf_set[1], perf_set[2],
[result['average']], result['units'])
def _SetupIndividualTestTimeoutScale(self, test):
timeout_scale = self._GetIndividualTestTimeoutScale(test)
valgrind_tools.SetChromeTimeoutScale(self.adb, timeout_scale)
def _GetIndividualTestTimeoutScale(self, test):
"""Returns the timeout scale for the given |test|."""
annotations = self.apks[0].GetTestAnnotations(test)
timeout_scale = 1
if 'TimeoutScale' in annotations:
for annotation in annotations:
scale_match = re.match('TimeoutScale:([0-9]+)', annotation)
if scale_match:
timeout_scale = int(scale_match.group(1))
if self.wait_for_debugger:
timeout_scale *= 100
return timeout_scale
def _GetIndividualTestTimeoutSecs(self, test):
"""Returns the timeout in seconds for the given |test|."""
annotations = self.apks[0].GetTestAnnotations(test)
if 'Manual' in annotations:
return 600 * 60
if 'External' in annotations:
return 10 * 60
if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
return 5 * 60
if 'MediumTest' in annotations:
return 3 * 60
return 1 * 60
def RunTests(self):
"""Runs the tests, generating the coverage if needed.
Returns:
A TestResults object.
"""
instrumentation_path = (self.instrumentation_class_path +
'/android.test.InstrumentationTestRunner')
instrumentation_args = self._GetInstrumentationArgs()
for test in self._GetTestsIter():
test_result = None
start_date_ms = None
try:
self.TestSetup(test)
start_date_ms = int(time.time()) * 1000
args_with_filter = dict(instrumentation_args)
args_with_filter['class'] = test
# |test_results| is a list that should contain
# a single TestResult object.
logging.warn(args_with_filter)
(test_results, _) = self.adb.Adb().StartInstrumentation(
instrumentation_path=instrumentation_path,
instrumentation_args=args_with_filter,
timeout_time=(self._GetIndividualTestTimeoutSecs(test) *
self._GetIndividualTestTimeoutScale(test) *
self.tool.GetTimeoutScale()))
duration_ms = int(time.time()) * 1000 - start_date_ms
assert len(test_results) == 1
test_result = test_results[0]
status_code = test_result.GetStatusCode()
if status_code:
log = test_result.GetFailureReason()
if not log:
log = 'No information.'
if self.screenshot_failures or log.find('INJECT_EVENTS perm') >= 0:
self._TakeScreenshot(test)
self.test_results.failed += [SingleTestResult(test, start_date_ms,
duration_ms, log)]
else:
result = [SingleTestResult(test, start_date_ms, duration_ms)]
self.test_results.ok += result
# Catch exceptions thrown by StartInstrumentation().
# See ../../third_party/android/testrunner/adb_interface.py
except (errors.WaitForResponseTimedOutError,
errors.DeviceUnresponsiveError,
errors.InstrumentationError), e:
if start_date_ms:
duration_ms = int(time.time()) * 1000 - start_date_ms
else:
start_date_ms = int(time.time()) * 1000
duration_ms = 0
message = str(e)
if not message:
message = 'No information.'
self.test_results.crashed += [SingleTestResult(test, start_date_ms,
duration_ms,
message)]
test_result = None
self.TestTeardown(test, test_result)
return self.test_results