class TestRunner(BaseTestRunner):
    """Responsible for running a series of tests connected to a single device."""

    _DEVICE_DATA_DIR = 'chrome/test/data'
    _EMMA_JAR = os.path.join(os.environ.get('ANDROID_BUILD_TOP', ''),
                             'external/emma/lib/emma.jar')
    _COVERAGE_MERGED_FILENAME = 'unittest_coverage.es'
    _COVERAGE_WEB_ROOT_DIR = os.environ.get('EMMA_WEB_ROOTDIR')
    _COVERAGE_FILENAME = 'coverage.ec'
    _COVERAGE_RESULT_PATH = (
        '/data/data/com.google.android.apps.chrome/files/' +
        _COVERAGE_FILENAME)
    _COVERAGE_META_INFO_PATH = os.path.join(
        os.environ.get('ANDROID_BUILD_TOP', ''), 'out/target/common/obj/APPS',
        'Chrome_intermediates/coverage.em')
    _HOSTMACHINE_PERF_OUTPUT_FILE = '/tmp/chrome-profile'
    _DEVICE_PERF_OUTPUT_SEARCH_PREFIX = (constants.DEVICE_PERF_OUTPUT_DIR +
                                         '/chrome-profile*')
    _DEVICE_HAS_TEST_FILES = {}

    def __init__(self, options, device, tests_iter, coverage, shard_index,
                 apks, ports_to_forward):
        """Create a new TestRunner.

    Args:
      options: An options object with the following required attributes:
      -  build_type: 'Release' or 'Debug'.
      -  install_apk: Re-installs the apk if opted.
      -  save_perf_json: Whether or not to save the JSON file from UI perf
            tests.
      -  screenshot_failures: Take a screenshot for a test failure
      -  tool: Name of the Valgrind tool.
      -  wait_for_debugger: blocks until the debugger is connected.
      -  disable_assertions: Whether to disable java assertions on the device.
      device: Attached android device.
      tests_iter: A list of tests to be run.
      coverage: Collects coverage information if opted.
      shard_index: shard # for this TestRunner, used to create unique port
          numbers.
      apks: A list of ApkInfo objects need to be installed. The first element
            should be the tests apk, the rests could be the apks used in test.
            The default is ChromeTest.apk.
      ports_to_forward: A list of port numbers for which to set up forwarders.
                        Can be optionally requested by a test case.
    Raises:
      FatalTestException: if coverage metadata is not available.
    """
        BaseTestRunner.__init__(self, device, options.tool, shard_index,
                                options.build_type)

        if not apks:
            apks = [
                apk_info.ApkInfo(options.test_apk_path,
                                 options.test_apk_jar_path)
            ]

        self.build_type = options.build_type
        self.install_apk = options.install_apk
        self.test_data = options.test_data
        self.save_perf_json = options.save_perf_json
        self.screenshot_failures = options.screenshot_failures
        self.wait_for_debugger = options.wait_for_debugger
        self.disable_assertions = options.disable_assertions

        self.tests_iter = tests_iter
        self.coverage = coverage
        self.apks = apks
        self.test_apk = apks[0]
        self.instrumentation_class_path = self.test_apk.GetPackageName()
        self.ports_to_forward = ports_to_forward

        self.test_results = TestResults()
        self.forwarder = None

        if self.coverage:
            if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
                os.remove(TestRunner._COVERAGE_MERGED_FILENAME)
            if not os.path.exists(TestRunner._COVERAGE_META_INFO_PATH):
                raise FatalTestException('FATAL ERROR in ' + sys.argv[0] +
                                         ' : Coverage meta info [' +
                                         TestRunner._COVERAGE_META_INFO_PATH +
                                         '] does not exist.')
            if (not TestRunner._COVERAGE_WEB_ROOT_DIR
                    or not os.path.exists(TestRunner._COVERAGE_WEB_ROOT_DIR)):
                raise FatalTestException(
                    'FATAL ERROR in ' + sys.argv[0] +
                    ' : Path specified in $EMMA_WEB_ROOTDIR [' +
                    TestRunner._COVERAGE_WEB_ROOT_DIR + '] does not exist.')

    def _GetTestsIter(self):
        if not self.tests_iter:
            # multiprocessing.Queue can't be pickled across processes if we have it as
            # a member set during constructor.  Grab one here instead.
            self.tests_iter = (BaseTestSharder.tests_container)
        assert self.tests_iter
        return self.tests_iter

    def CopyTestFilesOnce(self):
        """Pushes the test data files to the device. Installs the apk if opted."""
        if TestRunner._DEVICE_HAS_TEST_FILES.get(self.device, False):
            logging.warning(
                'Already copied test files to device %s, skipping.',
                self.device)
            return
        for dest_host_pair in self.test_data:
            dst_src = dest_host_pair.split(':', 1)
            dst_layer = dst_src[0]
            host_src = dst_src[1]
            host_test_files_path = constants.CHROME_DIR + '/' + host_src
            if os.path.exists(host_test_files_path):
                self.adb.PushIfNeeded(
                    host_test_files_path,
                    self.adb.GetExternalStorage() + '/' +
                    TestRunner._DEVICE_DATA_DIR + '/' + dst_layer)
        if self.install_apk:
            for apk in self.apks:
                self.adb.ManagedInstall(apk.GetApkPath(),
                                        package_name=apk.GetPackageName())
        self.tool.CopyFiles()
        TestRunner._DEVICE_HAS_TEST_FILES[self.device] = True

    def SaveCoverageData(self, test):
        """Saves the Emma coverage data before it's overwritten by the next test.

    Args:
      test: the test whose coverage data is collected.
    """
        if not self.coverage:
            return
        if not self.adb.Adb().Pull(TestRunner._COVERAGE_RESULT_PATH,
                                   constants.CHROME_DIR):
            logging.error('ERROR: Unable to find file ' +
                          TestRunner._COVERAGE_RESULT_PATH +
                          ' on the device for test ' + test)
        pulled_coverage_file = os.path.join(constants.CHROME_DIR,
                                            TestRunner._COVERAGE_FILENAME)
        if os.path.exists(TestRunner._COVERAGE_MERGED_FILENAME):
            cmd = [
                'java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'merge',
                '-in', pulled_coverage_file, '-in',
                TestRunner._COVERAGE_MERGED_FILENAME, '-out',
                TestRunner._COVERAGE_MERGED_FILENAME
            ]
            cmd_helper.RunCmd(cmd)
        else:
            shutil.copy(pulled_coverage_file,
                        TestRunner._COVERAGE_MERGED_FILENAME)
        os.remove(pulled_coverage_file)

    def GenerateCoverageReportIfNeeded(self):
        """Uses the Emma to generate a coverage report and a html page."""
        if not self.coverage:
            return
        cmd = [
            'java', '-classpath', TestRunner._EMMA_JAR, 'emma', 'report', '-r',
            'html', '-in', TestRunner._COVERAGE_MERGED_FILENAME, '-in',
            TestRunner._COVERAGE_META_INFO_PATH
        ]
        cmd_helper.RunCmd(cmd)
        new_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
                               time.strftime('Coverage_for_%Y_%m_%d_%a_%H:%M'))
        shutil.copytree('coverage', new_dir)

        latest_dir = os.path.join(TestRunner._COVERAGE_WEB_ROOT_DIR,
                                  'Latest_Coverage_Run')
        if os.path.exists(latest_dir):
            shutil.rmtree(latest_dir)
        os.mkdir(latest_dir)
        webserver_new_index = os.path.join(new_dir, 'index.html')
        webserver_new_files = os.path.join(new_dir, '_files')
        webserver_latest_index = os.path.join(latest_dir, 'index.html')
        webserver_latest_files = os.path.join(latest_dir, '_files')
        # Setup new softlinks to last result.
        os.symlink(webserver_new_index, webserver_latest_index)
        os.symlink(webserver_new_files, webserver_latest_files)
        cmd_helper.RunCmd(['chmod', '755', '-R', latest_dir, new_dir])

    def _GetInstrumentationArgs(self):
        ret = {}
        if self.coverage:
            ret['coverage'] = 'true'
        if self.wait_for_debugger:
            ret['debug'] = 'true'
        return ret

    def _TakeScreenshot(self, test):
        """Takes a screenshot from the device."""
        screenshot_name = os.path.join(constants.SCREENSHOTS_DIR,
                                       test + '.png')
        logging.info('Taking screenshot named %s', screenshot_name)
        self.adb.TakeScreenshot(screenshot_name)

    def SetUp(self):
        """Sets up the test harness and device before all tests are run."""
        super(TestRunner, self).SetUp()
        if not self.adb.IsRootEnabled():
            logging.warning(
                'Unable to enable java asserts for %s, non rooted device',
                self.device)
        else:
            if self.adb.SetJavaAssertsEnabled(
                    enable=not self.disable_assertions):
                self.adb.Reboot(full_reboot=False)

        # We give different default value to launch HTTP server based on shard index
        # because it may have race condition when multiple processes are trying to
        # launch lighttpd with same port at same time.
        http_server_ports = self.LaunchTestHttpServer(
            os.path.join(constants.CHROME_DIR),
            (constants.LIGHTTPD_RANDOM_PORT_FIRST + self.shard_index))
        if self.ports_to_forward:
            port_pairs = [(port, port) for port in self.ports_to_forward]
            # We need to remember which ports the HTTP server is using, since the
            # forwarder will stomp on them otherwise.
            port_pairs.append(http_server_ports)
            self.forwarder = Forwarder(self.adb, self.build_type)
            self.forwarder.Run(port_pairs, self.tool, '127.0.0.1')
        self.CopyTestFilesOnce()
        self.flags.AddFlags(['--enable-test-intents'])

    def TearDown(self):
        """Cleans up the test harness and saves outstanding data from test run."""
        if self.forwarder:
            self.forwarder.Close()
        self.GenerateCoverageReportIfNeeded()
        super(TestRunner, self).TearDown()

    def TestSetup(self, test):
        """Sets up the test harness for running a particular test.

    Args:
      test: The name of the test that will be run.
    """
        self.SetupPerfMonitoringIfNeeded(test)
        self._SetupIndividualTestTimeoutScale(test)
        self.tool.SetupEnvironment()

        # Make sure the forwarder is still running.
        self.RestartHttpServerForwarderIfNecessary()

    def _IsPerfTest(self, test):
        """Determines whether a test is a performance test.

    Args:
      test: The name of the test to be checked.

    Returns:
      Whether the test is annotated as a performance test.
    """
        return _PERF_TEST_ANNOTATION in self.test_apk.GetTestAnnotations(test)

    def SetupPerfMonitoringIfNeeded(self, test):
        """Sets up performance monitoring if the specified test requires it.

    Args:
      test: The name of the test to be run.
    """
        if not self._IsPerfTest(test):
            return
        self.adb.Adb().SendCommand(
            'shell rm ' + TestRunner._DEVICE_PERF_OUTPUT_SEARCH_PREFIX)
        self.adb.StartMonitoringLogcat()

    def TestTeardown(self, test, test_result):
        """Cleans up the test harness after running a particular test.

    Depending on the options of this TestRunner this might handle coverage
    tracking or performance tracking.  This method will only be called if the
    test passed.

    Args:
      test: The name of the test that was just run.
      test_result: result for this test.
    """

        self.tool.CleanUpEnvironment()

        # The logic below relies on the test passing.
        if not test_result or test_result.GetStatusCode():
            return

        self.TearDownPerfMonitoring(test)
        self.SaveCoverageData(test)

    def TearDownPerfMonitoring(self, test):
        """Cleans up performance monitoring if the specified test required it.

    Args:
      test: The name of the test that was just run.
    Raises:
      FatalTestException: if there's anything wrong with the perf data.
    """
        if not self._IsPerfTest(test):
            return
        raw_test_name = test.split('#')[1]

        # Wait and grab annotation data so we can figure out which traces to parse
        regex = self.adb.WaitForLogMatch(
            re.compile('\*\*PERFANNOTATION\(' + raw_test_name + '\)\:(.*)'),
            None)

        # If the test is set to run on a specific device type only (IE: only
        # tablet or phone) and it is being run on the wrong device, the test
        # just quits and does not do anything.  The java test harness will still
        # print the appropriate annotation for us, but will add --NORUN-- for
        # us so we know to ignore the results.
        # The --NORUN-- tag is managed by MainActivityTestBase.java
        if regex.group(1) != '--NORUN--':

            # Obtain the relevant perf data.  The data is dumped to a
            # JSON formatted file.
            json_string = self.adb.GetFileContents(
                '/data/data/com.google.android.apps.chrome/files/PerfTestData.txt'
            )

            if json_string:
                json_string = '\n'.join(json_string)
            else:
                raise FatalTestException(
                    'Perf file does not exist or is empty')

            if self.save_perf_json:
                json_local_file = '/tmp/chromium-android-perf-json-' + raw_test_name
                with open(json_local_file, 'w') as f:
                    f.write(json_string)
                logging.info('Saving Perf UI JSON from test ' + test + ' to ' +
                             json_local_file)

            raw_perf_data = regex.group(1).split(';')

            for raw_perf_set in raw_perf_data:
                if raw_perf_set:
                    perf_set = raw_perf_set.split(',')
                    if len(perf_set) != 3:
                        raise FatalTestException(
                            'Unexpected number of tokens in '
                            'perf annotation string: ' + raw_perf_set)

                    # Process the performance data
                    result = GetAverageRunInfoFromJSONString(
                        json_string, perf_set[0])

                    PrintPerfResult(perf_set[1], perf_set[2],
                                    [result['average']], result['units'])

    def _SetupIndividualTestTimeoutScale(self, test):
        timeout_scale = self._GetIndividualTestTimeoutScale(test)
        valgrind_tools.SetChromeTimeoutScale(self.adb, timeout_scale)

    def _GetIndividualTestTimeoutScale(self, test):
        """Returns the timeout scale for the given |test|."""
        annotations = self.apks[0].GetTestAnnotations(test)
        timeout_scale = 1
        if 'TimeoutScale' in annotations:
            for annotation in annotations:
                scale_match = re.match('TimeoutScale:([0-9]+)', annotation)
                if scale_match:
                    timeout_scale = int(scale_match.group(1))
        if self.wait_for_debugger:
            timeout_scale *= 100
        return timeout_scale

    def _GetIndividualTestTimeoutSecs(self, test):
        """Returns the timeout in seconds for the given |test|."""
        annotations = self.apks[0].GetTestAnnotations(test)
        if 'Manual' in annotations:
            return 600 * 60
        if 'External' in annotations:
            return 10 * 60
        if 'LargeTest' in annotations or _PERF_TEST_ANNOTATION in annotations:
            return 5 * 60
        if 'MediumTest' in annotations:
            return 3 * 60
        return 1 * 60

    def RunTests(self):
        """Runs the tests, generating the coverage if needed.

    Returns:
      A TestResults object.
    """
        instrumentation_path = (self.instrumentation_class_path +
                                '/android.test.InstrumentationTestRunner')
        instrumentation_args = self._GetInstrumentationArgs()
        for test in self._GetTestsIter():
            test_result = None
            start_date_ms = None
            try:
                self.TestSetup(test)
                start_date_ms = int(time.time()) * 1000
                args_with_filter = dict(instrumentation_args)
                args_with_filter['class'] = test
                # |test_results| is a list that should contain
                # a single TestResult object.
                logging.warn(args_with_filter)
                (test_results, _) = self.adb.Adb().StartInstrumentation(
                    instrumentation_path=instrumentation_path,
                    instrumentation_args=args_with_filter,
                    timeout_time=(self._GetIndividualTestTimeoutSecs(test) *
                                  self._GetIndividualTestTimeoutScale(test) *
                                  self.tool.GetTimeoutScale()))
                duration_ms = int(time.time()) * 1000 - start_date_ms
                assert len(test_results) == 1
                test_result = test_results[0]
                status_code = test_result.GetStatusCode()
                if status_code:
                    log = test_result.GetFailureReason()
                    if not log:
                        log = 'No information.'
                    if self.screenshot_failures or log.find(
                            'INJECT_EVENTS perm') >= 0:
                        self._TakeScreenshot(test)
                    self.test_results.failed += [
                        SingleTestResult(test, start_date_ms, duration_ms, log)
                    ]
                else:
                    result = [
                        SingleTestResult(test, start_date_ms, duration_ms)
                    ]
                    self.test_results.ok += result
            # Catch exceptions thrown by StartInstrumentation().
            # See ../../third_party/android/testrunner/adb_interface.py
            except (errors.WaitForResponseTimedOutError,
                    errors.DeviceUnresponsiveError,
                    errors.InstrumentationError), e:
                if start_date_ms:
                    duration_ms = int(time.time()) * 1000 - start_date_ms
                else:
                    start_date_ms = int(time.time()) * 1000
                    duration_ms = 0
                message = str(e)
                if not message:
                    message = 'No information.'
                self.test_results.crashed += [
                    SingleTestResult(test, start_date_ms, duration_ms, message)
                ]
                test_result = None
            self.TestTeardown(test, test_result)
        return self.test_results
Beispiel #2
0
 def _ForwardPorts(self, port_pairs):
     """Forwards a port."""
     Forwarder.Map(port_pairs, self.adb, self.tool)
 def SetUp(self):
   """Run once before all tests are run."""
   Forwarder.KillDevice(self.adb, self.tool)
   self.PushDependencies()
Beispiel #4
0
 def _UnmapPorts(self, port_pairs):
     """Unmap previously forwarded ports."""
     for (device_port, _) in port_pairs:
         Forwarder.UnmapDevicePort(device_port, self.adb)
 def _CreateAndRunForwarder(
     self, adb, port_pairs, tool, host_name, build_type):
   """Creates and run a forwarder."""
   forwarder = Forwarder(adb, build_type)
   forwarder.Run(port_pairs, tool, host_name)
   return forwarder
Beispiel #6
0
 def _KillHostForwarder(self):
   Forwarder.KillHost(self.build_type)
class BaseTestRunner(object):
    """Base class for running tests on a single device.

  A subclass should implement RunTests() with no parameter, so that calling
  the Run() method will set up tests, run them and tear them down.
  """
    def __init__(self, device, tool, build_type):
        """
      Args:
        device: Tests will run on the device of this ID.
        shard_index: Index number of the shard on which the test suite will run.
        build_type: 'Release' or 'Debug'.
    """
        self.device = device
        self.adb = android_commands.AndroidCommands(device=device)
        self.tool = CreateTool(tool, self.adb)
        self._http_server = None
        self._forwarder = None
        self._forwarder_device_port = 8000
        self.forwarder_base_url = ('http://localhost:%d' %
                                   self._forwarder_device_port)
        self.flags = FlagChanger(self.adb)
        self.flags.AddFlags(['--disable-fre'])
        self._spawning_server = None
        self._spawner_forwarder = None
        # We will allocate port for test server spawner when calling method
        # LaunchChromeTestServerSpawner and allocate port for test server when
        # starting it in TestServerThread.
        self.test_server_spawner_port = 0
        self.test_server_port = 0
        self.build_type = build_type

    def _PushTestServerPortInfoToDevice(self):
        """Pushes the latest port information to device."""
        self.adb.SetFileContents(
            self.adb.GetExternalStorage() + '/' +
            NET_TEST_SERVER_PORT_INFO_FILE,
            '%d:%d' % (self.test_server_spawner_port, self.test_server_port))

    def RunTest(self, test):
        """Runs a test. Needs to be overridden.

    Args:
      test: A test to run.

    Returns:
      Tuple containing:
        (base_test_result.TestRunResults, tests to rerun or None)
    """
        raise NotImplementedError

    def PushDependencies(self):
        """Push all dependencies to device once before all tests are run."""
        pass

    def SetUp(self):
        """Run once before all tests are run."""
        Forwarder.KillDevice(self.adb, self.tool)
        self.PushDependencies()

    def TearDown(self):
        """Run once after all tests are run."""
        self.ShutdownHelperToolsForTestSuite()

    def CopyTestData(self, test_data_paths, dest_dir):
        """Copies |test_data_paths| list of files/directories to |dest_dir|.

    Args:
      test_data_paths: A list of files or directories relative to |dest_dir|
          which should be copied to the device. The paths must exist in
          |DIR_SOURCE_ROOT|.
      dest_dir: Absolute path to copy to on the device.
    """
        for p in test_data_paths:
            self.adb.PushIfNeeded(os.path.join(constants.DIR_SOURCE_ROOT, p),
                                  os.path.join(dest_dir, p))

    def LaunchTestHttpServer(self,
                             document_root,
                             port=None,
                             extra_config_contents=None):
        """Launches an HTTP server to serve HTTP tests.

    Args:
      document_root: Document root of the HTTP server.
      port: port on which we want to the http server bind.
      extra_config_contents: Extra config contents for the HTTP server.
    """
        self._http_server = lighttpd_server.LighttpdServer(
            document_root,
            port=port,
            extra_config_contents=extra_config_contents)
        if self._http_server.StartupHttpServer():
            logging.info('http server started: http://localhost:%s',
                         self._http_server.port)
        else:
            logging.critical('Failed to start http server')
        self.StartForwarderForHttpServer()
        return (self._forwarder_device_port, self._http_server.port)

    def _ForwardPort(self, port_pairs):
        """Creates a forwarder instance if needed and forward a port."""
        if not self._forwarder:
            self._forwarder = Forwarder(self.adb, self.build_type)
        self._forwarder.Run(port_pairs, self.tool, '127.0.0.1')

    def StartForwarder(self, port_pairs):
        """Starts TCP traffic forwarding for the given |port_pairs|.

    Args:
      host_port_pairs: A list of (device_port, local_port) tuples to forward.
    """
        self._ForwardPort(port_pairs)

    def StartForwarderForHttpServer(self):
        """Starts a forwarder for the HTTP server.

    The forwarder forwards HTTP requests and responses between host and device.
    """
        self._ForwardPort([(self._forwarder_device_port,
                            self._http_server.port)])

    def RestartHttpServerForwarderIfNecessary(self):
        """Restarts the forwarder if it's not open."""
        # Checks to see if the http server port is being used.  If not forwards the
        # request.
        # TODO(dtrainor): This is not always reliable because sometimes the port
        # will be left open even after the forwarder has been killed.
        if not ports.IsDevicePortUsed(self.adb, self._forwarder_device_port):
            self.StartForwarderForHttpServer()

    def ShutdownHelperToolsForTestSuite(self):
        """Shuts down the server and the forwarder."""
        # Forwarders should be killed before the actual servers they're forwarding
        # to as they are clients potentially with open connections and to allow for
        # proper hand-shake/shutdown.
        Forwarder.KillDevice(self.adb, self.tool)
        if self._forwarder:
            self._forwarder.Close()
        if self._http_server:
            self._http_server.ShutdownHttpServer()
        if self._spawning_server:
            self._spawning_server.Stop()
        self.flags.Restore()

    def CleanupSpawningServerState(self):
        """Tells the spawning server to clean up any state.

    If the spawning server is reused for multiple tests, this should be called
    after each test to prevent tests affecting each other.
    """
        if self._spawning_server:
            self._spawning_server.CleanupState()

    def LaunchChromeTestServerSpawner(self):
        """Launches test server spawner."""
        server_ready = False
        error_msgs = []
        # TODO(pliard): deflake this function. The for loop should be removed as
        # well as IsHttpServerConnectable(). spawning_server.Start() should also
        # block until the server is ready.
        # Try 3 times to launch test spawner server.
        for i in xrange(0, 3):
            self.test_server_spawner_port = ports.AllocateTestServerPort()
            self._ForwardPort([(self.test_server_spawner_port,
                                self.test_server_spawner_port)])
            self._spawning_server = SpawningServer(
                self.test_server_spawner_port, self.adb, self.tool,
                self._forwarder, self.build_type)
            self._spawning_server.Start()
            server_ready, error_msg = ports.IsHttpServerConnectable(
                '127.0.0.1',
                self.test_server_spawner_port,
                path='/ping',
                expected_read='ready')
            if server_ready:
                break
            else:
                error_msgs.append(error_msg)
            self._spawning_server.Stop()
            # Wait for 2 seconds then restart.
            time.sleep(2)
        if not server_ready:
            logging.error(';'.join(error_msgs))
            raise Exception('Can not start the test spawner server.')
        self._PushTestServerPortInfoToDevice()
 def _ForwardPort(self, port_pairs):
     """Creates a forwarder instance if needed and forward a port."""
     if not self._forwarder:
         self._forwarder = Forwarder(self.adb, self.build_type)
     self._forwarder.Run(port_pairs, self.tool, '127.0.0.1')
Beispiel #9
0
 def SetUp(self):
     """Called before tests run."""
     Forwarder.KillDevice(self.adb, self.tool)