class JUnitTestRunner(MochitestDesktop): """ A test harness to run geckoview junit tests on a remote device. """ def __init__(self, log, options): self.log = log self.verbose = False if ( options.log_tbpl_level == "debug" or options.log_mach_level == "debug" or options.verbose ): self.verbose = True self.device = ADBDeviceFactory( adb=options.adbPath or "adb", device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=self.verbose, run_as_package=options.app, ) self.options = options self.log.debug("options=%s" % vars(options)) update_mozinfo() self.remote_profile = posixpath.join(self.device.test_root, "junit-profile") self.remote_filter_list = posixpath.join( self.device.test_root, "junit-filters.list" ) if self.options.coverage and not self.options.coverage_output_dir: raise UserError( "--coverage-output-dir is required when using --enable-coverage" ) if self.options.coverage: self.remote_coverage_output_file = posixpath.join( self.device.test_root, "junit-coverage.ec" ) self.coverage_output_file = os.path.join( self.options.coverage_output_dir, "junit-coverage.ec" ) self.server_init() self.cleanup() self.device.clear_logcat() self.build_profile() self.startServers(self.options, debuggerInfo=None, public=True) self.log.debug("Servers started") def collectLogcatForCurrentTest(self): # These are unique start and end markers logged by GeckoSessionTestRule.java START_MARKER = "1f0befec-3ff2-40ff-89cf-b127eb38b1ec" END_MARKER = "c5ee677f-bc83-49bd-9e28-2d35f3d0f059" logcat = self.device.get_logcat() test_logcat = "" started = False for l in logcat: if START_MARKER in l and self.test_name in l: started = True if started: test_logcat += l + "\n" if started and END_MARKER in l: return test_logcat def needsWebsocketProcessBridge(self, options): """ Overrides MochitestDesktop.needsWebsocketProcessBridge and always returns False as the junit tests do not use the websocket process bridge. This is needed to satisfy MochitestDesktop.startServers. """ return False def server_init(self): """ Additional initialization required to satisfy MochitestDesktop.startServers """ self._locations = None self.server = None self.wsserver = None self.websocketProcessBridge = None self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get("debug") else 90 if self.options.remoteWebServer is None: self.options.remoteWebServer = moznetwork.get_ip() self.options.webServer = self.options.remoteWebServer self.options.webSocketPort = "9988" self.options.httpdPath = None self.options.keep_open = False self.options.pidFile = "" self.options.subsuite = None self.options.xrePath = None if build_obj and "MOZ_HOST_BIN" in os.environ: self.options.xrePath = os.environ["MOZ_HOST_BIN"] if not self.options.utilityPath: self.options.utilityPath = self.options.xrePath if not self.options.xrePath: self.options.xrePath = self.options.utilityPath if build_obj: self.options.certPath = os.path.join( build_obj.topsrcdir, "build", "pgo", "certs" ) def build_profile(self): """ Create a local profile with test prefs and proxy definitions and push it to the remote device. """ self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options)) self.options.profilePath = self.profile.profile # Set preferences self.merge_base_profiles(self.options, "geckoview-junit") prefs = parse_preferences(self.options.extra_prefs) self.profile.set_preferences(prefs) if self.fillCertificateDB(self.options): self.log.error("Certificate integration failed") self.device.push(self.profile.profile, self.remote_profile) self.log.debug( "profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile)) ) def cleanup(self): try: self.stopServers() self.log.debug("Servers stopped") self.device.stop_application(self.options.app) self.device.rm(self.remote_profile, force=True, recursive=True) if hasattr(self, "profile"): del self.profile self.device.rm(self.remote_filter_list, force=True) except Exception: traceback.print_exc() self.log.info("Caught and ignored an exception during cleanup") def build_command_line(self, test_filters_file, test_filters): """ Construct and return the 'am instrument' command line. """ cmd = "am instrument -w -r" # profile location cmd = cmd + " -e args '-profile %s'" % self.remote_profile # chunks (shards) shards = self.options.totalChunks shard = self.options.thisChunk if shards is not None and shard is not None: shard -= 1 # shard index is 0 based cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard) # test filters: limit run to specific test(s) # filter can be class-name or 'class-name#method-name' (single test) # Multiple filters must be specified as a line-separated text file # and then pushed to the device. filter_list_name = None if test_filters_file: # We specified a pre-existing file, so use that filter_list_name = test_filters_file elif test_filters: if len(test_filters) > 1: # Generate the list file from test_filters with tempfile.NamedTemporaryFile(delete=False, mode="w") as filter_list: for f in test_filters: print(f, file=filter_list) filter_list_name = filter_list.name else: # A single filter may be directly appended to the command line cmd = cmd + " -e class %s" % test_filters[0] if filter_list_name: self.device.push(filter_list_name, self.remote_filter_list) if test_filters: # We only remove the filter list if we generated it as a # temporary file. os.remove(filter_list_name) cmd = cmd + " -e testFile %s" % self.remote_filter_list # enable code coverage reports if self.options.coverage: cmd = cmd + " -e coverage true" cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_file # environment env = {} env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["XPCOM_DEBUG_BREAK"] = "stack" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["MOZ_IN_AUTOMATION"] = "1" env["R_LOG_VERBOSE"] = "1" env["R_LOG_LEVEL"] = "6" env["R_LOG_DESTINATION"] = "stderr" if self.options.enable_webrender: env["MOZ_WEBRENDER"] = "1" else: env["MOZ_WEBRENDER"] = "0" if self.options.enable_fission: env["MOZ_FORCE_ENABLE_FISSION"] = "1" # Add additional env variables for [key, value] in [p.split("=", 1) for p in self.options.add_env]: env[key] = value for (env_count, (env_key, env_val)) in enumerate(six.iteritems(env)): cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val) # runner cmd = cmd + " %s/%s" % (self.options.app, self.options.runner) return cmd @property def locations(self): if self._locations is not None: return self._locations locations_file = os.path.join(here, "server-locations.txt") self._locations = ServerLocations(locations_file) return self._locations def need_more_runs(self): if self.options.run_until_failure and (self.fail_count == 0): return True if self.runs <= self.options.repeat: return True return False def run_tests(self, test_filters_file=None, test_filters=None): """ Run the tests. """ if not self.device.is_app_installed(self.options.app): raise UserError("%s is not installed" % self.options.app) if self.device.process_exist(self.options.app): raise UserError( "%s already running before starting tests" % self.options.app ) # test_filters_file and test_filters must be mutually-exclusive if test_filters_file and test_filters: raise UserError( "Test filters may not be specified when test-filters-file is provided" ) self.test_started = False self.pass_count = 0 self.fail_count = 0 self.todo_count = 0 self.total_count = 0 self.runs = 0 self.seen_last_test = False def callback(line): # Output callback: Parse the raw junit log messages, translating into # treeherder-friendly test start/pass/fail messages. line = six.ensure_str(line) self.log.process_output(self.options.app, str(line)) # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something" match = re.match(r"INSTRUMENTATION_STATUS:\s*class=(.*)", line) if match: self.class_name = match.group(1) # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something" match = re.match(r"INSTRUMENTATION_STATUS:\s*test=(.*)", line) if match: self.test_name = match.group(1) match = re.match(r"INSTRUMENTATION_STATUS:\s*numtests=(.*)", line) if match: self.total_count = int(match.group(1)) match = re.match(r"INSTRUMENTATION_STATUS:\s*current=(.*)", line) if match: self.current_test_id = int(match.group(1)) match = re.match(r"INSTRUMENTATION_STATUS:\s*stack=(.*)", line) if match: self.exception_message = match.group(1) if ( "org.mozilla.geckoview.test.rule.TestHarnessException" in self.exception_message ): # This is actually a problem in the test harness itself raise JavaTestHarnessException(self.exception_message) # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..." match = re.match(r"INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)", line) if match: status = match.group(1) full_name = "%s#%s" % (self.class_name, self.test_name) if full_name == self.current_full_name: # A crash in the test harness might cause us to ignore tests, # so we double check that we've actually ran all the tests if self.total_count == self.current_test_id: self.seen_last_test = True if status == "0": message = "" status = "PASS" expected = "PASS" self.pass_count += 1 if self.verbose: self.log.info("Printing logcat for test:") print(self.collectLogcatForCurrentTest()) elif status == "-3": # ignored (skipped) message = "" status = "SKIP" expected = "SKIP" self.todo_count += 1 elif status == "-4": # known fail message = "" status = "FAIL" expected = "FAIL" self.todo_count += 1 else: if self.exception_message: message = self.exception_message else: message = "status %s" % status status = "FAIL" expected = "PASS" self.fail_count += 1 self.log.info("Printing logcat for test:") print(self.collectLogcatForCurrentTest()) self.log.test_end(full_name, status, expected, message) self.test_started = False else: if self.test_started: # next test started without reporting previous status self.fail_count += 1 status = "FAIL" expected = "PASS" self.log.test_end( self.current_full_name, status, expected, "missing test completion status", ) self.log.test_start(full_name) self.test_started = True self.current_full_name = full_name # Ideally all test names should be reported to suite_start, but these test # names are not known in advance. self.log.suite_start(["geckoview-junit"]) try: self.device.grant_runtime_permissions(self.options.app) cmd = self.build_command_line( test_filters_file=test_filters_file, test_filters=test_filters ) while self.need_more_runs(): self.class_name = "" self.exception_message = "" self.test_name = "" self.current_full_name = "" self.current_test_id = 0 self.runs += 1 self.log.info("launching %s" % cmd) p = self.device.shell( cmd, timeout=self.options.max_time, stdout_callback=callback ) if p.timedout: self.log.error( "TEST-UNEXPECTED-TIMEOUT | runjunit.py | " "Timed out after %d seconds" % self.options.max_time ) self.log.info("Passed: %d" % self.pass_count) self.log.info("Failed: %d" % self.fail_count) self.log.info("Todo: %d" % self.todo_count) if not self.seen_last_test: self.log.error( "TEST-UNEXPECTED-FAIL | runjunit.py | " "Some tests did not run (probably due to a crash in the harness)" ) finally: self.log.suite_end() if self.check_for_crashes(): self.fail_count = 1 if self.options.coverage: try: self.device.pull( self.remote_coverage_output_file, self.coverage_output_file ) except ADBError: # Avoid a task retry in case the code coverage file is not found. self.log.error( "No code coverage file (%s) found on remote device" % self.remote_coverage_output_file ) return -1 return 1 if self.fail_count else 0 def check_for_crashes(self): symbols_path = self.options.symbolsPath try: dump_dir = tempfile.mkdtemp() remote_dir = posixpath.join(self.remote_profile, "minidumps") if not self.device.is_dir(remote_dir): return False self.device.pull(remote_dir, dump_dir) crashed = mozcrash.log_crashes( self.log, dump_dir, symbols_path, test=self.current_full_name ) finally: try: shutil.rmtree(dump_dir) except Exception: self.log.warning("unable to remove directory: %s" % dump_dir) return crashed
class JUnitTestRunner(MochitestDesktop): """ A test harness to run geckoview junit tests on a remote device. """ def __init__(self, log, options): self.log = log verbose = False if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug': verbose = True self.device = ADBDevice(adb=options.adbPath or 'adb', device=options.deviceSerial, test_root=options.remoteTestRoot, verbose=verbose) self.options = options self.log.debug("options=%s" % vars(options)) update_mozinfo() self.remote_profile = posixpath.join(self.device.test_root, 'junit-profile') if self.options.coverage and not self.options.coverage_output_dir: raise Exception("--coverage-output-dir is required when using --enable-coverage") if self.options.coverage: self.remote_coverage_output_file = posixpath.join(self.device.test_root, 'junit-coverage.ec') self.coverage_output_file = os.path.join(self.options.coverage_output_dir, 'junit-coverage.ec') self.server_init() self.cleanup() self.device.clear_logcat() self.build_profile() self.startServers( self.options, debuggerInfo=None, public=True) self.log.debug("Servers started") def server_init(self): """ Additional initialization required to satisfy MochitestDesktop.startServers """ self._locations = None self.server = None self.wsserver = None self.websocketProcessBridge = None self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90 if self.options.remoteWebServer is None: if os.name != "nt": self.options.remoteWebServer = moznetwork.get_ip() else: raise Exception("--remote-webserver must be specified") self.options.webServer = self.options.remoteWebServer self.options.webSocketPort = '9988' self.options.httpdPath = None self.options.keep_open = False self.options.pidFile = "" self.options.subsuite = None self.options.xrePath = None if build_obj and 'MOZ_HOST_BIN' in os.environ: self.options.xrePath = os.environ['MOZ_HOST_BIN'] if not self.options.utilityPath: self.options.utilityPath = self.options.xrePath if not self.options.xrePath: self.options.xrePath = self.options.utilityPath if build_obj: self.options.certPath = os.path.join(build_obj.topsrcdir, 'build', 'pgo', 'certs') def build_profile(self): """ Create a local profile with test prefs and proxy definitions and push it to the remote device. """ self.profile = Profile(locations=self.locations, proxy=self.proxy(self.options)) self.options.profilePath = self.profile.profile # Set preferences self.merge_base_profiles(self.options, 'geckoview-junit') prefs = parse_preferences(self.options.extra_prefs) self.profile.set_preferences(prefs) if self.fillCertificateDB(self.options): self.log.error("Certificate integration failed") self.device.push(self.profile.profile, self.remote_profile) self.log.debug("profile %s -> %s" % (str(self.profile.profile), str(self.remote_profile))) def cleanup(self): try: self.stopServers() self.log.debug("Servers stopped") self.device.stop_application(self.options.app) self.device.rm(self.remote_profile, force=True, recursive=True, root=True) if hasattr(self, 'profile'): del self.profile except Exception: traceback.print_exc() self.log.info("Caught and ignored an exception during cleanup") def build_command_line(self, test_filters): """ Construct and return the 'am instrument' command line. """ cmd = "am instrument -w -r" # profile location cmd = cmd + " -e args '-profile %s'" % self.remote_profile # multi-process e10s = 'true' if self.options.e10s else 'false' cmd = cmd + " -e use_multiprocess %s" % e10s # chunks (shards) shards = self.options.totalChunks shard = self.options.thisChunk if shards is not None and shard is not None: shard -= 1 # shard index is 0 based cmd = cmd + " -e numShards %d -e shardIndex %d" % (shards, shard) # test filters: limit run to specific test(s) for f in test_filters: # filter can be class-name or 'class-name#method-name' (single test) cmd = cmd + " -e class %s" % f # enable code coverage reports if self.options.coverage: cmd = cmd + " -e coverage true" cmd = cmd + " -e coverageFile %s" % self.remote_coverage_output_file # environment env = {} env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["XPCOM_DEBUG_BREAK"] = "stack" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["MOZ_IN_AUTOMATION"] = "1" env["R_LOG_VERBOSE"] = "1" env["R_LOG_LEVEL"] = "6" env["R_LOG_DESTINATION"] = "stderr" if self.options.enable_webrender: env["MOZ_WEBRENDER"] = '1' else: env["MOZ_WEBRENDER"] = '0' for (env_count, (env_key, env_val)) in enumerate(env.iteritems()): cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val) # runner cmd = cmd + " %s/%s" % (self.options.app, self.options.runner) return cmd @property def locations(self): if self._locations is not None: return self._locations locations_file = os.path.join(here, 'server-locations.txt') self._locations = ServerLocations(locations_file) return self._locations def need_more_runs(self): if self.options.run_until_failure and (self.fail_count == 0): return True if self.runs <= self.options.repeat: return True return False def run_tests(self, test_filters=None): """ Run the tests. """ if not self.device.is_app_installed(self.options.app): raise Exception("%s is not installed" % self.options.app) if self.device.process_exist(self.options.app): raise Exception("%s already running before starting tests" % self.options.app) self.test_started = False self.pass_count = 0 self.fail_count = 0 self.todo_count = 0 self.runs = 0 def callback(line): # Output callback: Parse the raw junit log messages, translating into # treeherder-friendly test start/pass/fail messages. self.log.process_output(self.options.app, str(line)) # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something" match = re.match(r'INSTRUMENTATION_STATUS:\s*class=(.*)', line) if match: self.class_name = match.group(1) # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something" match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line) if match: self.test_name = match.group(1) match = re.match(r'INSTRUMENTATION_STATUS:\s*stack=(.*)', line) if match: self.exception_message = match.group(1) if "org.mozilla.geckoview.test.rule.TestHarnessException" in self.exception_message: # This is actually a problem in the test harness itself raise JavaTestHarnessException(self.exception_message) # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..." match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line) if match: status = match.group(1) full_name = "%s.%s" % (self.class_name, self.test_name) if full_name == self.current_full_name: if status == '0': message = '' status = 'PASS' expected = 'PASS' self.pass_count += 1 elif status == '-3': # ignored (skipped) message = '' status = 'SKIP' expected = 'SKIP' self.todo_count += 1 elif status == '-4': # known fail message = '' status = 'FAIL' expected = 'FAIL' self.todo_count += 1 else: if self.exception_message: message = self.exception_message else: message = 'status %s' % status status = 'FAIL' expected = 'PASS' self.fail_count += 1 self.log.test_end(full_name, status, expected, message) self.test_started = False else: if self.test_started: # next test started without reporting previous status self.fail_count += 1 status = 'FAIL' expected = 'PASS' self.log.test_end(self.current_full_name, status, expected, "missing test completion status") self.log.test_start(full_name) self.test_started = True self.current_full_name = full_name # Ideally all test names should be reported to suite_start, but these test # names are not known in advance. self.log.suite_start(["geckoview-junit"]) try: self.device.grant_runtime_permissions(self.options.app) cmd = self.build_command_line(test_filters) while self.need_more_runs(): self.class_name = "" self.exception_message = "" self.test_name = "" self.current_full_name = "" self.runs += 1 self.log.info("launching %s" % cmd) p = self.device.shell(cmd, timeout=self.options.max_time, stdout_callback=callback) if p.timedout: self.log.error("TEST-UNEXPECTED-TIMEOUT | runjunit.py | " "Timed out after %d seconds" % self.options.max_time) self.log.info("Passed: %d" % self.pass_count) self.log.info("Failed: %d" % self.fail_count) self.log.info("Todo: %d" % self.todo_count) finally: self.log.suite_end() if self.check_for_crashes(): self.fail_count = 1 if self.options.coverage: try: self.device.pull(self.remote_coverage_output_file, self.coverage_output_file) except ADBError: # Avoid a task retry in case the code coverage file is not found. self.log.error("No code coverage file (%s) found on remote device" % self.remote_coverage_output_file) return -1 return 1 if self.fail_count else 0 def check_for_crashes(self): symbols_path = self.options.symbolsPath try: dump_dir = tempfile.mkdtemp() remote_dir = posixpath.join(self.remote_profile, 'minidumps') if not self.device.is_dir(remote_dir): return False self.device.pull(remote_dir, dump_dir) crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path, test=self.current_full_name) finally: try: shutil.rmtree(dump_dir) except Exception: self.log.warning("unable to remove directory: %s" % dump_dir) return crashed