async def run(self, headless=True): LOG("Building %s x %s" % (self.scenario, self.customization_data["name"])) with self.env.get_device(2828, verbose=True) as device: try: with self.env.get_browser(): metadata = await self.build_profile(device, headless) finally: self.env.dump_logs() if not self.archive: return LOG("Creating archive") archiver = Archiver(self.scenario, self.env.profile, self.archive) # the archive name is of the form # profile-<platform>-<scenario>-<customization>.tgz name = "profile-%(platform)s-%(name)s-%(customization)s.tgz" name = name % metadata archive_name = os.path.join(self.archive, name) dir = os.path.dirname(archive_name) if not os.path.exists(dir): os.makedirs(dir) archiver.create_archive(archive_name) LOG("Archive created at %s" % archive_name) statinfo = os.stat(archive_name) LOG("Current size is %d" % statinfo.st_size) self.changelog.append("update", **metadata)
def __init__(self, profile_dir): self.metadata_file = os.path.join(profile_dir, METADATA_NAME) LOG("Reading existing metadata at %s" % self.metadata_file) if not os.path.exists(self.metadata_file): LOG("Could not find the metadata file in that profile") self._data = {} else: with open(self.metadata_file) as f: self._data = json.loads(f.read())
def dump_logs(self): logcat = self.device.get_logcat() if logcat: # local path, not using posixpath logfile = os.path.join(self.archive, "logcat.log") LOG("Writing logcat at %s" % logfile) with open(logfile, "w") as f: for line in logcat: f.write(line + "\n") else: LOG("logcat came back empty")
async def start(self): port = free_port() await self._check_version() LOG("Running Webdriver on port %d" % port) LOG("Running Marionette on port 2828") pargs = [ self.binary, "-vv", "--port", str(port), "--marionette-port", "2828" ] return await subprocess_based_service(pargs, f"http://localhost:{port}", self.log_file)
async def build_profile(self, device, headless): scenario = self.scenario profile = self.env.profile customization_data = self.customization_data scenario_func = scenarii[scenario] if scenario in customization_data.get("scenario", {}): options = customization_data["scenario"][scenario] LOG("Loaded options for that scenario %s" % str(options)) else: options = {} # Adding general options options["platform"] = self.env.target_platform if not self.force_new: try: custom_name = customization_data["name"] get_profile(profile, self.env.target_platform, scenario, custom_name) except ProfileNotFoundError: # XXX we'll use a fresh profile for now fresh_profile(profile, customization_data) else: fresh_profile(profile, customization_data) LOG("Updating profile located at %r" % profile) metadata = Metadata(profile) LOG("Starting the Gecko app...") self.env.prepare(logfile=self._log_filename("adb")) geckodriver_logs = self._log_filename("geckodriver") LOG("Writing geckodriver logs in %s" % geckodriver_logs) try: firefox_instance = Firefox(**self.env.get_browser_args(headless)) with open(geckodriver_logs, "w") as glog: async with get_session( self.env.get_geckodriver(log_file=glog), firefox_instance ) as session: self.env.check_session(session) LOG("Running the %s scenario" % scenario) metadata.update(await scenario_func(session, options)) LOG("%s scenario done." % scenario) except Exception: ERROR("%s scenario broke!" % scenario) self.env.stop_browser() self.env.collect_profile() # writing metadata metadata.write( name=self.scenario, customization=self.customization_data["name"], version=self.env.get_browser_version(), platform=self.env.target_platform, ) LOG("Profile at %s" % profile) LOG("Done.") return metadata
def dump_logs(self): LOG("Dumping Android logs") try: logcat = self.device.get_logcat() if logcat: # local path, not using posixpath logfile = os.path.join(self.archive, "logcat.log") LOG("Writing logcat at %s" % logfile) with open(logfile, "wb") as f: for line in logcat: f.write(line.encode("utf8", errors="replace") + b"\n") else: LOG("logcat came back empty") except Exception: ERROR("Could not extract the logcat")
def stop_browser(self): LOG("Stopping %s" % self.app_name) self.device.stop_application(self.app_name, root=True) time.sleep(5) if self.device.process_exist(self.app_name): LOG("%s still running, trying SIGKILL" % self.app_name) num_tries = 0 while self.device.process_exist(self.app_name) and num_tries < 5: try: self.device.pkill(self.app_name, root=True) except ADBError: pass num_tries += 1 time.sleep(1) LOG("%s stopped" % self.app_name)
async def run_all(args): if args.scenario != "all": selected_scenario = [args.scenario] else: selected_scenario = scenarii.keys() # this is the loop that generates all combinations of profile # for the current platform when "all" is selected res = [] failures = 0 for scenario in selected_scenario: if args.customization != "all": try: res.append(await one_run(scenario, args.customization)) except Exception: failures += 1 ERROR("Something went wrong on this one.") if args.strict: raise else: for customization in get_customizations(): LOG("Customization %s" % customization) try: res.append(await one_run(scenario, customization)) except Exception: failures += 1 ERROR("Something went wrong on this one.") if args.strict: raise return failures, [one_res for one_res in res if one_res]
def dump_logs(self): try: logcat = self.device.get_logcat() except ADBError: ERROR("logcat call failure") return if logcat: # local path, not using posixpath logfile = os.path.join(self.archive, "logcat.log") LOG("Writing logcat at %s" % logfile) with open(logfile, "wb") as f: for line in logcat: f.write(line.encode("utf8") + b"\n") else: LOG("logcat came back empty")
def close(self): self._unset_adb_logger() if self.device is None: return try: self.device.remove_forwards("tcp:%d" % self.marionette_port) except ADBError: LOG("Could not remove forward port")
def write(self, **extras): # writing metadata LOG("Creating metadata...") self._data.update(**extras) ts = str(datetime.datetime.now()) if "created" not in self._data: self._data["created"] = ts self._data["updated"] = ts # XXX need android arch version here days = self._delta(self._data["created"], self._data["updated"]) self._data["days"] = days self._data["age"] = self._days2age(days) # adding info about the firefox version # XXX build ID ?? # XXX android ?? LOG("Saving metadata file in %s" % self.metadata_file) with open(self.metadata_file, "w") as f: f.write(json.dumps(self._data))
def _set_adb_logger(self, log_file): self.log_file = log_file if self.log_file is None: return LOG("Setting ADB log file to %s" % self.log_file) adb_logger = logging.getLogger("adb") adb_logger.setLevel(logging.DEBUG) self._adb_fh = logging.FileHandler(self.log_file) self._adb_fh.setLevel(logging.DEBUG) adb_logger.addHandler(self._adb_fh)
async def start(self): port = free_port() await self._check_version() LOG("Running Webdriver on port %d" % port) LOG("Running Marionette on port 2828") pargs = [ self.binary, "--log", "trace", "--port", str(port), "--marionette-port", "2828", ] LOG("Connecting on Android device") pargs.append("--connect-existing") return await subprocess_based_service(pargs, f"http://localhost:{port}", self.log_file)
def read_changelog(platform, repo="mozilla-central"): params = {"platform": platform, "repo": repo} changelog_url = CHANGELOG_LINK % params LOG("Getting %s" % changelog_url) download_dir = tempfile.mkdtemp() downloaded_changelog = os.path.join(download_dir, "changelog.json") try: download_file(changelog_url, target=downloaded_changelog) except ArchiveNotFound: shutil.rmtree(download_dir) raise ProfileNotFoundError(changelog_url) return Changelog(download_dir)
def prepare(self, profile, logfile): self._set_adb_logger(logfile) try: self.device = ADBDevice(verbose=self.verbose, logger_name="adb") except Exception: ERROR("Cannot initialize device") raise device = self.device self.profile = profile # checking that the app is installed if not device.is_app_installed(self.app_name): raise Exception("%s is not installed" % self.app_name) # debug flag LOG("Setting %s as the debug app on the phone" % self.app_name) device.shell("am set-debug-app --persistent %s" % self.app_name, stdout_callback=LOG) # creating the profile on the device LOG("Creating the profile on the device") remote_test_root = posixpath.join(device.test_root, "condprof") remote_profile = posixpath.join(remote_test_root, "profile") LOG("The profile on the phone will be at %s" % remote_profile) device.rm(remote_test_root, force=True, recursive=True) device.mkdir(remote_test_root) device.chmod(remote_test_root, recursive=True, root=True) device.rm(remote_profile, force=True, recursive=True) LOG("Pushing %s on the phone" % self.profile) device.push(profile, remote_profile) device.chmod(remote_profile, recursive=True, root=True) self.profile = profile self.remote_profile = remote_profile # creating the yml file yml_data = { "args": ["-marionette", "-profile", self.remote_profile], "prefs": DEFAULT_PREFS, "env": { "LOG_VERBOSE": 1, "R_LOG_LEVEL": 6, "MOZ_LOG": "" }, } yml_name = "%s-geckoview-config.yaml" % self.app_name yml_on_host = posixpath.join(tempfile.mkdtemp(), yml_name) write_yml_file(yml_on_host, yml_data) tmp_on_device = posixpath.join("/data", "local", "tmp") if not device.exists(tmp_on_device): raise IOError("%s does not exists on the device" % tmp_on_device) yml_on_device = posixpath.join(tmp_on_device, yml_name) try: device.rm(yml_on_device, force=True, recursive=True) device.push(yml_on_host, yml_on_device) device.chmod(yml_on_device, recursive=True, root=True) except Exception: LOG("could not create the yaml file on device. Permission issue?") raise # command line 'extra' args not used with geckoview apps; instead we use # an on-device config.yml file intent = "android.intent.action.VIEW" device.stop_application(self.app_name) device.launch_application(self.app_name, self.activity, intent, extras=None, url="about:blank") if not device.process_exist(self.app_name): raise Exception("Could not start %s" % self.app_name) LOG("Creating socket forwarding on port %d" % self.marionette_port) device.forward( local="tcp:%d" % self.marionette_port, remote="tcp:%d" % self.marionette_port, ) # we don't have a clean way for now to check that GV or Fenix # is ready to handle our tests. So here we just wait 30s LOG("Sleeping for 30s") time.sleep(30)
def get_profile( target_dir, platform, scenario, customization="default", task_id=None, download_cache=True, repo="mozilla-central", ): """Extract a conditioned profile in the target directory. If task_id is provided, will grab the profile from that task. when not provided (default) will grab the latest profile. """ # XXX assert values params = { "platform": platform, "scenario": scenario, "customization": customization, "task_id": task_id, "repo": repo, } filename = ARTIFACT_NAME % params if task_id is None: url = TC_LINK % params + filename else: url = DIRECT_LINK % params + filename if not download_cache: download_dir = tempfile.mkdtemp() else: # using a cache dir in the user home dir download_dir = os.path.expanduser(CONDPROF_CACHE) if not os.path.exists(download_dir): os.makedirs(download_dir) downloaded_archive = os.path.join(download_dir, filename) retries = 0 while retries < RETRIES: try: LOG("Getting %s" % url) try: archive = download_file(url, target=downloaded_archive) except ArchiveNotFound: raise ProfileNotFoundError(url) try: with tarfile.open(archive, "r:gz") as tar: LOG("Extracting the tarball content in %s" % target_dir) size = len(list(tar)) with progress.Bar(expected_size=size) as bar: def _extract(self, *args, **kw): if not TASK_CLUSTER: bar.show(bar.last_progress + 1) return self.old(*args, **kw) tar.old = tar.extract tar.extract = functools.partial(_extract, tar) tar.extractall(target_dir) except (OSError, tarfile.ReadError) as e: LOG("Failed to extract the tarball") if download_cache and os.path.exists(archive): LOG("Removing cached file to attempt a new download") os.remove(archive) raise ProfileNotFoundError(str(e)) finally: if not download_cache: shutil.rmtree(download_dir) LOG("Success, we have a profile to work with") return target_dir except Exception: LOG("Failed to get the profile.") retries += 1 if os.path.exists(downloaded_archive): try: os.remove(downloaded_archive) except Exception: ERROR("Could not remove the file") time.sleep(RETRY_PAUSE) # If we reach that point, it means all attempts failed ERROR("All attempt failed") raise ProfileNotFoundError(url)
def main(args=sys.argv[1:]): parser = argparse.ArgumentParser(description="Profile Creator") parser.add_argument("archive", help="Archives Dir", type=str, default=None) parser.add_argument("--firefox", help="Firefox Binary", type=str, default=None) parser.add_argument("--scenario", help="Scenario to use", type=str, default="all") parser.add_argument("--profile", help="Existing profile Dir", type=str, default=None) parser.add_argument("--customization", help="Profile customization to use", type=str, default="all") parser.add_argument( "--fresh-profile", help="Create a fresh profile", action="store_true", default=False, ) parser.add_argument("--visible", help="Don't use headless mode", action="store_true", default=False) parser.add_argument("--archives-dir", help="Archives local dir", type=str, default="/tmp/archives") parser.add_argument("--force-new", help="Create from scratch", action="store_true", default=False) parser.add_argument( "--strict", help="Errors out immediatly on a scenario failure", action="store_true", default=True, ) parser.add_argument( "--geckodriver", help="Path to the geckodriver binary", type=str, default=sys.platform.startswith("win") and "geckodriver.exe" or "geckodriver", ) parser.add_argument("--device-name", help="Name of the device", type=str, default=None) args = parser.parse_args(args=args) # unpacking a dmg # XXX do something similar if we get an apk (but later) # XXX we want to do # adb install -r target.apk # and get the installed app name if args.firefox is not None and args.firefox.endswith("dmg"): target = os.path.join(os.path.dirname(args.firefox), "firefox.app") extract_from_dmg(args.firefox, target) args.firefox = os.path.join(target, "Contents", "MacOS", "firefox") args.android = args.firefox is not None and args.firefox.startswith( "org.mozilla") if not args.android and args.firefox is not None: LOG("Verifying Desktop Firefox binary") # we want to verify we do have a firefox binary # XXX so lame if not os.path.exists(args.firefox): if "MOZ_FETCHES_DIR" in os.environ: target = os.path.join(os.environ["MOZ_FETCHES_DIR"], args.firefox) if os.path.exists(target): args.firefox = target if not os.path.exists(args.firefox): raise IOError("Cannot find %s" % args.firefox) version = get_version(args.firefox) LOG("Working with Firefox %s" % version) LOG(os.environ) args.archive = os.path.abspath(args.archive) LOG("Archives directory is %s" % args.archive) if not os.path.exists(args.archive): os.makedirs(args.archive, exist_ok=True) LOG("Verifying Geckodriver binary presence") if shutil.which(args.geckodriver) is None and not os.path.exists( args.geckodriver): raise IOError("Cannot find %s" % args.geckodriver) try: plat = args.android and "android" or get_current_platform() changelog = read_changelog(plat) LOG("Got the changelog from TaskCluster") except ProfileNotFoundError: LOG("changelog not found on TaskCluster, creating a local one.") changelog = Changelog(args.archive) loop = asyncio.get_event_loop() async def one_run(scenario, customization): if args.android: env = AndroidEnv( args.profile, args.firefox, args.geckodriver, args.archive, args.device_name, ) else: env = DesktopEnv( args.profile, args.firefox, args.geckodriver, args.archive, args.device_name, ) return await ProfileCreator(scenario, customization, args.archive, changelog, args.force_new, env).run(not args.visible) async def run_all(args): if args.scenario != "all": return await one_run(args.scenario, args.customization) # this is the loop that generates all combinations of profile # for the current platform when "all" is selected res = [] for scenario in scenarii.keys(): if args.customization != "all": try: res.append(await one_run(scenario, args.customization)) except Exception: ERROR("Something went wrong on this one.") if args.strict: raise else: for customization in get_customizations(): try: res.append(await one_run(scenario, customization)) except Exception: ERROR("Something went wrong on this one.") if args.strict: raise return res try: loop.run_until_complete(run_all(args)) LOG("Saving changelog in %s" % args.archive) changelog.save(args.archive) finally: loop.close()
def collect_profile(self): LOG("Collecting profile from %s" % self.remote_profile) self.device.pull(self.remote_profile, self.profile)