async def __call__(self, session): metadata = {} tabs = TabSwitcher(session, self.options) await tabs.create_windows() visited = 0 for current, (url, word) in enumerate(self.urls): logger.info("%d/%d %s" % (current + 1, self.max_urls, url)) retries = 0 while retries < 3: try: await self._visit_url(current, session, url, word) visited += 1 break except CallErrors: await asyncio.sleep(1.0) retries += 1 if current == self.max_urls - 1: break # switch to the next tab try: await asyncio.wait_for(tabs.switch(), 5) except CallErrors: # if we can't switch, it's ok pass metadata["visited_url"] = visited await self.sync(session, metadata) return metadata
def onerror(error): logger.info("Failed to get the profile.") if os.path.exists(downloaded_archive): try: os.remove(downloaded_archive) except Exception: logger.error("Could not remove the file")
def __init__(self, options): self.options = options self.words = self._read_lines("words.txt") self.urls = self._build_url_list(self._read_lines("urls.txt")) self.sync_js = self._load_script("sync") self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS) self.bookmark_js = self._load_script("bookmark") self.platform = options.get("platform", "") self.mobile = is_mobile(self.platform) self.max_urls = options.get("max_urls", MAX_URLS) # see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar if self.mobile: self.max_urls = min(self.max_urls, 20) logger.info("platform: %s" % self.platform) logger.info("max_urls: %s" % self.max_urls) self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY) # we're syncing only on desktop for now self.syncing = not self.mobile if self.syncing: self.username, self.password = get_credentials() if self.username is None: raise ValueError( "Sync operations need an FxA username and password") else: self.username, self.password = None, None
async def close_extra_windows(session): logger.info("Closing all tabs") handles = await session.get_window_handles() # we're closing all tabs except the last one for handle in handles[:-1]: await session.switch_to_window(handle) await session._request(url="/window", method="DELETE")
async def run_all(self): """Runs the conditioned profile builders""" if self.scenario != "all": selected_scenario = [self.scenario] else: selected_scenario = scenarii.keys() # this is the loop that generates all combinations of profile # for the current platform when "all" is selected res = [] failures = 0 for scenario in selected_scenario: if self.customization != "all": try: res.append(await self.one_run(scenario, self.customization)) except Exception: failures += 1 self.display_error(scenario, self.customization) else: for customization in get_customizations(): logger.info("Customization %s" % customization) try: res.append(await self.one_run(scenario, customization)) except Exception: failures += 1 self.display_error(scenario, customization) return failures, [one_res for one_res in res if one_res]
def run( archive, firefox=None, scenario="all", profile=None, customization="all", visible=False, archives_dir="/tmp/archives", force_new=False, strict=True, geckodriver="geckodriver", device_name=None, ): runner = Runner(profile, firefox, geckodriver, archive, device_name, strict, force_new, visible) runner.prepare(scenario, customization) loop = asyncio.get_event_loop() try: failures, results = loop.run_until_complete(runner.run_all()) logger.info("Saving changelog in %s" % archive) runner.save() if failures > 0: raise Exception("At least one scenario failed") finally: loop.close()
def _keep_pref(name, value): for item in to_remove: if not name.startswith(item): continue logger.info("Removing pref %s: %s" % (name, value)) return False return True
def _get_profile(): logger.info("Getting %s" % url) try: archive = download_file(url, target=downloaded_archive) except ArchiveNotFound: raise ProfileNotFoundError(url) try: with tarfile.open(archive, "r:gz") as tar: logger.info("Extracting the tarball content in %s" % target_dir) size = len(list(tar)) with progress.Bar(expected_size=size) as bar: def _extract(self, *args, **kw): if not TASK_CLUSTER: bar.show(bar.last_progress + 1) return self.old(*args, **kw) tar.old = tar.extract tar.extract = functools.partial(_extract, tar) tar.extractall(target_dir) except (OSError, tarfile.ReadError) as e: logger.info("Failed to extract the tarball") if download_cache and os.path.exists(archive): logger.info("Removing cached file to attempt a new download") os.remove(archive) raise ProfileNotFoundError(str(e)) finally: if not download_cache: shutil.rmtree(download_dir) _check_profile(target_dir) logger.info("Success, we have a profile to work with") return target_dir
def __init__(self, profile_dir): self.metadata_file = os.path.join(profile_dir, METADATA_NAME) logger.info("Reading existing metadata at %s" % self.metadata_file) if not os.path.exists(self.metadata_file): logger.info("Could not find the metadata file in that profile") self._data = {} else: with open(self.metadata_file) as f: self._data = json.loads(f.read())
def _set_adb_logger(self, log_file): self.log_file = log_file if self.log_file is None: return logger.info("Setting ADB log file to %s" % self.log_file) adb_logger = logging.getLogger("adb") adb_logger.setLevel(logging.DEBUG) self._adb_fh = logging.FileHandler(self.log_file) self._adb_fh.setLevel(logging.DEBUG) adb_logger.addHandler(self._adb_fh)
def read_changelog(platform, repo="mozilla-central"): params = {"platform": platform, "repo": repo} changelog_url = CHANGELOG_LINK % params logger.info("Getting %s" % changelog_url) download_dir = tempfile.mkdtemp() downloaded_changelog = os.path.join(download_dir, "changelog.json") try: download_file(changelog_url, target=downloaded_changelog) except ArchiveNotFound: shutil.rmtree(download_dir) raise ProfileNotFoundError(changelog_url) return Changelog(download_dir)
async def start(self): port = free_port() await self._check_version() logger.info("Running Webdriver on port %d" % port) logger.info("Running Marionette on port 2828") pargs = [ self.binary, "--log", "trace", "--port", str(port), "--marionette-port", "2828", ] return await subprocess_based_service(pargs, f"http://localhost:{port}", self.log_file)
def write(self, **extras): # writing metadata logger.info("Creating metadata...") self._data.update(**extras) ts = str(datetime.datetime.now()) if "created" not in self._data: self._data["created"] = ts self._data["updated"] = ts # XXX need android arch version here days = self._delta(self._data["created"], self._data["updated"]) self._data["days"] = days self._data["age"] = self._days2age(days) # adding info about the firefox version # XXX build ID ?? # XXX android ?? logger.info("Saving metadata file in %s" % self.metadata_file) with open(self.metadata_file, "w") as f: f.write(json.dumps(self._data))
def _retries(callable, onerror=None): retries = 0 pause = RETRY_PAUSE while retries < RETRIES: try: return callable() except Exception as e: if onerror is not None: onerror(e) logger.info("Failed, retrying") retries += 1 time.sleep(pause) pause *= 1.5 # If we reach that point, it means all attempts failed logger.error("All attempt failed") raise RetriesError()
async def sync(self, session, metadata): if not self.syncing: return # now that we've visited all pages, we want to upload to FXSync logger.info("Syncing profile to FxSync") logger.info("Username is %s, password is %s" % (self.username, self.password)) script_res = await execute_async_script( session, self.sync_js, self.username, self.password, "https://accounts.stage.mozaws.net", ) if script_res is None: script_res = {} metadata["logs"] = script_res.get("logs", {}) metadata["result"] = script_res.get("result", 0) metadata["result_message"] = script_res.get("result_message", "SUCCESS") return metadata
def stop_browser(self): logger.info("Stopping %s" % self.app_name) try: self.device.stop_application(self.app_name) except ADBError: logger.info("Could not stop the application using force-stop") time.sleep(5) if self.device.process_exist(self.app_name): logger.info("%s still running, trying SIGKILL" % self.app_name) num_tries = 0 while self.device.process_exist(self.app_name) and num_tries < 5: try: self.device.pkill(self.app_name) except ADBError: pass num_tries += 1 time.sleep(1) logger.info("%s stopped" % self.app_name)
def dump_logs(self): logger.info("Dumping Android logs") try: logcat = self.device.get_logcat() if logcat: # local path, not using posixpath logfile = os.path.join(self.archive, "logcat.log") logger.info("Writing logcat at %s" % logfile) with open(logfile, "wb") as f: for line in logcat: f.write(line.encode("utf8", errors="replace") + b"\n") else: logger.info("logcat came back empty") except Exception: logger.error("Could not extract the logcat", exc_info=True)
async def execute_async_script(session, script, *args): # switch to the right context if needed current_context = await session._request(url="/moz/context", method="GET") if current_context != "chrome": logger.info("Switching to chrome context") await session._request(url="/moz/context", method="POST", data={"context": "chrome"}) switch_back = True else: switch_back = False logger.info("Setting up script timeout") await session._request(url="/timeouts", method="POST", data={"script": _SCRIPT_TIMEOUT}) try: attempts = 0 while True: logger.info("Running triggerSync()") try: return await session._request( url="/execute/async", method="POST", data={ "script": script, "args": list(args) }, ) except Exception as e: attempts += 1 logger.error("The script failed.", exc_info=True) if attempts > 2: return { "result": 1, "result_message": str(e), "result_exc": e, "logs": {}, } finally: if switch_back: await session._request(url="/moz/context", method="POST", data={"context": current_context})
async def add_bookmark(self, session, url, title): logger.info("Adding bookmark to %s" % url) return await execute_async_script(session, self.bookmark_js, url, title, self.max_bookmarks)
def collect_profile(self): logger.info("Collecting profile from %s" % self.remote_profile) self.device.pull(self.remote_profile, self.profile)
def prepare(self, scenario, customization): self.scenario = scenario self.customization = customization # early checks to avoid extra work if self.customization != "all": if find_customization(self.customization) is None: raise IOError("Cannot find customization %r" % self.customization) if self.scenario != "all" and self.scenario not in scenarii: raise IOError("Cannot find scenario %r" % self.scenario) if not self.android and self.firefox is not None: logger.info("Verifying Desktop Firefox binary") # we want to verify we do have a firefox binary # XXX so lame if not os.path.exists(self.firefox): if "MOZ_FETCHES_DIR" in os.environ: target = os.path.join(os.environ["MOZ_FETCHES_DIR"], self.firefox) if os.path.exists(target): self.firefox = target if not os.path.exists(self.firefox): raise IOError("Cannot find %s" % self.firefox) version = get_version(self.firefox) logger.info("Working with Firefox %s" % version) logger.info(os.environ) self.archive = os.path.abspath(self.archive) logger.info("Archives directory is %s" % self.archive) if not os.path.exists(self.archive): os.makedirs(self.archive, exist_ok=True) logger.info("Verifying Geckodriver binary presence") if shutil.which(self.geckodriver) is None and not os.path.exists( self.geckodriver): raise IOError("Cannot find %s" % self.geckodriver) try: if self.android: plat = "%s-%s" % ( self.device_name, self.firefox.split("org.mozilla.")[-1], ) else: plat = get_current_platform() self.changelog = read_changelog(plat) logger.info("Got the changelog from TaskCluster") except ProfileNotFoundError: logger.info( "changelog not found on TaskCluster, creating a local one.") self.changelog = Changelog(self.archive)
async def build_profile(self, device, headless): scenario = self.scenario profile = self.env.profile customization_data = self.customization_data scenario_func = scenarii[scenario] if scenario in customization_data.get("scenario", {}): options = customization_data["scenario"][scenario] logger.info("Loaded options for that scenario %s" % str(options)) else: options = {} # Adding general options options["platform"] = self.env.target_platform if not self.force_new: try: custom_name = customization_data["name"] get_profile(profile, self.env.target_platform, scenario, custom_name) except ProfileNotFoundError: # XXX we'll use a fresh profile for now fresh_profile(profile, customization_data) else: fresh_profile(profile, customization_data) logger.info("Updating profile located at %r" % profile) metadata = Metadata(profile) logger.info("Starting the Gecko app...") adb_logs = self._log_filename("adb") self.env.prepare(logfile=adb_logs) geckodriver_logs = self._log_filename("geckodriver") logger.info("Writing geckodriver logs in %s" % geckodriver_logs) step = START try: firefox_instance = Firefox(**self.env.get_browser_args(headless)) step = INIT_GECKODRIVER with open(geckodriver_logs, "w") as glog: geckodriver = self.env.get_geckodriver(log_file=glog) step = START_SESSION async with get_session(geckodriver, firefox_instance) as session: step = START_SCENARIO self.env.check_session(session) logger.info("Running the %s scenario" % scenario) metadata.update(await scenario_func(session, options)) logger.info("%s scenario done." % scenario) await close_extra_windows(session) except Exception: logger.error("%s scenario broke!" % scenario) if step == START: logger.info("Could not initialize the browser") elif step == INIT_GECKODRIVER: logger.info("Could not initialize Geckodriver") elif step == START_SESSION: logger.info("Could not start the session, check %s first" % geckodriver_logs) else: logger.info( "Could not run the scenario, probably a faulty scenario") raise finally: self.env.stop_browser() for logfile in (adb_logs, geckodriver_logs): if os.path.exists(logfile): obfuscate_file(logfile) self.env.collect_profile() # writing metadata metadata.write( name=self.scenario, customization=self.customization_data["name"], version=self.env.get_browser_version(), platform=self.env.target_platform, ) logger.info("Profile at %s.\nDone." % profile) return metadata
async def run(self, headless=True): logger.info("Building %s x %s" % (self.scenario, self.customization_data["name"])) if self.scenario in self.customization_data.get("ignore_scenario", []): logger.info("Skipping (ignored scenario in that customization)") return filter_by_platform = self.customization_data.get("platforms") if filter_by_platform and self.env.target_platform not in filter_by_platform: logger.info("Skipping (ignored platform in that customization)") return with self.env.get_device(2828, verbose=True) as device: try: with self.env.get_browser(): metadata = await self.build_profile(device, headless) finally: self.env.dump_logs() if not self.archive: return logger.info("Creating archive") archiver = Archiver(self.scenario, self.env.profile, self.archive) # the archive name is of the form # profile-<platform>-<scenario>-<customization>.tgz name = "profile-%(platform)s-%(name)s-%(customization)s.tgz" name = name % metadata archive_name = os.path.join(self.archive, name) dir = os.path.dirname(archive_name) if not os.path.exists(dir): os.makedirs(dir) archiver.create_archive(archive_name) logger.info("Archive created at %s" % archive_name) statinfo = os.stat(archive_name) logger.info("Current size is %d" % statinfo.st_size) logger.info("Extracting logs") if "logs" in metadata: logs = metadata.pop("logs") for prefix, prefixed_logs in logs.items(): for log in prefixed_logs: content = obfuscate(log["content"])[1] with open(os.path.join(dir, prefix + "-" + log["name"]), "wb") as f: f.write(content.encode("utf-8")) if metadata.get("result", 0) != 0: logger.info("The scenario returned a bad exit code") raise Exception(metadata.get("result_message", "scenario error")) self.changelog.append("update", **metadata)
def prepare(self, profile, logfile): self._set_adb_logger(logfile) try: # See android_emulator_pgo.py run_tests for more # details on why test_root must be /sdcard/test_root # for android pgo due to Android 4.3. self.device = ADBDeviceFactory(verbose=self.verbose, logger_name="adb", test_root="/sdcard/test_root") except Exception: logger.error("Cannot initialize device") raise device = self.device self.profile = profile # checking that the app is installed if not device.is_app_installed(self.app_name): raise Exception("%s is not installed" % self.app_name) # debug flag logger.info("Setting %s as the debug app on the phone" % self.app_name) device.shell( "am set-debug-app --persistent %s" % self.app_name, stdout_callback=logger.info, ) # creating the profile on the device logger.info("Creating the profile on the device") remote_test_root = posixpath.join(device.test_root, "condprof") remote_profile = posixpath.join(remote_test_root, "profile") logger.info("The profile on the phone will be at %s" % remote_profile) device.rm(remote_test_root, force=True, recursive=True) device.mkdir(remote_test_root) device.rm(remote_profile, force=True, recursive=True) logger.info("Pushing %s on the phone" % self.profile) device.push(profile, remote_profile) device.chmod(remote_profile, recursive=True) self.profile = profile self.remote_profile = remote_profile # creating the yml file yml_data = { "args": ["-marionette", "-profile", self.remote_profile], "prefs": DEFAULT_PREFS, "env": { "LOG_VERBOSE": 1, "R_LOG_LEVEL": 6, "MOZ_LOG": "" }, } yml_name = "%s-geckoview-config.yaml" % self.app_name yml_on_host = posixpath.join(tempfile.mkdtemp(), yml_name) write_yml_file(yml_on_host, yml_data) tmp_on_device = posixpath.join("/data", "local", "tmp") if not device.exists(tmp_on_device): raise IOError("%s does not exists on the device" % tmp_on_device) yml_on_device = posixpath.join(tmp_on_device, yml_name) try: device.rm(yml_on_device, force=True, recursive=True) device.push(yml_on_host, yml_on_device) device.chmod(yml_on_device, recursive=True) except Exception: logger.info( "could not create the yaml file on device. Permission issue?") raise # command line 'extra' args not used with geckoview apps; instead we use # an on-device config.yml file intent = "android.intent.action.VIEW" device.stop_application(self.app_name) if self.fennec: # XXX does the Fennec app picks up the YML file ? extra_args = [ "-profile", self.remote_profile, "--es", "env0", "LOG_VERBOSE=1", "--es", "env1", "R_LOG_LEVEL=6", "--es", "env2", "MOZ_WEBRENDER=0", ] device.launch_fennec( self.app_name, extra_args=extra_args, url="about:blank", fail_if_running=False, ) else: device.launch_application(self.app_name, self.activity, intent, extras=None, url="about:blank") if not device.process_exist(self.app_name): raise Exception("Could not start %s" % self.app_name) logger.info("Creating socket forwarding on port %d" % self.marionette_port) device.forward( local="tcp:%d" % self.marionette_port, remote="tcp:%d" % self.marionette_port, ) # we don't have a clean way for now to check that GV or Fenix # is ready to handle our tests. So here we just wait 30s logger.info("Sleeping for 30s") time.sleep(30)
def get_profile( target_dir, platform, scenario, customization="default", task_id=None, download_cache=True, repo="mozilla-central", ): """Extract a conditioned profile in the target directory. If task_id is provided, will grab the profile from that task. when not provided (default) will grab the latest profile. """ # XXX assert values params = { "platform": platform, "scenario": scenario, "customization": customization, "task_id": task_id, "repo": repo, } logger.info("Getting conditioned profile with arguments: %s" % params) filename = ARTIFACT_NAME % params if task_id is None: url = TC_LINK % params + filename _check_service(TC_SERVICE) else: url = DIRECT_LINK % params + filename _check_service(ARTIFACTS_SERVICE) logger.info("preparing download dir") if not download_cache: download_dir = tempfile.mkdtemp() else: # using a cache dir in the user home dir download_dir = os.path.expanduser(CONDPROF_CACHE) if not os.path.exists(download_dir): os.makedirs(download_dir) downloaded_archive = os.path.join(download_dir, filename) logger.info("Downloaded archive path: %s" % downloaded_archive) def _get_profile(): logger.info("Getting %s" % url) try: archive = download_file(url, target=downloaded_archive) except ArchiveNotFound: raise ProfileNotFoundError(url) try: with tarfile.open(archive, "r:gz") as tar: logger.info("Extracting the tarball content in %s" % target_dir) size = len(list(tar)) with progress.Bar(expected_size=size) as bar: def _extract(self, *args, **kw): if not TASK_CLUSTER: bar.show(bar.last_progress + 1) return self.old(*args, **kw) tar.old = tar.extract tar.extract = functools.partial(_extract, tar) tar.extractall(target_dir) except (OSError, tarfile.ReadError) as e: logger.info("Failed to extract the tarball") if download_cache and os.path.exists(archive): logger.info("Removing cached file to attempt a new download") os.remove(archive) raise ProfileNotFoundError(str(e)) finally: if not download_cache: shutil.rmtree(download_dir) _check_profile(target_dir) logger.info("Success, we have a profile to work with") return target_dir def onerror(error): logger.info("Failed to get the profile.") if os.path.exists(downloaded_archive): try: os.remove(downloaded_archive) except Exception: logger.error("Could not remove the file") try: return _retries(_get_profile, onerror) except RetriesError: raise ProfileNotFoundError(url)