async def build_profile(self, device, headless): scenario = self.scenario profile = self.env.profile customization_data = self.customization_data scenario_func = scenarii[scenario] if scenario in customization_data.get("scenario", {}): options = customization_data["scenario"][scenario] LOG("Loaded options for that scenario %s" % str(options)) else: options = {} # Adding general options options["platform"] = self.env.target_platform if not self.force_new: try: custom_name = customization_data["name"] get_profile(profile, self.env.target_platform, scenario, custom_name) except ProfileNotFoundError: # XXX we'll use a fresh profile for now fresh_profile(profile, customization_data) else: fresh_profile(profile, customization_data) LOG("Updating profile located at %r" % profile) metadata = Metadata(profile) LOG("Starting the Gecko app...") self.env.prepare(logfile=self._log_filename("adb")) geckodriver_logs = self._log_filename("geckodriver") LOG("Writing geckodriver logs in %s" % geckodriver_logs) try: firefox_instance = Firefox(**self.env.get_browser_args(headless)) with open(geckodriver_logs, "w") as glog: async with get_session( self.env.get_geckodriver(log_file=glog), firefox_instance ) as session: self.env.check_session(session) LOG("Running the %s scenario" % scenario) metadata.update(await scenario_func(session, options)) LOG("%s scenario done." % scenario) except Exception: ERROR("%s scenario broke!" % scenario) self.env.stop_browser() self.env.collect_profile() # writing metadata metadata.write( name=self.scenario, customization=self.customization_data["name"], version=self.env.get_browser_version(), platform=self.env.target_platform, ) LOG("Profile at %s" % profile) LOG("Done.") return metadata
def test_cache(self): download_dir = os.path.expanduser("~/.condprof-cache") if os.path.exists(download_dir): num_elmts = len(os.listdir(download_dir)) else: num_elmts = 0 get_profile(self.target, "win64", "settled", "default") # grabbing a profile should generate two files self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) # we do at least two network calls when getting a file, # a HEAD and a GET and possibly a TC secret self.assertTrue(len(responses.calls) >= 2) # reseting the response counters responses.calls.reset() # and we should reuse them without downloading the file again get_profile(self.target, "win64", "settled", "default") # grabbing a profile should not download new stuff self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) # and do a single extra HEAD call, everything else is cached, # even the TC secret self.assertEqual(len(responses.calls), 1)
def _get_conditioned_profile(self): platform = self.get_arg("conditioned-platform") if platform is None: platform = get_current_platform() scenario = self.get_arg("conditioned-scenario") project = self.get_arg("conditioned-project") alternate_project = "mozilla-central" if project != "mozilla-central" else "try" temp_dir = tempfile.mkdtemp() try: condprof = get_profile(temp_dir, platform, scenario, repo=project) except ProfileNotFoundError: condprof = get_profile(temp_dir, platform, scenario, repo=alternate_project) except Exception: raise # now get the full directory path to our fetched conditioned profile condprof = Path(temp_dir, condprof) if not condprof.exists(): raise OSError(str(condprof)) return condprof
def get_conditioned_profile(self): """Downloads a platform-specific conditioned profile, using the condprofile client API; returns a self.conditioned_profile_dir""" # create a temp file to help ensure uniqueness temp_download_dir = tempfile.mkdtemp() LOG.info( "Making temp_download_dir from inside get_conditioned_profile {}". format(temp_download_dir)) # call condprof's client API to yield our platform-specific # conditioned-profile binary if isinstance(self, PerftestAndroid): android_app = self.config["binary"].split("org.mozilla.")[-1] device_name = self.config.get("device_name") if device_name is None: device_name = "g5" platform = "%s-%s" % (device_name, android_app) else: platform = get_current_platform() LOG.info("Platform used: %s" % platform) try: cond_prof_target_dir = get_profile(temp_download_dir, platform, "settled") except ProfileNotFoundError: # If we can't find the profile on mozilla-central, we look on try cond_prof_target_dir = get_profile(temp_download_dir, platform, "settled", repo="try") except Exception: # any other error is a showstopper LOG.critical("Could not get the conditioned profile") traceback.print_exc() raise # now get the full directory path to our fetched conditioned profile self.conditioned_profile_dir = os.path.join(temp_download_dir, cond_prof_target_dir) if not os.path.exists(cond_prof_target_dir): LOG.critical("Can't find target_dir {}, from get_profile()" "temp_download_dir {}, platform {}, settled".format( cond_prof_target_dir, temp_download_dir, platform)) raise OSError LOG.info("self.conditioned_profile_dir is now set: {}".format( self.conditioned_profile_dir)) shutil.rmtree(temp_download_dir) return self.conditioned_profile_dir
def test_cache(self): download_dir = os.path.expanduser("~/.condprof-cache") if os.path.exists(download_dir): num_elmts = len(os.listdir(download_dir)) else: num_elmts = 0 get_profile(self.target, "win64", "settled", "default") # grabbing a profile should generate two files self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) # we do at least two network calls when getting a file, # a HEAD and a GET and possibly a TC secret self.assertTrue(len(responses.calls) >= 2) # reseting the response counters responses.calls.reset() # and we should reuse them without downloading the file again get_profile(self.target, "win64", "settled", "default") # grabbing a profile should not download new stuff self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) # and do a single extra HEAD call, everything else is cached, # even the TC secret self.assertEqual(len(responses.calls), 2) prefs_js = os.path.join(self.target, "prefs.js") prefs = Preferences.read_prefs(prefs_js) # check that the gfx.blacklist prefs where cleaned out for name, value in prefs: self.assertFalse(name.startswith("gfx.blacklist")) # check that we have the startupScanScopes option forced prefs = dict(prefs) self.assertEqual(prefs["extensions.startupScanScopes"], 1) # make sure we don't have any marionette option set user_js = os.path.join(self.target, "user.js") for name, value in Preferences.read_prefs(user_js): self.assertFalse(name.startswith("marionette."))
async def build_profile(args): scenarii = scenario[args.scenarii] if not args.force_new: get_profile(args) logger.msg("Updating profile located at %r" % args.profile) metadata_file = os.path.join(args.profile, ".hp.json") with open(metadata_file) as f: metadata = json.loads(f.read()) f_args = ["-profile", args.profile] if platform.system() != "Darwin": f_args.append("-headless") caps = {"moz:firefoxOptions": {"args": f_args}} if args.firefox is not None: caps["moz:firefoxOptions"]["binary"] = args.firefox logger.msg("Starting the Fox...") with open("gecko.log", "a+") as glog: async with get_session(CustomGeckodriver(log_file=glog), Firefox(**caps)) as session: logger.msg("Running the %s scenario" % args.scenarii) metadata.update(await scenarii(session, args)) # writing metadata logger.msg("Creating metadata...") ts = str(datetime.datetime.now()) if "created" not in metadata: metadata["created"] = ts metadata["updated"] = ts metadata["name"] = args.scenarii metadata["platform"] = sys.platform metadata["age"] = get_age(metadata) metadata["version"] = "69.0a1" # add the build id XXX metadata["customization"] = "vanilla" # add themes with open(metadata_file, "w") as f: f.write(json.dumps(metadata)) logger.msg("Profile at %s" % args.profile) logger.msg("Done.")
def fetch( self, target_dir, platform, scenario, customization, task_id, download_cache, repo, ): self._init() from condprof.client import get_profile from condprof.util import get_current_platform if platform is None: platform = get_current_platform() if target_dir is None: target_dir = tempfile.mkdtemp() get_profile(target_dir, platform, scenario, customization, task_id, download_cache, repo)
def test_cache(self): download_dir = os.path.expanduser("~/.condprof-cache") if os.path.exists(download_dir): num_elmts = len(os.listdir(download_dir)) else: num_elmts = 0 get_profile(self.target, "win64", "settled", "default") # grabbing a profile should generate two files self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) # we do two network calls when getting a file, a HEAD and a GET response_calls = len(responses.calls) self.assertEqual(response_calls, 2) # and we should reuse them without downloading the file again get_profile(self.target, "win64", "settled", "default") # grabbing a profile should not download new stuff self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2) # and do a single extra HEAD call self.assertEqual(len(responses.calls), response_calls + 1)
def get_conditioned_profile(self): """Downloads a platform-specific conditioned profile, using the condprofile client API; returns a self.conditioned_profile_dir""" if self.conditioned_profile_dir: # We already have a directory, so provide a copy that # will get deleted after it's done with return self.conditioned_profile_copy # create a temp file to help ensure uniqueness temp_download_dir = self._get_temp_dir() LOG.info( "Making temp_download_dir from inside get_conditioned_profile {}". format(temp_download_dir)) # call condprof's client API to yield our platform-specific # conditioned-profile binary if isinstance(self, PerftestAndroid): android_app = self.config["binary"].split("org.mozilla.")[-1] device_name = self.config.get("device_name") if device_name is None: device_name = "g5" platform = "%s-%s" % (device_name, android_app) else: platform = get_current_platform() LOG.info("Platform used: %s" % platform) # when running under mozharness, the --project value # is set to match the project (try, mozilla-central, etc.) # By default it's mozilla-central, even on local runs. # We use it to prioritize conditioned profiles indexed # into the same project when it runs on the CI repo = self.config["project"] # we fall back to mozilla-central in all cases. If it # was already mozilla-central, we fall back to try alternate_repo = "mozilla-central" if repo != "mozilla-central" else "try" LOG.info("Getting profile from project %s" % repo) profile_scenario = self.config.get("conditioned_profile") try: cond_prof_target_dir = get_profile(temp_download_dir, platform, profile_scenario, repo=repo) except ProfileNotFoundError: cond_prof_target_dir = get_profile(temp_download_dir, platform, profile_scenario, repo=alternate_repo) except Exception: # any other error is a showstopper LOG.critical("Could not get the conditioned profile") traceback.print_exc() raise # now get the full directory path to our fetched conditioned profile self.conditioned_profile_dir = os.path.join(temp_download_dir, cond_prof_target_dir) if not os.path.exists(cond_prof_target_dir): LOG.critical( "Can't find target_dir {}, from get_profile()" "temp_download_dir {}, platform {}, scenario {}".format( cond_prof_target_dir, temp_download_dir, platform, profile_scenario)) raise OSError LOG.info("Original self.conditioned_profile_dir is now set: {}".format( self.conditioned_profile_dir)) return self.conditioned_profile_copy
async def build_profile(self, device, headless): scenario = self.scenario profile = self.env.profile customization_data = self.customization_data scenario_func = scenarii[scenario] if scenario in customization_data.get("scenario", {}): options = customization_data["scenario"][scenario] logger.info("Loaded options for that scenario %s" % str(options)) else: options = {} # Adding general options options["platform"] = self.env.target_platform if not self.force_new: try: custom_name = customization_data["name"] get_profile(profile, self.env.target_platform, scenario, custom_name) except ProfileNotFoundError: # XXX we'll use a fresh profile for now fresh_profile(profile, customization_data) else: fresh_profile(profile, customization_data) logger.info("Updating profile located at %r" % profile) metadata = Metadata(profile) logger.info("Starting the Gecko app...") adb_logs = self._log_filename("adb") self.env.prepare(logfile=adb_logs) geckodriver_logs = self._log_filename("geckodriver") logger.info("Writing geckodriver logs in %s" % geckodriver_logs) step = START try: firefox_instance = Firefox(**self.env.get_browser_args(headless)) step = INIT_GECKODRIVER with open(geckodriver_logs, "w") as glog: geckodriver = self.env.get_geckodriver(log_file=glog) step = START_SESSION async with get_session(geckodriver, firefox_instance) as session: step = START_SCENARIO self.env.check_session(session) logger.info("Running the %s scenario" % scenario) metadata.update(await scenario_func(session, options)) logger.info("%s scenario done." % scenario) await close_extra_windows(session) except Exception: logger.error("%s scenario broke!" % scenario) if step == START: logger.info("Could not initialize the browser") elif step == INIT_GECKODRIVER: logger.info("Could not initialize Geckodriver") elif step == START_SESSION: logger.info("Could not start the session, check %s first" % geckodriver_logs) else: logger.info( "Could not run the scenario, probably a faulty scenario") raise finally: self.env.stop_browser() for logfile in (adb_logs, geckodriver_logs): if os.path.exists(logfile): obfuscate_file(logfile) self.env.collect_profile() # writing metadata metadata.write( name=self.scenario, customization=self.customization_data["name"], version=self.env.get_browser_version(), platform=self.env.target_platform, ) logger.info("Profile at %s.\nDone." % profile) return metadata