async def build_profile(self, device, headless):
scenario = self.scenario
profile = self.env.profile
customization_data = self.customization_data
scenario_func = scenarii[scenario]
if scenario in customization_data.get("scenario", {}):
options = customization_data["scenario"][scenario]
LOG("Loaded options for that scenario %s" % str(options))
else:
options = {}
# Adding general options
options["platform"] = self.env.target_platform
if not self.force_new:
try:
custom_name = customization_data["name"]
get_profile(profile, self.env.target_platform, scenario, custom_name)
except ProfileNotFoundError:
# XXX we'll use a fresh profile for now
fresh_profile(profile, customization_data)
else:
fresh_profile(profile, customization_data)
LOG("Updating profile located at %r" % profile)
metadata = Metadata(profile)
LOG("Starting the Gecko app...")
self.env.prepare(logfile=self._log_filename("adb"))
geckodriver_logs = self._log_filename("geckodriver")
LOG("Writing geckodriver logs in %s" % geckodriver_logs)
try:
firefox_instance = Firefox(**self.env.get_browser_args(headless))
with open(geckodriver_logs, "w") as glog:
async with get_session(
self.env.get_geckodriver(log_file=glog), firefox_instance
) as session:
self.env.check_session(session)
LOG("Running the %s scenario" % scenario)
metadata.update(await scenario_func(session, options))
LOG("%s scenario done." % scenario)
except Exception:
ERROR("%s scenario broke!" % scenario)
self.env.stop_browser()
self.env.collect_profile()
# writing metadata
metadata.write(
name=self.scenario,
customization=self.customization_data["name"],
version=self.env.get_browser_version(),
platform=self.env.target_platform,
)
LOG("Profile at %s" % profile)
LOG("Done.")
return metadata
def test_cache(self):
download_dir = os.path.expanduser("~/.condprof-cache")
if os.path.exists(download_dir):
num_elmts = len(os.listdir(download_dir))
else:
num_elmts = 0
get_profile(self.target, "win64", "settled", "default")
# grabbing a profile should generate two files
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# we do at least two network calls when getting a file,
# a HEAD and a GET and possibly a TC secret
self.assertTrue(len(responses.calls) >= 2)
# reseting the response counters
responses.calls.reset()
# and we should reuse them without downloading the file again
get_profile(self.target, "win64", "settled", "default")
# grabbing a profile should not download new stuff
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# and do a single extra HEAD call, everything else is cached,
# even the TC secret
self.assertEqual(len(responses.calls), 1)
def _get_conditioned_profile(self):
platform = self.get_arg("conditioned-platform")
if platform is None:
platform = get_current_platform()
scenario = self.get_arg("conditioned-scenario")
project = self.get_arg("conditioned-project")
alternate_project = "mozilla-central" if project != "mozilla-central" else "try"
temp_dir = tempfile.mkdtemp()
try:
condprof = get_profile(temp_dir, platform, scenario, repo=project)
except ProfileNotFoundError:
condprof = get_profile(temp_dir,
platform,
scenario,
repo=alternate_project)
except Exception:
raise
# now get the full directory path to our fetched conditioned profile
condprof = Path(temp_dir, condprof)
if not condprof.exists():
raise OSError(str(condprof))
return condprof
def get_conditioned_profile(self):
"""Downloads a platform-specific conditioned profile, using the
condprofile client API; returns a self.conditioned_profile_dir"""
# create a temp file to help ensure uniqueness
temp_download_dir = tempfile.mkdtemp()
LOG.info(
"Making temp_download_dir from inside get_conditioned_profile {}".
format(temp_download_dir))
# call condprof's client API to yield our platform-specific
# conditioned-profile binary
if isinstance(self, PerftestAndroid):
android_app = self.config["binary"].split("org.mozilla.")[-1]
device_name = self.config.get("device_name")
if device_name is None:
device_name = "g5"
platform = "%s-%s" % (device_name, android_app)
else:
platform = get_current_platform()
LOG.info("Platform used: %s" % platform)
try:
cond_prof_target_dir = get_profile(temp_download_dir, platform,
"settled")
except ProfileNotFoundError:
# If we can't find the profile on mozilla-central, we look on try
cond_prof_target_dir = get_profile(temp_download_dir,
platform,
"settled",
repo="try")
except Exception:
# any other error is a showstopper
LOG.critical("Could not get the conditioned profile")
traceback.print_exc()
raise
# now get the full directory path to our fetched conditioned profile
self.conditioned_profile_dir = os.path.join(temp_download_dir,
cond_prof_target_dir)
if not os.path.exists(cond_prof_target_dir):
LOG.critical("Can't find target_dir {}, from get_profile()"
"temp_download_dir {}, platform {}, settled".format(
cond_prof_target_dir, temp_download_dir,
platform))
raise OSError
LOG.info("self.conditioned_profile_dir is now set: {}".format(
self.conditioned_profile_dir))
shutil.rmtree(temp_download_dir)
return self.conditioned_profile_dir
def test_cache(self):
download_dir = os.path.expanduser("~/.condprof-cache")
if os.path.exists(download_dir):
num_elmts = len(os.listdir(download_dir))
else:
num_elmts = 0
get_profile(self.target, "win64", "settled", "default")
# grabbing a profile should generate two files
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# we do at least two network calls when getting a file,
# a HEAD and a GET and possibly a TC secret
self.assertTrue(len(responses.calls) >= 2)
# reseting the response counters
responses.calls.reset()
# and we should reuse them without downloading the file again
get_profile(self.target, "win64", "settled", "default")
# grabbing a profile should not download new stuff
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# and do a single extra HEAD call, everything else is cached,
# even the TC secret
self.assertEqual(len(responses.calls), 2)
prefs_js = os.path.join(self.target, "prefs.js")
prefs = Preferences.read_prefs(prefs_js)
# check that the gfx.blacklist prefs where cleaned out
for name, value in prefs:
self.assertFalse(name.startswith("gfx.blacklist"))
# check that we have the startupScanScopes option forced
prefs = dict(prefs)
self.assertEqual(prefs["extensions.startupScanScopes"], 1)
# make sure we don't have any marionette option set
user_js = os.path.join(self.target, "user.js")
for name, value in Preferences.read_prefs(user_js):
self.assertFalse(name.startswith("marionette."))
async def build_profile(args):
scenarii = scenario[args.scenarii]
if not args.force_new:
get_profile(args)
logger.msg("Updating profile located at %r" % args.profile)
metadata_file = os.path.join(args.profile, ".hp.json")
with open(metadata_file) as f:
metadata = json.loads(f.read())
f_args = ["-profile", args.profile]
if platform.system() != "Darwin":
f_args.append("-headless")
caps = {"moz:firefoxOptions": {"args": f_args}}
if args.firefox is not None:
caps["moz:firefoxOptions"]["binary"] = args.firefox
logger.msg("Starting the Fox...")
with open("gecko.log", "a+") as glog:
async with get_session(CustomGeckodriver(log_file=glog),
Firefox(**caps)) as session:
logger.msg("Running the %s scenario" % args.scenarii)
metadata.update(await scenarii(session, args))
# writing metadata
logger.msg("Creating metadata...")
ts = str(datetime.datetime.now())
if "created" not in metadata:
metadata["created"] = ts
metadata["updated"] = ts
metadata["name"] = args.scenarii
metadata["platform"] = sys.platform
metadata["age"] = get_age(metadata)
metadata["version"] = "69.0a1" # add the build id XXX
metadata["customization"] = "vanilla" # add themes
with open(metadata_file, "w") as f:
f.write(json.dumps(metadata))
logger.msg("Profile at %s" % args.profile)
logger.msg("Done.")
def fetch(
self,
target_dir,
platform,
scenario,
customization,
task_id,
download_cache,
repo,
):
self._init()
from condprof.client import get_profile
from condprof.util import get_current_platform
if platform is None:
platform = get_current_platform()
if target_dir is None:
target_dir = tempfile.mkdtemp()
get_profile(target_dir, platform, scenario, customization, task_id,
download_cache, repo)
def test_cache(self):
download_dir = os.path.expanduser("~/.condprof-cache")
if os.path.exists(download_dir):
num_elmts = len(os.listdir(download_dir))
else:
num_elmts = 0
get_profile(self.target, "win64", "settled", "default")
# grabbing a profile should generate two files
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# we do two network calls when getting a file, a HEAD and a GET
response_calls = len(responses.calls)
self.assertEqual(response_calls, 2)
# and we should reuse them without downloading the file again
get_profile(self.target, "win64", "settled", "default")
# grabbing a profile should not download new stuff
self.assertEqual(len(os.listdir(download_dir)), num_elmts + 2)
# and do a single extra HEAD call
self.assertEqual(len(responses.calls), response_calls + 1)
def get_conditioned_profile(self):
"""Downloads a platform-specific conditioned profile, using the
condprofile client API; returns a self.conditioned_profile_dir"""
if self.conditioned_profile_dir:
# We already have a directory, so provide a copy that
# will get deleted after it's done with
return self.conditioned_profile_copy
# create a temp file to help ensure uniqueness
temp_download_dir = self._get_temp_dir()
LOG.info(
"Making temp_download_dir from inside get_conditioned_profile {}".
format(temp_download_dir))
# call condprof's client API to yield our platform-specific
# conditioned-profile binary
if isinstance(self, PerftestAndroid):
android_app = self.config["binary"].split("org.mozilla.")[-1]
device_name = self.config.get("device_name")
if device_name is None:
device_name = "g5"
platform = "%s-%s" % (device_name, android_app)
else:
platform = get_current_platform()
LOG.info("Platform used: %s" % platform)
# when running under mozharness, the --project value
# is set to match the project (try, mozilla-central, etc.)
# By default it's mozilla-central, even on local runs.
# We use it to prioritize conditioned profiles indexed
# into the same project when it runs on the CI
repo = self.config["project"]
# we fall back to mozilla-central in all cases. If it
# was already mozilla-central, we fall back to try
alternate_repo = "mozilla-central" if repo != "mozilla-central" else "try"
LOG.info("Getting profile from project %s" % repo)
profile_scenario = self.config.get("conditioned_profile")
try:
cond_prof_target_dir = get_profile(temp_download_dir,
platform,
profile_scenario,
repo=repo)
except ProfileNotFoundError:
cond_prof_target_dir = get_profile(temp_download_dir,
platform,
profile_scenario,
repo=alternate_repo)
except Exception:
# any other error is a showstopper
LOG.critical("Could not get the conditioned profile")
traceback.print_exc()
raise
# now get the full directory path to our fetched conditioned profile
self.conditioned_profile_dir = os.path.join(temp_download_dir,
cond_prof_target_dir)
if not os.path.exists(cond_prof_target_dir):
LOG.critical(
"Can't find target_dir {}, from get_profile()"
"temp_download_dir {}, platform {}, scenario {}".format(
cond_prof_target_dir, temp_download_dir, platform,
profile_scenario))
raise OSError
LOG.info("Original self.conditioned_profile_dir is now set: {}".format(
self.conditioned_profile_dir))
return self.conditioned_profile_copy
async def build_profile(self, device, headless):
scenario = self.scenario
profile = self.env.profile
customization_data = self.customization_data
scenario_func = scenarii[scenario]
if scenario in customization_data.get("scenario", {}):
options = customization_data["scenario"][scenario]
logger.info("Loaded options for that scenario %s" % str(options))
else:
options = {}
# Adding general options
options["platform"] = self.env.target_platform
if not self.force_new:
try:
custom_name = customization_data["name"]
get_profile(profile, self.env.target_platform, scenario,
custom_name)
except ProfileNotFoundError:
# XXX we'll use a fresh profile for now
fresh_profile(profile, customization_data)
else:
fresh_profile(profile, customization_data)
logger.info("Updating profile located at %r" % profile)
metadata = Metadata(profile)
logger.info("Starting the Gecko app...")
adb_logs = self._log_filename("adb")
self.env.prepare(logfile=adb_logs)
geckodriver_logs = self._log_filename("geckodriver")
logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
step = START
try:
firefox_instance = Firefox(**self.env.get_browser_args(headless))
step = INIT_GECKODRIVER
with open(geckodriver_logs, "w") as glog:
geckodriver = self.env.get_geckodriver(log_file=glog)
step = START_SESSION
async with get_session(geckodriver,
firefox_instance) as session:
step = START_SCENARIO
self.env.check_session(session)
logger.info("Running the %s scenario" % scenario)
metadata.update(await scenario_func(session, options))
logger.info("%s scenario done." % scenario)
await close_extra_windows(session)
except Exception:
logger.error("%s scenario broke!" % scenario)
if step == START:
logger.info("Could not initialize the browser")
elif step == INIT_GECKODRIVER:
logger.info("Could not initialize Geckodriver")
elif step == START_SESSION:
logger.info("Could not start the session, check %s first" %
geckodriver_logs)
else:
logger.info(
"Could not run the scenario, probably a faulty scenario")
raise
finally:
self.env.stop_browser()
for logfile in (adb_logs, geckodriver_logs):
if os.path.exists(logfile):
obfuscate_file(logfile)
self.env.collect_profile()
# writing metadata
metadata.write(
name=self.scenario,
customization=self.customization_data["name"],
version=self.env.get_browser_version(),
platform=self.env.target_platform,
)
logger.info("Profile at %s.\nDone." % profile)
return metadata
