async def run(self, headless=True):
LOG("Building %s x %s" % (self.scenario, self.customization_data["name"]))
with self.env.get_device(2828, verbose=True) as device:
try:
with self.env.get_browser():
metadata = await self.build_profile(device, headless)
finally:
self.env.dump_logs()
if not self.archive:
return
LOG("Creating archive")
archiver = Archiver(self.scenario, self.env.profile, self.archive)
# the archive name is of the form
# profile-<platform>-<scenario>-<customization>.tgz
name = "profile-%(platform)s-%(name)s-%(customization)s.tgz"
name = name % metadata
archive_name = os.path.join(self.archive, name)
dir = os.path.dirname(archive_name)
if not os.path.exists(dir):
os.makedirs(dir)
archiver.create_archive(archive_name)
LOG("Archive created at %s" % archive_name)
statinfo = os.stat(archive_name)
LOG("Current size is %d" % statinfo.st_size)
self.changelog.append("update", **metadata)
def __init__(self, profile_dir):
self.metadata_file = os.path.join(profile_dir, METADATA_NAME)
LOG("Reading existing metadata at %s" % self.metadata_file)
if not os.path.exists(self.metadata_file):
LOG("Could not find the metadata file in that profile")
self._data = {}
else:
with open(self.metadata_file) as f:
self._data = json.loads(f.read())
def dump_logs(self):
logcat = self.device.get_logcat()
if logcat:
# local path, not using posixpath
logfile = os.path.join(self.archive, "logcat.log")
LOG("Writing logcat at %s" % logfile)
with open(logfile, "w") as f:
for line in logcat:
f.write(line + "\n")
else:
LOG("logcat came back empty")
async def start(self):
port = free_port()
await self._check_version()
LOG("Running Webdriver on port %d" % port)
LOG("Running Marionette on port 2828")
pargs = [
self.binary, "-vv", "--port",
str(port), "--marionette-port", "2828"
]
return await subprocess_based_service(pargs,
f"http://localhost:{port}",
self.log_file)
async def build_profile(self, device, headless):
scenario = self.scenario
profile = self.env.profile
customization_data = self.customization_data
scenario_func = scenarii[scenario]
if scenario in customization_data.get("scenario", {}):
options = customization_data["scenario"][scenario]
LOG("Loaded options for that scenario %s" % str(options))
else:
options = {}
# Adding general options
options["platform"] = self.env.target_platform
if not self.force_new:
try:
custom_name = customization_data["name"]
get_profile(profile, self.env.target_platform, scenario, custom_name)
except ProfileNotFoundError:
# XXX we'll use a fresh profile for now
fresh_profile(profile, customization_data)
else:
fresh_profile(profile, customization_data)
LOG("Updating profile located at %r" % profile)
metadata = Metadata(profile)
LOG("Starting the Gecko app...")
self.env.prepare(logfile=self._log_filename("adb"))
geckodriver_logs = self._log_filename("geckodriver")
LOG("Writing geckodriver logs in %s" % geckodriver_logs)
try:
firefox_instance = Firefox(**self.env.get_browser_args(headless))
with open(geckodriver_logs, "w") as glog:
async with get_session(
self.env.get_geckodriver(log_file=glog), firefox_instance
) as session:
self.env.check_session(session)
LOG("Running the %s scenario" % scenario)
metadata.update(await scenario_func(session, options))
LOG("%s scenario done." % scenario)
except Exception:
ERROR("%s scenario broke!" % scenario)
self.env.stop_browser()
self.env.collect_profile()
# writing metadata
metadata.write(
name=self.scenario,
customization=self.customization_data["name"],
version=self.env.get_browser_version(),
platform=self.env.target_platform,
)
LOG("Profile at %s" % profile)
LOG("Done.")
return metadata
def dump_logs(self):
LOG("Dumping Android logs")
try:
logcat = self.device.get_logcat()
if logcat:
# local path, not using posixpath
logfile = os.path.join(self.archive, "logcat.log")
LOG("Writing logcat at %s" % logfile)
with open(logfile, "wb") as f:
for line in logcat:
f.write(line.encode("utf8", errors="replace") + b"\n")
else:
LOG("logcat came back empty")
except Exception:
ERROR("Could not extract the logcat")
def stop_browser(self):
LOG("Stopping %s" % self.app_name)
self.device.stop_application(self.app_name, root=True)
time.sleep(5)
if self.device.process_exist(self.app_name):
LOG("%s still running, trying SIGKILL" % self.app_name)
num_tries = 0
while self.device.process_exist(self.app_name) and num_tries < 5:
try:
self.device.pkill(self.app_name, root=True)
except ADBError:
pass
num_tries += 1
time.sleep(1)
LOG("%s stopped" % self.app_name)
async def run_all(args):
if args.scenario != "all":
selected_scenario = [args.scenario]
else:
selected_scenario = scenarii.keys()
# this is the loop that generates all combinations of profile
# for the current platform when "all" is selected
res = []
failures = 0
for scenario in selected_scenario:
if args.customization != "all":
try:
res.append(await one_run(scenario, args.customization))
except Exception:
failures += 1
ERROR("Something went wrong on this one.")
if args.strict:
raise
else:
for customization in get_customizations():
LOG("Customization %s" % customization)
try:
res.append(await one_run(scenario, customization))
except Exception:
failures += 1
ERROR("Something went wrong on this one.")
if args.strict:
raise
return failures, [one_res for one_res in res if one_res]
def dump_logs(self):
try:
logcat = self.device.get_logcat()
except ADBError:
ERROR("logcat call failure")
return
if logcat:
# local path, not using posixpath
logfile = os.path.join(self.archive, "logcat.log")
LOG("Writing logcat at %s" % logfile)
with open(logfile, "wb") as f:
for line in logcat:
f.write(line.encode("utf8") + b"\n")
else:
LOG("logcat came back empty")
def close(self):
self._unset_adb_logger()
if self.device is None:
return
try:
self.device.remove_forwards("tcp:%d" % self.marionette_port)
except ADBError:
LOG("Could not remove forward port")
def write(self, **extras):
# writing metadata
LOG("Creating metadata...")
self._data.update(**extras)
ts = str(datetime.datetime.now())
if "created" not in self._data:
self._data["created"] = ts
self._data["updated"] = ts
# XXX need android arch version here
days = self._delta(self._data["created"], self._data["updated"])
self._data["days"] = days
self._data["age"] = self._days2age(days)
# adding info about the firefox version
# XXX build ID ??
# XXX android ??
LOG("Saving metadata file in %s" % self.metadata_file)
with open(self.metadata_file, "w") as f:
f.write(json.dumps(self._data))
def _set_adb_logger(self, log_file):
self.log_file = log_file
if self.log_file is None:
return
LOG("Setting ADB log file to %s" % self.log_file)
adb_logger = logging.getLogger("adb")
adb_logger.setLevel(logging.DEBUG)
self._adb_fh = logging.FileHandler(self.log_file)
self._adb_fh.setLevel(logging.DEBUG)
adb_logger.addHandler(self._adb_fh)
async def start(self):
port = free_port()
await self._check_version()
LOG("Running Webdriver on port %d" % port)
LOG("Running Marionette on port 2828")
pargs = [
self.binary,
"--log",
"trace",
"--port",
str(port),
"--marionette-port",
"2828",
]
LOG("Connecting on Android device")
pargs.append("--connect-existing")
return await subprocess_based_service(pargs,
f"http://localhost:{port}",
self.log_file)
def read_changelog(platform, repo="mozilla-central"):
params = {"platform": platform, "repo": repo}
changelog_url = CHANGELOG_LINK % params
LOG("Getting %s" % changelog_url)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
def prepare(self, profile, logfile):
self._set_adb_logger(logfile)
try:
self.device = ADBDevice(verbose=self.verbose, logger_name="adb")
except Exception:
ERROR("Cannot initialize device")
raise
device = self.device
self.profile = profile
# checking that the app is installed
if not device.is_app_installed(self.app_name):
raise Exception("%s is not installed" % self.app_name)
# debug flag
LOG("Setting %s as the debug app on the phone" % self.app_name)
device.shell("am set-debug-app --persistent %s" % self.app_name,
stdout_callback=LOG)
# creating the profile on the device
LOG("Creating the profile on the device")
remote_test_root = posixpath.join(device.test_root, "condprof")
remote_profile = posixpath.join(remote_test_root, "profile")
LOG("The profile on the phone will be at %s" % remote_profile)
device.rm(remote_test_root, force=True, recursive=True)
device.mkdir(remote_test_root)
device.chmod(remote_test_root, recursive=True, root=True)
device.rm(remote_profile, force=True, recursive=True)
LOG("Pushing %s on the phone" % self.profile)
device.push(profile, remote_profile)
device.chmod(remote_profile, recursive=True, root=True)
self.profile = profile
self.remote_profile = remote_profile
# creating the yml file
yml_data = {
"args": ["-marionette", "-profile", self.remote_profile],
"prefs": DEFAULT_PREFS,
"env": {
"LOG_VERBOSE": 1,
"R_LOG_LEVEL": 6,
"MOZ_LOG": ""
},
}
yml_name = "%s-geckoview-config.yaml" % self.app_name
yml_on_host = posixpath.join(tempfile.mkdtemp(), yml_name)
write_yml_file(yml_on_host, yml_data)
tmp_on_device = posixpath.join("/data", "local", "tmp")
if not device.exists(tmp_on_device):
raise IOError("%s does not exists on the device" % tmp_on_device)
yml_on_device = posixpath.join(tmp_on_device, yml_name)
try:
device.rm(yml_on_device, force=True, recursive=True)
device.push(yml_on_host, yml_on_device)
device.chmod(yml_on_device, recursive=True, root=True)
except Exception:
LOG("could not create the yaml file on device. Permission issue?")
raise
# command line 'extra' args not used with geckoview apps; instead we use
# an on-device config.yml file
intent = "android.intent.action.VIEW"
device.stop_application(self.app_name)
device.launch_application(self.app_name,
self.activity,
intent,
extras=None,
url="about:blank")
if not device.process_exist(self.app_name):
raise Exception("Could not start %s" % self.app_name)
LOG("Creating socket forwarding on port %d" % self.marionette_port)
device.forward(
local="tcp:%d" % self.marionette_port,
remote="tcp:%d" % self.marionette_port,
)
# we don't have a clean way for now to check that GV or Fenix
# is ready to handle our tests. So here we just wait 30s
LOG("Sleeping for 30s")
time.sleep(30)
def get_profile(
target_dir,
platform,
scenario,
customization="default",
task_id=None,
download_cache=True,
repo="mozilla-central",
):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
"repo": repo,
}
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
else:
url = DIRECT_LINK % params + filename
if not download_cache:
download_dir = tempfile.mkdtemp()
else:
# using a cache dir in the user home dir
download_dir = os.path.expanduser(CONDPROF_CACHE)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
downloaded_archive = os.path.join(download_dir, filename)
retries = 0
while retries < RETRIES:
try:
LOG("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
LOG("Extracting the tarball content in %s" % target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
LOG("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
LOG("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
LOG("Success, we have a profile to work with")
return target_dir
except Exception:
LOG("Failed to get the profile.")
retries += 1
if os.path.exists(downloaded_archive):
try:
os.remove(downloaded_archive)
except Exception:
ERROR("Could not remove the file")
time.sleep(RETRY_PAUSE)
# If we reach that point, it means all attempts failed
ERROR("All attempt failed")
raise ProfileNotFoundError(url)
def main(args=sys.argv[1:]):
parser = argparse.ArgumentParser(description="Profile Creator")
parser.add_argument("archive", help="Archives Dir", type=str, default=None)
parser.add_argument("--firefox",
help="Firefox Binary",
type=str,
default=None)
parser.add_argument("--scenario",
help="Scenario to use",
type=str,
default="all")
parser.add_argument("--profile",
help="Existing profile Dir",
type=str,
default=None)
parser.add_argument("--customization",
help="Profile customization to use",
type=str,
default="all")
parser.add_argument(
"--fresh-profile",
help="Create a fresh profile",
action="store_true",
default=False,
)
parser.add_argument("--visible",
help="Don't use headless mode",
action="store_true",
default=False)
parser.add_argument("--archives-dir",
help="Archives local dir",
type=str,
default="/tmp/archives")
parser.add_argument("--force-new",
help="Create from scratch",
action="store_true",
default=False)
parser.add_argument(
"--strict",
help="Errors out immediatly on a scenario failure",
action="store_true",
default=True,
)
parser.add_argument(
"--geckodriver",
help="Path to the geckodriver binary",
type=str,
default=sys.platform.startswith("win") and "geckodriver.exe"
or "geckodriver",
)
parser.add_argument("--device-name",
help="Name of the device",
type=str,
default=None)
args = parser.parse_args(args=args)
# unpacking a dmg
# XXX do something similar if we get an apk (but later)
# XXX we want to do
# adb install -r target.apk
# and get the installed app name
if args.firefox is not None and args.firefox.endswith("dmg"):
target = os.path.join(os.path.dirname(args.firefox), "firefox.app")
extract_from_dmg(args.firefox, target)
args.firefox = os.path.join(target, "Contents", "MacOS", "firefox")
args.android = args.firefox is not None and args.firefox.startswith(
"org.mozilla")
if not args.android and args.firefox is not None:
LOG("Verifying Desktop Firefox binary")
# we want to verify we do have a firefox binary
# XXX so lame
if not os.path.exists(args.firefox):
if "MOZ_FETCHES_DIR" in os.environ:
target = os.path.join(os.environ["MOZ_FETCHES_DIR"],
args.firefox)
if os.path.exists(target):
args.firefox = target
if not os.path.exists(args.firefox):
raise IOError("Cannot find %s" % args.firefox)
version = get_version(args.firefox)
LOG("Working with Firefox %s" % version)
LOG(os.environ)
args.archive = os.path.abspath(args.archive)
LOG("Archives directory is %s" % args.archive)
if not os.path.exists(args.archive):
os.makedirs(args.archive, exist_ok=True)
LOG("Verifying Geckodriver binary presence")
if shutil.which(args.geckodriver) is None and not os.path.exists(
args.geckodriver):
raise IOError("Cannot find %s" % args.geckodriver)
try:
plat = args.android and "android" or get_current_platform()
changelog = read_changelog(plat)
LOG("Got the changelog from TaskCluster")
except ProfileNotFoundError:
LOG("changelog not found on TaskCluster, creating a local one.")
changelog = Changelog(args.archive)
loop = asyncio.get_event_loop()
async def one_run(scenario, customization):
if args.android:
env = AndroidEnv(
args.profile,
args.firefox,
args.geckodriver,
args.archive,
args.device_name,
)
else:
env = DesktopEnv(
args.profile,
args.firefox,
args.geckodriver,
args.archive,
args.device_name,
)
return await ProfileCreator(scenario, customization, args.archive,
changelog, args.force_new,
env).run(not args.visible)
async def run_all(args):
if args.scenario != "all":
return await one_run(args.scenario, args.customization)
# this is the loop that generates all combinations of profile
# for the current platform when "all" is selected
res = []
for scenario in scenarii.keys():
if args.customization != "all":
try:
res.append(await one_run(scenario, args.customization))
except Exception:
ERROR("Something went wrong on this one.")
if args.strict:
raise
else:
for customization in get_customizations():
try:
res.append(await one_run(scenario, customization))
except Exception:
ERROR("Something went wrong on this one.")
if args.strict:
raise
return res
try:
loop.run_until_complete(run_all(args))
LOG("Saving changelog in %s" % args.archive)
changelog.save(args.archive)
finally:
loop.close()
def collect_profile(self):
LOG("Collecting profile from %s" % self.remote_profile)
self.device.pull(self.remote_profile, self.profile)
