async def __call__(self, session):
metadata = {}
tabs = TabSwitcher(session, self.options)
await tabs.create_windows()
visited = 0
for current, (url, word) in enumerate(self.urls):
logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
retries = 0
while retries < 3:
try:
await self._visit_url(current, session, url, word)
visited += 1
break
except CallErrors:
await asyncio.sleep(1.0)
retries += 1
if current == self.max_urls - 1:
break
# switch to the next tab
try:
await asyncio.wait_for(tabs.switch(), 5)
except CallErrors:
# if we can't switch, it's ok
pass
metadata["visited_url"] = visited
await self.sync(session, metadata)
return metadata
def onerror(error):
logger.info("Failed to get the profile.")
if os.path.exists(downloaded_archive):
try:
os.remove(downloaded_archive)
except Exception:
logger.error("Could not remove the file")
def __init__(self, options):
self.options = options
self.words = self._read_lines("words.txt")
self.urls = self._build_url_list(self._read_lines("urls.txt"))
self.sync_js = self._load_script("sync")
self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS)
self.bookmark_js = self._load_script("bookmark")
self.platform = options.get("platform", "")
self.mobile = is_mobile(self.platform)
self.max_urls = options.get("max_urls", MAX_URLS)
# see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar
if self.mobile:
self.max_urls = min(self.max_urls, 20)
logger.info("platform: %s" % self.platform)
logger.info("max_urls: %s" % self.max_urls)
self.bookmark_frequency = options.get("bookmark_frequency",
BOOKMARK_FREQUENCY)
# we're syncing only on desktop for now
self.syncing = not self.mobile
if self.syncing:
self.username, self.password = get_credentials()
if self.username is None:
raise ValueError(
"Sync operations need an FxA username and password")
else:
self.username, self.password = None, None
async def close_extra_windows(session):
logger.info("Closing all tabs")
handles = await session.get_window_handles()
# we're closing all tabs except the last one
for handle in handles[:-1]:
await session.switch_to_window(handle)
await session._request(url="/window", method="DELETE")
async def run_all(self):
"""Runs the conditioned profile builders"""
if self.scenario != "all":
selected_scenario = [self.scenario]
else:
selected_scenario = scenarii.keys()
# this is the loop that generates all combinations of profile
# for the current platform when "all" is selected
res = []
failures = 0
for scenario in selected_scenario:
if self.customization != "all":
try:
res.append(await self.one_run(scenario,
self.customization))
except Exception:
failures += 1
self.display_error(scenario, self.customization)
else:
for customization in get_customizations():
logger.info("Customization %s" % customization)
try:
res.append(await self.one_run(scenario, customization))
except Exception:
failures += 1
self.display_error(scenario, customization)
return failures, [one_res for one_res in res if one_res]
def run(
archive,
firefox=None,
scenario="all",
profile=None,
customization="all",
visible=False,
archives_dir="/tmp/archives",
force_new=False,
strict=True,
geckodriver="geckodriver",
device_name=None,
):
runner = Runner(profile, firefox, geckodriver, archive, device_name,
strict, force_new, visible)
runner.prepare(scenario, customization)
loop = asyncio.get_event_loop()
try:
failures, results = loop.run_until_complete(runner.run_all())
logger.info("Saving changelog in %s" % archive)
runner.save()
if failures > 0:
raise Exception("At least one scenario failed")
finally:
loop.close()
def _keep_pref(name, value):
for item in to_remove:
if not name.startswith(item):
continue
logger.info("Removing pref %s: %s" % (name, value))
return False
return True
def _get_profile():
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
_check_profile(target_dir)
logger.info("Success, we have a profile to work with")
return target_dir
def __init__(self, profile_dir):
self.metadata_file = os.path.join(profile_dir, METADATA_NAME)
logger.info("Reading existing metadata at %s" % self.metadata_file)
if not os.path.exists(self.metadata_file):
logger.info("Could not find the metadata file in that profile")
self._data = {}
else:
with open(self.metadata_file) as f:
self._data = json.loads(f.read())
def _set_adb_logger(self, log_file):
self.log_file = log_file
if self.log_file is None:
return
logger.info("Setting ADB log file to %s" % self.log_file)
adb_logger = logging.getLogger("adb")
adb_logger.setLevel(logging.DEBUG)
self._adb_fh = logging.FileHandler(self.log_file)
self._adb_fh.setLevel(logging.DEBUG)
adb_logger.addHandler(self._adb_fh)
def read_changelog(platform, repo="mozilla-central"):
params = {"platform": platform, "repo": repo}
changelog_url = CHANGELOG_LINK % params
logger.info("Getting %s" % changelog_url)
download_dir = tempfile.mkdtemp()
downloaded_changelog = os.path.join(download_dir, "changelog.json")
try:
download_file(changelog_url, target=downloaded_changelog)
except ArchiveNotFound:
shutil.rmtree(download_dir)
raise ProfileNotFoundError(changelog_url)
return Changelog(download_dir)
async def start(self):
port = free_port()
await self._check_version()
logger.info("Running Webdriver on port %d" % port)
logger.info("Running Marionette on port 2828")
pargs = [
self.binary,
"--log",
"trace",
"--port",
str(port),
"--marionette-port",
"2828",
]
return await subprocess_based_service(pargs,
f"http://localhost:{port}",
self.log_file)
def write(self, **extras):
# writing metadata
logger.info("Creating metadata...")
self._data.update(**extras)
ts = str(datetime.datetime.now())
if "created" not in self._data:
self._data["created"] = ts
self._data["updated"] = ts
# XXX need android arch version here
days = self._delta(self._data["created"], self._data["updated"])
self._data["days"] = days
self._data["age"] = self._days2age(days)
# adding info about the firefox version
# XXX build ID ??
# XXX android ??
logger.info("Saving metadata file in %s" % self.metadata_file)
with open(self.metadata_file, "w") as f:
f.write(json.dumps(self._data))
def _retries(callable, onerror=None):
retries = 0
pause = RETRY_PAUSE
while retries < RETRIES:
try:
return callable()
except Exception as e:
if onerror is not None:
onerror(e)
logger.info("Failed, retrying")
retries += 1
time.sleep(pause)
pause *= 1.5
# If we reach that point, it means all attempts failed
logger.error("All attempt failed")
raise RetriesError()
async def sync(self, session, metadata):
if not self.syncing:
return
# now that we've visited all pages, we want to upload to FXSync
logger.info("Syncing profile to FxSync")
logger.info("Username is %s, password is %s" % (self.username, self.password))
script_res = await execute_async_script(
session,
self.sync_js,
self.username,
self.password,
"https://accounts.stage.mozaws.net",
)
if script_res is None:
script_res = {}
metadata["logs"] = script_res.get("logs", {})
metadata["result"] = script_res.get("result", 0)
metadata["result_message"] = script_res.get("result_message", "SUCCESS")
return metadata
def stop_browser(self):
logger.info("Stopping %s" % self.app_name)
try:
self.device.stop_application(self.app_name)
except ADBError:
logger.info("Could not stop the application using force-stop")
time.sleep(5)
if self.device.process_exist(self.app_name):
logger.info("%s still running, trying SIGKILL" % self.app_name)
num_tries = 0
while self.device.process_exist(self.app_name) and num_tries < 5:
try:
self.device.pkill(self.app_name)
except ADBError:
pass
num_tries += 1
time.sleep(1)
logger.info("%s stopped" % self.app_name)
def dump_logs(self):
logger.info("Dumping Android logs")
try:
logcat = self.device.get_logcat()
if logcat:
# local path, not using posixpath
logfile = os.path.join(self.archive, "logcat.log")
logger.info("Writing logcat at %s" % logfile)
with open(logfile, "wb") as f:
for line in logcat:
f.write(line.encode("utf8", errors="replace") + b"\n")
else:
logger.info("logcat came back empty")
except Exception:
logger.error("Could not extract the logcat", exc_info=True)
async def execute_async_script(session, script, *args):
# switch to the right context if needed
current_context = await session._request(url="/moz/context", method="GET")
if current_context != "chrome":
logger.info("Switching to chrome context")
await session._request(url="/moz/context",
method="POST",
data={"context": "chrome"})
switch_back = True
else:
switch_back = False
logger.info("Setting up script timeout")
await session._request(url="/timeouts",
method="POST",
data={"script": _SCRIPT_TIMEOUT})
try:
attempts = 0
while True:
logger.info("Running triggerSync()")
try:
return await session._request(
url="/execute/async",
method="POST",
data={
"script": script,
"args": list(args)
},
)
except Exception as e:
attempts += 1
logger.error("The script failed.", exc_info=True)
if attempts > 2:
return {
"result": 1,
"result_message": str(e),
"result_exc": e,
"logs": {},
}
finally:
if switch_back:
await session._request(url="/moz/context",
method="POST",
data={"context": current_context})
async def add_bookmark(self, session, url, title):
logger.info("Adding bookmark to %s" % url)
return await execute_async_script(session, self.bookmark_js, url,
title, self.max_bookmarks)
def collect_profile(self):
logger.info("Collecting profile from %s" % self.remote_profile)
self.device.pull(self.remote_profile, self.profile)
def prepare(self, scenario, customization):
self.scenario = scenario
self.customization = customization
# early checks to avoid extra work
if self.customization != "all":
if find_customization(self.customization) is None:
raise IOError("Cannot find customization %r" %
self.customization)
if self.scenario != "all" and self.scenario not in scenarii:
raise IOError("Cannot find scenario %r" % self.scenario)
if not self.android and self.firefox is not None:
logger.info("Verifying Desktop Firefox binary")
# we want to verify we do have a firefox binary
# XXX so lame
if not os.path.exists(self.firefox):
if "MOZ_FETCHES_DIR" in os.environ:
target = os.path.join(os.environ["MOZ_FETCHES_DIR"],
self.firefox)
if os.path.exists(target):
self.firefox = target
if not os.path.exists(self.firefox):
raise IOError("Cannot find %s" % self.firefox)
version = get_version(self.firefox)
logger.info("Working with Firefox %s" % version)
logger.info(os.environ)
self.archive = os.path.abspath(self.archive)
logger.info("Archives directory is %s" % self.archive)
if not os.path.exists(self.archive):
os.makedirs(self.archive, exist_ok=True)
logger.info("Verifying Geckodriver binary presence")
if shutil.which(self.geckodriver) is None and not os.path.exists(
self.geckodriver):
raise IOError("Cannot find %s" % self.geckodriver)
try:
if self.android:
plat = "%s-%s" % (
self.device_name,
self.firefox.split("org.mozilla.")[-1],
)
else:
plat = get_current_platform()
self.changelog = read_changelog(plat)
logger.info("Got the changelog from TaskCluster")
except ProfileNotFoundError:
logger.info(
"changelog not found on TaskCluster, creating a local one.")
self.changelog = Changelog(self.archive)
async def build_profile(self, device, headless):
scenario = self.scenario
profile = self.env.profile
customization_data = self.customization_data
scenario_func = scenarii[scenario]
if scenario in customization_data.get("scenario", {}):
options = customization_data["scenario"][scenario]
logger.info("Loaded options for that scenario %s" % str(options))
else:
options = {}
# Adding general options
options["platform"] = self.env.target_platform
if not self.force_new:
try:
custom_name = customization_data["name"]
get_profile(profile, self.env.target_platform, scenario,
custom_name)
except ProfileNotFoundError:
# XXX we'll use a fresh profile for now
fresh_profile(profile, customization_data)
else:
fresh_profile(profile, customization_data)
logger.info("Updating profile located at %r" % profile)
metadata = Metadata(profile)
logger.info("Starting the Gecko app...")
adb_logs = self._log_filename("adb")
self.env.prepare(logfile=adb_logs)
geckodriver_logs = self._log_filename("geckodriver")
logger.info("Writing geckodriver logs in %s" % geckodriver_logs)
step = START
try:
firefox_instance = Firefox(**self.env.get_browser_args(headless))
step = INIT_GECKODRIVER
with open(geckodriver_logs, "w") as glog:
geckodriver = self.env.get_geckodriver(log_file=glog)
step = START_SESSION
async with get_session(geckodriver,
firefox_instance) as session:
step = START_SCENARIO
self.env.check_session(session)
logger.info("Running the %s scenario" % scenario)
metadata.update(await scenario_func(session, options))
logger.info("%s scenario done." % scenario)
await close_extra_windows(session)
except Exception:
logger.error("%s scenario broke!" % scenario)
if step == START:
logger.info("Could not initialize the browser")
elif step == INIT_GECKODRIVER:
logger.info("Could not initialize Geckodriver")
elif step == START_SESSION:
logger.info("Could not start the session, check %s first" %
geckodriver_logs)
else:
logger.info(
"Could not run the scenario, probably a faulty scenario")
raise
finally:
self.env.stop_browser()
for logfile in (adb_logs, geckodriver_logs):
if os.path.exists(logfile):
obfuscate_file(logfile)
self.env.collect_profile()
# writing metadata
metadata.write(
name=self.scenario,
customization=self.customization_data["name"],
version=self.env.get_browser_version(),
platform=self.env.target_platform,
)
logger.info("Profile at %s.\nDone." % profile)
return metadata
async def run(self, headless=True):
logger.info("Building %s x %s" %
(self.scenario, self.customization_data["name"]))
if self.scenario in self.customization_data.get("ignore_scenario", []):
logger.info("Skipping (ignored scenario in that customization)")
return
filter_by_platform = self.customization_data.get("platforms")
if filter_by_platform and self.env.target_platform not in filter_by_platform:
logger.info("Skipping (ignored platform in that customization)")
return
with self.env.get_device(2828, verbose=True) as device:
try:
with self.env.get_browser():
metadata = await self.build_profile(device, headless)
finally:
self.env.dump_logs()
if not self.archive:
return
logger.info("Creating archive")
archiver = Archiver(self.scenario, self.env.profile, self.archive)
# the archive name is of the form
# profile-<platform>-<scenario>-<customization>.tgz
name = "profile-%(platform)s-%(name)s-%(customization)s.tgz"
name = name % metadata
archive_name = os.path.join(self.archive, name)
dir = os.path.dirname(archive_name)
if not os.path.exists(dir):
os.makedirs(dir)
archiver.create_archive(archive_name)
logger.info("Archive created at %s" % archive_name)
statinfo = os.stat(archive_name)
logger.info("Current size is %d" % statinfo.st_size)
logger.info("Extracting logs")
if "logs" in metadata:
logs = metadata.pop("logs")
for prefix, prefixed_logs in logs.items():
for log in prefixed_logs:
content = obfuscate(log["content"])[1]
with open(os.path.join(dir, prefix + "-" + log["name"]),
"wb") as f:
f.write(content.encode("utf-8"))
if metadata.get("result", 0) != 0:
logger.info("The scenario returned a bad exit code")
raise Exception(metadata.get("result_message", "scenario error"))
self.changelog.append("update", **metadata)
def prepare(self, profile, logfile):
self._set_adb_logger(logfile)
try:
# See android_emulator_pgo.py run_tests for more
# details on why test_root must be /sdcard/test_root
# for android pgo due to Android 4.3.
self.device = ADBDeviceFactory(verbose=self.verbose,
logger_name="adb",
test_root="/sdcard/test_root")
except Exception:
logger.error("Cannot initialize device")
raise
device = self.device
self.profile = profile
# checking that the app is installed
if not device.is_app_installed(self.app_name):
raise Exception("%s is not installed" % self.app_name)
# debug flag
logger.info("Setting %s as the debug app on the phone" % self.app_name)
device.shell(
"am set-debug-app --persistent %s" % self.app_name,
stdout_callback=logger.info,
)
# creating the profile on the device
logger.info("Creating the profile on the device")
remote_test_root = posixpath.join(device.test_root, "condprof")
remote_profile = posixpath.join(remote_test_root, "profile")
logger.info("The profile on the phone will be at %s" % remote_profile)
device.rm(remote_test_root, force=True, recursive=True)
device.mkdir(remote_test_root)
device.rm(remote_profile, force=True, recursive=True)
logger.info("Pushing %s on the phone" % self.profile)
device.push(profile, remote_profile)
device.chmod(remote_profile, recursive=True)
self.profile = profile
self.remote_profile = remote_profile
# creating the yml file
yml_data = {
"args": ["-marionette", "-profile", self.remote_profile],
"prefs": DEFAULT_PREFS,
"env": {
"LOG_VERBOSE": 1,
"R_LOG_LEVEL": 6,
"MOZ_LOG": ""
},
}
yml_name = "%s-geckoview-config.yaml" % self.app_name
yml_on_host = posixpath.join(tempfile.mkdtemp(), yml_name)
write_yml_file(yml_on_host, yml_data)
tmp_on_device = posixpath.join("/data", "local", "tmp")
if not device.exists(tmp_on_device):
raise IOError("%s does not exists on the device" % tmp_on_device)
yml_on_device = posixpath.join(tmp_on_device, yml_name)
try:
device.rm(yml_on_device, force=True, recursive=True)
device.push(yml_on_host, yml_on_device)
device.chmod(yml_on_device, recursive=True)
except Exception:
logger.info(
"could not create the yaml file on device. Permission issue?")
raise
# command line 'extra' args not used with geckoview apps; instead we use
# an on-device config.yml file
intent = "android.intent.action.VIEW"
device.stop_application(self.app_name)
if self.fennec:
# XXX does the Fennec app picks up the YML file ?
extra_args = [
"-profile",
self.remote_profile,
"--es",
"env0",
"LOG_VERBOSE=1",
"--es",
"env1",
"R_LOG_LEVEL=6",
"--es",
"env2",
"MOZ_WEBRENDER=0",
]
device.launch_fennec(
self.app_name,
extra_args=extra_args,
url="about:blank",
fail_if_running=False,
)
else:
device.launch_application(self.app_name,
self.activity,
intent,
extras=None,
url="about:blank")
if not device.process_exist(self.app_name):
raise Exception("Could not start %s" % self.app_name)
logger.info("Creating socket forwarding on port %d" %
self.marionette_port)
device.forward(
local="tcp:%d" % self.marionette_port,
remote="tcp:%d" % self.marionette_port,
)
# we don't have a clean way for now to check that GV or Fenix
# is ready to handle our tests. So here we just wait 30s
logger.info("Sleeping for 30s")
time.sleep(30)
def get_profile(
target_dir,
platform,
scenario,
customization="default",
task_id=None,
download_cache=True,
repo="mozilla-central",
):
"""Extract a conditioned profile in the target directory.
If task_id is provided, will grab the profile from that task. when not
provided (default) will grab the latest profile.
"""
# XXX assert values
params = {
"platform": platform,
"scenario": scenario,
"customization": customization,
"task_id": task_id,
"repo": repo,
}
logger.info("Getting conditioned profile with arguments: %s" % params)
filename = ARTIFACT_NAME % params
if task_id is None:
url = TC_LINK % params + filename
_check_service(TC_SERVICE)
else:
url = DIRECT_LINK % params + filename
_check_service(ARTIFACTS_SERVICE)
logger.info("preparing download dir")
if not download_cache:
download_dir = tempfile.mkdtemp()
else:
# using a cache dir in the user home dir
download_dir = os.path.expanduser(CONDPROF_CACHE)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
downloaded_archive = os.path.join(download_dir, filename)
logger.info("Downloaded archive path: %s" % downloaded_archive)
def _get_profile():
logger.info("Getting %s" % url)
try:
archive = download_file(url, target=downloaded_archive)
except ArchiveNotFound:
raise ProfileNotFoundError(url)
try:
with tarfile.open(archive, "r:gz") as tar:
logger.info("Extracting the tarball content in %s" %
target_dir)
size = len(list(tar))
with progress.Bar(expected_size=size) as bar:
def _extract(self, *args, **kw):
if not TASK_CLUSTER:
bar.show(bar.last_progress + 1)
return self.old(*args, **kw)
tar.old = tar.extract
tar.extract = functools.partial(_extract, tar)
tar.extractall(target_dir)
except (OSError, tarfile.ReadError) as e:
logger.info("Failed to extract the tarball")
if download_cache and os.path.exists(archive):
logger.info("Removing cached file to attempt a new download")
os.remove(archive)
raise ProfileNotFoundError(str(e))
finally:
if not download_cache:
shutil.rmtree(download_dir)
_check_profile(target_dir)
logger.info("Success, we have a profile to work with")
return target_dir
def onerror(error):
logger.info("Failed to get the profile.")
if os.path.exists(downloaded_archive):
try:
os.remove(downloaded_archive)
except Exception:
logger.error("Could not remove the file")
try:
return _retries(_get_profile, onerror)
except RetriesError:
raise ProfileNotFoundError(url)