def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames)})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
proxy=True,
preferences=preferences)
self.profile.set_preferences({
"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False
})
self.runner = FirefoxRunner(
profile=self.profile,
binary=self.binary,
cmdargs=[cmd_arg("marionette"), "about:blank"],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=self.debug_args,
interactive=self.interactive)
self.logger.debug("Firefox Started")
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
profile = FirefoxProfile(locations=locations, proxy=True, preferences=preferences)
profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False})
self.runner = FirefoxRunner(profile=profile,
binary=self.binary,
cmdargs=[cmd_arg("marionette"), "about:blank"],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
self.logger.debug("Firefox Started")
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({
"marionette.enabled":
True,
"marionette.port":
self.marionette_port,
"dom.disable_open_during_load":
False,
"network.dns.localDomains":
",".join(hostnames),
"network.proxy.type":
0,
"places.history.enabled":
False
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.binary, [cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def start(self):
profile = FirefoxProfile()
profile.set_preferences({"dom.disable_open_during_load": False,
"marionette.defaultPrefs.enabled": True})
self.logger.debug("Creating device runner")
self.runner = mozrunner.B2GDeviceRunner(profile=profile)
self.logger.debug("Starting device runner")
self.runner.start()
self.logger.debug("Device runner started")
def start(self):
profile = FirefoxProfile()
profile.set_preferences({"dom.disable_open_during_load": False,
"marionette.defaultPrefs.enabled": True})
self.logger.debug("Creating device runner")
self.runner = mozrunner.B2GDeviceRunner(profile=profile)
self.logger.debug("Starting device runner")
self.runner.start()
self.logger.debug("Device runner started")
def install_greasemonkey(profile_path):
profile = FirefoxProfile(str(profile_path))
gm_amo_url = 'https://services.addons.mozilla.org/fr/firefox/api/1.5/addon/748'
xpi_url = profile.addon_manager.get_amo_install_path(gm_amo_url)
print("Installing Greasemonkey from:", xpi_url)
profile.addon_manager.install_from_path(xpi_url)
# avoid showing Greasemonkey appearing in anyway in the UI
gm_prefs = {
'extensions.greasemonkey.haveInsertedToolbarbutton': True,
'extensions.greasemonkey.stats.prompted': True
}
profile.set_persistent_preferences(gm_prefs)
import ipdb; ipdb.set_trace()
def install_greasemonkey(profile_path):
profile = FirefoxProfile(str(profile_path))
gm_amo_url = 'https://services.addons.mozilla.org/fr/firefox/api/1.5/addon/748'
xpi_url = profile.addon_manager.get_amo_install_path(gm_amo_url)
print("Installing Greasemonkey from:", xpi_url)
profile.addon_manager.install_from_path(xpi_url)
# avoid showing Greasemonkey appearing in anyway in the UI
gm_prefs = {
'extensions.greasemonkey.haveInsertedToolbarbutton': True,
'extensions.greasemonkey.stats.prompted': True
}
profile.set_persistent_preferences(gm_prefs)
import ipdb
ipdb.set_trace()
def test_relative_path(self):
tempdir = tempfile.mkdtemp()
# make a dummy profile
profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
restore=False)
self.assertTrue(
os.path.exists(os.path.join(tempdir, 'testprofilepath',
'user.js')))
# make a dummy test
test = """function test() { };"""
f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
f.write(test)
f.close()
# run mozmill on it
process = ProcessHandler(
['mozmill', '-t', 'test_dummy.js', '--profile=testprofilepath'],
cwd=tempdir,
# stop mozmill from printing output to console
processOutputLine=[lambda line: None])
process.run()
process.wait()
self.assertNotEqual(process.proc.poll(), None)
# cleanup
shutil.rmtree(tempdir)
def launch_firefox(path, profile=None, url=None, args=None):
"""Launch the app with optional args for profile, windows, URI, etc.
:param path: Firefox path.
:param profile: Firefox profile.
:param url: URL to be loaded.
:param args: Optional list of arguments.
:return: List of Firefox flags.
"""
if args is None:
args = []
if profile is None:
raise APIHelperError('No profile name present, aborting run.')
profile = FirefoxProfile(profile=profile)
args.append('-foreground')
args.append('-no-remote')
if url is not None:
args.append('-new-tab')
args.append(url)
process_args = {'stream': None}
logger.debug('Creating Firefox runner ...')
try:
runner = FirefoxRunner(binary=path, profile=profile, cmdargs=args, process_args=process_args)
logger.debug('Firefox runner successfully created.')
logger.debug('Running Firefox with command: "%s"' % ','.join(runner.command))
return runner
except errors.RunnerNotStartedError:
raise APIHelperError('Error creating Firefox runner.')
def test_relative_path(self):
tempdir = tempfile.mkdtemp()
# make a dummy profile
profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
restore=False)
self.assertTrue(
os.path.exists(os.path.join(tempdir, 'testprofilepath',
'user.js')))
# make a dummy test
test = """test1 = function() { };"""
f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
f.write(test)
f.close()
# run mozmill on it
process = ProcessHandler(
['mozmill', '-t', 'test_dummy.js', '--profile=testprofilepath'],
cwd=tempdir)
code = process.waitForFinish(timeout=120)
self.assertEqual(code, 0)
# cleanup
shutil.rmtree(tempdir)
def create_profile(self, custom_addons=[], custom_prefs=None, root=True):
# Create, install and initialize the profile to be
# used in the test.
#
# Extensions are not currently supported therefore we will
# need to kill the process and ignore that fact.
self.dm.pkill(self.build.app_name, root=root)
if isinstance(custom_prefs, dict):
prefs = dict(self.preferences.items() + custom_prefs.items())
else:
prefs = self.preferences
profile = FirefoxProfile(preferences=prefs, addons=custom_addons)
if not self.install_profile(profile):
return False
self.loggerdeco.debug('Attempt to Initialize profile')
self.run_fennec_with_profile(self.build.app_name, self._initialize_url)
self.wait_for_fennec(max_wait_time=10,
wait_time=5,
kill_wait_time=5)
# minidumps not created?
self.handle_crashes()
return True
def get_content(target):
"""
Get html, css and cookies from the target site
:param target: the target site
:return: content_cookies, content_html
"""
print("{}[-] Retrieving website content {}".format(Bcolors.RESET,
Bcolors.RESET))
# create a new profile so as not to mix the user's browsing info with that of the analysis
profile_conf_name = "/tmp/gdpr-analyzer/gdpr-analyzer.default"
FirefoxProfile(profile=profile_conf_name)
# define profile preferences
browser = Browser('firefox',
headless=True,
profile=profile_conf_name,
timeout=1000,
wait_time=200,
profile_preferences={"network.cookie.cookieBehavior": 0})
# navigation run
with browser:
browser.visit(target)
# only gives us first party cookies
# content_cookies = browser.cookies.all(verbose=True)
# sad trick shot to access cookies database only work for linux because of path
paterform = platform.system()
if paterform == "Darwin":
profile_repo = glob.glob('/var/folders/sd/*/T/rust_mozprofile*')
else:
profile_repo = glob.glob('/tmp/rust_mozprofile*')
latest_profile_repo = max(profile_repo, key=os.path.getctime)
# copy database because we can not access to the one which is temporary create
db_source = latest_profile_repo + "/cookies.sqlite"
db_destination = "/tmp/gdpr-analyzer/cookies.sqlite"
shutil.copyfile(db_source, db_destination)
content_html = browser.html
# get cookie content from db
con = sqlite3.connect(db_destination)
cur = con.cursor()
cur.execute("SELECT * FROM moz_cookies")
rows = cur.fetchall()
content_cookies = []
for cookie in rows:
content_cookies.append(cookie)
con.close()
print("{}[-] Website content obtained {}".format(Bcolors.GREEN,
Bcolors.RESET))
return content_cookies, content_html
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False,
"dom.send_after_paint_to_content": True})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary,
self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def __init__(self, methodName, **kwargs):
unittest.TestCase.__init__(self, methodName)
self.loglines = []
self.duration = 0
self.expected = kwargs.pop('expected', 'pass')
self.logger = get_default_logger()
self.profile = FirefoxProfile()
self.binary = kwargs.pop('binary', None)
def create(self):
"""Create a Firefox profile and return the mozprofile Profile object pointing at that
profile"""
preferences = self._load_prefs()
profile = FirefoxProfile(preferences=preferences)
self._set_required_prefs(profile)
if self.ca_certificate_path is not None:
self._setup_ssl(profile)
return profile
def prepare_phone(self, build_metadata):
prefs = {
'browser.firstrun.show.localepicker': False,
'browser.sessionstore.resume_from_crash': False,
'browser.firstrun.show.uidiscovery': False,
'shell.checkDefaultClient': False,
'browser.warnOnQuit': False,
'browser.EULA.override': True,
'toolkit.telemetry.prompted': 999,
'toolkit.telemetry.notifiedOptOut': 999
}
profile = FirefoxProfile(preferences=prefs)
self.install_profile(profile)
def create(self, **kwargs):
"""Create a Firefox profile and return the mozprofile Profile object pointing at that
profile
:param kwargs: Additional arguments to pass into the profile constructor
"""
preferences = self._load_prefs()
profile = FirefoxProfile(preferences=preferences, **kwargs)
self._set_required_prefs(profile)
if self.ca_certificate_path is not None:
self._setup_ssl(profile)
return profile
def youtube_video(video_id):
config["playback_recordings"] = "%s.playback" % video_id
config["playback_tool_args"] = [
"--set", "upstream_cert=false",
"-S", "/tmp/testing/mozproxy/%s.playback" % video_id,
]
proxy = get_playback(config)
if proxy is None:
raise Exception("Could not start Proxy")
try:
prefs = {"media.autoplay.default": 0}
prefs["browser.newtabpage.activity-stream.feeds.snippets"] = 0
prefs["browser.newtabpage.activity-stream.disableSnippets"] = 1
prefs["network.proxy.type"] = 1
prefs["network.proxy.http"] = config["host"]
prefs["network.proxy.http_port"] = 8080
prefs["network.proxy.ssl"] = config["host"]
prefs["network.proxy.ssl_port"] = 8080
prefs["network.proxy.no_proxies_on"] = config["host"]
profile = FirefoxProfile(profile="/tmp/mozprof", preferences=prefs, addons=[])
browser = apps["fxdesktop"].create(
profile=profile, app="fxdesktop", bin=config["binary"],
app_args=["about:blank"]
)
browser.start()
except Exception:
proxy.stop()
raise
try:
page = YoutubePage(video_id)
except Exception:
try:
browser.close()
finally:
proxy.stop()
raise
try:
yield page
finally:
try:
page.close()
browser.close()
finally:
proxy.stop()
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.enabled": True,
"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary,
self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def run(self):
'''
Starts Firefox thread with Marionette turned on.
'''
self.profile = FirefoxProfile()
self.profile.set_preferences({"marionette.defaultPrefs.enabled" : True,
"marionette.defaultPrefs.port": 2828,
"browser.startup.page": 0,
"browser.startup.homepage": "about:blank",
})
self.runner = FirefoxRunner(profile = self.profile,
binary = self.binary,
kp_kwargs = {'processOutputLine' : [self.logger]})
self.runner.start()
self._firefoxRunningEvent.set()
self.runner.wait()
def create_profile(self, build_metadata, custom_prefs=None):
# Create, install and initialize the profile to be
# used in the test.
telemetry_prompt = 999
if build_metadata['blddate'] < '2013-01-03':
telemetry_prompt = 2
prefs = {
'browser.firstrun.show.localepicker': False,
'browser.sessionstore.resume_from_crash': False,
'dom.ipc.plugins.flash.subprocess.crashreporter.enabled': False,
'browser.firstrun.show.uidiscovery': False,
'shell.checkDefaultClient': False,
'browser.warnOnQuit': False,
'browser.EULA.override': True,
'toolkit.telemetry.prompted': telemetry_prompt,
'toolkit.telemetry.notifiedOptOut': telemetry_prompt,
'datareporting.healthreport.service.enabled': False,
}
if isinstance(custom_prefs, dict):
prefs = dict(prefs.items() + custom_prefs.items())
profile = FirefoxProfile(preferences=prefs,
addons='%s/xpi/quitter.xpi' % os.getcwd())
if not self.install_profile(profile):
return False
appname = build_metadata['androidprocname']
buildid = build_metadata['buildid']
success = False
for attempt in range(self.user_cfg[PHONE_RETRY_LIMIT]):
self.loggerdeco.debug('Attempt %d Initializing profile' % attempt)
self.run_fennec_with_profile(appname, self._initialize_url)
if self.wait_for_fennec(build_metadata):
success = True
break
sleep(self.user_cfg[PHONE_RETRY_WAIT])
if not success:
msg = 'Failure initializing profile for build %s' % buildid
self.loggerdeco.error(msg)
self.set_status(msg=msg)
return success
def install_profile(self, profile=None, root=True):
if not profile:
profile = FirefoxProfile()
profile_path_parent = os.path.split(self.profile_path)[0]
success = False
for attempt in range(1, self.options.phone_retry_limit + 1):
try:
self.loggerdeco.debug('Attempt %d installing profile', attempt)
if self.dm.exists(self.profile_path, root=root):
# If the profile already exists, chmod it to make sure
# we have permission to delete it.
self.dm.chmod(self.profile_path, recursive=True, root=root)
self.dm.rm(self.profile_path,
recursive=True,
force=True,
root=root)
self.dm.chmod(profile_path_parent, root=root)
self.dm.mkdir(self.profile_path, root=root)
self.dm.chmod(self.profile_path, root=root)
self.dm.push(profile.profile, self.profile_path)
self.dm.chmod(self.profile_path, recursive=True, root=root)
success = True
break
except ADBError:
self.loggerdeco.exception('Attempt %d Exception installing '
'profile to %s' %
(attempt, self.profile_path))
sleep(self.options.phone_retry_wait)
if not success:
self.add_failure(
self.name, TestStatus.TEST_UNEXPECTED_FAIL,
'Failure installing profile to %s' % self.profile_path,
TreeherderStatus.TESTFAILED)
return success
def create_profile(self, custom_addons=[], custom_prefs=None, root=True):
# Create, install and initialize the profile to be
# used in the test.
temp_addons = ['quitter.xpi']
temp_addons.extend(custom_addons)
addons = ['%s/xpi/%s' % (os.getcwd(), addon) for addon in temp_addons]
# make sure firefox isn't running when we try to
# install the profile.
self.dm.pkill(self.build.app_name, root=root)
if isinstance(custom_prefs, dict):
prefs = dict(self.preferences.items() + custom_prefs.items())
else:
prefs = self.preferences
profile = FirefoxProfile(preferences=prefs, addons=addons)
if not self.install_profile(profile):
return False
success = False
for attempt in range(1, self.options.phone_retry_limit + 1):
self.loggerdeco.debug('Attempt %d Initializing profile', attempt)
self.run_fennec_with_profile(self.build.app_name,
self._initialize_url)
if self.wait_for_fennec():
success = True
break
sleep(self.options.phone_retry_wait)
if not success or self.handle_crashes():
self.add_failure(self.name, TestStatus.TEST_UNEXPECTED_FAIL,
'Failure initializing profile',
TreeherderStatus.TESTFAILED)
return success
def createprofile(self, targetpath, delete=False):
if not self.savedpath:
self.save(targetpath)
profiledir = "profile" + self._platformsuffix
profilepath = os.path.join(targetpath, self.parent.version, profiledir)
if delete and os.path.exists(profilepath):
shutil.rmtree(profilepath)
# TODO non-firefox, multiple files
profileparams = {
"profile": profilepath,
"addons": [self.savedpath],
"preferences": {
"xpinstall.signatures.required": False,
# Enable browser toolbox to monitor network requests
"devtools.chrome.enabled": True,
"devtools.debugger.remote-enabled": True
},
"restore": False
}
self.profile = FirefoxProfile(**profileparams)
return self.profile
class FirefoxBrowser(Browser):
used_ports = set()
def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.marionette_port = None
self.used_ports.add(self.marionette_port)
self.runner = None
self.debug_args = debug_args
self.interactive = interactive
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
self.certutil_binary = certutil_binary
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
ports = {"http": "8000",
"https": "8443",
"ws": "8888"}
self.profile = FirefoxProfile(locations=locations,
proxy=ports,
preferences=preferences)
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False})
if self.ca_certificate_path is not None:
self.setup_ssl()
self.runner = FirefoxRunner(profile=self.profile,
binary=self.binary,
cmdargs=[cmd_arg("marionette"), "about:blank"],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
if os.path.exists(prefs_path):
preferences = Preferences.read_prefs(prefs_path)
else:
self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
preferences = []
return preferences
def stop(self):
self.logger.debug("Stopping browser")
if self.runner is not None:
try:
self.runner.stop()
except OSError:
# This can happen on Windows if the process is already dead
pass
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
self.logger.process_output(self.pid(),
line.decode("utf8", "replace"),
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
mozcrash.log_crashes(self.logger,
dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process,
test=test)
def setup_ssl(self):
"""Create a certificate database to use in the test profile. This is configured
to trust the CA Certificate that has signed the web-platform.test server
certificate."""
self.logger.info("Setting up ssl")
# Make sure the certutil libraries from the source tree are loaded when using a
# local copy of certutil
# TODO: Maybe only set this if certutil won't launch?
env = os.environ.copy()
certutil_dir = os.path.dirname(self.binary)
env["LD_LIBRARY_PATH"] = certutil_dir
env["PATH"] = os.path.pathsep.join([certutil_dir, env["PATH"]])
def certutil(*args):
cmd = [self.certutil_binary] + list(args)
self.logger.process_output("certutil",
subprocess.check_output(cmd,
env=env,
stderr=subprocess.STDOUT),
" ".join(cmd))
pw_path = os.path.join(self.profile.profile, ".crtdbpw")
with open(pw_path, "w") as f:
# Use empty password for certificate db
f.write("\n")
cert_db_path = self.profile.profile
# Create a new certificate db
certutil("-N", "-d", cert_db_path, "-f", pw_path)
# Add the CA certificate to the database and mark as trusted to issue server certs
certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
"-n", "web-platform-tests", "-i", self.ca_certificate_path)
# List all certs in the database
certutil("-L", "-d", cert_db_path)
class FirefoxAndroidBrowser(FirefoxBrowser):
init_timeout = 300
shutdown_timeout = 60
def __init__(self,
logger,
prefs_root,
test_type,
package_name="org.mozilla.geckoview.test",
device_serial="emulator-5444",
**kwargs):
FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type,
**kwargs)
self.package_name = package_name
self.device_serial = device_serial
self.tests_root = kwargs["tests_root"]
self.install_fonts = kwargs["install_fonts"]
self.stackwalk_binary = kwargs["stackwalk_binary"]
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port()
env = {}
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.enable_webrender:
env["MOZ_WEBRENDER"] = "1"
else:
env["MOZ_WEBRENDER"] = "0"
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"network.dns.localDomains":
",".join(self.config.domains_set),
"dom.disable_open_during_load":
False,
"places.history.enabled":
False,
"dom.send_after_paint_to_content":
True,
"network.preload":
True,
})
if self.test_type == "reftest":
self.logger.info("Setting android reftest preferences")
self.profile.set_preferences({
"browser.viewport.desktopWidth":
800,
# Disable high DPI
"layout.css.devPixelsPerPx":
"1.0",
# Ensure that the full browser element
# appears in the screenshot
"apz.allow_zooming":
False,
"android.widget_paints_background":
False,
# Ensure that scrollbars are always painted
"layout.testing.overlay-scrollbars.always-visible":
True,
})
if self.install_fonts:
self.logger.debug("Copying Ahem font to profile")
font_dir = os.path.join(self.profile.profile, "fonts")
if not os.path.exists(font_dir):
os.makedirs(font_dir)
with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"),
"rb") as src:
with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
dest.write(src.read())
self.leak_report_file = None
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.package_name, self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"], self.debug_info)
self.runner = FennecEmulatorRunner(
app=self.package_name,
profile=self.profile,
cmdargs=cmd[1:],
env=env,
symbols_path=self.symbols_path,
serial=self.device_serial,
# TODO - choose appropriate log dir
logdir=os.getcwd())
self.logger.debug("Starting %s" % self.package_name)
# connect to a running emulator
self.runner.device.connect()
self.runner.stop()
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.runner.device.device.forward(
local="tcp:{}".format(self.marionette_port),
remote="tcp:{}".format(self.marionette_port))
for ports in self.config.ports.values():
for port in ports:
self.runner.device.device.reverse(local="tcp:{}".format(port),
remote="tcp:{}".format(port))
self.logger.debug("%s Started" % self.package_name)
def stop(self, force=False):
if self.runner is not None:
if self.runner.device.connected:
try:
self.runner.device.device.remove_forwards()
self.runner.device.device.remove_reverses()
except Exception as e:
self.logger.warning(
"Failed to remove forwarded or reversed ports: %s" % e)
# We assume that stopping the runner prompts the
# browser to shut down.
self.runner.stop()
self.logger.debug("stopped")
def check_crash(self, process, test):
if not os.environ.get("MINIDUMP_STACKWALK",
"") and self.stackwalk_binary:
os.environ["MINIDUMP_STACKWALK"] = self.stackwalk_binary
return bool(self.runner.check_for_crashes(test_name=test))
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60
shutdown_timeout = 60
def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False, stackfix_dir=None,
binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1,
chaos_mode_flags=None, config=None):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.test_type = test_type
self.extra_prefs = extra_prefs
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
self.certutil_binary = certutil_binary
self.e10s = e10s
self.binary_args = binary_args
self.config = config
if stackfix_dir:
self.stack_fixer = get_stack_fixer_function(stackfix_dir,
self.symbols_path)
else:
self.stack_fixer = None
if timeout_multiplier:
self.init_timeout = self.init_timeout * timeout_multiplier
self.leak_report_file = None
self.leak_check = leak_check
self.stylo_threads = stylo_threads
self.chaos_mode_flags = chaos_mode_flags
def settings(self, test):
return {"check_leaks": self.leak_check and not test.leaks}
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(self.config.domains_set),
"network.proxy.type": 0,
"places.history.enabled": False,
"dom.send_after_paint_to_content": True,
"network.preload": True})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary,
self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs = Preferences()
pref_paths = []
prefs_general = os.path.join(self.prefs_root, 'prefs_general.js')
if os.path.isfile(prefs_general):
# Old preference file used in Firefox 60 and earlier (remove when no longer supported)
pref_paths.append(prefs_general)
profiles = os.path.join(self.prefs_root, 'profiles.json')
if os.path.isfile(profiles):
with open(profiles, 'r') as fh:
for name in json.load(fh)['web-platform-tests']:
pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
for path in pref_paths:
if os.path.exists(path):
prefs.add(Preferences.read_prefs(path))
else:
self.logger.warning("Failed to find base prefs file in %s" % path)
# Add any custom preferences
prefs.add(self.extra_prefs, cast=True)
return prefs()
def stop(self, force=False):
if self.runner is not None and self.runner.is_running():
try:
# For Firefox we assume that stopping the runner prompts the
# browser to shut down. This allows the leak log to be written
for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)),
(False, lambda: self.runner.stop(signal.SIGTERM)),
(False, lambda: self.runner.stop(signal.SIGKILL))]:
if not force or not clean:
retcode = stop_f()
if retcode is not None:
self.logger.info("Browser exited with return code %s" % retcode)
break
except OSError:
# This can happen on Windows if the process is already dead
pass
self.logger.debug("stopped")
def process_leaks(self):
self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
if self.leak_report_file is None:
return
mozleak.process_leak_log(
self.leak_report_file,
leak_thresholds={
"default": 0,
"tab": 10000, # See dependencies of bug 1051230.
# GMP rarely gets a log, but when it does, it leaks a little.
"geckomediaplugin": 20000,
},
ignore_missing_leaks=["geckomediaplugin"],
log=self.logger,
stack_fixer=self.stack_fixer
)
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
if "GLib-GObject-CRITICAL" in line:
return
if line:
data = line.decode("utf8", "replace")
if self.stack_fixer:
data = self.stack_fixer(data)
self.logger.process_output(self.pid(),
data,
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
self.process_leaks()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def check_for_crashes(self):
dump_dir = os.path.join(self.profile.profile, "minidumps")
return bool(mozcrash.check_for_crashes(dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
quiet=True))
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
mozcrash.log_crashes(self.logger,
dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process,
test=test)
def setup_ssl(self):
"""Create a certificate database to use in the test profile. This is configured
to trust the CA Certificate that has signed the web-platform.test server
certificate."""
if self.certutil_binary is None:
self.logger.info("--certutil-binary not supplied; Firefox will not check certificates")
return
self.logger.info("Setting up ssl")
# Make sure the certutil libraries from the source tree are loaded when using a
# local copy of certutil
# TODO: Maybe only set this if certutil won't launch?
env = os.environ.copy()
certutil_dir = os.path.dirname(self.binary)
if mozinfo.isMac:
env_var = "DYLD_LIBRARY_PATH"
elif mozinfo.isUnix:
env_var = "LD_LIBRARY_PATH"
else:
env_var = "PATH"
env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
if env_var in env else certutil_dir).encode(
sys.getfilesystemencoding() or 'utf-8', 'replace')
def certutil(*args):
cmd = [self.certutil_binary] + list(args)
self.logger.process_output("certutil",
subprocess.check_output(cmd,
env=env,
stderr=subprocess.STDOUT),
" ".join(cmd))
pw_path = os.path.join(self.profile.profile, ".crtdbpw")
with open(pw_path, "w") as f:
# Use empty password for certificate db
f.write("\n")
cert_db_path = self.profile.profile
# Create a new certificate db
certutil("-N", "-d", cert_db_path, "-f", pw_path)
# Add the CA certificate to the database and mark as trusted to issue server certs
certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
"-n", "web-platform-tests", "-i", self.ca_certificate_path)
# List all certs in the database
certutil("-L", "-d", cert_db_path)
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, "build")
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo"))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(self.topsrcdir, "testing",
"profiles")
with open(os.path.join(profile_data_dir, "profiles.json"),
"r") as fh:
base_profiles = json.load(fh)["valgrind"]
prefpaths = [
os.path.join(profile_data_dir, profile, "user.js")
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(self.topsrcdir, "tools", "quitter",
"*****@*****.**")
locations = ServerLocations()
locations.add_host(host="127.0.0.1",
port=httpd.httpd.server_port,
options="primary")
profile = FirefoxProfile(
profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations,
)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env["G_SLICE"] = "always-malloc"
env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
outputHandler = OutputHandler(self.log)
kp_kwargs = {
"processOutputLine": [outputHandler],
"universal_newlines": True,
}
valgrind = "valgrind"
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
"--sym-offsets=yes",
"--smc-check=all-non-file",
"--vex-iropt-register-updates=allregs-at-mem-access",
"--gen-suppressions=all",
"--num-callers=36",
"--leak-check=full",
"--show-possibly-lost=no",
"--track-origins=yes",
"--trace-children=yes",
"-v", # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
"--fair-sched=yes",
# Keep debuginfo after library unmap. See bug 1382280.
"--keep-debuginfo=yes",
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
"--expensive-definedness-checks=yes",
# Compensate for the compiler inlining `new` but not `delete`
# or vice versa.
"--show-mismatched-frees=no",
]
for s in suppressions:
valgrind_args.append("--suppressions=" + s)
supps_dir = os.path.join(build_dir, "valgrind")
supps_file1 = os.path.join(supps_dir, "cross-architecture.sup")
valgrind_args.append("--suppressions=" + supps_file1)
if mozinfo.os == "linux":
machtype = {
"x86_64": "x86_64-pc-linux-gnu",
"x86": "i386-pc-linux-gnu",
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + ".sup")
if os.path.isfile(supps_file2):
valgrind_args.append("--suppressions=" + supps_file2)
exitcode = None
timeout = 1800
binary_not_found_exception = None
try:
runner = FirefoxRunner(
profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs,
)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
except BinaryNotFoundException as e:
binary_not_found_exception = e
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR,
"valgrind-fail-parsing",
{
"errs": errs,
"supps": supps
},
"TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors "
"seen, but {supps} generated suppressions seen",
)
elif errs == 0:
status = 0
self.log(
logging.INFO,
"valgrind-pass",
{},
"TEST-PASS | valgrind-test | valgrind found no errors",
)
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if binary_not_found_exception:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{"error": str(binary_not_found_exception)},
"TEST-UNEXPECTED-FAIL | valgrind-test | {error}",
)
self.log(
logging.INFO,
"valgrind-fail-errors",
{"help": binary_not_found_exception.help()},
"{help}",
)
elif exitcode is None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-timeout",
{"timeout": timeout},
"TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out "
"(reached {timeout} second limit)",
)
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{"exitcode": exitcode},
"TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code "
"from Valgrind: {exitcode}",
)
httpd.stop()
return status
for profile in base_profiles]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {"server": "%s:%d" % httpd.httpd.server_address,
"OOP": "false"}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[os.path.join(
build.topsrcdir, 'tools', 'quitter',
'*****@*****.**')],
locations=locations)
env = os.environ.copy()
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
# For VC12+, make sure we can find the right bitness of pgort1x0.dll
if not substs.get('HAVE_64BIT_BUILD'):
for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
if e not in env:
continue
vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
if os.path.exists(vcdir):
class FirefoxBrowser(Browser):
used_ports = set()
def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
symbols_path=None, stackwalk_binary=None):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.marionette_port = None
self.used_ports.add(self.marionette_port)
self.runner = None
self.debug_args = debug_args
self.interactive = interactive
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
ports = {"http": "8000",
"https": "8443",
"ws": "8888"}
self.profile = FirefoxProfile(locations=locations,
proxy=ports,
preferences=preferences)
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False})
self.runner = FirefoxRunner(profile=self.profile,
binary=self.binary,
cmdargs=[cmd_arg("marionette"), "about:blank"],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
if os.path.exists(prefs_path):
preferences = Preferences.read_prefs(prefs_path)
else:
self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
preferences = []
return preferences
def stop(self):
self.logger.debug("Stopping browser")
if self.runner is not None:
try:
self.runner.stop()
except OSError:
# This can happen on Windows if the process is already dead
pass
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
self.logger.process_output(self.pid(),
line.decode("utf8", "replace"),
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process, test=test)
try:
#TODO: refactor this into mozprofile
prefpath = os.path.join(build.topsrcdir, "testing", "profiles",
"prefs_general.js")
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
"OOP": "false"
}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
profile = FirefoxProfile(
profile=profilePath,
preferences=prefs,
addons=[os.path.join(build.distdir, 'xpi-stage', 'quitter')],
locations=locations)
env = os.environ.copy()
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
jarlog = os.getenv("JARLOG_FILE")
if jarlog:
env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]
cmdargs = ["http://localhost:%d/index.html" % PORT]
runner = FirefoxRunner(
profile=profile,
binary=build.get_binary_path(where="staged-package"),
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60
shutdown_timeout = 60
def __init__(self,
logger,
binary,
prefs_root,
test_type,
extra_prefs=None,
debug_info=None,
symbols_path=None,
stackwalk_binary=None,
certutil_binary=None,
ca_certificate_path=None,
e10s=False,
stackfix_dir=None,
binary_args=None,
timeout_multiplier=None,
leak_check=False,
stylo_threads=1,
chaos_mode_flags=None):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.test_type = test_type
self.extra_prefs = extra_prefs
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
self.certutil_binary = certutil_binary
self.e10s = e10s
self.binary_args = binary_args
if stackfix_dir:
self.stack_fixer = get_stack_fixer_function(
stackfix_dir, self.symbols_path)
else:
self.stack_fixer = None
if timeout_multiplier:
self.init_timeout = self.init_timeout * timeout_multiplier
self.leak_report_file = None
self.leak_check = leak_check
self.stylo_threads = stylo_threads
self.chaos_mode_flags = chaos_mode_flags
def settings(self, test):
return {"check_leaks": self.leak_check and not test.leaks}
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"dom.disable_open_during_load":
False,
"network.dns.localDomains":
",".join(hostnames),
"network.proxy.type":
0,
"places.history.enabled":
False,
"dom.send_after_paint_to_content":
True,
"network.preload":
True
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences(
{"layout.interruptible-reflow.enabled": False})
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile,
"runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.binary, self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"], self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs = Preferences()
prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
if os.path.exists(prefs_path):
prefs.add(Preferences.read_prefs(prefs_path))
else:
self.logger.warning("Failed to find base prefs file in %s" %
prefs_path)
# Add any custom preferences
prefs.add(self.extra_prefs, cast=True)
return prefs()
def stop(self, force=False):
if self.runner is not None and self.runner.is_running():
try:
# For Firefox we assume that stopping the runner prompts the
# browser to shut down. This allows the leak log to be written
for clean, stop_f in [
(True, lambda: self.runner.wait(self.shutdown_timeout)),
(False, lambda: self.runner.stop(signal.SIGTERM)),
(False, lambda: self.runner.stop(signal.SIGKILL))
]:
if not force or not clean:
retcode = stop_f()
if retcode is not None:
self.logger.info(
"Browser exited with return code %s" % retcode)
break
except OSError:
# This can happen on Windows if the process is already dead
pass
self.logger.debug("stopped")
def process_leaks(self):
self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
if self.leak_report_file is None:
return
mozleak.process_leak_log(
self.leak_report_file,
leak_thresholds={
"default": 0,
"tab": 10000, # See dependencies of bug 1051230.
# GMP rarely gets a log, but when it does, it leaks a little.
"geckomediaplugin": 20000,
},
ignore_missing_leaks=["geckomediaplugin"],
log=self.logger,
stack_fixer=self.stack_fixer)
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
data = line.decode("utf8", "replace")
if self.stack_fixer:
data = self.stack_fixer(data)
self.logger.process_output(self.pid(),
data,
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
self.process_leaks()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def check_for_crashes(self):
dump_dir = os.path.join(self.profile.profile, "minidumps")
return bool(
mozcrash.check_for_crashes(dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
quiet=True))
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
mozcrash.log_crashes(self.logger,
dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process,
test=test)
def setup_ssl(self):
"""Create a certificate database to use in the test profile. This is configured
to trust the CA Certificate that has signed the web-platform.test server
certificate."""
if self.certutil_binary is None:
self.logger.info(
"--certutil-binary not supplied; Firefox will not check certificates"
)
return
self.logger.info("Setting up ssl")
# Make sure the certutil libraries from the source tree are loaded when using a
# local copy of certutil
# TODO: Maybe only set this if certutil won't launch?
env = os.environ.copy()
certutil_dir = os.path.dirname(self.binary)
if mozinfo.isMac:
env_var = "DYLD_LIBRARY_PATH"
elif mozinfo.isUnix:
env_var = "LD_LIBRARY_PATH"
else:
env_var = "PATH"
env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
if env_var in env else certutil_dir).encode(
sys.getfilesystemencoding() or 'utf-8', 'replace')
def certutil(*args):
cmd = [self.certutil_binary] + list(args)
self.logger.process_output(
"certutil",
subprocess.check_output(cmd, env=env,
stderr=subprocess.STDOUT),
" ".join(cmd))
pw_path = os.path.join(self.profile.profile, ".crtdbpw")
with open(pw_path, "w") as f:
# Use empty password for certificate db
f.write("\n")
cert_db_path = self.profile.profile
# Create a new certificate db
certutil("-N", "-d", cert_db_path, "-f", pw_path)
# Add the CA certificate to the database and mark as trusted to issue server certs
certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n",
"web-platform-tests", "-i", self.ca_certificate_path)
# List all certs in the database
certutil("-L", "-d", cert_db_path)
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(self.topsrcdir, 'testing',
'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['valgrind']
prefpaths = [
os.path.join(profile_data_dir, profile, 'user.js')
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
'server': '%s:%d' % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(self.topsrcdir, 'tools', 'quitter',
'*****@*****.**')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--sym-offsets=yes',
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
'-v', # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
'--fair-sched=yes',
# Keep debuginfo after library unmap. See bug 1382280.
'--keep-debuginfo=yes',
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
'--expensive-definedness-checks=yes',
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
if mozinfo.os == 'linux':
machtype = {
'x86_64': 'x86_64-pc-linux-gnu',
'x86': 'i386-pc-linux-gnu',
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR, 'valgrind-fail-parsing', {
'errs': errs,
'supps': supps
},
'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors '
'seen, but {supps} generated suppressions seen')
elif errs == 0:
status = 0
self.log(
logging.INFO, 'valgrind-pass', {},
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode is None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR, 'valgrind-fail-timeout',
{'timeout': timeout},
'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out '
'(reached {timeout} second limit)')
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR, 'valgrind-fail-errors',
{'exitcode': exitcode},
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code '
'from Valgrind: {exitcode}')
httpd.stop()
return status
class FennecBrowser(FirefoxBrowser):
used_ports = set()
init_timeout = 300
shutdown_timeout = 60
def __init__(self, logger, prefs_root, test_type, package_name=None,
device_serial="emulator-5444", **kwargs):
FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
self._package_name = package_name
self.device_serial = device_serial
self.tests_root = kwargs["tests_root"]
self.install_fonts = kwargs["install_fonts"]
@property
def package_name(self):
"""
Name of app to run on emulator.
"""
if self._package_name is None:
self._package_name = "org.mozilla.fennec"
user = os.getenv("USER")
if user:
self._package_name += "_" + user
return self._package_name
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = {}
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"places.history.enabled": False,
"dom.send_after_paint_to_content": True,
"network.preload": True})
if self.test_type == "reftest":
self.logger.info("Setting android reftest preferences")
self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
# Disable high DPI
"layout.css.devPixelsPerPx": "1.0",
# Ensure that the full browser element
# appears in the screenshot
"apz.allow_zooming": False,
"android.widget_paints_background": False,
# Ensure that scrollbars are always painted
"ui.scrollbarFadeBeginDelay": 100000})
if self.install_fonts:
self.logger.debug("Copying Ahem font to profile")
font_dir = os.path.join(self.profile.profile, "fonts")
if not os.path.exists(font_dir):
os.makedirs(font_dir)
with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
dest.write(src.read())
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.package_name,
self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FennecEmulatorRunner(app=self.package_name,
profile=self.profile,
cmdargs=cmd[1:],
env=env,
symbols_path=self.symbols_path,
serial=self.device_serial,
# TODO - choose appropriate log dir
logdir=os.getcwd(),
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting %s" % self.package_name)
# connect to a running emulator
self.runner.device.connect()
write_hosts_file(self.config, self.runner.device.device)
self.runner.stop()
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.runner.device.device.forward(
local="tcp:{}".format(self.marionette_port),
remote="tcp:{}".format(self.marionette_port))
self.logger.debug("%s Started" % self.package_name)
def stop(self, force=False):
if self.runner is not None:
if (self.runner.device.connected and
len(self.runner.device.device.list_forwards()) > 0):
try:
self.runner.device.device.remove_forwards(
"tcp:{}".format(self.marionette_port))
except Exception:
self.logger.warning("Failed to remove port forwarding")
# We assume that stopping the runner prompts the
# browser to shut down. This allows the leak log to be written
self.runner.stop()
self.logger.debug("stopped")
def run():
parser = argparse.ArgumentParser(description="Run crawler")
parser.add_argument("-b", "--binary", type=str, help="path to the Firefox binary")
parser.add_argument("-a", "--abpdir", type=str, help="path to the Adblock Plus repository")
parser.add_argument(
"-f",
"--filters",
metavar="url",
type=str,
nargs="+",
default=[
"https://easylist-downloads.adblockplus.org/easylist.txt",
"https://easylist-downloads.adblockplus.org/exceptionrules.txt",
],
help="filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.",
)
parser.add_argument("-t", "--timeout", type=int, default=300, help="Load timeout (seconds)")
parser.add_argument("-x", "--maxtabs", type=int, default=15, help="Maximal number of tabs to open in parallel")
parser.add_argument("list", type=str, help="URL list to process")
parser.add_argument("outdir", type=str, help="directory to write data into")
parameters = parser.parse_args()
import buildtools.packagerGecko as packager
cleanup = []
try:
base_dir = os.path.dirname(os.path.abspath(__file__))
handle, crawlerxpi = tempfile.mkstemp(suffix=".xpi")
os.close(handle)
cleanup.append(crawlerxpi)
packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)
abpxpi = "https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi"
if parameters.abpdir:
handle, abpxpi = tempfile.mkstemp(suffix=".xpi")
os.close(handle)
cleanup.append(abpxpi)
packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True)
profile = FirefoxProfile(
addons=[crawlerxpi, abpxpi],
preferences={
"browser.startup.homepage": "about:blank",
"browser.tabs.warnOnCloseOtherTabs": False,
"browser.uitour.enabled": False,
"prompts.tab_modal.enabled": False,
"startup.homepage_welcome_url": "about:blank",
"startup.homepage_welcome_url.additional": "about:blank",
"xpinstall.signatures.required": False,
},
)
abpsettings = os.path.join(profile.profile, "adblockplus")
os.makedirs(abpsettings)
with open(os.path.join(abpsettings, "patterns.ini"), "w") as handle:
print >> handle, "# Adblock Plus preferences"
print >> handle, "version=4"
for url in parameters.filters:
if "=" in url:
path, url = url.split("=", 1)
with open(path, "r") as source:
data = source.read()
else:
data = urllib.urlopen(url).read()
print >> handle, "[Subscription]"
print >> handle, "url=%s" % url
print >> handle, "[Subscription filters]"
print >> handle, "\n".join(data.splitlines()[1:])
finally:
for path in cleanup:
os.unlink(path)
server = None
try:
port = random.randrange(2000, 60000)
print "Communicating with client on port %i" % port
app = CrawlerApp(parameters)
server = make_server("localhost", port, app)
app.server = server
threading.Thread(target=lambda: server.serve_forever()).start()
runner = FirefoxRunner(
profile=profile,
binary=parameters.binary,
cmdargs=["--crawler-port", str(port)],
env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE="1"),
)
while app.urls:
runner.start()
runner.wait()
finally:
if server:
server.shutdown()
profile.cleanup()
def start(self, **kwargs):
if self.marionette_port is None:
with FennecBrowser.used_ports_lock:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = {}
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"places.history.enabled": False,
"dom.send_after_paint_to_content": True,
"network.preload": True})
if self.test_type == "reftest":
self.logger.info("Setting android reftest preferences")
self.profile.set_preferences({"browser.viewport.desktopWidth": 800,
# Disable high DPI
"layout.css.devPixelsPerPx": "1.0",
# Ensure that the full browser element
# appears in the screenshot
"apz.allow_zooming": False,
"android.widget_paints_background": False,
# Ensure that scrollbars are always painted
"layout.testing.overlay-scrollbars.always-visible": True})
if self.install_fonts:
self.logger.debug("Copying Ahem font to profile")
font_dir = os.path.join(self.profile.profile, "fonts")
if not os.path.exists(font_dir):
os.makedirs(font_dir)
with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
dest.write(src.read())
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.package_name,
self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FennecEmulatorRunner(app=self.package_name,
profile=self.profile,
cmdargs=cmd[1:],
env=env,
symbols_path=self.symbols_path,
serial=self.device_serial,
# TODO - choose appropriate log dir
logdir=os.getcwd(),
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting %s" % self.package_name)
# connect to a running emulator
self.runner.device.connect()
write_hosts_file(self.config, self.runner.device.device)
self.runner.stop()
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.runner.device.device.forward(
local="tcp:{}".format(self.marionette_port),
remote="tcp:{}".format(self.marionette_port))
self.logger.debug("%s Started" % self.package_name)
def start(self, group_metadata=None, **kwargs):
if group_metadata is None:
group_metadata = {}
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
if self.asan:
print "Setting up LSAN"
self.lsan_handler = mozleak.LSANLeaks(self.logger,
scope=group_metadata.get("scope", "/"),
allowed=self.lsan_allowed,
maxNumRecordedFrames=self.lsan_max_stack_depth)
env = test_environment(xrePath=os.path.dirname(self.binary),
debugger=self.debug_info is not None,
log=self.logger,
lsanPath=self.prefs_root)
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.headless:
env["MOZ_HEADLESS"] = "1"
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({
"marionette.port": self.marionette_port,
"network.dns.localDomains": ",".join(self.config.domains_set),
"dom.file.createInChild": True,
# TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
"network.proxy.type": 0,
"places.history.enabled": False,
"network.preload": True,
})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
if self.leak_check:
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary,
args,
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port()
env = {}
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.enable_webrender:
env["MOZ_WEBRENDER"] = "1"
else:
env["MOZ_WEBRENDER"] = "0"
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"network.dns.localDomains":
",".join(self.config.domains_set),
"dom.disable_open_during_load":
False,
"places.history.enabled":
False,
"dom.send_after_paint_to_content":
True,
"network.preload":
True,
})
if self.test_type == "reftest":
self.logger.info("Setting android reftest preferences")
self.profile.set_preferences({
"browser.viewport.desktopWidth":
800,
# Disable high DPI
"layout.css.devPixelsPerPx":
"1.0",
# Ensure that the full browser element
# appears in the screenshot
"apz.allow_zooming":
False,
"android.widget_paints_background":
False,
# Ensure that scrollbars are always painted
"layout.testing.overlay-scrollbars.always-visible":
True,
})
if self.install_fonts:
self.logger.debug("Copying Ahem font to profile")
font_dir = os.path.join(self.profile.profile, "fonts")
if not os.path.exists(font_dir):
os.makedirs(font_dir)
with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"),
"rb") as src:
with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
dest.write(src.read())
self.leak_report_file = None
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.package_name, self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"], self.debug_info)
self.runner = FennecEmulatorRunner(
app=self.package_name,
profile=self.profile,
cmdargs=cmd[1:],
env=env,
symbols_path=self.symbols_path,
serial=self.device_serial,
# TODO - choose appropriate log dir
logdir=os.getcwd())
self.logger.debug("Starting %s" % self.package_name)
# connect to a running emulator
self.runner.device.connect()
self.runner.stop()
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.runner.device.device.forward(
local="tcp:{}".format(self.marionette_port),
remote="tcp:{}".format(self.marionette_port))
for ports in self.config.ports.values():
for port in ports:
self.runner.device.device.reverse(local="tcp:{}".format(port),
remote="tcp:{}".format(port))
self.logger.debug("%s Started" % self.package_name)
def start(self, group_metadata=None, **kwargs):
if group_metadata is None:
group_metadata = {}
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
if self.asan:
print "Setting up LSAN"
self.lsan_handler = mozleak.LSANLeaks(
self.logger,
scope=group_metadata.get("scope", "/"),
allowed=self.lsan_allowed,
maxNumRecordedFrames=self.lsan_max_stack_depth)
env = test_environment(xrePath=os.path.dirname(self.binary),
debugger=self.debug_info is not None,
log=self.logger,
lsanPath=self.prefs_root)
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.headless:
env["MOZ_HEADLESS"] = "1"
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"network.dns.localDomains":
",".join(self.config.domains_set),
"dom.file.createInChild":
True,
# TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
"network.proxy.type":
0,
"places.history.enabled":
False,
"network.preload":
True,
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences(
{"layout.interruptible-reflow.enabled": False})
if self.leak_check:
self.leak_report_file = os.path.join(
self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary, args, self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
class FirefoxBrowser(Browser):
init_timeout = 70
shutdown_timeout = 70
def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False, lsan_dir=None, stackfix_dir=None,
binary_args=None, timeout_multiplier=None, leak_check=False, asan=False,
stylo_threads=1, chaos_mode_flags=None, config=None, browser_channel="nightly", headless=None, **kwargs):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.test_type = test_type
self.extra_prefs = extra_prefs
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
self.certutil_binary = certutil_binary
self.e10s = e10s
self.binary_args = binary_args
self.config = config
if stackfix_dir:
self.stack_fixer = get_stack_fixer_function(stackfix_dir,
self.symbols_path)
else:
self.stack_fixer = None
if timeout_multiplier:
self.init_timeout = self.init_timeout * timeout_multiplier
self.asan = asan
self.lsan_dir = lsan_dir
self.lsan_allowed = None
self.lsan_max_stack_depth = None
self.mozleak_allowed = None
self.mozleak_thresholds = None
self.leak_check = leak_check
self.leak_report_file = None
self.lsan_handler = None
self.stylo_threads = stylo_threads
self.chaos_mode_flags = chaos_mode_flags
self.browser_channel = browser_channel
self.headless = headless
def settings(self, test):
return {"check_leaks": self.leak_check and not test.leaks,
"lsan_allowed": test.lsan_allowed,
"lsan_max_stack_depth": test.lsan_max_stack_depth,
"mozleak_allowed": self.leak_check and test.mozleak_allowed,
"mozleak_thresholds": self.leak_check and test.mozleak_threshold}
def start(self, group_metadata=None, **kwargs):
if group_metadata is None:
group_metadata = {}
self.group_metadata = group_metadata
self.lsan_allowed = kwargs.get("lsan_allowed")
self.lsan_max_stack_depth = kwargs.get("lsan_max_stack_depth")
self.mozleak_allowed = kwargs.get("mozleak_allowed")
self.mozleak_thresholds = kwargs.get("mozleak_thresholds")
if self.marionette_port is None:
self.marionette_port = get_free_port()
if self.asan:
self.lsan_handler = mozleak.LSANLeaks(self.logger,
scope=group_metadata.get("scope", "/"),
allowed=self.lsan_allowed,
maxNumRecordedFrames=self.lsan_max_stack_depth)
env = test_environment(xrePath=os.path.dirname(self.binary),
debugger=self.debug_info is not None,
log=self.logger,
lsanPath=self.lsan_dir)
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.headless:
env["MOZ_HEADLESS"] = "1"
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({
"marionette.port": self.marionette_port,
"network.dns.localDomains": ",".join(self.config.domains_set),
"dom.file.createInChild": True,
# TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
"network.proxy.type": 0,
"places.history.enabled": False,
"network.preload": True,
})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})
if self.leak_check:
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary,
args,
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs = Preferences()
pref_paths = []
profiles = os.path.join(self.prefs_root, 'profiles.json')
if os.path.isfile(profiles):
with open(profiles, 'r') as fh:
for name in json.load(fh)['web-platform-tests']:
if self.browser_channel in (None, 'nightly'):
pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
elif name != 'unittest-features':
pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
else:
# Old preference files used before the creation of profiles.json (remove when no longer supported)
legacy_pref_paths = (
os.path.join(self.prefs_root, 'prefs_general.js'), # Used in Firefox 60 and below
os.path.join(self.prefs_root, 'common', 'user.js'), # Used in Firefox 61
)
for path in legacy_pref_paths:
if os.path.isfile(path):
pref_paths.append(path)
for path in pref_paths:
if os.path.exists(path):
prefs.add(Preferences.read_prefs(path))
else:
self.logger.warning("Failed to find base prefs file in %s" % path)
# Add any custom preferences
prefs.add(self.extra_prefs, cast=True)
return prefs()
def stop(self, force=False):
if self.runner is not None and self.runner.is_running():
try:
# For Firefox we assume that stopping the runner prompts the
# browser to shut down. This allows the leak log to be written
for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)),
(False, lambda: self.runner.stop(signal.SIGTERM)),
(False, lambda: self.runner.stop(signal.SIGKILL))]:
if not force or not clean:
retcode = stop_f()
if retcode is not None:
self.logger.info("Browser exited with return code %s" % retcode)
break
except OSError:
# This can happen on Windows if the process is already dead
pass
self.process_leaks()
self.logger.debug("stopped")
def process_leaks(self):
self.logger.info("PROCESS LEAKS %s" % self.leak_report_file)
if self.lsan_handler:
self.lsan_handler.process()
if self.leak_report_file is not None:
mozleak.process_leak_log(
self.leak_report_file,
leak_thresholds=self.mozleak_thresholds,
ignore_missing_leaks=["gmplugin"],
log=self.logger,
stack_fixer=self.stack_fixer,
scope=self.group_metadata.get("scope"),
allowed=self.mozleak_allowed
)
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
if "GLib-GObject-CRITICAL" in line:
return
if line:
data = line.decode("utf8", "replace")
if self.stack_fixer:
data = self.stack_fixer(data)
if self.lsan_handler:
data = self.lsan_handler.log(data)
if data is not None:
self.logger.process_output(self.pid(),
data,
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self, force=False):
self.stop(force)
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def check_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
return bool(mozcrash.log_crashes(self.logger,
dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process,
test=test))
def setup_ssl(self):
"""Create a certificate database to use in the test profile. This is configured
to trust the CA Certificate that has signed the web-platform.test server
certificate."""
if self.certutil_binary is None:
self.logger.info("--certutil-binary not supplied; Firefox will not check certificates")
return
self.logger.info("Setting up ssl")
# Make sure the certutil libraries from the source tree are loaded when using a
# local copy of certutil
# TODO: Maybe only set this if certutil won't launch?
env = os.environ.copy()
certutil_dir = os.path.dirname(self.binary or self.certutil_binary)
if mozinfo.isMac:
env_var = "DYLD_LIBRARY_PATH"
elif mozinfo.isUnix:
env_var = "LD_LIBRARY_PATH"
else:
env_var = "PATH"
env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
if env_var in env else certutil_dir).encode(
sys.getfilesystemencoding() or 'utf-8', 'replace')
def certutil(*args):
cmd = [self.certutil_binary] + list(args)
self.logger.process_output("certutil",
subprocess.check_output(cmd,
env=env,
stderr=subprocess.STDOUT),
" ".join(cmd))
pw_path = os.path.join(self.profile.profile, ".crtdbpw")
with open(pw_path, "w") as f:
# Use empty password for certificate db
f.write("\n")
cert_db_path = self.profile.profile
# Create a new certificate db
certutil("-N", "-d", cert_db_path, "-f", pw_path)
# Add the CA certificate to the database and mark as trusted to issue server certs
certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
"-n", "web-platform-tests", "-i", self.ca_certificate_path)
# List all certs in the database
certutil("-L", "-d", cert_db_path)
def run():
parser = argparse.ArgumentParser(description='Run crawler')
parser.add_argument(
'-b', '--binary', type=str,
help='path to the Firefox binary'
)
parser.add_argument(
'-a', '--abpdir', type=str,
help='path to the Adblock Plus repository'
)
parser.add_argument(
'-f', '--filters', metavar='url', type=str, nargs='+',
default=["https://easylist-downloads.adblockplus.org/easylist.txt", "https://easylist-downloads.adblockplus.org/exceptionrules.txt"],
help='filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.'
)
parser.add_argument(
'-t', '--timeout', type=int, default=300,
help='Load timeout (seconds)'
)
parser.add_argument(
'-x', '--maxtabs', type=int, default=15,
help='Maximal number of tabs to open in parallel'
)
parser.add_argument(
'list', type=str,
help='URL list to process'
)
parser.add_argument(
'outdir', type=str,
help='directory to write data into'
)
parameters = parser.parse_args()
import buildtools.packagerGecko as packager
cleanup = []
try:
base_dir = os.path.dirname(__file__)
handle, crawlerxpi = tempfile.mkstemp(suffix='.xpi')
os.close(handle)
cleanup.append(crawlerxpi)
packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)
abpxpi = 'https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi'
if parameters.abpdir:
handle, abpxpi = tempfile.mkstemp(suffix='.xpi')
os.close(handle)
cleanup.append(abpxpi)
packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True)
profile = FirefoxProfile(
addons=[
crawlerxpi,
abpxpi,
],
preferences={
'browser.uitour.enabled': False,
'prompts.tab_modal.enabled': False,
}
)
abpsettings = os.path.join(profile.profile, 'adblockplus')
os.makedirs(abpsettings)
with open(os.path.join(abpsettings, 'patterns.ini'), 'w') as handle:
print >>handle, '# Adblock Plus preferences'
print >>handle, 'version=4'
for url in parameters.filters:
if '=' in url:
path, url = url.split('=', 1)
with open(path, 'r') as source:
data = source.read()
else:
data = urllib.urlopen(url).read()
print >>handle, '[Subscription]'
print >>handle, 'url=%s' % url
print >>handle, '[Subscription filters]'
print >>handle, '\n'.join(data.splitlines()[1:])
finally:
for path in cleanup:
os.unlink(path)
server = None
try:
port = random.randrange(2000, 60000)
print "Communicating with client on port %i" % port
app = CrawlerApp(parameters)
server = make_server('localhost', port, app)
app.server = server
threading.Thread(target=lambda: server.serve_forever()).start()
runner = FirefoxRunner(
profile=profile,
binary=parameters.binary,
cmdargs=['--crawler-port', str(port)],
env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE='1'),
)
while app.urls:
runner.start()
runner.wait()
finally:
if server:
server.shutdown()
profile.cleanup()
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"dom.disable_open_during_load":
False,
"network.dns.localDomains":
",".join(hostnames),
"network.proxy.type":
0,
"places.history.enabled":
False,
"dom.send_after_paint_to_content":
True,
"network.preload":
True
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences(
{"layout.interruptible-reflow.enabled": False})
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile,
"runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.binary, self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"], self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60
def __init__(self, logger, binary, prefs_root, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
self.certutil_binary = certutil_binary
self.e10s = e10s
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft") and
'5.1' in platform.version()):
self.profile.set_preferences({"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
if os.path.exists(prefs_path):
preferences = Preferences.read_prefs(prefs_path)
else:
self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
preferences = []
return preferences
def stop(self):
self.logger.debug("Stopping browser")
if self.runner is not None:
try:
self.runner.stop()
except OSError:
# This can happen on Windows if the process is already dead
pass
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
self.logger.process_output(self.pid(),
line.decode("utf8", "replace"),
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
mozcrash.log_crashes(self.logger,
dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process,
test=test)
def setup_ssl(self):
"""Create a certificate database to use in the test profile. This is configured
to trust the CA Certificate that has signed the web-platform.test server
certificate."""
self.logger.info("Setting up ssl")
# Make sure the certutil libraries from the source tree are loaded when using a
# local copy of certutil
# TODO: Maybe only set this if certutil won't launch?
env = os.environ.copy()
certutil_dir = os.path.dirname(self.binary)
if mozinfo.isMac:
env_var = "DYLD_LIBRARY_PATH"
elif mozinfo.isUnix:
env_var = "LD_LIBRARY_PATH"
else:
env_var = "PATH"
env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
if env_var in env else certutil_dir).encode(
sys.getfilesystemencoding() or 'utf-8', 'replace')
def certutil(*args):
cmd = [self.certutil_binary] + list(args)
self.logger.process_output("certutil",
subprocess.check_output(cmd,
env=env,
stderr=subprocess.STDOUT),
" ".join(cmd))
pw_path = os.path.join(self.profile.profile, ".crtdbpw")
with open(pw_path, "w") as f:
# Use empty password for certificate db
f.write("\n")
cert_db_path = self.profile.profile
# Create a new certificate db
certutil("-N", "-d", cert_db_path, "-f", pw_path)
# Add the CA certificate to the database and mark as trusted to issue server certs
certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
"-n", "web-platform-tests", "-i", self.ca_certificate_path)
# List all certs in the database
certutil("-L", "-d", cert_db_path)
class FirefoxThread(Thread):
def __init__(self, binary, marionette_port = 2828):
Thread.__init__(self)
self.binary = binary
self.marionette_port = marionette_port
self.logger = FirefoxThreadLogger(None)
self._firefoxRunningEvent = Event()
def run(self):
'''
Starts Firefox thread with Marionette turned on.
'''
self.profile = FirefoxProfile()
self.profile.set_preferences({"marionette.defaultPrefs.enabled" : True,
"marionette.defaultPrefs.port": 2828,
"browser.startup.page": 0,
"browser.startup.homepage": "about:blank",
})
self.runner = FirefoxRunner(profile = self.profile,
binary = self.binary,
kp_kwargs = {'processOutputLine' : [self.logger]})
self.runner.start()
self._firefoxRunningEvent.set()
self.runner.wait()
def stop(self):
'''
Stops Firefox/Nightly. To be called by external thread.
'''
self.runner.stop()
def getPID(self):
'''
This is called by external threads, and blocks until PID is available in FirefoxRunner,
which is shortly after start() has been called.
'''
self._firefoxRunningEvent.wait()
return self.runner.process_handler.proc.pid
def waitForMarionettePortOpenReady(self, timeout):
'''
This method can be run by an external thread. Returns True when the port is open, or False on timeout.
It's active waiting with 1 sec heartbeat, if you know better solution please mail me.
Originally taken from:
https://github.com/mozilla/marionette_client/blob/master/marionette/emulator.py#L246
'''
starttime = datetime.datetime.now()
while datetime.datetime.now() - starttime < datetime.timedelta(seconds=timeout):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(('127.0.0.1', self.marionette_port))
data = sock.recv(16)
sock.close()
if '"from"' in data:
return True
except:
#import traceback
#print traceback.format_exc()
pass
time.sleep(1)
print ''
return False
class FennecBrowser(FirefoxBrowser):
used_ports = set()
init_timeout = 300
shutdown_timeout = 60
def __init__(self, logger, prefs_root, test_type, package_name=None,
device_serial="emulator-5444", **kwargs):
FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
self._package_name = package_name
self.device_serial = device_serial
self.tests_root = kwargs["tests_root"]
self.install_fonts = kwargs["install_fonts"]
@property
def package_name(self):
"""
Name of app to run on emulator.
"""
if self._package_name is None:
self._package_name = "org.mozilla.fennec"
user = os.getenv("USER")
if user:
self._package_name += "_" + user
return self._package_name
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = {}
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({"marionette.port": self.marionette_port,
"dom.disable_open_during_load": False,
"places.history.enabled": False,
"dom.send_after_paint_to_content": True,
"network.preload": True})
if self.test_type == "reftest":
self.logger.info("Setting android reftest preferences")
self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
# Disable high DPI
"layout.css.devPixelsPerPx": "1.0",
# Ensure that the full browser element
# appears in the screenshot
"apz.allow_zooming": False,
"android.widget_paints_background": False,
# Ensure that scrollbars are always painted
"ui.scrollbarFadeBeginDelay": 100000})
if self.install_fonts:
self.logger.debug("Copying Ahem font to profile")
font_dir = os.path.join(self.profile.profile, "fonts")
if not os.path.exists(font_dir):
os.makedirs(font_dir)
with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
dest.write(src.read())
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.package_name,
self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FennecEmulatorRunner(app=self.package_name,
profile=self.profile,
cmdargs=cmd[1:],
env=env,
symbols_path=self.symbols_path,
serial=self.device_serial,
# TODO - choose appropriate log dir
logdir=os.getcwd(),
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting %s" % self.package_name)
# connect to a running emulator
self.runner.device.connect()
write_hosts_file(self.config, self.runner.device.device)
self.runner.stop()
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.runner.device.device.forward(
local="tcp:{}".format(self.marionette_port),
remote="tcp:{}".format(self.marionette_port))
self.logger.debug("%s Started" % self.package_name)
def stop(self, force=False):
if self.runner is not None:
if (self.runner.device.connected and
len(self.runner.device.device.list_forwards()) > 0):
try:
self.runner.device.device.remove_forwards(
"tcp:{}".format(self.marionette_port))
except Exception:
self.logger.warning("Failed to remove port forwarding")
# We assume that stopping the runner prompts the
# browser to shut down. This allows the leak log to be written
self.runner.stop()
self.logger.debug("stopped")