def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames)})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def run(self, test, phase='phase1', ignore_unused_engines=True):
self.profile.set_preferences({
'testing.tps.testFile':
os.path.abspath(test),
'testing.tps.testPhase':
phase,
'testing.tps.ignoreUnusedEngines':
ignore_unused_engines,
})
args = ['-marionette']
process_args = {'processOutputLine': [self._log]}
self.logger.info('Running: {} {}'.format(self.firefox, ' '.join(args)))
self.logger.info('Using profile at: {}'.format(self.profile.profile))
runner = FirefoxRunner(binary=self.firefox,
cmdargs=args,
profile=self.profile,
process_args=process_args)
runner.start(timeout=TIMEOUT)
runner.wait(timeout=TIMEOUT)
self.firefox_log.close()
with open(self.tps_log) as f:
for line in f.readlines():
if 'CROSSWEAVE ERROR: ' in line:
raise TPSError(line.partition('CROSSWEAVE ERROR: ')[-1])
with open(self.tps_log) as f:
assert 'test phase {}: PASS'.format(phase) in f.read()
def launch(self, args=None):
"""Launch the app with optional args for profile, windows, URI, etc.
:param url: URL to be loaded.
:param args: Optional list of arguments.
:return: List of Firefox flags.
"""
if args is None:
args = []
args.append('-foreground')
args.append('-no-remote')
args.append('-new-tab')
args.append(self.url)
process_args = {'stream': None}
logger.debug('Creating Firefox runner ...')
try:
runner = FirefoxRunner(binary=self.application.path,
profile=self.profile,
cmdargs=args,
process_args=process_args)
logger.debug('Firefox runner successfully created.')
logger.debug('Running Firefox with command: "%s"' %
','.join(runner.command))
except run_errors.RunnerNotStartedError:
raise APIHelperError('Error creating Firefox runner.')
else:
return runner
def launch_control_center():
profile_path = os.path.join(get_core_args().workdir, 'cc_profile')
fx_path = PathManager.get_local_firefox_path()
if fx_path is None:
logger.error(
'Can\'t find local Firefox installation, aborting Iris run.')
return False, None
args = ['http://127.0.0.1:%s' % get_core_args().port]
process_args = {'stream': None}
profile = MozProfile(profile=profile_path,
preferences=Settings.default_fx_prefs)
fx_runner = FirefoxRunner(binary=fx_path,
profile=profile,
cmdargs=args,
process_args=process_args)
fx_runner.start()
server = LocalWebServer(get_core_args().workdir, get_core_args().port)
server.stop()
time.sleep(Settings.DEFAULT_UI_DELAY)
if OSHelper.is_mac():
type(text='q', modifier=KeyModifier.CMD)
elif OSHelper.is_windows():
type(text='w', modifier=[KeyModifier.CTRL, KeyModifier.SHIFT])
else:
type(text='q', modifier=KeyModifier.CTRL)
try:
fx_runner.stop()
except Exception as e:
logger.debug('Error stopping fx_runner')
logger.debug(e)
return server.result
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
proxy=True,
preferences=preferences)
self.profile.set_preferences({
"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False
})
self.runner = FirefoxRunner(
profile=self.profile,
binary=self.binary,
cmdargs=[cmd_arg("marionette"), "about:blank"],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=self.debug_args,
interactive=self.interactive)
self.logger.debug("Firefox Started")
def launch_firefox(path, profile=None, url=None, args=None):
"""Launch the app with optional args for profile, windows, URI, etc.
:param path: Firefox path.
:param profile: Firefox profile.
:param url: URL to be loaded.
:param args: Optional list of arguments.
:return: List of Firefox flags.
"""
if args is None:
args = []
if profile is None:
raise APIHelperError('No profile name present, aborting run.')
profile = FirefoxProfile(profile=profile)
args.append('-foreground')
args.append('-no-remote')
if url is not None:
args.append('-new-tab')
args.append(url)
process_args = {'stream': None}
logger.debug('Creating Firefox runner ...')
try:
runner = FirefoxRunner(binary=path, profile=profile, cmdargs=args, process_args=process_args)
logger.debug('Firefox runner successfully created.')
logger.debug('Running Firefox with command: "%s"' % ','.join(runner.command))
return runner
except errors.RunnerNotStartedError:
raise APIHelperError('Error creating Firefox runner.')
def run(self, test, phase='phase1', ignore_unused_engines=True):
args = [
'-tps={}'.format(os.path.abspath(test)),
'-tpsphase={}'.format(phase), '-marionette'
]
if ignore_unused_engines:
args.append('--ignore-unused-engines')
process_args = {'processOutputLine': [self._log]}
self.logger.info('Running: {} {}'.format(self.firefox, ' '.join(args)))
self.logger.info('Using profile at: {}'.format(self.profile.profile))
runner = FirefoxRunner(binary=self.firefox,
cmdargs=args,
profile=self.profile,
process_args=process_args)
runner.start(timeout=TIMEOUT)
runner.wait(timeout=TIMEOUT)
self.firefox_log.close()
with open(self.tps_log) as f:
for line in f.readlines():
if 'CROSSWEAVE ERROR: ' in line:
raise TPSError(line.partition('CROSSWEAVE ERROR: ')[-1])
with open(self.tps_log) as f:
assert 'test phase {}: PASS'.format(phase) in f.read()
def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None):
"""Runs the given FirefoxRunner with the given Profile, waits
for completion, then returns the process exit code
"""
if profile is None:
profile = Profile()
self.profile = profile
if self.binary is None and self.url:
self.binary = self.download_build()
if self.runner is None:
self.runner = FirefoxRunner(self.profile, binary=self.binary)
self.runner.profile = self.profile
if env is not None:
self.runner.env.update(env)
if args is not None:
self.runner.cmdargs = copy.copy(args)
self.runner.start()
returncode = self.runner.wait(timeout)
return returncode
def runprofile(binary, fileobj):
try:
runner = FirefoxRunner(binary=binary, profile=fileobj.profile)
runner.start()
runner.wait()
except KeyboardInterrupt:
pass
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({
"marionette.enabled":
True,
"marionette.port":
self.marionette_port,
"dom.disable_open_during_load":
False,
"network.dns.localDomains":
",".join(hostnames),
"network.proxy.type":
0,
"places.history.enabled":
False
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.binary, [cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def launch_control_center():
profile_path = os.path.join(PathManager.get_working_dir(), "cc_profile")
fx_path = PathManager.get_local_firefox_path()
if fx_path is None:
logger.error("Can't find local Firefox installation, aborting Iris run.")
return False, None
args = ["http://127.0.0.1:%s" % get_core_args().port]
process_args = {"stream": None}
profile = MozProfile(profile=profile_path, preferences=get_fx_prefs())
if OSHelper.is_windows():
process = subprocess.Popen(
[
fx_path,
"-no-remote",
"-new-tab",
args,
"--wait-for-browser",
"-foreground",
"-profile",
profile.profile,
],
shell=False,
)
else:
fx_runner = FirefoxRunner(
binary=fx_path, profile=profile, cmdargs=args, process_args=process_args
)
fx_runner.start()
logger.debug("Launching web server for directory %s" % PathManager.get_working_dir())
server = LocalWebServer(PathManager.get_working_dir(), get_core_args().port)
server.stop()
time.sleep(Settings.DEFAULT_UI_DELAY)
if OSHelper.is_mac():
type(text="q", modifier=KeyModifier.CMD)
elif OSHelper.is_windows():
type(text="w", modifier=[KeyModifier.CTRL, KeyModifier.SHIFT])
else:
type(text="q", modifier=KeyModifier.CTRL)
if OSHelper.is_windows():
if process.pid is not None:
try:
logger.debug("Closing Firefox process ID: %s" % process.pid)
process = psutil.Process(process.pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
except psutil.NoSuchProcess:
pass
else:
try:
fx_runner.stop()
except Exception as e:
logger.debug("Error stopping fx_runner")
logger.debug(e)
return server.result
def start(self):
"""Start an instance of Firefox, returning a BrowserInstance handle"""
profile = self.base_profile.clone(self.base_profile.profile)
marionette_port = get_free_port()
profile.set_preferences({"marionette.port": marionette_port})
env = test_environment(xrePath=os.path.dirname(self.binary),
debugger=self.debug_info is not None,
useLSan=True,
log=self.logger)
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.headless:
env["MOZ_HEADLESS"] = "1"
if self.enable_webrender:
env["MOZ_WEBRENDER"] = "1"
env["MOZ_ACCELERATED"] = "1"
# Set MOZ_X_SYNC and GDK_SYNCHRONIZE for investigation; bug 1625250.
env["MOZ_X_SYNC"] = "1"
env["GDK_SYNCHRONIZE"] = "1"
else:
env["MOZ_WEBRENDER"] = "0"
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary, args, self.debug_info)
if self.leak_check:
leak_report_file = os.path.join(
profile.profile, "runtests_leaks_%s.log" % os.getpid())
if os.path.exists(leak_report_file):
os.remove(leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = leak_report_file
else:
leak_report_file = None
output_handler = OutputHandler(self.logger, self.stackfix_dir,
self.symbols_path, self.asan)
runner = FirefoxRunner(
profile=profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=cast_env(env),
process_class=ProcessHandler,
process_args={"processOutputLine": [output_handler]})
instance = BrowserInstance(self.logger, runner, marionette_port,
output_handler, leak_report_file)
self.logger.debug("Starting Firefox")
runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
return instance
def run_tests(self, test_path, options):
reftestlist = self.getManifestPath(test_path)
if not reftestlist.startswith('file://'):
reftestlist = 'file://%s' % reftestlist
self.profile = self.create_profile(options,
reftestlist,
profile_to_clone=options.profile)
env = self.buildBrowserEnv(options, self.profile.profile)
kp_kwargs = {
'processOutputLine': [self._on_output],
'onTimeout': [self._on_timeout],
'kill_on_timeout': False
}
if not options.debugger:
if not options.timeout:
if mozinfo.info['debug']:
options.timeout = 420
else:
options.timeout = 300
self.timeout = options.timeout + 30.0
log.info("%s | Running tests: start.", os.path.basename(__file__))
cmd, args = self.build_command_line(
options.app,
ignore_window_size=options.ignoreWindowSize,
browser_arg=options.browser_arg)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd,
cmdargs=args,
env=env,
process_class=ProcessHandler,
process_args=kp_kwargs,
symbols_path=options.symbolsPath)
status = 0
try:
self.runner.start(outputTimeout=self.timeout)
log.info("%s | Application pid: %d", os.path.basename(__file__),
self.runner.process_handler.pid)
# kick starts the reftest harness
self.run_marionette_script()
status = self.runner.wait()
finally:
self.runner.check_for_crashes(test_name=self.last_test)
self.runner.cleanup()
if status > 0:
log.testFail("%s | application terminated with exit code %s",
self.last_test, status)
elif status < 0:
log.info("%s | application killed with signal %s", self.last_test,
-status)
log.info("%s | Running tests: end.", os.path.basename(__file__))
return status
def launch_control_center():
profile_path = os.path.join(get_core_args().workdir, 'cc_profile')
fx_path = PathManager.get_local_firefox_path()
if fx_path is None:
logger.error(
'Can\'t find local Firefox installation, aborting Iris run.')
return False, None
args = ['http://127.0.0.1:%s' % get_core_args().port]
process_args = {'stream': None}
profile = MozProfile(profile=profile_path,
preferences=Settings.default_fx_prefs)
if OSHelper.is_windows():
process = subprocess.Popen([
fx_path, '-no-remote', '-new-tab', args, '--wait-for-browser',
'-foreground', '-profile', profile.profile
],
shell=False)
else:
fx_runner = FirefoxRunner(binary=fx_path,
profile=profile,
cmdargs=args,
process_args=process_args)
fx_runner.start()
server = LocalWebServer(get_core_args().workdir, get_core_args().port)
server.stop()
time.sleep(Settings.DEFAULT_UI_DELAY)
if OSHelper.is_mac():
type(text='q', modifier=KeyModifier.CMD)
elif OSHelper.is_windows():
type(text='w', modifier=[KeyModifier.CTRL, KeyModifier.SHIFT])
else:
type(text='q', modifier=KeyModifier.CTRL)
if OSHelper.is_windows():
if process.pid is not None:
try:
logger.debug('Closing Firefox process ID: %s' % process.pid)
process = psutil.Process(process.pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
except psutil.NoSuchProcess:
pass
else:
try:
fx_runner.stop()
except Exception as e:
logger.debug('Error stopping fx_runner')
logger.debug(e)
return server.result
def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None):
"""Runs the given FirefoxRunner with the given Profile, waits
for completion, then returns the process exit code
"""
if profile is None:
profile = Profile()
self.profile = profile
if self.binary is None and self.url:
self.binary = self.download_build()
runner = FirefoxRunner(profile=self.profile,
binary=self.binary,
env=env,
cmdargs=args)
runner.start(timeout=timeout)
return runner.wait()
def start(self):
"""Start an instance of Firefox, returning a BrowserInstance handle"""
profile = self.base_profile.clone(self.base_profile.profile)
marionette_port = get_free_port()
profile.set_preferences({"marionette.port": marionette_port})
env = get_environ(self.logger, self.binary, self.debug_info,
self.stylo_threads, self.headless,
self.enable_webrender, self.chaos_mode_flags)
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary, args, self.debug_info)
if self.leak_check:
leak_report_file = os.path.join(
profile.profile, "runtests_leaks_%s.log" % os.getpid())
if os.path.exists(leak_report_file):
os.remove(leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = leak_report_file
else:
leak_report_file = None
output_handler = OutputHandler(self.logger, self.stackfix_dir,
self.symbols_path, self.asan)
runner = FirefoxRunner(
profile=profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [output_handler]})
instance = BrowserInstance(self.logger, runner, marionette_port,
output_handler, leak_report_file)
self.logger.debug("Starting Firefox")
runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
return instance
def start_firefox(self):
env = os.environ.copy()
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
profile = Profile()
profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
"dom.max_script_run_time": 0})
self.firefox_runner = FirefoxRunner(profile,
self.firefox_binary,
cmdargs=["--marionette"],
env=env,
kp_kwargs = {"processOutputLine":[self.on_output]},
process_class=self.process_cls)
self.logger.debug("Starting Firefox")
self.firefox_runner.start()
self.logger.debug("Firefox Started")
def start(self):
"""Start an instance of Firefox, returning a BrowserInstance handle"""
profile = self.base_profile.clone(self.base_profile.profile)
marionette_port = get_free_port()
profile.set_preferences({"marionette.port": marionette_port})
env = get_environ(self.logger, self.binary, self.debug_info,
self.stylo_threads, self.headless,
self.chaos_mode_flags)
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary, args, self.debug_info)
leak_report_file = setup_leak_report(self.leak_check, profile, env)
output_handler = FirefoxOutputHandler(
self.logger,
cmd,
stackfix_dir=self.stackfix_dir,
symbols_path=self.symbols_path,
asan=self.asan,
leak_report_file=leak_report_file)
runner = FirefoxRunner(
profile=profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [output_handler]})
instance = BrowserInstance(self.logger, runner, marionette_port,
output_handler, leak_report_file)
self.logger.debug("Starting Firefox")
runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
output_handler.after_process_start(runner.process_handler.pid)
self.logger.debug("Firefox Started")
return instance
env["XPCOM_DEBUG_BREAK"] = "warn"
# For VC12+, make sure we can find the right bitness of pgort1x0.dll
if not substs.get('HAVE_64BIT_BUILD'):
for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
if e not in env:
continue
vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
if os.path.exists(vcdir):
env['PATH'] = '%s;%s' % (vcdir, env['PATH'])
break
# Run Firefox a first time to initialize its profile
runner = FirefoxRunner(profile=profile,
binary=binary,
cmdargs=['data:text/html,<script>Quitter.quit()</script>'],
env=env)
runner.start()
ret = runner.wait()
if ret:
print("Firefox exited with code %d during profile initialization"
% ret)
httpd.stop()
sys.exit(ret)
jarlog = os.getenv("JARLOG_FILE")
if jarlog:
env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"])
cmdargs = ["http://localhost:%d/index.html" % PORT]
def start(self, **kwargs):
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_CRASHREPORTER"] = "1"
env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
locations = ServerLocations(
filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"dom.disable_open_during_load":
False,
"network.dns.localDomains":
",".join(hostnames),
"network.proxy.type":
0,
"places.history.enabled":
False,
"dom.send_after_paint_to_content":
True,
"network.preload":
True
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences(
{"layout.interruptible-reflow.enabled": False})
if self.leak_check and kwargs.get("check_leaks", True):
self.leak_report_file = os.path.join(self.profile.profile,
"runtests_leaks.log")
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(
self.binary, self.binary_args if self.binary_args else [] +
[cmd_arg("marionette"), "about:blank"], self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, "build")
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo"))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(self.topsrcdir, "testing",
"profiles")
with open(os.path.join(profile_data_dir, "profiles.json"),
"r") as fh:
base_profiles = json.load(fh)["valgrind"]
prefpaths = [
os.path.join(profile_data_dir, profile, "user.js")
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
"server": "%s:%d" % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(self.topsrcdir, "tools", "quitter",
"*****@*****.**")
locations = ServerLocations()
locations.add_host(host="127.0.0.1",
port=httpd.httpd.server_port,
options="primary")
profile = FirefoxProfile(
profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations,
)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env["G_SLICE"] = "always-malloc"
env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1"
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
outputHandler = OutputHandler(self.log)
kp_kwargs = {
"processOutputLine": [outputHandler],
"universal_newlines": True,
}
valgrind = "valgrind"
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
"--sym-offsets=yes",
"--smc-check=all-non-file",
"--vex-iropt-register-updates=allregs-at-mem-access",
"--gen-suppressions=all",
"--num-callers=36",
"--leak-check=full",
"--show-possibly-lost=no",
"--track-origins=yes",
"--trace-children=yes",
"-v", # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
"--fair-sched=yes",
# Keep debuginfo after library unmap. See bug 1382280.
"--keep-debuginfo=yes",
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
"--expensive-definedness-checks=yes",
# Compensate for the compiler inlining `new` but not `delete`
# or vice versa.
"--show-mismatched-frees=no",
]
for s in suppressions:
valgrind_args.append("--suppressions=" + s)
supps_dir = os.path.join(build_dir, "valgrind")
supps_file1 = os.path.join(supps_dir, "cross-architecture.sup")
valgrind_args.append("--suppressions=" + supps_file1)
if mozinfo.os == "linux":
machtype = {
"x86_64": "x86_64-pc-linux-gnu",
"x86": "i386-pc-linux-gnu",
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + ".sup")
if os.path.isfile(supps_file2):
valgrind_args.append("--suppressions=" + supps_file2)
exitcode = None
timeout = 1800
binary_not_found_exception = None
try:
runner = FirefoxRunner(
profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs,
)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
except BinaryNotFoundException as e:
binary_not_found_exception = e
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR,
"valgrind-fail-parsing",
{
"errs": errs,
"supps": supps
},
"TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors "
"seen, but {supps} generated suppressions seen",
)
elif errs == 0:
status = 0
self.log(
logging.INFO,
"valgrind-pass",
{},
"TEST-PASS | valgrind-test | valgrind found no errors",
)
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if binary_not_found_exception:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{"error": str(binary_not_found_exception)},
"TEST-UNEXPECTED-FAIL | valgrind-test | {error}",
)
self.log(
logging.INFO,
"valgrind-fail-errors",
{"help": binary_not_found_exception.help()},
"{help}",
)
elif exitcode is None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-timeout",
{"timeout": timeout},
"TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out "
"(reached {timeout} second limit)",
)
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR,
"valgrind-fail-errors",
{"exitcode": exitcode},
"TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code "
"from Valgrind: {exitcode}",
)
httpd.stop()
return status
# For VC12+, make sure we can find the right bitness of pgort1x0.dll
if not substs.get('HAVE_64BIT_BUILD'):
for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
if e not in env:
continue
vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
if os.path.exists(vcdir):
env['PATH'] = '%s;%s' % (vcdir, env['PATH'])
break
# Run Firefox a first time to initialize its profile
runner = FirefoxRunner(profile=profile,
binary=build.get_binary_path(
where="staged-package"),
cmdargs=['javascript:Quitter.quit()'],
env=env)
runner.start()
runner.wait()
jarlog = os.getenv("JARLOG_FILE")
if jarlog:
env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]
cmdargs = ["http://localhost:%d/index.html" % PORT]
runner = FirefoxRunner(profile=profile,
binary=build.get_binary_path(
where="staged-package"),
cmdargs=cmdargs,
def valgrind_test(self, suppressions):
import json
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
#TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js')
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = { 'server': '%s:%d' % httpd.httpd.server_address,
'OOP': 'false'}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', '*****@*****.**')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
'-v', # Enable verbosity to get the list of used suppressions
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(logging.ERROR, 'valgrind-fail-parsing',
{'errs': errs, 'supps': supps},
'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen')
elif errs == 0:
status = 0
self.log(logging.INFO, 'valgrind-pass', {},
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode == None:
status = 2 # turns the TBPL job red
self.log(logging.ERROR, 'valgrind-fail-timeout',
{'timeout': timeout},
'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)')
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(logging.ERROR, 'valgrind-fail-errors', {},
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind')
httpd.stop()
return status
"OOP": "false"
}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
profile = FirefoxProfile(
profile=profilePath,
preferences=prefs,
addons=[os.path.join(build.distdir, 'xpi-stage', 'quitter')],
locations=locations)
env = os.environ.copy()
env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
env["XPCOM_DEBUG_BREAK"] = "warn"
jarlog = os.getenv("JARLOG_FILE")
if jarlog:
env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]
cmdargs = ["http://localhost:%d/index.html" % PORT]
runner = FirefoxRunner(
profile=profile,
binary=build.get_binary_path(where="staged-package"),
cmdargs=cmdargs,
env=env)
runner.start(debug_args=debug_args, interactive=interactive)
runner.wait()
httpd.stop()
finally:
shutil.rmtree(profilePath)
def start(self, group_metadata=None, **kwargs):
if group_metadata is None:
group_metadata = {}
if self.marionette_port is None:
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
if self.asan:
print "Setting up LSAN"
self.lsan_handler = mozleak.LSANLeaks(
self.logger,
scope=group_metadata.get("scope", "/"),
allowed=self.lsan_allowed,
maxNumRecordedFrames=self.lsan_max_stack_depth)
env = test_environment(xrePath=os.path.dirname(self.binary),
debugger=self.debug_info is not None,
log=self.logger,
lsanPath=self.prefs_root)
env["STYLO_THREADS"] = str(self.stylo_threads)
if self.chaos_mode_flags is not None:
env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
if self.headless:
env["MOZ_HEADLESS"] = "1"
preferences = self.load_prefs()
self.profile = FirefoxProfile(preferences=preferences)
self.profile.set_preferences({
"marionette.port":
self.marionette_port,
"network.dns.localDomains":
",".join(self.config.domains_set),
"dom.file.createInChild":
True,
# TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
"network.proxy.type":
0,
"places.history.enabled":
False,
"network.preload":
True,
})
if self.e10s:
self.profile.set_preferences(
{"browser.tabs.remote.autostart": True})
if self.test_type == "reftest":
self.profile.set_preferences(
{"layout.interruptible-reflow.enabled": False})
if self.leak_check:
self.leak_report_file = os.path.join(
self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
if os.path.exists(self.leak_report_file):
os.remove(self.leak_report_file)
env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
else:
self.leak_report_file = None
# Bug 1262954: winxp + e10s, disable hwaccel
if (self.e10s and platform.system() in ("Windows", "Microsoft")
and '5.1' in platform.version()):
self.profile.set_preferences(
{"layers.acceleration.disabled": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
args = self.binary_args[:] if self.binary_args else []
args += [cmd_arg("marionette"), "about:blank"]
debug_args, cmd = browser_command(self.binary, args, self.debug_info)
self.runner = FirefoxRunner(
profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args,
interactive=self.debug_info
and self.debug_info.interactive)
self.logger.debug("Firefox Started")
env["MOZ_DISABLE_VR_SANDBOX"] = "1"
# Ensure different pids write to different files
env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw"
# Write to an output file if we're running in automation
process_args = {"universal_newlines": True}
if "UPLOAD_PATH" in env:
process_args["logfile"] = os.path.join(env["UPLOAD_PATH"],
"profile-run-1.log")
# Run Firefox a first time to initialize its profile
runner = FirefoxRunner(
profile=profile,
binary=binary,
cmdargs=["data:text/html,<script>Quitter.quit()</script>"],
env=env,
process_args=process_args,
)
runner.start()
ret = runner.wait()
if ret:
print("Firefox exited with code %d during profile initialization" %
ret)
logfile = process_args.get("logfile")
if logfile:
print("Firefox output (%s):" % logfile)
with open(logfile) as f:
print(f.read())
httpd.stop()
get_crashreports(profilePath, name="Profile initialization")
def valgrind_test(self, suppressions):
import json
import re
import sys
import tempfile
from mozbuild.base import MozbuildObject
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
#TODO: refactor this into mozprofile
prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles',
'prefs_general.js')
prefs = {}
prefs.update(Preferences.read_prefs(prefpath))
interpolation = {
'server': '%s:%d' % httpd.httpd.server_address,
'OOP': 'false'
}
prefs = json.loads(json.dumps(prefs) % interpolation)
for pref in prefs:
prefs[pref] = Preferences.cast(prefs[pref])
quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
class OutputHandler(object):
def __init__(self):
self.found_errors = False
def __call__(self, line):
print(line)
m = re.match(
r'.*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts',
line)
if m:
self.found_errors = True
outputHandler = OutputHandler()
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind, '--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all', '--num-callers=20',
'--leak-check=full', '--show-possibly-lost=no',
'--track-origins=yes'
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
# MACHTYPE is an odd bash-only environment variable that doesn't
# show up in os.environ, so we have to get it another way.
machtype = subprocess.check_output(
['bash', '-c', 'echo $MACHTYPE']).rstrip()
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
kp_kwargs=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait()
finally:
if not outputHandler.found_errors:
status = 0
print(
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
print(
'TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors'
)
if exitcode != 0:
status = 2 # turns the TBPL job red
print(
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind'
)
httpd.stop()
return status
def valgrind_test(self, suppressions):
from mozfile import TemporaryDirectory
from mozhttpd import MozHttpd
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import findInPath
from six import string_types
from valgrind.output_handler import OutputHandler
build_dir = os.path.join(self.topsrcdir, 'build')
# XXX: currently we just use the PGO inputs for Valgrind runs. This may
# change in the future.
httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
httpd.start(block=False)
with TemporaryDirectory() as profilePath:
# TODO: refactor this into mozprofile
profile_data_dir = os.path.join(self.topsrcdir, 'testing',
'profiles')
with open(os.path.join(profile_data_dir, 'profiles.json'),
'r') as fh:
base_profiles = json.load(fh)['valgrind']
prefpaths = [
os.path.join(profile_data_dir, profile, 'user.js')
for profile in base_profiles
]
prefs = {}
for path in prefpaths:
prefs.update(Preferences.read_prefs(path))
interpolation = {
'server': '%s:%d' % httpd.httpd.server_address,
}
for k, v in prefs.items():
if isinstance(v, string_types):
v = v.format(**interpolation)
prefs[k] = Preferences.cast(v)
quitter = os.path.join(self.topsrcdir, 'tools', 'quitter',
'*****@*****.**')
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=httpd.httpd.server_port,
options='primary')
profile = FirefoxProfile(profile=profilePath,
preferences=prefs,
addons=[quitter],
locations=locations)
firefox_args = [httpd.get_url()]
env = os.environ.copy()
env['G_SLICE'] = 'always-malloc'
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'warn'
env.update(self.extra_environment_variables)
outputHandler = OutputHandler(self.log)
kp_kwargs = {'processOutputLine': [outputHandler]}
valgrind = 'valgrind'
if not os.path.exists(valgrind):
valgrind = findInPath(valgrind)
valgrind_args = [
valgrind,
'--sym-offsets=yes',
'--smc-check=all-non-file',
'--vex-iropt-register-updates=allregs-at-mem-access',
'--gen-suppressions=all',
'--num-callers=36',
'--leak-check=full',
'--show-possibly-lost=no',
'--track-origins=yes',
'--trace-children=yes',
'-v', # Enable verbosity to get the list of used suppressions
# Avoid excessive delays in the presence of spinlocks.
# See bug 1309851.
'--fair-sched=yes',
# Keep debuginfo after library unmap. See bug 1382280.
'--keep-debuginfo=yes',
# Reduce noise level on rustc and/or LLVM compiled code.
# See bug 1365915
'--expensive-definedness-checks=yes',
]
for s in suppressions:
valgrind_args.append('--suppressions=' + s)
supps_dir = os.path.join(build_dir, 'valgrind')
supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
valgrind_args.append('--suppressions=' + supps_file1)
if mozinfo.os == 'linux':
machtype = {
'x86_64': 'x86_64-pc-linux-gnu',
'x86': 'i386-pc-linux-gnu',
}.get(mozinfo.processor)
if machtype:
supps_file2 = os.path.join(supps_dir, machtype + '.sup')
if os.path.isfile(supps_file2):
valgrind_args.append('--suppressions=' + supps_file2)
exitcode = None
timeout = 1800
try:
runner = FirefoxRunner(profile=profile,
binary=self.get_binary_path(),
cmdargs=firefox_args,
env=env,
process_args=kp_kwargs)
runner.start(debug_args=valgrind_args)
exitcode = runner.wait(timeout=timeout)
finally:
errs = outputHandler.error_count
supps = outputHandler.suppression_count
if errs != supps:
status = 1 # turns the TBPL job orange
self.log(
logging.ERROR, 'valgrind-fail-parsing', {
'errs': errs,
'supps': supps
},
'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors '
'seen, but {supps} generated suppressions seen')
elif errs == 0:
status = 0
self.log(
logging.INFO, 'valgrind-pass', {},
'TEST-PASS | valgrind-test | valgrind found no errors')
else:
status = 1 # turns the TBPL job orange
# We've already printed details of the errors.
if exitcode is None:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR, 'valgrind-fail-timeout',
{'timeout': timeout},
'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out '
'(reached {timeout} second limit)')
elif exitcode != 0:
status = 2 # turns the TBPL job red
self.log(
logging.ERROR, 'valgrind-fail-errors',
{'exitcode': exitcode},
'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code '
'from Valgrind: {exitcode}')
httpd.stop()
return status
# Create profile for mozrunner and start the Firebug tests
self.log.info("Starting Firebug Tests")
try:
self.log.debug(
"Creating Firefox profile and installing extensions")
prefs = {"extensions.update.enabled": "false"}
mozProfile = FirefoxProfile(profile=self.profile,
addons=["firebug.xpi", "fbtest.xpi"],
preferences=prefs)
self.profile = mozProfile.profile
self.log.debug("Creating Firefox runner")
mozRunner = FirefoxRunner(
profile=mozProfile,
binary=self.binary,
cmdargs=["-no-remote", "-runFBTests", self.testlist],
env=mozEnv)
self.binary = mozRunner.binary
if not self.appVersion:
self.appdir, self.appVersion = self.get_app_info()
# Disable the compatibility check on startup
self.disable_compatibility_check()
self.log.debug("Running '" + self.binary +
" -no-remote -runFBTests " + self.testlist + "'")
mozRunner.start()
except Exception:
self.log.error("Could not start Firefox")
self.log.error(traceback.format_exc())
def run():
parser = argparse.ArgumentParser(description='Run crawler')
parser.add_argument('-b',
'--binary',
type=str,
help='path to the Firefox binary')
parser.add_argument('-a',
'--abpdir',
type=str,
help='path to the Adblock Plus repository')
parser.add_argument(
'-f',
'--filters',
metavar='url',
type=str,
nargs='+',
default=[
"https://easylist-downloads.adblockplus.org/easylist.txt",
"https://easylist-downloads.adblockplus.org/exceptionrules.txt"
],
help=
'filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.'
)
parser.add_argument('-t',
'--timeout',
type=int,
default=300,
help='Load timeout (seconds)')
parser.add_argument('-x',
'--maxtabs',
type=int,
default=15,
help='Maximal number of tabs to open in parallel')
parser.add_argument('list', type=str, help='URL list to process')
parser.add_argument('outdir',
type=str,
help='directory to write data into')
parameters = parser.parse_args()
import buildtools.packagerGecko as packager
cleanup = []
try:
base_dir = os.path.dirname(__file__)
handle, crawlerxpi = tempfile.mkstemp(suffix='.xpi')
os.close(handle)
cleanup.append(crawlerxpi)
packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)
abpxpi = 'https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi'
if parameters.abpdir:
handle, abpxpi = tempfile.mkstemp(suffix='.xpi')
os.close(handle)
cleanup.append(abpxpi)
packager.createBuild(parameters.abpdir,
outFile=abpxpi,
releaseBuild=True)
profile = FirefoxProfile(addons=[
crawlerxpi,
abpxpi,
],
preferences={
'browser.uitour.enabled': False,
'prompts.tab_modal.enabled': False,
})
abpsettings = os.path.join(profile.profile, 'adblockplus')
os.makedirs(abpsettings)
with open(os.path.join(abpsettings, 'patterns.ini'), 'w') as handle:
print >> handle, '# Adblock Plus preferences'
print >> handle, 'version=4'
for url in parameters.filters:
if '=' in url:
path, url = url.split('=', 1)
with open(path, 'r') as source:
data = source.read()
else:
data = urllib.urlopen(url).read()
print >> handle, '[Subscription]'
print >> handle, 'url=%s' % url
print >> handle, '[Subscription filters]'
print >> handle, '\n'.join(data.splitlines()[1:])
finally:
for path in cleanup:
os.unlink(path)
server = None
try:
port = random.randrange(2000, 60000)
print "Communicating with client on port %i" % port
app = CrawlerApp(parameters)
server = make_server('localhost', port, app)
app.server = server
threading.Thread(target=lambda: server.serve_forever()).start()
runner = FirefoxRunner(
profile=profile,
binary=parameters.binary,
cmdargs=['--crawler-port', str(port)],
env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE='1'),
)
while app.urls:
runner.start()
runner.wait()
finally:
if server:
server.shutdown()
profile.cleanup()
