def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) env = os.environ.copy() env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations(filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, preferences=preferences) self.profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames)}) if self.ca_certificate_path is not None: self.setup_ssl() debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"], self.debug_info) self.runner = FirefoxRunner(profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started")
def run(self, test, phase='phase1', ignore_unused_engines=True): self.profile.set_preferences({ 'testing.tps.testFile': os.path.abspath(test), 'testing.tps.testPhase': phase, 'testing.tps.ignoreUnusedEngines': ignore_unused_engines, }) args = ['-marionette'] process_args = {'processOutputLine': [self._log]} self.logger.info('Running: {} {}'.format(self.firefox, ' '.join(args))) self.logger.info('Using profile at: {}'.format(self.profile.profile)) runner = FirefoxRunner(binary=self.firefox, cmdargs=args, profile=self.profile, process_args=process_args) runner.start(timeout=TIMEOUT) runner.wait(timeout=TIMEOUT) self.firefox_log.close() with open(self.tps_log) as f: for line in f.readlines(): if 'CROSSWEAVE ERROR: ' in line: raise TPSError(line.partition('CROSSWEAVE ERROR: ')[-1]) with open(self.tps_log) as f: assert 'test phase {}: PASS'.format(phase) in f.read()
def launch(self, args=None): """Launch the app with optional args for profile, windows, URI, etc. :param url: URL to be loaded. :param args: Optional list of arguments. :return: List of Firefox flags. """ if args is None: args = [] args.append('-foreground') args.append('-no-remote') args.append('-new-tab') args.append(self.url) process_args = {'stream': None} logger.debug('Creating Firefox runner ...') try: runner = FirefoxRunner(binary=self.application.path, profile=self.profile, cmdargs=args, process_args=process_args) logger.debug('Firefox runner successfully created.') logger.debug('Running Firefox with command: "%s"' % ','.join(runner.command)) except run_errors.RunnerNotStartedError: raise APIHelperError('Error creating Firefox runner.') else: return runner
def launch_control_center(): profile_path = os.path.join(get_core_args().workdir, 'cc_profile') fx_path = PathManager.get_local_firefox_path() if fx_path is None: logger.error( 'Can\'t find local Firefox installation, aborting Iris run.') return False, None args = ['http://127.0.0.1:%s' % get_core_args().port] process_args = {'stream': None} profile = MozProfile(profile=profile_path, preferences=Settings.default_fx_prefs) fx_runner = FirefoxRunner(binary=fx_path, profile=profile, cmdargs=args, process_args=process_args) fx_runner.start() server = LocalWebServer(get_core_args().workdir, get_core_args().port) server.stop() time.sleep(Settings.DEFAULT_UI_DELAY) if OSHelper.is_mac(): type(text='q', modifier=KeyModifier.CMD) elif OSHelper.is_windows(): type(text='w', modifier=[KeyModifier.CTRL, KeyModifier.SHIFT]) else: type(text='q', modifier=KeyModifier.CTRL) try: fx_runner.stop() except Exception as e: logger.debug('Error stopping fx_runner') logger.debug(e) return server.result
def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations( filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, proxy=True, preferences=preferences) self.profile.set_preferences({ "marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False }) self.runner = FirefoxRunner( profile=self.profile, binary=self.binary, cmdargs=[cmd_arg("marionette"), "about:blank"], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=self.debug_args, interactive=self.interactive) self.logger.debug("Firefox Started")
def launch_firefox(path, profile=None, url=None, args=None): """Launch the app with optional args for profile, windows, URI, etc. :param path: Firefox path. :param profile: Firefox profile. :param url: URL to be loaded. :param args: Optional list of arguments. :return: List of Firefox flags. """ if args is None: args = [] if profile is None: raise APIHelperError('No profile name present, aborting run.') profile = FirefoxProfile(profile=profile) args.append('-foreground') args.append('-no-remote') if url is not None: args.append('-new-tab') args.append(url) process_args = {'stream': None} logger.debug('Creating Firefox runner ...') try: runner = FirefoxRunner(binary=path, profile=profile, cmdargs=args, process_args=process_args) logger.debug('Firefox runner successfully created.') logger.debug('Running Firefox with command: "%s"' % ','.join(runner.command)) return runner except errors.RunnerNotStartedError: raise APIHelperError('Error creating Firefox runner.')
def run(self, test, phase='phase1', ignore_unused_engines=True): args = [ '-tps={}'.format(os.path.abspath(test)), '-tpsphase={}'.format(phase), '-marionette' ] if ignore_unused_engines: args.append('--ignore-unused-engines') process_args = {'processOutputLine': [self._log]} self.logger.info('Running: {} {}'.format(self.firefox, ' '.join(args))) self.logger.info('Using profile at: {}'.format(self.profile.profile)) runner = FirefoxRunner(binary=self.firefox, cmdargs=args, profile=self.profile, process_args=process_args) runner.start(timeout=TIMEOUT) runner.wait(timeout=TIMEOUT) self.firefox_log.close() with open(self.tps_log) as f: for line in f.readlines(): if 'CROSSWEAVE ERROR: ' in line: raise TPSError(line.partition('CROSSWEAVE ERROR: ')[-1]) with open(self.tps_log) as f: assert 'test phase {}: PASS'.format(phase) in f.read()
def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None): """Runs the given FirefoxRunner with the given Profile, waits for completion, then returns the process exit code """ if profile is None: profile = Profile() self.profile = profile if self.binary is None and self.url: self.binary = self.download_build() if self.runner is None: self.runner = FirefoxRunner(self.profile, binary=self.binary) self.runner.profile = self.profile if env is not None: self.runner.env.update(env) if args is not None: self.runner.cmdargs = copy.copy(args) self.runner.start() returncode = self.runner.wait(timeout) return returncode
def runprofile(binary, fileobj): try: runner = FirefoxRunner(binary=binary, profile=fileobj.profile) runner.start() runner.wait() except KeyboardInterrupt: pass
def start(self): self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) env = os.environ.copy() env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" locations = ServerLocations( filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, preferences=preferences) self.profile.set_preferences({ "marionette.enabled": True, "marionette.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames), "network.proxy.type": 0, "places.history.enabled": False }) if self.e10s: self.profile.set_preferences( {"browser.tabs.remote.autostart": True}) # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences( {"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() debug_args, cmd = browser_command( self.binary, [cmd_arg("marionette"), "about:blank"], self.debug_info) self.runner = FirefoxRunner( profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started")
def launch_control_center(): profile_path = os.path.join(PathManager.get_working_dir(), "cc_profile") fx_path = PathManager.get_local_firefox_path() if fx_path is None: logger.error("Can't find local Firefox installation, aborting Iris run.") return False, None args = ["http://127.0.0.1:%s" % get_core_args().port] process_args = {"stream": None} profile = MozProfile(profile=profile_path, preferences=get_fx_prefs()) if OSHelper.is_windows(): process = subprocess.Popen( [ fx_path, "-no-remote", "-new-tab", args, "--wait-for-browser", "-foreground", "-profile", profile.profile, ], shell=False, ) else: fx_runner = FirefoxRunner( binary=fx_path, profile=profile, cmdargs=args, process_args=process_args ) fx_runner.start() logger.debug("Launching web server for directory %s" % PathManager.get_working_dir()) server = LocalWebServer(PathManager.get_working_dir(), get_core_args().port) server.stop() time.sleep(Settings.DEFAULT_UI_DELAY) if OSHelper.is_mac(): type(text="q", modifier=KeyModifier.CMD) elif OSHelper.is_windows(): type(text="w", modifier=[KeyModifier.CTRL, KeyModifier.SHIFT]) else: type(text="q", modifier=KeyModifier.CTRL) if OSHelper.is_windows(): if process.pid is not None: try: logger.debug("Closing Firefox process ID: %s" % process.pid) process = psutil.Process(process.pid) for proc in process.children(recursive=True): proc.kill() process.kill() except psutil.NoSuchProcess: pass else: try: fx_runner.stop() except Exception as e: logger.debug("Error stopping fx_runner") logger.debug(e) return server.result
def start(self): """Start an instance of Firefox, returning a BrowserInstance handle""" profile = self.base_profile.clone(self.base_profile.profile) marionette_port = get_free_port() profile.set_preferences({"marionette.port": marionette_port}) env = test_environment(xrePath=os.path.dirname(self.binary), debugger=self.debug_info is not None, useLSan=True, log=self.logger) env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) if self.headless: env["MOZ_HEADLESS"] = "1" if self.enable_webrender: env["MOZ_WEBRENDER"] = "1" env["MOZ_ACCELERATED"] = "1" # Set MOZ_X_SYNC and GDK_SYNCHRONIZE for investigation; bug 1625250. env["MOZ_X_SYNC"] = "1" env["GDK_SYNCHRONIZE"] = "1" else: env["MOZ_WEBRENDER"] = "0" args = self.binary_args[:] if self.binary_args else [] args += [cmd_arg("marionette"), "about:blank"] debug_args, cmd = browser_command(self.binary, args, self.debug_info) if self.leak_check: leak_report_file = os.path.join( profile.profile, "runtests_leaks_%s.log" % os.getpid()) if os.path.exists(leak_report_file): os.remove(leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = leak_report_file else: leak_report_file = None output_handler = OutputHandler(self.logger, self.stackfix_dir, self.symbols_path, self.asan) runner = FirefoxRunner( profile=profile, binary=cmd[0], cmdargs=cmd[1:], env=cast_env(env), process_class=ProcessHandler, process_args={"processOutputLine": [output_handler]}) instance = BrowserInstance(self.logger, runner, marionette_port, output_handler, leak_report_file) self.logger.debug("Starting Firefox") runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") return instance
def run_tests(self, test_path, options): reftestlist = self.getManifestPath(test_path) if not reftestlist.startswith('file://'): reftestlist = 'file://%s' % reftestlist self.profile = self.create_profile(options, reftestlist, profile_to_clone=options.profile) env = self.buildBrowserEnv(options, self.profile.profile) kp_kwargs = { 'processOutputLine': [self._on_output], 'onTimeout': [self._on_timeout], 'kill_on_timeout': False } if not options.debugger: if not options.timeout: if mozinfo.info['debug']: options.timeout = 420 else: options.timeout = 300 self.timeout = options.timeout + 30.0 log.info("%s | Running tests: start.", os.path.basename(__file__)) cmd, args = self.build_command_line( options.app, ignore_window_size=options.ignoreWindowSize, browser_arg=options.browser_arg) self.runner = FirefoxRunner(profile=self.profile, binary=cmd, cmdargs=args, env=env, process_class=ProcessHandler, process_args=kp_kwargs, symbols_path=options.symbolsPath) status = 0 try: self.runner.start(outputTimeout=self.timeout) log.info("%s | Application pid: %d", os.path.basename(__file__), self.runner.process_handler.pid) # kick starts the reftest harness self.run_marionette_script() status = self.runner.wait() finally: self.runner.check_for_crashes(test_name=self.last_test) self.runner.cleanup() if status > 0: log.testFail("%s | application terminated with exit code %s", self.last_test, status) elif status < 0: log.info("%s | application killed with signal %s", self.last_test, -status) log.info("%s | Running tests: end.", os.path.basename(__file__)) return status
def launch_control_center(): profile_path = os.path.join(get_core_args().workdir, 'cc_profile') fx_path = PathManager.get_local_firefox_path() if fx_path is None: logger.error( 'Can\'t find local Firefox installation, aborting Iris run.') return False, None args = ['http://127.0.0.1:%s' % get_core_args().port] process_args = {'stream': None} profile = MozProfile(profile=profile_path, preferences=Settings.default_fx_prefs) if OSHelper.is_windows(): process = subprocess.Popen([ fx_path, '-no-remote', '-new-tab', args, '--wait-for-browser', '-foreground', '-profile', profile.profile ], shell=False) else: fx_runner = FirefoxRunner(binary=fx_path, profile=profile, cmdargs=args, process_args=process_args) fx_runner.start() server = LocalWebServer(get_core_args().workdir, get_core_args().port) server.stop() time.sleep(Settings.DEFAULT_UI_DELAY) if OSHelper.is_mac(): type(text='q', modifier=KeyModifier.CMD) elif OSHelper.is_windows(): type(text='w', modifier=[KeyModifier.CTRL, KeyModifier.SHIFT]) else: type(text='q', modifier=KeyModifier.CTRL) if OSHelper.is_windows(): if process.pid is not None: try: logger.debug('Closing Firefox process ID: %s' % process.pid) process = psutil.Process(process.pid) for proc in process.children(recursive=True): proc.kill() process.kill() except psutil.NoSuchProcess: pass else: try: fx_runner.stop() except Exception as e: logger.debug('Error stopping fx_runner') logger.debug(e) return server.result
def run(self, profile=None, timeout=PROCESS_TIMEOUT, env=None, args=None): """Runs the given FirefoxRunner with the given Profile, waits for completion, then returns the process exit code """ if profile is None: profile = Profile() self.profile = profile if self.binary is None and self.url: self.binary = self.download_build() runner = FirefoxRunner(profile=self.profile, binary=self.binary, env=env, cmdargs=args) runner.start(timeout=timeout) return runner.wait()
def start(self): """Start an instance of Firefox, returning a BrowserInstance handle""" profile = self.base_profile.clone(self.base_profile.profile) marionette_port = get_free_port() profile.set_preferences({"marionette.port": marionette_port}) env = get_environ(self.logger, self.binary, self.debug_info, self.stylo_threads, self.headless, self.enable_webrender, self.chaos_mode_flags) args = self.binary_args[:] if self.binary_args else [] args += [cmd_arg("marionette"), "about:blank"] debug_args, cmd = browser_command(self.binary, args, self.debug_info) if self.leak_check: leak_report_file = os.path.join( profile.profile, "runtests_leaks_%s.log" % os.getpid()) if os.path.exists(leak_report_file): os.remove(leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = leak_report_file else: leak_report_file = None output_handler = OutputHandler(self.logger, self.stackfix_dir, self.symbols_path, self.asan) runner = FirefoxRunner( profile=profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [output_handler]}) instance = BrowserInstance(self.logger, runner, marionette_port, output_handler, leak_report_file) self.logger.debug("Starting Firefox") runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started") return instance
def start_firefox(self): env = os.environ.copy() env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' profile = Profile() profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False, "dom.max_script_run_time": 0}) self.firefox_runner = FirefoxRunner(profile, self.firefox_binary, cmdargs=["--marionette"], env=env, kp_kwargs = {"processOutputLine":[self.on_output]}, process_class=self.process_cls) self.logger.debug("Starting Firefox") self.firefox_runner.start() self.logger.debug("Firefox Started")
def start(self): """Start an instance of Firefox, returning a BrowserInstance handle""" profile = self.base_profile.clone(self.base_profile.profile) marionette_port = get_free_port() profile.set_preferences({"marionette.port": marionette_port}) env = get_environ(self.logger, self.binary, self.debug_info, self.stylo_threads, self.headless, self.chaos_mode_flags) args = self.binary_args[:] if self.binary_args else [] args += [cmd_arg("marionette"), "about:blank"] debug_args, cmd = browser_command(self.binary, args, self.debug_info) leak_report_file = setup_leak_report(self.leak_check, profile, env) output_handler = FirefoxOutputHandler( self.logger, cmd, stackfix_dir=self.stackfix_dir, symbols_path=self.symbols_path, asan=self.asan, leak_report_file=leak_report_file) runner = FirefoxRunner( profile=profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [output_handler]}) instance = BrowserInstance(self.logger, runner, marionette_port, output_handler, leak_report_file) self.logger.debug("Starting Firefox") runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) output_handler.after_process_start(runner.process_handler.pid) self.logger.debug("Firefox Started") return instance
env["XPCOM_DEBUG_BREAK"] = "warn" # For VC12+, make sure we can find the right bitness of pgort1x0.dll if not substs.get('HAVE_64BIT_BUILD'): for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'): if e not in env: continue vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin')) if os.path.exists(vcdir): env['PATH'] = '%s;%s' % (vcdir, env['PATH']) break # Run Firefox a first time to initialize its profile runner = FirefoxRunner(profile=profile, binary=binary, cmdargs=['data:text/html,<script>Quitter.quit()</script>'], env=env) runner.start() ret = runner.wait() if ret: print("Firefox exited with code %d during profile initialization" % ret) httpd.stop() sys.exit(ret) jarlog = os.getenv("JARLOG_FILE") if jarlog: env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog) print("jarlog: %s" % env["MOZ_JAR_LOG_FILE"]) cmdargs = ["http://localhost:%d/index.html" % PORT]
def start(self, **kwargs): if self.marionette_port is None: self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) env = os.environ.copy() env["MOZ_CRASHREPORTER"] = "1" env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) locations = ServerLocations( filename=os.path.join(here, "server-locations.txt")) preferences = self.load_prefs() self.profile = FirefoxProfile(locations=locations, preferences=preferences) self.profile.set_preferences({ "marionette.port": self.marionette_port, "dom.disable_open_during_load": False, "network.dns.localDomains": ",".join(hostnames), "network.proxy.type": 0, "places.history.enabled": False, "dom.send_after_paint_to_content": True, "network.preload": True }) if self.e10s: self.profile.set_preferences( {"browser.tabs.remote.autostart": True}) if self.test_type == "reftest": self.profile.set_preferences( {"layout.interruptible-reflow.enabled": False}) if self.leak_check and kwargs.get("check_leaks", True): self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log") if os.path.exists(self.leak_report_file): os.remove(self.leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file else: self.leak_report_file = None # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences( {"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() debug_args, cmd = browser_command( self.binary, self.binary_args if self.binary_args else [] + [cmd_arg("marionette"), "about:blank"], self.debug_info) self.runner = FirefoxRunner( profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started")
def valgrind_test(self, suppressions): from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from six import string_types from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, "build") # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo")) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile profile_data_dir = os.path.join(self.topsrcdir, "testing", "profiles") with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh: base_profiles = json.load(fh)["valgrind"] prefpaths = [ os.path.join(profile_data_dir, profile, "user.js") for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { "server": "%s:%d" % httpd.httpd.server_address, } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) quitter = os.path.join(self.topsrcdir, "tools", "quitter", "*****@*****.**") locations = ServerLocations() locations.add_host(host="127.0.0.1", port=httpd.httpd.server_port, options="primary") profile = FirefoxProfile( profile=profilePath, preferences=prefs, addons=[quitter], locations=locations, ) firefox_args = [httpd.get_url()] env = os.environ.copy() env["G_SLICE"] = "always-malloc" env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1" env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1" env["XPCOM_DEBUG_BREAK"] = "warn" outputHandler = OutputHandler(self.log) kp_kwargs = { "processOutputLine": [outputHandler], "universal_newlines": True, } valgrind = "valgrind" if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, "--sym-offsets=yes", "--smc-check=all-non-file", "--vex-iropt-register-updates=allregs-at-mem-access", "--gen-suppressions=all", "--num-callers=36", "--leak-check=full", "--show-possibly-lost=no", "--track-origins=yes", "--trace-children=yes", "-v", # Enable verbosity to get the list of used suppressions # Avoid excessive delays in the presence of spinlocks. # See bug 1309851. "--fair-sched=yes", # Keep debuginfo after library unmap. See bug 1382280. "--keep-debuginfo=yes", # Reduce noise level on rustc and/or LLVM compiled code. # See bug 1365915 "--expensive-definedness-checks=yes", # Compensate for the compiler inlining `new` but not `delete` # or vice versa. "--show-mismatched-frees=no", ] for s in suppressions: valgrind_args.append("--suppressions=" + s) supps_dir = os.path.join(build_dir, "valgrind") supps_file1 = os.path.join(supps_dir, "cross-architecture.sup") valgrind_args.append("--suppressions=" + supps_file1) if mozinfo.os == "linux": machtype = { "x86_64": "x86_64-pc-linux-gnu", "x86": "i386-pc-linux-gnu", }.get(mozinfo.processor) if machtype: supps_file2 = os.path.join(supps_dir, machtype + ".sup") if os.path.isfile(supps_file2): valgrind_args.append("--suppressions=" + supps_file2) exitcode = None timeout = 1800 binary_not_found_exception = None try: runner = FirefoxRunner( profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs, ) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) except BinaryNotFoundException as e: binary_not_found_exception = e finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log( logging.ERROR, "valgrind-fail-parsing", { "errs": errs, "supps": supps }, "TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors " "seen, but {supps} generated suppressions seen", ) elif errs == 0: status = 0 self.log( logging.INFO, "valgrind-pass", {}, "TEST-PASS | valgrind-test | valgrind found no errors", ) else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if binary_not_found_exception: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-errors", {"error": str(binary_not_found_exception)}, "TEST-UNEXPECTED-FAIL | valgrind-test | {error}", ) self.log( logging.INFO, "valgrind-fail-errors", {"help": binary_not_found_exception.help()}, "{help}", ) elif exitcode is None: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-timeout", {"timeout": timeout}, "TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out " "(reached {timeout} second limit)", ) elif exitcode != 0: status = 2 # turns the TBPL job red self.log( logging.ERROR, "valgrind-fail-errors", {"exitcode": exitcode}, "TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code " "from Valgrind: {exitcode}", ) httpd.stop() return status
# For VC12+, make sure we can find the right bitness of pgort1x0.dll if not substs.get('HAVE_64BIT_BUILD'): for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'): if e not in env: continue vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin')) if os.path.exists(vcdir): env['PATH'] = '%s;%s' % (vcdir, env['PATH']) break # Run Firefox a first time to initialize its profile runner = FirefoxRunner(profile=profile, binary=build.get_binary_path( where="staged-package"), cmdargs=['javascript:Quitter.quit()'], env=env) runner.start() runner.wait() jarlog = os.getenv("JARLOG_FILE") if jarlog: env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog) print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"] cmdargs = ["http://localhost:%d/index.html" % PORT] runner = FirefoxRunner(profile=profile, binary=build.get_binary_path( where="staged-package"), cmdargs=cmdargs,
def valgrind_test(self, suppressions): import json import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, 'OOP': 'false'} prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', '*****@*****.**') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' env.update(self.extra_environment_variables) outputHandler = OutputHandler(self.log) kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=36', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes', '--trace-children=yes', '-v', # Enable verbosity to get the list of used suppressions ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip() supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None timeout = 1800 try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log(logging.ERROR, 'valgrind-fail-parsing', {'errs': errs, 'supps': supps}, 'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen') elif errs == 0: status = 0 self.log(logging.INFO, 'valgrind-pass', {}, 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode == None: status = 2 # turns the TBPL job red self.log(logging.ERROR, 'valgrind-fail-timeout', {'timeout': timeout}, 'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)') elif exitcode != 0: status = 2 # turns the TBPL job red self.log(logging.ERROR, 'valgrind-fail-errors', {}, 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind') httpd.stop() return status
"OOP": "false" } prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) profile = FirefoxProfile( profile=profilePath, preferences=prefs, addons=[os.path.join(build.distdir, 'xpi-stage', 'quitter')], locations=locations) env = os.environ.copy() env["MOZ_CRASHREPORTER_NO_REPORT"] = "1" env["XPCOM_DEBUG_BREAK"] = "warn" jarlog = os.getenv("JARLOG_FILE") if jarlog: env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog) print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"] cmdargs = ["http://localhost:%d/index.html" % PORT] runner = FirefoxRunner( profile=profile, binary=build.get_binary_path(where="staged-package"), cmdargs=cmdargs, env=env) runner.start(debug_args=debug_args, interactive=interactive) runner.wait() httpd.stop() finally: shutil.rmtree(profilePath)
def start(self, group_metadata=None, **kwargs): if group_metadata is None: group_metadata = {} if self.marionette_port is None: self.marionette_port = get_free_port(2828, exclude=self.used_ports) self.used_ports.add(self.marionette_port) if self.asan: print "Setting up LSAN" self.lsan_handler = mozleak.LSANLeaks( self.logger, scope=group_metadata.get("scope", "/"), allowed=self.lsan_allowed, maxNumRecordedFrames=self.lsan_max_stack_depth) env = test_environment(xrePath=os.path.dirname(self.binary), debugger=self.debug_info is not None, log=self.logger, lsanPath=self.prefs_root) env["STYLO_THREADS"] = str(self.stylo_threads) if self.chaos_mode_flags is not None: env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags) if self.headless: env["MOZ_HEADLESS"] = "1" preferences = self.load_prefs() self.profile = FirefoxProfile(preferences=preferences) self.profile.set_preferences({ "marionette.port": self.marionette_port, "network.dns.localDomains": ",".join(self.config.domains_set), "dom.file.createInChild": True, # TODO: Remove preferences once Firefox 64 is stable (Bug 905404) "network.proxy.type": 0, "places.history.enabled": False, "network.preload": True, }) if self.e10s: self.profile.set_preferences( {"browser.tabs.remote.autostart": True}) if self.test_type == "reftest": self.profile.set_preferences( {"layout.interruptible-reflow.enabled": False}) if self.leak_check: self.leak_report_file = os.path.join( self.profile.profile, "runtests_leaks_%s.log" % os.getpid()) if os.path.exists(self.leak_report_file): os.remove(self.leak_report_file) env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file else: self.leak_report_file = None # Bug 1262954: winxp + e10s, disable hwaccel if (self.e10s and platform.system() in ("Windows", "Microsoft") and '5.1' in platform.version()): self.profile.set_preferences( {"layers.acceleration.disabled": True}) if self.ca_certificate_path is not None: self.setup_ssl() args = self.binary_args[:] if self.binary_args else [] args += [cmd_arg("marionette"), "about:blank"] debug_args, cmd = browser_command(self.binary, args, self.debug_info) self.runner = FirefoxRunner( profile=self.profile, binary=cmd[0], cmdargs=cmd[1:], env=env, process_class=ProcessHandler, process_args={"processOutputLine": [self.on_output]}) self.logger.debug("Starting Firefox") self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive) self.logger.debug("Firefox Started")
env["MOZ_DISABLE_VR_SANDBOX"] = "1" # Ensure different pids write to different files env["LLVM_PROFILE_FILE"] = "default_%p_random_%m.profraw" # Write to an output file if we're running in automation process_args = {"universal_newlines": True} if "UPLOAD_PATH" in env: process_args["logfile"] = os.path.join(env["UPLOAD_PATH"], "profile-run-1.log") # Run Firefox a first time to initialize its profile runner = FirefoxRunner( profile=profile, binary=binary, cmdargs=["data:text/html,<script>Quitter.quit()</script>"], env=env, process_args=process_args, ) runner.start() ret = runner.wait() if ret: print("Firefox exited with code %d during profile initialization" % ret) logfile = process_args.get("logfile") if logfile: print("Firefox output (%s):" % logfile) with open(logfile) as f: print(f.read()) httpd.stop() get_crashreports(profilePath, name="Profile initialization")
def valgrind_test(self, suppressions): import json import re import sys import tempfile from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: #TODO: refactor this into mozprofile prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') prefs = {} prefs.update(Preferences.read_prefs(prefpath)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, 'OOP': 'false' } prefs = json.loads(json.dumps(prefs) % interpolation) for pref in prefs: prefs[pref] = Preferences.cast(prefs[pref]) quitter = os.path.join(self.distdir, 'xpi-stage', 'quitter') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['XPCOM_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' class OutputHandler(object): def __init__(self): self.found_errors = False def __call__(self, line): print(line) m = re.match( r'.*ERROR SUMMARY: [1-9]\d* errors from \d+ contexts', line) if m: self.found_errors = True outputHandler = OutputHandler() kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=20', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes' ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) # MACHTYPE is an odd bash-only environment variable that doesn't # show up in os.environ, so we have to get it another way. machtype = subprocess.check_output( ['bash', '-c', 'echo $MACHTYPE']).rstrip() supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, kp_kwargs=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait() finally: if not outputHandler.found_errors: status = 0 print( 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange print( 'TEST-UNEXPECTED-FAIL | valgrind-test | valgrind found errors' ) if exitcode != 0: status = 2 # turns the TBPL job red print( 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind' ) httpd.stop() return status
def valgrind_test(self, suppressions): from mozfile import TemporaryDirectory from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner from mozrunner.utils import findInPath from six import string_types from valgrind.output_handler import OutputHandler build_dir = os.path.join(self.topsrcdir, 'build') # XXX: currently we just use the PGO inputs for Valgrind runs. This may # change in the future. httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) httpd.start(block=False) with TemporaryDirectory() as profilePath: # TODO: refactor this into mozprofile profile_data_dir = os.path.join(self.topsrcdir, 'testing', 'profiles') with open(os.path.join(profile_data_dir, 'profiles.json'), 'r') as fh: base_profiles = json.load(fh)['valgrind'] prefpaths = [ os.path.join(profile_data_dir, profile, 'user.js') for profile in base_profiles ] prefs = {} for path in prefpaths: prefs.update(Preferences.read_prefs(path)) interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, } for k, v in prefs.items(): if isinstance(v, string_types): v = v.format(**interpolation) prefs[k] = Preferences.cast(v) quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', '*****@*****.**') locations = ServerLocations() locations.add_host(host='127.0.0.1', port=httpd.httpd.server_port, options='primary') profile = FirefoxProfile(profile=profilePath, preferences=prefs, addons=[quitter], locations=locations) firefox_args = [httpd.get_url()] env = os.environ.copy() env['G_SLICE'] = 'always-malloc' env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1' env['XPCOM_DEBUG_BREAK'] = 'warn' env.update(self.extra_environment_variables) outputHandler = OutputHandler(self.log) kp_kwargs = {'processOutputLine': [outputHandler]} valgrind = 'valgrind' if not os.path.exists(valgrind): valgrind = findInPath(valgrind) valgrind_args = [ valgrind, '--sym-offsets=yes', '--smc-check=all-non-file', '--vex-iropt-register-updates=allregs-at-mem-access', '--gen-suppressions=all', '--num-callers=36', '--leak-check=full', '--show-possibly-lost=no', '--track-origins=yes', '--trace-children=yes', '-v', # Enable verbosity to get the list of used suppressions # Avoid excessive delays in the presence of spinlocks. # See bug 1309851. '--fair-sched=yes', # Keep debuginfo after library unmap. See bug 1382280. '--keep-debuginfo=yes', # Reduce noise level on rustc and/or LLVM compiled code. # See bug 1365915 '--expensive-definedness-checks=yes', ] for s in suppressions: valgrind_args.append('--suppressions=' + s) supps_dir = os.path.join(build_dir, 'valgrind') supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') valgrind_args.append('--suppressions=' + supps_file1) if mozinfo.os == 'linux': machtype = { 'x86_64': 'x86_64-pc-linux-gnu', 'x86': 'i386-pc-linux-gnu', }.get(mozinfo.processor) if machtype: supps_file2 = os.path.join(supps_dir, machtype + '.sup') if os.path.isfile(supps_file2): valgrind_args.append('--suppressions=' + supps_file2) exitcode = None timeout = 1800 try: runner = FirefoxRunner(profile=profile, binary=self.get_binary_path(), cmdargs=firefox_args, env=env, process_args=kp_kwargs) runner.start(debug_args=valgrind_args) exitcode = runner.wait(timeout=timeout) finally: errs = outputHandler.error_count supps = outputHandler.suppression_count if errs != supps: status = 1 # turns the TBPL job orange self.log( logging.ERROR, 'valgrind-fail-parsing', { 'errs': errs, 'supps': supps }, 'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors ' 'seen, but {supps} generated suppressions seen') elif errs == 0: status = 0 self.log( logging.INFO, 'valgrind-pass', {}, 'TEST-PASS | valgrind-test | valgrind found no errors') else: status = 1 # turns the TBPL job orange # We've already printed details of the errors. if exitcode is None: status = 2 # turns the TBPL job red self.log( logging.ERROR, 'valgrind-fail-timeout', {'timeout': timeout}, 'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out ' '(reached {timeout} second limit)') elif exitcode != 0: status = 2 # turns the TBPL job red self.log( logging.ERROR, 'valgrind-fail-errors', {'exitcode': exitcode}, 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code ' 'from Valgrind: {exitcode}') httpd.stop() return status
# Create profile for mozrunner and start the Firebug tests self.log.info("Starting Firebug Tests") try: self.log.debug( "Creating Firefox profile and installing extensions") prefs = {"extensions.update.enabled": "false"} mozProfile = FirefoxProfile(profile=self.profile, addons=["firebug.xpi", "fbtest.xpi"], preferences=prefs) self.profile = mozProfile.profile self.log.debug("Creating Firefox runner") mozRunner = FirefoxRunner( profile=mozProfile, binary=self.binary, cmdargs=["-no-remote", "-runFBTests", self.testlist], env=mozEnv) self.binary = mozRunner.binary if not self.appVersion: self.appdir, self.appVersion = self.get_app_info() # Disable the compatibility check on startup self.disable_compatibility_check() self.log.debug("Running '" + self.binary + " -no-remote -runFBTests " + self.testlist + "'") mozRunner.start() except Exception: self.log.error("Could not start Firefox") self.log.error(traceback.format_exc())
def run(): parser = argparse.ArgumentParser(description='Run crawler') parser.add_argument('-b', '--binary', type=str, help='path to the Firefox binary') parser.add_argument('-a', '--abpdir', type=str, help='path to the Adblock Plus repository') parser.add_argument( '-f', '--filters', metavar='url', type=str, nargs='+', default=[ "https://easylist-downloads.adblockplus.org/easylist.txt", "https://easylist-downloads.adblockplus.org/exceptionrules.txt" ], help= 'filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.' ) parser.add_argument('-t', '--timeout', type=int, default=300, help='Load timeout (seconds)') parser.add_argument('-x', '--maxtabs', type=int, default=15, help='Maximal number of tabs to open in parallel') parser.add_argument('list', type=str, help='URL list to process') parser.add_argument('outdir', type=str, help='directory to write data into') parameters = parser.parse_args() import buildtools.packagerGecko as packager cleanup = [] try: base_dir = os.path.dirname(__file__) handle, crawlerxpi = tempfile.mkstemp(suffix='.xpi') os.close(handle) cleanup.append(crawlerxpi) packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True) abpxpi = 'https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi' if parameters.abpdir: handle, abpxpi = tempfile.mkstemp(suffix='.xpi') os.close(handle) cleanup.append(abpxpi) packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True) profile = FirefoxProfile(addons=[ crawlerxpi, abpxpi, ], preferences={ 'browser.uitour.enabled': False, 'prompts.tab_modal.enabled': False, }) abpsettings = os.path.join(profile.profile, 'adblockplus') os.makedirs(abpsettings) with open(os.path.join(abpsettings, 'patterns.ini'), 'w') as handle: print >> handle, '# Adblock Plus preferences' print >> handle, 'version=4' for url in parameters.filters: if '=' in url: path, url = url.split('=', 1) with open(path, 'r') as source: data = source.read() else: data = urllib.urlopen(url).read() print >> handle, '[Subscription]' print >> handle, 'url=%s' % url print >> handle, '[Subscription filters]' print >> handle, '\n'.join(data.splitlines()[1:]) finally: for path in cleanup: os.unlink(path) server = None try: port = random.randrange(2000, 60000) print "Communicating with client on port %i" % port app = CrawlerApp(parameters) server = make_server('localhost', port, app) app.server = server threading.Thread(target=lambda: server.serve_forever()).start() runner = FirefoxRunner( profile=profile, binary=parameters.binary, cmdargs=['--crawler-port', str(port)], env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE='1'), ) while app.urls: runner.start() runner.wait() finally: if server: server.shutdown() profile.cleanup()