示例#1
0
    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames)})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#2
0
    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(
            filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      proxy=True,
                                      preferences=preferences)
        self.profile.set_preferences({
            "marionette.defaultPrefs.enabled": True,
            "marionette.defaultPrefs.port": self.marionette_port,
            "dom.disable_open_during_load": False
        })

        self.runner = FirefoxRunner(
            profile=self.profile,
            binary=self.binary,
            cmdargs=[cmd_arg("marionette"), "about:blank"],
            env=env,
            process_class=ProcessHandler,
            process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args,
                          interactive=self.interactive)
        self.logger.debug("Firefox Started")
示例#3
0
    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        profile = FirefoxProfile(locations=locations, proxy=True, preferences=preferences)
        profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                 "marionette.defaultPrefs.port": self.marionette_port,
                                 "dom.disable_open_during_load": False})

        self.runner = FirefoxRunner(profile=profile,
                                    binary=self.binary,
                                    cmdargs=[cmd_arg("marionette"), "about:blank"],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
        self.logger.debug("Firefox Started")
示例#4
0
    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
        self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(
            filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({
            "marionette.enabled":
            True,
            "marionette.port":
            self.marionette_port,
            "dom.disable_open_during_load":
            False,
            "network.dns.localDomains":
            ",".join(hostnames),
            "network.proxy.type":
            0,
            "places.history.enabled":
            False
        })
        if self.e10s:
            self.profile.set_preferences(
                {"browser.tabs.remote.autostart": True})

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft")
                and '5.1' in platform.version()):
            self.profile.set_preferences(
                {"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(
            self.binary, [cmd_arg("marionette"), "about:blank"],
            self.debug_info)

        self.runner = FirefoxRunner(
            profile=self.profile,
            binary=cmd[0],
            cmdargs=cmd[1:],
            env=env,
            process_class=ProcessHandler,
            process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args,
                          interactive=self.debug_info
                          and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#5
0
    def start(self):
        profile = FirefoxProfile()

        profile.set_preferences({"dom.disable_open_during_load": False,
                                 "marionette.defaultPrefs.enabled": True})

        self.logger.debug("Creating device runner")
        self.runner = mozrunner.B2GDeviceRunner(profile=profile)
        self.logger.debug("Starting device runner")
        self.runner.start()
        self.logger.debug("Device runner started")
示例#6
0
    def start(self):
        profile = FirefoxProfile()

        profile.set_preferences({"dom.disable_open_during_load": False,
                                 "marionette.defaultPrefs.enabled": True})

        self.logger.debug("Creating device runner")
        self.runner = mozrunner.B2GDeviceRunner(profile=profile)
        self.logger.debug("Starting device runner")
        self.runner.start()
        self.logger.debug("Device runner started")
def install_greasemonkey(profile_path):
    profile = FirefoxProfile(str(profile_path))
    gm_amo_url = 'https://services.addons.mozilla.org/fr/firefox/api/1.5/addon/748'
    xpi_url = profile.addon_manager.get_amo_install_path(gm_amo_url)
    print("Installing Greasemonkey from:", xpi_url)
    profile.addon_manager.install_from_path(xpi_url)
    # avoid showing Greasemonkey appearing in anyway in the UI
    gm_prefs = {
        'extensions.greasemonkey.haveInsertedToolbarbutton': True,
        'extensions.greasemonkey.stats.prompted': True
    }
    profile.set_persistent_preferences(gm_prefs)
    import ipdb; ipdb.set_trace()
def install_greasemonkey(profile_path):
    profile = FirefoxProfile(str(profile_path))
    gm_amo_url = 'https://services.addons.mozilla.org/fr/firefox/api/1.5/addon/748'
    xpi_url = profile.addon_manager.get_amo_install_path(gm_amo_url)
    print("Installing Greasemonkey from:", xpi_url)
    profile.addon_manager.install_from_path(xpi_url)
    # avoid showing Greasemonkey appearing in anyway in the UI
    gm_prefs = {
        'extensions.greasemonkey.haveInsertedToolbarbutton': True,
        'extensions.greasemonkey.stats.prompted': True
    }
    profile.set_persistent_preferences(gm_prefs)
    import ipdb
    ipdb.set_trace()
示例#9
0
    def test_relative_path(self):
        tempdir = tempfile.mkdtemp()

        # make a dummy profile
        profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
                                 restore=False)
        self.assertTrue(
            os.path.exists(os.path.join(tempdir, 'testprofilepath',
                                        'user.js')))

        # make a dummy test
        test = """function test() { };"""
        f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
        f.write(test)
        f.close()

        # run mozmill on it
        process = ProcessHandler(
            ['mozmill', '-t', 'test_dummy.js', '--profile=testprofilepath'],
            cwd=tempdir,
            # stop mozmill from printing output to console
            processOutputLine=[lambda line: None])
        process.run()
        process.wait()

        self.assertNotEqual(process.proc.poll(), None)

        # cleanup
        shutil.rmtree(tempdir)
示例#10
0
文件: general.py 项目: Sylvia23/iris
def launch_firefox(path, profile=None, url=None, args=None):
    """Launch the app with optional args for profile, windows, URI, etc.

    :param path: Firefox path.
    :param profile: Firefox profile.
    :param url: URL to be loaded.
    :param args: Optional list of arguments.
    :return: List of Firefox flags.
    """
    if args is None:
        args = []

    if profile is None:
        raise APIHelperError('No profile name present, aborting run.')

    profile = FirefoxProfile(profile=profile)

    args.append('-foreground')
    args.append('-no-remote')

    if url is not None:
        args.append('-new-tab')
        args.append(url)

    process_args = {'stream': None}
    logger.debug('Creating Firefox runner ...')
    try:
        runner = FirefoxRunner(binary=path, profile=profile, cmdargs=args, process_args=process_args)
        logger.debug('Firefox runner successfully created.')
        logger.debug('Running Firefox with command: "%s"' % ','.join(runner.command))
        return runner
    except errors.RunnerNotStartedError:
        raise APIHelperError('Error creating Firefox runner.')
    def test_relative_path(self):
        tempdir = tempfile.mkdtemp()

        # make a dummy profile
        profile = FirefoxProfile(os.path.join(tempdir, 'testprofilepath'),
                                 restore=False)
        self.assertTrue(
            os.path.exists(os.path.join(tempdir, 'testprofilepath',
                                        'user.js')))

        # make a dummy test
        test = """test1 = function() { };"""
        f = file(os.path.join(tempdir, 'test_dummy.js'), 'w')
        f.write(test)
        f.close()

        # run mozmill on it
        process = ProcessHandler(
            ['mozmill', '-t', 'test_dummy.js', '--profile=testprofilepath'],
            cwd=tempdir)
        code = process.waitForFinish(timeout=120)
        self.assertEqual(code, 0)

        # cleanup
        shutil.rmtree(tempdir)
示例#12
0
    def create_profile(self, custom_addons=[], custom_prefs=None, root=True):
        # Create, install and initialize the profile to be
        # used in the test.
        #
        # Extensions are not currently supported therefore we will
        # need to kill the process and ignore that fact.

        self.dm.pkill(self.build.app_name, root=root)
        if isinstance(custom_prefs, dict):
            prefs = dict(self.preferences.items() + custom_prefs.items())
        else:
            prefs = self.preferences
        profile = FirefoxProfile(preferences=prefs, addons=custom_addons)
        if not self.install_profile(profile):
            return False

        self.loggerdeco.debug('Attempt to Initialize profile')
        self.run_fennec_with_profile(self.build.app_name, self._initialize_url)

        self.wait_for_fennec(max_wait_time=10,
                             wait_time=5,
                             kill_wait_time=5)

        # minidumps not created?
        self.handle_crashes()
        return True
示例#13
0
def get_content(target):
    """
    Get html, css and cookies from the target site
    :param target: the target site
    :return: content_cookies, content_html
    """

    print("{}[-] Retrieving website content {}".format(Bcolors.RESET,
                                                       Bcolors.RESET))
    # create a new profile so as not to mix the user's browsing info with that of the analysis
    profile_conf_name = "/tmp/gdpr-analyzer/gdpr-analyzer.default"
    FirefoxProfile(profile=profile_conf_name)

    # define profile preferences
    browser = Browser('firefox',
                      headless=True,
                      profile=profile_conf_name,
                      timeout=1000,
                      wait_time=200,
                      profile_preferences={"network.cookie.cookieBehavior": 0})

    # navigation run
    with browser:
        browser.visit(target)

        # only gives us first party cookies
        # content_cookies = browser.cookies.all(verbose=True)

        # sad trick shot to access cookies database only work for linux because of path
        paterform = platform.system()
        if paterform == "Darwin":
            profile_repo = glob.glob('/var/folders/sd/*/T/rust_mozprofile*')
        else:
            profile_repo = glob.glob('/tmp/rust_mozprofile*')

        latest_profile_repo = max(profile_repo, key=os.path.getctime)

        # copy database because we can not access to the one which is temporary create
        db_source = latest_profile_repo + "/cookies.sqlite"
        db_destination = "/tmp/gdpr-analyzer/cookies.sqlite"
        shutil.copyfile(db_source, db_destination)

        content_html = browser.html

    # get cookie content from db
    con = sqlite3.connect(db_destination)
    cur = con.cursor()
    cur.execute("SELECT * FROM moz_cookies")
    rows = cur.fetchall()

    content_cookies = []
    for cookie in rows:
        content_cookies.append(cookie)

    con.close()

    print("{}[-] Website content obtained {}".format(Bcolors.GREEN,
                                                     Bcolors.RESET))

    return content_cookies, content_html
示例#14
0
    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#15
0
 def __init__(self, methodName, **kwargs):
     unittest.TestCase.__init__(self, methodName)
     self.loglines = []
     self.duration = 0
     self.expected = kwargs.pop('expected', 'pass')
     self.logger = get_default_logger()
     self.profile = FirefoxProfile()
     self.binary = kwargs.pop('binary', None)
示例#16
0
文件: firefox.py 项目: ZachT1711/wpt
    def create(self):
        """Create a Firefox profile and return the mozprofile Profile object pointing at that
        profile"""
        preferences = self._load_prefs()

        profile = FirefoxProfile(preferences=preferences)
        self._set_required_prefs(profile)
        if self.ca_certificate_path is not None:
            self._setup_ssl(profile)

        return profile
示例#17
0
 def prepare_phone(self, build_metadata):
     prefs = {
         'browser.firstrun.show.localepicker': False,
         'browser.sessionstore.resume_from_crash': False,
         'browser.firstrun.show.uidiscovery': False,
         'shell.checkDefaultClient': False,
         'browser.warnOnQuit': False,
         'browser.EULA.override': True,
         'toolkit.telemetry.prompted': 999,
         'toolkit.telemetry.notifiedOptOut': 999
     }
     profile = FirefoxProfile(preferences=prefs)
     self.install_profile(profile)
示例#18
0
    def create(self, **kwargs):
        """Create a Firefox profile and return the mozprofile Profile object pointing at that
        profile

        :param kwargs: Additional arguments to pass into the profile constructor
        """
        preferences = self._load_prefs()

        profile = FirefoxProfile(preferences=preferences, **kwargs)
        self._set_required_prefs(profile)
        if self.ca_certificate_path is not None:
            self._setup_ssl(profile)

        return profile
示例#19
0
def youtube_video(video_id):
    config["playback_recordings"] = "%s.playback" % video_id
    config["playback_tool_args"] = [
    "--set", "upstream_cert=false",
    "-S", "/tmp/testing/mozproxy/%s.playback" % video_id,
    ]
    proxy = get_playback(config)
    if proxy is None:
        raise Exception("Could not start Proxy")
    try:
        prefs = {"media.autoplay.default": 0}
        prefs["browser.newtabpage.activity-stream.feeds.snippets"] = 0
        prefs["browser.newtabpage.activity-stream.disableSnippets"] = 1
        prefs["network.proxy.type"] = 1
        prefs["network.proxy.http"] = config["host"]
        prefs["network.proxy.http_port"] = 8080
        prefs["network.proxy.ssl"] = config["host"]
        prefs["network.proxy.ssl_port"] = 8080
        prefs["network.proxy.no_proxies_on"] = config["host"]
        profile = FirefoxProfile(profile="/tmp/mozprof", preferences=prefs, addons=[])
        browser = apps["fxdesktop"].create(
            profile=profile, app="fxdesktop", bin=config["binary"],
            app_args=["about:blank"]
        )
        browser.start()
    except Exception:
        proxy.stop()
        raise

    try:
        page = YoutubePage(video_id)
    except Exception:
        try:
            browser.close()
        finally:
            proxy.stop()
        raise

    try:
        yield page
    finally:
        try:
            page.close()
            browser.close()
        finally:
            proxy.stop()
示例#20
0
    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
        self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.enabled": True,
                                      "marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#21
0
    def run(self):
        '''
        Starts Firefox thread with Marionette turned on.
        '''
        
        self.profile = FirefoxProfile()
        self.profile.set_preferences({"marionette.defaultPrefs.enabled" : True,
                                      "marionette.defaultPrefs.port": 2828,
                                      "browser.startup.page": 0,
                                      "browser.startup.homepage": "about:blank",
                                      })
        self.runner = FirefoxRunner(profile = self.profile,
                                    binary = self.binary,
                                    kp_kwargs = {'processOutputLine' : [self.logger]})

        self.runner.start()
        self._firefoxRunningEvent.set()
        self.runner.wait()
示例#22
0
    def create_profile(self, build_metadata, custom_prefs=None):
        # Create, install and initialize the profile to be
        # used in the test.

        telemetry_prompt = 999
        if build_metadata['blddate'] < '2013-01-03':
            telemetry_prompt = 2
        prefs = {
            'browser.firstrun.show.localepicker': False,
            'browser.sessionstore.resume_from_crash': False,
            'dom.ipc.plugins.flash.subprocess.crashreporter.enabled': False,
            'browser.firstrun.show.uidiscovery': False,
            'shell.checkDefaultClient': False,
            'browser.warnOnQuit': False,
            'browser.EULA.override': True,
            'toolkit.telemetry.prompted': telemetry_prompt,
            'toolkit.telemetry.notifiedOptOut': telemetry_prompt,
            'datareporting.healthreport.service.enabled': False,
        }
        if isinstance(custom_prefs, dict):
            prefs = dict(prefs.items() + custom_prefs.items())
        profile = FirefoxProfile(preferences=prefs,
                                 addons='%s/xpi/quitter.xpi' % os.getcwd())
        if not self.install_profile(profile):
            return False

        appname = build_metadata['androidprocname']
        buildid = build_metadata['buildid']
        success = False
        for attempt in range(self.user_cfg[PHONE_RETRY_LIMIT]):
            self.loggerdeco.debug('Attempt %d Initializing profile' % attempt)
            self.run_fennec_with_profile(appname, self._initialize_url)
            if self.wait_for_fennec(build_metadata):
                success = True
                break
            sleep(self.user_cfg[PHONE_RETRY_WAIT])

        if not success:
            msg = 'Failure initializing profile for build %s' % buildid
            self.loggerdeco.error(msg)
            self.set_status(msg=msg)

        return success
示例#23
0
    def install_profile(self, profile=None, root=True):
        if not profile:
            profile = FirefoxProfile()

        profile_path_parent = os.path.split(self.profile_path)[0]
        success = False
        for attempt in range(1, self.options.phone_retry_limit + 1):
            try:
                self.loggerdeco.debug('Attempt %d installing profile', attempt)
                if self.dm.exists(self.profile_path, root=root):
                    # If the profile already exists, chmod it to make sure
                    # we have permission to delete it.
                    self.dm.chmod(self.profile_path, recursive=True, root=root)
                    self.dm.rm(self.profile_path,
                               recursive=True,
                               force=True,
                               root=root)
                self.dm.chmod(profile_path_parent, root=root)
                self.dm.mkdir(self.profile_path, root=root)
                self.dm.chmod(self.profile_path, root=root)
                self.dm.push(profile.profile, self.profile_path)
                self.dm.chmod(self.profile_path, recursive=True, root=root)
                success = True
                break
            except ADBError:
                self.loggerdeco.exception('Attempt %d Exception installing '
                                          'profile to %s' %
                                          (attempt, self.profile_path))
                sleep(self.options.phone_retry_wait)

        if not success:
            self.add_failure(
                self.name, TestStatus.TEST_UNEXPECTED_FAIL,
                'Failure installing profile to %s' % self.profile_path,
                TreeherderStatus.TESTFAILED)

        return success
示例#24
0
    def create_profile(self, custom_addons=[], custom_prefs=None, root=True):
        # Create, install and initialize the profile to be
        # used in the test.

        temp_addons = ['quitter.xpi']
        temp_addons.extend(custom_addons)
        addons = ['%s/xpi/%s' % (os.getcwd(), addon) for addon in temp_addons]

        # make sure firefox isn't running when we try to
        # install the profile.

        self.dm.pkill(self.build.app_name, root=root)
        if isinstance(custom_prefs, dict):
            prefs = dict(self.preferences.items() + custom_prefs.items())
        else:
            prefs = self.preferences
        profile = FirefoxProfile(preferences=prefs, addons=addons)
        if not self.install_profile(profile):
            return False

        success = False
        for attempt in range(1, self.options.phone_retry_limit + 1):
            self.loggerdeco.debug('Attempt %d Initializing profile', attempt)
            self.run_fennec_with_profile(self.build.app_name,
                                         self._initialize_url)

            if self.wait_for_fennec():
                success = True
                break
            sleep(self.options.phone_retry_wait)

        if not success or self.handle_crashes():
            self.add_failure(self.name, TestStatus.TEST_UNEXPECTED_FAIL,
                             'Failure initializing profile',
                             TreeherderStatus.TESTFAILED)

        return success
示例#25
0
    def createprofile(self, targetpath, delete=False):
        if not self.savedpath:
            self.save(targetpath)

        profiledir = "profile" + self._platformsuffix
        profilepath = os.path.join(targetpath, self.parent.version, profiledir)
        if delete and os.path.exists(profilepath):
            shutil.rmtree(profilepath)

        # TODO non-firefox, multiple files
        profileparams = {
            "profile": profilepath,
            "addons": [self.savedpath],
            "preferences": {
                "xpinstall.signatures.required": False,

                # Enable browser toolbox to monitor network requests
                "devtools.chrome.enabled": True,
                "devtools.debugger.remote-enabled": True
            },
            "restore": False
        }
        self.profile = FirefoxProfile(**profileparams)
        return self.profile
示例#26
0
class FirefoxBrowser(Browser):
    used_ports = set()

    def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.marionette_port = None
        self.used_ports.add(self.marionette_port)
        self.runner = None
        self.debug_args = debug_args
        self.interactive = interactive
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        ports = {"http": "8000",
                 "https": "8443",
                 "ws": "8888"}

        self.profile = FirefoxProfile(locations=locations,
                                      proxy=ports,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=self.binary,
                                    cmdargs=[cmd_arg("marionette"), "about:blank"],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            preferences = Preferences.read_prefs(prefs_path)
        else:
            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
            preferences = []

        return preferences

    def stop(self):
        self.logger.debug("Stopping browser")
        if self.runner is not None:
            try:
                self.runner.stop()
            except OSError:
                # This can happen on Windows if the process is already dead
                pass

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        self.logger.process_output(self.pid(),
                                   line.decode("utf8", "replace"),
                                   command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def log_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        mozcrash.log_crashes(self.logger,
                             dump_dir,
                             symbols_path=self.symbols_path,
                             stackwalk_binary=self.stackwalk_binary,
                             process=process,
                             test=test)

    def setup_ssl(self):
        """Create a certificate database to use in the test profile. This is configured
        to trust the CA Certificate that has signed the web-platform.test server
        certificate."""

        self.logger.info("Setting up ssl")

        # Make sure the certutil libraries from the source tree are loaded when using a
        # local copy of certutil
        # TODO: Maybe only set this if certutil won't launch?
        env = os.environ.copy()
        certutil_dir = os.path.dirname(self.binary)
        env["LD_LIBRARY_PATH"] = certutil_dir
        env["PATH"] = os.path.pathsep.join([certutil_dir, env["PATH"]])

        def certutil(*args):
            cmd = [self.certutil_binary] + list(args)
            self.logger.process_output("certutil",
                                       subprocess.check_output(cmd,
                                                               env=env,
                                                               stderr=subprocess.STDOUT),
                                       " ".join(cmd))

        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
        with open(pw_path, "w") as f:
            # Use empty password for certificate db
            f.write("\n")

        cert_db_path = self.profile.profile

        # Create a new certificate db
        certutil("-N", "-d", cert_db_path, "-f", pw_path)

        # Add the CA certificate to the database and mark as trusted to issue server certs
        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
                 "-n", "web-platform-tests", "-i", self.ca_certificate_path)

        # List all certs in the database
        certutil("-L", "-d", cert_db_path)
示例#27
0
class FirefoxAndroidBrowser(FirefoxBrowser):
    init_timeout = 300
    shutdown_timeout = 60

    def __init__(self,
                 logger,
                 prefs_root,
                 test_type,
                 package_name="org.mozilla.geckoview.test",
                 device_serial="emulator-5444",
                 **kwargs):
        FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type,
                                **kwargs)
        self.package_name = package_name
        self.device_serial = device_serial
        self.tests_root = kwargs["tests_root"]
        self.install_fonts = kwargs["install_fonts"]
        self.stackwalk_binary = kwargs["stackwalk_binary"]

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port()

        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.enable_webrender:
            env["MOZ_WEBRENDER"] = "1"
        else:
            env["MOZ_WEBRENDER"] = "0"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port":
            self.marionette_port,
            "network.dns.localDomains":
            ",".join(self.config.domains_set),
            "dom.disable_open_during_load":
            False,
            "places.history.enabled":
            False,
            "dom.send_after_paint_to_content":
            True,
            "network.preload":
            True,
        })

        if self.test_type == "reftest":
            self.logger.info("Setting android reftest preferences")
            self.profile.set_preferences({
                "browser.viewport.desktopWidth":
                800,
                # Disable high DPI
                "layout.css.devPixelsPerPx":
                "1.0",
                # Ensure that the full browser element
                # appears in the screenshot
                "apz.allow_zooming":
                False,
                "android.widget_paints_background":
                False,
                # Ensure that scrollbars are always painted
                "layout.testing.overlay-scrollbars.always-visible":
                True,
            })

        if self.install_fonts:
            self.logger.debug("Copying Ahem font to profile")
            font_dir = os.path.join(self.profile.profile, "fonts")
            if not os.path.exists(font_dir):
                os.makedirs(font_dir)
            with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"),
                      "rb") as src:
                with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
                    dest.write(src.read())

        self.leak_report_file = None

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(
            self.package_name, self.binary_args if self.binary_args else [] +
            [cmd_arg("marionette"), "about:blank"], self.debug_info)

        self.runner = FennecEmulatorRunner(
            app=self.package_name,
            profile=self.profile,
            cmdargs=cmd[1:],
            env=env,
            symbols_path=self.symbols_path,
            serial=self.device_serial,
            # TODO - choose appropriate log dir
            logdir=os.getcwd())

        self.logger.debug("Starting %s" % self.package_name)
        # connect to a running emulator
        self.runner.device.connect()

        self.runner.stop()
        self.runner.start(debug_args=debug_args,
                          interactive=self.debug_info
                          and self.debug_info.interactive)

        self.runner.device.device.forward(
            local="tcp:{}".format(self.marionette_port),
            remote="tcp:{}".format(self.marionette_port))

        for ports in self.config.ports.values():
            for port in ports:
                self.runner.device.device.reverse(local="tcp:{}".format(port),
                                                  remote="tcp:{}".format(port))

        self.logger.debug("%s Started" % self.package_name)

    def stop(self, force=False):
        if self.runner is not None:
            if self.runner.device.connected:
                try:
                    self.runner.device.device.remove_forwards()
                    self.runner.device.device.remove_reverses()
                except Exception as e:
                    self.logger.warning(
                        "Failed to remove forwarded or reversed ports: %s" % e)
            # We assume that stopping the runner prompts the
            # browser to shut down.
            self.runner.stop()
        self.logger.debug("stopped")

    def check_crash(self, process, test):
        if not os.environ.get("MINIDUMP_STACKWALK",
                              "") and self.stackwalk_binary:
            os.environ["MINIDUMP_STACKWALK"] = self.stackwalk_binary
        return bool(self.runner.check_for_crashes(test_name=test))
示例#28
0
class FirefoxBrowser(Browser):
    used_ports = set()
    init_timeout = 60
    shutdown_timeout = 60

    def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None, e10s=False, stackfix_dir=None,
                 binary_args=None, timeout_multiplier=None, leak_check=False, stylo_threads=1,
                 chaos_mode_flags=None, config=None):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.test_type = test_type
        self.extra_prefs = extra_prefs
        self.marionette_port = None
        self.runner = None
        self.debug_info = debug_info
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary
        self.e10s = e10s
        self.binary_args = binary_args
        self.config = config
        if stackfix_dir:
            self.stack_fixer = get_stack_fixer_function(stackfix_dir,
                                                        self.symbols_path)
        else:
            self.stack_fixer = None

        if timeout_multiplier:
            self.init_timeout = self.init_timeout * timeout_multiplier

        self.leak_report_file = None
        self.leak_check = leak_check
        self.stylo_threads = stylo_threads
        self.chaos_mode_flags = chaos_mode_flags

    def settings(self, test):
        return {"check_leaks": self.leak_check and not test.leaks}

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(self.config.domains_set),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True,
                                      "network.preload": True})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs = Preferences()

        pref_paths = []
        prefs_general = os.path.join(self.prefs_root, 'prefs_general.js')
        if os.path.isfile(prefs_general):
            # Old preference file used in Firefox 60 and earlier (remove when no longer supported)
            pref_paths.append(prefs_general)

        profiles = os.path.join(self.prefs_root, 'profiles.json')
        if os.path.isfile(profiles):
            with open(profiles, 'r') as fh:
                for name in json.load(fh)['web-platform-tests']:
                    pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))

        for path in pref_paths:
            if os.path.exists(path):
                prefs.add(Preferences.read_prefs(path))
            else:
                self.logger.warning("Failed to find base prefs file in %s" % path)

        # Add any custom preferences
        prefs.add(self.extra_prefs, cast=True)

        return prefs()

    def stop(self, force=False):
        if self.runner is not None and self.runner.is_running():
            try:
                # For Firefox we assume that stopping the runner prompts the
                # browser to shut down. This allows the leak log to be written
                for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)),
                                      (False, lambda: self.runner.stop(signal.SIGTERM)),
                                      (False, lambda: self.runner.stop(signal.SIGKILL))]:
                    if not force or not clean:
                        retcode = stop_f()
                        if retcode is not None:
                            self.logger.info("Browser exited with return code %s" % retcode)
                            break
            except OSError:
                # This can happen on Windows if the process is already dead
                pass
        self.logger.debug("stopped")

    def process_leaks(self):
        self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
        if self.leak_report_file is None:
            return
        mozleak.process_leak_log(
            self.leak_report_file,
            leak_thresholds={
                "default": 0,
                "tab": 10000,  # See dependencies of bug 1051230.
                # GMP rarely gets a log, but when it does, it leaks a little.
                "geckomediaplugin": 20000,
            },
            ignore_missing_leaks=["geckomediaplugin"],
            log=self.logger,
            stack_fixer=self.stack_fixer
        )

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        if "GLib-GObject-CRITICAL" in line:
            return
        if line:
            data = line.decode("utf8", "replace")
            if self.stack_fixer:
                data = self.stack_fixer(data)
            self.logger.process_output(self.pid(),
                                      data,
                                      command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()
        self.process_leaks()

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def check_for_crashes(self):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        return bool(mozcrash.check_for_crashes(dump_dir,
                                               symbols_path=self.symbols_path,
                                               stackwalk_binary=self.stackwalk_binary,
                                               quiet=True))

    def log_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        mozcrash.log_crashes(self.logger,
                             dump_dir,
                             symbols_path=self.symbols_path,
                             stackwalk_binary=self.stackwalk_binary,
                             process=process,
                             test=test)

    def setup_ssl(self):
        """Create a certificate database to use in the test profile. This is configured
        to trust the CA Certificate that has signed the web-platform.test server
        certificate."""
        if self.certutil_binary is None:
            self.logger.info("--certutil-binary not supplied; Firefox will not check certificates")
            return

        self.logger.info("Setting up ssl")

        # Make sure the certutil libraries from the source tree are loaded when using a
        # local copy of certutil
        # TODO: Maybe only set this if certutil won't launch?
        env = os.environ.copy()
        certutil_dir = os.path.dirname(self.binary)
        if mozinfo.isMac:
            env_var = "DYLD_LIBRARY_PATH"
        elif mozinfo.isUnix:
            env_var = "LD_LIBRARY_PATH"
        else:
            env_var = "PATH"


        env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
                        if env_var in env else certutil_dir).encode(
                            sys.getfilesystemencoding() or 'utf-8', 'replace')

        def certutil(*args):
            cmd = [self.certutil_binary] + list(args)
            self.logger.process_output("certutil",
                                       subprocess.check_output(cmd,
                                                               env=env,
                                                               stderr=subprocess.STDOUT),
                                       " ".join(cmd))

        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
        with open(pw_path, "w") as f:
            # Use empty password for certificate db
            f.write("\n")

        cert_db_path = self.profile.profile

        # Create a new certificate db
        certutil("-N", "-d", cert_db_path, "-f", pw_path)

        # Add the CA certificate to the database and mark as trusted to issue server certs
        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
                 "-n", "web-platform-tests", "-i", self.ca_certificate_path)

        # List all certs in the database
        certutil("-L", "-d", cert_db_path)
    def valgrind_test(self, suppressions):

        from mozfile import TemporaryDirectory
        from mozhttpd import MozHttpd
        from mozprofile import FirefoxProfile, Preferences
        from mozprofile.permissions import ServerLocations
        from mozrunner import FirefoxRunner
        from mozrunner.utils import findInPath
        from six import string_types
        from valgrind.output_handler import OutputHandler

        build_dir = os.path.join(self.topsrcdir, "build")

        # XXX: currently we just use the PGO inputs for Valgrind runs.  This may
        # change in the future.
        httpd = MozHttpd(docroot=os.path.join(build_dir, "pgo"))
        httpd.start(block=False)

        with TemporaryDirectory() as profilePath:
            # TODO: refactor this into mozprofile
            profile_data_dir = os.path.join(self.topsrcdir, "testing",
                                            "profiles")
            with open(os.path.join(profile_data_dir, "profiles.json"),
                      "r") as fh:
                base_profiles = json.load(fh)["valgrind"]

            prefpaths = [
                os.path.join(profile_data_dir, profile, "user.js")
                for profile in base_profiles
            ]
            prefs = {}
            for path in prefpaths:
                prefs.update(Preferences.read_prefs(path))

            interpolation = {
                "server": "%s:%d" % httpd.httpd.server_address,
            }
            for k, v in prefs.items():
                if isinstance(v, string_types):
                    v = v.format(**interpolation)
                prefs[k] = Preferences.cast(v)

            quitter = os.path.join(self.topsrcdir, "tools", "quitter",
                                   "*****@*****.**")

            locations = ServerLocations()
            locations.add_host(host="127.0.0.1",
                               port=httpd.httpd.server_port,
                               options="primary")

            profile = FirefoxProfile(
                profile=profilePath,
                preferences=prefs,
                addons=[quitter],
                locations=locations,
            )

            firefox_args = [httpd.get_url()]

            env = os.environ.copy()
            env["G_SLICE"] = "always-malloc"
            env["MOZ_CC_RUN_DURING_SHUTDOWN"] = "1"
            env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
            env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
            env["XPCOM_DEBUG_BREAK"] = "warn"

            outputHandler = OutputHandler(self.log)
            kp_kwargs = {
                "processOutputLine": [outputHandler],
                "universal_newlines": True,
            }

            valgrind = "valgrind"
            if not os.path.exists(valgrind):
                valgrind = findInPath(valgrind)

            valgrind_args = [
                valgrind,
                "--sym-offsets=yes",
                "--smc-check=all-non-file",
                "--vex-iropt-register-updates=allregs-at-mem-access",
                "--gen-suppressions=all",
                "--num-callers=36",
                "--leak-check=full",
                "--show-possibly-lost=no",
                "--track-origins=yes",
                "--trace-children=yes",
                "-v",  # Enable verbosity to get the list of used suppressions
                # Avoid excessive delays in the presence of spinlocks.
                # See bug 1309851.
                "--fair-sched=yes",
                # Keep debuginfo after library unmap.  See bug 1382280.
                "--keep-debuginfo=yes",
                # Reduce noise level on rustc and/or LLVM compiled code.
                # See bug 1365915
                "--expensive-definedness-checks=yes",
                # Compensate for the compiler inlining `new` but not `delete`
                # or vice versa.
                "--show-mismatched-frees=no",
            ]

            for s in suppressions:
                valgrind_args.append("--suppressions=" + s)

            supps_dir = os.path.join(build_dir, "valgrind")
            supps_file1 = os.path.join(supps_dir, "cross-architecture.sup")
            valgrind_args.append("--suppressions=" + supps_file1)

            if mozinfo.os == "linux":
                machtype = {
                    "x86_64": "x86_64-pc-linux-gnu",
                    "x86": "i386-pc-linux-gnu",
                }.get(mozinfo.processor)
                if machtype:
                    supps_file2 = os.path.join(supps_dir, machtype + ".sup")
                    if os.path.isfile(supps_file2):
                        valgrind_args.append("--suppressions=" + supps_file2)

            exitcode = None
            timeout = 1800
            binary_not_found_exception = None
            try:
                runner = FirefoxRunner(
                    profile=profile,
                    binary=self.get_binary_path(),
                    cmdargs=firefox_args,
                    env=env,
                    process_args=kp_kwargs,
                )
                runner.start(debug_args=valgrind_args)
                exitcode = runner.wait(timeout=timeout)
            except BinaryNotFoundException as e:
                binary_not_found_exception = e
            finally:
                errs = outputHandler.error_count
                supps = outputHandler.suppression_count
                if errs != supps:
                    status = 1  # turns the TBPL job orange
                    self.log(
                        logging.ERROR,
                        "valgrind-fail-parsing",
                        {
                            "errs": errs,
                            "supps": supps
                        },
                        "TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors "
                        "seen, but {supps} generated suppressions seen",
                    )

                elif errs == 0:
                    status = 0
                    self.log(
                        logging.INFO,
                        "valgrind-pass",
                        {},
                        "TEST-PASS | valgrind-test | valgrind found no errors",
                    )
                else:
                    status = 1  # turns the TBPL job orange
                    # We've already printed details of the errors.

                if binary_not_found_exception:
                    status = 2  # turns the TBPL job red
                    self.log(
                        logging.ERROR,
                        "valgrind-fail-errors",
                        {"error": str(binary_not_found_exception)},
                        "TEST-UNEXPECTED-FAIL | valgrind-test | {error}",
                    )
                    self.log(
                        logging.INFO,
                        "valgrind-fail-errors",
                        {"help": binary_not_found_exception.help()},
                        "{help}",
                    )
                elif exitcode is None:
                    status = 2  # turns the TBPL job red
                    self.log(
                        logging.ERROR,
                        "valgrind-fail-timeout",
                        {"timeout": timeout},
                        "TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out "
                        "(reached {timeout} second limit)",
                    )
                elif exitcode != 0:
                    status = 2  # turns the TBPL job red
                    self.log(
                        logging.ERROR,
                        "valgrind-fail-errors",
                        {"exitcode": exitcode},
                        "TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code "
                        "from Valgrind: {exitcode}",
                    )

                httpd.stop()

            return status
                     for profile in base_profiles]

        prefs = {}
        for path in prefpaths:
            prefs.update(Preferences.read_prefs(path))

        interpolation = {"server": "%s:%d" % httpd.httpd.server_address,
                         "OOP": "false"}
        for k, v in prefs.items():
            if isinstance(v, string_types):
                v = v.format(**interpolation)
            prefs[k] = Preferences.cast(v)

        profile = FirefoxProfile(profile=profilePath,
                                 preferences=prefs,
                                 addons=[os.path.join(
                                     build.topsrcdir, 'tools', 'quitter',
                                     '*****@*****.**')],
                                 locations=locations)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["XPCOM_DEBUG_BREAK"] = "warn"

        # For VC12+, make sure we can find the right bitness of pgort1x0.dll
        if not substs.get('HAVE_64BIT_BUILD'):
            for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
                if e not in env:
                    continue

                vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
                if os.path.exists(vcdir):
示例#31
0
class FirefoxBrowser(Browser):
    used_ports = set()

    def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
                 symbols_path=None, stackwalk_binary=None):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.marionette_port = None
        self.used_ports.add(self.marionette_port)
        self.runner = None
        self.debug_args = debug_args
        self.interactive = interactive
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        ports = {"http": "8000",
                 "https": "8443",
                 "ws": "8888"}

        self.profile = FirefoxProfile(locations=locations,
                                      proxy=ports,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False})

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=self.binary,
                                    cmdargs=[cmd_arg("marionette"), "about:blank"],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")
        self.runner.start(debug_args=self.debug_args, interactive=self.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            preferences = Preferences.read_prefs(prefs_path)
        else:
            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
            preferences = []

        return preferences

    def stop(self):
        self.logger.debug("Stopping browser")
        if self.runner is not None:
            try:
                self.runner.stop()
            except OSError:
                # This can happen on Windows if the process is already dead
                pass

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        self.logger.process_output(self.pid(),
                                   line.decode("utf8", "replace"),
                                   command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def log_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")
        mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path,
                             stackwalk_binary=self.stackwalk_binary,
                             process=process, test=test)
示例#32
0
    try:
        #TODO: refactor this into mozprofile
        prefpath = os.path.join(build.topsrcdir, "testing", "profiles",
                                "prefs_general.js")
        prefs = {}
        prefs.update(Preferences.read_prefs(prefpath))
        interpolation = {
            "server": "%s:%d" % httpd.httpd.server_address,
            "OOP": "false"
        }
        prefs = json.loads(json.dumps(prefs) % interpolation)
        for pref in prefs:
            prefs[pref] = Preferences.cast(prefs[pref])
        profile = FirefoxProfile(
            profile=profilePath,
            preferences=prefs,
            addons=[os.path.join(build.distdir, 'xpi-stage', 'quitter')],
            locations=locations)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
        env["XPCOM_DEBUG_BREAK"] = "warn"
        jarlog = os.getenv("JARLOG_FILE")
        if jarlog:
            env["MOZ_JAR_LOG_FILE"] = os.path.abspath(jarlog)
            print "jarlog: %s" % env["MOZ_JAR_LOG_FILE"]

        cmdargs = ["http://localhost:%d/index.html" % PORT]
        runner = FirefoxRunner(
            profile=profile,
            binary=build.get_binary_path(where="staged-package"),
示例#33
0
class FirefoxBrowser(Browser):
    used_ports = set()
    init_timeout = 60
    shutdown_timeout = 60

    def __init__(self,
                 logger,
                 binary,
                 prefs_root,
                 test_type,
                 extra_prefs=None,
                 debug_info=None,
                 symbols_path=None,
                 stackwalk_binary=None,
                 certutil_binary=None,
                 ca_certificate_path=None,
                 e10s=False,
                 stackfix_dir=None,
                 binary_args=None,
                 timeout_multiplier=None,
                 leak_check=False,
                 stylo_threads=1,
                 chaos_mode_flags=None):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.test_type = test_type
        self.extra_prefs = extra_prefs
        self.marionette_port = None
        self.runner = None
        self.debug_info = debug_info
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary
        self.e10s = e10s
        self.binary_args = binary_args
        if stackfix_dir:
            self.stack_fixer = get_stack_fixer_function(
                stackfix_dir, self.symbols_path)
        else:
            self.stack_fixer = None

        if timeout_multiplier:
            self.init_timeout = self.init_timeout * timeout_multiplier

        self.leak_report_file = None
        self.leak_check = leak_check
        self.stylo_threads = stylo_threads
        self.chaos_mode_flags = chaos_mode_flags

    def settings(self, test):
        return {"check_leaks": self.leak_check and not test.leaks}

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        locations = ServerLocations(
            filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({
            "marionette.port":
            self.marionette_port,
            "dom.disable_open_during_load":
            False,
            "network.dns.localDomains":
            ",".join(hostnames),
            "network.proxy.type":
            0,
            "places.history.enabled":
            False,
            "dom.send_after_paint_to_content":
            True,
            "network.preload":
            True
        })
        if self.e10s:
            self.profile.set_preferences(
                {"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences(
                {"layout.interruptible-reflow.enabled": False})

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile,
                                                 "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft")
                and '5.1' in platform.version()):
            self.profile.set_preferences(
                {"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(
            self.binary, self.binary_args if self.binary_args else [] +
            [cmd_arg("marionette"), "about:blank"], self.debug_info)

        self.runner = FirefoxRunner(
            profile=self.profile,
            binary=cmd[0],
            cmdargs=cmd[1:],
            env=env,
            process_class=ProcessHandler,
            process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args,
                          interactive=self.debug_info
                          and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs = Preferences()

        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            prefs.add(Preferences.read_prefs(prefs_path))
        else:
            self.logger.warning("Failed to find base prefs file in %s" %
                                prefs_path)

        # Add any custom preferences
        prefs.add(self.extra_prefs, cast=True)

        return prefs()

    def stop(self, force=False):
        if self.runner is not None and self.runner.is_running():
            try:
                # For Firefox we assume that stopping the runner prompts the
                # browser to shut down. This allows the leak log to be written
                for clean, stop_f in [
                    (True, lambda: self.runner.wait(self.shutdown_timeout)),
                    (False, lambda: self.runner.stop(signal.SIGTERM)),
                    (False, lambda: self.runner.stop(signal.SIGKILL))
                ]:
                    if not force or not clean:
                        retcode = stop_f()
                        if retcode is not None:
                            self.logger.info(
                                "Browser exited with return code %s" % retcode)
                            break
            except OSError:
                # This can happen on Windows if the process is already dead
                pass
        self.logger.debug("stopped")

    def process_leaks(self):
        self.logger.debug("PROCESS LEAKS %s" % self.leak_report_file)
        if self.leak_report_file is None:
            return
        mozleak.process_leak_log(
            self.leak_report_file,
            leak_thresholds={
                "default": 0,
                "tab": 10000,  # See dependencies of bug 1051230.
                # GMP rarely gets a log, but when it does, it leaks a little.
                "geckomediaplugin": 20000,
            },
            ignore_missing_leaks=["geckomediaplugin"],
            log=self.logger,
            stack_fixer=self.stack_fixer)

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        data = line.decode("utf8", "replace")
        if self.stack_fixer:
            data = self.stack_fixer(data)
        self.logger.process_output(self.pid(),
                                   data,
                                   command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()
        self.process_leaks()

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def check_for_crashes(self):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        return bool(
            mozcrash.check_for_crashes(dump_dir,
                                       symbols_path=self.symbols_path,
                                       stackwalk_binary=self.stackwalk_binary,
                                       quiet=True))

    def log_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        mozcrash.log_crashes(self.logger,
                             dump_dir,
                             symbols_path=self.symbols_path,
                             stackwalk_binary=self.stackwalk_binary,
                             process=process,
                             test=test)

    def setup_ssl(self):
        """Create a certificate database to use in the test profile. This is configured
        to trust the CA Certificate that has signed the web-platform.test server
        certificate."""
        if self.certutil_binary is None:
            self.logger.info(
                "--certutil-binary not supplied; Firefox will not check certificates"
            )
            return

        self.logger.info("Setting up ssl")

        # Make sure the certutil libraries from the source tree are loaded when using a
        # local copy of certutil
        # TODO: Maybe only set this if certutil won't launch?
        env = os.environ.copy()
        certutil_dir = os.path.dirname(self.binary)
        if mozinfo.isMac:
            env_var = "DYLD_LIBRARY_PATH"
        elif mozinfo.isUnix:
            env_var = "LD_LIBRARY_PATH"
        else:
            env_var = "PATH"

        env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
                        if env_var in env else certutil_dir).encode(
                            sys.getfilesystemencoding() or 'utf-8', 'replace')

        def certutil(*args):
            cmd = [self.certutil_binary] + list(args)
            self.logger.process_output(
                "certutil",
                subprocess.check_output(cmd, env=env,
                                        stderr=subprocess.STDOUT),
                " ".join(cmd))

        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
        with open(pw_path, "w") as f:
            # Use empty password for certificate db
            f.write("\n")

        cert_db_path = self.profile.profile

        # Create a new certificate db
        certutil("-N", "-d", cert_db_path, "-f", pw_path)

        # Add the CA certificate to the database and mark as trusted to issue server certs
        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,", "-n",
                 "web-platform-tests", "-i", self.ca_certificate_path)

        # List all certs in the database
        certutil("-L", "-d", cert_db_path)
示例#34
0
    def valgrind_test(self, suppressions):

        from mozfile import TemporaryDirectory
        from mozhttpd import MozHttpd
        from mozprofile import FirefoxProfile, Preferences
        from mozprofile.permissions import ServerLocations
        from mozrunner import FirefoxRunner
        from mozrunner.utils import findInPath
        from six import string_types
        from valgrind.output_handler import OutputHandler

        build_dir = os.path.join(self.topsrcdir, 'build')

        # XXX: currently we just use the PGO inputs for Valgrind runs.  This may
        # change in the future.
        httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo'))
        httpd.start(block=False)

        with TemporaryDirectory() as profilePath:
            # TODO: refactor this into mozprofile
            profile_data_dir = os.path.join(self.topsrcdir, 'testing',
                                            'profiles')
            with open(os.path.join(profile_data_dir, 'profiles.json'),
                      'r') as fh:
                base_profiles = json.load(fh)['valgrind']

            prefpaths = [
                os.path.join(profile_data_dir, profile, 'user.js')
                for profile in base_profiles
            ]
            prefs = {}
            for path in prefpaths:
                prefs.update(Preferences.read_prefs(path))

            interpolation = {
                'server': '%s:%d' % httpd.httpd.server_address,
            }
            for k, v in prefs.items():
                if isinstance(v, string_types):
                    v = v.format(**interpolation)
                prefs[k] = Preferences.cast(v)

            quitter = os.path.join(self.topsrcdir, 'tools', 'quitter',
                                   '*****@*****.**')

            locations = ServerLocations()
            locations.add_host(host='127.0.0.1',
                               port=httpd.httpd.server_port,
                               options='primary')

            profile = FirefoxProfile(profile=profilePath,
                                     preferences=prefs,
                                     addons=[quitter],
                                     locations=locations)

            firefox_args = [httpd.get_url()]

            env = os.environ.copy()
            env['G_SLICE'] = 'always-malloc'
            env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
            env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
            env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
            env['XPCOM_DEBUG_BREAK'] = 'warn'

            env.update(self.extra_environment_variables)

            outputHandler = OutputHandler(self.log)
            kp_kwargs = {'processOutputLine': [outputHandler]}

            valgrind = 'valgrind'
            if not os.path.exists(valgrind):
                valgrind = findInPath(valgrind)

            valgrind_args = [
                valgrind,
                '--sym-offsets=yes',
                '--smc-check=all-non-file',
                '--vex-iropt-register-updates=allregs-at-mem-access',
                '--gen-suppressions=all',
                '--num-callers=36',
                '--leak-check=full',
                '--show-possibly-lost=no',
                '--track-origins=yes',
                '--trace-children=yes',
                '-v',  # Enable verbosity to get the list of used suppressions
                # Avoid excessive delays in the presence of spinlocks.
                # See bug 1309851.
                '--fair-sched=yes',
                # Keep debuginfo after library unmap.  See bug 1382280.
                '--keep-debuginfo=yes',
                # Reduce noise level on rustc and/or LLVM compiled code.
                # See bug 1365915
                '--expensive-definedness-checks=yes',
            ]

            for s in suppressions:
                valgrind_args.append('--suppressions=' + s)

            supps_dir = os.path.join(build_dir, 'valgrind')
            supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup')
            valgrind_args.append('--suppressions=' + supps_file1)

            if mozinfo.os == 'linux':
                machtype = {
                    'x86_64': 'x86_64-pc-linux-gnu',
                    'x86': 'i386-pc-linux-gnu',
                }.get(mozinfo.processor)
                if machtype:
                    supps_file2 = os.path.join(supps_dir, machtype + '.sup')
                    if os.path.isfile(supps_file2):
                        valgrind_args.append('--suppressions=' + supps_file2)

            exitcode = None
            timeout = 1800
            try:
                runner = FirefoxRunner(profile=profile,
                                       binary=self.get_binary_path(),
                                       cmdargs=firefox_args,
                                       env=env,
                                       process_args=kp_kwargs)
                runner.start(debug_args=valgrind_args)
                exitcode = runner.wait(timeout=timeout)

            finally:
                errs = outputHandler.error_count
                supps = outputHandler.suppression_count
                if errs != supps:
                    status = 1  # turns the TBPL job orange
                    self.log(
                        logging.ERROR, 'valgrind-fail-parsing', {
                            'errs': errs,
                            'supps': supps
                        },
                        'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors '
                        'seen, but {supps} generated suppressions seen')

                elif errs == 0:
                    status = 0
                    self.log(
                        logging.INFO, 'valgrind-pass', {},
                        'TEST-PASS | valgrind-test | valgrind found no errors')
                else:
                    status = 1  # turns the TBPL job orange
                    # We've already printed details of the errors.

                if exitcode is None:
                    status = 2  # turns the TBPL job red
                    self.log(
                        logging.ERROR, 'valgrind-fail-timeout',
                        {'timeout': timeout},
                        'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out '
                        '(reached {timeout} second limit)')
                elif exitcode != 0:
                    status = 2  # turns the TBPL job red
                    self.log(
                        logging.ERROR, 'valgrind-fail-errors',
                        {'exitcode': exitcode},
                        'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code '
                        'from Valgrind: {exitcode}')

                httpd.stop()

            return status
示例#35
0
class FennecBrowser(FirefoxBrowser):
    used_ports = set()
    init_timeout = 300
    shutdown_timeout = 60

    def __init__(self, logger, prefs_root, test_type, package_name=None,
                 device_serial="emulator-5444", **kwargs):
        FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
        self._package_name = package_name
        self.device_serial = device_serial
        self.tests_root = kwargs["tests_root"]
        self.install_fonts = kwargs["install_fonts"]

    @property
    def package_name(self):
        """
        Name of app to run on emulator.
        """
        if self._package_name is None:
            self._package_name = "org.mozilla.fennec"
            user = os.getenv("USER")
            if user:
                self._package_name += "_" + user
        return self._package_name

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True,
                                      "network.preload": True})
        if self.test_type == "reftest":
            self.logger.info("Setting android reftest preferences")
            self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
                                          # Disable high DPI
                                          "layout.css.devPixelsPerPx": "1.0",
                                          # Ensure that the full browser element
                                          # appears in the screenshot
                                          "apz.allow_zooming": False,
                                          "android.widget_paints_background": False,
                                          # Ensure that scrollbars are always painted
                                          "ui.scrollbarFadeBeginDelay": 100000})

        if self.install_fonts:
            self.logger.debug("Copying Ahem font to profile")
            font_dir = os.path.join(self.profile.profile, "fonts")
            if not os.path.exists(font_dir):
                os.makedirs(font_dir)
            with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
                with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
                    dest.write(src.read())

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.package_name,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FennecEmulatorRunner(app=self.package_name,
                                           profile=self.profile,
                                           cmdargs=cmd[1:],
                                           env=env,
                                           symbols_path=self.symbols_path,
                                           serial=self.device_serial,
                                           # TODO - choose appropriate log dir
                                           logdir=os.getcwd(),
                                           process_class=ProcessHandler,
                                           process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting %s" % self.package_name)
        # connect to a running emulator
        self.runner.device.connect()

        write_hosts_file(self.config, self.runner.device.device)

        self.runner.stop()
        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)

        self.runner.device.device.forward(
            local="tcp:{}".format(self.marionette_port),
            remote="tcp:{}".format(self.marionette_port))

        self.logger.debug("%s Started" % self.package_name)

    def stop(self, force=False):
        if self.runner is not None:
            if (self.runner.device.connected and
                len(self.runner.device.device.list_forwards()) > 0):
                try:
                    self.runner.device.device.remove_forwards(
                        "tcp:{}".format(self.marionette_port))
                except Exception:
                    self.logger.warning("Failed to remove port forwarding")
            # We assume that stopping the runner prompts the
            # browser to shut down. This allows the leak log to be written
            self.runner.stop()
        self.logger.debug("stopped")
示例#36
0
def run():
    parser = argparse.ArgumentParser(description="Run crawler")
    parser.add_argument("-b", "--binary", type=str, help="path to the Firefox binary")
    parser.add_argument("-a", "--abpdir", type=str, help="path to the Adblock Plus repository")
    parser.add_argument(
        "-f",
        "--filters",
        metavar="url",
        type=str,
        nargs="+",
        default=[
            "https://easylist-downloads.adblockplus.org/easylist.txt",
            "https://easylist-downloads.adblockplus.org/exceptionrules.txt",
        ],
        help="filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.",
    )
    parser.add_argument("-t", "--timeout", type=int, default=300, help="Load timeout (seconds)")
    parser.add_argument("-x", "--maxtabs", type=int, default=15, help="Maximal number of tabs to open in parallel")
    parser.add_argument("list", type=str, help="URL list to process")
    parser.add_argument("outdir", type=str, help="directory to write data into")
    parameters = parser.parse_args()

    import buildtools.packagerGecko as packager

    cleanup = []
    try:
        base_dir = os.path.dirname(os.path.abspath(__file__))
        handle, crawlerxpi = tempfile.mkstemp(suffix=".xpi")
        os.close(handle)
        cleanup.append(crawlerxpi)
        packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)

        abpxpi = "https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi"
        if parameters.abpdir:
            handle, abpxpi = tempfile.mkstemp(suffix=".xpi")
            os.close(handle)
            cleanup.append(abpxpi)
            packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True)

        profile = FirefoxProfile(
            addons=[crawlerxpi, abpxpi],
            preferences={
                "browser.startup.homepage": "about:blank",
                "browser.tabs.warnOnCloseOtherTabs": False,
                "browser.uitour.enabled": False,
                "prompts.tab_modal.enabled": False,
                "startup.homepage_welcome_url": "about:blank",
                "startup.homepage_welcome_url.additional": "about:blank",
                "xpinstall.signatures.required": False,
            },
        )

        abpsettings = os.path.join(profile.profile, "adblockplus")
        os.makedirs(abpsettings)
        with open(os.path.join(abpsettings, "patterns.ini"), "w") as handle:
            print >> handle, "# Adblock Plus preferences"
            print >> handle, "version=4"
            for url in parameters.filters:
                if "=" in url:
                    path, url = url.split("=", 1)
                    with open(path, "r") as source:
                        data = source.read()
                else:
                    data = urllib.urlopen(url).read()
                print >> handle, "[Subscription]"
                print >> handle, "url=%s" % url
                print >> handle, "[Subscription filters]"
                print >> handle, "\n".join(data.splitlines()[1:])
    finally:
        for path in cleanup:
            os.unlink(path)

    server = None
    try:
        port = random.randrange(2000, 60000)
        print "Communicating with client on port %i" % port

        app = CrawlerApp(parameters)
        server = make_server("localhost", port, app)
        app.server = server
        threading.Thread(target=lambda: server.serve_forever()).start()

        runner = FirefoxRunner(
            profile=profile,
            binary=parameters.binary,
            cmdargs=["--crawler-port", str(port)],
            env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE="1"),
        )
        while app.urls:
            runner.start()
            runner.wait()
    finally:
        if server:
            server.shutdown()
        profile.cleanup()
示例#37
0
    def start(self, **kwargs):
        if self.marionette_port is None:
            with FennecBrowser.used_ports_lock:
                self.marionette_port = get_free_port(2828, exclude=self.used_ports)
                self.used_ports.add(self.marionette_port)

        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True,
                                      "network.preload": True})
        if self.test_type == "reftest":
            self.logger.info("Setting android reftest preferences")
            self.profile.set_preferences({"browser.viewport.desktopWidth": 800,
                                          # Disable high DPI
                                          "layout.css.devPixelsPerPx": "1.0",
                                          # Ensure that the full browser element
                                          # appears in the screenshot
                                          "apz.allow_zooming": False,
                                          "android.widget_paints_background": False,
                                          # Ensure that scrollbars are always painted
                                          "layout.testing.overlay-scrollbars.always-visible": True})

        if self.install_fonts:
            self.logger.debug("Copying Ahem font to profile")
            font_dir = os.path.join(self.profile.profile, "fonts")
            if not os.path.exists(font_dir):
                os.makedirs(font_dir)
            with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
                with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
                    dest.write(src.read())

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.package_name,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FennecEmulatorRunner(app=self.package_name,
                                           profile=self.profile,
                                           cmdargs=cmd[1:],
                                           env=env,
                                           symbols_path=self.symbols_path,
                                           serial=self.device_serial,
                                           # TODO - choose appropriate log dir
                                           logdir=os.getcwd(),
                                           process_class=ProcessHandler,
                                           process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting %s" % self.package_name)
        # connect to a running emulator
        self.runner.device.connect()

        write_hosts_file(self.config, self.runner.device.device)

        self.runner.stop()
        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)

        self.runner.device.device.forward(
            local="tcp:{}".format(self.marionette_port),
            remote="tcp:{}".format(self.marionette_port))

        self.logger.debug("%s Started" % self.package_name)
示例#38
0
    def start(self, group_metadata=None, **kwargs):
        if group_metadata is None:
            group_metadata = {}

        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        if self.asan:
            print "Setting up LSAN"
            self.lsan_handler = mozleak.LSANLeaks(self.logger,
                                                  scope=group_metadata.get("scope", "/"),
                                                  allowed=self.lsan_allowed,
                                                  maxNumRecordedFrames=self.lsan_max_stack_depth)

        env = test_environment(xrePath=os.path.dirname(self.binary),
                               debugger=self.debug_info is not None,
                               log=self.logger,
                               lsanPath=self.prefs_root)

        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.headless:
            env["MOZ_HEADLESS"] = "1"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port": self.marionette_port,
            "network.dns.localDomains": ",".join(self.config.domains_set),
            "dom.file.createInChild": True,
            # TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
            "network.proxy.type": 0,
            "places.history.enabled": False,
            "network.preload": True,
        })
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})

        if self.leak_check:
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        args = self.binary_args[:] if self.binary_args else []
        args += [cmd_arg("marionette"), "about:blank"]

        debug_args, cmd = browser_command(self.binary,
                                          args,
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#39
0
    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port()

        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.enable_webrender:
            env["MOZ_WEBRENDER"] = "1"
        else:
            env["MOZ_WEBRENDER"] = "0"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port":
            self.marionette_port,
            "network.dns.localDomains":
            ",".join(self.config.domains_set),
            "dom.disable_open_during_load":
            False,
            "places.history.enabled":
            False,
            "dom.send_after_paint_to_content":
            True,
            "network.preload":
            True,
        })

        if self.test_type == "reftest":
            self.logger.info("Setting android reftest preferences")
            self.profile.set_preferences({
                "browser.viewport.desktopWidth":
                800,
                # Disable high DPI
                "layout.css.devPixelsPerPx":
                "1.0",
                # Ensure that the full browser element
                # appears in the screenshot
                "apz.allow_zooming":
                False,
                "android.widget_paints_background":
                False,
                # Ensure that scrollbars are always painted
                "layout.testing.overlay-scrollbars.always-visible":
                True,
            })

        if self.install_fonts:
            self.logger.debug("Copying Ahem font to profile")
            font_dir = os.path.join(self.profile.profile, "fonts")
            if not os.path.exists(font_dir):
                os.makedirs(font_dir)
            with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"),
                      "rb") as src:
                with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
                    dest.write(src.read())

        self.leak_report_file = None

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(
            self.package_name, self.binary_args if self.binary_args else [] +
            [cmd_arg("marionette"), "about:blank"], self.debug_info)

        self.runner = FennecEmulatorRunner(
            app=self.package_name,
            profile=self.profile,
            cmdargs=cmd[1:],
            env=env,
            symbols_path=self.symbols_path,
            serial=self.device_serial,
            # TODO - choose appropriate log dir
            logdir=os.getcwd())

        self.logger.debug("Starting %s" % self.package_name)
        # connect to a running emulator
        self.runner.device.connect()

        self.runner.stop()
        self.runner.start(debug_args=debug_args,
                          interactive=self.debug_info
                          and self.debug_info.interactive)

        self.runner.device.device.forward(
            local="tcp:{}".format(self.marionette_port),
            remote="tcp:{}".format(self.marionette_port))

        for ports in self.config.ports.values():
            for port in ports:
                self.runner.device.device.reverse(local="tcp:{}".format(port),
                                                  remote="tcp:{}".format(port))

        self.logger.debug("%s Started" % self.package_name)
示例#40
0
文件: firefox.py 项目: DJMcNab/servo
    def start(self, group_metadata=None, **kwargs):
        if group_metadata is None:
            group_metadata = {}

        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        if self.asan:
            print "Setting up LSAN"
            self.lsan_handler = mozleak.LSANLeaks(
                self.logger,
                scope=group_metadata.get("scope", "/"),
                allowed=self.lsan_allowed,
                maxNumRecordedFrames=self.lsan_max_stack_depth)

        env = test_environment(xrePath=os.path.dirname(self.binary),
                               debugger=self.debug_info is not None,
                               log=self.logger,
                               lsanPath=self.prefs_root)

        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.headless:
            env["MOZ_HEADLESS"] = "1"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port":
            self.marionette_port,
            "network.dns.localDomains":
            ",".join(self.config.domains_set),
            "dom.file.createInChild":
            True,
            # TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
            "network.proxy.type":
            0,
            "places.history.enabled":
            False,
            "network.preload":
            True,
        })
        if self.e10s:
            self.profile.set_preferences(
                {"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences(
                {"layout.interruptible-reflow.enabled": False})

        if self.leak_check:
            self.leak_report_file = os.path.join(
                self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft")
                and '5.1' in platform.version()):
            self.profile.set_preferences(
                {"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        args = self.binary_args[:] if self.binary_args else []
        args += [cmd_arg("marionette"), "about:blank"]

        debug_args, cmd = browser_command(self.binary, args, self.debug_info)

        self.runner = FirefoxRunner(
            profile=self.profile,
            binary=cmd[0],
            cmdargs=cmd[1:],
            env=env,
            process_class=ProcessHandler,
            process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args,
                          interactive=self.debug_info
                          and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#41
0
class FirefoxBrowser(Browser):
    init_timeout = 70
    shutdown_timeout = 70

    def __init__(self, logger, binary, prefs_root, test_type, extra_prefs=None, debug_info=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None, e10s=False, lsan_dir=None, stackfix_dir=None,
                 binary_args=None, timeout_multiplier=None, leak_check=False, asan=False,
                 stylo_threads=1, chaos_mode_flags=None, config=None, browser_channel="nightly", headless=None, **kwargs):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.test_type = test_type
        self.extra_prefs = extra_prefs
        self.marionette_port = None
        self.runner = None
        self.debug_info = debug_info
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary
        self.e10s = e10s
        self.binary_args = binary_args
        self.config = config
        if stackfix_dir:
            self.stack_fixer = get_stack_fixer_function(stackfix_dir,
                                                        self.symbols_path)
        else:
            self.stack_fixer = None

        if timeout_multiplier:
            self.init_timeout = self.init_timeout * timeout_multiplier

        self.asan = asan
        self.lsan_dir = lsan_dir
        self.lsan_allowed = None
        self.lsan_max_stack_depth = None
        self.mozleak_allowed = None
        self.mozleak_thresholds = None
        self.leak_check = leak_check
        self.leak_report_file = None
        self.lsan_handler = None
        self.stylo_threads = stylo_threads
        self.chaos_mode_flags = chaos_mode_flags
        self.browser_channel = browser_channel
        self.headless = headless

    def settings(self, test):
        return {"check_leaks": self.leak_check and not test.leaks,
                "lsan_allowed": test.lsan_allowed,
                "lsan_max_stack_depth": test.lsan_max_stack_depth,
                "mozleak_allowed": self.leak_check and test.mozleak_allowed,
                "mozleak_thresholds": self.leak_check and test.mozleak_threshold}

    def start(self, group_metadata=None, **kwargs):
        if group_metadata is None:
            group_metadata = {}

        self.group_metadata = group_metadata
        self.lsan_allowed = kwargs.get("lsan_allowed")
        self.lsan_max_stack_depth = kwargs.get("lsan_max_stack_depth")
        self.mozleak_allowed = kwargs.get("mozleak_allowed")
        self.mozleak_thresholds = kwargs.get("mozleak_thresholds")

        if self.marionette_port is None:
            self.marionette_port = get_free_port()

        if self.asan:
            self.lsan_handler = mozleak.LSANLeaks(self.logger,
                                                  scope=group_metadata.get("scope", "/"),
                                                  allowed=self.lsan_allowed,
                                                  maxNumRecordedFrames=self.lsan_max_stack_depth)

        env = test_environment(xrePath=os.path.dirname(self.binary),
                               debugger=self.debug_info is not None,
                               log=self.logger,
                               lsanPath=self.lsan_dir)

        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)
        if self.headless:
            env["MOZ_HEADLESS"] = "1"

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({
            "marionette.port": self.marionette_port,
            "network.dns.localDomains": ",".join(self.config.domains_set),
            "dom.file.createInChild": True,
            # TODO: Remove preferences once Firefox 64 is stable (Bug 905404)
            "network.proxy.type": 0,
            "places.history.enabled": False,
            "network.preload": True,
        })
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences({"layout.interruptible-reflow.enabled": False})

        if self.leak_check:
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks_%s.log" % os.getpid())
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        args = self.binary_args[:] if self.binary_args else []
        args += [cmd_arg("marionette"), "about:blank"]

        debug_args, cmd = browser_command(self.binary,
                                          args,
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs = Preferences()

        pref_paths = []

        profiles = os.path.join(self.prefs_root, 'profiles.json')
        if os.path.isfile(profiles):
            with open(profiles, 'r') as fh:
                for name in json.load(fh)['web-platform-tests']:
                    if self.browser_channel in (None, 'nightly'):
                        pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
                    elif name != 'unittest-features':
                        pref_paths.append(os.path.join(self.prefs_root, name, 'user.js'))
        else:
            # Old preference files used before the creation of profiles.json (remove when no longer supported)
            legacy_pref_paths = (
                os.path.join(self.prefs_root, 'prefs_general.js'),   # Used in Firefox 60 and below
                os.path.join(self.prefs_root, 'common', 'user.js'),  # Used in Firefox 61
            )
            for path in legacy_pref_paths:
                if os.path.isfile(path):
                    pref_paths.append(path)

        for path in pref_paths:
            if os.path.exists(path):
                prefs.add(Preferences.read_prefs(path))
            else:
                self.logger.warning("Failed to find base prefs file in %s" % path)

        # Add any custom preferences
        prefs.add(self.extra_prefs, cast=True)

        return prefs()

    def stop(self, force=False):
        if self.runner is not None and self.runner.is_running():
            try:
                # For Firefox we assume that stopping the runner prompts the
                # browser to shut down. This allows the leak log to be written
                for clean, stop_f in [(True, lambda: self.runner.wait(self.shutdown_timeout)),
                                      (False, lambda: self.runner.stop(signal.SIGTERM)),
                                      (False, lambda: self.runner.stop(signal.SIGKILL))]:
                    if not force or not clean:
                        retcode = stop_f()
                        if retcode is not None:
                            self.logger.info("Browser exited with return code %s" % retcode)
                            break
            except OSError:
                # This can happen on Windows if the process is already dead
                pass
        self.process_leaks()
        self.logger.debug("stopped")

    def process_leaks(self):
        self.logger.info("PROCESS LEAKS %s" % self.leak_report_file)
        if self.lsan_handler:
            self.lsan_handler.process()
        if self.leak_report_file is not None:
            mozleak.process_leak_log(
                self.leak_report_file,
                leak_thresholds=self.mozleak_thresholds,
                ignore_missing_leaks=["gmplugin"],
                log=self.logger,
                stack_fixer=self.stack_fixer,
                scope=self.group_metadata.get("scope"),
                allowed=self.mozleak_allowed
            )

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        if "GLib-GObject-CRITICAL" in line:
            return
        if line:
            data = line.decode("utf8", "replace")
            if self.stack_fixer:
                data = self.stack_fixer(data)
            if self.lsan_handler:
                data = self.lsan_handler.log(data)
            if data is not None:
                self.logger.process_output(self.pid(),
                                           data,
                                           command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self, force=False):
        self.stop(force)

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def check_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        return bool(mozcrash.log_crashes(self.logger,
                                         dump_dir,
                                         symbols_path=self.symbols_path,
                                         stackwalk_binary=self.stackwalk_binary,
                                         process=process,
                                         test=test))

    def setup_ssl(self):
        """Create a certificate database to use in the test profile. This is configured
        to trust the CA Certificate that has signed the web-platform.test server
        certificate."""
        if self.certutil_binary is None:
            self.logger.info("--certutil-binary not supplied; Firefox will not check certificates")
            return

        self.logger.info("Setting up ssl")

        # Make sure the certutil libraries from the source tree are loaded when using a
        # local copy of certutil
        # TODO: Maybe only set this if certutil won't launch?
        env = os.environ.copy()
        certutil_dir = os.path.dirname(self.binary or self.certutil_binary)
        if mozinfo.isMac:
            env_var = "DYLD_LIBRARY_PATH"
        elif mozinfo.isUnix:
            env_var = "LD_LIBRARY_PATH"
        else:
            env_var = "PATH"


        env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
                        if env_var in env else certutil_dir).encode(
                            sys.getfilesystemencoding() or 'utf-8', 'replace')

        def certutil(*args):
            cmd = [self.certutil_binary] + list(args)
            self.logger.process_output("certutil",
                                       subprocess.check_output(cmd,
                                                               env=env,
                                                               stderr=subprocess.STDOUT),
                                       " ".join(cmd))

        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
        with open(pw_path, "w") as f:
            # Use empty password for certificate db
            f.write("\n")

        cert_db_path = self.profile.profile

        # Create a new certificate db
        certutil("-N", "-d", cert_db_path, "-f", pw_path)

        # Add the CA certificate to the database and mark as trusted to issue server certs
        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
                 "-n", "web-platform-tests", "-i", self.ca_certificate_path)

        # List all certs in the database
        certutil("-L", "-d", cert_db_path)
示例#42
0
def run():
  parser = argparse.ArgumentParser(description='Run crawler')
  parser.add_argument(
    '-b', '--binary', type=str,
    help='path to the Firefox binary'
  )
  parser.add_argument(
    '-a', '--abpdir', type=str,
    help='path to the Adblock Plus repository'
  )
  parser.add_argument(
    '-f', '--filters', metavar='url', type=str, nargs='+',
    default=["https://easylist-downloads.adblockplus.org/easylist.txt", "https://easylist-downloads.adblockplus.org/exceptionrules.txt"],
    help='filter lists to install in Adblock Plus. The arguments can also have the format path=url, the data will be read from the specified path then.'
  )
  parser.add_argument(
    '-t', '--timeout', type=int, default=300,
    help='Load timeout (seconds)'
  )
  parser.add_argument(
    '-x', '--maxtabs', type=int, default=15,
    help='Maximal number of tabs to open in parallel'
  )
  parser.add_argument(
    'list', type=str,
    help='URL list to process'
  )
  parser.add_argument(
    'outdir', type=str,
    help='directory to write data into'
  )
  parameters = parser.parse_args()

  import buildtools.packagerGecko as packager
  cleanup = []
  try:
    base_dir = os.path.dirname(__file__)
    handle, crawlerxpi = tempfile.mkstemp(suffix='.xpi')
    os.close(handle)
    cleanup.append(crawlerxpi)
    packager.createBuild(base_dir, outFile=crawlerxpi, releaseBuild=True)

    abpxpi = 'https://addons.mozilla.org/firefox/downloads/latest/1865/addon-1865-latest.xpi'
    if parameters.abpdir:
      handle, abpxpi = tempfile.mkstemp(suffix='.xpi')
      os.close(handle)
      cleanup.append(abpxpi)
      packager.createBuild(parameters.abpdir, outFile=abpxpi, releaseBuild=True)

    profile = FirefoxProfile(
      addons=[
        crawlerxpi,
        abpxpi,
      ],
      preferences={
        'browser.uitour.enabled': False,
        'prompts.tab_modal.enabled': False,
      }
    )

    abpsettings = os.path.join(profile.profile, 'adblockplus')
    os.makedirs(abpsettings)
    with open(os.path.join(abpsettings, 'patterns.ini'), 'w') as handle:
      print >>handle, '# Adblock Plus preferences'
      print >>handle, 'version=4'
      for url in parameters.filters:
        if '=' in url:
          path, url = url.split('=', 1)
          with open(path, 'r') as source:
            data = source.read()
        else:
          data = urllib.urlopen(url).read()
        print >>handle, '[Subscription]'
        print >>handle, 'url=%s' % url
        print >>handle, '[Subscription filters]'
        print >>handle, '\n'.join(data.splitlines()[1:])
  finally:
    for path in cleanup:
      os.unlink(path)

  server = None
  try:
    port = random.randrange(2000, 60000)
    print "Communicating with client on port %i" % port

    app = CrawlerApp(parameters)
    server = make_server('localhost', port, app)
    app.server = server
    threading.Thread(target=lambda: server.serve_forever()).start()

    runner = FirefoxRunner(
      profile=profile,
      binary=parameters.binary,
      cmdargs=['--crawler-port', str(port)],
      env=dict(os.environ, MOZ_CRASHREPORTER_DISABLE='1'),
    )
    while app.urls:
      runner.start()
      runner.wait()
  finally:
    if server:
      server.shutdown()
    profile.cleanup()
示例#43
0
    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        locations = ServerLocations(
            filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({
            "marionette.port":
            self.marionette_port,
            "dom.disable_open_during_load":
            False,
            "network.dns.localDomains":
            ",".join(hostnames),
            "network.proxy.type":
            0,
            "places.history.enabled":
            False,
            "dom.send_after_paint_to_content":
            True,
            "network.preload":
            True
        })
        if self.e10s:
            self.profile.set_preferences(
                {"browser.tabs.remote.autostart": True})

        if self.test_type == "reftest":
            self.profile.set_preferences(
                {"layout.interruptible-reflow.enabled": False})

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile,
                                                 "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft")
                and '5.1' in platform.version()):
            self.profile.set_preferences(
                {"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(
            self.binary, self.binary_args if self.binary_args else [] +
            [cmd_arg("marionette"), "about:blank"], self.debug_info)

        self.runner = FirefoxRunner(
            profile=self.profile,
            binary=cmd[0],
            cmdargs=cmd[1:],
            env=env,
            process_class=ProcessHandler,
            process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args,
                          interactive=self.debug_info
                          and self.debug_info.interactive)
        self.logger.debug("Firefox Started")
示例#44
0
class FirefoxBrowser(Browser):
    used_ports = set()
    init_timeout = 60

    def __init__(self, logger, binary, prefs_root, debug_info=None,
                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                 ca_certificate_path=None, e10s=False):
        Browser.__init__(self, logger)
        self.binary = binary
        self.prefs_root = prefs_root
        self.marionette_port = None
        self.runner = None
        self.debug_info = debug_info
        self.profile = None
        self.symbols_path = symbols_path
        self.stackwalk_binary = stackwalk_binary
        self.ca_certificate_path = ca_certificate_path
        self.certutil_binary = certutil_binary
        self.e10s = e10s

    def start(self):
        self.marionette_port = get_free_port(2828, exclude=self.used_ports)
        self.used_ports.add(self.marionette_port)

        env = os.environ.copy()
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"

        locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(locations=locations,
                                      preferences=preferences)
        self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                      "marionette.defaultPrefs.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "network.dns.localDomains": ",".join(hostnames),
                                      "network.proxy.type": 0,
                                      "places.history.enabled": False})
        if self.e10s:
            self.profile.set_preferences({"browser.tabs.remote.autostart": True})

        # Bug 1262954: winxp + e10s, disable hwaccel
        if (self.e10s and platform.system() in ("Windows", "Microsoft") and
            '5.1' in platform.version()):
            self.profile.set_preferences({"layers.acceleration.disabled": True})

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FirefoxRunner(profile=self.profile,
                                    binary=cmd[0],
                                    cmdargs=cmd[1:],
                                    env=env,
                                    process_class=ProcessHandler,
                                    process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting Firefox")

        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
        self.logger.debug("Firefox Started")

    def load_prefs(self):
        prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
        if os.path.exists(prefs_path):
            preferences = Preferences.read_prefs(prefs_path)
        else:
            self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
            preferences = []

        return preferences

    def stop(self):
        self.logger.debug("Stopping browser")
        if self.runner is not None:
            try:
                self.runner.stop()
            except OSError:
                # This can happen on Windows if the process is already dead
                pass

    def pid(self):
        if self.runner.process_handler is None:
            return None

        try:
            return self.runner.process_handler.pid
        except AttributeError:
            return None

    def on_output(self, line):
        """Write a line of output from the firefox process to the log"""
        self.logger.process_output(self.pid(),
                                   line.decode("utf8", "replace"),
                                   command=" ".join(self.runner.command))

    def is_alive(self):
        if self.runner:
            return self.runner.is_running()
        return False

    def cleanup(self):
        self.stop()

    def executor_browser(self):
        assert self.marionette_port is not None
        return ExecutorBrowser, {"marionette_port": self.marionette_port}

    def log_crash(self, process, test):
        dump_dir = os.path.join(self.profile.profile, "minidumps")

        mozcrash.log_crashes(self.logger,
                             dump_dir,
                             symbols_path=self.symbols_path,
                             stackwalk_binary=self.stackwalk_binary,
                             process=process,
                             test=test)

    def setup_ssl(self):
        """Create a certificate database to use in the test profile. This is configured
        to trust the CA Certificate that has signed the web-platform.test server
        certificate."""

        self.logger.info("Setting up ssl")

        # Make sure the certutil libraries from the source tree are loaded when using a
        # local copy of certutil
        # TODO: Maybe only set this if certutil won't launch?
        env = os.environ.copy()
        certutil_dir = os.path.dirname(self.binary)
        if mozinfo.isMac:
            env_var = "DYLD_LIBRARY_PATH"
        elif mozinfo.isUnix:
            env_var = "LD_LIBRARY_PATH"
        else:
            env_var = "PATH"


        env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
                        if env_var in env else certutil_dir).encode(
                                sys.getfilesystemencoding() or 'utf-8', 'replace')

        def certutil(*args):
            cmd = [self.certutil_binary] + list(args)
            self.logger.process_output("certutil",
                                       subprocess.check_output(cmd,
                                                               env=env,
                                                               stderr=subprocess.STDOUT),
                                       " ".join(cmd))

        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
        with open(pw_path, "w") as f:
            # Use empty password for certificate db
            f.write("\n")

        cert_db_path = self.profile.profile

        # Create a new certificate db
        certutil("-N", "-d", cert_db_path, "-f", pw_path)

        # Add the CA certificate to the database and mark as trusted to issue server certs
        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
                 "-n", "web-platform-tests", "-i", self.ca_certificate_path)

        # List all certs in the database
        certutil("-L", "-d", cert_db_path)
示例#45
0
class FirefoxThread(Thread):
    def __init__(self, binary, marionette_port = 2828):
        Thread.__init__(self)
        self.binary = binary
        self.marionette_port = marionette_port
        self.logger = FirefoxThreadLogger(None)
        self._firefoxRunningEvent = Event()
        
        
    def run(self):
        '''
        Starts Firefox thread with Marionette turned on.
        '''
        
        self.profile = FirefoxProfile()
        self.profile.set_preferences({"marionette.defaultPrefs.enabled" : True,
                                      "marionette.defaultPrefs.port": 2828,
                                      "browser.startup.page": 0,
                                      "browser.startup.homepage": "about:blank",
                                      })
        self.runner = FirefoxRunner(profile = self.profile,
                                    binary = self.binary,
                                    kp_kwargs = {'processOutputLine' : [self.logger]})

        self.runner.start()
        self._firefoxRunningEvent.set()
        self.runner.wait()
        
    def stop(self):
        '''
        Stops Firefox/Nightly. To be called by external thread.
        '''
        self.runner.stop()
        
    def getPID(self):
        '''
        This is called by external threads, and blocks until PID is available in FirefoxRunner,
        which is shortly after start() has been called. 
        '''
        self._firefoxRunningEvent.wait()
        return self.runner.process_handler.proc.pid
        
        
    def waitForMarionettePortOpenReady(self, timeout):
        '''
        This method can be run by an external thread. Returns True when the port is open, or False on timeout.
        It's active waiting with 1 sec heartbeat, if you know better solution please mail me.
        
        Originally taken from:
        https://github.com/mozilla/marionette_client/blob/master/marionette/emulator.py#L246
        '''
        starttime = datetime.datetime.now()
        while datetime.datetime.now() - starttime < datetime.timedelta(seconds=timeout):
            try:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.connect(('127.0.0.1', self.marionette_port))
                data = sock.recv(16)
                sock.close()
                if '"from"' in data:
                    return True
            except:
                #import traceback
                #print traceback.format_exc()
                pass
            
            time.sleep(1)
        print ''
        return False
        
        
示例#46
0
文件: fennec.py 项目: Coder206/servo
class FennecBrowser(FirefoxBrowser):
    used_ports = set()
    init_timeout = 300
    shutdown_timeout = 60

    def __init__(self, logger, prefs_root, test_type, package_name=None,
                 device_serial="emulator-5444", **kwargs):
        FirefoxBrowser.__init__(self, logger, None, prefs_root, test_type, **kwargs)
        self._package_name = package_name
        self.device_serial = device_serial
        self.tests_root = kwargs["tests_root"]
        self.install_fonts = kwargs["install_fonts"]

    @property
    def package_name(self):
        """
        Name of app to run on emulator.
        """
        if self._package_name is None:
            self._package_name = "org.mozilla.fennec"
            user = os.getenv("USER")
            if user:
                self._package_name += "_" + user
        return self._package_name

    def start(self, **kwargs):
        if self.marionette_port is None:
            self.marionette_port = get_free_port(2828, exclude=self.used_ports)
            self.used_ports.add(self.marionette_port)

        env = {}
        env["MOZ_CRASHREPORTER"] = "1"
        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
        env["STYLO_THREADS"] = str(self.stylo_threads)
        if self.chaos_mode_flags is not None:
            env["MOZ_CHAOSMODE"] = str(self.chaos_mode_flags)

        preferences = self.load_prefs()

        self.profile = FirefoxProfile(preferences=preferences)
        self.profile.set_preferences({"marionette.port": self.marionette_port,
                                      "dom.disable_open_during_load": False,
                                      "places.history.enabled": False,
                                      "dom.send_after_paint_to_content": True,
                                      "network.preload": True})
        if self.test_type == "reftest":
            self.logger.info("Setting android reftest preferences")
            self.profile.set_preferences({"browser.viewport.desktopWidth": 600,
                                          # Disable high DPI
                                          "layout.css.devPixelsPerPx": "1.0",
                                          # Ensure that the full browser element
                                          # appears in the screenshot
                                          "apz.allow_zooming": False,
                                          "android.widget_paints_background": False,
                                          # Ensure that scrollbars are always painted
                                          "ui.scrollbarFadeBeginDelay": 100000})

        if self.install_fonts:
            self.logger.debug("Copying Ahem font to profile")
            font_dir = os.path.join(self.profile.profile, "fonts")
            if not os.path.exists(font_dir):
                os.makedirs(font_dir)
            with open(os.path.join(self.tests_root, "fonts", "Ahem.ttf"), "rb") as src:
                with open(os.path.join(font_dir, "Ahem.ttf"), "wb") as dest:
                    dest.write(src.read())

        if self.leak_check and kwargs.get("check_leaks", True):
            self.leak_report_file = os.path.join(self.profile.profile, "runtests_leaks.log")
            if os.path.exists(self.leak_report_file):
                os.remove(self.leak_report_file)
            env["XPCOM_MEM_BLOAT_LOG"] = self.leak_report_file
        else:
            self.leak_report_file = None

        if self.ca_certificate_path is not None:
            self.setup_ssl()

        debug_args, cmd = browser_command(self.package_name,
                                          self.binary_args if self.binary_args else [] +
                                          [cmd_arg("marionette"), "about:blank"],
                                          self.debug_info)

        self.runner = FennecEmulatorRunner(app=self.package_name,
                                           profile=self.profile,
                                           cmdargs=cmd[1:],
                                           env=env,
                                           symbols_path=self.symbols_path,
                                           serial=self.device_serial,
                                           # TODO - choose appropriate log dir
                                           logdir=os.getcwd(),
                                           process_class=ProcessHandler,
                                           process_args={"processOutputLine": [self.on_output]})

        self.logger.debug("Starting %s" % self.package_name)
        # connect to a running emulator
        self.runner.device.connect()

        write_hosts_file(self.config, self.runner.device.device)

        self.runner.stop()
        self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)

        self.runner.device.device.forward(
            local="tcp:{}".format(self.marionette_port),
            remote="tcp:{}".format(self.marionette_port))

        self.logger.debug("%s Started" % self.package_name)

    def stop(self, force=False):
        if self.runner is not None:
            if (self.runner.device.connected and
                len(self.runner.device.device.list_forwards()) > 0):
                try:
                    self.runner.device.device.remove_forwards(
                        "tcp:{}".format(self.marionette_port))
                except Exception:
                    self.logger.warning("Failed to remove port forwarding")
            # We assume that stopping the runner prompts the
            # browser to shut down. This allows the leak log to be written
            self.runner.stop()
        self.logger.debug("stopped")