示例#1
0
    def install_from_path(self, path, unpack=False):
        """
        Installs addon from a filepath, url or directory of addons in the profile.

        :param path: url, path to .xpi, or directory of addons
        :param unpack: whether to unpack unless specified otherwise in the install.rdf
        """

        # if the addon is a URL, download it
        # note that this won't work with protocols urllib2 doesn't support
        if mozfile.is_url(path):
            path = self.download(path)
            self.downloaded_addons.append(path)

        addons = [path]

        # if path is not an add-on, try to install all contained add-ons
        try:
            self.addon_details(path)
        except AddonFormatError, e:
            module_logger.warning('Could not install %s: %s' % (path, str(e)))

            # If the path doesn't exist, then we don't really care, just return
            if not os.path.isdir(path):
                return

            addons = [os.path.join(path, x) for x in os.listdir(path) if
                      self.is_addon(os.path.join(path, x))]
            addons.sort()
示例#2
0
    def read(cls, path):
        """read preferences from a file"""

        section = None  # for .ini files
        basename = os.path.basename(path)
        if ":" in basename:
            # section of INI file
            path, section = path.rsplit(":", 1)

        if not os.path.exists(path) and not mozfile.is_url(path):
            raise PreferencesReadError("'%s' does not exist" % path)

        if section:
            try:
                return cls.read_ini(path, section)
            except PreferencesReadError:
                raise
            except Exception as e:
                raise PreferencesReadError(str(e))

        # try both JSON and .ini format
        try:
            return cls.read_json(path)
        except Exception as e:
            try:
                return cls.read_ini(path)
            except Exception as f:
                for exception in e, f:
                    if isinstance(exception, PreferencesReadError):
                        raise exception
                raise PreferencesReadError("Could not recognize format of %s" % path)
示例#3
0
文件: prefs.py 项目: emilio/gecko-dev
    def read(cls, path):
        """read preferences from a file"""

        section = None  # for .ini files
        basename = os.path.basename(path)
        if ":" in basename:
            # section of INI file
            path, section = path.rsplit(":", 1)

        if not os.path.exists(path) and not mozfile.is_url(path):
            raise PreferencesReadError("'%s' does not exist" % path)

        if section:
            try:
                return cls.read_ini(path, section)
            except PreferencesReadError:
                raise
            except Exception as e:
                raise PreferencesReadError(str(e))

        # try both JSON and .ini format
        try:
            return cls.read_json(path)
        except Exception as e:
            try:
                return cls.read_ini(path)
            except Exception as f:
                for exception in e, f:
                    if isinstance(exception, PreferencesReadError):
                        raise exception
                raise PreferencesReadError("Could not recognize format of %s" % path)
示例#4
0
    def install_from_path(self, path, unpack=False):
        """
        Installs addon from a filepath, url or directory of addons in the profile.

        :param path: url, path to .xpi, or directory of addons
        :param unpack: whether to unpack unless specified otherwise in the install.rdf
        """

        # if the addon is a URL, download it
        # note that this won't work with protocols urllib2 doesn't support
        if mozfile.is_url(path):
            path = self.download(path)
            self.downloaded_addons.append(path)

        addons = [path]

        # if path is not an add-on, try to install all contained add-ons
        try:
            self.addon_details(path)
        except AddonFormatError, e:
            module_logger.warning('Could not install %s: %s' % (path, str(e)))

            # If the path doesn't exist, then we don't really care, just return
            if not os.path.isdir(path):
                return

            addons = [
                os.path.join(path, x) for x in os.listdir(path)
                if self.is_addon(os.path.join(path, x))
            ]
            addons.sort()
示例#5
0
 def _get_symbols(self):
     # This updates self.symbols_path so we only download once
     if self.symbols_path and mozfile.is_url(self.symbols_path):
         self.remove_symbols = True
         self.logger.info("Downloading symbols from: %s" % self.symbols_path)
         # Get the symbols and write them to a temporary zipfile
         data = urllib2.urlopen(self.symbols_path)
         with tempfile.TemporaryFile() as symbols_file:
             symbols_file.write(data.read())
             # extract symbols to a temporary directory (which we'll delete after
             # processing all crashes)
             self.symbols_path = tempfile.mkdtemp()
             with zipfile.ZipFile(symbols_file, 'r') as zfile:
                 mozfile.extract_zip(zfile, self.symbols_path)
示例#6
0
 def _get_symbols(self):
     # This updates self.symbols_path so we only download once
     if self.symbols_path and mozfile.is_url(self.symbols_path):
         self.remove_symbols = True
         self.logger.info("Downloading symbols from: %s" % self.symbols_path)
         # Get the symbols and write them to a temporary zipfile
         data = urllib2.urlopen(self.symbols_path)
         with tempfile.TemporaryFile() as symbols_file:
             symbols_file.write(data.read())
             # extract symbols to a temporary directory (which we'll delete after
             # processing all crashes)
             self.symbols_path = tempfile.mkdtemp()
             with zipfile.ZipFile(symbols_file, 'r') as zfile:
                 mozfile.extract_zip(zfile, self.symbols_path)
示例#7
0
    def read(cls, path):
        """read preferences from a file"""

        section = None # for .ini files
        basename = os.path.basename(path)
        if ':' in basename:
            # section of INI file
            path, section = path.rsplit(':', 1)

        if not os.path.exists(path) and not mozfile.is_url(path):
            raise PreferencesReadError("'%s' does not exist" % path)

        if section:
            try:
                return cls.read_ini(path, section)
            except PreferencesReadError:
                raise
            except Exception, e:
                raise PreferencesReadError(str(e))
示例#8
0
    def read(cls, path):
        """read preferences from a file"""

        section = None  # for .ini files
        basename = os.path.basename(path)
        if ':' in basename:
            # section of INI file
            path, section = path.rsplit(':', 1)

        if not os.path.exists(path) and not mozfile.is_url(path):
            raise PreferencesReadError("'%s' does not exist" % path)

        if section:
            try:
                return cls.read_ini(path, section)
            except PreferencesReadError:
                raise
            except Exception, e:
                raise PreferencesReadError(str(e))
示例#9
0
    def _get_symbols(self):
        # If no symbols path has been set create a temporary folder to let the
        # minidump stackwalk download the symbols.
        if not self.symbols_path:
            self.symbols_path = tempfile.mkdtemp()
            self.remove_symbols = True

        # This updates self.symbols_path so we only download once.
        if mozfile.is_url(self.symbols_path):
            self.remove_symbols = True
            self.logger.info("Downloading symbols from: %s" % self.symbols_path)
            # Get the symbols and write them to a temporary zipfile
            data = urlopen(self.symbols_path)
            with tempfile.TemporaryFile() as symbols_file:
                symbols_file.write(data.read())
                # extract symbols to a temporary directory (which we'll delete after
                # processing all crashes)
                self.symbols_path = tempfile.mkdtemp()
                with zipfile.ZipFile(symbols_file, 'r') as zfile:
                    mozfile.extract_zip(zfile, self.symbols_path)
示例#10
0
文件: addons.py 项目: AlexxNica/gecko
    def install_from_path(self, path, unpack=False):
        """
        Installs addon from a filepath, url or directory of addons in the profile.

        :param path: url, path to .xpi, or directory of addons
        :param unpack: whether to unpack unless specified otherwise in the install.rdf
        """

        # if the addon is a URL, download it
        # note that this won't work with protocols urllib2 doesn't support
        if mozfile.is_url(path):
            path = self.download(path)
            self.downloaded_addons.append(path)

        addons = [path]

        # if path is not an add-on, try to install all contained add-ons
        try:
            self.addon_details(path)
        except AddonFormatError as e:
            module_logger.warning('Could not install %s: %s' % (path, str(e)))

            # If the path doesn't exist, then we don't really care, just return
            if not os.path.isdir(path):
                return

            addons = [os.path.join(path, x) for x in os.listdir(path) if
                      self.is_addon(os.path.join(path, x))]
            addons.sort()

        # install each addon
        for addon in addons:
            # determine the addon id
            addon_details = self.addon_details(addon)
            addon_id = addon_details.get('id')

            # if the add-on has to be unpacked force it now
            # note: we might want to let Firefox do it in case of addon details
            orig_path = None
            if os.path.isfile(addon) and (unpack or addon_details['unpack']):
                orig_path = addon
                addon = tempfile.mkdtemp()
                mozfile.extract(orig_path, addon)

            # copy the addon to the profile
            extensions_path = os.path.join(self.profile, 'extensions', 'staged')
            addon_path = os.path.join(extensions_path, addon_id)

            if os.path.isfile(addon):
                addon_path += '.xpi'

                # move existing xpi file to backup location to restore later
                if os.path.exists(addon_path):
                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
                    shutil.move(addon_path, self.backup_dir)

                # copy new add-on to the extension folder
                if not os.path.exists(extensions_path):
                    os.makedirs(extensions_path)
                shutil.copy(addon, addon_path)
            else:
                # move existing folder to backup location to restore later
                if os.path.exists(addon_path):
                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
                    shutil.move(addon_path, self.backup_dir)

                # copy new add-on to the extension folder
                shutil.copytree(addon, addon_path, symlinks=True)

            # if we had to extract the addon, remove the temporary directory
            if orig_path:
                mozfile.remove(addon)
                addon = orig_path

            self._addons.append(addon_id)
            self.installed_addons.append(addon)
示例#11
0
    def symbolicate(self):
        """
        Symbolicate Gecko profiling data for one pagecycle.

        """
        profiles = self.collect_profiles()
        if len(profiles) == 0:
            LOG.error("No profiles collected")
            return

        symbolicator = ProfileSymbolicator({
            # Trace-level logging (verbose)
            "enableTracing": 0,
            # Fallback server if symbol is not found locally
            "remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
            # Maximum number of symbol files to keep in memory
            "maxCacheEntries": 2000000,
            # Frequency of checking for recent symbols to
            # cache (in hours)
            "prefetchInterval": 12,
            # Oldest file age to prefetch (in hours)
            "prefetchThreshold": 48,
            # Maximum number of library versions to pre-fetch
            # per library
            "prefetchMaxSymbolsPerLib": 3,
            # Default symbol lookup directories
            "defaultApp": "FIREFOX",
            "defaultOs": "WINDOWS",
            # Paths to .SYM files, expressed internally as a
            # mapping of app or platform names to directories
            # Note: App & OS names from requests are converted
            # to all-uppercase internally
            "symbolPaths": self.symbol_paths,
        })

        if self.raptor_config.get("symbols_path") is not None:
            if mozfile.is_url(self.raptor_config["symbols_path"]):
                symbolicator.integrate_symbol_zip_from_url(
                    self.raptor_config["symbols_path"])
            elif os.path.isfile(self.raptor_config["symbols_path"]):
                symbolicator.integrate_symbol_zip_from_file(
                    self.raptor_config["symbols_path"])
            elif os.path.isdir(self.raptor_config["symbols_path"]):
                sym_path = self.raptor_config["symbols_path"]
                symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
                self.cleanup = False

        missing_symbols_zip = os.path.join(self.upload_dir,
                                           "missingsymbols.zip")

        try:
            mode = zipfile.ZIP_DEFLATED
        except NameError:
            mode = zipfile.ZIP_STORED

        with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
            for profile_filename in profiles:
                testname = profile_filename
                if testname.endswith(".profile"):
                    testname = testname[0:-8]
                profile_path = os.path.join(self.gecko_profile_dir,
                                            profile_filename)
                self._save_gecko_profile(symbolicator, missing_symbols_zip,
                                         profile_path)

                # Our zip will contain one directory per test,
                # and each directory will contain one or more
                # *.profile files - one for each pagecycle
                path_in_zip = os.path.join(
                    "profile_{0}".format(self.test_config["name"]),
                    testname + ".profile",
                )
                LOG.info("Adding profile {0} to archive {1}".format(
                    path_in_zip, self.profile_arcname))
                try:
                    arc.write(profile_path, path_in_zip)
                except Exception:
                    LOG.exception("Failed to copy profile {0} as {1} to"
                                  " archive {2}".format(
                                      profile_path, path_in_zip,
                                      self.profile_arcname))
            # save the latest gecko profile archive to an env var, so later on
            # it can be viewed automatically via the view-gecko-profile tool
            os.environ[
                "RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
示例#12
0
 def test_is_url(self):
     self.assertTrue(is_url('http://mozilla.org'))
     self.assertFalse(is_url('/usr/bin/mozilla.org'))
     self.assertTrue(is_url('file:///usr/bin/mozilla.org'))
     self.assertFalse(is_url('c:\foo\bar'))
示例#13
0
 def test_is_url(self):
     self.assertTrue(is_url('http://mozilla.org'))
     self.assertFalse(is_url('/usr/bin/mozilla.org'))
     self.assertTrue(is_url('file:///usr/bin/mozilla.org'))
     self.assertFalse(is_url('c:\foo\bar'))
示例#14
0
    def symbolicate(self, cycle):
        """
        Symbolicate sps profiling data for one cycle.

        :param cycle: the number of the cycle of the test currently run.
        """
        symbolicator = symbolication.ProfileSymbolicator({
            # Trace-level logging (verbose)
            "enableTracing":
            0,
            # Fallback server if symbol is not found locally
            "remoteSymbolServer":
            "http://symbolapi.mozilla.org:80/talos/",
            # Maximum number of symbol files to keep in memory
            "maxCacheEntries":
            2000000,
            # Frequency of checking for recent symbols to
            # cache (in hours)
            "prefetchInterval":
            12,
            # Oldest file age to prefetch (in hours)
            "prefetchThreshold":
            48,
            # Maximum number of library versions to pre-fetch
            # per library
            "prefetchMaxSymbolsPerLib":
            3,
            # Default symbol lookup directories
            "defaultApp":
            "FIREFOX",
            "defaultOs":
            "WINDOWS",
            # Paths to .SYM files, expressed internally as a
            # mapping of app or platform names to directories
            # Note: App & OS names from requests are converted
            # to all-uppercase internally
            "symbolPaths":
            self.symbol_paths
        })

        if self.browser_config['symbols_path']:
            if mozfile.is_url(self.browser_config['symbols_path']):
                symbolicator.integrate_symbol_zip_from_url(
                    self.browser_config['symbols_path'])
            else:
                symbolicator.integrate_symbol_zip_from_file(
                    self.browser_config['symbols_path'])

        missing_symbols_zip = os.path.join(self.upload_dir,
                                           "missingsymbols.zip")

        try:
            mode = zipfile.ZIP_DEFLATED
        except NameError:
            mode = zipfile.ZIP_STORED

        sps_profile_dir = self.option('dir')

        with zipfile.ZipFile(self.profile_arcname, 'a', mode) as arc:
            # Collect all individual profiles that the test
            # has put into sps_profile_dir.
            for profile_filename in os.listdir(sps_profile_dir):
                testname = profile_filename
                if testname.endswith(".sps"):
                    testname = testname[0:-4]
                profile_path = os.path.join(sps_profile_dir, profile_filename)
                self._save_sps_profile(cycle, symbolicator,
                                       missing_symbols_zip, profile_path)

                # Our zip will contain one directory per subtest,
                # and each subtest directory will contain one or
                # more cycle_i.sps files. For example, with
                # test_config['name'] == 'tscrollx',
                # profile_filename == 'iframe.svg.sps', i == 0,
                # we'll get path_in_zip ==
                # 'profile_tscrollx/iframe.svg/cycle_0.sps'.
                cycle_name = "cycle_{0}.sps".format(cycle)
                path_in_zip = \
                    os.path.join(
                        "profile_{0}".format(self.test_config['name']),
                        testname,
                        cycle_name
                    )
                LOG.info("Adding profile {0} to archive {1}".format(
                    path_in_zip, self.profile_arcname))
                try:
                    arc.write(profile_path, path_in_zip)
                except Exception:
                    LOG.exception("Failed to copy profile {0} as {1} to"
                                  " archive {2}".format(
                                      profile_path, path_in_zip,
                                      self.profile_arcname))
示例#15
0
    def install_from_path(self, path, unpack=False):
        """
        Installs addon from a filepath, url or directory of addons in the profile.

        :param path: url, path to .xpi, or directory of addons
        :param unpack: whether to unpack unless specified otherwise in the install.rdf
        """

        # if the addon is a URL, download it
        # note that this won't work with protocols urllib2 doesn't support
        if mozfile.is_url(path):
            response = urllib2.urlopen(path)
            fd, path = tempfile.mkstemp(suffix=".xpi")
            os.write(fd, response.read())
            os.close(fd)

            self.downloaded_addons.append(path)

        addons = [path]

        # if path is not an add-on, try to install all contained add-ons
        if not self.is_addon(path):
            # If the path doesn't exist, then we don't really care, just return
            if not os.path.isdir(path):
                return
            addons = [os.path.join(path, x) for x in os.listdir(path) if self.is_addon(os.path.join(path, x))]
            addons.sort()

        # install each addon
        for addon in addons:
            # determine the addon id
            addon_details = self.addon_details(addon)
            addon_id = addon_details.get("id")
            assert addon_id, "The addon id could not be found: %s" % addon

            # if the add-on has to be unpacked force it now
            # note: we might want to let Firefox do it in case of addon details
            orig_path = None
            if os.path.isfile(addon) and (unpack or addon_details["unpack"]):
                orig_path = addon
                addon = tempfile.mkdtemp()
                mozfile.extract(orig_path, addon)

            # copy the addon to the profile
            extensions_path = os.path.join(self.profile, "extensions", "staged")
            addon_path = os.path.join(extensions_path, addon_id)

            if os.path.isfile(addon):
                addon_path += ".xpi"

                # save existing xpi file to restore later
                if os.path.exists(addon_path):
                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
                    shutil.copy(addon_path, self.backup_dir)

                if not os.path.exists(extensions_path):
                    os.makedirs(extensions_path)
                shutil.copy(addon, addon_path)
            else:
                # save existing dir to restore later
                if os.path.exists(addon_path):
                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
                    dest = os.path.join(self.backup_dir, os.path.basename(addon_path))
                    dir_util.copy_tree(addon_path, dest, preserve_symlinks=1)

                dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)

            # if we had to extract the addon, remove the temporary directory
            if orig_path:
                dir_util.remove_tree(addon)
                addon = orig_path

            self._addons.append(addon_id)
            self.installed_addons.append(addon)
示例#16
0
    def symbolicate(self, cycle):
        """
        Symbolicate Gecko profiling data for one cycle.

        :param cycle: the number of the cycle of the test currently run.
        """
        symbolicator = ProfileSymbolicator({
            # Trace-level logging (verbose)
            "enableTracing": 0,
            # Fallback server if symbol is not found locally
            "remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
            # Maximum number of symbol files to keep in memory
            "maxCacheEntries": 2000000,
            # Frequency of checking for recent symbols to
            # cache (in hours)
            "prefetchInterval": 12,
            # Oldest file age to prefetch (in hours)
            "prefetchThreshold": 48,
            # Maximum number of library versions to pre-fetch
            # per library
            "prefetchMaxSymbolsPerLib": 3,
            # Default symbol lookup directories
            "defaultApp": "FIREFOX",
            "defaultOs": "WINDOWS",
            # Paths to .SYM files, expressed internally as a
            # mapping of app or platform names to directories
            # Note: App & OS names from requests are converted
            # to all-uppercase internally
            "symbolPaths": self.symbol_paths,
        })

        if self.browser_config["symbols_path"]:
            if mozfile.is_url(self.browser_config["symbols_path"]):
                symbolicator.integrate_symbol_zip_from_url(
                    self.browser_config["symbols_path"])
            elif os.path.isfile(self.browser_config["symbols_path"]):
                symbolicator.integrate_symbol_zip_from_file(
                    self.browser_config["symbols_path"])
            elif os.path.isdir(self.browser_config["symbols_path"]):
                sym_path = self.browser_config["symbols_path"]
                symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
                self.cleanup = False

        missing_symbols_zip = os.path.join(self.upload_dir,
                                           "missingsymbols.zip")

        try:
            mode = zipfile.ZIP_DEFLATED
        except NameError:
            mode = zipfile.ZIP_STORED

        gecko_profile_dir = self.option("dir")

        with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
            # Collect all individual profiles that the test
            # has put into gecko_profile_dir.
            for profile_filename in os.listdir(gecko_profile_dir):
                testname = profile_filename
                if testname.endswith(".profile"):
                    testname = testname[0:-8]
                profile_path = os.path.join(gecko_profile_dir,
                                            profile_filename)
                self._save_gecko_profile(cycle, symbolicator,
                                         missing_symbols_zip, profile_path)

                # Our zip will contain one directory per subtest,
                # and each subtest directory will contain one or
                # more cycle_i.profile files. For example, with
                # test_config['name'] == 'tscrollx',
                # profile_filename == 'iframe.svg.profile', i == 0,
                # we'll get path_in_zip ==
                # 'profile_tscrollx/iframe.svg/cycle_0.profile'.
                cycle_name = "cycle_{0}.profile".format(cycle)
                path_in_zip = os.path.join(
                    "profile_{0}".format(self.test_config["name"]), testname,
                    cycle_name)
                LOG.info("Adding profile {0} to archive {1}".format(
                    path_in_zip, self.profile_arcname))
                try:
                    arc.write(profile_path, path_in_zip)
                except Exception:
                    LOG.exception("Failed to copy profile {0} as {1} to"
                                  " archive {2}".format(
                                      profile_path, path_in_zip,
                                      self.profile_arcname))
            # save the latest gecko profile archive to an env var, so later on
            # it can be viewed automatically via the view-gecko-profile tool
            os.environ[
                "TALOS_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
示例#17
0
    def symbolicate(self, cycle):
        """
        Symbolicate Gecko profiling data for one cycle.

        :param cycle: the number of the cycle of the test currently run.
        """
        symbolicator = symbolication.ProfileSymbolicator({
            # Trace-level logging (verbose)
            "enableTracing": 0,
            # Fallback server if symbol is not found locally
            "remoteSymbolServer":
                "http://symbolapi.mozilla.org:80/talos/",
            # Maximum number of symbol files to keep in memory
            "maxCacheEntries": 2000000,
            # Frequency of checking for recent symbols to
            # cache (in hours)
            "prefetchInterval": 12,
            # Oldest file age to prefetch (in hours)
            "prefetchThreshold": 48,
            # Maximum number of library versions to pre-fetch
            # per library
            "prefetchMaxSymbolsPerLib": 3,
            # Default symbol lookup directories
            "defaultApp": "FIREFOX",
            "defaultOs": "WINDOWS",
            # Paths to .SYM files, expressed internally as a
            # mapping of app or platform names to directories
            # Note: App & OS names from requests are converted
            # to all-uppercase internally
            "symbolPaths": self.symbol_paths
        })

        if self.browser_config['symbols_path']:
            if mozfile.is_url(self.browser_config['symbols_path']):
                symbolicator.integrate_symbol_zip_from_url(
                    self.browser_config['symbols_path']
                )
            else:
                symbolicator.integrate_symbol_zip_from_file(
                    self.browser_config['symbols_path']
                )

        missing_symbols_zip = os.path.join(self.upload_dir,
                                           "missingsymbols.zip")

        try:
            mode = zipfile.ZIP_DEFLATED
        except NameError:
            mode = zipfile.ZIP_STORED

        gecko_profile_dir = self.option('dir')

        with zipfile.ZipFile(self.profile_arcname, 'a', mode) as arc:
            # Collect all individual profiles that the test
            # has put into gecko_profile_dir.
            for profile_filename in os.listdir(gecko_profile_dir):
                testname = profile_filename
                if testname.endswith(".profile"):
                    testname = testname[0:-8]
                profile_path = os.path.join(gecko_profile_dir, profile_filename)
                self._save_gecko_profile(cycle, symbolicator,
                                         missing_symbols_zip,
                                         profile_path)

                # Our zip will contain one directory per subtest,
                # and each subtest directory will contain one or
                # more cycle_i.profile files. For example, with
                # test_config['name'] == 'tscrollx',
                # profile_filename == 'iframe.svg.profile', i == 0,
                # we'll get path_in_zip ==
                # 'profile_tscrollx/iframe.svg/cycle_0.profile'.
                cycle_name = "cycle_{0}.profile".format(cycle)
                path_in_zip = \
                    os.path.join(
                        "profile_{0}".format(self.test_config['name']),
                        testname,
                        cycle_name
                    )
                LOG.info(
                    "Adding profile {0} to archive {1}"
                    .format(path_in_zip, self.profile_arcname)
                )
                try:
                    arc.write(profile_path, path_in_zip)
                except Exception:
                    LOG.exception(
                        "Failed to copy profile {0} as {1} to"
                        " archive {2}".format(profile_path,
                                              path_in_zip,
                                              self.profile_arcname)
                    )
示例#18
0
    def symbolicate(self):
        """
        Symbolicate Gecko profiling data for one pagecycle.

        """
        profiles = self.collect_profiles()
        if len(profiles) == 0:
            LOG.error("No profiles collected")
            return

        symbolicator = ProfileSymbolicator({
            # Trace-level logging (verbose)
            "enableTracing": 0,
            # Fallback server if symbol is not found locally
            "remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
            # Maximum number of symbol files to keep in memory
            "maxCacheEntries": 2000000,
            # Frequency of checking for recent symbols to
            # cache (in hours)
            "prefetchInterval": 12,
            # Oldest file age to prefetch (in hours)
            "prefetchThreshold": 48,
            # Maximum number of library versions to pre-fetch
            # per library
            "prefetchMaxSymbolsPerLib": 3,
            # Default symbol lookup directories
            "defaultApp": "FIREFOX",
            "defaultOs": "WINDOWS",
            # Paths to .SYM files, expressed internally as a
            # mapping of app or platform names to directories
            # Note: App & OS names from requests are converted
            # to all-uppercase internally
            "symbolPaths": self.symbol_paths,
        })

        if self.raptor_config.get("symbols_path") is not None:
            if mozfile.is_url(self.raptor_config["symbols_path"]):
                symbolicator.integrate_symbol_zip_from_url(
                    self.raptor_config["symbols_path"])
            elif os.path.isfile(self.raptor_config["symbols_path"]):
                symbolicator.integrate_symbol_zip_from_file(
                    self.raptor_config["symbols_path"])
            elif os.path.isdir(self.raptor_config["symbols_path"]):
                sym_path = self.raptor_config["symbols_path"]
                symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
                self.cleanup = False

        missing_symbols_zip = os.path.join(self.upload_dir,
                                           "missingsymbols.zip")

        try:
            mode = zipfile.ZIP_DEFLATED
        except NameError:
            mode = zipfile.ZIP_STORED

        with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
            for profile_info in profiles:
                profile_path = profile_info["path"]

                LOG.info("Opening profile at %s" % profile_path)
                profile = self._open_gecko_profile(profile_path)

                LOG.info("Symbolicating profile from %s" % profile_path)
                symbolicated_profile = self._symbolicate_profile(
                    profile, missing_symbols_zip, symbolicator)

                try:
                    # Write the profiles into a set of folders formatted as:
                    # <TEST-NAME>-<TEST_TYPE>. The file names have a count prefixed
                    # to them to prevent any naming conflicts. The count is the
                    # number of files already in the folder.
                    folder_name = "%s-%s" % (
                        self.test_config["name"],
                        profile_info["type"],
                    )
                    profile_name = "-".join([
                        str(
                            len([
                                f for f in arc.namelist() if folder_name in f
                            ]) + 1),
                        os.path.split(profile_path)[-1],
                    ])
                    path_in_zip = os.path.join(folder_name, profile_name)

                    LOG.info("Adding profile %s to archive %s as %s" %
                             (profile_path, self.profile_arcname, path_in_zip))
                    arc.writestr(
                        path_in_zip,
                        json.dumps(symbolicated_profile,
                                   ensure_ascii=False).encode("utf-8"),
                    )
                except Exception:
                    LOG.exception(
                        "Failed to add symbolicated profile %s to archive %s" %
                        (profile_path, self.profile_arcname))
                    raise

        # save the latest gecko profile archive to an env var, so later on
        # it can be viewed automatically via the view-gecko-profile tool
        os.environ[
            "RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
示例#19
0
    def install_from_path(self, path, unpack=False):
        """
        Installs addon from a filepath, url or directory of addons in the profile.

        :param path: url, path to .xpi, or directory of addons
        :param unpack: whether to unpack unless specified otherwise in the install.rdf
        """

        # if the addon is a URL, download it
        # note that this won't work with protocols urllib2 doesn't support
        if mozfile.is_url(path):
            response = urllib2.urlopen(path)
            fd, path = tempfile.mkstemp(suffix='.xpi')
            os.write(fd, response.read())
            os.close(fd)

            self.downloaded_addons.append(path)

        addons = [path]

        # if path is not an add-on, try to install all contained add-ons
        if not self.is_addon(path):
            # If the path doesn't exist, then we don't really care, just return
            if not os.path.isdir(path):
                return
            addons = [
                os.path.join(path, x) for x in os.listdir(path)
                if self.is_addon(os.path.join(path, x))
            ]
            addons.sort()

        # install each addon
        for addon in addons:
            # determine the addon id
            addon_details = self.addon_details(addon)
            addon_id = addon_details.get('id')
            assert addon_id, 'The addon id could not be found: %s' % addon

            # if the add-on has to be unpacked force it now
            # note: we might want to let Firefox do it in case of addon details
            orig_path = None
            if os.path.isfile(addon) and (unpack or addon_details['unpack']):
                orig_path = addon
                addon = tempfile.mkdtemp()
                mozfile.extract(orig_path, addon)

            # copy the addon to the profile
            extensions_path = os.path.join(self.profile, 'extensions',
                                           'staged')
            addon_path = os.path.join(extensions_path, addon_id)

            if os.path.isfile(addon):
                addon_path += '.xpi'

                # save existing xpi file to restore later
                if os.path.exists(addon_path):
                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
                    shutil.copy(addon_path, self.backup_dir)

                if not os.path.exists(extensions_path):
                    os.makedirs(extensions_path)
                shutil.copy(addon, addon_path)
            else:
                # save existing dir to restore later
                if os.path.exists(addon_path):
                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
                    dest = os.path.join(self.backup_dir,
                                        os.path.basename(addon_path))
                    dir_util.copy_tree(addon_path, dest, preserve_symlinks=1)

                dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)

            # if we had to extract the addon, remove the temporary directory
            if orig_path:
                dir_util.remove_tree(addon)
                addon = orig_path

            self._addons.append(addon_id)
            self.installed_addons.append(addon)
示例#20
0
 def test_is_url(self):
     self.assertTrue(is_url("http://mozilla.org"))
     self.assertFalse(is_url("/usr/bin/mozilla.org"))
     self.assertTrue(is_url("file:///usr/bin/mozilla.org"))
     self.assertFalse(is_url("c:\foo\bar"))
示例#21
0
def check_for_crashes(dump_directory, symbols_path,
                      stackwalk_binary=None,
                      dump_save_path=None,
                      test_name=None,
                      quiet=False):
    """
    Print a stack trace for minidump files left behind by a crashing program.

    `dump_directory` will be searched for minidump files. Any minidump files found will
    have `stackwalk_binary` executed on them, with `symbols_path` passed as an extra
    argument.

    `stackwalk_binary` should be a path to the minidump_stackwalk binary.
    If `stackwalk_binary` is not set, the MINIDUMP_STACKWALK environment variable
    will be checked and its value used if it is not empty.

    `symbols_path` should be a path to a directory containing symbols to use for
    dump processing. This can either be a path to a directory containing Breakpad-format
    symbols, or a URL to a zip file containing a set of symbols.

    If `dump_save_path` is set, it should be a path to a directory in which to copy minidump
    files for safekeeping after a stack trace has been printed. If not set, the environment
    variable MINIDUMP_SAVE_PATH will be checked and its value used if it is not empty.

    If `test_name` is set it will be used as the test name in log output. If not set the
    filename of the calling function will be used.

    If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a
    crash is detected.

    Returns True if any minidumps were found, False otherwise.
    """
    dumps = glob.glob(os.path.join(dump_directory, '*.dmp'))
    if not dumps:
        return False

    if stackwalk_binary is None:
        stackwalk_binary = os.environ.get('MINIDUMP_STACKWALK', None)

    # try to get the caller's filename if no test name is given
    if test_name is None:
        try:
            test_name = os.path.basename(sys._getframe(1).f_code.co_filename)
        except:
            test_name = "unknown"

    try:
        log = mozlog.getLogger('mozcrash')
        remove_symbols = False
        # If our symbols are at a remote URL, download them now
        # We want to download URLs like http://... but not Windows paths like c:\...
        if symbols_path and mozfile.is_url(symbols_path):
            log.info("Downloading symbols from: %s", symbols_path)
            remove_symbols = True
            # Get the symbols and write them to a temporary zipfile
            data = urllib2.urlopen(symbols_path)
            symbols_file = tempfile.TemporaryFile()
            symbols_file.write(data.read())
            # extract symbols to a temporary directory (which we'll delete after
            # processing all crashes)
            symbols_path = tempfile.mkdtemp()
            zfile = zipfile.ZipFile(symbols_file, 'r')
            mozfile.extract_zip(zfile, symbols_path)
            zfile.close()

        for d in dumps:
            extra = os.path.splitext(d)[0] + '.extra'

            stackwalk_output = []
            stackwalk_output.append("Crash dump filename: " + d)
            top_frame = None
            if symbols_path and stackwalk_binary and os.path.exists(stackwalk_binary):
                # run minidump_stackwalk
                p = subprocess.Popen([stackwalk_binary, d, symbols_path],
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE)
                (out, err) = p.communicate()
                if len(out) > 3:
                    # minidump_stackwalk is chatty,
                    # so ignore stderr when it succeeds.
                    stackwalk_output.append(out)
                    # The top frame of the crash is always the line after "Thread N (crashed)"
                    # Examples:
                    #  0  libc.so + 0xa888
                    #  0  libnss3.so!nssCertificate_Destroy [certificate.c : 102 + 0x0]
                    #  0  mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0]
                    #  0  libxul.so!void js::gc::MarkInternal<JSObject>(JSTracer*, JSObject**) [Marking.cpp : 92 + 0x28]
                    lines = out.splitlines()
                    for i, line in enumerate(lines):
                        if "(crashed)" in line:
                            match = re.search(r"^ 0  (?:.*!)?(?:void )?([^\[]+)", lines[i+1])
                            if match:
                                top_frame = "@ %s" % match.group(1).strip()
                            break
                else:
                    stackwalk_output.append("stderr from minidump_stackwalk:")
                    stackwalk_output.append(err)
                if p.returncode != 0:
                    stackwalk_output.append("minidump_stackwalk exited with return code %d" % p.returncode)
            else:
                if not symbols_path:
                    stackwalk_output.append("No symbols path given, can't process dump.")
                if not stackwalk_binary:
                    stackwalk_output.append("MINIDUMP_STACKWALK not set, can't process dump.")
                elif stackwalk_binary and not os.path.exists(stackwalk_binary):
                    stackwalk_output.append("MINIDUMP_STACKWALK binary not found: %s" % stackwalk_binary)
            if not top_frame:
                top_frame = "Unknown top frame"
            if not quiet:
                print "PROCESS-CRASH | %s | application crashed [%s]" % (test_name, top_frame)
                print '\n'.join(stackwalk_output)
            if dump_save_path is None:
                dump_save_path = os.environ.get('MINIDUMP_SAVE_PATH', None)
            if dump_save_path:
                # This code did not previously create the directory,
                # so there may be a file hanging out with its name.
                if os.path.isfile(dump_save_path):
                    os.unlink(dump_save_path)
                if not os.path.isdir(dump_save_path):
                    try:
                        os.makedirs(dump_save_path)
                    except OSError:
                        pass

                shutil.move(d, dump_save_path)
                log.info("Saved minidump as %s",
                         os.path.join(dump_save_path, os.path.basename(d)))

                if os.path.isfile(extra):
                    shutil.move(extra, dump_save_path)
                    log.info("Saved app info as %s",
                             os.path.join(dump_save_path, os.path.basename(extra)))
            else:
                mozfile.remove(d)
                mozfile.remove(extra)
    finally:
        if remove_symbols:
            mozfile.remove(symbols_path)

    return True
示例#22
0
def symbolicate_profile_json(profile_path, objdir_path):
    """
    Symbolicate a single JSON profile.
    """
    temp_dir = tempfile.mkdtemp()
    firefox_symbol_path = os.path.join(temp_dir, "firefox")
    windows_symbol_path = os.path.join(temp_dir, "windows")
    os.mkdir(firefox_symbol_path)
    os.mkdir(windows_symbol_path)

    symbol_paths = {"FIREFOX": firefox_symbol_path, "WINDOWS": windows_symbol_path}

    symbolicator = ProfileSymbolicator(
        {
            # Trace-level logging (verbose)
            "enableTracing": 0,
            # Fallback server if symbol is not found locally
            "remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
            # Maximum number of symbol files to keep in memory
            "maxCacheEntries": 2000000,
            # Frequency of checking for recent symbols to
            # cache (in hours)
            "prefetchInterval": 12,
            # Oldest file age to prefetch (in hours)
            "prefetchThreshold": 48,
            # Maximum number of library versions to pre-fetch
            # per library
            "prefetchMaxSymbolsPerLib": 3,
            # Default symbol lookup directories
            "defaultApp": "FIREFOX",
            "defaultOs": "WINDOWS",
            # Paths to .SYM files, expressed internally as a
            # mapping of app or platform names to directories
            # Note: App & OS names from requests are converted
            # to all-uppercase internally
            "symbolPaths": symbol_paths,
        }
    )

    symbol_path = os.path.join(objdir_path, "dist", "crashreporter-symbols")
    missing_symbols_zip = os.path.join(tempfile.mkdtemp(), "missingsymbols.zip")

    if mozfile.is_url(symbol_path):
        symbolicator.integrate_symbol_zip_from_url(symbol_path)
    elif os.path.isfile(symbol_path):
        symbolicator.integrate_symbol_zip_from_file(symbol_path)
    elif os.path.isdir(symbol_path):
        symbol_paths["FIREFOX"] = symbol_path

    LOG.info(
        "Symbolicating the performance profile... This could take a couple "
        "of minutes."
    )

    try:
        with open(profile_path, "r") as profile_file:
            profile = json.load(profile_file)
        symbolicator.dump_and_integrate_missing_symbols(profile, missing_symbols_zip)
        symbolicator.symbolicate_profile(profile)
        # Overwrite the profile in place.
        save_gecko_profile(profile, profile_path)
    except MemoryError:
        LOG.error(
            "Ran out of memory while trying"
            " to symbolicate profile {0}".format(profile_path)
        )
    except Exception as e:
        LOG.error("Encountered an exception during profile symbolication")
        LOG.error(e)

    shutil.rmtree(temp_dir)