def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
path = self.download(path)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
try:
self.addon_details(path)
except AddonFormatError, e:
module_logger.warning('Could not install %s: %s' % (path, str(e)))
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [os.path.join(path, x) for x in os.listdir(path) if
self.is_addon(os.path.join(path, x))]
addons.sort()
def read(cls, path):
"""read preferences from a file"""
section = None # for .ini files
basename = os.path.basename(path)
if ":" in basename:
# section of INI file
path, section = path.rsplit(":", 1)
if not os.path.exists(path) and not mozfile.is_url(path):
raise PreferencesReadError("'%s' does not exist" % path)
if section:
try:
return cls.read_ini(path, section)
except PreferencesReadError:
raise
except Exception as e:
raise PreferencesReadError(str(e))
# try both JSON and .ini format
try:
return cls.read_json(path)
except Exception as e:
try:
return cls.read_ini(path)
except Exception as f:
for exception in e, f:
if isinstance(exception, PreferencesReadError):
raise exception
raise PreferencesReadError("Could not recognize format of %s" % path)
def read(cls, path):
"""read preferences from a file"""
section = None # for .ini files
basename = os.path.basename(path)
if ":" in basename:
# section of INI file
path, section = path.rsplit(":", 1)
if not os.path.exists(path) and not mozfile.is_url(path):
raise PreferencesReadError("'%s' does not exist" % path)
if section:
try:
return cls.read_ini(path, section)
except PreferencesReadError:
raise
except Exception as e:
raise PreferencesReadError(str(e))
# try both JSON and .ini format
try:
return cls.read_json(path)
except Exception as e:
try:
return cls.read_ini(path)
except Exception as f:
for exception in e, f:
if isinstance(exception, PreferencesReadError):
raise exception
raise PreferencesReadError("Could not recognize format of %s" % path)
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
path = self.download(path)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
try:
self.addon_details(path)
except AddonFormatError, e:
module_logger.warning('Could not install %s: %s' % (path, str(e)))
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [
os.path.join(path, x) for x in os.listdir(path)
if self.is_addon(os.path.join(path, x))
]
addons.sort()
def _get_symbols(self):
# This updates self.symbols_path so we only download once
if self.symbols_path and mozfile.is_url(self.symbols_path):
self.remove_symbols = True
self.logger.info("Downloading symbols from: %s" % self.symbols_path)
# Get the symbols and write them to a temporary zipfile
data = urllib2.urlopen(self.symbols_path)
with tempfile.TemporaryFile() as symbols_file:
symbols_file.write(data.read())
# extract symbols to a temporary directory (which we'll delete after
# processing all crashes)
self.symbols_path = tempfile.mkdtemp()
with zipfile.ZipFile(symbols_file, 'r') as zfile:
mozfile.extract_zip(zfile, self.symbols_path)
def _get_symbols(self):
# This updates self.symbols_path so we only download once
if self.symbols_path and mozfile.is_url(self.symbols_path):
self.remove_symbols = True
self.logger.info("Downloading symbols from: %s" % self.symbols_path)
# Get the symbols and write them to a temporary zipfile
data = urllib2.urlopen(self.symbols_path)
with tempfile.TemporaryFile() as symbols_file:
symbols_file.write(data.read())
# extract symbols to a temporary directory (which we'll delete after
# processing all crashes)
self.symbols_path = tempfile.mkdtemp()
with zipfile.ZipFile(symbols_file, 'r') as zfile:
mozfile.extract_zip(zfile, self.symbols_path)
def read(cls, path):
"""read preferences from a file"""
section = None # for .ini files
basename = os.path.basename(path)
if ':' in basename:
# section of INI file
path, section = path.rsplit(':', 1)
if not os.path.exists(path) and not mozfile.is_url(path):
raise PreferencesReadError("'%s' does not exist" % path)
if section:
try:
return cls.read_ini(path, section)
except PreferencesReadError:
raise
except Exception, e:
raise PreferencesReadError(str(e))
def read(cls, path):
"""read preferences from a file"""
section = None # for .ini files
basename = os.path.basename(path)
if ':' in basename:
# section of INI file
path, section = path.rsplit(':', 1)
if not os.path.exists(path) and not mozfile.is_url(path):
raise PreferencesReadError("'%s' does not exist" % path)
if section:
try:
return cls.read_ini(path, section)
except PreferencesReadError:
raise
except Exception, e:
raise PreferencesReadError(str(e))
def _get_symbols(self):
# If no symbols path has been set create a temporary folder to let the
# minidump stackwalk download the symbols.
if not self.symbols_path:
self.symbols_path = tempfile.mkdtemp()
self.remove_symbols = True
# This updates self.symbols_path so we only download once.
if mozfile.is_url(self.symbols_path):
self.remove_symbols = True
self.logger.info("Downloading symbols from: %s" % self.symbols_path)
# Get the symbols and write them to a temporary zipfile
data = urlopen(self.symbols_path)
with tempfile.TemporaryFile() as symbols_file:
symbols_file.write(data.read())
# extract symbols to a temporary directory (which we'll delete after
# processing all crashes)
self.symbols_path = tempfile.mkdtemp()
with zipfile.ZipFile(symbols_file, 'r') as zfile:
mozfile.extract_zip(zfile, self.symbols_path)
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
path = self.download(path)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
try:
self.addon_details(path)
except AddonFormatError as e:
module_logger.warning('Could not install %s: %s' % (path, str(e)))
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [os.path.join(path, x) for x in os.listdir(path) if
self.is_addon(os.path.join(path, x))]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions', 'staged')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# move existing xpi file to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# move existing folder to backup location to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.move(addon_path, self.backup_dir)
# copy new add-on to the extension folder
shutil.copytree(addon, addon_path, symlinks=True)
# if we had to extract the addon, remove the temporary directory
if orig_path:
mozfile.remove(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
def symbolicate(self):
"""
Symbolicate Gecko profiling data for one pagecycle.
"""
profiles = self.collect_profiles()
if len(profiles) == 0:
LOG.error("No profiles collected")
return
symbolicator = ProfileSymbolicator({
# Trace-level logging (verbose)
"enableTracing": 0,
# Fallback server if symbol is not found locally
"remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
# Maximum number of symbol files to keep in memory
"maxCacheEntries": 2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval": 12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold": 48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib": 3,
# Default symbol lookup directories
"defaultApp": "FIREFOX",
"defaultOs": "WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths": self.symbol_paths,
})
if self.raptor_config.get("symbols_path") is not None:
if mozfile.is_url(self.raptor_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_url(
self.raptor_config["symbols_path"])
elif os.path.isfile(self.raptor_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_file(
self.raptor_config["symbols_path"])
elif os.path.isdir(self.raptor_config["symbols_path"]):
sym_path = self.raptor_config["symbols_path"]
symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
self.cleanup = False
missing_symbols_zip = os.path.join(self.upload_dir,
"missingsymbols.zip")
try:
mode = zipfile.ZIP_DEFLATED
except NameError:
mode = zipfile.ZIP_STORED
with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
for profile_filename in profiles:
testname = profile_filename
if testname.endswith(".profile"):
testname = testname[0:-8]
profile_path = os.path.join(self.gecko_profile_dir,
profile_filename)
self._save_gecko_profile(symbolicator, missing_symbols_zip,
profile_path)
# Our zip will contain one directory per test,
# and each directory will contain one or more
# *.profile files - one for each pagecycle
path_in_zip = os.path.join(
"profile_{0}".format(self.test_config["name"]),
testname + ".profile",
)
LOG.info("Adding profile {0} to archive {1}".format(
path_in_zip, self.profile_arcname))
try:
arc.write(profile_path, path_in_zip)
except Exception:
LOG.exception("Failed to copy profile {0} as {1} to"
" archive {2}".format(
profile_path, path_in_zip,
self.profile_arcname))
# save the latest gecko profile archive to an env var, so later on
# it can be viewed automatically via the view-gecko-profile tool
os.environ[
"RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
def test_is_url(self):
self.assertTrue(is_url('http://mozilla.org'))
self.assertFalse(is_url('/usr/bin/mozilla.org'))
self.assertTrue(is_url('file:///usr/bin/mozilla.org'))
self.assertFalse(is_url('c:\foo\bar'))
def test_is_url(self):
self.assertTrue(is_url('http://mozilla.org'))
self.assertFalse(is_url('/usr/bin/mozilla.org'))
self.assertTrue(is_url('file:///usr/bin/mozilla.org'))
self.assertFalse(is_url('c:\foo\bar'))
def symbolicate(self, cycle):
"""
Symbolicate sps profiling data for one cycle.
:param cycle: the number of the cycle of the test currently run.
"""
symbolicator = symbolication.ProfileSymbolicator({
# Trace-level logging (verbose)
"enableTracing":
0,
# Fallback server if symbol is not found locally
"remoteSymbolServer":
"http://symbolapi.mozilla.org:80/talos/",
# Maximum number of symbol files to keep in memory
"maxCacheEntries":
2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval":
12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold":
48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib":
3,
# Default symbol lookup directories
"defaultApp":
"FIREFOX",
"defaultOs":
"WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths":
self.symbol_paths
})
if self.browser_config['symbols_path']:
if mozfile.is_url(self.browser_config['symbols_path']):
symbolicator.integrate_symbol_zip_from_url(
self.browser_config['symbols_path'])
else:
symbolicator.integrate_symbol_zip_from_file(
self.browser_config['symbols_path'])
missing_symbols_zip = os.path.join(self.upload_dir,
"missingsymbols.zip")
try:
mode = zipfile.ZIP_DEFLATED
except NameError:
mode = zipfile.ZIP_STORED
sps_profile_dir = self.option('dir')
with zipfile.ZipFile(self.profile_arcname, 'a', mode) as arc:
# Collect all individual profiles that the test
# has put into sps_profile_dir.
for profile_filename in os.listdir(sps_profile_dir):
testname = profile_filename
if testname.endswith(".sps"):
testname = testname[0:-4]
profile_path = os.path.join(sps_profile_dir, profile_filename)
self._save_sps_profile(cycle, symbolicator,
missing_symbols_zip, profile_path)
# Our zip will contain one directory per subtest,
# and each subtest directory will contain one or
# more cycle_i.sps files. For example, with
# test_config['name'] == 'tscrollx',
# profile_filename == 'iframe.svg.sps', i == 0,
# we'll get path_in_zip ==
# 'profile_tscrollx/iframe.svg/cycle_0.sps'.
cycle_name = "cycle_{0}.sps".format(cycle)
path_in_zip = \
os.path.join(
"profile_{0}".format(self.test_config['name']),
testname,
cycle_name
)
LOG.info("Adding profile {0} to archive {1}".format(
path_in_zip, self.profile_arcname))
try:
arc.write(profile_path, path_in_zip)
except Exception:
LOG.exception("Failed to copy profile {0} as {1} to"
" archive {2}".format(
profile_path, path_in_zip,
self.profile_arcname))
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
response = urllib2.urlopen(path)
fd, path = tempfile.mkstemp(suffix=".xpi")
os.write(fd, response.read())
os.close(fd)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
if not self.is_addon(path):
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [os.path.join(path, x) for x in os.listdir(path) if self.is_addon(os.path.join(path, x))]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get("id")
assert addon_id, "The addon id could not be found: %s" % addon
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details["unpack"]):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, "extensions", "staged")
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += ".xpi"
# save existing xpi file to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.copy(addon_path, self.backup_dir)
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# save existing dir to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
dest = os.path.join(self.backup_dir, os.path.basename(addon_path))
dir_util.copy_tree(addon_path, dest, preserve_symlinks=1)
dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)
# if we had to extract the addon, remove the temporary directory
if orig_path:
dir_util.remove_tree(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
def symbolicate(self, cycle):
"""
Symbolicate Gecko profiling data for one cycle.
:param cycle: the number of the cycle of the test currently run.
"""
symbolicator = ProfileSymbolicator({
# Trace-level logging (verbose)
"enableTracing": 0,
# Fallback server if symbol is not found locally
"remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
# Maximum number of symbol files to keep in memory
"maxCacheEntries": 2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval": 12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold": 48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib": 3,
# Default symbol lookup directories
"defaultApp": "FIREFOX",
"defaultOs": "WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths": self.symbol_paths,
})
if self.browser_config["symbols_path"]:
if mozfile.is_url(self.browser_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_url(
self.browser_config["symbols_path"])
elif os.path.isfile(self.browser_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_file(
self.browser_config["symbols_path"])
elif os.path.isdir(self.browser_config["symbols_path"]):
sym_path = self.browser_config["symbols_path"]
symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
self.cleanup = False
missing_symbols_zip = os.path.join(self.upload_dir,
"missingsymbols.zip")
try:
mode = zipfile.ZIP_DEFLATED
except NameError:
mode = zipfile.ZIP_STORED
gecko_profile_dir = self.option("dir")
with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
# Collect all individual profiles that the test
# has put into gecko_profile_dir.
for profile_filename in os.listdir(gecko_profile_dir):
testname = profile_filename
if testname.endswith(".profile"):
testname = testname[0:-8]
profile_path = os.path.join(gecko_profile_dir,
profile_filename)
self._save_gecko_profile(cycle, symbolicator,
missing_symbols_zip, profile_path)
# Our zip will contain one directory per subtest,
# and each subtest directory will contain one or
# more cycle_i.profile files. For example, with
# test_config['name'] == 'tscrollx',
# profile_filename == 'iframe.svg.profile', i == 0,
# we'll get path_in_zip ==
# 'profile_tscrollx/iframe.svg/cycle_0.profile'.
cycle_name = "cycle_{0}.profile".format(cycle)
path_in_zip = os.path.join(
"profile_{0}".format(self.test_config["name"]), testname,
cycle_name)
LOG.info("Adding profile {0} to archive {1}".format(
path_in_zip, self.profile_arcname))
try:
arc.write(profile_path, path_in_zip)
except Exception:
LOG.exception("Failed to copy profile {0} as {1} to"
" archive {2}".format(
profile_path, path_in_zip,
self.profile_arcname))
# save the latest gecko profile archive to an env var, so later on
# it can be viewed automatically via the view-gecko-profile tool
os.environ[
"TALOS_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
def symbolicate(self, cycle):
"""
Symbolicate Gecko profiling data for one cycle.
:param cycle: the number of the cycle of the test currently run.
"""
symbolicator = symbolication.ProfileSymbolicator({
# Trace-level logging (verbose)
"enableTracing": 0,
# Fallback server if symbol is not found locally
"remoteSymbolServer":
"http://symbolapi.mozilla.org:80/talos/",
# Maximum number of symbol files to keep in memory
"maxCacheEntries": 2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval": 12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold": 48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib": 3,
# Default symbol lookup directories
"defaultApp": "FIREFOX",
"defaultOs": "WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths": self.symbol_paths
})
if self.browser_config['symbols_path']:
if mozfile.is_url(self.browser_config['symbols_path']):
symbolicator.integrate_symbol_zip_from_url(
self.browser_config['symbols_path']
)
else:
symbolicator.integrate_symbol_zip_from_file(
self.browser_config['symbols_path']
)
missing_symbols_zip = os.path.join(self.upload_dir,
"missingsymbols.zip")
try:
mode = zipfile.ZIP_DEFLATED
except NameError:
mode = zipfile.ZIP_STORED
gecko_profile_dir = self.option('dir')
with zipfile.ZipFile(self.profile_arcname, 'a', mode) as arc:
# Collect all individual profiles that the test
# has put into gecko_profile_dir.
for profile_filename in os.listdir(gecko_profile_dir):
testname = profile_filename
if testname.endswith(".profile"):
testname = testname[0:-8]
profile_path = os.path.join(gecko_profile_dir, profile_filename)
self._save_gecko_profile(cycle, symbolicator,
missing_symbols_zip,
profile_path)
# Our zip will contain one directory per subtest,
# and each subtest directory will contain one or
# more cycle_i.profile files. For example, with
# test_config['name'] == 'tscrollx',
# profile_filename == 'iframe.svg.profile', i == 0,
# we'll get path_in_zip ==
# 'profile_tscrollx/iframe.svg/cycle_0.profile'.
cycle_name = "cycle_{0}.profile".format(cycle)
path_in_zip = \
os.path.join(
"profile_{0}".format(self.test_config['name']),
testname,
cycle_name
)
LOG.info(
"Adding profile {0} to archive {1}"
.format(path_in_zip, self.profile_arcname)
)
try:
arc.write(profile_path, path_in_zip)
except Exception:
LOG.exception(
"Failed to copy profile {0} as {1} to"
" archive {2}".format(profile_path,
path_in_zip,
self.profile_arcname)
)
def symbolicate(self):
"""
Symbolicate Gecko profiling data for one pagecycle.
"""
profiles = self.collect_profiles()
if len(profiles) == 0:
LOG.error("No profiles collected")
return
symbolicator = ProfileSymbolicator({
# Trace-level logging (verbose)
"enableTracing": 0,
# Fallback server if symbol is not found locally
"remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
# Maximum number of symbol files to keep in memory
"maxCacheEntries": 2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval": 12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold": 48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib": 3,
# Default symbol lookup directories
"defaultApp": "FIREFOX",
"defaultOs": "WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths": self.symbol_paths,
})
if self.raptor_config.get("symbols_path") is not None:
if mozfile.is_url(self.raptor_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_url(
self.raptor_config["symbols_path"])
elif os.path.isfile(self.raptor_config["symbols_path"]):
symbolicator.integrate_symbol_zip_from_file(
self.raptor_config["symbols_path"])
elif os.path.isdir(self.raptor_config["symbols_path"]):
sym_path = self.raptor_config["symbols_path"]
symbolicator.options["symbolPaths"]["FIREFOX"] = sym_path
self.cleanup = False
missing_symbols_zip = os.path.join(self.upload_dir,
"missingsymbols.zip")
try:
mode = zipfile.ZIP_DEFLATED
except NameError:
mode = zipfile.ZIP_STORED
with zipfile.ZipFile(self.profile_arcname, "a", mode) as arc:
for profile_info in profiles:
profile_path = profile_info["path"]
LOG.info("Opening profile at %s" % profile_path)
profile = self._open_gecko_profile(profile_path)
LOG.info("Symbolicating profile from %s" % profile_path)
symbolicated_profile = self._symbolicate_profile(
profile, missing_symbols_zip, symbolicator)
try:
# Write the profiles into a set of folders formatted as:
# <TEST-NAME>-<TEST_TYPE>. The file names have a count prefixed
# to them to prevent any naming conflicts. The count is the
# number of files already in the folder.
folder_name = "%s-%s" % (
self.test_config["name"],
profile_info["type"],
)
profile_name = "-".join([
str(
len([
f for f in arc.namelist() if folder_name in f
]) + 1),
os.path.split(profile_path)[-1],
])
path_in_zip = os.path.join(folder_name, profile_name)
LOG.info("Adding profile %s to archive %s as %s" %
(profile_path, self.profile_arcname, path_in_zip))
arc.writestr(
path_in_zip,
json.dumps(symbolicated_profile,
ensure_ascii=False).encode("utf-8"),
)
except Exception:
LOG.exception(
"Failed to add symbolicated profile %s to archive %s" %
(profile_path, self.profile_arcname))
raise
# save the latest gecko profile archive to an env var, so later on
# it can be viewed automatically via the view-gecko-profile tool
os.environ[
"RAPTOR_LATEST_GECKO_PROFILE_ARCHIVE"] = self.profile_arcname
def install_from_path(self, path, unpack=False):
"""
Installs addon from a filepath, url or directory of addons in the profile.
:param path: url, path to .xpi, or directory of addons
:param unpack: whether to unpack unless specified otherwise in the install.rdf
"""
# if the addon is a URL, download it
# note that this won't work with protocols urllib2 doesn't support
if mozfile.is_url(path):
response = urllib2.urlopen(path)
fd, path = tempfile.mkstemp(suffix='.xpi')
os.write(fd, response.read())
os.close(fd)
self.downloaded_addons.append(path)
addons = [path]
# if path is not an add-on, try to install all contained add-ons
if not self.is_addon(path):
# If the path doesn't exist, then we don't really care, just return
if not os.path.isdir(path):
return
addons = [
os.path.join(path, x) for x in os.listdir(path)
if self.is_addon(os.path.join(path, x))
]
addons.sort()
# install each addon
for addon in addons:
# determine the addon id
addon_details = self.addon_details(addon)
addon_id = addon_details.get('id')
assert addon_id, 'The addon id could not be found: %s' % addon
# if the add-on has to be unpacked force it now
# note: we might want to let Firefox do it in case of addon details
orig_path = None
if os.path.isfile(addon) and (unpack or addon_details['unpack']):
orig_path = addon
addon = tempfile.mkdtemp()
mozfile.extract(orig_path, addon)
# copy the addon to the profile
extensions_path = os.path.join(self.profile, 'extensions',
'staged')
addon_path = os.path.join(extensions_path, addon_id)
if os.path.isfile(addon):
addon_path += '.xpi'
# save existing xpi file to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
shutil.copy(addon_path, self.backup_dir)
if not os.path.exists(extensions_path):
os.makedirs(extensions_path)
shutil.copy(addon, addon_path)
else:
# save existing dir to restore later
if os.path.exists(addon_path):
self.backup_dir = self.backup_dir or tempfile.mkdtemp()
dest = os.path.join(self.backup_dir,
os.path.basename(addon_path))
dir_util.copy_tree(addon_path, dest, preserve_symlinks=1)
dir_util.copy_tree(addon, addon_path, preserve_symlinks=1)
# if we had to extract the addon, remove the temporary directory
if orig_path:
dir_util.remove_tree(addon)
addon = orig_path
self._addons.append(addon_id)
self.installed_addons.append(addon)
def test_is_url(self):
self.assertTrue(is_url("http://mozilla.org"))
self.assertFalse(is_url("/usr/bin/mozilla.org"))
self.assertTrue(is_url("file:///usr/bin/mozilla.org"))
self.assertFalse(is_url("c:\foo\bar"))
def check_for_crashes(dump_directory, symbols_path,
stackwalk_binary=None,
dump_save_path=None,
test_name=None,
quiet=False):
"""
Print a stack trace for minidump files left behind by a crashing program.
`dump_directory` will be searched for minidump files. Any minidump files found will
have `stackwalk_binary` executed on them, with `symbols_path` passed as an extra
argument.
`stackwalk_binary` should be a path to the minidump_stackwalk binary.
If `stackwalk_binary` is not set, the MINIDUMP_STACKWALK environment variable
will be checked and its value used if it is not empty.
`symbols_path` should be a path to a directory containing symbols to use for
dump processing. This can either be a path to a directory containing Breakpad-format
symbols, or a URL to a zip file containing a set of symbols.
If `dump_save_path` is set, it should be a path to a directory in which to copy minidump
files for safekeeping after a stack trace has been printed. If not set, the environment
variable MINIDUMP_SAVE_PATH will be checked and its value used if it is not empty.
If `test_name` is set it will be used as the test name in log output. If not set the
filename of the calling function will be used.
If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a
crash is detected.
Returns True if any minidumps were found, False otherwise.
"""
dumps = glob.glob(os.path.join(dump_directory, '*.dmp'))
if not dumps:
return False
if stackwalk_binary is None:
stackwalk_binary = os.environ.get('MINIDUMP_STACKWALK', None)
# try to get the caller's filename if no test name is given
if test_name is None:
try:
test_name = os.path.basename(sys._getframe(1).f_code.co_filename)
except:
test_name = "unknown"
try:
log = mozlog.getLogger('mozcrash')
remove_symbols = False
# If our symbols are at a remote URL, download them now
# We want to download URLs like http://... but not Windows paths like c:\...
if symbols_path and mozfile.is_url(symbols_path):
log.info("Downloading symbols from: %s", symbols_path)
remove_symbols = True
# Get the symbols and write them to a temporary zipfile
data = urllib2.urlopen(symbols_path)
symbols_file = tempfile.TemporaryFile()
symbols_file.write(data.read())
# extract symbols to a temporary directory (which we'll delete after
# processing all crashes)
symbols_path = tempfile.mkdtemp()
zfile = zipfile.ZipFile(symbols_file, 'r')
mozfile.extract_zip(zfile, symbols_path)
zfile.close()
for d in dumps:
extra = os.path.splitext(d)[0] + '.extra'
stackwalk_output = []
stackwalk_output.append("Crash dump filename: " + d)
top_frame = None
if symbols_path and stackwalk_binary and os.path.exists(stackwalk_binary):
# run minidump_stackwalk
p = subprocess.Popen([stackwalk_binary, d, symbols_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = p.communicate()
if len(out) > 3:
# minidump_stackwalk is chatty,
# so ignore stderr when it succeeds.
stackwalk_output.append(out)
# The top frame of the crash is always the line after "Thread N (crashed)"
# Examples:
# 0 libc.so + 0xa888
# 0 libnss3.so!nssCertificate_Destroy [certificate.c : 102 + 0x0]
# 0 mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0]
# 0 libxul.so!void js::gc::MarkInternal<JSObject>(JSTracer*, JSObject**) [Marking.cpp : 92 + 0x28]
lines = out.splitlines()
for i, line in enumerate(lines):
if "(crashed)" in line:
match = re.search(r"^ 0 (?:.*!)?(?:void )?([^\[]+)", lines[i+1])
if match:
top_frame = "@ %s" % match.group(1).strip()
break
else:
stackwalk_output.append("stderr from minidump_stackwalk:")
stackwalk_output.append(err)
if p.returncode != 0:
stackwalk_output.append("minidump_stackwalk exited with return code %d" % p.returncode)
else:
if not symbols_path:
stackwalk_output.append("No symbols path given, can't process dump.")
if not stackwalk_binary:
stackwalk_output.append("MINIDUMP_STACKWALK not set, can't process dump.")
elif stackwalk_binary and not os.path.exists(stackwalk_binary):
stackwalk_output.append("MINIDUMP_STACKWALK binary not found: %s" % stackwalk_binary)
if not top_frame:
top_frame = "Unknown top frame"
if not quiet:
print "PROCESS-CRASH | %s | application crashed [%s]" % (test_name, top_frame)
print '\n'.join(stackwalk_output)
if dump_save_path is None:
dump_save_path = os.environ.get('MINIDUMP_SAVE_PATH', None)
if dump_save_path:
# This code did not previously create the directory,
# so there may be a file hanging out with its name.
if os.path.isfile(dump_save_path):
os.unlink(dump_save_path)
if not os.path.isdir(dump_save_path):
try:
os.makedirs(dump_save_path)
except OSError:
pass
shutil.move(d, dump_save_path)
log.info("Saved minidump as %s",
os.path.join(dump_save_path, os.path.basename(d)))
if os.path.isfile(extra):
shutil.move(extra, dump_save_path)
log.info("Saved app info as %s",
os.path.join(dump_save_path, os.path.basename(extra)))
else:
mozfile.remove(d)
mozfile.remove(extra)
finally:
if remove_symbols:
mozfile.remove(symbols_path)
return True
def symbolicate_profile_json(profile_path, objdir_path):
"""
Symbolicate a single JSON profile.
"""
temp_dir = tempfile.mkdtemp()
firefox_symbol_path = os.path.join(temp_dir, "firefox")
windows_symbol_path = os.path.join(temp_dir, "windows")
os.mkdir(firefox_symbol_path)
os.mkdir(windows_symbol_path)
symbol_paths = {"FIREFOX": firefox_symbol_path, "WINDOWS": windows_symbol_path}
symbolicator = ProfileSymbolicator(
{
# Trace-level logging (verbose)
"enableTracing": 0,
# Fallback server if symbol is not found locally
"remoteSymbolServer": "https://symbols.mozilla.org/symbolicate/v4",
# Maximum number of symbol files to keep in memory
"maxCacheEntries": 2000000,
# Frequency of checking for recent symbols to
# cache (in hours)
"prefetchInterval": 12,
# Oldest file age to prefetch (in hours)
"prefetchThreshold": 48,
# Maximum number of library versions to pre-fetch
# per library
"prefetchMaxSymbolsPerLib": 3,
# Default symbol lookup directories
"defaultApp": "FIREFOX",
"defaultOs": "WINDOWS",
# Paths to .SYM files, expressed internally as a
# mapping of app or platform names to directories
# Note: App & OS names from requests are converted
# to all-uppercase internally
"symbolPaths": symbol_paths,
}
)
symbol_path = os.path.join(objdir_path, "dist", "crashreporter-symbols")
missing_symbols_zip = os.path.join(tempfile.mkdtemp(), "missingsymbols.zip")
if mozfile.is_url(symbol_path):
symbolicator.integrate_symbol_zip_from_url(symbol_path)
elif os.path.isfile(symbol_path):
symbolicator.integrate_symbol_zip_from_file(symbol_path)
elif os.path.isdir(symbol_path):
symbol_paths["FIREFOX"] = symbol_path
LOG.info(
"Symbolicating the performance profile... This could take a couple "
"of minutes."
)
try:
with open(profile_path, "r") as profile_file:
profile = json.load(profile_file)
symbolicator.dump_and_integrate_missing_symbols(profile, missing_symbols_zip)
symbolicator.symbolicate_profile(profile)
# Overwrite the profile in place.
save_gecko_profile(profile, profile_path)
except MemoryError:
LOG.error(
"Ran out of memory while trying"
" to symbolicate profile {0}".format(profile_path)
)
except Exception as e:
LOG.error("Encountered an exception during profile symbolication")
LOG.error(e)
shutil.rmtree(temp_dir)