def _get_primary_file_path(self, reponame, repourl):
self._setup_dirs(reponame)
repomdfilename = self._get_repo_file_path(reponame,
repourl,
"repodata/repomd.xml")
rplh = RepomdPrimaryLocationHandler()
repomdparser = xml.sax.make_parser()
repomdparser.setContentHandler(rplh)
try:
mdfp = open(repomdfilename, "r")
except Exception as ex:
raise FafError("Reading: {0}".format(str(ex)))
else:
with mdfp:
try:
repomdparser.parse(mdfp)
except SAXException as ex:
raise FafError("Failed to parse repomd.xml: {0}"
.format(str(ex)))
return self._get_repo_file_path(reponame,
repourl,
rplh.primary_location)
def store_rpm_provides(db: Database, package: Package, nogpgcheck: bool = False) -> None:
"""
Save RPM provides of `package` to storage.
Expects pyfaf.storage.opsys.Package object.
"""
pkg_id = package.id
ts = rpm.ts()
rpm_file = package.get_lob_fd("package")
if not rpm_file:
raise FafError("Package {0} has no lob stored".format(package.name))
if nogpgcheck:
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) #pylint: disable=protected-access
try:
header = ts.hdrFromFdno(rpm_file.fileno())
except rpm.error as exc:
rpm_file.close()
raise FafError("rpm error: {0}".format(exc)) from exc
files = header.fiFromHeader()
log.debug("%s contains %d files", package.nvra(), len(files))
# Invalid name for type variable
# pylint: disable-msg=C0103
for f in files:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = f[0]
new.flags = 0
db.session.add(new)
provides = header.dsFromHeader('providename')
for p in provides:
if len(p.N()) > 1024:
log.warning("Provides item in RPM header of %s longer than 1024 "
"characters. Skipping", package.name)
continue
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = p.N()
new.flags = p.Flags()
evr = p.EVR()
if evr:
try:
new.epoch, new.version, new.release = parse_evr(evr)
except ValueError as ex:
log.warning("Unparsable EVR ‘%s’ of %s in Provides of %s: %s. "
"Skipping",
evr, p.N(), package.name, ex)
continue
db.session.add(new)
rpm_file.close()
db.session.flush()
def _parse_kernel_build_id(self, build_id, archs):
"""
Parses the kernel build string such as
3.10.0-3.fc19.x86_64
3.10.0-3.fc19.armv7hl.tegra
2.6.32-358.14.1.el6.i686.PAE
3.15.6-200.fc20.i686+PAE
"""
arch = None
flavour = None
splitby = "+" if "+" in build_id else "."
head, tail = build_id.rsplit(splitby, 1)
if tail in archs:
arch = tail
else:
flavour = tail
head, tail = head.rsplit(".", 1)
if not tail in archs:
raise FafError("Unable to determine architecture from '{0}'"
.format(build_id))
arch = tail
try:
version, release = head.rsplit("-", 1)
except ValueError:
raise FafError("Unable to determine release from '{0}'"
.format(head))
return version, release, arch, flavour
def save_lob(self,
name,
data,
binary=True,
overwrite=False,
truncate=False):
lobpath = self.get_lob_path(name)
if not overwrite and os.path.isfile(lobpath):
raise FafError("Lob '{0}' already exists".format(name))
maxlen = self.__lobs__[name]
mode = "w"
if binary:
mode += "b"
with open(lobpath, mode) as lob:
if type(data) in [str, unicode]:
self._save_lob_string(lob, data, maxlen, truncate)
elif hasattr(data, "read"):
if not truncate:
raise FafError(
"When saving from file, truncate must be enabled")
self._save_lob_file(lob, data, maxlen)
else:
raise FafError(
"Data must be either str, unicode or file-like object")
def save_lob(self,
name: str,
data: Union[bytes, BinaryIO],
overwrite: bool = False,
truncate: bool = False):
lobpath = self.get_lob_path(name)
if not isinstance(data, bytes) and not hasattr(data, "read"):
raise FafError("Cannot save LOB '{0}'. Data must be either "
"a bytestring or a file-like object".format(name))
if not overwrite and os.path.isfile(lobpath):
raise FafError("Lob '{0}' already exists".format(name))
maxlen = self.__lobs__[name]
with open(lobpath, "wb") as lob:
if hasattr(data, "read"):
if not truncate:
raise FafError(
"When saving from file, truncate must be enabled")
data = cast(BinaryIO, data)
self._write_lob_file(lob, data, maxlen)
else:
data = cast(bytes, data)
self._write_lob_bytes(lob, data, maxlen, truncate)
def _get_tasks(self, cmdline, db):
result = set()
# no arguments - pull everything for non-EOL releases
if not cmdline.opsys:
for osplugin in systems.values():
db_opsys = get_opsys_by_name(db, osplugin.nice_name)
if db_opsys is None:
raise FafError("Operating system '{0}' is not defined in "
"storage".format(osplugin.nice_name))
for db_release in db_opsys.releases:
if db_release.status != "EOL":
result.add((osplugin, db_release))
# a single opsys - respect opsysrelease
elif len(cmdline.opsys) == 1:
if cmdline.opsys[0] not in systems:
raise FafError("Operating system '{0}' is not supported"
.format(cmdline.opsys[0]))
osplugin = systems[cmdline.opsys[0]]
db_opsys = get_opsys_by_name(db, osplugin.nice_name)
if db_opsys is None:
raise FafError("Operating system '{0}' is not defined in "
"storage".format(osplugin.nice_name))
if len(cmdline.opsys_release) < 1:
for db_release in db_opsys.releases:
result.add((osplugin, db_release))
else:
for release in cmdline.opsys_release:
db_release = get_osrelease(db, osplugin.nice_name, release)
if db_release is None:
self.log_warn("Operating system '{0} {1}' is not "
"supported".format(osplugin.nice_name,
release))
continue
result.add((osplugin, db_release))
# multiple opsys - pull all of their releases
else:
for opsys_name in cmdline.opsys:
if not opsys_name in systems:
self.log_warn("Operating system '{0}' is not supported"
.format(opsys_name))
continue
osplugin = systems[opsys_name]
db_opsys = get_opsys_by_name(db, osplugin.nice_name)
if db_opsys is None:
self.log_warn("Operating system '{0}' is not defined in "
"storage".format(osplugin.nice_name))
continue
for db_release in db_opsys.releases:
result.add((osplugin, db_release))
return sorted(result, key=lambda p_r: (p_r[1].opsys.name, p_r[1].version))
def save_lob(self,
name,
data,
binary=True,
overwrite=False,
truncate=False):
lobpath = self.get_lob_path(name)
if not overwrite and os.path.isfile(lobpath):
raise FafError("Lob '{0}' already exists".format(name))
maxlen = self.__lobs__[name]
mode = "w"
if binary:
mode += "b"
with open(lobpath, mode) as lob:
if sys.version_info.major == 3 and isinstance(data, bytes):
data = data.decode("utf-8")
if isinstance(data, six.string_types):
self._save_lob_string(lob, data, maxlen, truncate)
elif hasattr(data, "read"):
if not truncate:
raise FafError(
"When saving from file, truncate must be enabled")
self._save_lob_file(lob, data, maxlen)
else:
raise FafError(
"Data must be either str, unicode or file-like object")
def _db_backtrace_find_crash_thread(self, db_backtrace):
if len(db_backtrace.threads) == 1:
return db_backtrace.threads[0]
db_threads = [t for t in db_backtrace.threads if t.crashthread]
if not db_threads:
raise FafError("No crash thread could be found for backtrace #{0}"
.format(db_backtrace.id))
if len(db_threads) > 1:
raise FafError("Multiple crash threads found for backtrace #{0}"
.format(db_backtrace.id))
return db_threads[0]
def _hash_backtrace(self,
db_backtrace,
hashbase=None,
offset=False) -> str:
if hashbase is None:
hashbase = []
crashthreads = [t for t in db_backtrace.threads if t.crashthread]
if not crashthreads:
raise FafError("No crash thread found")
if len(crashthreads) > 1:
raise FafError("Multiple crash threads found")
frames = [f for f in crashthreads[0].frames if not f.inlined][:16]
hasnames = all([
f.symbolsource.symbol is not None
and f.symbolsource.symbol.name is not None
and f.symbolsource.symbol.name != "??" for f in frames
])
hashashes = all([f.symbolsource.hash is not None for f in frames])
# use function names if available
if hasnames:
# also hash offset for reports that use it as line numbers
# these reports always have function names
if offset:
hashbase.extend([
"{0} @ {1} + {2}".format(f.symbolsource.symbol.name,
f.symbolsource.path,
f.symbolsource.offset)
for f in frames
])
else:
hashbase.extend([
"{0} @ {1}".format(f.symbolsource.symbol.name,
f.symbolsource.path) for f in frames
])
# fallback to hashes
elif hashashes:
hashbase.extend([
"{0} @ {1}".format(f.symbolsource.hash, f.symbolsource.path)
for f in frames
])
else:
raise FafError("either function names or hashes are required")
return hash_list(hashbase)
def _get_crash_thread(self, stacktrace):
"""
Searches for a single crash thread and return it. Raises FafError if
there is no crash thread or if there are multiple crash threads.
"""
crashthreads = [t for t in stacktrace if ("crash_thread" in t and
t["crash_thread"])]
if not crashthreads:
raise FafError("No crash thread found")
if len(crashthreads) > 1:
raise FafError("Multiple crash threads found")
return crashthreads[0]["frames"]
def validate_packages(self, packages) -> bool:
CentOS.packages_checker.check(packages)
affected = False
for package in packages:
if "package_role" in package:
if package["package_role"] not in CentOS.pkg_roles:
raise FafError("Only the following package roles are allowed: "
"{0}".format(", ".join(CentOS.pkg_roles)))
if package["package_role"] == "affected":
affected = True
if not(affected or self.allow_unpackaged):
raise FafError("uReport must contain affected package")
return True
def _tar_xz(self,
archive_name,
archive_dir,
filepaths,
unlink=True) -> None:
archive_path = os.path.join(archive_dir, archive_name)
archive_path_tmp = os.path.join(archive_dir,
"{0}.tmp".format(archive_name))
tmpdir = get_temp_dir()
tmpsubdir = os.path.join(tmpdir, "archive")
unlink_paths = list(filepaths)
try:
os.makedirs(tmpsubdir)
except OSError as ex:
if ex.errno == errno.EEXIST:
raise FafError("The directory '{0}' already exists".format(
tmpsubdir)) from ex
raise
untar = None
if os.path.isfile(archive_path):
self.log_info("An existing archive found, will merge the contents")
untar = self._untar_xz(archive_path)
with os.scandir(untar) as iterator:
for entry in iterator:
if entry.is_dir() and entry.name.endswith("archive"):
filepaths = [
os.path.join(entry.path, f)
for f in os.listdir(entry.path)
] + filepaths
break
self.log_info("Creating symlinks")
for filepath in filepaths:
linkpath = os.path.join(tmpsubdir, os.path.basename(filepath))
# merge - do not overwrite already archived data
try:
self.log_debug("%s ~> %s", filepath, linkpath)
os.symlink(filepath, linkpath)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
self.log_debug("Already exists")
self.log_info("Running tar")
safe_popen("tar", "chJf", archive_path_tmp, "-C", tmpdir, "archive")
os.rename(archive_path_tmp, archive_path)
self.log_info("Cleaning up")
if untar is not None:
shutil.rmtree(untar, ignore_errors=True)
if unlink:
for path in unlink_paths:
os.unlink(path)
shutil.rmtree(tmpsubdir)
def unpack_rpm_to_tmp(path, prefix="faf"):
"""
Unpack RPM package to a temp directory. The directory is either specified
in storage.tmpdir config option or use the system default temp directory.
"""
tmpdir = config.get("storage.tmpdir", None)
result = tempfile.mkdtemp(prefix=prefix, dir=tmpdir)
for dirname in ["bin", "lib", "lib64", "sbin"]:
os.makedirs(os.path.join(result, "usr", dirname))
os.symlink(os.path.join("usr", dirname), os.path.join(result, dirname))
rpm2cpio = Popen(["rpm2cpio", path], stdout=PIPE, stderr=PIPE)
cpio = Popen(["cpio", "-id", "--quiet"],
stdin=rpm2cpio.stdout,
stderr=PIPE,
cwd=result)
#FIXME: false positive by pylint # pylint: disable=fixme
rpm2cpio.stdout.close() # pylint: disable=no-member
try:
# generous timeout of 15 minutes (kernel unpacking)
cpio.communicate(timeout=900)
except TimeoutExpired:
cpio.kill()
cpio.communicate()
finally:
if cpio.returncode != 0:
shutil.rmtree(result)
raise FafError("Failed to unpack RPM '{0}'".format(path))
return result
def save(db, ureport, create_component=False, timestamp=None, count=1):
"""
Save uReport based on ureport_version element assuming the given uReport "
is valid. Flush the database at the end.
"""
if timestamp is None:
timestamp = datetime.datetime.utcnow()
ver = get_version(ureport)
if ver == 1:
save_ureport1(db,
ureport,
create_component=create_component,
timestamp=timestamp,
count=count)
elif ver == 2:
save_ureport2(db,
ureport,
create_component=create_component,
timestamp=timestamp,
count=count)
else:
raise FafError("uReport version {0} is not supported".format(ver))
db.session.flush()
def unpack_rpm_to_tmp(path, prefix="faf"):
"""
Unpack RPM package to a temp directory. The directory is either specified
in storage.tmpdir config option or use the system default temp directory.
"""
tmpdir = None
if "storage.tmpdir" in config:
tmpdir = config["storage.tmpdir"]
result = tempfile.mkdtemp(prefix=prefix, dir=tmpdir)
for dirname in ["bin", "lib", "lib64", "sbin"]:
os.makedirs(os.path.join(result, "usr", dirname))
os.symlink(os.path.join("usr", dirname), os.path.join(result, dirname))
rpm2cpio = Popen(["rpm2cpio", path], stdout=PIPE, stderr=PIPE)
cpio = Popen(["cpio", "-id", "--quiet"],
stdin=rpm2cpio.stdout,
stderr=PIPE,
cwd=result)
# do not check rpm2cpio exitcode as there may be a bug for large files
# https://bugzilla.redhat.com/show_bug.cgi?id=790396
rpm2cpio.wait()
if cpio.wait() != 0:
shutil.rmtree(result)
raise FafError("Failed to unpack RPM '{0}'".format(path))
return result
def __init__(self,
debug=False,
dry=False,
session_kwargs=None,
create_schema=False) -> None:
if Database.__instance__ and Database.__instancecheck__(
Database.__instance__):
raise FafError(
"Database is a singleton and has already been instantiated. "
"If you have lost the reference, you can access the object "
"from Database.__instance__ .")
if not session_kwargs:
session_kwargs = {"autoflush": False, "autocommit": True}
self._db = create_engine(get_connect_string())
self._db.echo = self._debug = debug
self._dry = dry
GenericTable.metadata.bind = self._db
self.session = Session(self._db, **session_kwargs)
self.session._flush_orig = self.session.flush #pylint: disable=protected-access
self.session.flush = self._flush_session
if create_schema:
GenericTable.metadata.create_all()
Database.__instance__ = self
def del_lob(self, name):
lobpath = self.get_lob_path(name)
if not os.path.isfile(lobpath):
raise FafError("Lob '{0}' does not exist".format(name))
os.unlink(lobpath)
def _save_lob_string(self, dest, data, maxlen=0, truncate=False):
if maxlen > 0 and len(data) > maxlen:
if truncate:
data = data[:maxlen]
else:
raise FafError("Data is too long, '{0}' only allows length of {1}".format(dest.name, maxlen))
dest.write(data)
def get_base_address(binary_path):
"""
Runs eu-unstrip on a binary to get the address used
as base for calculating relative offsets.
"""
child = safe_popen("eu-unstrip", "-n", "-e", binary_path)
if child is None:
raise FafError("eu-unstrip failed")
match = RE_UNSTRIP_BASE_OFFSET.match(child.stdout)
if match is None:
raise FafError("Unexpected output from eu-unstrip: '{0}'".format(
child.stdout))
return int(match.group(1), 16)
def _get_repo_file_path(self,
reponame,
repourl,
remote,
local=None) -> str:
url = os.path.join(repourl, remote)
if url.startswith("file://"):
return url[len("file://"):]
if url.startswith("/"):
return url
if local is None:
local = os.path.basename(remote)
cachename = os.path.join(self.cachedir, self.name, reponame, local)
mtime = 0
try:
mtime = os.path.getmtime(cachename)
except OSError as ex:
if errno.ENOENT != ex.errno:
raise FafError("Cannot access cache: {0}".format(str(ex)))
if (mtime + self.cacheperiod) <= time.time():
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url.encode("ascii", "ignore"))
try:
fp = open(cachename, "wb")
except Exception as ex:
raise FafError(
"Creating cache file {0} filed with: {1}".format(
cachename, str(ex)))
else:
with fp:
curl.setopt(pycurl.WRITEDATA, fp)
try:
curl.perform()
except pycurl.error as ex:
raise FafError("Downloading failed: {0}".format(
str(ex)))
finally:
curl.close()
return cachename
def _setup_dirs(self, reponame):
dirname = os.path.join(self.cachedir, self.name, reponame)
try:
os.makedirs(dirname)
except OSError as ex:
if ex.errno != 17:
raise FafError("Cache directories error '{0}': {1}".format(
self.cachedir, str(ex)))
return dirname
def _write_lob_bytes(self, dest: BinaryIO, data: bytes, maxlen: int = 0,
truncate: bool = False):
if len(data) > maxlen > 0:
if truncate:
data = data[:maxlen]
else:
raise FafError("Data is too long, '{0}' only allows length of {1}".format(dest.name, maxlen))
dest.write(data)
def validate_packages(self, packages):
Fedora.packages_checker.check(packages)
for package in packages:
if ("package_role" in package
and package["package_role"] not in Fedora.pkg_roles):
raise FafError("Only the following package roles are allowed: "
"{0}".format(", ".join(Fedora.pkg_roles)))
return True
def pkstr(self):
parts = []
for column in self.__table__._columns:
if column.primary_key:
parts.append(str(self.__getattribute__(column.name)))
if not parts:
raise FafError("No primary key found for object '{0}'".format(self.__class__.__name__))
return "-".join(parts)
def pkstr(self) -> str:
parts = []
for col in self.__table__._columns: #pylint: disable=no-member, protected-access
if col.primary_key:
parts.append(str(self.__getattribute__(col.name)))
if not parts:
raise FafError("No primary key found for object '{0}'".format(self.__class__.__name__))
return "-".join(parts)
def _get_repo_file_path(self,
reponame: str,
repourl: str,
remote: str,
local: Optional[str] = None) -> str:
url = os.path.join(repourl, remote)
if url.startswith("file://"):
return url[len("file://"):]
if url.startswith("/"):
return url
if local is None:
local = os.path.basename(remote)
cachename = os.path.join(self.cachedir, self.name, reponame, local)
last_modified: float = 0
try:
last_modified = os.path.getmtime(cachename)
except OSError as ex:
if errno.ENOENT != ex.errno:
raise FafError("Cannot access cache: {0}".format(
str(ex))) from ex
# Check for cache expiration.
if (last_modified + self.cacheperiod) <= time.time():
try:
cache_file = open(cachename, "wb")
except Exception as ex:
raise FafError(
"Creating cache file {0} filed with: {1}".format(
cachename, str(ex))) from ex
with cache_file:
try:
with urlopen(url) as response:
cache_file.write(response.read())
except HTTPError as ex:
raise FafError("Downloading failed: {0}".format(
str(ex))) from ex
return cachename
def precompute_history(report_ids, period, count=20):
today = datetime.date.today()
if period == 'day':
table = ReportHistoryDaily
first_day = today - datetime.timedelta(days=count - 1)
elif period == 'week':
table = ReportHistoryWeekly
# Last Monday or today if it's Monday.
last_day = today - datetime.timedelta(days=today.weekday())
first_day = last_day - datetime.timedelta(days=(count - 1) * 7)
elif period == 'month':
table = ReportHistoryMonthly
# First day of this month.
last_day = datetime.date(today.year, today.month, 1)
first_day = last_day - relativedelta(months=count - 1)
else:
raise FafError(f'Invalid time period "{period}"')
date_column = getattr(table, period)
q = (db.session.query(table.opsysrelease_id,
date_column.label('date'),
func.sum(table.count).label('total_count'),
func.sum(table.unique).label('total_unique'))
.filter(table.report_id.in_(report_ids))
.filter(date_column >= first_day)
.group_by(date_column, table.opsysrelease_id)
.subquery())
histories = (db.session.query(OpSysRelease,
q.c.date,
q.c.total_count,
q.c.total_unique)
.join(q)
.order_by(OpSysRelease.id, q.c.date)
.all())
# Preprocessing to unify output format for all periods and for easier plotting.
by_opsys = {}
for osr, entries in groupby(histories, itemgetter(0)):
counts = [{'date': e.date,
'count': e.total_count,
'unique': e.total_unique}
for e in entries]
by_opsys[str(osr)] = counts
result = {
'by_opsys': by_opsys,
'from_date': first_day,
'period_count': count
}
return result
def addr2line(binary_path, address, debuginfo_dir):
"""
Calls eu-addr2line on a binary, address and directory with debuginfo.
Returns an ordered list of triplets (function name, source file, line no).
The last element is always the symbol given to retrace. The elements
before are inlined symbols that should be placed above the given symbol
(assuming that entry point is on the bottom of the stacktrace).
"""
result = []
child = safe_popen("eu-addr2line", "--executable", binary_path,
"--debuginfo-path", debuginfo_dir, "--functions",
str(address))
if child is None:
raise FafError("eu-add2line failed")
line1, line2 = child.stdout.splitlines()
line2_parts = line2.split(":", 1)
line2_srcfile = line2_parts[0]
line2_srcline = int(line2_parts[1])
match = RE_ADDR2LINE_LINE1.match(line1)
if match is None:
raise FafError(
"Unexpected output from eu-addr2line: '{0}'".format(line1))
if match.group(3) is None:
funcname = match.group(1)
srcfile = line2_srcfile
srcline = line2_srcline
else:
funcname = match.group(6)
srcfile = match.group(4)
srcline = int(match.group(5))
result.append((match.group(1), line2_srcfile, line2_srcline))
result.append((funcname, srcfile, srcline))
return result
def get_opsys_name(self, cmdline_opsys):
"""
Get correct opsys name from user passed opsys
or raise FafError if not available
"""
cmdline_opsys = cmdline_opsys.lower()
if not cmdline_opsys in systems:
raise FafError(
"Operating system '{0}' does not exist".format(cmdline_opsys))
return systems[cmdline_opsys].nice_name
def save_lob(self, name, data, overwrite=False, truncate=False):
lobpath = self.get_lob_path(name)
if not isinstance(data, bytes) and not hasattr(data, "read"):
raise FafError(
"Data must be either a bytestring or a file-like object")
if not overwrite and os.path.isfile(lobpath):
raise FafError("Lob '{0}' already exists".format(name))
maxlen = self.__lobs__[name]
with open(lobpath, "wb") as lob:
if hasattr(data, "read"):
if not truncate:
raise FafError(
"When saving from file, truncate must be enabled")
self._write_lob_file(lob, data, maxlen)
else:
self._write_lob_bytes(lob, data, maxlen, truncate)
