def _save_history(self, db, events, new_bug_id):
"""
Save bug history to the database.
Expects list of `events` and ID of the bug as `new_bug_id`.
"""
total = len(events)
for num, event in enumerate(events):
self.log_debug("Processing history event {0}/{1}".format(num + 1,
total))
user_email = event["who"]
user = queries.get_bz_user(db, user_email)
if not user:
self.log_debug("History changed by unknown user #{0}".format(
user_email))
downloaded = self._download_user(user_email)
if not downloaded:
self.log_error("Unable to download user, skipping.")
continue
user = self._save_user(db, downloaded)
for change in event["changes"]:
chtime = self._convert_datetime(event["when"])
ch = (
db.session.query(BzBugHistory)
.filter((BzBugHistory.user == user) &
(BzBugHistory.time == chtime) &
(BzBugHistory.field == change["field_name"]) &
(BzBugHistory.added == change["added"]) &
(BzBugHistory.removed == change["removed"]))
.first())
if ch:
self.log_debug("Skipping existing history event "
"#{0}".format(ch.id))
continue
new = BzBugHistory()
new.bug_id = new_bug_id
new.user = user
new.time = chtime
new.field = change["field_name"]
new.added = change["added"][:column_len(BzBugHistory, "added")]
new.removed = change["removed"][:column_len(BzBugHistory, "removed")]
db.session.add(new)
db.session.flush()
def _find_crashfn(self, db, problemplugin, query_all=False):
db_backtraces = get_backtraces_by_type(db, problemplugin.name,
query_all=query_all)
db_backtraces_count = db_backtraces.count()
i = 0
for db_backtrace in db_backtraces.yield_per(100):
i += 1
try:
crashfn = (demangle(problemplugin.find_crash_function(
db_backtrace))[:column_len(ReportBacktrace, "crashfn")])
except Exception as ex:
self.log_warn("Unable to find crash function: {0}"
.format(str(ex)))
continue
if db_backtrace.crashfn != crashfn:
self.log_info("[{0} / {1}] Updating backtrace #{2}: {3} ~> {4}"
.format(i, db_backtraces_count, db_backtrace.id,
db_backtrace.crashfn, crashfn))
db_backtrace.crashfn = crashfn
else:
self.log_info("[{0} / {1}] Backtrace #{2} up to date: {3}"
.format(i, db_backtraces_count, db_backtrace.id,
db_backtrace.crashfn))
if (i % 100) == 0:
db.session.flush()
db.session.flush()
ReportURL,
column_len)
from pyfaf.ureport_compat import ureport1to2
from sqlalchemy.exc import IntegrityError
log = log.getChildLogger(__name__)
__all__ = ["get_version", "save", "ureport2",
"validate", "validate_attachment"]
UREPORT_CHECKER = DictChecker({
"os": DictChecker({
"name": StringChecker(allowed=systems.keys()),
"version": StringChecker(pattern=r"^[a-zA-Z0-9_\.\-\+~]+$",
maxlen=column_len(OpSysRelease,
"version")),
"architecture": StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
# Anything else will be checked by the plugin
}),
# The checker for packages depends on operating system
"packages": ListChecker(Checker(object)),
"problem": DictChecker({
"type": StringChecker(allowed=problemtypes.keys()),
# Anything else will be checked by the plugin
}),
"reason": StringChecker(maxlen=column_len(ReportReason, "reason")),
class CoredumpProblem(ProblemType):
name = "core"
nice_name = "Crash of user-space binary"
checker = DictChecker({
# no need to check type twice, the toplevel checker already did it
# "type": StringChecker(allowed=[CoredumpProblem.name]),
"signal":
IntChecker(minval=0),
"component":
StringChecker(pattern=r"^[a-zA-Z0-9\-\._]+$",
maxlen=column_len(OpSysComponent, "name")),
"executable":
StringChecker(maxlen=column_len(ReportExecutable, "path")),
"user":
DictChecker({
"root": Checker(bool),
"local": Checker(bool),
}),
"stacktrace":
ListChecker(DictChecker({
"crash_thread":
Checker(bool, mandatory=False),
"frames":
ListChecker(DictChecker({
"address":
IntChecker(minval=0),
"build_id_offset":
IntChecker(minval=0),
"file_name":
StringChecker(maxlen=column_len(SymbolSource, "path")),
"build_id":
StringChecker(pattern=r"^[a-fA-F0-9]+$",
maxlen=column_len(SymbolSource, "build_id"),
mandatory=False),
"fingerprint":
StringChecker(pattern=r"^[a-fA-F0-9]+$",
maxlen=column_len(ReportBtHash, "hash"),
mandatory=False),
"function_name":
StringChecker(maxlen=column_len(Symbol, "nice_name"),
mandatory=False)
}),
minlen=1)
}),
minlen=1)
})
def __init__(self, *args, **kwargs):
super(CoredumpProblem, self).__init__()
hashkeys = ["processing.corehashframes", "processing.hashframes"]
self.load_config_to_self("hashframes", hashkeys, 16, callback=int)
cmpkeys = [
"processing.corecmpframes", "processing.cmpframes",
"processing.clusterframes"
]
self.load_config_to_self("cmpframes", cmpkeys, 16, callback=int)
cutkeys = ["processing.corecutthreshold", "processing.cutthreshold"]
self.load_config_to_self("cutthreshold", cutkeys, 0.3, callback=float)
normkeys = ["processing.corenormalize", "processing.normalize"]
self.load_config_to_self("normalize",
normkeys,
True,
callback=str2bool)
skipkeys = ["retrace.coreskipsource", "retrace.skipsource"]
self.load_config_to_self("skipsrc", skipkeys, True, callback=str2bool)
def _get_crash_thread(self, stacktrace):
"""
Searches for a single crash thread and return it. Raises FafError if
there is no crash thread or if there are multiple crash threads.
"""
crashthreads = [
t for t in stacktrace
if ("crash_thread" in t and t["crash_thread"])
]
if len(crashthreads) < 1:
raise FafError("No crash thread found")
if len(crashthreads) > 1:
raise FafError("Multiple crash threads found")
return crashthreads[0]["frames"]
def _hash_backtrace(self, backtrace):
result = []
for key in ["function_name", "fingerprint", "build_id_offset"]:
hashbase = []
threads_sane = []
for thread in backtrace:
threads_sane.append(all(key in f for f in thread["frames"]))
if not all(threads_sane):
continue
for thread in backtrace:
if "crash_thread" in thread and thread["crash_thread"]:
hashbase.append("Crash Thread")
else:
hashbase.append("Thread")
for frame in thread["frames"]:
if "build_id" in frame:
build_id = frame["build_id"]
else:
build_id = None
hashbase.append(" {0} @ {1} ({2})".format(
frame[key],
frame["file_name"].encode("ascii",
"ignore"), build_id))
result.append(hash_list(hashbase))
return result
def _db_thread_to_satyr(self, db_thread):
thread = satyr.GdbThread()
thread.number = db_thread.number
for db_frame in db_thread.frames:
frame = satyr.GdbFrame()
frame.address = db_frame.symbolsource.offset
frame.library_name = \
db_frame.symbolsource.path.encode("ascii", "ignore")
frame.number = db_frame.order
if db_frame.symbolsource.symbol is not None:
frame.function_name = db_frame.symbolsource.symbol.name
else:
frame.function_name = "??"
if db_frame.symbolsource.source_path is not None:
frame.source_file = \
db_frame.symbolsource.source_path.encode("ascii", "ignore")
if db_frame.symbolsource.line_number is not None:
frame.source_line = db_frame.symbolsource.line_number
thread.frames.append(frame)
if self.normalize:
stacktrace = satyr.GdbStacktrace()
stacktrace.threads.append(thread)
stacktrace.normalize()
return thread
def _db_thread_validate(self, db_thread):
if len(db_thread.frames) == 1:
db_frame = db_thread.frames[0]
if (db_frame.symbolsource.symbol is not None and
db_frame.symbolsource.symbol.name == "anonymous function"
and db_frame.symbolsource.symbol.normalized_path
== "unknown filename"):
return False
return True
def _db_report_to_satyr(self, db_report):
if len(db_report.backtraces) < 1:
self.log_warn("Report #{0} has no usable backtraces".format(
db_report.id))
return None
if len(db_report.backtraces[0].threads) < 1:
self.log_warn("Report #{0} has no usable threads".format(
db_report.id))
return None
for db_thread in db_report.backtraces[0].threads:
if not db_thread.crashthread:
continue
if self._db_thread_validate(db_thread):
return self._db_thread_to_satyr(db_thread)
else:
self.log_warn("Report #{0} has only one bad frame".format(
db_report.id))
return None
self.log_warn("Report #{0} has no crash thread".format(db_report.id))
return None
def _get_file_name_from_build_id(self, build_id):
return "/usr/lib/debug/.build-id/{0}/{1}.debug".format(
build_id[:2], build_id[2:])
def validate_ureport(self, ureport):
# Frames calling JIT compiled functions usually do not contain
# function name nor file name. This would result to the uReport being
# rejected. However the stack above is often the relevant part and we
# do not want to reject such uReports.
# This code tries to detect calling JIT compiled code and filling
# the frames with file name (the JIT caller) and function name
# (anonymous function).
if "stacktrace" in ureport and isinstance(ureport["stacktrace"], list):
for thread in ureport["stacktrace"]:
if not isinstance(thread, dict):
continue
jit_fname = None
if "frames" in thread and isinstance(thread["frames"], list):
for frame in thread["frames"]:
if not isinstance(frame, dict):
continue
if ("file_name" in frame and "function_name" in frame
and "jit" in frame["function_name"].lower()):
jit_fname = frame["file_name"]
if "file_name" not in frame and jit_fname is not None:
frame["file_name"] = jit_fname
if ("function_name" not in frame
or frame["function_name"] == "??"):
frame["function_name"] = "anonymous function"
if len(thread["frames"]) > 0:
last_frame = thread["frames"][-1]
if isinstance(last_frame, dict):
if "file_name" not in last_frame:
last_frame["file_name"] = "unknown filename"
if ("function_name" not in last_frame
or frame["function_name"] == "??"):
last_frame[
"function_name"] = "anonymous function"
CoredumpProblem.checker.check(ureport)
# just to be sure there is exactly one crash thread
self._get_crash_thread(ureport["stacktrace"])
return True
def hash_ureport(self, ureport):
crashthread = self._get_crash_thread(ureport["stacktrace"])
hashbase = [ureport["component"]]
if all("function_name" in f for f in crashthread):
key = "function_name"
elif all("fingerprint" in f for f in crashthread):
key = "fingerprint"
else:
key = "build_id_offset"
for i, frame in enumerate(crashthread):
# Instance of 'CoredumpProblem' has no 'hashframes' member
# pylint: disable-msg=E1101
if i >= self.hashframes:
break
hashbase.append("{0} @ {1}".format(
frame[key], frame["file_name"].encode("ascii", "ignore")))
return hash_list(hashbase)
def save_ureport(self, db, db_report, ureport, flush=False, count=1):
db_report.errname = str(ureport["signal"])
db_reportexe = get_reportexe(db, db_report, ureport["executable"])
if db_reportexe is None:
db_reportexe = ReportExecutable()
db_reportexe.path = ureport["executable"]
db_reportexe.report = db_report
db_reportexe.count = 0
db.session.add(db_reportexe)
db_reportexe.count += count
bthashes = self._hash_backtrace(ureport["stacktrace"])
if len(bthashes) < 1:
raise FafError("Unable to get backtrace hash")
if len(db_report.backtraces) < 1:
new_symbols = {}
new_symbolsources = {}
db_backtrace = ReportBacktrace()
db_backtrace.report = db_report
db.session.add(db_backtrace)
for bthash in bthashes:
db_bthash = ReportBtHash()
db_bthash.backtrace = db_backtrace
db_bthash.type = "NAMES"
db_bthash.hash = bthash
db.session.add(db_bthash)
tid = 0
for thread in ureport["stacktrace"]:
tid += 1
crash = "crash_thread" in thread and thread["crash_thread"]
db_thread = ReportBtThread()
db_thread.backtrace = db_backtrace
db_thread.number = tid
db_thread.crashthread = crash
db.session.add(db_thread)
fid = 0
for frame in thread["frames"]:
# OK, this is totally ugly.
# Frames may contain inlined functions, that would normally
# require shifting all frames by 1 and inserting a new one.
# There is no way to do this efficiently with SQL Alchemy
# (you need to go one by one and flush after each) so
# creating a space for additional frames is a huge speed
# optimization.
fid += 10
if "build_id" in frame:
build_id = frame["build_id"]
else:
build_id = None
if "fingerprint" in frame:
fingerprint = frame["fingerprint"]
else:
fingerprint = None
path = os.path.abspath(frame["file_name"])
offset = frame["build_id_offset"]
db_symbol = None
if "function_name" in frame:
norm_path = get_libname(path)
db_symbol = \
get_symbol_by_name_path(db,
frame["function_name"],
norm_path)
if db_symbol is None:
key = (frame["function_name"], norm_path)
if key in new_symbols:
db_symbol = new_symbols[key]
else:
db_symbol = Symbol()
db_symbol.name = frame["function_name"]
db_symbol.normalized_path = norm_path
db.session.add(db_symbol)
new_symbols[key] = db_symbol
db_symbolsource = get_ssource_by_bpo(
db, build_id, path, offset)
if db_symbolsource is None:
key = (build_id, path, offset)
if key in new_symbolsources:
db_symbolsource = new_symbolsources[key]
else:
db_symbolsource = SymbolSource()
db_symbolsource.symbol = db_symbol
db_symbolsource.build_id = build_id
db_symbolsource.path = path
db_symbolsource.offset = offset
db_symbolsource.hash = fingerprint
db.session.add(db_symbolsource)
new_symbolsources[key] = db_symbolsource
db_frame = ReportBtFrame()
db_frame.thread = db_thread
db_frame.order = fid
db_frame.symbolsource = db_symbolsource
db_frame.inlined = False
db.session.add(db_frame)
if flush:
db.session.flush()
def save_ureport_post_flush(self):
self.log_debug("save_ureport_post_flush is not required for coredumps")
def get_component_name(self, ureport):
return ureport["component"]
def compare(self, db_report1, db_report2):
satyr_report1 = self._db_report_to_satyr(db_report1)
satyr_report2 = self._db_report_to_satyr(db_report2)
return satyr_report1.distance(satyr_report2)
def compare_many(self, db_reports):
self.log_info("Loading reports")
reports = []
ret_db_reports = []
i = 0
for db_report in db_reports:
i += 1
self.log_debug("[{0} / {1}] Loading report #{2}".format(
i, len(db_reports), db_report.id))
report = self._db_report_to_satyr(db_report)
if report is None:
self.log_debug("Unable to build satyr.GdbStacktrace")
continue
reports.append(report)
ret_db_reports.append(db_report)
self.log_info("Calculating distances")
distances = satyr.Distances(reports, len(reports))
return ret_db_reports, distances
def check_btpath_match(self, ureport, parser):
for thread in ureport["stacktrace"]:
if "crash_thread" not in thread or not thread["crash_thread"]:
continue
for frame in thread["frames"]:
match = parser.match(frame["file_name"])
if match is not None:
return True
return False
def _get_ssources_for_retrace_query(self, db):
core_syms = (db.session.query(
SymbolSource.id).join(ReportBtFrame).join(ReportBtThread).join(
ReportBacktrace).join(Report).filter(
Report.type == CoredumpProblem.name).subquery())
q = (db.session.query(SymbolSource).filter(
SymbolSource.id.in_(core_syms)).filter(
SymbolSource.build_id != None).filter(
(SymbolSource.symbol == None)
| (SymbolSource.source_path == None)
| (SymbolSource.line_number == None)))
return q
def find_packages_for_ssource(self, db, db_ssource):
self.log_debug("Build-id: {0}".format(db_ssource.build_id))
filename = self._get_file_name_from_build_id(db_ssource.build_id)
self.log_debug("File name: {0}".format(filename))
db_debug_package = get_package_by_file(db, filename)
if db_debug_package is None:
debug_nvra = "Not found"
else:
debug_nvra = db_debug_package.nvra()
self.log_debug("Debug Package: {0}".format(debug_nvra))
db_bin_package = None
if db_debug_package is not None:
paths = [db_ssource.path]
if os.path.sep in db_ssource.path:
paths.append(usrmove(db_ssource.path))
paths.append(os.path.abspath(db_ssource.path))
paths.append(usrmove(os.path.abspath(db_ssource.path)))
db_build = db_debug_package.build
db_arch = db_debug_package.arch
for path in paths:
db_bin_package = get_package_by_file_build_arch(
db, path, db_build, db_arch)
if db_bin_package is not None:
# Do not fix UsrMove in the DB - it's a terrible slow-down
# Rather fake it with symlinks when unpacking
#if db_ssource.path != path:
# self.log_debug("Fixing path: {0} ~> {1}"
# .format(db_ssource.path, path))
# build_id = db_ssource.build_id
# db_ssource_fixed = get_ssource_by_bpo(db,
# build_id,
# path,
# db_ssource.offset)
# if db_ssource_fixed is None:
# db_ssource.path = path
# else:
# db_ssource = db_ssource_fixed
break
if db_bin_package is None:
bin_nvra = "Not found"
else:
bin_nvra = db_bin_package.nvra()
self.log_debug("Binary Package: {0}".format(bin_nvra))
db_src_package = None
if not self.skipsrc and db_debug_package is not None:
db_build = db_debug_package.build
db_src_package = get_src_package_by_build(db, db_build)
if db_src_package is None:
src_nvra = "Not found"
else:
src_nvra = db_src_package.nvra()
self.log_debug("Source Package: {0}".format(src_nvra))
# indicate incomplete result
if db_bin_package is None:
db_debug_package = None
return db_ssource, (db_debug_package, db_bin_package, db_src_package)
def retrace(self, db, task):
new_symbols = {}
new_symbolsources = {}
for bin_pkg, db_ssources in task.binary_packages.items():
self.log_info("Retracing symbols from package {0}".format(
bin_pkg.nvra))
i = 0
for db_ssource in db_ssources:
i += 1
self.log_debug("[{0} / {1}] Processing '{2}' @ '{3}'".format(
i, len(db_ssources), ssource2funcname(db_ssource),
db_ssource.path))
norm_path = get_libname(db_ssource.path)
binary = os.path.join(bin_pkg.unpacked_path,
db_ssource.path[1:])
try:
address = get_base_address(binary) + db_ssource.offset
except FafError as ex:
self.log_debug("get_base_address failed: {0}".format(
str(ex)))
db_ssource.retrace_fail_count += 1
continue
try:
debug_path = os.path.join(task.debuginfo.unpacked_path,
"usr", "lib", "debug")
results = addr2line(binary, address, debug_path)
results.reverse()
except Exception as ex:
self.log_debug("addr2line failed: {0}".format(str(ex)))
db_ssource.retrace_fail_count += 1
continue
inl_id = 0
while len(results) > 1:
inl_id += 1
funcname, srcfile, srcline = results.pop()
self.log_debug(
"Unwinding inlined function '{0}'".format(funcname))
# hack - we have no offset for inlined symbols
# let's use minus source line to avoid collisions
offset = -srcline
db_ssource_inl = get_ssource_by_bpo(
db, db_ssource.build_id, db_ssource.path, offset)
if db_ssource_inl is None:
key = (db_ssource.build_id, db_ssource.path, offset)
if key in new_symbolsources:
db_ssource_inl = new_symbolsources[key]
else:
db_symbol_inl = get_symbol_by_name_path(
db, funcname, norm_path)
if db_symbol_inl is None:
sym_key = (funcname, norm_path)
if sym_key in new_symbols:
db_symbol_inl = new_symbols[sym_key]
else:
db_symbol_inl = Symbol()
db_symbol_inl.name = funcname
db_symbol_inl.normalized_path = norm_path
db.session.add(db_symbol_inl)
new_symbols[sym_key] = db_symbol_inl
db_ssource_inl = SymbolSource()
db_ssource_inl.symbol = db_symbol_inl
db_ssource_inl.build_id = db_ssource.build_id
db_ssource_inl.path = db_ssource.path
db_ssource_inl.offset = offset
db_ssource_inl.source_path = srcfile
db_ssource_inl.line_number = srcline
db.session.add(db_ssource_inl)
new_symbolsources[key] = db_ssource_inl
for db_frame in db_ssource.frames:
db_frames = sorted(db_frame.thread.frames,
key=lambda f: f.order)
idx = db_frames.index(db_frame)
if idx > 0:
prevframe = db_frame.thread.frames[idx - 1]
if (prevframe.inlined and prevframe.symbolsource
== db_ssource_inl):
continue
db_newframe = ReportBtFrame()
db_newframe.symbolsource = db_ssource_inl
db_newframe.thread = db_frame.thread
db_newframe.inlined = True
db_newframe.order = db_frame.order - inl_id
db.session.add(db_newframe)
funcname, srcfile, srcline = results.pop()
self.log_debug("Result: {0}".format(funcname))
db_symbol = get_symbol_by_name_path(db, funcname, norm_path)
if db_symbol is None:
key = (funcname, norm_path)
if key in new_symbols:
db_symbol = new_symbols[key]
else:
self.log_debug(
"Creating new symbol '{0}' @ '{1}'".format(
funcname, db_ssource.path))
db_symbol = Symbol()
db_symbol.name = funcname
db_symbol.normalized_path = norm_path
db.session.add(db_symbol)
new_symbols[key] = db_symbol
if db_symbol.nice_name is None:
db_symbol.nice_name = demangle(funcname)
db_ssource.symbol = db_symbol
db_ssource.source_path = srcfile
db_ssource.line_number = srcline
if task.debuginfo.unpacked_path is not None:
self.log_debug("Removing {0}".format(task.debuginfo.unpacked_path))
shutil.rmtree(task.debuginfo.unpacked_path, ignore_errors=True)
if task.source is not None and task.source.unpacked_path is not None:
self.log_debug("Removing {0}".format(task.source.unpacked_path))
shutil.rmtree(task.source.unpacked_path, ignore_errors=True)
for bin_pkg in task.binary_packages.keys():
if bin_pkg.unpacked_path is not None:
self.log_debug("Removing {0}".format(bin_pkg.unpacked_path))
shutil.rmtree(bin_pkg.unpacked_path, ignore_errors=True)
def find_crash_function(self, db_backtrace):
for db_thread in db_backtrace.threads:
if not db_thread.crashthread:
continue
satyr_thread = self._db_thread_to_satyr(db_thread)
satyr_stacktrace = satyr.GdbStacktrace()
satyr_stacktrace.threads.append(satyr_thread)
return satyr_stacktrace.find_crash_frame().function_name
self.log_warn("Backtrace #{0} has no crash thread".format(
db_backtrace.id))
return None
class KerneloopsProblem(ProblemType):
name = "kerneloops"
nice_name = "Kernel oops"
tainted_flags = {
"module_proprietary": ("P", "Proprietary module has been loaded"),
"forced_module": ("F", "Module has been forcibly loaded"),
"smp_unsafe": ("S", "SMP with CPUs not designed for SMP"),
"forced_removal": ("R", "User forced a module unload"),
"mce": ("M", "System experienced a machine check exception"),
"page_release": ("B", "System has hit bad_page"),
"userspace": ("U", "Userspace-defined naughtiness"),
"died_recently": ("D", "Kernel has oopsed before"),
"acpi_overridden": ("A", "ACPI table overridden"),
"warning": ("W", "Taint on warning"),
"staging_driver": ("C", "Modules from drivers/staging are loaded"),
"firmware_workaround": ("I", "Working around severe firmware bug"),
"module_out_of_tree": ("O", "Out-of-tree module has been loaded"),
}
checker = DictChecker({
# no need to check type twice, the toplevel checker already did it
# "type": StringChecker(allowed=[KerneloopsProblem.name]),
"component": StringChecker(pattern=r"^kernel(-[a-zA-Z0-9\-\._]+)?$",
maxlen=column_len(OpSysComponent,
"name")),
"version": StringChecker(pattern=(r"^[0-9]+\.[0-9]+\.[0-9]+"
r"(.[^\-]+)?(\-.*)?$"),
maxlen=column_len(SymbolSource,
"build_id")),
"taint_flags": ListChecker(StringChecker(allowed=tainted_flags.keys())),
"modules": ListChecker(StringChecker(pattern=r"^[a-zA-Z0-9_]+(\([A-Z\+\-]+\))?$",
maxlen=column_len(KernelModule,
"name")),
mandatory=False),
"raw_oops": StringChecker(maxlen=Report.__lobs__["oops"],
mandatory=False),
"frames": ListChecker(DictChecker({
"address": IntChecker(minval=0, maxval=((1 << 64) - 1)),
"reliable": Checker(bool),
"function_name": StringChecker(pattern=r"^[a-zA-Z0-9_\.]+$",
maxlen=column_len(Symbol,
"name")),
"function_offset": IntChecker(minval=0, maxval=((1 << 63) - 1)),
"function_length": IntChecker(minval=0, maxval=((1 << 63) - 1)),
"module_name": StringChecker(pattern=r"^[a-zA-Z0-9_]+(\([A-Z\+\-]+\))?$",
mandatory=False),
}), minlen=1)
})
@classmethod
def install(cls, db, logger=None):
if logger is None:
logger = log.getChildLogger(cls.__name__)
for flag, (char, nice_name) in cls.tainted_flags.items():
if get_taint_flag_by_ureport_name(db, flag) is None:
logger.info("Adding kernel taint flag '{0}': {1}"
.format(char, nice_name))
new = KernelTaintFlag()
new.character = char
new.ureport_name = flag
new.nice_name = nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db):
for flag in cls.tainted_flags.keys():
if get_taint_flag_by_ureport_name(db, flag) is None:
return False
return True
def __init__(self, *args, **kwargs):
super(KerneloopsProblem, self).__init__()
hashkeys = ["processing.oopshashframes", "processing.hashframes"]
self.load_config_to_self("hashframes", hashkeys, 16, callback=int)
cmpkeys = ["processing.oopscmpframes", "processing.cmpframes",
"processing.clusterframes"]
self.load_config_to_self("cmpframes", cmpkeys, 16, callback=int)
cutkeys = ["processing.oopscutthreshold", "processing.cutthreshold"]
self.load_config_to_self("cutthreshold", cutkeys, 0.3, callback=float)
normkeys = ["processing.oopsnormalize", "processing.normalize"]
self.load_config_to_self("normalize", normkeys, True, callback=str2bool)
skipkeys = ["retrace.oopsskipsource", "retrace.skipsource"]
self.load_config_to_self("skipsrc", skipkeys, True, callback=str2bool)
self.add_lob = {}
self._kernel_pkg_map = {}
self.archnames = None
def _hash_koops(self, koops, taintflags=None, skip_unreliable=False):
if taintflags is None:
taintflags = []
if skip_unreliable:
frames = filter(lambda f: f["reliable"], koops)
else:
frames = koops
if len(frames) < 1:
return None
hashbase = list(taintflags)
for frame in frames:
if not "module_name" in frame:
module = "vmlinux"
else:
module = frame["module_name"]
hashbase.append("{0} {1}+{2}/{3} @ {4}"
.format(frame["address"], frame["function_name"],
frame["function_offset"],
frame["function_length"], module))
return sha1("\n".join(hashbase)).hexdigest()
def _db_backtrace_to_satyr(self, db_backtrace):
stacktrace = satyr.Kerneloops()
if len(db_backtrace.threads) < 1:
self.log_warn("Backtrace #{0} has no usable threads"
.format(db_backtrace.id))
return None
db_thread = db_backtrace.threads[0]
if len(db_thread.frames) < 1:
self.log_warn("Thread #{0} has no usable frames"
.format(db_thread.id))
return None
for db_frame in db_thread.frames:
frame = satyr.KerneloopsFrame()
if db_frame.symbolsource.symbol is not None:
frame.function_name = db_frame.symbolsource.symbol.name
else:
frame.function_name = "??"
frame.address = db_frame.symbolsource.offset
frame.function_offset = db_frame.symbolsource.func_offset
frame.reliable = db_frame.reliable
if frame.address < 0:
frame.address += (1 << 64)
stacktrace.frames.append(frame)
if self.normalize:
stacktrace.normalize()
return stacktrace
def _db_report_to_satyr(self, db_report):
if len(db_report.backtraces) < 1:
self.log_warn("Report #{0} has no usable backtraces"
.format(db_report.id))
return None
return self._db_backtrace_to_satyr(db_report.backtraces[0])
def _parse_kernel_build_id(self, build_id, archs):
"""
Parses the kernel build string such as
3.10.0-3.fc19.x86_64
3.10.0-3.fc19.armv7hl.tegra
2.6.32-358.14.1.el6.i686.PAE
3.15.6-200.fc20.i686+PAE
"""
arch = None
flavour = None
splitby = "+" if "+" in build_id else "."
head, tail = build_id.rsplit(splitby, 1)
if tail in archs:
arch = tail
else:
flavour = tail
head, tail = head.rsplit(".", 1)
if not tail in archs:
raise FafError("Unable to determine architecture from '{0}'"
.format(build_id))
arch = tail
try:
version, release = head.rsplit("-", 1)
except ValueError:
raise FafError("Unable to determine release from '{0}'"
.format(head))
return version, release, arch, flavour
def _get_debug_path(self, db, module, db_package):
"""
Return the path of debuginfo file for
a given module or None if not found.
"""
if module == "vmlinux":
filename = module
else:
filename = "{0}.ko.debug".format(module)
dep = (db.session.query(PackageDependency)
.filter(PackageDependency.package == db_package)
.filter(PackageDependency.type == "PROVIDES")
.filter(PackageDependency.name.like("/%%/{0}"
.format(filename)))
.first())
if dep is None:
filename = "{0}.ko.debug".format(module.replace("_", "-"))
dep = (db.session.query(PackageDependency)
.filter(PackageDependency.package == db_package)
.filter(PackageDependency.type == "PROVIDES")
.filter(PackageDependency.name.like("/%%/{0}"
.format(filename)))
.first())
if dep is None:
self.log_debug("Unable to find debuginfo for module '{0}'"
.format(module))
return None
return dep.name
def validate_ureport(self, ureport):
# we want to keep unreliable frames without function name RHBZ#1119072
if "frames" in ureport:
for frame in ureport["frames"]:
if ("function_name" not in frame and
"reliable" in frame and
not frame["reliable"]):
frame["function_name"] = "_unknown_"
KerneloopsProblem.checker.check(ureport)
return True
def hash_ureport(self, ureport):
hashbase = [ureport["component"]]
hashbase.extend(ureport["taint_flags"])
for i, frame in enumerate(ureport["frames"]):
# Instance of 'KerneloopsProblem' has no 'hashframes' member
# pylint: disable-msg=E1101
if i >= self.hashframes:
break
if not "module_name" in frame:
module = "vmlinux"
else:
module = frame["module_name"]
hashbase.append("{0} @ {1}".format(frame["function_name"], module))
return sha1("\n".join(hashbase)).hexdigest()
def save_ureport(self, db, db_report, ureport, flush=False, count=1):
bthash1 = self._hash_koops(ureport["frames"], skip_unreliable=False)
bthash2 = self._hash_koops(ureport["frames"], skip_unreliable=True)
if len(db_report.backtraces) < 1:
db_backtrace = ReportBacktrace()
db_backtrace.report = db_report
db.session.add(db_backtrace)
db_thread = ReportBtThread()
db_thread.backtrace = db_backtrace
db_thread.crashthread = True
db.session.add(db_thread)
db_bthash1 = ReportBtHash()
db_bthash1.backtrace = db_backtrace
db_bthash1.hash = bthash1
db_bthash1.type = "NAMES"
db.session.add(db_bthash1)
if bthash2 is not None and bthash1 != bthash2:
db_bthash2 = ReportBtHash()
db_bthash2.backtrace = db_backtrace
db_bthash2.hash = bthash2
db_bthash2.type = "NAMES"
db.session.add(db_bthash2)
new_symbols = {}
new_symbolsources = {}
i = 0
for frame in ureport["frames"]:
# OK, this is totally ugly.
# Frames may contain inlined functions, that would normally
# require shifting all frames by 1 and inserting a new one.
# There is no way to do this efficiently with SQL Alchemy
# (you need to go one by one and flush after each) so
# creating a space for additional frames is a huge speed
# optimization.
i += 10
# nah, another hack, deals with wrong parsing
if frame["function_name"].startswith("0x"):
continue
if not "module_name" in frame:
module = "vmlinux"
else:
module = frame["module_name"]
db_symbol = get_symbol_by_name_path(db, frame["function_name"],
module)
if db_symbol is None:
key = (frame["function_name"], module)
if key in new_symbols:
db_symbol = new_symbols[key]
else:
db_symbol = Symbol()
db_symbol.name = frame["function_name"]
db_symbol.normalized_path = module
db.session.add(db_symbol)
new_symbols[key] = db_symbol
# this doesn't work well. on 64bit, kernel maps to
# the end of address space (64bit unsigned), but in
# postgres bigint is 64bit signed and can't save
# the value - let's just map it to signed
if frame["address"] >= (1 << 63):
address = frame["address"] - (1 << 64)
else:
address = frame["address"]
db_symbolsource = get_ssource_by_bpo(db, ureport["version"],
module, address)
if db_symbolsource is None:
key = (ureport["version"], module, address)
if key in new_symbolsources:
db_symbolsource = new_symbolsources[key]
else:
db_symbolsource = SymbolSource()
db_symbolsource.path = module
db_symbolsource.offset = address
db_symbolsource.func_offset = frame["function_offset"]
db_symbolsource.symbol = db_symbol
db_symbolsource.build_id = ureport["version"]
db.session.add(db_symbolsource)
new_symbolsources[key] = db_symbolsource
db_frame = ReportBtFrame()
db_frame.thread = db_thread
db_frame.order = i
db_frame.symbolsource = db_symbolsource
db_frame.inlined = False
db_frame.reliable = frame["reliable"]
db.session.add(db_frame)
for taintflag in ureport["taint_flags"]:
db_taintflag = get_taint_flag_by_ureport_name(db, taintflag)
if db_taintflag is None:
self.log_warn("Skipping unsupported taint flag '{0}'"
.format(taintflag))
continue
db_bttaintflag = ReportBtTaintFlag()
db_bttaintflag.backtrace = db_backtrace
db_bttaintflag.taintflag = db_taintflag
db.session.add(db_bttaintflag)
if "modules" in ureport:
new_modules = {}
# use set() to remove duplicates
for module in set(ureport["modules"]):
idx = module.find("(")
if idx >= 0:
module = module[:idx]
db_module = get_kernelmodule_by_name(db, module)
if db_module is None:
if module in new_modules:
db_module = new_modules[module]
else:
db_module = KernelModule()
db_module.name = module
db.session.add(db_module)
new_modules[module] = db_module
db_btmodule = ReportBtKernelModule()
db_btmodule.kernelmodule = db_module
db_btmodule.backtrace = db_backtrace
db.session.add(db_btmodule)
# do not overwrite an existing oops
if not db_report.has_lob("oops"):
# do not append here, but create a new dict
# we only want save_ureport_post_flush process the most
# recently saved report
self.add_lob = {db_report: ureport["raw_oops"].encode("utf-8")}
if flush:
db.session.flush()
def save_ureport_post_flush(self):
for report, raw_oops in self.add_lob.items():
report.save_lob("oops", raw_oops)
# clear the list so that re-calling does not make problems
self.add_lob = {}
def get_component_name(self, ureport):
return ureport["component"]
def compare(self, db_report1, db_report2):
satyr_report1 = self._db_report_to_satyr(db_report1)
satyr_report2 = self._db_report_to_satyr(db_report2)
return satyr_report1.distance(satyr_report2)
def compare_many(self, db_reports):
self.log_info("Loading reports")
reports = []
ret_db_reports = []
i = 0
for db_report in db_reports:
i += 1
self.log_debug("[{0} / {1}] Loading report #{2}"
.format(i, len(db_reports), db_report.id))
report = self._db_report_to_satyr(db_report)
if report is None:
self.log_debug("Unable to build satyr.Kerneloops")
continue
reports.append(report)
ret_db_reports.append(db_report)
self.log_info("Calculating distances")
distances = satyr.Distances(reports, len(reports))
return ret_db_reports, distances
def get_ssources_for_retrace(self, db, max_fail_count=-1):
q = (db.session.query(SymbolSource)
.join(ReportBtFrame)
.join(ReportBtThread)
.join(ReportBacktrace)
.join(Report)
.filter(Report.type == KerneloopsProblem.name)
.filter((SymbolSource.source_path == None) |
(SymbolSource.line_number == None))
.filter(SymbolSource.symbol_id != None))
if max_fail_count >= 0:
q = q.filter(SymbolSource.retrace_fail_count <= max_fail_count)
return q.all()
def find_packages_for_ssource(self, db, db_ssource):
if db_ssource.build_id is None:
self.log_debug("No kernel information for '{0}' @ '{1}'"
.format(db_ssource.symbol.name, db_ssource.path))
return db_ssource, (None, None, None)
if db_ssource.build_id in self._kernel_pkg_map:
return db_ssource, self._kernel_pkg_map[db_ssource.build_id]
if self.archnames is None:
self.archnames = set(arch.name for arch in get_archs(db))
kernelver = self._parse_kernel_build_id(db_ssource.build_id, self.archnames)
version, release, arch, flavour = kernelver
if flavour is not None:
basename = "kernel-{0}-debuginfo".format(flavour)
else:
basename = "kernel-debuginfo"
db_debug_pkg = get_package_by_nevra(db, basename, 0,
version, release, arch)
nvra = "{0}-{1}-{2}.{3}".format(basename, version, release, arch)
db_src_pkg = None
if db_debug_pkg is None:
self.log_debug("Package {0} not found in storage".format(nvra))
elif not self.skipsrc:
srcname = "kernel-debuginfo-common-{0}".format(arch)
db_src_pkg = get_package_by_name_build_arch(db, srcname,
db_debug_pkg.build,
db_debug_pkg.arch)
if db_src_pkg is None:
self.log_debug("Package {0}-{1}-{2}.{3} not found in storage"
.format(srcname, version, release, arch))
result = db_debug_pkg, db_debug_pkg, db_src_pkg
self._kernel_pkg_map[db_ssource.build_id] = result
return db_ssource, result
def retrace(self, db, task):
new_symbols = {}
new_symbolsources = {}
debug_paths = set(os.path.join(task.debuginfo.unpacked_path, fname[1:])
for fname in task.debuginfo.debug_files)
if task.debuginfo.debug_files is not None:
db_debug_pkg = task.debuginfo.db_package
if db_debug_pkg.has_lob("offset_map"):
with db_debug_pkg.get_lob_fd("offset_map") as fd:
offset_map = pickle.load(fd)
else:
offset_map = get_function_offset_map(debug_paths)
db_debug_pkg.save_lob("offset_map", pickle.dumps(offset_map))
else:
offset_map = {}
for bin_pkg, db_ssources in task.binary_packages.items():
i = 0
for db_ssource in db_ssources:
i += 1
module = db_ssource.path
self.log_info(u"[{0} / {1}] Processing '{2}' @ '{3}'"
.format(i, len(db_ssources),
db_ssource.symbol.name, module))
if db_ssource.path == "vmlinux":
address = db_ssource.offset
if address < 0:
address += (1 << 64)
else:
if module not in offset_map:
self.log_debug("Module '{0}' not found in package '{1}'"
.format(module, task.debuginfo.nvra))
db_ssource.retrace_fail_count += 1
continue
module_map = offset_map[module]
symbol_name = db_ssource.symbol.name
if symbol_name not in module_map:
symbol_name = symbol_name.lstrip("_")
if symbol_name not in module_map:
self.log_debug("Function '{0}' not found in module "
"'{1}'".format(db_ssource.symbol.name,
module))
db_ssource.retrace_fail_count += 1
continue
address = module_map[symbol_name] + db_ssource.func_offset
debug_dir = os.path.join(task.debuginfo.unpacked_path,
"usr", "lib", "debug")
debug_path = self._get_debug_path(db, module,
task.debuginfo.db_package)
if debug_path is None:
db_ssource.retrace_fail_count += 1
continue
try:
abspath = os.path.join(task.debuginfo.unpacked_path,
debug_path[1:])
results = addr2line(abspath, address, debug_dir)
results.reverse()
except FafError as ex:
self.log_debug("addr2line failed: {0}".format(str(ex)))
db_ssource.retrace_fail_count += 1
continue
inl_id = 0
while len(results) > 1:
inl_id += 1
funcname, srcfile, srcline = results.pop()
self.log_debug("Unwinding inlined function '{0}'"
.format(funcname))
# hack - we have no offset for inlined symbols
# let's use minus source line to avoid collisions
offset = -srcline
db_ssource_inl = get_ssource_by_bpo(db, db_ssource.build_id,
db_ssource.path, offset)
if db_ssource_inl is None:
key = (db_ssource.build_id, db_ssource.path, offset)
if key in new_symbolsources:
db_ssource_inl = new_symbolsources[key]
else:
db_symbol_inl = get_symbol_by_name_path(db,
funcname,
module)
if db_symbol_inl is None:
sym_key = (funcname, module)
if sym_key in new_symbols:
db_symbol_inl = new_symbols[sym_key]
else:
db_symbol_inl = Symbol()
db_symbol_inl.name = funcname
db_symbol_inl.normalized_path = module
db.session.add(db_symbol_inl)
new_symbols[sym_key] = db_symbol_inl
db_ssource_inl = SymbolSource()
db_ssource_inl.symbol = db_symbol_inl
db_ssource_inl.build_id = db_ssource.build_id
db_ssource_inl.path = module
db_ssource_inl.offset = offset
db_ssource_inl.source_path = srcfile
db_ssource_inl.line_number = srcline
db.session.add(db_ssource_inl)
new_symbolsources[key] = db_ssource_inl
for db_frame in db_ssource.frames:
db_frames = sorted(db_frame.thread.frames,
key=lambda f: f.order)
idx = db_frames.index(db_frame)
if idx > 0:
prevframe = db_frame.thread.frames[idx - 1]
if (prevframe.inlined and
prevframe.symbolsource == db_ssource_inl):
continue
db_newframe = ReportBtFrame()
db_newframe.symbolsource = db_ssource_inl
db_newframe.thread = db_frame.thread
db_newframe.inlined = True
db_newframe.order = db_frame.order - inl_id
db.session.add(db_newframe)
funcname, srcfile, srcline = results.pop()
self.log_debug("Result: {0}".format(funcname))
db_symbol = get_symbol_by_name_path(db, funcname, module)
if db_symbol is None:
key = (funcname, module)
if key in new_symbols:
db_symbol = new_symbols[key]
else:
self.log_debug("Creating new symbol '{0}' @ '{1}'"
.format(funcname, module))
db_symbol = Symbol()
db_symbol.name = funcname
db_symbol.normalized_path = module
db.session.add(db_symbol)
new_symbols[key] = db_symbol
if db_symbol.nice_name is None:
db_symbol.nice_name = demangle(funcname)
db_ssource.symbol = db_symbol
db_ssource.source_path = srcfile
db_ssource.line_number = srcline
if task.debuginfo is not None:
self.log_debug("Removing {0}".format(task.debuginfo.unpacked_path))
shutil.rmtree(task.debuginfo.unpacked_path, ignore_errors=True)
if task.source is not None and task.source.unpacked_path is not None:
self.log_debug("Removing {0}".format(task.source.unpacked_path))
shutil.rmtree(task.source.unpacked_path, ignore_errors=True)
def check_btpath_match(self, ureport, parser):
for frame in ureport["frames"]:
# vmlinux
if not "module_name" in frame:
continue
match = parser.match(frame["module_name"])
if match is not None:
return True
return False
def find_crash_function(self, db_backtrace):
satyr_koops = self._db_backtrace_to_satyr(db_backtrace)
return satyr_koops.frames[0].function_name
ReportReason,
column_len)
from pyfaf.ureport_compat import ureport1to2
from sqlalchemy.exc import IntegrityError
log = log.getChildLogger(__name__)
__all__ = ["get_version", "save", "ureport2",
"validate", "validate_attachment"]
UREPORT_CHECKER = DictChecker({
"os": DictChecker({
"name": StringChecker(allowed=systems.keys()),
"version": StringChecker(pattern=r"^[a-zA-Z0-9_\.\-\+~]+$",
maxlen=column_len(OpSysRelease,
"version")),
"architecture": StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
# Anything else will be checked by the plugin
}),
# The checker for packages depends on operating system
"packages": ListChecker(Checker(object)),
"problem": DictChecker({
"type": StringChecker(allowed=problemtypes.keys()),
# Anything else will be checked by the plugin
}),
"reason": StringChecker(maxlen=column_len(ReportReason, "reason")),
class PythonProblem(ProblemType):
name = "python"
nice_name = "Unhandled Python exception"
checker = DictChecker({
# no need to check type twice, the toplevel checker already did it
# "type": StringChecker(allowed=[PythonProblem.name]),
"exception_name": StringChecker(pattern=r"^[a-zA-Z0-9_\.]+$", maxlen=256),
"component": StringChecker(pattern=r"^[a-zA-Z0-9\-\._]+$",
maxlen=column_len(OpSysComponent,
"name")),
"stacktrace": ListChecker(DictChecker({
"file_name": StringChecker(maxlen=column_len(SymbolSource,
"path")),
"file_line": IntChecker(minval=1, mandatory=False),
"line_contents": StringChecker(maxlen=column_len(SymbolSource,
"srcline"),
mandatory=False),
"function_name": StringChecker(pattern=r"^([a-zA-Z0-9_]+|[a-z ']+)",
maxlen=column_len(Symbol, "name"),
mandatory=False),
"special_function": StringChecker(pattern=r"^[a-zA-Z0-9_]+",
maxlen=column_len(Symbol, "name"),
mandatory=False),
}), minlen=1)
})
def __init__(self, *args, **kwargs):
super(PythonProblem, self).__init__()
hashkeys = ["processing.pythonhashframes", "processing.hashframes"]
self.hashframes = None
self.load_config_to_self("hashframes", hashkeys, 16, callback=int)
cmpkeys = ["processing.pythoncmpframes", "processing.cmpframes",
"processing.clusterframes"]
self.cmpframes = None
self.load_config_to_self("cmpframes", cmpkeys, 16, callback=int)
cutkeys = ["processing.pythoncutthreshold", "processing.cutthreshold"]
self.cutthreshold = None
self.load_config_to_self("cutthreshold", cutkeys, 0.3, callback=float)
normkeys = ["processing.pythonnormalize", "processing.normalize"]
self.normalize = None
self.load_config_to_self("normalize", normkeys, True, callback=str2bool)
skipkeys = ["retrace.pythonskipsource", "retrace.skipsource"]
self.skipsrc = None
self.load_config_to_self("skipsrc", skipkeys, True, callback=str2bool)
def _hash_traceback(self, traceback):
hashbase = []
for frame in traceback:
if "special_function" in frame:
funcname = "<{0}>".format(frame["special_function"])
else:
funcname = frame["function_name"]
hashbase.append("{0} @ {1} + {2}".format(funcname,
frame["file_name"],
frame["file_line"]))
return hash_list(hashbase)
def db_report_to_satyr(self, db_report):
if not db_report.backtraces:
self.log_warn("Report #{0} has no usable backtraces"
.format(db_report.id))
return None
db_backtrace = db_report.backtraces[0]
if not db_backtrace.threads:
self.log_warn("Backtrace #{0} has no usable threads"
.format(db_backtrace.id))
return None
db_thread = db_backtrace.threads[0]
if not db_thread.frames:
self.log_warn("Thread #{0} has no usable frames"
.format(db_thread.id))
return None
stacktrace = satyr.PythonStacktrace()
if db_report.errname is not None:
stacktrace.exception_name = db_report.errname
for db_frame in db_thread.frames:
frame = satyr.PythonFrame()
funcname = db_frame.symbolsource.symbol.name
if funcname.startswith("<") and funcname.endswith(">"):
frame.special_function = funcname[1:-1]
else:
frame.function_name = funcname
frame.file_line = db_frame.symbolsource.offset
frame.file_name = db_frame.symbolsource.path
if db_frame.symbolsource.srcline is not None:
frame.line_contents = db_frame.symbolsource.srcline
stacktrace.frames.append(frame)
return stacktrace
def validate_ureport(self, ureport):
PythonProblem.checker.check(ureport)
for frame in ureport["stacktrace"]:
if "function_name" not in frame and "special_function" not in frame:
raise CheckError("Either `function_name` or `special_function`"
" is required")
return True
def hash_ureport(self, ureport):
hashbase = [ureport["component"]]
for i, frame in enumerate(ureport["stacktrace"]):
# Instance of 'PythonProblem' has no 'hashframes' member
# pylint: disable-msg=E1101
if i >= self.hashframes:
break
if "special_function" in frame:
funcname = "<{0}>".format(frame["special_function"])
else:
funcname = frame["function_name"]
hashbase.append("{0} @ {1} + {2}".format(funcname,
frame["file_name"],
frame["file_line"]))
return hash_list(hashbase)
def get_component_name(self, ureport):
return ureport["component"]
def save_ureport(self, db, db_report, ureport, flush=False, count=1):
crashframe = ureport["stacktrace"][0]
if "special_function" in crashframe:
crashfn = "<{0}>".format(crashframe["special_function"])
else:
crashfn = crashframe["function_name"]
if not db_report.errname:
db_report.errname = ureport["exception_name"]
elif ureport["exception_name"] and (ureport["exception_name"][0] in ascii_uppercase
or "." in ureport["exception_name"]):
# Only overwrite errname if the new one begins with an uppercase
# letter or contains a ".", i.e. is probably a valid exception type
db_report.errname = ureport["exception_name"]
db_reportexe = get_reportexe(db, db_report, crashframe["file_name"])
if db_reportexe is None:
db_reportexe = ReportExecutable()
db_reportexe.report = db_report
db_reportexe.path = crashframe["file_name"]
db_reportexe.count = 0
db.session.add(db_reportexe)
db_reportexe.count += count
bthash = self._hash_traceback(ureport["stacktrace"])
if not db_report.backtraces:
db_backtrace = ReportBacktrace()
db_backtrace.report = db_report
db_backtrace.crashfn = crashfn
db.session.add(db_backtrace)
db_bthash = ReportBtHash()
db_bthash.type = "NAMES"
db_bthash.hash = bthash
db_bthash.backtrace = db_backtrace
db_thread = ReportBtThread()
db_thread.backtrace = db_backtrace
db_thread.crashthread = True
db.session.add(db_thread)
new_symbols = {}
new_symbolsources = {}
i = 0
for frame in ureport["stacktrace"]:
i += 1
if "special_function" in frame:
function_name = "<{0}>".format(frame["special_function"])
else:
function_name = frame["function_name"]
if "special_file" in frame:
file_name = "<{0}>".format(frame["special_file"])
else:
file_name = frame["file_name"]
norm_path = get_libname(file_name)
db_symbol = get_symbol_by_name_path(db, function_name,
norm_path)
if db_symbol is None:
key = (function_name, norm_path)
if key in new_symbols:
db_symbol = new_symbols[key]
else:
db_symbol = Symbol()
db_symbol.name = function_name
db_symbol.normalized_path = norm_path
db.session.add(db_symbol)
new_symbols[key] = db_symbol
db_symbolsource = get_symbolsource(db, db_symbol,
file_name,
frame["file_line"])
if db_symbolsource is None:
key = (function_name, file_name,
frame["file_line"])
if key in new_symbolsources:
db_symbolsource = new_symbolsources[key]
else:
db_symbolsource = SymbolSource()
db_symbolsource.path = file_name
db_symbolsource.offset = frame["file_line"]
db_symbolsource.source_path = file_name
db_symbolsource.symbol = db_symbol
if "line_contents" in frame:
db_symbolsource.srcline = frame["line_contents"]
if "file_line" in frame:
db_symbolsource.line_number = frame["file_line"]
db.session.add(db_symbolsource)
new_symbolsources[key] = db_symbolsource
db_frame = ReportBtFrame()
db_frame.order = i
db_frame.inlined = False
db_frame.symbolsource = db_symbolsource
db_frame.thread = db_thread
db.session.add(db_frame)
if flush:
db.session.flush()
def save_ureport_post_flush(self):
self.log_debug("save_ureport_post_flush is not required for python")
def _get_ssources_for_retrace_query(self, db):
return None
def find_packages_for_ssource(self, db, db_ssource):
self.log_info("Retracing is not required for Python exceptions")
return None, (None, None, None)
def retrace(self, db, task):
self.log_info("Retracing is not required for Python exceptions")
def compare(self, db_report1, db_report2):
satyr_report1 = self.db_report_to_satyr(db_report1)
satyr_report2 = self.db_report_to_satyr(db_report2)
return satyr_report1.distance(satyr_report2)
def check_btpath_match(self, ureport, parser):
for frame in ureport["stacktrace"]:
if "special_file" in frame:
file_name = "<{0}>".format(frame["special_file"])
else:
file_name = frame["file_name"]
match = parser.match(file_name)
if match is not None:
return True
return False
def find_crash_function(self, db_backtrace):
crashthreads = [t for t in db_backtrace.threads if t.crashthread]
if not crashthreads:
self.log_debug("crashthread not found")
return None
if len(crashthreads) > 1:
self.log_debug("multiple crash threads found")
return None
db_symbol = crashthreads[0].frames[0].symbolsource.symbol
return db_symbol.nice_name or db_symbol.name
class Fedora(System):
name = "fedora"
nice_name = "Fedora"
packages_checker = ListChecker(DictChecker({
"name":
StringChecker(pattern=r"^[a-zA-Z0-9_\-\.\+~]+$",
maxlen=column_len(Package, "name")),
"epoch":
IntChecker(minval=0),
"version":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+~]+$",
maxlen=column_len(Build, "version")),
"release":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "release")),
"architecture":
StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
}),
minlen=0)
ureport_checker = DictChecker({
# no need to check name, version and architecture twice
# the toplevel checker already did it
# "name": StringChecker(allowed=[Fedora.name])
# "version": StringChecker()
# "architecture": StringChecker()
"desktop":
StringChecker(mandatory=False,
pattern=r"^[a-zA-Z0-9_/-]+$",
maxlen=column_len(ReportReleaseDesktop, "desktop"))
})
pkg_roles = ["affected", "related", "selinux_policy"]
@classmethod
def install(cls, db, logger=None) -> None:
if logger is None:
logger = log.getChild(cls.__name__)
logger.info("Adding Fedora operating system")
new = OpSys()
new.name = cls.nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db) -> bool:
return bool(get_opsys_by_name(db, cls.nice_name))
def __init__(self) -> None:
super().__init__()
self.eol = None
self.pdc_url = None
self.pagure_url = None
self.build_aging_days = None
self.koji_url = None
self.ignored_releases = []
self.allow_unpackaged = None
self.load_config_to_self("eol", ["fedora.supporteol"],
False,
callback=str2bool)
self.load_config_to_self("pdc_url", ["fedora.fedorapdc"],
"https://pdc.fedoraproject.org/rest_api/v1/")
self.load_config_to_self("pagure_url", ["fedora.pagureapi"],
"https://src.fedoraproject.org/api/0/")
self.load_config_to_self("build_aging_days",
["fedora.build-aging-days"],
7,
callback=int)
self.load_config_to_self("koji_url", ["fedora.koji-url"], None)
self.load_config_to_self("ignored_releases",
["fedora.ignored-releases"], [],
callback=words2list)
self.load_config_to_self("allow_unpackaged",
["ureport.allow-unpackaged"],
False,
callback=str2bool)
def _save_packages(self, db, db_report, packages, count=1) -> None:
for package in packages:
role = "RELATED"
if "package_role" in package:
if package["package_role"] == "affected":
role = "CRASHED"
elif package["package_role"] == "selinux_policy":
role = "SELINUX_POLICY"
db_package = get_package_by_nevra(db,
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["architecture"])
if db_package is None:
self.log_warn("Package {0}-{1}:{2}-{3}.{4} not found in "
"storage".format(package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"]))
db_unknown_pkg = get_unknown_package(db, db_report, role,
package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"])
if db_unknown_pkg is None:
db_arch = get_arch_by_name(db, package["architecture"])
if db_arch is None:
continue
db_unknown_pkg = ReportUnknownPackage()
db_unknown_pkg.report = db_report
db_unknown_pkg.name = package["name"]
db_unknown_pkg.epoch = package["epoch"]
db_unknown_pkg.version = package["version"]
db_unknown_pkg.release = package["release"]
db_unknown_pkg.semver = to_semver(package["version"])
db_unknown_pkg.semrel = to_semver(package["release"])
db_unknown_pkg.arch = db_arch
db_unknown_pkg.type = role
db_unknown_pkg.count = 0
db.session.add(db_unknown_pkg)
db_unknown_pkg.count += count
continue
db_reportpackage = get_reportpackage(db, db_report, db_package)
if db_reportpackage is None:
db_reportpackage = ReportPackage()
db_reportpackage.report = db_report
db_reportpackage.installed_package = db_package
db_reportpackage.count = 0
db_reportpackage.type = role
db.session.add(db_reportpackage)
db_reportpackage.count += count
def validate_ureport(self, ureport) -> bool:
Fedora.ureport_checker.check(ureport)
return True
def validate_packages(self, packages) -> bool:
affected = False
Fedora.packages_checker.check(packages)
for package in packages:
if "package_role" in package:
if package["package_role"] not in Fedora.pkg_roles:
raise FafError(
"Only the following package roles are allowed: "
"{0}".format(", ".join(Fedora.pkg_roles)))
if package["package_role"] == "affected":
affected = True
if not (affected or self.allow_unpackaged):
raise FafError("uReport must contain affected package")
return True
def save_ureport(self,
db,
db_report,
ureport,
packages,
flush=False,
count=1) -> None:
if "desktop" in ureport:
db_release = get_osrelease(db, Fedora.nice_name,
ureport["version"])
if db_release is None:
self.log_warn("Release '{0} {1}' not found".format(
Fedora.nice_name, ureport["version"]))
else:
db_reldesktop = get_report_release_desktop(
db, db_report, db_release, ureport["desktop"])
if db_reldesktop is None:
db_reldesktop = ReportReleaseDesktop()
db_reldesktop.report = db_report
db_reldesktop.release = db_release
db_reldesktop.desktop = ureport["desktop"]
db_reldesktop.count = 0
db.session.add(db_reldesktop)
db_reldesktop.count += count
self._save_packages(db, db_report, packages, count=count)
if flush:
db.session.flush()
def get_releases(self) -> Dict[str, Dict[str, str]]:
result = {}
# Page size -1 means, that all results are on one page
url = f"{self.pdc_url}releases/?page_size=-1&short={Fedora.name}"
with urlopen(url) as response:
releases = json.load(response)
for release in releases:
ver = release["version"].lower()
# only accept Fedora version with decimals (or rawhide)
if not ver.isdecimal() and ver != "rawhide":
continue
# Only accept GA releases, i.e. skip updates and updates-testing
# pseudo-releases. Moreover check for specifically ignored releases.
if release.get("release_type") != "ga" or self._is_ignored(ver):
continue
result[ver] = {"status": "ACTIVE" if release["active"] else "EOL"}
return result
def get_components(self, release) -> List[str]:
branch = self._release_to_branch(release)
result = []
url = (f"{self.pdc_url}component-branches/?name={branch}&page_size=-1"
"&fields=global_component&type=rpm")
with urlopen(url) as response:
components = json.load(response)
for item in components:
result.append(item["global_component"])
return result
def get_component_acls(self, component) -> Dict[str, Dict[str, bool]]:
result: Dict[str, Dict[str, bool]] = {}
url = f"{self.pagure_url}/rpms/{component}"
try:
with urlopen(url) as response:
acls = json.load(response)
except HTTPError as ex:
self.log_error(
"Unable to get package information for component '%s': %s\n\tURL: %s",
component, str(ex), url)
return result
for user_g in acls["access_users"]:
for user in acls["access_users"][user_g]:
result[user] = {"commit": True, "watchbugzilla": False}
# Check for watchers
url += "/watchers"
try:
with urlopen(url) as response:
watchers = json.load(response)
except HTTPError as ex:
self.log_error(
"Unable to get watchers for component '%s': %s\n\tURL: %s",
component, str(ex), url)
return result
for user in watchers["watchers"]:
if user in result.keys():
result[user]["watchbugzilla"] = True
else:
result[user] = {"commit": False, "watchbugzilla": True}
return result
def get_build_candidates(self, db) -> List[Build]:
return (db.session.query(Build).filter(
Build.release.like("%%.fc%%")).all())
def check_pkgname_match(self, packages, parser) -> bool:
for package in packages:
if (not "package_role" in package
or package["package_role"].lower() != "affected"):
continue
nvra = "{0}-{1}-{2}.{3}".format(package["name"],
package["version"],
package["release"],
package["architecture"])
match = parser.match(nvra)
if match is not None:
return True
return False
def _release_to_branch(self, release) -> str:
"""
Convert faf's release to branch name
"""
if not isinstance(release, str):
release = str(release)
if release.lower() == "rawhide":
branch = "rawhide"
elif release.isdigit():
int_release = int(release)
if int_release < 6:
branch = "FC-{0}".format(int_release)
elif int_release == 6:
branch = "fc{0}".format(int_release)
else:
branch = "f{0}".format(int_release)
else:
raise FafError("{0} is not a valid Fedora version".format(release))
return branch
def get_released_builds(
self, release) -> List[Dict[str, Union[str, int, datetime]]]:
session = koji.ClientSession(self.koji_url)
builds_release = session.listTagged(tag="f{0}".format(release),
inherit=False)
builds_updates = session.listTagged(tag="f{0}-updates".format(release),
inherit=False)
return [{
"name":
b["name"],
"epoch":
b["epoch"] if b["epoch"] is not None else 0,
"version":
b["version"],
"release":
b["release"],
"nvr":
b["nvr"],
"completion_time":
datetime.strptime(b["completion_time"], "%Y-%m-%d %H:%M:%S.%f")
} for b in sorted(builds_release + builds_updates,
key=lambda b: b["completion_time"],
reverse=True)]
def _is_ignored(self, ver) -> bool:
"""
Check if the release version matches any of the glob-like patterns specified
in the configuration option 'ignored-releases'.
"""
for pattern in self.ignored_releases:
if fnmatch.fnmatchcase(ver, pattern):
return True
return False
class Fedora(System):
name = "fedora"
nice_name = "Fedora"
supported_repos = ["fedora-koji"]
packages_checker = ListChecker(DictChecker({
"name":
StringChecker(pattern=r"^[a-zA-Z0-9_\-\.\+~]+$",
maxlen=column_len(Package, "name")),
"epoch":
IntChecker(minval=0),
"version":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+~]+$",
maxlen=column_len(Build, "version")),
"release":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "release")),
"architecture":
StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
}),
minlen=0)
ureport_checker = DictChecker({
# no need to check name, version and architecture twice
# the toplevel checker already did it
# "name": StringChecker(allowed=[Fedora.name])
# "version": StringChecker()
# "architecture": StringChecker()
"desktop":
StringChecker(mandatory=False,
pattern=r"^[a-zA-Z0-9_/-]+$",
maxlen=column_len(ReportReleaseDesktop, "desktop"))
})
pkg_roles = ["affected", "related", "selinux_policy"]
@classmethod
def install(cls, db, logger=None):
if logger is None:
logger = log.getChildLogger(cls.__name__)
logger.info("Adding Fedora operating system")
new = OpSys()
new.name = cls.nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db):
return bool(get_opsys_by_name(db, cls.nice_name))
def __init__(self):
super(Fedora, self).__init__()
self.eol = None
self.pdc_url = None
self.pagure_url = None
self.build_aging_days = None
self.koji_url = None
self.allow_unpackaged = None
self.load_config_to_self("eol", ["fedora.supporteol"],
False,
callback=str2bool)
self.load_config_to_self("pdc_url", ["fedora.fedorapdc"],
"https://pdc.fedoraproject.org/rest_api/v1/")
self.load_config_to_self("pagure_url", ["fedora.pagureapi"],
"https://src.fedoraproject.org/api/0/")
self.load_config_to_self("build_aging_days",
["fedora.build-aging-days"],
7,
callback=int)
self.load_config_to_self("koji_url", ["fedora.koji-url"], None)
self.load_config_to_self("allow_unpackaged",
["ureport.allow-unpackaged"],
False,
callback=str2bool)
def _save_packages(self, db, db_report, packages, count=1):
for package in packages:
role = "RELATED"
if "package_role" in package:
if package["package_role"] == "affected":
role = "CRASHED"
elif package["package_role"] == "selinux_policy":
role = "SELINUX_POLICY"
db_package = get_package_by_nevra(db,
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["architecture"])
if db_package is None:
self.log_warn("Package {0}-{1}:{2}-{3}.{4} not found in "
"storage".format(package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"]))
db_unknown_pkg = get_unknown_package(db, db_report, role,
package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"])
if db_unknown_pkg is None:
db_arch = get_arch_by_name(db, package["architecture"])
if db_arch is None:
continue
db_unknown_pkg = ReportUnknownPackage()
db_unknown_pkg.report = db_report
db_unknown_pkg.name = package["name"]
db_unknown_pkg.epoch = package["epoch"]
db_unknown_pkg.version = package["version"]
db_unknown_pkg.release = package["release"]
db_unknown_pkg.semver = to_semver(package["version"])
db_unknown_pkg.semrel = to_semver(package["release"])
db_unknown_pkg.arch = db_arch
db_unknown_pkg.type = role
db_unknown_pkg.count = 0
db.session.add(db_unknown_pkg)
db_unknown_pkg.count += count
continue
db_reportpackage = get_reportpackage(db, db_report, db_package)
if db_reportpackage is None:
db_reportpackage = ReportPackage()
db_reportpackage.report = db_report
db_reportpackage.installed_package = db_package
db_reportpackage.count = 0
db_reportpackage.type = role
db.session.add(db_reportpackage)
db_reportpackage.count += count
def validate_ureport(self, ureport):
Fedora.ureport_checker.check(ureport)
return True
def validate_packages(self, packages):
affected = False
Fedora.packages_checker.check(packages)
for package in packages:
if "package_role" in package:
if package["package_role"] not in Fedora.pkg_roles:
raise FafError(
"Only the following package roles are allowed: "
"{0}".format(", ".join(Fedora.pkg_roles)))
if package["package_role"] == "affected":
affected = True
if not (affected or self.allow_unpackaged):
raise FafError("uReport must contain affected package")
return True
def save_ureport(self,
db,
db_report,
ureport,
packages,
flush=False,
count=1):
if "desktop" in ureport:
db_release = get_osrelease(db, Fedora.nice_name,
ureport["version"])
if db_release is None:
self.log_warn("Release '{0} {1}' not found".format(
Fedora.nice_name, ureport["version"]))
else:
db_reldesktop = get_report_release_desktop(
db, db_report, db_release, ureport["desktop"])
if db_reldesktop is None:
db_reldesktop = ReportReleaseDesktop()
db_reldesktop.report = db_report
db_reldesktop.release = db_release
db_reldesktop.desktop = ureport["desktop"]
db_reldesktop.count = 0
db.session.add(db_reldesktop)
db_reldesktop.count += count
self._save_packages(db, db_report, packages, count=count)
if flush:
db.session.flush()
def get_releases(self):
result = {}
# Page size -1 means, that all results are on one page
url = self.pdc_url + "releases/?page_size=-1"
response = json.load(urllib.request.urlopen(url))
for release in response:
if release["short"] != Fedora.name:
continue
ver = release["version"].lower()
if "epel" in ver:
continue
result[ver] = {"status": "ACTIVE" if release["active"] else "EOL"}
return result
def get_components(self, release):
branch = self._release_to_branch(release)
result = []
url = self.pdc_url + "component-branches/"
url += "?name={0}&page_size=-1&fields=global_component&type=rpm".format(
branch)
response = json.load(urllib.request.urlopen(url))
for item in response:
result.append(item["global_component"])
return result
def get_component_acls(self, component):
result = {}
url = self.pagure_url + "/rpms/{0}".format(component)
response = json.load(urllib.request.urlopen(url))
if "error" in response:
self.log_error("Unable to get package information for component"
" {0}, error was: {1}".format(
component, response["error"]))
return result
for user_g in response["access_users"]:
for user in response["access_users"][user_g]:
result[user] = {"commit": True, "watchbugzilla": False}
# Check for watchers
url += "/watchers"
response = json.load(urllib.request.urlopen(url))
if "error" in response:
self.log_error("Unable to get package information for component"
" {0}, error was: {1}".format(
component, response["error"]))
return result
for user in response["watchers"]:
if user in result.keys():
result[user]["watchbugzilla"] = True
else:
result[user] = {"commit": False, "watchbugzilla": True}
return result
def get_build_candidates(self, db):
return (db.session.query(Build).filter(
Build.release.like("%%.fc%%")).all())
def check_pkgname_match(self, packages, parser):
for package in packages:
if (not "package_role" in package
or package["package_role"].lower() != "affected"):
continue
nvra = "{0}-{1}-{2}.{3}".format(package["name"],
package["version"],
package["release"],
package["architecture"])
match = parser.match(nvra)
if match is not None:
return True
return False
def _release_to_branch(self, release):
"""
Convert faf's release to branch name
"""
if not isinstance(release, str):
release = str(release)
# "rawhide" is called "master"
if release.lower() == "rawhide":
branch = "master"
elif release.isdigit():
int_release = int(release)
if int_release < 6:
branch = "FC-{0}".format(int_release)
elif int_release == 6:
branch = "fc{0}".format(int_release)
else:
branch = "f{0}".format(int_release)
else:
raise FafError("{0} is not a valid Fedora version")
return branch
def get_released_builds(self, release):
session = koji.ClientSession(self.koji_url)
builds_release = session.listTagged(tag="f{0}".format(release),
inherit=False)
builds_updates = session.listTagged(tag="f{0}-updates".format(release),
inherit=False)
return [{
"name":
b["name"],
"epoch":
b["epoch"] if b["epoch"] is not None else 0,
"version":
b["version"],
"release":
b["release"],
"nvr":
b["nvr"],
"completion_time":
datetime.strptime(b["completion_time"], "%Y-%m-%d %H:%M:%S.%f")
} for b in sorted(builds_release + builds_updates,
key=lambda b: b["completion_time"],
reverse=True)]
class CentOS(System):
name = "centos"
nice_name = "CentOS"
packages_checker = ListChecker(DictChecker({
"name":
StringChecker(pattern=r"^[a-zA-Z0-9_\-\.\+~]+$",
maxlen=column_len(Package, "name")),
"epoch":
IntChecker(minval=0),
"version":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+~]+$",
maxlen=column_len(Build, "version")),
"release":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "release")),
"architecture":
StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
}),
minlen=0)
ureport_checker = DictChecker({
# no need to check name, version and architecture twice
# the toplevel checker already did it
# "name": StringChecker(allowed=[CentOS.name])
# "version": StringChecker()
# "architecture": StringChecker()
})
pkg_roles = ["affected", "related", "selinux_policy"]
@classmethod
def install(cls, db, logger=None) -> None:
if logger is None:
logger = log.getChild(cls.__name__)
logger.info("Adding CentOS")
new = OpSys()
new.name = cls.nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db) -> bool:
return bool(get_opsys_by_name(db, cls.nice_name))
def __init__(self) -> None:
super().__init__()
self.eol = None
self.repo_urls = []
self.allow_unpackaged = None
self.inactive_releases = None
self.active_releases = None
self.load_config_to_self("repo_urls", ["centos.repo-urls"], [],
callback=words2list)
self.load_config_to_self("allow_unpackaged",
["ureport.allow-unpackaged"],
False,
callback=str2bool)
self.load_config_to_self("inactive_releases",
["centos.inactive-releases"])
self.load_config_to_self("active_releases", ["centos.active-releases"])
def _save_packages(self, db, db_report, packages, count=1) -> None:
for package in packages:
role = "RELATED"
if "package_role" in package:
if package["package_role"] == "affected":
role = "CRASHED"
elif package["package_role"] == "selinux_policy":
role = "SELINUX_POLICY"
db_package = get_package_by_nevra(db,
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["architecture"])
if db_package is None:
self.log_warn("Package {0}-{1}:{2}-{3}.{4} not found in "
"storage".format(package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"]))
db_unknown_pkg = get_unknown_package(db, db_report, role,
package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"])
if db_unknown_pkg is None:
db_arch = get_arch_by_name(db, package["architecture"])
if db_arch is None:
continue
db_unknown_pkg = ReportUnknownPackage()
db_unknown_pkg.report = db_report
db_unknown_pkg.name = package["name"]
db_unknown_pkg.epoch = package["epoch"]
db_unknown_pkg.version = package["version"]
db_unknown_pkg.release = package["release"]
db_unknown_pkg.semver = to_semver(package["version"])
db_unknown_pkg.semrel = to_semver(package["release"])
db_unknown_pkg.arch = db_arch
db_unknown_pkg.type = role
db_unknown_pkg.count = 0
db.session.add(db_unknown_pkg)
db_unknown_pkg.count += count
continue
db_reportpackage = get_reportpackage(db, db_report, db_package)
if db_reportpackage is None:
db_reportpackage = ReportPackage()
db_reportpackage.report = db_report
db_reportpackage.installed_package = db_package
db_reportpackage.count = 0
db_reportpackage.type = role
db.session.add(db_reportpackage)
db_reportpackage.count += count
def validate_ureport(self, ureport) -> bool:
CentOS.ureport_checker.check(ureport)
return True
def validate_packages(self, packages) -> bool:
CentOS.packages_checker.check(packages)
affected = False
for package in packages:
if "package_role" in package:
if package["package_role"] not in CentOS.pkg_roles:
raise FafError(
"Only the following package roles are allowed: "
"{0}".format(", ".join(CentOS.pkg_roles)))
if package["package_role"] == "affected":
affected = True
if not (affected or self.allow_unpackaged):
raise FafError("uReport must contain affected package")
return True
def save_ureport(self,
db,
db_report,
ureport,
packages,
flush=False,
count=1) -> None:
self._save_packages(db, db_report, packages, count=count)
if flush:
db.session.flush()
def get_releases(self) -> Dict[str, Dict[str, str]]:
result = {}
for release in re.findall(r"[\w\.]+", self.inactive_releases):
result[release] = {"status": "EOL"}
for release in re.findall(r"[\w\.]+", self.active_releases):
result[release] = {"status": "ACTIVE"}
return result
def get_components(self, release) -> List[str]:
if not self.repo_urls:
self.log_info("No repository URLs were found.")
return []
urls = [
repo.replace("$releasever", release) for repo in self.repo_urls
]
components = []
if "dnf" in repo_types:
from pyfaf.repos.dnf import Dnf
dnf = Dnf(self.name, *urls)
components.extend(
list(set(pkg["name"] for pkg in dnf.list_packages(["src"]))))
else:
raise FafError("No repo type available")
return components
def get_build_candidates(self, db) -> List[Build]:
return (db.session.query(Build).filter(
Build.release.like("%%.el%%")).all())
def check_pkgname_match(self, packages, parser) -> bool:
for package in packages:
if ("package_role" not in package
or package["package_role"].lower() != "affected"):
continue
nvra = "{0}-{1}-{2}.{3}".format(package["name"],
package["version"],
package["release"],
package["architecture"])
match = parser.match(nvra)
if match is not None:
return True
return False
class JavaProblem(ProblemType):
name = "java"
nice_name = "Unhandled Java exception"
checker = DictChecker({
# no need to check type twice, the toplevel checker already did it
# "type": StringChecker(allowed=[JavaProblem.name]),
"component":
StringChecker(pattern=r"^[a-zA-Z0-9\-\._]+$",
maxlen=column_len(OpSysComponent, "name")),
"threads":
ListChecker(DictChecker({
"name":
StringChecker(),
"frames":
ListChecker(DictChecker({
"name":
StringChecker(maxlen=column_len(Symbol, "name")),
"is_native":
Checker(bool),
"is_exception":
Checker(bool),
}),
minlen=1),
}),
minlen=1)
})
default_frame_checker = DictChecker({
"file_name":
StringChecker(maxlen=column_len(SymbolSource, "source_path")),
"file_line":
IntChecker(minval=1),
"class_path":
StringChecker(maxlen=column_len(SymbolSource, "path")),
})
exception = "Exception thrown"
native = "Native function call"
unknown = "Unknown"
def __init__(self, *args, **kwargs):
super(JavaProblem, self).__init__()
hashkeys = ["processing.javahashframes", "processing.hashframes"]
self.load_config_to_self("hashframes", hashkeys, 16, callback=int)
cmpkeys = [
"processing.javacmpframes", "processing.cmpframes",
"processing.clusterframes"
]
self.load_config_to_self("cmpframes", cmpkeys, 16, callback=int)
cutkeys = ["processing.javacutthreshold", "processing.cutthreshold"]
self.load_config_to_self("cutthreshold", cutkeys, 0.3, callback=float)
normkeys = ["processing.javanormalize", "processing.normalize"]
self.load_config_to_self("normalize",
normkeys,
True,
callback=str2bool)
skipkeys = ["retrace.javaskipsource", "retrace.skipsource"]
self.load_config_to_self("skipsrc", skipkeys, True, callback=str2bool)
def _hash_backtrace(self, threads):
hashbase = []
for thread in threads:
hashbase.append("Thread")
for frame in thread["frames"]:
# Instance of 'JavaProblem' has no 'hashframes' member
# pylint: disable-msg=E1101
if "class_path" in frame:
hashbase.append("{0} @ {1}".format(frame["name"],
frame["class_path"]))
continue
hashbase.append(frame["name"])
return hash_list(hashbase)
def _db_backtrace_find_crash_thread(self, db_backtrace):
if len(db_backtrace.threads) == 1:
return db_backtrace.threads[0]
db_threads = [t for t in db_backtrace.threads if t.crashthread]
if len(db_threads) < 1:
raise FafError(
"No crash thread could be found for backtrace #{0}".format(
db_backtrace.id))
if len(db_threads) > 1:
raise FafError(
"Multiple crash threads found for backtrace #{0}".format(
db_backtrace.id))
return db_threads[0]
def _db_frame_to_satyr(self, db_frame):
class_path = db_frame.symbolsource.path
result = satyr.JavaFrame()
result.name = db_frame.symbolsource.symbol.name
result.is_native = class_path == JavaProblem.native
result.is_exception = class_path == JavaProblem.exception
if class_path not in [
JavaProblem.exception, JavaProblem.native, JavaProblem.unknown
]:
result.class_path = class_path
if db_frame.symbolsource.source_path is not None:
result.file_name = db_frame.symbolsource.source_path
result.file_line = db_frame.symbolsource.line_number
return result
def _db_thread_to_satyr(self, db_thread):
if len(db_thread.frames) < 1:
self.log_warn("Thread #{0} has no usable frames".format(
db_thread.id))
return None
result = satyr.JavaThread()
result.name = "Thread #{0}".format(db_thread.number)
for db_frame in db_thread.frames:
frame = self._db_frame_to_satyr(db_frame)
if frame is None:
continue
result.frames.append(frame)
return result
def _db_backtrace_to_satyr(self, db_backtrace):
if len(db_backtrace.threads) < 1:
self.log_warn("Backtrace #{0} has no usable threads".format(
db_backtrace.id))
return None
if len(db_backtrace.threads) > 1:
self.log_warn("Backtrace #{0} has several threads".format(
db_backtrace.id))
return self._db_thread_to_satyr(db_backtrace.threads[0])
def _db_report_to_satyr(self, db_report):
if len(db_report.backtraces) < 1:
self.log_warn("Report #{0} has no usable backtraces".format(
db_report.id))
return None
return self._db_backtrace_to_satyr(db_report.backtraces[0])
def validate_ureport(self, ureport):
JavaProblem.checker.check(ureport)
for thread in ureport["threads"]:
for frame in thread["frames"]:
if not frame["is_native"] and not frame["is_exception"]:
JavaProblem.default_frame_checker.check(frame)
return True
def hash_ureport(self, ureport):
hashbase = [ureport["component"]]
# at the moment we only send crash thread
# we may need to identify the crash thread in the future
for i, frame in enumerate(ureport["threads"][0]["frames"]):
# Instance of 'JavaProblem' has no 'hashframes' member
# pylint: disable-msg=E1101
if i >= self.hashframes:
break
if "class_path" in frame:
hashbase.append("{0} @ {1}".format(frame["name"],
frame["class_path"]))
continue
hashbase.append(frame["name"])
return hash_list(hashbase)
def get_component_name(self, ureport):
return ureport["component"]
def save_ureport(self, db, db_report, ureport, flush=False, count=1):
# at the moment we only send crash thread
# we may need to identify the crash thread in the future
crashthread = ureport["threads"][0]
crashfn = None
for frame in crashthread["frames"]:
if not frame["is_exception"]:
crashfn = frame["name"]
break
if crashfn is not None and "." in crashfn:
crashfn = crashfn.rsplit(".", 1)[1]
errname = None
for frame in crashthread["frames"]:
if frame["is_exception"]:
errname = frame["name"]
break
if "." in errname:
errname = errname.rsplit(".", 1)[1]
db_report.errname = errname
bthash = self._hash_backtrace(ureport["threads"])
if len(db_report.backtraces) < 1:
db_backtrace = ReportBacktrace()
db_backtrace.report = db_report
db_backtrace.crashfn = crashfn
db.session.add(db_backtrace)
db_bthash = ReportBtHash()
db_bthash.type = "NAMES"
db_bthash.hash = bthash
db_bthash.backtrace = db_backtrace
new_symbols = {}
new_symbolsources = {}
j = 0
for thread in ureport["threads"]:
j += 1
db_thread = ReportBtThread()
db_thread.backtrace = db_backtrace
db_thread.crashthread = thread == crashthread
db_thread.number = j
db.session.add(db_thread)
i = 0
for frame in thread["frames"]:
i += 1
function_name = frame["name"]
if "class_path" in frame:
file_name = frame["class_path"]
elif frame["is_exception"]:
file_name = JavaProblem.exception
elif frame["is_native"]:
file_name = JavaProblem.native
else:
file_name = JavaProblem.unknown
if "file_line" in frame:
file_line = frame["file_line"]
else:
file_line = 0
db_symbol = get_symbol_by_name_path(
db, function_name, file_name)
if db_symbol is None:
key = (function_name, file_name)
if key in new_symbols:
db_symbol = new_symbols[key]
else:
db_symbol = Symbol()
db_symbol.name = function_name
db_symbol.normalized_path = file_name
db.session.add(db_symbol)
new_symbols[key] = db_symbol
db_symbolsource = get_symbolsource(db, db_symbol,
file_name, file_line)
if db_symbolsource is None:
key = (function_name, file_name, file_line)
if key in new_symbolsources:
db_symbolsource = new_symbolsources[key]
else:
db_symbolsource = SymbolSource()
db_symbolsource.path = file_name
db_symbolsource.offset = file_line
if "file_name" in frame:
db_symbolsource.source_path = frame[
"file_name"]
db_symbolsource.line_number = file_line
db_symbolsource.symbol = db_symbol
db.session.add(db_symbolsource)
new_symbolsources[key] = db_symbolsource
db_frame = ReportBtFrame()
db_frame.order = i
db_frame.inlined = False
db_frame.symbolsource = db_symbolsource
db_frame.thread = db_thread
db.session.add(db_frame)
if flush:
db.session.flush()
def save_ureport_post_flush(self):
self.log_debug("save_ureport_post_flush is not required for java")
def _get_ssources_for_retrace_query(self, db):
return None
def find_packages_for_ssource(self, db, db_ssource):
self.log_info("Retracing is not required for Java exceptions")
return None, (None, None, None)
def retrace(self, db, task):
self.log_info("Retracing is not required for Java exceptions")
def compare(self, db_report1, db_report2):
satyr_report1 = self._db_report_to_satyr(db_report1)
satyr_report2 = self._db_report_to_satyr(db_report2)
return satyr_report1.distance(satyr_report2)
def compare_many(self, db_reports):
self.log_info("Loading reports")
reports = []
ret_db_reports = []
i = 0
for db_report in db_reports:
i += 1
self.log_debug("[{0} / {1}] Loading report #{2}".format(
i, len(db_reports), db_report.id))
report = self._db_report_to_satyr(db_report)
if report is None:
self.log_debug("Unable to build satyr.JavaStacktrace")
continue
reports.append(report)
ret_db_reports.append(db_report)
self.log_info("Calculating distances")
distances = satyr.Distances(reports, len(reports))
return ret_db_reports, distances
def check_btpath_match(self, ureport, parser):
for thread in ureport["threads"]:
for frame in thread["frames"]:
for key in ["class_path", "file_name"]:
if key in frame:
match = parser.match(frame[key])
if match is not None:
return True
return False
def find_crash_function(self, db_backtrace):
crash_thread = self._db_backtrace_find_crash_thread(db_backtrace)
return crash_thread.frames[0].symbolsource.symbol.name
class RHEL(System):
name = "rhel"
nice_name = "Red Hat Enterprise Linux"
packages_checker = ListChecker(DictChecker({
"name":
StringChecker(pattern=r"^[a-zA-Z0-9_\-\.\+~]+$",
maxlen=column_len(Package, "name")),
"epoch":
IntChecker(minval=0),
"version":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "version")),
"release":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "release")),
"architecture":
StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
}),
minlen=1)
ureport_checker = DictChecker({
# no need to check name, version and architecture twice
# the toplevel checker already did it
# "name": StringChecker(allowed=[RHEL.name])
# "version": StringChecker()
# "architecture": StringChecker()
})
pkg_roles = ["affected", "related", "selinux_policy"]
@classmethod
def install(cls, db, logger=None):
if logger is None:
logger = log.getChildLogger(cls.__name__)
logger.info("Adding Red Hat Enterprise Linux operating system")
new = OpSys()
new.name = cls.nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db):
return bool(get_opsys_by_name(db, cls.nice_name))
def __init__(self):
super(RHEL, self).__init__()
def _save_packages(self, db, db_report, packages, count=1):
for package in packages:
role = "RELATED"
if "package_role" in package:
if package["package_role"] == "affected":
role = "CRASHED"
elif package["package_role"] == "selinux_policy":
role = "SELINUX_POLICY"
db_package = get_package_by_nevra(db,
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["architecture"])
if db_package is None:
self.log_warn("Package {0}-{1}:{2}-{3}.{4} not found in "
"storage".format(package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"]))
db_unknown_pkg = get_unknown_package(db, db_report, role,
package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"])
if db_unknown_pkg is None:
db_arch = get_arch_by_name(db, package["architecture"])
if db_arch is None:
continue
db_unknown_pkg = ReportUnknownPackage()
db_unknown_pkg.report = db_report
db_unknown_pkg.name = package["name"]
db_unknown_pkg.epoch = package["epoch"]
db_unknown_pkg.version = package["version"]
db_unknown_pkg.release = package["release"]
db_unknown_pkg.arch = db_arch
db_unknown_pkg.type = role
db_unknown_pkg.count = 0
db.session.add(db_unknown_pkg)
db_unknown_pkg.count += count
continue
db_reportpackage = get_reportpackage(db, db_report, db_package)
if db_reportpackage is None:
db_reportpackage = ReportPackage()
db_reportpackage.report = db_report
db_reportpackage.installed_package = db_package
db_reportpackage.count = 0
db_reportpackage.type = role
db.session.add(db_reportpackage)
db_reportpackage.count += count
def validate_ureport(self, ureport):
RHEL.ureport_checker.check(ureport)
return True
def validate_packages(self, packages):
RHEL.packages_checker.check(packages)
for package in packages:
if ("package_role" in package
and package["package_role"] not in RHEL.pkg_roles):
raise FafError("Only the following package roles are allowed: "
"{0}".format(", ".join(RHEL.pkg_roles)))
return True
def save_ureport(self,
db,
db_report,
ureport,
packages,
flush=False,
count=1):
self._save_packages(db, db_report, packages, count=count)
if flush:
db.session.flush()
def get_releases(self):
return []
def get_components(self, release):
return []
def get_component_acls(self, component, release=None):
return {}
def get_build_candidates(self, db):
return (db.session.query(Build).filter(
Build.release.like("%%.el%%")).all())
def check_pkgname_match(self, packages, parser):
for package in packages:
if (not "package_role" in package
or package["package_role"].lower() != "affected"):
continue
nvra = "{0}-{1}-{2}.{3}".format(package["name"],
package["version"],
package["release"],
package["architecture"])
match = parser.match(nvra)
if match is not None:
return True
return False
def get_released_builds(self, release):
return []
class CentOS(System):
name = "centos"
nice_name = "CentOS"
packages_checker = ListChecker(
DictChecker({
"name": StringChecker(pattern=r"^[a-zA-Z0-9_\-\.\+~]+$",
maxlen=column_len(Package,
"name")),
"epoch": IntChecker(minval=0),
"version": StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "version")),
"release": StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "release")),
"architecture": StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
}), minlen=0
)
ureport_checker = DictChecker({
# no need to check name, version and architecture twice
# the toplevel checker already did it
# "name": StringChecker(allowed=[CentOS.name])
# "version": StringChecker()
# "architecture": StringChecker()
})
pkg_roles = ["affected", "related", "selinux_policy"]
@classmethod
def install(cls, db, logger=None):
if logger is None:
logger = log.getChildLogger(cls.__name__)
logger.info("Adding CentOS")
new = OpSys()
new.name = cls.nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db):
return bool(get_opsys_by_name(db, cls.nice_name))
def __init__(self):
super(CentOS, self).__init__()
self.load_config_to_self("base_repo_url", ["centos.base-repo-url"],
"http://vault.centos.org/centos/$releasever/"
"os/Source/")
self.load_config_to_self("updates_repo_url", ["centos.updates-repo-url"],
"http://vault.centos.org/centos/$releasever/"
"updates/Source/")
self.load_config_to_self("allow_unpackaged",
["ureport.allow-unpackaged"], False,
callback=str2bool)
def _save_packages(self, db, db_report, packages, count=1):
for package in packages:
role = "RELATED"
if "package_role" in package:
if package["package_role"] == "affected":
role = "CRASHED"
elif package["package_role"] == "selinux_policy":
role = "SELINUX_POLICY"
db_package = get_package_by_nevra(db,
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["architecture"])
if db_package is None:
self.log_warn("Package {0}-{1}:{2}-{3}.{4} not found in "
"storage".format(package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"]))
db_unknown_pkg = get_unknown_package(db,
db_report,
role,
package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"])
if db_unknown_pkg is None:
db_arch = get_arch_by_name(db, package["architecture"])
if db_arch is None:
continue
db_unknown_pkg = ReportUnknownPackage()
db_unknown_pkg.report = db_report
db_unknown_pkg.name = package["name"]
db_unknown_pkg.epoch = package["epoch"]
db_unknown_pkg.version = package["version"]
db_unknown_pkg.release = package["release"]
db_unknown_pkg.arch = db_arch
db_unknown_pkg.type = role
db_unknown_pkg.count = 0
db.session.add(db_unknown_pkg)
db_unknown_pkg.count += count
continue
db_reportpackage = get_reportpackage(db, db_report, db_package)
if db_reportpackage is None:
db_reportpackage = ReportPackage()
db_reportpackage.report = db_report
db_reportpackage.installed_package = db_package
db_reportpackage.count = 0
db_reportpackage.type = role
db.session.add(db_reportpackage)
db_reportpackage.count += count
def validate_ureport(self, ureport):
CentOS.ureport_checker.check(ureport)
return True
def validate_packages(self, packages):
CentOS.packages_checker.check(packages)
affected = False
for package in packages:
if "package_role" in package:
if package["package_role"] not in CentOS.pkg_roles:
raise FafError("Only the following package roles are allowed: "
"{0}".format(", ".join(CentOS.pkg_roles)))
if package["package_role"] == "affected":
affected = True
if not(affected or self.allow_unpackaged):
raise FafError("uReport must contain affected package")
return True
def save_ureport(self, db, db_report, ureport, packages, flush=False, count=1):
self._save_packages(db, db_report, packages, count=count)
if flush:
db.session.flush()
def get_releases(self):
return {"7": {"status": "ACTIVE"}}
def get_components(self, release):
urls = [repo.replace("$releasever", release)
for repo in [self.base_repo_url, self.updates_repo_url]]
yum = Yum(self.name, *urls)
components = list(set(pkg["name"]
for pkg in yum.list_packages(["src"])))
return components
#def get_component_acls(self, component, release=None):
# return {}
def get_build_candidates(self, db):
return (db.session.query(Build)
.filter(Build.release.like("%%.el%%"))
.all())
def check_pkgname_match(self, packages, parser):
for package in packages:
if ("package_role" not in package or
package["package_role"].lower() != "affected"):
continue
nvra = "{0}-{1}-{2}.{3}".format(package["name"],
package["version"],
package["release"],
package["architecture"])
match = parser.match(nvra)
if match is not None:
return True
return False
class Fedora(System):
name = "fedora"
nice_name = "Fedora"
supported_repos = ["fedora-koji"]
packages_checker = ListChecker(DictChecker({
"name":
StringChecker(pattern=r"^[a-zA-Z0-9_\-\.\+~]+$",
maxlen=column_len(Package, "name")),
"epoch":
IntChecker(minval=0),
"version":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "version")),
"release":
StringChecker(pattern=r"^[a-zA-Z0-9_\.\+]+$",
maxlen=column_len(Build, "release")),
"architecture":
StringChecker(pattern=r"^[a-zA-Z0-9_]+$",
maxlen=column_len(Arch, "name")),
}),
minlen=0)
ureport_checker = DictChecker({
# no need to check name, version and architecture twice
# the toplevel checker already did it
# "name": StringChecker(allowed=[Fedora.name])
# "version": StringChecker()
# "architecture": StringChecker()
"desktop":
StringChecker(mandatory=False,
pattern=r"^[a-zA-Z0-9_/-]+$",
maxlen=column_len(ReportReleaseDesktop, "desktop"))
})
pkg_roles = ["affected", "related", "selinux_policy"]
@classmethod
def install(cls, db, logger=None):
if logger is None:
logger = log.getChildLogger(cls.__name__)
logger.info("Adding Fedora operating system")
new = OpSys()
new.name = cls.nice_name
db.session.add(new)
db.session.flush()
@classmethod
def installed(cls, db):
return bool(get_opsys_by_name(db, cls.nice_name))
def __init__(self):
super(Fedora, self).__init__()
self.load_config_to_self("eol", ["fedora.supporteol"],
False,
callback=str2bool)
self.load_config_to_self("pkgdb_url", ["fedora.pkgdburl"],
"https://admin.fedoraproject.org/pkgdb/")
self._pkgdb = pkgdb2client.PkgDB(url=self.pkgdb_url)
self.load_config_to_self("build_aging_days",
["fedora.build-aging-days"],
7,
callback=int)
self.load_config_to_self("koji_url", ["fedora.koji-url"], None)
self.load_config_to_self("allow_unpackaged",
["ureport.allow-unpackaged"],
False,
callback=str2bool)
def _save_packages(self, db, db_report, packages, count=1):
for package in packages:
role = "RELATED"
if "package_role" in package:
if package["package_role"] == "affected":
role = "CRASHED"
elif package["package_role"] == "selinux_policy":
role = "SELINUX_POLICY"
db_package = get_package_by_nevra(db,
name=package["name"],
epoch=package["epoch"],
version=package["version"],
release=package["release"],
arch=package["architecture"])
if db_package is None:
self.log_warn("Package {0}-{1}:{2}-{3}.{4} not found in "
"storage".format(package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"]))
db_unknown_pkg = get_unknown_package(db, db_report, role,
package["name"],
package["epoch"],
package["version"],
package["release"],
package["architecture"])
if db_unknown_pkg is None:
db_arch = get_arch_by_name(db, package["architecture"])
if db_arch is None:
continue
db_unknown_pkg = ReportUnknownPackage()
db_unknown_pkg.report = db_report
db_unknown_pkg.name = package["name"]
db_unknown_pkg.epoch = package["epoch"]
db_unknown_pkg.version = package["version"]
db_unknown_pkg.release = package["release"]
db_unknown_pkg.arch = db_arch
db_unknown_pkg.type = role
db_unknown_pkg.count = 0
db.session.add(db_unknown_pkg)
db_unknown_pkg.count += count
continue
db_reportpackage = get_reportpackage(db, db_report, db_package)
if db_reportpackage is None:
db_reportpackage = ReportPackage()
db_reportpackage.report = db_report
db_reportpackage.installed_package = db_package
db_reportpackage.count = 0
db_reportpackage.type = role
db.session.add(db_reportpackage)
db_reportpackage.count += count
def validate_ureport(self, ureport):
Fedora.ureport_checker.check(ureport)
return True
def validate_packages(self, packages):
affected = False
Fedora.packages_checker.check(packages)
for package in packages:
if ("package_role" in package):
if (package["package_role"] not in Fedora.pkg_roles):
raise FafError(
"Only the following package roles are allowed: "
"{0}".format(", ".join(Fedora.pkg_roles)))
if (package["package_role"] == "affected"):
affected = True
if not (affected or self.allow_unpackaged):
raise FafError("uReport must contain affected package")
return True
def save_ureport(self,
db,
db_report,
ureport,
packages,
flush=False,
count=1):
if "desktop" in ureport:
db_release = get_osrelease(db, Fedora.nice_name,
ureport["version"])
if db_release is None:
self.log_warn("Release '{0} {1}' not found".format(
Fedora.nice_name, ureport["version"]))
else:
db_reldesktop = get_report_release_desktop(
db, db_report, db_release, ureport["desktop"])
if db_reldesktop is None:
db_reldesktop = ReportReleaseDesktop()
db_reldesktop.report = db_report
db_reldesktop.release = db_release
db_reldesktop.desktop = ureport["desktop"]
db_reldesktop.count = 0
db.session.add(db_reldesktop)
db_reldesktop.count += count
self._save_packages(db, db_report, packages, count=count)
if flush:
db.session.flush()
def get_releases(self):
result = {}
collections = self._pkgdb.get_collections()["collections"]
for collection in collections:
# there is EPEL in collections, we are only interested in Fedora
if collection["name"].lower() != Fedora.name:
continue
# "devel" is called "rawhide" on all other places
if collection["version"].lower() == "devel":
collection["version"] = "rawhide"
result[collection["version"]] = {
"status": collection["status"].upper().replace(' ', '_'),
"kojitag": collection["koji_name"],
"shortname": collection["branchname"],
}
return result
def get_components(self, release):
branch = self._release_to_pkgdb_branch(release)
try:
pkgs = self._pkgdb.get_packages(branches=branch,
page='all',
eol=self.eol)
except pkgdb2client.PkgDBException as e:
raise FafError(
"Unable to get components for {0}, error was: {1}".format(
release, e))
return [pkg["name"] for pkg in pkgs["packages"]]
def get_component_acls(self, component, release=None):
branch = None
if release:
branch = self._release_to_pkgdb_branch(release)
result = {}
try:
packages = self._pkgdb.get_package(component,
branches=branch,
eol=self.eol)
except pkgdb2client.PkgDBException as e:
self.log_error("Unable to get package information for component"
" {0}, error was: {1}".format(component, e))
return result
for pkg in packages["packages"]:
acls = {
pkg["point_of_contact"]: {
"owner": True,
},
}
if not "acls" in pkg:
continue
for acl in pkg["acls"]:
aclname = acl["acl"]
person = acl["fas_name"]
status = acl["status"] == "Approved"
if person in acls:
acls[person][aclname] = status
else:
acls[person] = {aclname: status}
if release:
return acls
branch = pkg["branchname"]
relname = self._pkgdb_branch_to_release(branch)
result[relname] = acls
return result
def get_build_candidates(self, db):
return (db.session.query(Build).filter(
Build.release.like("%%.fc%%")).all())
def check_pkgname_match(self, packages, parser):
for package in packages:
if (not "package_role" in package
or package["package_role"].lower() != "affected"):
continue
nvra = "{0}-{1}-{2}.{3}".format(package["name"],
package["version"],
package["release"],
package["architecture"])
match = parser.match(nvra)
if match is not None:
return True
return False
def _release_to_pkgdb_branch(self, release):
"""
Convert faf's release to pkgdb2 branch name
"""
if not isinstance(release, basestring):
release = str(release)
# "rawhide" is called "master" in pkgdb2
if release.lower() == "rawhide":
branch = "master"
elif release.isdigit():
int_release = int(release)
if int_release < 6:
branch = "FC-{0}".format(int_release)
elif int_release == 6:
branch = "fc{0}".format(int_release)
else:
branch = "f{0}".format(int_release)
else:
raise FafError("{0} is not a valid Fedora version")
return branch
def _pkgdb_branch_to_release(self, branch):
"""
Convert pkgdb2 branch name to faf's release
"""
if branch == "master":
return "rawhide"
if branch.startswith("fc"):
return branch[2:]
if branch.startswith("FC-"):
return branch[3:]
return branch[1:]
def get_released_builds(self, release):
session = koji.ClientSession(self.koji_url)
builds_release = session.listTagged(tag="f{0}".format(release),
inherit=False)
builds_updates = session.listTagged(tag="f{0}-updates".format(release),
inherit=False)
return [{
"name":
b["name"],
"epoch":
b["epoch"] if b["epoch"] is not None else 0,
"version":
b["version"],
"release":
b["release"],
"nvr":
b["nvr"],
"completion_time":
datetime.strptime(b["completion_time"], "%Y-%m-%d %H:%M:%S.%f")
} for b in sorted(builds_release + builds_updates,
key=lambda b: b["completion_time"],
reverse=True)]
