def prepare_tasks(db, debuginfo_map):
"""
Creates tasks from debuginfo map
"""
result = []
for debuginfo in debuginfo_map:
packages = []
for package in debuginfo_map[debuginfo]:
pkg_entry = { "package": package,
"nvra": package.nvra(),
"rpm_path": package.get_lob_path("package"),
"symbols": debuginfo_map[debuginfo][package] }
packages.append(pkg_entry)
if debuginfo.name.startswith("kernel"):
common = "kernel-debuginfo-common-{0}".format(debuginfo.arch.name)
source = db.session.query(Package) \
.join(Arch) \
.join(Build) \
.filter((Build.id == debuginfo.build_id) &
(Package.arch == debuginfo.arch) &
(Package.name == common)) \
.one()
else:
try:
source = db.session.query(Package) \
.join(Arch) \
.join(Build) \
.filter((Build.id == debuginfo.build_id) &
(Arch.name == "src")) \
.one()
except Exception as ex:
logging.error("Unable to find source RPM: {0}".format(str(ex)))
continue
task = { "debuginfo": { "package": debuginfo,
"nvra": debuginfo.nvra(),
"rpm_path": debuginfo.get_lob_path("package") },
"source": { "package": source,
"nvra": source.nvra(),
"rpm_path": source.get_lob_path("package") },
"packages": packages }
result.append(task)
return result
def prepare_debuginfo_map(db):
"""
Prepares the mapping debuginfo ~> packages ~> symbols
"""
result = {}
symbolsources = db.session.query(SymbolSource) \
.filter(SymbolSource.source_path == None).all()
todelete = set()
total = len(symbolsources)
i = 0
for symbolsource in symbolsources:
i += 1
try:
if not symbolsource.symbol:
logging.info("Empty symbol for symbolsource #{0} @ '{1}'" \
.format(symbolsource.id, symbolsource.path))
continue
except:
continue
logging.info("[{0}/{1}] Processing {2} @ '{3}'" \
.format(i, total, symbolsource.symbol.name, symbolsource.path))
if not symbolsource.frames:
logging.info("No frames found")
continue
if symbolsource.frames[0].backtrace.report.type.lower() == "kerneloops":
try:
version, release, arch, flavour = parse_kernel_build_id(symbolsource.build_id)
except Exception as ex:
logging.error(str(ex))
continue
logging.debug("Version = {0}; Release = {1}; Arch = {2}; Flavour = {3}" \
.format(version, release, arch, flavour))
pkgname = "kernel"
if not flavour is None:
pkgname = "kernel-{0}".format(flavour)
debugpkgname = "{0}-debuginfo".format(pkgname)
debuginfo = db.session.query(Package) \
.join(Build) \
.join(Arch) \
.filter((Package.name == debugpkgname) &
(Build.version == version) &
(Build.release == release) &
(Arch.name == arch)) \
.first()
if not debuginfo:
logging.debug("Matching kernel debuginfo not found")
continue
if not os.path.isfile(debuginfo.get_lob_path("package")):
logging.debug("Package metadata found, but the actual package "
"is not available in storage.")
continue
if not debuginfo in result:
result[debuginfo] = {}
# ugly, but whatever - there is no binary package required
if not debuginfo in result[debuginfo]:
result[debuginfo][debuginfo] = set()
result[debuginfo][debuginfo].add(symbolsource)
continue
if symbolsource.frames[0].backtrace.report.type.lower() != "userspace":
logging.info("Skipping non-userspace symbol")
continue
if symbolsource.build_id is None:
logging.info("No build-id available")
continue
debug_file = get_debug_file(symbolsource.build_id)
debuginfos = db.session.query(Package) \
.join(PackageDependency) \
.filter((PackageDependency.name == debug_file) &
(PackageDependency.type == "PROVIDES")) \
.all()
logging.debug("Found {0} debuginfo packages".format(len(debuginfos)))
for debuginfo in debuginfos:
package = db.session.query(Package) \
.join(PackageDependency) \
.filter((PackageDependency.name == symbolsource.path) &
(Package.arch == debuginfo.arch) &
(Package.build_id == debuginfo.build_id)) \
.first()
if not package:
logging.debug("Trying AbsPath fix")
abspath = os.path.abspath(symbolsource.path)
if symbolsource.path != abspath:
logging.info("Applying AbsPath fix")
conflict = db.session.query(SymbolSource) \
.filter((SymbolSource.path == abspath) &
(SymbolSource.offset == symbolsource.offset) &
(SymbolSource.build_id == symbolsource.build_id)) \
.first()
if conflict:
db.session.execute("UPDATE {0} SET symbolsource_id = :newid " \
"WHERE symbolsource_id = :oldid" \
.format(ReportBtFrame.__tablename__),
{"oldid": symbolsource.id, "newid": conflict.id })
todelete.add(symbolsource.id)
db.session.expunge(symbolsource)
symbolsource = conflict
else:
symbolsource.path = abspath
package = db.session.query(Package) \
.join(PackageDependency) \
.filter((PackageDependency.name == symbolsource.path) &
(Package.arch == debuginfo.arch) &
(Package.build_id == debuginfo.build_id)) \
.first()
if not package:
logging.debug("Trying UsrMove fix")
if symbolsource.path.startswith("/usr"):
newpath = symbolsource.path[4:]
else:
newpath = "/usr{0}".format(symbolsource.path)
package = db.session.query(Package) \
.join(PackageDependency) \
.filter((PackageDependency.name == newpath) &
(Package.arch == debuginfo.arch) &
(Package.build_id == debuginfo.build_id)) \
.first()
if package:
logging.info("Applying UsrMove fix")
conflict = db.session.query(SymbolSource) \
.filter((SymbolSource.path == newpath) &
(SymbolSource.offset == symbolsource.offset) &
(SymbolSource.build_id == symbolsource.build_id)) \
.first()
if conflict:
db.session.execute("UPDATE {0} SET symbolsource_id = :newid " \
"WHERE symbolsource_id = :oldid" \
.format(ReportBtFrame.__tablename__),
{"oldid": symbolsource.id, "newid": conflict.id })
todelete.add(symbolsource.id)
db.session.expunge(symbolsource)
symbolsource = conflict
else:
symbolsource.path = newpath
db.session.flush()
if not package:
logging.debug("Matching binary package not found")
continue
if not os.path.isfile(package.get_lob_path("package")) or \
not os.path.isfile(debuginfo.get_lob_path("package")):
logging.debug("Package metadata found, but the actual package "
"is not available in storage.")
continue
if not debuginfo in result:
result[debuginfo] = {}
if not package in result[debuginfo]:
result[debuginfo][package] = set()
result[debuginfo][package].add(symbolsource)
break
else:
logging.warn("Unable to find a suitable package combination")
db.session.flush()
if todelete:
# cond1 | cond2 | cond3 | ... | cond_n-1 | cond_n
# delete by slices of 100 members - reduce is recursive
todelete = list(todelete)
for i in xrange((len(todelete) + 99) / 100):
part = todelete[(100 * i):(100 * (i + 1))]
cond = reduce(lambda x, y: x | y, [SymbolSource.id == id for id in part])
try:
db.session.query(SymbolSource).filter(cond).delete()
except Exception as ex:
logging.error("Unable to delete: {0}".format(str(ex)))
continue
return result