def store_rpm_provides(db: Database, package: Package, nogpgcheck: bool = False) -> None:
"""
Save RPM provides of `package` to storage.
Expects pyfaf.storage.opsys.Package object.
"""
pkg_id = package.id
ts = rpm.ts()
rpm_file = package.get_lob_fd("package")
if not rpm_file:
raise FafError("Package {0} has no lob stored".format(package.name))
if nogpgcheck:
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) #pylint: disable=protected-access
try:
header = ts.hdrFromFdno(rpm_file.fileno())
except rpm.error as exc:
rpm_file.close()
raise FafError("rpm error: {0}".format(exc)) from exc
files = header.fiFromHeader()
log.debug("%s contains %d files", package.nvra(), len(files))
# Invalid name for type variable
# pylint: disable-msg=C0103
for f in files:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = f[0]
new.flags = 0
db.session.add(new)
provides = header.dsFromHeader('providename')
for p in provides:
if len(p.N()) > 1024:
log.warning("Provides item in RPM header of %s longer than 1024 "
"characters. Skipping", package.name)
continue
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = p.N()
new.flags = p.Flags()
evr = p.EVR()
if evr:
try:
new.epoch, new.version, new.release = parse_evr(evr)
except ValueError as ex:
log.warning("Unparsable EVR ‘%s’ of %s in Provides of %s: %s. "
"Skipping",
evr, p.N(), package.name, ex)
continue
db.session.add(new)
rpm_file.close()
db.session.flush()
def _get_pkgname_parsers(self, db, db_opsys=None):
"""
Query pyfaf.storage.SfPrefilterPackageName objects and turn them into
python regexp parsers. Return a dictionary {parser: solution}.
"""
result = {}
db_pkgnames = get_sf_prefilter_pkgnames(db, db_opsys=db_opsys)
for db_pkgname in db_pkgnames:
try:
parser = re.compile(db_pkgname.pattern)
except re.error as ex:
log.warning("Unable to compile pattern '%s': %s",
db_pkgname.pattern, str(ex))
continue
result[parser] = db_pkgname.solution
return result
def _get_pkgname_parsers(self, db, db_opsys=None):
"""
Query pyfaf.storage.SfPrefilterPackageName objects and turn them into
python regexp parsers. Return a dictionary {parser: solution}.
"""
result = {}
db_pkgnames = get_sf_prefilter_pkgnames(db, db_opsys=db_opsys)
for db_pkgname in db_pkgnames:
try:
parser = re.compile(db_pkgname.pattern)
except re.error as ex:
log.warning("Unable to compile pattern '%s': %s",
db_pkgname.pattern, str(ex))
continue
result[parser] = db_pkgname.solution
return result
def _get_btpath_parsers(self,
db,
db_opsys=None) -> Dict[re.Pattern, Solution]:
"""
Query pyfaf.storage.SfPrefilterBacktracePath objects and turn them into
python regexp parsers. Return a dictionary {parser: solution}.
"""
result = {}
db_btpaths = get_sf_prefilter_btpaths(db, db_opsys=db_opsys)
for db_btpath in db_btpaths:
try:
parser = re.compile(db_btpath.pattern)
except re.error as ex:
log.warning("Unable to compile pattern '%s': %s",
db_btpath.pattern, str(ex))
continue
result[parser] = db_btpath.solution
return result
def find_solution_ureport(self,
db,
ureport,
osr=None) -> Optional[SfPrefilterSolution]:
"""
Check whether uReport matches a knowledgebase
entry. Return a pyfaf.storage.SfPrefilterSolution object or None.
"""
if "ureport_version" in ureport and ureport["ureport_version"] == 1:
ureport = ureport1to2(ureport)
validate(ureport)
db_opsys = None
if osr is not None:
db_opsys = osr.opsys
osname = ureport["os"]["name"]
if osname not in systems:
log.warning("Operating system '%s' is not supported", osname)
else:
osplugin = systems[osname]
db_opsys = get_opsys_by_name(db, osplugin.nice_name)
if db_opsys is None:
log.warning(
"Operaring system '%s' is not installed in storage",
osplugin.nice_name)
else:
pkgname_parsers = self._get_pkgname_parsers(db,
db_opsys=db_opsys)
for parser, solution in pkgname_parsers.items():
if osplugin.check_pkgname_match(ureport["packages"],
parser):
return self._sfps_to_solution(solution)
ptype = ureport["problem"]["type"]
if ptype not in problemtypes:
log.warning("Problem type '%s' is not supported", ptype)
else:
problemplugin = problemtypes[ptype]
btpath_parsers = self._get_btpath_parsers(db, db_opsys=db_opsys)
for parser, solution in btpath_parsers.items():
if problemplugin.check_btpath_match(ureport["problem"],
parser):
return self._sfps_to_solution(solution)
return None
def find_solution_ureport(self, db, ureport, osr=None):
"""
Check whether uReport matches a knowledgebase
entry. Return a pyfaf.storage.SfPrefilterSolution object or None.
"""
if "ureport_version" in ureport and ureport["ureport_version"] == 1:
ureport = ureport1to2(ureport)
validate(ureport)
db_opsys = None
if osr is not None:
db_opsys = osr.opsys
osname = ureport["os"]["name"]
if osname not in systems:
log.warning("Operating system '%s' is not supported", osname)
else:
osplugin = systems[osname]
db_opsys = get_opsys_by_name(db, osplugin.nice_name)
if db_opsys is None:
log.warning("Operaring system '%s' is not installed in storage",
osplugin.nice_name)
else:
pkgname_parsers = self._get_pkgname_parsers(db, db_opsys=db_opsys)
for parser, solution in pkgname_parsers.items():
if osplugin.check_pkgname_match(ureport["packages"], parser):
return self._sfps_to_solution(solution)
ptype = ureport["problem"]["type"]
if ptype not in problemtypes:
log.warning("Problem type '%s' is not supported", ptype)
else:
problemplugin = problemtypes[ptype]
btpath_parsers = self._get_btpath_parsers(db, db_opsys=db_opsys)
for parser, solution in btpath_parsers.items():
if problemplugin.check_btpath_match(ureport["problem"], parser):
return self._sfps_to_solution(solution)
return None
def save_attachment(db, attachment):
atype = attachment["type"].lower()
if not attachment_type_allowed(atype):
raise FafError(
"Attachment type '{}' not allowed on this server".format(atype))
report = get_report(db, attachment["bthash"])
if not report:
raise FafError("Report for given bthash not found")
if atype in ["rhbz", "fedora-bugzilla", "rhel-bugzilla"]:
bug_id = int(attachment["data"])
reportbug = (db.session.query(
ReportBz).filter((ReportBz.report_id == report.id)
& (ReportBz.bzbug_id == bug_id)).first())
if reportbug:
log.debug("Skipping existing attachment")
return
bug = get_bz_bug(db, bug_id)
if not bug:
if atype in bugtrackers:
# download from bugtracker identified by atype
tracker = bugtrackers[atype]
if not tracker.installed(db):
raise FafError("Bugtracker used in this attachment"
" is not installed")
bug = tracker.download_bug_to_storage(db, bug_id)
elif atype == "rhbz":
# legacy value
# - we need to guess the bugtracker:
# either fedora-bugzilla or rhel-bugzilla,
# former is more probable
for possible_tracker in ["fedora-bugzilla", "rhel-bugzilla"]:
if possible_tracker not in bugtrackers:
continue
tracker = bugtrackers[possible_tracker]
if not tracker.installed(db):
continue
bug = tracker.download_bug_to_storage(db, bug_id)
if bug:
break
if bug:
new = ReportBz()
new.report = report
new.bzbug = bug
db.session.add(new)
db.session.flush()
else:
log.error("Failed to fetch bug #{0} from '{1}'".format(
bug_id, atype))
elif atype == "centos-mantisbt":
bug_id = int(attachment["data"])
reportbug = (db.session.query(ReportMantis).filter(
(ReportMantis.report_id == report.id)
& (ReportMantis.mantisbug_id == bug_id)).first())
if reportbug:
log.debug("Skipping existing attachment")
return
bug = get_mantis_bug(db, bug_id)
if not bug:
if atype in bugtrackers:
# download from bugtracker identified by atype
tracker = bugtrackers[atype]
if not tracker.installed(db):
raise FafError("Bugtracker used in this attachment"
" is not installed")
bug = tracker.download_bug_to_storage(db, bug_id)
if bug:
new = ReportMantis()
new.report = report
new.mantisbug = bug
db.session.add(new)
db.session.flush()
else:
log.error("Failed to fetch bug #{0} from '{1}'".format(
bug_id, atype))
elif atype == "comment":
comment = ReportComment()
comment.report = report
comment.text = attachment["data"]
comment.saved = datetime.datetime.utcnow()
db.session.add(comment)
db.session.flush()
elif atype == "email":
db_contact_email = get_contact_email(db, attachment["data"])
if db_contact_email is None:
db_contact_email = ContactEmail()
db_contact_email.email_address = attachment["data"]
db.session.add(db_contact_email)
db_report_contact_email = ReportContactEmail()
db_report_contact_email.contact_email = db_contact_email
db_report_contact_email.report = report
db.session.add(db_report_contact_email)
else:
db_report_contact_email = \
get_report_contact_email(db, db_contact_email.id, report.id)
if db_report_contact_email is None:
db_report_contact_email = ReportContactEmail()
db_report_contact_email.contact_email = db_contact_email
db_report_contact_email.report = report
db.session.add(db_report_contact_email)
try:
db.session.flush()
except IntegrityError:
raise FafError("Email address already assigned to the report")
elif atype == "url":
url = attachment["data"]
# 0ne URL can be attached to many Reports, but every reports must
# have unique url's
db_url = (db.session.query(ReportURL).filter(
ReportURL.url == url).filter(
ReportURL.report_id == report.id).first())
if db_url:
log.debug("Skipping existing URL")
return
db_url = ReportURL()
db_url.report = report
db_url.url = url
db_url.saved = datetime.datetime.utcnow()
try:
db.session.flush()
except IntegrityError:
raise FafError("Unable to save URL")
else:
log.warning("Unknown attachment type")
def save_attachment(db, attachment):
atype = attachment["type"].lower()
if not attachment_type_allowed(atype):
raise FafError("Attachment type '{}' not allowed on this server"
.format(atype))
report = get_report(db, attachment["bthash"])
if not report:
raise FafError("Report for given bthash not found")
if atype in ["rhbz", "fedora-bugzilla", "rhel-bugzilla"]:
bug_id = int(attachment["data"])
reportbug = (db.session.query(ReportBz)
.filter(
(ReportBz.report_id == report.id) &
(ReportBz.bzbug_id == bug_id)
)
.first())
if reportbug:
log.debug("Skipping existing attachment")
return
bug = get_bz_bug(db, bug_id)
if not bug:
if atype in bugtrackers:
# download from bugtracker identified by atype
tracker = bugtrackers[atype]
if not tracker.installed(db):
raise FafError("Bugtracker used in this attachment"
" is not installed")
bug = tracker.download_bug_to_storage(db, bug_id)
elif atype == "rhbz":
# legacy value
# - we need to guess the bugtracker:
# either fedora-bugzilla or rhel-bugzilla,
# former is more probable
for possible_tracker in ["fedora-bugzilla", "rhel-bugzilla"]:
if possible_tracker not in bugtrackers:
continue
tracker = bugtrackers[possible_tracker]
if not tracker.installed(db):
continue
bug = tracker.download_bug_to_storage(db, bug_id)
if bug:
break
if bug:
new = ReportBz()
new.report = report
new.bzbug = bug
db.session.add(new)
db.session.flush()
else:
log.error("Failed to fetch bug #{0} from '{1}'"
.format(bug_id, atype))
elif atype == "centos-mantisbt":
bug_id = int(attachment["data"])
reportbug = (db.session.query(ReportMantis)
.filter(
(ReportMantis.report_id == report.id) &
(ReportMantis.mantisbug_id == bug_id))
.first())
if reportbug:
log.debug("Skipping existing attachment")
return
bug = get_mantis_bug(db, bug_id)
if not bug:
if atype in bugtrackers:
# download from bugtracker identified by atype
tracker = bugtrackers[atype]
if not tracker.installed(db):
raise FafError("Bugtracker used in this attachment"
" is not installed")
bug = tracker.download_bug_to_storage(db, bug_id)
if bug:
new = ReportMantis()
new.report = report
new.mantisbug = bug
db.session.add(new)
db.session.flush()
else:
log.error("Failed to fetch bug #{0} from '{1}'"
.format(bug_id, atype))
elif atype == "comment":
comment = ReportComment()
comment.report = report
comment.text = attachment["data"]
comment.saved = datetime.datetime.utcnow()
db.session.add(comment)
db.session.flush()
elif atype == "email":
db_contact_email = get_contact_email(db, attachment["data"])
if db_contact_email is None:
db_contact_email = ContactEmail()
db_contact_email.email_address = attachment["data"]
db.session.add(db_contact_email)
db_report_contact_email = ReportContactEmail()
db_report_contact_email.contact_email = db_contact_email
db_report_contact_email.report = report
db.session.add(db_report_contact_email)
else:
db_report_contact_email = \
get_report_contact_email(db, db_contact_email.id, report.id)
if db_report_contact_email is None:
db_report_contact_email = ReportContactEmail()
db_report_contact_email.contact_email = db_contact_email
db_report_contact_email.report = report
db.session.add(db_report_contact_email)
try:
db.session.flush()
except IntegrityError:
raise FafError("Email address already assigned to the report")
elif atype == "url":
url = attachment["data"]
# 0ne URL can be attached to many Reports, but every reports must
# have unique url's
db_url = (db.session.query(ReportURL)
.filter(ReportURL.url == url)
.filter(ReportURL.report_id == report.id)
.first())
if db_url:
log.debug("Skipping existing URL")
return
db_url = ReportURL()
db_url.report = report
db_url.url = url
db_url.saved = datetime.datetime.utcnow()
try:
db.session.flush()
except IntegrityError:
raise FafError("Unable to save URL")
else:
log.warning("Unknown attachment type")
def _flush_session(self, *args, **kwargs) -> None:
if self._dry:
log.warning("Dry run enabled, not flushing the database")
else:
self.session._flush_orig(*args, **kwargs) #pylint: disable=protected-access
def store_rpm_deps(db, package, nogpgcheck=False):
"""
Save RPM dependencies of `package` to
storage.
Expects pyfaf.storage.opsys.Package object.
"""
pkg_id = package.id
ts = rpm.ts()
rpm_file = package.get_lob_fd("package")
if not rpm_file:
log.warning("Package {0} has no lob stored".format(package.name))
return False
if nogpgcheck:
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) #pylint: disable=protected-access
try:
header = ts.hdrFromFdno(rpm_file.fileno())
except rpm.error as exc:
log.error("rpm error: {0}".format(exc))
return False
files = header.fiFromHeader()
log.debug("%s contains %d files", package.nvra(), len(files))
# Invalid name for type variable
# pylint: disable-msg=C0103
for f in files:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = f[0]
new.flags = 0
db.session.add(new)
provides = header.dsFromHeader('providename')
for p in provides:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = p.N()
new.flags = p.Flags()
evr = p.EVR()
if evr:
new.epoch, new.version, new.release = parse_evr(evr)
db.session.add(new)
requires = header.dsFromHeader('requirename')
for r in requires:
new = PackageDependency()
new.package_id = pkg_id
new.type = "REQUIRES"
new.name = r.N()
new.flags = r.Flags()
evr = r.EVR()
if evr:
new.epoch, new.version, new.release = parse_evr(evr)
db.session.add(new)
conflicts = header.dsFromHeader('conflictname')
for c in conflicts:
new = PackageDependency()
new.package_id = pkg_id
new.type = "CONFLICTS"
new.name = c.N()
new.flags = c.Flags()
evr = c.EVR()
if evr:
new.epoch, new.version, new.release = parse_evr(evr)
db.session.add(new)
# pylint: enable-msg=C0103
rpm_file.close()
db.session.flush()
return True
def store_rpm_deps(db, package, nogpgcheck=False):
"""
Save RPM dependencies of `package` to
storage.
Expects pyfaf.storage.opsys.Package object.
"""
pkg_id = package.id
ts = rpm.ts()
rpm_file = package.get_lob_fd("package")
if not rpm_file:
log.warning("Package {0} has no lob stored".format(package.name))
return False
if nogpgcheck:
ts.setVSFlags(rpm._RPMVSF_NOSIGNATURES) #pylint: disable=protected-access
try:
header = ts.hdrFromFdno(rpm_file.fileno())
except rpm.error as exc:
log.error("rpm error: {0}".format(exc))
return False
files = header.fiFromHeader()
log.debug("{0} contains {1} files".format(package.nvra(),
len(files)))
# Invalid name for type variable
# pylint: disable-msg=C0103
for f in files:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = f[0]
new.flags = 0
db.session.add(new)
provides = header.dsFromHeader('providename')
for p in provides:
new = PackageDependency()
new.package_id = pkg_id
new.type = "PROVIDES"
new.name = p.N()
new.flags = p.Flags()
evr = p.EVR()
if evr:
new.epoch, new.version, new.release = parse_evr(evr)
db.session.add(new)
requires = header.dsFromHeader('requirename')
for r in requires:
new = PackageDependency()
new.package_id = pkg_id
new.type = "REQUIRES"
new.name = r.N()
new.flags = r.Flags()
evr = r.EVR()
if evr:
new.epoch, new.version, new.release = parse_evr(evr)
db.session.add(new)
conflicts = header.dsFromHeader('conflictname')
for c in conflicts:
new = PackageDependency()
new.package_id = pkg_id
new.type = "CONFLICTS"
new.name = c.N()
new.flags = c.Flags()
evr = c.EVR()
if evr:
new.epoch, new.version, new.release = parse_evr(evr)
db.session.add(new)
# pylint: enable-msg=C0103
rpm_file.close()
db.session.flush()
return True
def _flush_session(self, *args, **kwargs):
if self._dry:
log.warning("Dry run enabled, not flushing the database")
else:
self.session._flush_orig(*args, **kwargs) #pylint: disable=protected-access
def _flush_session(self, *args, **kwargs):
if self._dry:
log.warning("Dry run enabled, not flushing the database")
else:
self.session._flush_orig(*args, **kwargs)