def main(root_or_arc_path, hid=None, verbosity=0, **kwargs):
"""
:param root_or_arc_path:
Path to the root dir of RPM DB files or Archive (tar.xz, tar.gz, zip,
etc.) of RPM DB files. Path might be a relative path from current dir.
:param hid:
Some identification info of the target host where original RPM DB data
was collected.
:param verbosity: Verbosity level: 0 (default), 1 (verbose), 2 (debug)
:param kwargs:
Extra keyword arguments other than `root_or_arc_path` passed to make an
instance of :class:`fleure.config.Host`
:return: Workdir where results exist or path to archive of results
"""
set_loglevel(verbosity)
host = configure(root_or_arc_path, hid, **kwargs)
if host is None:
LOG.error(_("Failed to configure the host: root=%s"),
root_or_arc_path)
return None
prepare(host)
if host.available:
LOG.info(_("Anaylize the host: %s"), host.hid)
analyze(host)
if kwargs.get("archive", False):
outname = "report-%s-%s.zip" % (host.hid, fleure.globals.TODAY)
return archive_report(host.workdir, outname)
else:
return host.workdir
def prepare(host):
"""
:param host: Configured host, an instance of :class:`fleure.config.Host`
:return: An instance of :class:`fleure.config.Host`
"""
if not host.has_valid_root():
LOG.error(_("Root dir is not ready. Error was: %s"), host.error)
return
LOG.info(_("%s: Start to initialize: root=%s, backend=%s"),
host.hid, host.root, host.backend)
base = host.init_base()
base.prepare()
LOG.info(_("%s[%s]: Initialization completed, start to analyze ..."),
host.hid, base.name)
host.installed = sorted(base.list_installed(),
key=itemgetter(*host.rpmkeys))
LOG.info(_("%s: Found %d (rebuilt=%d, replaced=%d) installed RPMs"),
host.hid, len(host.installed),
len([p for p in host.installed if p.get("rebuilt", False)]),
len([p for p in host.installed if p.get("replaced", False)]))
host.save(host.installed, "packages")
if base.ready():
host.available = True
def check_rpmdb_root(root, readonly=True, system=False,
dbnames=fleure.globals.RPMDB_FILENAMES):
"""
:param root: The pivot root directry where target's RPM DB files exist
:param readonly: Ensure RPM DB files readonly
:param system: Allow accessing system RPM DB in /var/lib/rpm
:return: True if necessary setup was done w/ success else False
"""
if system:
readonly = True
else:
assert root != "/", "Do not run this for host system's RPM DB!"
rpmdbdir = os.path.join(root, fleure.globals.RPMDB_SUBDIR)
if not os.path.exists(rpmdbdir):
LOG.error(_("RPM DB dir %s does not exist!"), rpmdbdir)
return False
pkgdb = os.path.join(rpmdbdir, "Packages")
if not _is_bsd_hashdb(pkgdb):
LOG.error(_("%s does not look a RPM DB (Packages) file!"), pkgdb)
return False
for dbn in dbnames:
dbpath = os.path.join(rpmdbdir, dbn)
if not os.path.exists(dbpath):
# NOTE: It's not an error at once.
LOG.info(_("RPM DB %s looks missing"), dbn)
if readonly and os.access(dbpath, os.W_OK) and not system:
os.chmod(dbpath, 0o444)
return True
def analyze(host):
"""
:param host: host object function :function:`prepare` returns
"""
metadata = dict(id=host.hid, root=host.root, workdir=host.workdir,
repos=host.repos, backend=host.base.name,
score=host.cvss_min_score, keywords=host.errata_keywords,
pkeywords=host.errata_pkeywords,
installed=len(host.installed), hosts=[host.hid, ],
generated=datetime.datetime.now().strftime("%F %T"),
period=host.period, refdir=host.refdir)
host.save(metadata, "metadata")
LOG.info(_("%s: Analyzing errata and packages ..."), host.hid)
host.updates = ups = host.base.list_updates()
p2na = itemgetter("name", "arch")
calls = (functools.partial(fleure.datasets.complement_an_errata,
updates=set(p2na(u) for u in ups),
to_update_fn=p2na,
score=host.cvss_min_score))
host.errata = ers = host.base.list_errata(calls)
host.save(ers, "errata")
host.save(ups, "updates")
LOG.info(_("%s: Found %d errata and %d updates, saved the lists"),
host.hid, len(ers), len(ups))
ips = host.installed
analyze_and_dump_results(host, ips, ers, ups)
LOG.info(_("%s: Saved analysis results in %s"), host.workdir)
if host.period:
(start, end) = host.period
LOG.info(_("%s: Analyzing errata and packages [%s ~ %s]"),
host.hid, start, end)
pdir = os.path.join(host.workdir, "%s_%s" % (start, end))
if not os.path.exists(pdir):
LOG.debug(_("%s: Creating period working dir %s"), host.hid, pdir)
os.makedirs(pdir)
pes = [e for e in ers
if fleure.dates.in_period(e["issue_date"], start, end)]
analyze_and_dump_results(host, ips, pes, ups, pdir)
LOG.info(_("%s [%s ~ %s]: Found %d errata and saved"),
host.hid, start, end, len(pes))
if host.refdir:
LOG.debug(_("%s [delta]: Analyze delta errata data by refering %s"),
host.hid, host.refdir)
(ers, ups) = fleure.datasets.compute_delta(host.refdir, ers, ups)
host.save(ers, "errata", subdir="delta")
host.save(ups, "updates", subdir="delta")
LOG.info(_("%s [delta]: Found %d errata and %d updates, save the "
"lists"), host.hid, len(ers), len(ups))
LOG.info(_("%s: Analyzing delta errata and packages ..."), host.hid)
analyze_and_dump_results(host, ips, ers, ups)
LOG.info(_("%s: Saved delta analysis results in %s"), host.workdir)
def cvss_metrics(cvss, metrics_map=None):
"""
TODO: Some of CVEs in Red Hat CVE database look having wrong CVSS
metrics data.
:param cvss: A string represents CVSS metrics,
ex. "AV:N/AC:H/Au:N/C:N/I:P/A:N"
:param metrics_map: CVSS metrics mappings :: dict
>>> ms_ref = [
... ("Access Vector", 3), ("Access Complexity", 1),
... ("Authentication", 3), ("Confidentiality Impact", 1),
... ("Integrity Impact", 2), ("Availability Impact", 1),
... ]
>>> ms0 = cvss_metrics("AV:N/AC:H/Au:N/C:N/I:P/A:N")
>>> assert ms0 == ms_ref, str(ms0)
>>> ms1 = cvss_metrics("AV:N/AC:H/AU:N/C:N/I:P/A:N") # CVE-2012-3406
>>> assert ms1 == ms_ref, str(ms1)
>>> ms2 = cvss_metrics("AV:N/AC:H/Au/N/C:N/I:P/A:N") # CVE-2012-5077
>>> assert ms2 == ms_ref, str(ms2)
>>> ms3 = cvss_metrics("AV:N/AC:N/Au/N/C:P/I:N/A:N") # CVE-2012-3375
>>> assert ms3 != ms_ref, str(ms3)
"""
if metrics_map is None:
metrics_map = CVSSS_METRICS_MAP
metrics = []
if "/AU:" in cvss:
cvss = cvss.replace("/AU:", "/Au:")
if "/Au/" in cvss:
cvss = cvss.replace("/Au/", "/Au:")
for lms in cvss.split("/"):
(key, val) = lms.split(":")
metric = metrics_map.get(key, False)
if not metric:
LOG.error(_("Unknown CVSS metric abbrev: %s"), key)
return metrics
label = metric["label"]
val = metric["metrics"].get(val, False)
if not val:
LOG.error(_("Uknown value for CVSS metric '%s': %s"), metric, val)
return metrics
metrics.append((label, val))
return metrics
def get_cvss_for_cve(cve):
"""
Get CVSS data for given cve from the Red Hat www site.
:param cve: CVE name, e.g. "CVE-2010-1585" :: str
:return: {"metrics": base_metric :: str, "score": base_score :: str}
See the HTML source of CVE www page for its format, e.g.
https://www.redhat.com/security/data/cve/CVE-2010-1585.html.
"""
match = re.match(r"CVE-(?P<year>\d{4})-(?P<id>\d{4})", cve)
if match:
year = int(match.groupdict()["year"])
if year < 2009: # No CVSS
return None
else:
LOG.warn(_("Invalid CVE: %s"), cve)
return None
def has_cvss_link(tag):
"""Does CVE has a link to CVSS base metrics?
"""
return tag.get("href", "").startswith("http://nvd.nist.gov/cvss.cfm")
def is_base_score(tag):
"""Is it CVSS base score?
"""
return tag.string == "Base Score:"
url_fmt = "http://nvd.nist.gov/cvss.cfm?version=2&name=%s&vector=(%s)"
try:
data = urlread(cve2url(cve))
soup = beautifulsoup.BeautifulSoup(data)
cvss_base_metrics = soup.findAll(has_cvss_link)[0].string
cvss_base_score = soup.findAll(is_base_score)[0].parent.td.string
# may fail to parse `cvss_base_metrics`
cvss_base_metrics_vec = cvss_metrics(cvss_base_metrics)
return dict(cve=cve,
metrics=cvss_base_metrics,
metrics_v=cvss_base_metrics_vec,
score=cvss_base_score,
url=url_fmt % (cve, cvss_base_metrics))
except Exception as exc:
LOG.warn(_("Could not get CVSS data: err=%s"), str(exc))
return None
def _errata_to_int(errata):
"""
Generate an int represents an errata to used as comparison key.
- RHSA > RHBA > RHEA (type)
- RHSA: Critical > Important > Moderate > Low (severity)
- RHBA-2013:0212 > RHBA-2012:1422 (year)
- RHBA-2013:0212 > RHBA-2013:0210 (sequential id)
- RHBA-2013:0212-1 > RHBA-2013:0212 (revision)
:param errata: A dict represents an errata
>>> _errata_to_int(dict(advisory="RHBA-2012:1422-1", ))
202012142201
>>> _errata_to_int(dict(advisory="RHSA-2014:0422", severity="Moderate"))
342014042200
"""
echars = dict(E=1, B=2, S=3)
sevs = collections.defaultdict(int, dict(Low=2, Moderate=4, Important=6,
Critical=8))
reg = re.compile(r"^RH(?P<echar>(E|B|S))A-(?P<year>\d{4}):"
r"(?P<seq>\d{4,5})(?:-(?P<rev>\d+))?$")
match = reg.match(errata["advisory"])
if not match:
LOG.warn(_("Not an errata advisory ? : %(advisory)s"), errata)
return 0
dic = match.groupdict()
rev = 0 if dic["rev"] is None else int(dic["rev"])
return int("%d%d%s%s%02d" % (echars[dic["echar"]],
sevs[errata.get("severity", 0)],
dic["year"], dic["seq"], rev))
def extract_rpmdb_archive(arc_path, root=None):
"""Try to extract given RPM DB files archive `arc_path`.
:param arc_path: Archive file path
:param root:
Path to dir to extract RPM DB files. These files will be put into
`root`/var/lib/rpm.
:return:
A tuple of (root, err) where root is an absolute path of root of RPM DB
files extracted or None, indicates extraction failed, and err is an
error message tells what's the problem of exraction failure.
"""
if not os.path.exists(arc_path):
return (None, "Not found an archive: " + arc_path)
if not os.path.isfile(arc_path):
return (None, "Not a file: " + arc_path)
if root is None:
root = tempfile.mkdtemp(dir="/tmp", prefix="%s-" % __name__)
LOG.info(_("Created a root dir of RPM DBs: %s"), root)
else:
root = os.path.abspath(root) # Ensure it's an absolute path.
rpmdbdir = os.path.join(root, fleure.globals.RPMDB_SUBDIR)
if not os.path.exists(rpmdbdir):
os.makedirs(rpmdbdir)
prefix = fleure.globals.RPMDB_SUBDIR
files = [os.path.join(prefix, fn) for fn in fleure.globals.RPMDB_FILENAMES]
return (root, _exract_fnc(arc_path)(arc_path, root, files))
def dump_depgraph(root, ers, workdir=None, outname="rpm_depgraph_gv", tpaths=fleure.globals.FLEURE_TEMPLATE_PATHS):
"""
Make up context to generate RPM dependency graph w/ graphviz (sfdp) from
the RPM database files for given host group.
:param root: Host group's root dir where 'var/lib/rpm' exists
:param ers: List of errata dict, see :func:`analyze_errata` in fleure.main
:param workdir: Working dir to dump result
:param outname: Output file base name
:param tpaths: A list of template search paths
"""
if workdir is None:
workdir = root
ctx = _make_depgraph_context(root, ers)
fleure.utils.json_dump(ctx, os.path.join(workdir, outname + ".json"))
output = os.path.join(workdir, outname + ".dot")
opts = dict(at_paths=tpaths, at_engine="jinja2", at_ask_missing=True)
anytemplate.render_to("rpm_depgraph_gv.dot.j2", ctx, output, **opts)
anytemplate.render_to("rpm_depgraph_gv.css.j2", ctx, os.path.join(workdir, outname + ".css"), **opts)
anytemplate.render_to("rpm_depgraph.html.j2", ctx, os.path.join(workdir, "rpm_depgraph.html"), **opts)
output2 = os.path.join(workdir, outname + ".svg")
cmd_s = "sfdp -Tsvg -o%s %s" % (output2, output)
(rcode, out, err) = fleure.utils.subproc_call(cmd_s, timeout=120)
if rcode != 0:
if not err:
err = "Maybe timeout occurs"
LOG.warn(_("Failed to generate a SVG file: in=%s, out=%s, " "out/err=%s/%s"), output, output2, out, err)
def archive_report(reportdir, output):
"""Archive analysis report.
:reportdir: Dir where generated report files exist
:output: Output filename
:return:
Absolute path of archive file made or None might indicates some
failures before/during making archive.
"""
filenames = fleure.globals.REPORT_FILES
if all(os.path.exists(os.path.join(reportdir, fn)) for fn in filenames):
arcpath = fleure.archive.archive_report(reportdir, output)
LOG.info(_("Archived results: %s"), arcpath)
return arcpath
LOG.warn(_("Reprot files (%s) do not exist. Do no make a report "
"archives"), ", ".join(filenames))
return None
def main(hosts_datadir, workdir=None, verbosity=0, multiproc=False, **kwargs):
"""
:param hosts_datadir:
Path to dir in which rpm db roots or its archive of hosts exist
:param workdir: Working dir to save results
:param verbosity: Verbosity level: 0 (default), 1 (verbose), 2 (debug)
:param multiproc: Utilize multiprocessing module to compute results
in parallel as much as possible if True
"""
fleure.main.set_loglevel(verbosity)
all_hosts = list(configure(hosts_datadir, workdir=workdir, **kwargs))
hosts = prepare(all_hosts)
LOG.info(_("Analyze %d/%d hosts"), len(hosts), len(all_hosts))
ilen = lambda h: len(h.installed)
hps = lambda h: [p2nevra(p) for p in h.installed]
gby = lambda xs, kf: itertools.groupby(sorted(xs, key=kf), kf)
# Group hosts by installed rpms to degenerate these hosts and avoid to
# analyze for same installed RPMs more than once. his :: [[[h]]]
his = [[list(t1[1]) for t1 in gby(t0[1], hps)] for t0 in gby(hosts, ilen)]
for hss in his:
hset = [(hs[0], hs[1:]) for hs in hss]
hsdata = [hs[0] for hs in hset]
if multiproc:
pool = multiprocessing.Pool(multiprocessing.cpu_count())
pool.map(fleure.main.analyze, hsdata)
else:
for host in hsdata:
fleure.main.analyze(host)
for hid, hsrest in hset:
if hsrest:
LOG.info(_("Skip to analyze %s as its installed RPMs are "
"exactly same as %s's"),
','.join(x.hid for x in hsrest), hid)
mk_symlinks_to_ref(hid, hsrest)
def configure(hosts_datadir, workdir=None, **kwargs):
"""
Scan and collect hosts' basic data (installed rpms list, etc.).
:param hosts_datadir: Dir in which rpm db roots of hosts exist
:param workdir: Working dir to save results
:return: A generator to yield a tuple,
(host_identity, host_rpmroot or None)
"""
if workdir is None:
LOG.info(_("Set workdir to hosts_datadir: %s"), hosts_datadir)
workdir = hosts_datadir
else:
if not os.path.exists(workdir):
LOG.debug(_("Creating working dir: %s"), workdir)
os.makedirs(workdir)
hpaths = sorted(glob.glob(os.path.join(hosts_datadir, '*')))
hids = _hids_from_apaths(hpaths)
cachedir = os.path.join(workdir, "_cache") # Use common cache dir.
for hid, hpath in itertools.izip(hids, hpaths):
hworkdir = os.path.join(workdir, hid)
if not os.path.exists(hworkdir):
os.makedirs(hworkdir)
if os.path.isdir(hpath):
hroot = hpath
else:
hroot = hworkdir
fleure.archive.extract_rpmdb_archive(hpath, hworkdir)
kwargs["hid"] = hid
kwargs["workdir"] = hworkdir
kwargs["cachedir"] = cachedir
yield fleure.main.configure(hroot, **kwargs)
def mk_symlinks_to_ref(href, hsrest):
"""
:param href: Reference host object
:param hsrest: A list of hosts having same installed rpms as `href`
"""
orgdir = os.path.abspath(os.curdir)
for hst in hsrest:
os.chdir(hst.workdir)
href_workdir = os.path.join('..', href.hid) # TODO: Keep consistency.
LOG.info(_("%s: Make symlinks to results in %s/"),
hst.hid, href_workdir)
for src in glob.glob(os.path.join(href_workdir, '*.*')):
dst = os.path.basename(src)
if not os.path.exists(dst):
LOG.debug(_("Make a symlink to %s"), src)
os.symlink(src, dst)
metadatafile = os.path.join(href_workdir, "metadata.json")
shutil.copy2(metadatafile, metadatafile + ".save")
metadata = fleure.utils.json_load(metadatafile)
metadata["hosts"].append(hst.hid)
fleure.utils.json_dump(metadata, metadatafile)
os.chdir(orgdir)
def _cve_socre_ge(cve, score=0, default=False):
"""
:param cve: A dict contains CVE and CVSS info.
:param score: Lowest score to select CVEs (float). It's Set to 4.0 (PCIDSS
limit) by default:
* NVD Vulnerability Severity Ratings: http://nvd.nist.gov/cvss.cfm
* PCIDSS: https://www.pcisecuritystandards.org
:param default: Default value if failed to get CVSS score to compare with
given score
:return: True if given CVE's socre is greater or equal to given score.
"""
if "score" not in cve:
LOG.warn(_("CVE %(cve)s lacks of CVSS base metrics and score"), cve)
return default
try:
return float(cve["score"]) >= float(score)
except (KeyError, ValueError):
LOG.warn(_("Failed to compare CVE's score: %s, score=%.1f"),
str(cve), score)
return default
def populate(self):
"""
Populates the package sack from the repositories.
Network access to yum repos will happen if any non-local repos
activated and it should be going to take some time to finish.
"""
if not self._populated:
LOG.info(_("Loading yum repo metadata from repos: %s"),
','.join(r.id for r in self.base.repos.listEnabled()))
# self.base._getTs()
self.base._getSacks()
self.base._getUpdates()
self._populated = True # TBD
def errata_of_keywords_g(ers, keywords=fleure.globals.ERRATA_KEYWORDS,
pkeywords=None, stemming=True):
"""
:param ers: A list of errata
:param keywords: A tuple of keywords to filter 'important' RHBAs
:param pkeywords: Similar to above but a dict gives the list per RPMs
:param stemming: Strict matching of keywords with using NLTK stemmer
:return:
A generator to yield errata of which description contains any of
given keywords
>>> ert0 = dict(advisory="RHSA-2015:XXX1",
... description="system hangs, or crash...")
>>> ert1 = dict(advisory="RHEA-2015:XXX2",
... description="some enhancement and changes")
>>> ers = list(errata_of_keywords_g([ert0], ("hang", ), stemming=True))
>>> ert0 in ers
True
>>> ers[0]["keywords"] # 'hangs' with stemming matches.
['hang']
>>> ers = list(errata_of_keywords_g([ert0, ert1], ("hang", "crash"),
... stemming=False))
>>> ert0 in ers
True
>>> ers[0]["keywords"] # 'hangs' w/o stemming does not match with 'hang'.
['crash']
>>> ert1 in ers
False
"""
if stemming:
_stem = _STEMMER.stem
if pkeywords is None:
pkeywords = fleure.globals.ERRATA_PKEYWORDS
for ert in ers:
tokens = set(nltk.wordpunct_tokenize(ert["description"]))
if stemming:
tokens = set(_stem(w.lower()) for w in tokens)
kwds = _errata_keywords(ert.get("package_names", []), keywords,
pkeywords)
matched = kwds & tokens
if matched:
LOG.debug(_("%s matched: keywords=%s"), ert["advisory"],
', '.join(matched))
ert["keywords"] = list(matched)
yield ert
def _fmt_bzs(bzs, summary=False):
"""
:param cves: List of CVE dict {cve, score, url, metrics} or str "cve".
:return: List of CVE strings
"""
def fmt(bze):
"""bugzilla entry formatter"""
return ("bz#%(id)s: "
"%(summary)s " if summary and "summary" in bze else ""
"(%(url)s)")
try:
bzs = [fmt(bz) % bz for bz in bzs]
except KeyError:
LOG.warn(_("BZ Key error: %s"), str(bzs))
return bzs
def set_loglevel(verbosity=0, backend=False):
"""
:param verbosity: Verbosity level = 0 | 1 | 2
:param backend: Set backend's log level also if True
"""
if verbosity in (0, 1, 2):
llvl = [logging.WARN, logging.INFO, logging.DEBUG][verbosity]
else:
LOG.warn(_("Wrong verbosity: %d"), verbosity)
llvl = logging.WARN
LOG.setLevel(llvl)
if not backend:
llvl = logging.WARN
for mod in fleure.config.BACKEND_MODULES:
getattr(mod, "LOG").setLevel(llvl)
def _is_bsd_hashdb(dbpath):
"""
TODO: Is this enough to check if given file ``dbpath`` is RPM DB file ?
And also, maybe some db files should be opened w/ bsddb.btopen instead of
bsddb.hashopen.
>>> if os.path.exists("/etc/redhat-release"):
... _is_bsd_hashdb("/var/lib/rpm/Packages")
True
"""
try:
if bsddb is None:
return True # bsddb is not avialable in python3.
bsddb.hashopen(dbpath, 'r')
except (OSError, IOError):
LOG.warn(_("Not a Berkley DB?: %s"), dbpath)
return False
return True
def configure(root_or_arc_path, hid=None, **kwargs):
"""
:param root_or_arc_path:
Path to the root dir of RPM DB files or Archive (tar.xz, tar.gz, zip,
etc.) of RPM DB files. Path might be a relative path from current dir.
:param hid:
Some identification info of the target host where original RPM DB data
was collected.
:param kwargs:
Extra keyword arguments other than `root_or_arc_path` passed to make an
instance of :class:`fleure.config.Host`
:return: An instance of :class:`fleure.config.Host`
"""
host = fleure.config.Host(root_or_arc_path, hid=hid, **kwargs)
host.configure() # Extract archive, setup root and repos, etc.
if not host.has_valid_root():
LOG.error(_("Root dir is not ready. Error was: %s"), host.error)
return None
return host
def hosts_rpmroot_g(hosts_datadir):
"""
List system names from assessment datadir.
This function expects that assessment data (rpm db files) of each hosts are
found under $host_identity/ in `datadir`, that is,
`datadir`/<host_identity>/var/lib/rpm/Packages exists. If rpm db file[s]
are not found for a host, that host will be simply ignored.
<host_identity> may be a hostname, host id, fqdn or something to
identify that host.
:param hosts_datadir: Dir in which rpm db roots of hosts exist
:return: A generator to yield a tuple,
(host_identity, host_rpmroot or None)
"""
for hostdir in glob.glob(os.path.join(hosts_datadir, '*')):
if fleure.rpmutils.check_rpmdb_root(hostdir):
yield (os.path.basename(hostdir), hostdir)
else:
LOG.warn(_("Failed to find RPM DBs under %s"), hostdir)
yield (os.path.basename(hostdir), None)
def safe_unzip(arcfile, destdir, files=None):
"""
Extract zip file safely similar to :func:`safe_untar`.
.. note::
zipfile.extract in python 2.7+ can process untrusted zip files safely:
https://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.extract
:param arcfile: Zip file path
:param destdir: Destination dir to extract files from `tarfile` to
:param files:
A list of files to extract. All files looks safe will be extracted if
it's None.
:return: A list of error messages if something goes wrong or []
"""
if files is None:
files = []
errors = []
with zipfile.ZipFile(arcfile) as zipf:
for filepath in zipf.namelist():
if files and filepath not in files:
LOG.info(_("Skip %s as not in the list"), filepath)
continue
if _is_bad_path(filepath):
errors.append("Skip as bad path: {}".format(filepath))
continue
zipf.extract(filepath, path=destdir)
path = os.path.join(destdir, filepath)
err = _remove_if_bad_file(path, destdir)
if err:
errors.append(err)
return errors
def compute_delta(refdir, ers, updates, nevra_keys=fleure.globals.RPM_KEYS):
"""
:param refdir: Dir has reference data files: packages.json, errata.json
and updates.json
:param ers: A list of errata
:param updates: A list of update packages
"""
_assert_if_not_exist(refdir, "data dir")
ref_es_file = os.path.join(refdir, "errata.json")
ref_us_file = os.path.join(refdir, "updates.json")
_assert_if_not_exist(ref_es_file, "errata file")
_assert_if_not_exist(ref_us_file, "updates file")
ref_es_data = fleure.utils.json_load(ref_es_file)
ref_us_data = fleure.utils.json_load(ref_us_file)
LOG.debug(_("Loaded reference errata and updates file"))
ref_eadvs = set(e["advisory"] for e in ref_es_data["data"])
ref_nevras = set((p[k] for k in nevra_keys) for p in ref_us_data["data"])
return ([e for e in ers if e["advisory"] not in ref_eadvs],
[u for u in updates
if (u[k] for k in nevra_keys) not in ref_nevras])
def analyze_and_dump_results(host, rpms, errata, updates, dumpdir=None):
"""
Analyze and dump package level static analysis results.
:param host: host object function :function:`prepare` returns
:param rpms: A list of installed RPMs
:param errata: A list of applicable errata
:param updates: A list of update RPMs
:param dumpdir: Dir to save results
"""
if dumpdir is None:
dumpdir = host.workdir
dargs = dict(score=host.cvss_min_score, keywords=host.errata_keywords,
pkeywords=host.errata_pkeywords, core_rpms=host.core_rpms)
rpmkeys = host.rpmkeys
installed = dict(list=rpms, list_rebuilt=[], list_replaced=[],
list_from_others=[])
for pkg in rpms:
for key in ("rebuilt", "replaced", "from_others"):
if pkg.get(key, False):
installed["list_" + key].append(pkg)
nps = len(rpms)
nus = len(updates)
ers = fleure.analysis.analyze_errata(errata, **dargs)
data = dict(errata=ers,
installed=installed,
updates=dict(list=updates,
rate=[(_("packages need updates"), nus),
(_("packages not need updates"),
nps - nus)]))
host.save(data, "summary", dumpdir)
fleure.depgraph.dump_depgraph(host.root, ers, host.workdir,
tpaths=host.tpaths)
# TODO: Keep DRY principle.
lrpmkeys = [_("name"), _("epoch"), _("version"), _("release"), _("arch")]
rpmdkeys = list(rpmkeys) + ["summary", "vendor", "buildhost"]
lrpmdkeys = lrpmkeys + [_("summary"), _("vendor"), _("buildhost")]
sekeys = ("advisory", "severity", "synopsis", "url", "update_names")
lsekeys = (_("advisory"), _("severity"), _("synopsis"), _("url"),
_("update_names"))
bekeys = ("advisory", "keywords", "synopsis", "url", "update_names")
lbekeys = (_("advisory"), _("keywords"), _("synopsis"), _("url"),
_("update_names"))
mds = [fleure.analysis.mk_overview_dataset(data, **dargs),
make_dataset((data["errata"]["rhsa"]["list_latest_critical"] +
data["errata"]["rhsa"]["list_latest_important"]),
_("Cri-Important RHSAs (latests)"), sekeys, lsekeys),
make_dataset(sorted(data["errata"]["rhsa"]["list_critical"],
key=itemgetter("update_names")) +
sorted(data["errata"]["rhsa"]["list_important"],
key=itemgetter("update_names")),
_("Critical or Important RHSAs"), sekeys, lsekeys),
make_dataset(data["errata"]["rhba"]["list_by_kwds_of_core_rpms"],
_("RHBAs (core rpms, keywords)"), bekeys, lbekeys),
make_dataset(data["errata"]["rhba"]["list_by_kwds"],
_("RHBAs (keyword)"), bekeys, lbekeys),
make_dataset(data["errata"]["rhba"]["list_latests_of_core_rpms"],
_("RHBAs (core rpms, latests)"), bekeys, lbekeys),
make_dataset(data["errata"]["rhsa"]["list_critical_updates"],
_("Update RPMs by RHSAs (Critical)"), rpmkeys,
lrpmkeys),
make_dataset(data["errata"]["rhsa"]["list_important_updates"],
_("Updates by RHSAs (Important)"), rpmkeys, lrpmkeys),
make_dataset(data["errata"]["rhba"]["list_updates_by_kwds"],
_("Updates by RHBAs (Keyword)"), rpmkeys, lrpmkeys)]
score = host.cvss_min_score
if score > 0:
cvss_ds = [
make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"],
_("RHSAs (CVSS score >= %.1f)") % score,
("advisory", "severity", "synopsis",
"cves", "cvsses_s", "url"),
(_("advisory"), _("severity"), _("synopsis"),
_("cves"), _("cvsses_s"), _("url"))),
make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"],
_("RHBAs (CVSS score >= %.1f)") % score,
("advisory", "synopsis", "cves", "cvsses_s", "url"),
(_("advisory"), _("synopsis"), _("cves"),
_("cvsses_s"), _("url")))]
mds.extend(cvss_ds)
for key, title in (("list_rebuilt", _("Rebuilt RPMs")),
("list_replaced", _("Replaced RPMs")),
("list_from_others", _("RPMs from other vendors"))):
if data["installed"][key]:
mds.append(make_dataset(data["installed"][key], title, rpmdkeys,
lrpmdkeys))
dump_xls(mds, os.path.join(dumpdir, "errata_summary.xls"))
if host.details:
dds = [make_dataset(errata, _("Errata Details"),
("advisory", "type", "severity", "synopsis",
"description", "issue_date", "update_date", "url",
"cves", "bzs", "update_names"),
(_("advisory"), _("type"), _("severity"),
_("synopsis"), _("description"), _("issue_date"),
_("update_date"), _("url"), _("cves"),
_("bzs"), _("update_names"))),
make_dataset(updates, _("Update RPMs"), rpmkeys, lrpmkeys),
make_dataset(rpms, _("Installed RPMs"), rpmdkeys, lrpmdkeys)]
dump_xls(dds, os.path.join(dumpdir, "errata_details.xls"))
def mk_overview_dataset(data, score=fleure.globals.CVSS_MIN_SCORE,
keywords=fleure.globals.ERRATA_KEYWORDS,
core_rpms=None, **kwargs):
"""
:param data: RPMs, Update RPMs and various errata data summarized
:param score: CVSS base metrics score limit
:param keywords: A tuple of keywords to filter 'important' RHBAs
:param core_rpms: Core RPMs to filter errata by them
:return: An instance of tablib.Dataset becomes a worksheet represents the
overview of analysys reuslts
"""
rows = [[_("Critical or Important RHSAs (Security Errata)")],
[_("# of Critical RHSAs"),
len(data["errata"]["rhsa"]["list_critical"])],
[_("# of Critical RHSAs (latests only)"),
len(data["errata"]["rhsa"]["list_latest_critical"])],
[_("# of Important RHSAs"),
len(data["errata"]["rhsa"]["list_important"])],
[_("# of Important RHSAs (latests only)"),
len(data["errata"]["rhsa"]["list_latest_important"])],
[_("Update RPMs by Critical or Important RHSAs at minimum")],
[_("# of Update RPMs by Critical RHSAs at minimum"),
len(data["errata"]["rhsa"]["list_critical_updates"])],
[_("# of Update RPMs by Important RHSAs at minimum"),
len(data["errata"]["rhsa"]["list_important_updates"])],
[],
[_("RHBAs (Bug Errata) by keywords: %s") % ", ".join(keywords)],
[_("# of RHBAs by keywords"),
len(data["errata"]["rhba"]["list_by_kwds"])],
[_("# of Update RPMs by RHBAs by keywords at minimum"),
len(data["errata"]["rhba"]["list_updates_by_kwds"])]]
if core_rpms is not None:
rows += [[],
[_("RHBAs of core rpms: %s") % ", ".join(core_rpms)],
[_("# of RHBAs of core rpms (latests only)"),
len(data["errata"]["rhba"]["list_latests_of_core_rpms"])]]
if score > 0:
rows += [[],
[_("RHSAs and RHBAs by CVSS score")],
[_("# of RHSAs of CVSS Score >= %.1f") % score,
len(data["errata"]["rhsa"]["list_higher_cvss_score"])],
[_("# of Update RPMs by the above RHSAs at minimum"),
len(data["errata"]["rhsa"]["list_higher_cvss_updates"])],
[_("# of RHBAs of CVSS Score >= %.1f") % score,
len(data["errata"]["rhba"]["list_higher_cvss_score"])],
[_("# of Update RPMs by the above RHBAs at minimum"),
len(data["errata"]["rhba"]["list_higher_cvss_updates"])]]
rows += [[],
[_("# of RHSAs"), len(data["errata"]["rhsa"]["list"])],
[_("# of RHBAs"), len(data["errata"]["rhba"]["list"])],
[_("# of RHEAs (Enhancement Errata)"),
len(data["errata"]["rhea"]["list"])],
[_("# of Update RPMs"), len(data["updates"]["list"])],
[_("# of Installed RPMs"), len(data["installed"]["list"])],
[],
[_("Origin of Installed RPMs")],
[_("# of Rebuilt RPMs"), len(data["installed"]["list_rebuilt"])],
[_("# of Replaced RPMs"),
len(data["installed"]["list_replaced"])],
[_("# of RPMs from other vendors (non Red Hat)"),
len(data["installed"]["list_from_others"])]]
headers = (_("Item"), _("Value"), _("Notes"))
dataset = tablib.Dataset(headers=headers)
dataset.title = _("Overview of analysis results")
mcols = len(headers)
for row in rows:
if row and len(row) == 1: # Special case: separator
dataset.append_separator(row[0])
else:
dataset.append(padding_row(row, mcols))
return dataset