def add_host_to_metadata(workdir, host):
metadatafile = os.path.join(workdir, "metadata.json")
shutil.copy2(metadatafile, metadatafile + ".save")
metadata = U.json_load(metadatafile)
metadata["hosts"].append(host)
U.json_dump(metadata, metadatafile)
def dump_graphs(root, workdir, tpaths=_TEMPLATE_PATHS, html=True):
"""
Make and dump RPM dependency graphs.
:param root: RPM Database root dir
:param workdir: Working directory to dump results
:param tpaths: Template path list
:param html: Generate HTML graph files if True
"""
g = make_dependency_graph(root)
dag = make_dependencies_dag(root)
trees = make_dependencies_trees(root, True)
if os.path.exists(workdir):
assert os.path.isdir(workdir)
else:
os.makedirs(workdir)
# see also: http://bl.ocks.org/mbostock/4062045.
logging.info("Make dependency graph and dump it")
g_data = dict()
g_data["name"] = "RPM Dependency graph: root=%s" % root
g_data["nodes"] = [dict(name=n, group=1) for n in g]
nodeids = dict((n["name"], i) for i, n in enumerate(g_data["nodes"]))
g_data["links"] = [
dict(source=nodeids[e[0]], target=nodeids[e[1]], value=1)
for e in g.edges_iter()
]
U.json_dump(g_data, os.path.join(workdir, "rpmdep_graph.json"))
# Likewise.
logging.info("Make dependency DAG and dump it")
dag_data = dict()
dag_data["name"] = "RPM Dependency DAG: root=%s" % root
dag_data["nodes"] = [dict(name=n, group=1) for n in dag]
nodeids = dict((n["name"], i) for i, n in enumerate(dag_data["nodes"]))
dag_data["links"] = [
dict(source=nodeids[e[0]], target=nodeids[e[1]], value=1)
for e in dag.edges_iter()
]
U.json_dump(dag_data, os.path.join(workdir, "rpmdep_dag.json"))
# see also: http://bl.ocks.org/mbostock/4063550 (flare.json)
logging.info("Make dependency trees and dump them")
for tree in trees:
f = os.path.join(workdir, "rpmdep_tree_%(name)s.json" % tree)
U.copen(f, 'w').write(str(tree))
# Render dependency graph w/ d3.js (force directed layout).
# see also: https://github.com/mbostock/d3/wiki/Force-Layout
if html:
logging.info("Generate HTML graph files")
t = "rpmdep_graph_d3_force_directed_graph.html.j2"
_renderfile(workdir, t, dict(root=root, graph_type="graph"), tpaths)
_renderfile(workdir, t, dict(root=root, graph_type="daga"), tpaths)
def prepare(root,
workdir=None,
repos=[],
did=None,
cachedir=None,
backend=DEFAULT_BACKEND,
backends=BACKENDS,
nevra_keys=NEVRA_KEYS):
"""
:param root: Root dir of RPM db, ex. / (/var/lib/rpm)
:param workdir: Working dir to save results
:param repos: List of yum repos to get updateinfo data (errata and updtes)
:param did: Identity of the data (ex. hostname) or empty str
:param cachedir: A dir to save metadata cache of yum repos
:param backend: Backend module to use to get updates and errata
:param backends: Backend list
:return: A bunch.Bunch object of (Base, workdir, installed_rpms_list)
"""
root = os.path.abspath(root) # Ensure it's absolute path.
if not repos:
repos = rpmkit.updateinfo.utils.guess_rhel_repos(root)
LOG.info(_("%s: Use guessed repos %s"), did, ', '.join(repos))
if workdir is None:
LOG.info(_("%s: Set workdir to root %s"), did, root)
workdir = root
else:
if not os.path.exists(workdir):
LOG.debug(_("%s: Creating working dir %s"), did, workdir)
os.makedirs(workdir)
host = bunch.bunchify(
dict(id=did,
root=root,
workdir=workdir,
repos=repos,
available=False,
cachedir=cachedir))
# pylint: disable=maybe-no-member
if not rpmkit.updateinfo.utils.check_rpmdb_root(root):
LOG.warn(_("%s: RPM DB not available and don't analyze %s"), host.id,
root)
return host
base = get_backend(backend)(host.root,
host.repos,
workdir=host.workdir,
cachedir=cachedir)
LOG.debug(_("%s: Initialized backend %s"), host.id, base.name)
host.base = base
LOG.debug(_("%s: Dump Installed RPMs list loaded from %s"), host.id,
host.root)
host.installed = sorted(host.base.list_installed(),
key=itemgetter(*nevra_keys))
LOG.info(_("%s: Found %d (rebuilt=%d, replaced=%d) Installed RPMs"),
host.id, len(host.installed),
len([p for p in host.installed if p.get("rebuilt", False)]),
len([p for p in host.installed if p.get("replaced", False)]))
U.json_dump(dict(data=host.installed, ), rpm_list_path(host.workdir))
host.available = True
# pylint: enable=maybe-no-member
return host
def dump_results(workdir,
rpms,
errata,
updates,
score=0,
keywords=ERRATA_KEYWORDS,
core_rpms=[],
details=True,
rpmkeys=NEVRA_KEYS,
vendor="redhat"):
"""
:param workdir: Working dir to dump the result
:param rpms: A list of installed RPMs
:param errata: A list of applicable errata
:param updates: A list of update RPMs
:param score: CVSS base metrics score
:param keywords: Keyword list to filter 'important' RHBAs
:param core_rpms: Core RPMs to filter errata by them
:param details: Dump details also if True
"""
rpms_rebuilt = [p for p in rpms if p.get("rebuilt", False)]
rpms_replaced = [p for p in rpms if p.get("replaced", False)]
rpms_from_others = [p for p in rpms if p.get("origin", '') != vendor]
rpms_by_vendor = [
p for p in rpms if p.get("origin", '') == vendor
and not p.get("rebuilt", False) and not p.get("replaced", False)
]
nps = len(rpms)
nus = len(updates)
data = dict(errata=analyze_errata(errata, updates, score, keywords,
core_rpms),
installed=dict(list=rpms,
list_rebuilt=rpms_rebuilt,
list_replaced=rpms_replaced,
list_from_others=rpms_from_others,
list_by_vendor=rpms_by_vendor),
updates=dict(list=updates,
rate=[(_("packages need updates"), nus),
(_("packages not need updates"), nps - nus)
]))
U.json_dump(data, os.path.join(workdir, "summary.json"))
# FIXME: How to keep DRY principle?
lrpmkeys = [_("name"), _("epoch"), _("version"), _("release"), _("arch")]
rpmdkeys = rpmkeys + ["summary", "vendor", "buildhost"]
lrpmdkeys = lrpmkeys + [_("summary"), _("vendor"), _("buildhost")]
sekeys = ("advisory", "severity", "synopsis", "url", "update_names")
lsekeys = (_("advisory"), _("severity"), _("synopsis"), _("url"),
_("update_names"))
bekeys = ("advisory", "keywords", "synopsis", "url", "update_names")
lbekeys = (_("advisory"), _("keywords"), _("synopsis"), _("url"),
_("update_names"))
ds = [
make_overview_dataset(workdir, data, score, keywords, core_rpms),
make_dataset((data["errata"]["rhsa"]["list_latest_critical"] +
data["errata"]["rhsa"]["list_latest_important"]),
_("Cri-Important RHSAs (latests)"), sekeys, lsekeys),
make_dataset(
sorted(data["errata"]["rhsa"]["list_critical"],
key=itemgetter("update_names")) +
sorted(data["errata"]["rhsa"]["list_important"],
key=itemgetter("update_names")),
_("Critical or Important RHSAs"), sekeys, lsekeys),
make_dataset(data["errata"]["rhba"]["list_by_kwds_of_core_rpms"],
_("RHBAs (core rpms, keywords)"), bekeys, lbekeys),
make_dataset(data["errata"]["rhba"]["list_by_kwds"],
_("RHBAs (keyword)"), bekeys, lbekeys),
make_dataset(data["errata"]["rhba"]["list_latests_of_core_rpms"],
_("RHBAs (core rpms, latests)"), bekeys, lbekeys),
make_dataset(data["errata"]["rhsa"]["list_critical_updates"],
_("Update RPMs by RHSAs (Critical)"), rpmkeys, lrpmkeys),
make_dataset(data["errata"]["rhsa"]["list_important_updates"],
_("Updates by RHSAs (Important)"), rpmkeys, lrpmkeys),
make_dataset(data["errata"]["rhba"]["list_updates_by_kwds"],
_("Updates by RHBAs (Keyword)"), rpmkeys, lrpmkeys)
]
if score > 0:
cvss_ds = [
make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"],
_("RHSAs (CVSS score >= %.1f)") % score,
("advisory", "severity", "synopsis", "cves",
"cvsses_s", "url"),
(_("advisory"), _("severity"), _("synopsis"),
_("cves"), _("cvsses_s"), _("url"))),
make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"],
_("RHBAs (CVSS score >= %.1f)") % score,
("advisory", "synopsis", "cves", "cvsses_s", "url"),
(_("advisory"), _("synopsis"), _("cves"),
_("cvsses_s"), _("url")))
]
ds.extend(cvss_ds)
if data["installed"]["list_rebuilt"]:
ds.append(
make_dataset(data["installed"]["list_rebuilt"], _("Rebuilt RPMs"),
rpmdkeys, lrpmdkeys))
if data["installed"]["list_replaced"]:
ds.append(
make_dataset(data["installed"]["list_replaced"],
_("Replaced RPMs"), rpmdkeys, lrpmdkeys))
if data["installed"]["list_from_others"]:
ds.append(
make_dataset(data["installed"]["list_from_others"],
_("RPMs from other vendors"), rpmdkeys, lrpmdkeys))
dump_xls(ds, os.path.join(workdir, "errata_summary.xls"))
if details:
dds = [
make_dataset(
errata, _("Errata Details"),
("advisory", "type", "severity", "synopsis", "description",
"issue_date", "update_date", "url", "cves", "bzs",
"update_names"),
(_("advisory"), _("type"), _("severity"), _("synopsis"),
_("description"), _("issue_date"), _("update_date"), _("url"),
_("cves"), _("bzs"), _("update_names"))),
make_dataset(updates, _("Update RPMs"), rpmkeys, lrpmkeys),
make_dataset(rpms, _("Installed RPMs"), rpmdkeys, lrpmdkeys)
]
dump_xls(dds, os.path.join(workdir, "errata_details.xls"))
def analyze(host,
score=0,
keywords=ERRATA_KEYWORDS,
core_rpms=[],
period=(),
refdir=None,
nevra_keys=NEVRA_KEYS):
"""
:param host: host object function :function:`prepare` returns
:param score: CVSS base metrics score
:param keywords: Keyword list to filter 'important' RHBAs
:param core_rpms: Core RPMs to filter errata by them
:param period: Period of errata in format of YYYY[-MM[-DD]],
ex. ("2014-10-01", "2014-11-01")
:param refdir: A dir holding reference data previously generated to
compute delta (updates since that data)
"""
base = host.base
workdir = host.workdir
timestamp = datetime.datetime.now().strftime("%F %T")
metadata = bunch.bunchify(
dict(id=host.id,
root=host.root,
workdir=host.workdir,
repos=host.repos,
backend=host.base.name,
score=score,
keywords=keywords,
installed=len(host.installed),
hosts=[
host.id,
],
generated=timestamp))
# pylint: disable=maybe-no-member
LOG.debug(_("%s: Dump metadata for %s"), host.id, host.root)
# pylint: enable=maybe-no-member
U.json_dump(metadata.toDict(), os.path.join(workdir, "metadata.json"))
us = U.uniq(base.list_updates(), key=itemgetter(*nevra_keys))
es = base.list_errata()
es = U.uniq(errata_complement_g(es, us, score),
key=itemgetter("id"),
reverse=True)
LOG.info(_("%s: Found %d Errata, %d Update RPMs"), host.id, len(es),
len(us))
LOG.debug(_("%s: Dump Errata and Update RPMs list..."), host.id)
U.json_dump(dict(data=es, ), errata_list_path(workdir))
U.json_dump(dict(data=us, ), updates_file_path(workdir))
host.errata = es
host.updates = us
ips = host.installed
LOG.info(_("%s: Analyze and dump results of errata data in %s"), host.id,
workdir)
dump_results(workdir, ips, es, us, score, keywords, core_rpms)
if period:
(start_date, end_date) = period_to_dates(*period)
LOG.info(_("%s: Analyze errata in period: %s ~ %s"), host.id,
start_date, end_date)
pes = [e for e in es if errata_in_period(e, start_date, end_date)]
pdir = os.path.join(workdir, "%s_%s" % (start_date, end_date))
if not os.path.exists(pdir):
LOG.debug(_("%s: Creating period working dir %s"), host.id, pdir)
os.makedirs(pdir)
dump_results(pdir, ips, pes, us, score, keywords, core_rpms, False)
if refdir:
LOG.debug(_("%s [delta]: Analyze delta errata data by refering %s"),
host.id, refdir)
(es, us) = compute_delta(refdir, es, us)
LOG.info(_("%s [delta]: Found %d Errata, %d Update RPMs"), host.id,
len(es), len(us))
deltadir = os.path.join(workdir, "delta")
if not os.path.exists(deltadir):
LOG.debug(_("%s: Creating delta working dir %s"), host.id,
deltadir)
os.makedirs(deltadir)
U.json_dump(dict(data=es, ), errata_list_path(deltadir))
U.json_dump(dict(data=us, ), updates_file_path(deltadir))
LOG.info(_("%s: Analyze and dump results of delta errata in %s"),
host.id, deltadir)
dump_results(workdir, ips, es, us, score, keywords, core_rpms)
