def add_host_to_metadata(workdir, host): metadatafile = os.path.join(workdir, "metadata.json") shutil.copy2(metadatafile, metadatafile + ".save") metadata = U.json_load(metadatafile) metadata["hosts"].append(host) U.json_dump(metadata, metadatafile)
def dump_graphs(root, workdir, tpaths=_TEMPLATE_PATHS, html=True): """ Make and dump RPM dependency graphs. :param root: RPM Database root dir :param workdir: Working directory to dump results :param tpaths: Template path list :param html: Generate HTML graph files if True """ g = make_dependency_graph(root) dag = make_dependencies_dag(root) trees = make_dependencies_trees(root, True) if os.path.exists(workdir): assert os.path.isdir(workdir) else: os.makedirs(workdir) # see also: http://bl.ocks.org/mbostock/4062045. logging.info("Make dependency graph and dump it") g_data = dict() g_data["name"] = "RPM Dependency graph: root=%s" % root g_data["nodes"] = [dict(name=n, group=1) for n in g] nodeids = dict((n["name"], i) for i, n in enumerate(g_data["nodes"])) g_data["links"] = [ dict(source=nodeids[e[0]], target=nodeids[e[1]], value=1) for e in g.edges_iter() ] U.json_dump(g_data, os.path.join(workdir, "rpmdep_graph.json")) # Likewise. logging.info("Make dependency DAG and dump it") dag_data = dict() dag_data["name"] = "RPM Dependency DAG: root=%s" % root dag_data["nodes"] = [dict(name=n, group=1) for n in dag] nodeids = dict((n["name"], i) for i, n in enumerate(dag_data["nodes"])) dag_data["links"] = [ dict(source=nodeids[e[0]], target=nodeids[e[1]], value=1) for e in dag.edges_iter() ] U.json_dump(dag_data, os.path.join(workdir, "rpmdep_dag.json")) # see also: http://bl.ocks.org/mbostock/4063550 (flare.json) logging.info("Make dependency trees and dump them") for tree in trees: f = os.path.join(workdir, "rpmdep_tree_%(name)s.json" % tree) U.copen(f, 'w').write(str(tree)) # Render dependency graph w/ d3.js (force directed layout). # see also: https://github.com/mbostock/d3/wiki/Force-Layout if html: logging.info("Generate HTML graph files") t = "rpmdep_graph_d3_force_directed_graph.html.j2" _renderfile(workdir, t, dict(root=root, graph_type="graph"), tpaths) _renderfile(workdir, t, dict(root=root, graph_type="daga"), tpaths)
def prepare(root, workdir=None, repos=[], did=None, cachedir=None, backend=DEFAULT_BACKEND, backends=BACKENDS, nevra_keys=NEVRA_KEYS): """ :param root: Root dir of RPM db, ex. / (/var/lib/rpm) :param workdir: Working dir to save results :param repos: List of yum repos to get updateinfo data (errata and updtes) :param did: Identity of the data (ex. hostname) or empty str :param cachedir: A dir to save metadata cache of yum repos :param backend: Backend module to use to get updates and errata :param backends: Backend list :return: A bunch.Bunch object of (Base, workdir, installed_rpms_list) """ root = os.path.abspath(root) # Ensure it's absolute path. if not repos: repos = rpmkit.updateinfo.utils.guess_rhel_repos(root) LOG.info(_("%s: Use guessed repos %s"), did, ', '.join(repos)) if workdir is None: LOG.info(_("%s: Set workdir to root %s"), did, root) workdir = root else: if not os.path.exists(workdir): LOG.debug(_("%s: Creating working dir %s"), did, workdir) os.makedirs(workdir) host = bunch.bunchify( dict(id=did, root=root, workdir=workdir, repos=repos, available=False, cachedir=cachedir)) # pylint: disable=maybe-no-member if not rpmkit.updateinfo.utils.check_rpmdb_root(root): LOG.warn(_("%s: RPM DB not available and don't analyze %s"), host.id, root) return host base = get_backend(backend)(host.root, host.repos, workdir=host.workdir, cachedir=cachedir) LOG.debug(_("%s: Initialized backend %s"), host.id, base.name) host.base = base LOG.debug(_("%s: Dump Installed RPMs list loaded from %s"), host.id, host.root) host.installed = sorted(host.base.list_installed(), key=itemgetter(*nevra_keys)) LOG.info(_("%s: Found %d (rebuilt=%d, replaced=%d) Installed RPMs"), host.id, len(host.installed), len([p for p in host.installed if p.get("rebuilt", False)]), len([p for p in host.installed if p.get("replaced", False)])) U.json_dump(dict(data=host.installed, ), rpm_list_path(host.workdir)) host.available = True # pylint: enable=maybe-no-member return host
def dump_results(workdir, rpms, errata, updates, score=0, keywords=ERRATA_KEYWORDS, core_rpms=[], details=True, rpmkeys=NEVRA_KEYS, vendor="redhat"): """ :param workdir: Working dir to dump the result :param rpms: A list of installed RPMs :param errata: A list of applicable errata :param updates: A list of update RPMs :param score: CVSS base metrics score :param keywords: Keyword list to filter 'important' RHBAs :param core_rpms: Core RPMs to filter errata by them :param details: Dump details also if True """ rpms_rebuilt = [p for p in rpms if p.get("rebuilt", False)] rpms_replaced = [p for p in rpms if p.get("replaced", False)] rpms_from_others = [p for p in rpms if p.get("origin", '') != vendor] rpms_by_vendor = [ p for p in rpms if p.get("origin", '') == vendor and not p.get("rebuilt", False) and not p.get("replaced", False) ] nps = len(rpms) nus = len(updates) data = dict(errata=analyze_errata(errata, updates, score, keywords, core_rpms), installed=dict(list=rpms, list_rebuilt=rpms_rebuilt, list_replaced=rpms_replaced, list_from_others=rpms_from_others, list_by_vendor=rpms_by_vendor), updates=dict(list=updates, rate=[(_("packages need updates"), nus), (_("packages not need updates"), nps - nus) ])) U.json_dump(data, os.path.join(workdir, "summary.json")) # FIXME: How to keep DRY principle? lrpmkeys = [_("name"), _("epoch"), _("version"), _("release"), _("arch")] rpmdkeys = rpmkeys + ["summary", "vendor", "buildhost"] lrpmdkeys = lrpmkeys + [_("summary"), _("vendor"), _("buildhost")] sekeys = ("advisory", "severity", "synopsis", "url", "update_names") lsekeys = (_("advisory"), _("severity"), _("synopsis"), _("url"), _("update_names")) bekeys = ("advisory", "keywords", "synopsis", "url", "update_names") lbekeys = (_("advisory"), _("keywords"), _("synopsis"), _("url"), _("update_names")) ds = [ make_overview_dataset(workdir, data, score, keywords, core_rpms), make_dataset((data["errata"]["rhsa"]["list_latest_critical"] + data["errata"]["rhsa"]["list_latest_important"]), _("Cri-Important RHSAs (latests)"), sekeys, lsekeys), make_dataset( sorted(data["errata"]["rhsa"]["list_critical"], key=itemgetter("update_names")) + sorted(data["errata"]["rhsa"]["list_important"], key=itemgetter("update_names")), _("Critical or Important RHSAs"), sekeys, lsekeys), make_dataset(data["errata"]["rhba"]["list_by_kwds_of_core_rpms"], _("RHBAs (core rpms, keywords)"), bekeys, lbekeys), make_dataset(data["errata"]["rhba"]["list_by_kwds"], _("RHBAs (keyword)"), bekeys, lbekeys), make_dataset(data["errata"]["rhba"]["list_latests_of_core_rpms"], _("RHBAs (core rpms, latests)"), bekeys, lbekeys), make_dataset(data["errata"]["rhsa"]["list_critical_updates"], _("Update RPMs by RHSAs (Critical)"), rpmkeys, lrpmkeys), make_dataset(data["errata"]["rhsa"]["list_important_updates"], _("Updates by RHSAs (Important)"), rpmkeys, lrpmkeys), make_dataset(data["errata"]["rhba"]["list_updates_by_kwds"], _("Updates by RHBAs (Keyword)"), rpmkeys, lrpmkeys) ] if score > 0: cvss_ds = [ make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"], _("RHSAs (CVSS score >= %.1f)") % score, ("advisory", "severity", "synopsis", "cves", "cvsses_s", "url"), (_("advisory"), _("severity"), _("synopsis"), _("cves"), _("cvsses_s"), _("url"))), make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"], _("RHBAs (CVSS score >= %.1f)") % score, ("advisory", "synopsis", "cves", "cvsses_s", "url"), (_("advisory"), _("synopsis"), _("cves"), _("cvsses_s"), _("url"))) ] ds.extend(cvss_ds) if data["installed"]["list_rebuilt"]: ds.append( make_dataset(data["installed"]["list_rebuilt"], _("Rebuilt RPMs"), rpmdkeys, lrpmdkeys)) if data["installed"]["list_replaced"]: ds.append( make_dataset(data["installed"]["list_replaced"], _("Replaced RPMs"), rpmdkeys, lrpmdkeys)) if data["installed"]["list_from_others"]: ds.append( make_dataset(data["installed"]["list_from_others"], _("RPMs from other vendors"), rpmdkeys, lrpmdkeys)) dump_xls(ds, os.path.join(workdir, "errata_summary.xls")) if details: dds = [ make_dataset( errata, _("Errata Details"), ("advisory", "type", "severity", "synopsis", "description", "issue_date", "update_date", "url", "cves", "bzs", "update_names"), (_("advisory"), _("type"), _("severity"), _("synopsis"), _("description"), _("issue_date"), _("update_date"), _("url"), _("cves"), _("bzs"), _("update_names"))), make_dataset(updates, _("Update RPMs"), rpmkeys, lrpmkeys), make_dataset(rpms, _("Installed RPMs"), rpmdkeys, lrpmdkeys) ] dump_xls(dds, os.path.join(workdir, "errata_details.xls"))
def analyze(host, score=0, keywords=ERRATA_KEYWORDS, core_rpms=[], period=(), refdir=None, nevra_keys=NEVRA_KEYS): """ :param host: host object function :function:`prepare` returns :param score: CVSS base metrics score :param keywords: Keyword list to filter 'important' RHBAs :param core_rpms: Core RPMs to filter errata by them :param period: Period of errata in format of YYYY[-MM[-DD]], ex. ("2014-10-01", "2014-11-01") :param refdir: A dir holding reference data previously generated to compute delta (updates since that data) """ base = host.base workdir = host.workdir timestamp = datetime.datetime.now().strftime("%F %T") metadata = bunch.bunchify( dict(id=host.id, root=host.root, workdir=host.workdir, repos=host.repos, backend=host.base.name, score=score, keywords=keywords, installed=len(host.installed), hosts=[ host.id, ], generated=timestamp)) # pylint: disable=maybe-no-member LOG.debug(_("%s: Dump metadata for %s"), host.id, host.root) # pylint: enable=maybe-no-member U.json_dump(metadata.toDict(), os.path.join(workdir, "metadata.json")) us = U.uniq(base.list_updates(), key=itemgetter(*nevra_keys)) es = base.list_errata() es = U.uniq(errata_complement_g(es, us, score), key=itemgetter("id"), reverse=True) LOG.info(_("%s: Found %d Errata, %d Update RPMs"), host.id, len(es), len(us)) LOG.debug(_("%s: Dump Errata and Update RPMs list..."), host.id) U.json_dump(dict(data=es, ), errata_list_path(workdir)) U.json_dump(dict(data=us, ), updates_file_path(workdir)) host.errata = es host.updates = us ips = host.installed LOG.info(_("%s: Analyze and dump results of errata data in %s"), host.id, workdir) dump_results(workdir, ips, es, us, score, keywords, core_rpms) if period: (start_date, end_date) = period_to_dates(*period) LOG.info(_("%s: Analyze errata in period: %s ~ %s"), host.id, start_date, end_date) pes = [e for e in es if errata_in_period(e, start_date, end_date)] pdir = os.path.join(workdir, "%s_%s" % (start_date, end_date)) if not os.path.exists(pdir): LOG.debug(_("%s: Creating period working dir %s"), host.id, pdir) os.makedirs(pdir) dump_results(pdir, ips, pes, us, score, keywords, core_rpms, False) if refdir: LOG.debug(_("%s [delta]: Analyze delta errata data by refering %s"), host.id, refdir) (es, us) = compute_delta(refdir, es, us) LOG.info(_("%s [delta]: Found %d Errata, %d Update RPMs"), host.id, len(es), len(us)) deltadir = os.path.join(workdir, "delta") if not os.path.exists(deltadir): LOG.debug(_("%s: Creating delta working dir %s"), host.id, deltadir) os.makedirs(deltadir) U.json_dump(dict(data=es, ), errata_list_path(deltadir)) U.json_dump(dict(data=us, ), updates_file_path(deltadir)) LOG.info(_("%s: Analyze and dump results of delta errata in %s"), host.id, deltadir) dump_results(workdir, ips, es, us, score, keywords, core_rpms)