def load_package_groups_data_g(paths=[], data=_DATA_0, profkey="system_profile"): """ :param paths: A list of package group data dirs """ sysprof = bunch.bunchify(data.get(profkey, {})) for path in paths: logging.debug("Loading profiles from: " + path) pgdata = load_profiles(path) for grp in pgdata.get("groups", []): instif = grp.get("install_if", '') grp["install_if"] = parse_install_pred(instif, sysprof, False) logging.debug("install_if: %s -> %s" % (instif, grp["install_if"])) # TODO: Is 'type' of the packages (mandatory | default | optional) # to be checked? inst_pkgs = RU.uniq( (p["name"] for p in grp.get("packages", []) if grp["install_if"]), use_set=True) uninst_pkgs = RU.uniq( (p["name"] for p in grp.get("packages", []) if not grp["install_if"]), use_set=True) grp["install_pkgs"] = inst_pkgs grp["remove_pkgs"] = uninst_pkgs yield grp
def load_package_groups_data_g(paths=[], data=_DATA_0, profkey="system_profile"): """ :param paths: A list of package group data dirs """ sysprof = bunch.bunchify(data.get(profkey, {})) for path in paths: logging.debug("Loading profiles from: " + path) pgdata = load_profiles(path) for grp in pgdata.get("groups", []): instif = grp.get("install_if", '') grp["install_if"] = parse_install_pred(instif, sysprof, False) logging.debug("install_if: %s -> %s" % (instif, grp["install_if"])) # TODO: Is 'type' of the packages (mandatory | default | optional) # to be checked? inst_pkgs = RU.uniq((p["name"] for p in grp.get("packages", []) if grp["install_if"]), use_set=True) uninst_pkgs = RU.uniq((p["name"] for p in grp.get("packages", []) if not grp["install_if"]), use_set=True) grp["install_pkgs"] = inst_pkgs grp["remove_pkgs"] = uninst_pkgs yield grp
def make_dependencies_dag(root, reqs=None, rreqs=None): """ Make directed acyclic graph of RPM dependencies. see also: * http://en.wikipedia.org/wiki/Directed_acyclic_graph * http://en.wikipedia.org/wiki/Strongly_connected_component :param root: RPM Database root dir :param reqs: A dict represents RPM deps, {x: [package_requires_x]}. :param rreqs: A dict represents RPM deps, {x: [package_required_by_x]}. :return: networkx.DiGraph instance represents the dag of rpm deps. """ if rreqs is None: rreqs = RU.make_reversed_requires_dict(root) if reqs is None: reqs = RU.make_requires_dict(root) g = make_dependency_graph(root, rreqs=rreqs) # Degenerate strongly connected components: for scc in list(NX.strongly_connected_components(g)): scc = sorted(U.uniq(scc)) # TODO: Is this needed? if len(scc) == 1: # Ignore sccs of which length is 1. continue _degenerate_nodes(g, scc, "Strongly Connected Components") # Degenerate cyclic nodes: for cns in NX.simple_cycles(g): cns = sorted(U.uniq(cns)) # TODO: Likewise # Should not happen as selc cyclic nodes were removed in advance. assert len(cns) != 1, "Self cyclic node: " + cns[0] _degenerate_nodes(g, cns, "Cyclic nodes") assert NX.is_directed_acyclic_graph(g), \ "I'm still missing something to make the dep. graph to dag..." return g
def _notice_to_errata(notice): """ Notice metadata examples: packages: 'pkglist': [ {'name': 'Red Hat Enterprise Linux Server (v. 6 for 64-bit x86_64)', 'packages': [ {'arch': 'x86_64', 'epoch': '0', 'filename': 'xorg-x11-drv-fbdev-0.4.3-16.el6.x86_64.rpm', 'name': 'xorg-x11-drv-fbdev', 'release': '16.el6', 'src': 'xorg-x11-drv-fbdev-0.4.3-16.el6.src.rpm', 'sum': ('sha256', '8f3da83bb19c3776053c543002c9...'), 'version': '0.4.3'}, ... }, ... ] cve in notice_metadata["references"]: {'href': 'https://www.redhat.com/security/data/cve/CVE-2013-1994.html', 'id': 'CVE-2013-1994', 'title': 'CVE-2013-1994', 'type': 'cve'} """ nmd = notice.get_metadata() errata = dict(advisory=nmd["update_id"], synopsis=nmd["title"], description=nmd["description"], update_date=nmd["updated"], issue_date=nmd["issued"], solution=nmd["solution"], type=nmd["type"], severity=nmd.get("severity", "N/A")) errata["bzs"] = [ normalize_bz(bz) for bz in filter( lambda r: r.get("type") == "bugzilla", nmd.get("references", [])) ] errata["cves"] = [ normalize_cve(cve) for cve in filter(lambda r: r.get("type") == "cve", nmd.get("references", [])) ] errata["packages"] = RU.concat(nps["packages"] for nps in nmd.get("pkglist", [])) errata["package_names"] = ','.join( RU.uniq(p["name"] for p in errata["packages"])) errata["url"] = rpmkit.updateinfo.utils.errata_url(errata["advisory"]) return errata
def make_dependencies_dag(root, reqs=None, rreqs=None): """ Make directed acyclic graph of RPM dependencies. see also: * http://en.wikipedia.org/wiki/Directed_acyclic_graph * http://en.wikipedia.org/wiki/Strongly_connected_component :param root: RPM Database root dir :param reqs: A dict represents RPM deps, {x: [package_requires_x]}. :param rreqs: A dict represents RPM deps, {x: [package_required_by_x]}. :return: networkx.DiGraph instance represents the dag of rpm deps. """ if rreqs is None: rreqs = RU.make_reversed_requires_dict(root) if reqs is None: reqs = RU.make_requires_dict(root) g = make_dependency_graph(root, rreqs=rreqs) # Degenerate strongly connected components: for scc in list(NX.strongly_connected_components(g)): scc = sorted(U.uniq(scc)) # TODO: Is this needed? if len(scc) == 1: # Ignore sccs of which length is 1. continue _degenerate_nodes(g, scc, "Strongly Connected Components") # Degenerate cyclic nodes: for cns in NX.simple_cycles(g): cns = sorted(U.uniq(cns)) # TODO: Likewise # Should not happen as selc cyclic nodes were removed in advance. assert len(cns) != 1, "Self cyclic node: " + cns[0] _degenerate_nodes(g, cns, "Cyclic nodes") assert NX.is_directed_acyclic_graph(g), "I'm still missing something to make the dep. graph to dag..." return g
def load_packages(pf): """ Load package info list from given file. :param pf: Packages list file. """ labels = RU.uniq(load_packages_g(pf)) LOG.info("Loaded %d RPM labels from %s" % (len(labels), pf)) return labels
def _notice_to_errata(notice): """ Notice metadata examples: packages: 'pkglist': [ {'name': 'Red Hat Enterprise Linux Server (v. 6 for 64-bit x86_64)', 'packages': [ {'arch': 'x86_64', 'epoch': '0', 'filename': 'xorg-x11-drv-fbdev-0.4.3-16.el6.x86_64.rpm', 'name': 'xorg-x11-drv-fbdev', 'release': '16.el6', 'src': 'xorg-x11-drv-fbdev-0.4.3-16.el6.src.rpm', 'sum': ('sha256', '8f3da83bb19c3776053c543002c9...'), 'version': '0.4.3'}, ... }, ... ] cve in notice_metadata["references"]: {'href': 'https://www.redhat.com/security/data/cve/CVE-2013-1994.html', 'id': 'CVE-2013-1994', 'title': 'CVE-2013-1994', 'type': 'cve'} """ nmd = notice.get_metadata() errata = dict(advisory=nmd["update_id"], synopsis=nmd["title"], description=nmd["description"], update_date=nmd["updated"], issue_date=nmd["issued"], solution=nmd["solution"], type=nmd["type"], severity=nmd.get("severity", "N/A")) errata["bzs"] = [normalize_bz(bz) for bz in filter(lambda r: r.get("type") == "bugzilla", nmd.get("references", []))] errata["cves"] = [normalize_cve(cve) for cve in filter(lambda r: r.get("type") == "cve", nmd.get("references", []))] errata["packages"] = RU.concat(nps["packages"] for nps in nmd.get("pkglist", [])) errata["package_names"] = ','.join(RU.uniq(p["name"] for p in errata["packages"])) errata["url"] = rpmkit.updateinfo.utils.errata_url(errata["advisory"]) return errata
def errata_complement_g(errata, updates, score=0): """ TODO: What should be complemented? :param errata: A list of errata :param updates: A list of update packages :param score: CVSS score """ unas = set(p2na(u) for u in updates) for e in errata: e["id"] = errata_to_int(e) # Sorting key e["updates"] = U.uniq(p for p in e.get("packages", []) if p2na(p) in unas) e["update_names"] = list(set(u["name"] for u in e["updates"])) # TODO: Dirty hack to strip extra white spaces at the top and the end # of synopsis of some errata. e["synopsis"] = e["synopsis"].strip() if score > 0: e["cves"] = [fetch_cve_details(cve) for cve in e.get("cves", [])] yield e
def ucat(xss): return RU.uniq(RU.concat(xss))
def analyze(host, score=0, keywords=ERRATA_KEYWORDS, core_rpms=[], period=(), refdir=None, nevra_keys=NEVRA_KEYS): """ :param host: host object function :function:`prepare` returns :param score: CVSS base metrics score :param keywords: Keyword list to filter 'important' RHBAs :param core_rpms: Core RPMs to filter errata by them :param period: Period of errata in format of YYYY[-MM[-DD]], ex. ("2014-10-01", "2014-11-01") :param refdir: A dir holding reference data previously generated to compute delta (updates since that data) """ base = host.base workdir = host.workdir timestamp = datetime.datetime.now().strftime("%F %T") metadata = bunch.bunchify(dict(id=host.id, root=host.root, workdir=host.workdir, repos=host.repos, backend=host.base.name, score=score, keywords=keywords, installed=len(host.installed), hosts=[host.id, ], generated=timestamp)) # pylint: disable=maybe-no-member LOG.debug(_("%s: Dump metadata for %s"), host.id, host.root) # pylint: enable=maybe-no-member U.json_dump(metadata.toDict(), os.path.join(workdir, "metadata.json")) us = U.uniq(base.list_updates(), key=itemgetter(*nevra_keys)) es = base.list_errata() es = U.uniq(errata_complement_g(es, us, score), key=itemgetter("id"), reverse=True) LOG.info(_("%s: Found %d Errata, %d Update RPMs"), host.id, len(es), len(us)) LOG.debug(_("%s: Dump Errata and Update RPMs list..."), host.id) U.json_dump(dict(data=es, ), errata_list_path(workdir)) U.json_dump(dict(data=us, ), updates_file_path(workdir)) host.errata = es host.updates = us ips = host.installed LOG.info(_("%s: Analyze and dump results of errata data in %s"), host.id, workdir) dump_results(workdir, ips, es, us, score, keywords, core_rpms) if period: (start_date, end_date) = period_to_dates(*period) LOG.info(_("%s: Analyze errata in period: %s ~ %s"), host.id, start_date, end_date) pes = [e for e in es if errata_in_period(e, start_date, end_date)] pdir = os.path.join(workdir, "%s_%s" % (start_date, end_date)) if not os.path.exists(pdir): LOG.debug(_("%s: Creating period working dir %s"), host.id, pdir) os.makedirs(pdir) dump_results(pdir, ips, pes, us, score, keywords, core_rpms, False) if refdir: LOG.debug(_("%s [delta]: Analyze delta errata data by refering %s"), host.id, refdir) (es, us) = compute_delta(refdir, es, us) LOG.info(_("%s [delta]: Found %d Errata, %d Update RPMs"), host.id, len(es), len(us)) deltadir = os.path.join(workdir, "delta") if not os.path.exists(deltadir): LOG.debug(_("%s: Creating delta working dir %s"), host.id, deltadir) os.makedirs(deltadir) U.json_dump(dict(data=es, ), errata_list_path(deltadir)) U.json_dump(dict(data=us, ), updates_file_path(deltadir)) LOG.info(_("%s: Analyze and dump results of delta errata in %s"), host.id, deltadir) dump_results(workdir, ips, es, us, score, keywords, core_rpms)
def analyze(host, score=0, keywords=ERRATA_KEYWORDS, core_rpms=[], period=(), refdir=None, nevra_keys=NEVRA_KEYS): """ :param host: host object function :function:`prepare` returns :param score: CVSS base metrics score :param keywords: Keyword list to filter 'important' RHBAs :param core_rpms: Core RPMs to filter errata by them :param period: Period of errata in format of YYYY[-MM[-DD]], ex. ("2014-10-01", "2014-11-01") :param refdir: A dir holding reference data previously generated to compute delta (updates since that data) """ base = host.base workdir = host.workdir timestamp = datetime.datetime.now().strftime("%F %T") metadata = bunch.bunchify( dict(id=host.id, root=host.root, workdir=host.workdir, repos=host.repos, backend=host.base.name, score=score, keywords=keywords, installed=len(host.installed), hosts=[ host.id, ], generated=timestamp)) # pylint: disable=maybe-no-member LOG.debug(_("%s: Dump metadata for %s"), host.id, host.root) # pylint: enable=maybe-no-member U.json_dump(metadata.toDict(), os.path.join(workdir, "metadata.json")) us = U.uniq(base.list_updates(), key=itemgetter(*nevra_keys)) es = base.list_errata() es = U.uniq(errata_complement_g(es, us, score), key=itemgetter("id"), reverse=True) LOG.info(_("%s: Found %d Errata, %d Update RPMs"), host.id, len(es), len(us)) LOG.debug(_("%s: Dump Errata and Update RPMs list..."), host.id) U.json_dump(dict(data=es, ), errata_list_path(workdir)) U.json_dump(dict(data=us, ), updates_file_path(workdir)) host.errata = es host.updates = us ips = host.installed LOG.info(_("%s: Analyze and dump results of errata data in %s"), host.id, workdir) dump_results(workdir, ips, es, us, score, keywords, core_rpms) if period: (start_date, end_date) = period_to_dates(*period) LOG.info(_("%s: Analyze errata in period: %s ~ %s"), host.id, start_date, end_date) pes = [e for e in es if errata_in_period(e, start_date, end_date)] pdir = os.path.join(workdir, "%s_%s" % (start_date, end_date)) if not os.path.exists(pdir): LOG.debug(_("%s: Creating period working dir %s"), host.id, pdir) os.makedirs(pdir) dump_results(pdir, ips, pes, us, score, keywords, core_rpms, False) if refdir: LOG.debug(_("%s [delta]: Analyze delta errata data by refering %s"), host.id, refdir) (es, us) = compute_delta(refdir, es, us) LOG.info(_("%s [delta]: Found %d Errata, %d Update RPMs"), host.id, len(es), len(us)) deltadir = os.path.join(workdir, "delta") if not os.path.exists(deltadir): LOG.debug(_("%s: Creating delta working dir %s"), host.id, deltadir) os.makedirs(deltadir) U.json_dump(dict(data=es, ), errata_list_path(deltadir)) U.json_dump(dict(data=us, ), updates_file_path(deltadir)) LOG.info(_("%s: Analyze and dump results of delta errata in %s"), host.id, deltadir) dump_results(workdir, ips, es, us, score, keywords, core_rpms)