Exemple #1
0
def main(root_or_arc_path, hid=None, verbosity=0, **kwargs):
    """
    :param root_or_arc_path:
        Path to the root dir of RPM DB files or Archive (tar.xz, tar.gz, zip,
        etc.) of RPM DB files. Path might be a relative path from current dir.
    :param hid:
        Some identification info of the target host where original RPM DB data
        was collected.
    :param verbosity: Verbosity level: 0 (default), 1 (verbose), 2 (debug)
    :param kwargs:
        Extra keyword arguments other than `root_or_arc_path` passed to make an
        instance of :class:`fleure.config.Host`

    :return: Workdir where results exist or path to archive of results
    """
    set_loglevel(verbosity)
    host = configure(root_or_arc_path, hid, **kwargs)
    if host is None:
        LOG.error(_("Failed to configure the host: root=%s"),
                  root_or_arc_path)
        return None

    prepare(host)

    if host.available:
        LOG.info(_("Anaylize the host: %s"), host.hid)
        analyze(host)

    if kwargs.get("archive", False):
        outname = "report-%s-%s.zip" % (host.hid, fleure.globals.TODAY)
        return archive_report(host.workdir, outname)
    else:
        return host.workdir
Exemple #2
0
def prepare(host):
    """
    :param host: Configured host, an instance of :class:`fleure.config.Host`
    :return: An instance of :class:`fleure.config.Host`
    """
    if not host.has_valid_root():
        LOG.error(_("Root dir is not ready. Error was: %s"), host.error)
        return

    LOG.info(_("%s: Start to initialize: root=%s, backend=%s"),
             host.hid, host.root, host.backend)
    base = host.init_base()
    base.prepare()
    LOG.info(_("%s[%s]: Initialization completed, start to analyze ..."),
             host.hid, base.name)

    host.installed = sorted(base.list_installed(),
                            key=itemgetter(*host.rpmkeys))
    LOG.info(_("%s: Found %d (rebuilt=%d, replaced=%d) installed RPMs"),
             host.hid, len(host.installed),
             len([p for p in host.installed if p.get("rebuilt", False)]),
             len([p for p in host.installed if p.get("replaced", False)]))

    host.save(host.installed, "packages")

    if base.ready():
        host.available = True
Exemple #3
0
def check_rpmdb_root(root, readonly=True, system=False,
                     dbnames=fleure.globals.RPMDB_FILENAMES):
    """
    :param root: The pivot root directry where target's RPM DB files exist
    :param readonly: Ensure RPM DB files readonly
    :param system: Allow accessing system RPM DB in /var/lib/rpm
    :return: True if necessary setup was done w/ success else False
    """
    if system:
        readonly = True
    else:
        assert root != "/", "Do not run this for host system's RPM DB!"

    rpmdbdir = os.path.join(root, fleure.globals.RPMDB_SUBDIR)

    if not os.path.exists(rpmdbdir):
        LOG.error(_("RPM DB dir %s does not exist!"), rpmdbdir)
        return False

    pkgdb = os.path.join(rpmdbdir, "Packages")
    if not _is_bsd_hashdb(pkgdb):
        LOG.error(_("%s does not look a RPM DB (Packages) file!"), pkgdb)
        return False

    for dbn in dbnames:
        dbpath = os.path.join(rpmdbdir, dbn)

        if not os.path.exists(dbpath):
            # NOTE: It's not an error at once.
            LOG.info(_("RPM DB %s looks missing"), dbn)

        if readonly and os.access(dbpath, os.W_OK) and not system:
            os.chmod(dbpath, 0o444)

    return True
Exemple #4
0
def analyze(host):
    """
    :param host: host object function :function:`prepare` returns
    """
    metadata = dict(id=host.hid, root=host.root, workdir=host.workdir,
                    repos=host.repos, backend=host.base.name,
                    score=host.cvss_min_score, keywords=host.errata_keywords,
                    pkeywords=host.errata_pkeywords,
                    installed=len(host.installed), hosts=[host.hid, ],
                    generated=datetime.datetime.now().strftime("%F %T"),
                    period=host.period, refdir=host.refdir)
    host.save(metadata, "metadata")

    LOG.info(_("%s: Analyzing errata and packages ..."), host.hid)
    host.updates = ups = host.base.list_updates()

    p2na = itemgetter("name", "arch")
    calls = (functools.partial(fleure.datasets.complement_an_errata,
                               updates=set(p2na(u) for u in ups),
                               to_update_fn=p2na,
                               score=host.cvss_min_score))
    host.errata = ers = host.base.list_errata(calls)

    host.save(ers, "errata")
    host.save(ups, "updates")
    LOG.info(_("%s: Found %d errata and %d updates, saved the lists"),
             host.hid, len(ers), len(ups))

    ips = host.installed
    analyze_and_dump_results(host, ips, ers, ups)
    LOG.info(_("%s: Saved analysis results in %s"), host.workdir)

    if host.period:
        (start, end) = host.period
        LOG.info(_("%s: Analyzing errata and packages [%s ~ %s]"),
                 host.hid, start, end)
        pdir = os.path.join(host.workdir, "%s_%s" % (start, end))
        if not os.path.exists(pdir):
            LOG.debug(_("%s: Creating period working dir %s"), host.hid, pdir)
            os.makedirs(pdir)

        pes = [e for e in ers
               if fleure.dates.in_period(e["issue_date"], start, end)]
        analyze_and_dump_results(host, ips, pes, ups, pdir)
        LOG.info(_("%s [%s ~ %s]: Found %d errata and saved"),
                 host.hid, start, end, len(pes))

    if host.refdir:
        LOG.debug(_("%s [delta]: Analyze delta errata data by refering %s"),
                  host.hid, host.refdir)
        (ers, ups) = fleure.datasets.compute_delta(host.refdir, ers, ups)
        host.save(ers, "errata", subdir="delta")
        host.save(ups, "updates", subdir="delta")
        LOG.info(_("%s [delta]: Found %d errata and %d updates, save the "
                   "lists"), host.hid, len(ers), len(ups))

        LOG.info(_("%s: Analyzing delta errata and packages ..."), host.hid)
        analyze_and_dump_results(host, ips, ers, ups)
        LOG.info(_("%s: Saved delta analysis results in %s"), host.workdir)
Exemple #5
0
def cvss_metrics(cvss, metrics_map=None):
    """
    TODO: Some of CVEs in Red Hat CVE database look having wrong CVSS
    metrics data.

    :param cvss: A string represents CVSS metrics,
        ex. "AV:N/AC:H/Au:N/C:N/I:P/A:N"
    :param metrics_map: CVSS metrics mappings :: dict

    >>> ms_ref = [
    ...     ("Access Vector", 3), ("Access Complexity", 1),
    ...     ("Authentication", 3), ("Confidentiality Impact", 1),
    ...     ("Integrity Impact", 2), ("Availability Impact", 1),
    ... ]
    >>> ms0 = cvss_metrics("AV:N/AC:H/Au:N/C:N/I:P/A:N")
    >>> assert ms0 == ms_ref, str(ms0)

    >>> ms1 = cvss_metrics("AV:N/AC:H/AU:N/C:N/I:P/A:N")  # CVE-2012-3406
    >>> assert ms1 == ms_ref, str(ms1)

    >>> ms2 = cvss_metrics("AV:N/AC:H/Au/N/C:N/I:P/A:N")  # CVE-2012-5077
    >>> assert ms2 == ms_ref, str(ms2)

    >>> ms3 = cvss_metrics("AV:N/AC:N/Au/N/C:P/I:N/A:N")  # CVE-2012-3375
    >>> assert ms3 != ms_ref, str(ms3)
    """
    if metrics_map is None:
        metrics_map = CVSSS_METRICS_MAP

    metrics = []

    if "/AU:" in cvss:
        cvss = cvss.replace("/AU:", "/Au:")

    if "/Au/" in cvss:
        cvss = cvss.replace("/Au/", "/Au:")

    for lms in cvss.split("/"):
        (key, val) = lms.split(":")
        metric = metrics_map.get(key, False)

        if not metric:
            LOG.error(_("Unknown CVSS metric abbrev: %s"), key)
            return metrics

        label = metric["label"]
        val = metric["metrics"].get(val, False)

        if not val:
            LOG.error(_("Uknown value for CVSS metric '%s': %s"), metric, val)
            return metrics

        metrics.append((label, val))

    return metrics
Exemple #6
0
def get_cvss_for_cve(cve):
    """
    Get CVSS data for given cve from the Red Hat www site.

    :param cve: CVE name, e.g. "CVE-2010-1585" :: str
    :return:  {"metrics": base_metric :: str, "score": base_score :: str}

    See the HTML source of CVE www page for its format, e.g.
    https://www.redhat.com/security/data/cve/CVE-2010-1585.html.
    """
    match = re.match(r"CVE-(?P<year>\d{4})-(?P<id>\d{4})", cve)
    if match:
        year = int(match.groupdict()["year"])
        if year < 2009:  # No CVSS
            return None
    else:
        LOG.warn(_("Invalid CVE: %s"), cve)
        return None

    def has_cvss_link(tag):
        """Does CVE has a link to CVSS base metrics?
        """
        return tag.get("href", "").startswith("http://nvd.nist.gov/cvss.cfm")

    def is_base_score(tag):
        """Is it CVSS base score?
        """
        return tag.string == "Base Score:"

    url_fmt = "http://nvd.nist.gov/cvss.cfm?version=2&name=%s&vector=(%s)"
    try:
        data = urlread(cve2url(cve))
        soup = beautifulsoup.BeautifulSoup(data)

        cvss_base_metrics = soup.findAll(has_cvss_link)[0].string
        cvss_base_score = soup.findAll(is_base_score)[0].parent.td.string

        # may fail to parse `cvss_base_metrics`
        cvss_base_metrics_vec = cvss_metrics(cvss_base_metrics)

        return dict(cve=cve,
                    metrics=cvss_base_metrics,
                    metrics_v=cvss_base_metrics_vec,
                    score=cvss_base_score,
                    url=url_fmt % (cve, cvss_base_metrics))

    except Exception as exc:
        LOG.warn(_("Could not get CVSS data: err=%s"), str(exc))

    return None
Exemple #7
0
def _errata_to_int(errata):
    """
    Generate an int represents an errata to used as comparison key.

    - RHSA > RHBA > RHEA (type)
    - RHSA: Critical > Important > Moderate > Low (severity)
    - RHBA-2013:0212 > RHBA-2012:1422 (year)
    - RHBA-2013:0212 > RHBA-2013:0210 (sequential id)
    - RHBA-2013:0212-1 > RHBA-2013:0212 (revision)

    :param errata: A dict represents an errata

    >>> _errata_to_int(dict(advisory="RHBA-2012:1422-1", ))
    202012142201
    >>> _errata_to_int(dict(advisory="RHSA-2014:0422", severity="Moderate"))
    342014042200
    """
    echars = dict(E=1, B=2, S=3)
    sevs = collections.defaultdict(int, dict(Low=2, Moderate=4, Important=6,
                                             Critical=8))
    reg = re.compile(r"^RH(?P<echar>(E|B|S))A-(?P<year>\d{4}):"
                     r"(?P<seq>\d{4,5})(?:-(?P<rev>\d+))?$")

    match = reg.match(errata["advisory"])
    if not match:
        LOG.warn(_("Not an errata advisory ? : %(advisory)s"), errata)
        return 0

    dic = match.groupdict()
    rev = 0 if dic["rev"] is None else int(dic["rev"])
    return int("%d%d%s%s%02d" % (echars[dic["echar"]],
                                 sevs[errata.get("severity", 0)],
                                 dic["year"], dic["seq"], rev))
Exemple #8
0
def extract_rpmdb_archive(arc_path, root=None):
    """Try to extract given RPM DB files archive `arc_path`.

    :param arc_path: Archive file path
    :param root:
        Path to dir to extract RPM DB files. These files will be put into
        `root`/var/lib/rpm.

    :return:
       A tuple of (root, err) where root is an absolute path of root of RPM DB
       files extracted or None, indicates extraction failed, and err is an
       error message tells what's the problem of exraction failure.
    """
    if not os.path.exists(arc_path):
        return (None, "Not found an archive: " + arc_path)

    if not os.path.isfile(arc_path):
        return (None, "Not a file: " + arc_path)

    if root is None:
        root = tempfile.mkdtemp(dir="/tmp", prefix="%s-" % __name__)
        LOG.info(_("Created a root dir of RPM DBs: %s"), root)
    else:
        root = os.path.abspath(root)  # Ensure it's an absolute path.

    rpmdbdir = os.path.join(root, fleure.globals.RPMDB_SUBDIR)
    if not os.path.exists(rpmdbdir):
        os.makedirs(rpmdbdir)

    prefix = fleure.globals.RPMDB_SUBDIR
    files = [os.path.join(prefix, fn) for fn in fleure.globals.RPMDB_FILENAMES]

    return (root, _exract_fnc(arc_path)(arc_path, root, files))
Exemple #9
0
def dump_depgraph(root, ers, workdir=None, outname="rpm_depgraph_gv", tpaths=fleure.globals.FLEURE_TEMPLATE_PATHS):
    """
    Make up context to generate RPM dependency graph w/ graphviz (sfdp) from
    the RPM database files for given host group.

    :param root: Host group's root dir where 'var/lib/rpm' exists
    :param ers: List of errata dict, see :func:`analyze_errata` in fleure.main
    :param workdir: Working dir to dump result
    :param outname: Output file base name
    :param tpaths: A list of template search paths
    """
    if workdir is None:
        workdir = root

    ctx = _make_depgraph_context(root, ers)
    fleure.utils.json_dump(ctx, os.path.join(workdir, outname + ".json"))

    output = os.path.join(workdir, outname + ".dot")
    opts = dict(at_paths=tpaths, at_engine="jinja2", at_ask_missing=True)
    anytemplate.render_to("rpm_depgraph_gv.dot.j2", ctx, output, **opts)
    anytemplate.render_to("rpm_depgraph_gv.css.j2", ctx, os.path.join(workdir, outname + ".css"), **opts)
    anytemplate.render_to("rpm_depgraph.html.j2", ctx, os.path.join(workdir, "rpm_depgraph.html"), **opts)

    output2 = os.path.join(workdir, outname + ".svg")
    cmd_s = "sfdp -Tsvg -o%s %s" % (output2, output)
    (rcode, out, err) = fleure.utils.subproc_call(cmd_s, timeout=120)
    if rcode != 0:
        if not err:
            err = "Maybe timeout occurs"
        LOG.warn(_("Failed to generate a SVG file: in=%s, out=%s, " "out/err=%s/%s"), output, output2, out, err)
Exemple #10
0
def archive_report(reportdir, output):
    """Archive analysis report.

    :reportdir: Dir where generated report files exist
    :output: Output filename
    :return:
        Absolute path of archive file made or None might indicates some
        failures before/during making archive.
    """
    filenames = fleure.globals.REPORT_FILES
    if all(os.path.exists(os.path.join(reportdir, fn)) for fn in filenames):
        arcpath = fleure.archive.archive_report(reportdir, output)
        LOG.info(_("Archived results: %s"), arcpath)
        return arcpath

    LOG.warn(_("Reprot files (%s) do not exist. Do no make a report "
               "archives"), ", ".join(filenames))
    return None
Exemple #11
0
def main(hosts_datadir, workdir=None, verbosity=0, multiproc=False, **kwargs):
    """
    :param hosts_datadir:
        Path to dir in which rpm db roots or its archive of hosts exist

    :param workdir: Working dir to save results
    :param verbosity: Verbosity level: 0 (default), 1 (verbose), 2 (debug)
    :param multiproc: Utilize multiprocessing module to compute results
        in parallel as much as possible if True
    """
    fleure.main.set_loglevel(verbosity)
    all_hosts = list(configure(hosts_datadir, workdir=workdir, **kwargs))
    hosts = prepare(all_hosts)

    LOG.info(_("Analyze %d/%d hosts"), len(hosts), len(all_hosts))
    ilen = lambda h: len(h.installed)
    hps = lambda h: [p2nevra(p) for p in h.installed]
    gby = lambda xs, kf: itertools.groupby(sorted(xs, key=kf), kf)

    # Group hosts by installed rpms to degenerate these hosts and avoid to
    # analyze for same installed RPMs more than once. his :: [[[h]]]
    his = [[list(t1[1]) for t1 in gby(t0[1], hps)] for t0 in gby(hosts, ilen)]

    for hss in his:
        hset = [(hs[0], hs[1:]) for hs in hss]
        hsdata = [hs[0] for hs in hset]

        if multiproc:
            pool = multiprocessing.Pool(multiprocessing.cpu_count())
            pool.map(fleure.main.analyze, hsdata)
        else:
            for host in hsdata:
                fleure.main.analyze(host)

        for hid, hsrest in hset:
            if hsrest:
                LOG.info(_("Skip to analyze %s as its installed RPMs are "
                           "exactly same as %s's"),
                         ','.join(x.hid for x in hsrest), hid)
                mk_symlinks_to_ref(hid, hsrest)
Exemple #12
0
def configure(hosts_datadir, workdir=None, **kwargs):
    """
    Scan and collect hosts' basic data (installed rpms list, etc.).

    :param hosts_datadir: Dir in which rpm db roots of hosts exist
    :param workdir: Working dir to save results

    :return: A generator to yield a tuple,
        (host_identity, host_rpmroot or None)
    """
    if workdir is None:
        LOG.info(_("Set workdir to hosts_datadir: %s"), hosts_datadir)
        workdir = hosts_datadir
    else:
        if not os.path.exists(workdir):
            LOG.debug(_("Creating working dir: %s"), workdir)
            os.makedirs(workdir)

    hpaths = sorted(glob.glob(os.path.join(hosts_datadir, '*')))
    hids = _hids_from_apaths(hpaths)
    cachedir = os.path.join(workdir, "_cache")  # Use common cache dir.

    for hid, hpath in itertools.izip(hids, hpaths):
        hworkdir = os.path.join(workdir, hid)
        if not os.path.exists(hworkdir):
            os.makedirs(hworkdir)

        if os.path.isdir(hpath):
            hroot = hpath
        else:
            hroot = hworkdir
            fleure.archive.extract_rpmdb_archive(hpath, hworkdir)

        kwargs["hid"] = hid
        kwargs["workdir"] = hworkdir
        kwargs["cachedir"] = cachedir

        yield fleure.main.configure(hroot, **kwargs)
Exemple #13
0
def mk_symlinks_to_ref(href, hsrest):
    """
    :param href: Reference host object
    :param hsrest: A list of hosts having same installed rpms as `href`
    """
    orgdir = os.path.abspath(os.curdir)
    for hst in hsrest:
        os.chdir(hst.workdir)
        href_workdir = os.path.join('..', href.hid)  # TODO: Keep consistency.
        LOG.info(_("%s: Make symlinks to results in %s/"),
                 hst.hid, href_workdir)
        for src in glob.glob(os.path.join(href_workdir, '*.*')):
            dst = os.path.basename(src)
            if not os.path.exists(dst):
                LOG.debug(_("Make a symlink to %s"), src)
                os.symlink(src, dst)

        metadatafile = os.path.join(href_workdir, "metadata.json")
        shutil.copy2(metadatafile, metadatafile + ".save")
        metadata = fleure.utils.json_load(metadatafile)
        metadata["hosts"].append(hst.hid)
        fleure.utils.json_dump(metadata, metadatafile)

        os.chdir(orgdir)
Exemple #14
0
def _cve_socre_ge(cve, score=0, default=False):
    """
    :param cve: A dict contains CVE and CVSS info.
    :param score: Lowest score to select CVEs (float). It's Set to 4.0 (PCIDSS
        limit) by default:

        * NVD Vulnerability Severity Ratings: http://nvd.nist.gov/cvss.cfm
        * PCIDSS: https://www.pcisecuritystandards.org

    :param default: Default value if failed to get CVSS score to compare with
        given score

    :return: True if given CVE's socre is greater or equal to given score.
    """
    if "score" not in cve:
        LOG.warn(_("CVE %(cve)s lacks of CVSS base metrics and score"), cve)
        return default
    try:
        return float(cve["score"]) >= float(score)
    except (KeyError, ValueError):
        LOG.warn(_("Failed to compare CVE's score: %s, score=%.1f"),
                 str(cve), score)

    return default
Exemple #15
0
    def populate(self):
        """
        Populates the package sack from the repositories.

        Network access to yum repos will happen if any non-local repos
        activated and it should be going to take some time to finish.
        """
        if not self._populated:
            LOG.info(_("Loading yum repo metadata from repos: %s"),
                     ','.join(r.id for r in self.base.repos.listEnabled()))
            # self.base._getTs()
            self.base._getSacks()
            self.base._getUpdates()

        self._populated = True  # TBD
Exemple #16
0
def errata_of_keywords_g(ers, keywords=fleure.globals.ERRATA_KEYWORDS,
                         pkeywords=None, stemming=True):
    """
    :param ers: A list of errata
    :param keywords: A tuple of keywords to filter 'important' RHBAs
    :param pkeywords: Similar to above but a dict gives the list per RPMs
    :param stemming: Strict matching of keywords with using NLTK stemmer
    :return:
        A generator to yield errata of which description contains any of
        given keywords

    >>> ert0 = dict(advisory="RHSA-2015:XXX1",
    ...             description="system hangs, or crash...")
    >>> ert1 = dict(advisory="RHEA-2015:XXX2",
    ...             description="some enhancement and changes")
    >>> ers = list(errata_of_keywords_g([ert0], ("hang", ), stemming=True))
    >>> ert0 in ers
    True
    >>> ers[0]["keywords"]  # 'hangs' with stemming matches.
    ['hang']
    >>> ers = list(errata_of_keywords_g([ert0, ert1], ("hang", "crash"),
    ...                                 stemming=False))
    >>> ert0 in ers
    True
    >>> ers[0]["keywords"]  # 'hangs' w/o stemming does not match with 'hang'.
    ['crash']
    >>> ert1 in ers
    False
    """
    if stemming:
        _stem = _STEMMER.stem

    if pkeywords is None:
        pkeywords = fleure.globals.ERRATA_PKEYWORDS

    for ert in ers:
        tokens = set(nltk.wordpunct_tokenize(ert["description"]))
        if stemming:
            tokens = set(_stem(w.lower()) for w in tokens)

        kwds = _errata_keywords(ert.get("package_names", []), keywords,
                                pkeywords)
        matched = kwds & tokens
        if matched:
            LOG.debug(_("%s matched: keywords=%s"), ert["advisory"],
                      ', '.join(matched))
            ert["keywords"] = list(matched)
            yield ert
Exemple #17
0
def _fmt_bzs(bzs, summary=False):
    """
    :param cves: List of CVE dict {cve, score, url, metrics} or str "cve".
    :return: List of CVE strings
    """
    def fmt(bze):
        """bugzilla entry formatter"""
        return ("bz#%(id)s: "
                "%(summary)s " if summary and "summary" in bze else ""
                "(%(url)s)")
    try:
        bzs = [fmt(bz) % bz for bz in bzs]
    except KeyError:
        LOG.warn(_("BZ Key error: %s"), str(bzs))

    return bzs
Exemple #18
0
def set_loglevel(verbosity=0, backend=False):
    """
    :param verbosity: Verbosity level = 0 | 1 | 2
    :param backend: Set backend's log level also if True
    """
    if verbosity in (0, 1, 2):
        llvl = [logging.WARN, logging.INFO, logging.DEBUG][verbosity]
    else:
        LOG.warn(_("Wrong verbosity: %d"), verbosity)
        llvl = logging.WARN

    LOG.setLevel(llvl)

    if not backend:
        llvl = logging.WARN

    for mod in fleure.config.BACKEND_MODULES:
        getattr(mod, "LOG").setLevel(llvl)
Exemple #19
0
def _is_bsd_hashdb(dbpath):
    """
    TODO: Is this enough to check if given file ``dbpath`` is RPM DB file ?
    And also, maybe some db files should be opened w/ bsddb.btopen instead of
    bsddb.hashopen.

    >>> if os.path.exists("/etc/redhat-release"):
    ...     _is_bsd_hashdb("/var/lib/rpm/Packages")
    True
    """
    try:
        if bsddb is None:
            return True  # bsddb is not avialable in python3.

        bsddb.hashopen(dbpath, 'r')
    except (OSError, IOError):
        LOG.warn(_("Not a Berkley DB?: %s"), dbpath)
        return False

    return True
Exemple #20
0
def configure(root_or_arc_path, hid=None, **kwargs):
    """
    :param root_or_arc_path:
        Path to the root dir of RPM DB files or Archive (tar.xz, tar.gz, zip,
        etc.) of RPM DB files. Path might be a relative path from current dir.
    :param hid:
        Some identification info of the target host where original RPM DB data
        was collected.
    :param kwargs:
        Extra keyword arguments other than `root_or_arc_path` passed to make an
        instance of :class:`fleure.config.Host`

    :return: An instance of :class:`fleure.config.Host`
    """
    host = fleure.config.Host(root_or_arc_path, hid=hid, **kwargs)
    host.configure()  # Extract archive, setup root and repos, etc.

    if not host.has_valid_root():
        LOG.error(_("Root dir is not ready. Error was: %s"), host.error)
        return None

    return host
Exemple #21
0
def hosts_rpmroot_g(hosts_datadir):
    """
    List system names from assessment datadir.

    This function expects that assessment data (rpm db files) of each hosts are
    found under $host_identity/ in `datadir`, that is,
    `datadir`/<host_identity>/var/lib/rpm/Packages exists. If rpm db file[s]
    are not found for a host, that host will be simply ignored.

    <host_identity> may be a hostname, host id, fqdn or something to
    identify that host.

    :param hosts_datadir: Dir in which rpm db roots of hosts exist
    :return: A generator to yield a tuple,
        (host_identity, host_rpmroot or None)
    """
    for hostdir in glob.glob(os.path.join(hosts_datadir, '*')):
        if fleure.rpmutils.check_rpmdb_root(hostdir):
            yield (os.path.basename(hostdir), hostdir)
        else:
            LOG.warn(_("Failed to find RPM DBs under %s"), hostdir)
            yield (os.path.basename(hostdir), None)
Exemple #22
0
def safe_unzip(arcfile, destdir, files=None):
    """
    Extract zip file safely similar to :func:`safe_untar`.

    .. note::
       zipfile.extract in python 2.7+ can process untrusted zip files safely:
       https://docs.python.org/2/library/zipfile.html#zipfile.ZipFile.extract

    :param arcfile: Zip file path
    :param destdir: Destination dir to extract files from `tarfile` to
    :param files:
        A list of files to extract. All files looks safe will be extracted if
        it's None.

    :return: A list of error messages if something goes wrong or []
    """
    if files is None:
        files = []

    errors = []
    with zipfile.ZipFile(arcfile) as zipf:
        for filepath in zipf.namelist():
            if files and filepath not in files:
                LOG.info(_("Skip %s as not in the list"), filepath)
                continue

            if _is_bad_path(filepath):
                errors.append("Skip as bad path: {}".format(filepath))
                continue

            zipf.extract(filepath, path=destdir)

            path = os.path.join(destdir, filepath)
            err = _remove_if_bad_file(path, destdir)
            if err:
                errors.append(err)

    return errors
Exemple #23
0
def compute_delta(refdir, ers, updates, nevra_keys=fleure.globals.RPM_KEYS):
    """
    :param refdir: Dir has reference data files: packages.json, errata.json
        and updates.json
    :param ers: A list of errata
    :param updates: A list of update packages
    """
    _assert_if_not_exist(refdir, "data dir")

    ref_es_file = os.path.join(refdir, "errata.json")
    ref_us_file = os.path.join(refdir, "updates.json")
    _assert_if_not_exist(ref_es_file, "errata file")
    _assert_if_not_exist(ref_us_file, "updates file")

    ref_es_data = fleure.utils.json_load(ref_es_file)
    ref_us_data = fleure.utils.json_load(ref_us_file)
    LOG.debug(_("Loaded reference errata and updates file"))

    ref_eadvs = set(e["advisory"] for e in ref_es_data["data"])
    ref_nevras = set((p[k] for k in nevra_keys) for p in ref_us_data["data"])

    return ([e for e in ers if e["advisory"] not in ref_eadvs],
            [u for u in updates
             if (u[k] for k in nevra_keys) not in ref_nevras])
Exemple #24
0
def analyze_and_dump_results(host, rpms, errata, updates, dumpdir=None):
    """
    Analyze and dump package level static analysis results.

    :param host: host object function :function:`prepare` returns
    :param rpms: A list of installed RPMs
    :param errata: A list of applicable errata
    :param updates: A list of update RPMs
    :param dumpdir: Dir to save results
    """
    if dumpdir is None:
        dumpdir = host.workdir

    dargs = dict(score=host.cvss_min_score, keywords=host.errata_keywords,
                 pkeywords=host.errata_pkeywords, core_rpms=host.core_rpms)
    rpmkeys = host.rpmkeys

    installed = dict(list=rpms, list_rebuilt=[], list_replaced=[],
                     list_from_others=[])
    for pkg in rpms:
        for key in ("rebuilt", "replaced", "from_others"):
            if pkg.get(key, False):
                installed["list_" + key].append(pkg)

    nps = len(rpms)
    nus = len(updates)

    ers = fleure.analysis.analyze_errata(errata, **dargs)
    data = dict(errata=ers,
                installed=installed,
                updates=dict(list=updates,
                             rate=[(_("packages need updates"), nus),
                                   (_("packages not need updates"),
                                    nps - nus)]))

    host.save(data, "summary", dumpdir)
    fleure.depgraph.dump_depgraph(host.root, ers, host.workdir,
                                  tpaths=host.tpaths)
    # TODO: Keep DRY principle.
    lrpmkeys = [_("name"), _("epoch"), _("version"), _("release"), _("arch")]

    rpmdkeys = list(rpmkeys) + ["summary", "vendor", "buildhost"]
    lrpmdkeys = lrpmkeys + [_("summary"), _("vendor"), _("buildhost")]

    sekeys = ("advisory", "severity", "synopsis", "url", "update_names")
    lsekeys = (_("advisory"), _("severity"), _("synopsis"), _("url"),
               _("update_names"))
    bekeys = ("advisory", "keywords", "synopsis", "url", "update_names")
    lbekeys = (_("advisory"), _("keywords"), _("synopsis"), _("url"),
               _("update_names"))

    mds = [fleure.analysis.mk_overview_dataset(data, **dargs),
           make_dataset((data["errata"]["rhsa"]["list_latest_critical"] +
                         data["errata"]["rhsa"]["list_latest_important"]),
                        _("Cri-Important RHSAs (latests)"), sekeys, lsekeys),
           make_dataset(sorted(data["errata"]["rhsa"]["list_critical"],
                               key=itemgetter("update_names")) +
                        sorted(data["errata"]["rhsa"]["list_important"],
                               key=itemgetter("update_names")),
                        _("Critical or Important RHSAs"), sekeys, lsekeys),
           make_dataset(data["errata"]["rhba"]["list_by_kwds_of_core_rpms"],
                        _("RHBAs (core rpms, keywords)"), bekeys, lbekeys),
           make_dataset(data["errata"]["rhba"]["list_by_kwds"],
                        _("RHBAs (keyword)"), bekeys, lbekeys),
           make_dataset(data["errata"]["rhba"]["list_latests_of_core_rpms"],
                        _("RHBAs (core rpms, latests)"), bekeys, lbekeys),
           make_dataset(data["errata"]["rhsa"]["list_critical_updates"],
                        _("Update RPMs by RHSAs (Critical)"), rpmkeys,
                        lrpmkeys),
           make_dataset(data["errata"]["rhsa"]["list_important_updates"],
                        _("Updates by RHSAs (Important)"), rpmkeys, lrpmkeys),
           make_dataset(data["errata"]["rhba"]["list_updates_by_kwds"],
                        _("Updates by RHBAs (Keyword)"), rpmkeys, lrpmkeys)]

    score = host.cvss_min_score
    if score > 0:
        cvss_ds = [
            make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"],
                         _("RHSAs (CVSS score >= %.1f)") % score,
                         ("advisory", "severity", "synopsis",
                          "cves", "cvsses_s", "url"),
                         (_("advisory"), _("severity"), _("synopsis"),
                          _("cves"), _("cvsses_s"), _("url"))),
            make_dataset(data["errata"]["rhsa"]["list_higher_cvss_score"],
                         _("RHBAs (CVSS score >= %.1f)") % score,
                         ("advisory", "synopsis", "cves", "cvsses_s", "url"),
                         (_("advisory"), _("synopsis"), _("cves"),
                          _("cvsses_s"), _("url")))]
        mds.extend(cvss_ds)

    for key, title in (("list_rebuilt", _("Rebuilt RPMs")),
                       ("list_replaced", _("Replaced RPMs")),
                       ("list_from_others", _("RPMs from other vendors"))):
        if data["installed"][key]:
            mds.append(make_dataset(data["installed"][key], title, rpmdkeys,
                                    lrpmdkeys))

    dump_xls(mds, os.path.join(dumpdir, "errata_summary.xls"))

    if host.details:
        dds = [make_dataset(errata, _("Errata Details"),
                            ("advisory", "type", "severity", "synopsis",
                             "description", "issue_date", "update_date", "url",
                             "cves", "bzs", "update_names"),
                            (_("advisory"), _("type"), _("severity"),
                             _("synopsis"), _("description"), _("issue_date"),
                             _("update_date"), _("url"), _("cves"),
                             _("bzs"), _("update_names"))),
               make_dataset(updates, _("Update RPMs"), rpmkeys, lrpmkeys),
               make_dataset(rpms, _("Installed RPMs"), rpmdkeys, lrpmdkeys)]

        dump_xls(dds, os.path.join(dumpdir, "errata_details.xls"))
Exemple #25
0
def mk_overview_dataset(data, score=fleure.globals.CVSS_MIN_SCORE,
                        keywords=fleure.globals.ERRATA_KEYWORDS,
                        core_rpms=None, **kwargs):
    """
    :param data: RPMs, Update RPMs and various errata data summarized
    :param score: CVSS base metrics score limit
    :param keywords: A tuple of keywords to filter 'important' RHBAs
    :param core_rpms: Core RPMs to filter errata by them

    :return: An instance of tablib.Dataset becomes a worksheet represents the
        overview of analysys reuslts
    """
    rows = [[_("Critical or Important RHSAs (Security Errata)")],
            [_("# of Critical RHSAs"),
             len(data["errata"]["rhsa"]["list_critical"])],
            [_("# of Critical RHSAs (latests only)"),
             len(data["errata"]["rhsa"]["list_latest_critical"])],
            [_("# of Important RHSAs"),
             len(data["errata"]["rhsa"]["list_important"])],
            [_("# of Important RHSAs (latests only)"),
             len(data["errata"]["rhsa"]["list_latest_important"])],
            [_("Update RPMs by Critical or Important RHSAs at minimum")],
            [_("# of Update RPMs by Critical RHSAs at minimum"),
             len(data["errata"]["rhsa"]["list_critical_updates"])],
            [_("# of Update RPMs by Important RHSAs at minimum"),
             len(data["errata"]["rhsa"]["list_important_updates"])],
            [],
            [_("RHBAs (Bug Errata) by keywords: %s") % ", ".join(keywords)],
            [_("# of RHBAs by keywords"),
             len(data["errata"]["rhba"]["list_by_kwds"])],
            [_("# of Update RPMs by RHBAs by keywords at minimum"),
             len(data["errata"]["rhba"]["list_updates_by_kwds"])]]

    if core_rpms is not None:
        rows += [[],
                 [_("RHBAs of core rpms: %s") % ", ".join(core_rpms)],
                 [_("# of RHBAs of core rpms (latests only)"),
                  len(data["errata"]["rhba"]["list_latests_of_core_rpms"])]]

    if score > 0:
        rows += [[],
                 [_("RHSAs and RHBAs by CVSS score")],
                 [_("# of RHSAs of CVSS Score >= %.1f") % score,
                  len(data["errata"]["rhsa"]["list_higher_cvss_score"])],
                 [_("# of Update RPMs by the above RHSAs at minimum"),
                  len(data["errata"]["rhsa"]["list_higher_cvss_updates"])],
                 [_("# of RHBAs of CVSS Score >= %.1f") % score,
                  len(data["errata"]["rhba"]["list_higher_cvss_score"])],
                 [_("# of Update RPMs by the above RHBAs at minimum"),
                  len(data["errata"]["rhba"]["list_higher_cvss_updates"])]]

    rows += [[],
             [_("# of RHSAs"), len(data["errata"]["rhsa"]["list"])],
             [_("# of RHBAs"), len(data["errata"]["rhba"]["list"])],
             [_("# of RHEAs (Enhancement Errata)"),
              len(data["errata"]["rhea"]["list"])],
             [_("# of Update RPMs"), len(data["updates"]["list"])],
             [_("# of Installed RPMs"), len(data["installed"]["list"])],
             [],
             [_("Origin of Installed RPMs")],
             [_("# of Rebuilt RPMs"), len(data["installed"]["list_rebuilt"])],
             [_("# of Replaced RPMs"),
              len(data["installed"]["list_replaced"])],
             [_("# of RPMs from other vendors (non Red Hat)"),
              len(data["installed"]["list_from_others"])]]

    headers = (_("Item"), _("Value"), _("Notes"))
    dataset = tablib.Dataset(headers=headers)
    dataset.title = _("Overview of analysis results")

    mcols = len(headers)
    for row in rows:
        if row and len(row) == 1:  # Special case: separator
            dataset.append_separator(row[0])
        else:
            dataset.append(padding_row(row, mcols))

    return dataset