Пример #1
0
    def reports(self, count=100):
        comps = self.ses.query(OpSysComponent).all()
        releases = self.ses.query(OpSysRelease).all()
        arches = self.ses.query(Arch).all()
        symbols = self.ses.query(SymbolSource).all()
        bugs = self.ses.query(BzBug).all()

        for rel in releases:
            self.begin('Reports for %s %s' % (rel.opsys.name, rel.version))
            since = rel.releasedate
            if since is None:
                since = datetime.now().date() + fuzzy_timedelta(
                    months=random.randrange(-6, 0))
            till = self.get_release_end_date(since, rel.opsys)

            for _ in range(count):
                report = Report()
                report.type = 'USERSPACE'
                report.count = random.randrange(1, 20)
                occ_date = self.get_occurrence_date(since, till)
                if occ_date > datetime.now():
                    # skipping reports from the future
                    continue
                report.first_occurrence = report.last_occurrence = occ_date
                report.component = random.choice(comps)
                self.add(report)

                report_bt = ReportBacktrace()
                report_bt.report = report
                self.add(report_bt)

                bthash = ReportBtHash()
                bthash.type = 'NAMES'
                bthash.hash = randutils.randhash()
                bthash.backtrace = report_bt
                self.add(bthash)

                for j in range(random.randrange(1, 40)):
                    btframe = ReportBtFrame()
                    btframe.backtrace = report_bt
                    btframe.order = j
                    btframe.symbolsource = random.choice(symbols)

                report_bt.crashfn = btframe.symbolsource.symbol.name

                current = []
                last_occ = occ_date

                for j in range(report.count):
                    if j > 1:
                        occ_date = self.get_occurrence_date(since, till)
                        if occ_date > datetime.now():
                            continue

                    if occ_date > last_occ:
                        last_occ = occ_date

                    arch = random.choice(arches)
                    day = occ_date.date()
                    week = day - timedelta(days=day.weekday())
                    month = day.replace(day=1)
                    smode = random.choice(['DISABLED', 'PERMISSIVE', 'ENFORCING'])

                    stat_map = [(ReportArch, [('arch', arch)]),
                                (ReportOpSysRelease, [('opsysrelease', rel)]),
                                (ReportHistoryMonthly, [('opsysrelease', rel),
                                                        ('month', month)]),
                                (ReportHistoryWeekly, [('opsysrelease', rel),
                                                       ('week', week)]),
                                (ReportHistoryDaily, [('opsysrelease', rel),
                                                      ('day', day)]),
                                (ReportSelinuxMode, [('mode', smode)]),
                               ]

                    # add crashed and related packages
                    for typ in ['CRASHED', 'RELATED']:
                        for pkg in locals()['{0}_pkgs'.format(typ.lower())]:
                            if randutils.toss():
                                stat_map.append(
                                    (ReportPackage,
                                     [('installed_package', pkg),
                                      ('running_package', pkg),
                                      ('type', typ)])
                                    )

                    if randutils.tosshigh():
                        stat_map.append((ReportUptime, [('uptime_exp',
                                                         int(math.log(random.randrange(1, 100000))))]))

                    if randutils.toss():
                        stat_map.append((ReportOpSysRelease,
                                         [('opsysrelease', random.choice(releases))]))

                    if randutils.tosslow():
                        stat_map.append((ReportBz,
                                         [('bzbug', random.choice(bugs))]))

                    for table, cols in stat_map:
                        for report_stat in current:
                            if not isinstance(report_stat, table):
                                continue
                            matching = True
                            for name, value in cols:
                                if getattr(report_stat, name) != value:
                                    matching = False
                            if matching:
                                report_stat.count += 1
                                break
                        else:
                            report_stat = table()
                            report_stat.report = report
                            for name, value in cols:
                                setattr(report_stat, name, value)
                            report_stat.count = 1
                            current.append(report_stat)

                self.extend(current)
                report.last_occurrence = last_occ
            self.commit()
Пример #2
0
    def reports(self, count=100):
        comps = self.ses.query(OpSysComponent).all()
        releases = self.ses.query(OpSysRelease).all()
        arches = self.ses.query(Arch).all()
        symbols = self.ses.query(SymbolSource).all()
        packages = self.ses.query(Package).all()
        bugs = self.ses.query(RhbzBug).all()

        for rel in releases:
            self.begin('Reports for %s %s' % (rel.opsys.name, rel.version))
            since = rel.releasedate
            if since is None:
                since = datetime.now().date() + fuzzy_timedelta(
                    months=random.randrange(-6, 0))
            till = self.get_release_end_date(since, rel.opsys)

            for i in range(count):
                report = Report()
                report.type = 'USERSPACE'
                report.count = random.randrange(1, 20)
                occ_date = self.get_occurence_date(since, till)
                if occ_date > datetime.now():
                    # skipping reports from the future
                    continue
                report.first_occurence = report.last_occurence = occ_date
                report.component = random.choice(comps)
                self.add(report)

                report_bt = ReportBacktrace()
                report_bt.report = report
                self.add(report_bt)

                bthash = ReportBtHash()
                bthash.type = 'NAMES'
                bthash.hash = randutils.randhash()
                bthash.backtrace = report_bt
                self.add(bthash)

                for j in range(random.randrange(1, 40)):
                    btframe = ReportBtFrame()
                    btframe.backtrace = report_bt
                    btframe.order = j
                    btframe.symbolsource = random.choice(symbols)

                report_bt.crashfn = btframe.symbolsource.symbol.name

                current = []
                last_occ = occ_date
                crashed_pkgs = random.sample(packages, random.randrange(1,4))
                related_pkgs = random.sample(packages, random.randrange(1,4))

                for j in range(report.count):
                    if j > 1:
                        occ_date = self.get_occurence_date(since, till)
                        if occ_date > datetime.now():
                            continue

                    if occ_date > last_occ:
                        last_occ = occ_date

                    arch = random.choice(arches)
                    day = occ_date.date()
                    week = day - timedelta(days=day.weekday())
                    month = day.replace(day=1)
                    smode = random.choice(['DISABLED', 'PERMISSIVE', 'ENFORCING'])

                    stat_map = [(ReportArch, [('arch', arch)]),
                                (ReportOpSysRelease, [('opsysrelease', rel)]),
                                (ReportHistoryMonthly, [('opsysrelease', rel),
                                    ('month', month)]),
                                (ReportHistoryWeekly, [('opsysrelease', rel),
                                    ('week', week)]),
                                (ReportHistoryDaily, [('opsysrelease', rel),
                                    ('day', day)]),
                                (ReportSelinuxMode, [('mode', smode)]),
                                ]

                    # add crashed and related packages
                    for typ in ['CRASHED', 'RELATED']:
                        for pkg in locals()['{0}_pkgs'.format(typ.lower())]:
                            if randutils.toss():
                                stat_map.append(
                                    (ReportPackage,
                                        [('installed_package', pkg),
                                        ('running_package', pkg),
                                        ('type', typ)])
                                    )

                    if randutils.tosshigh():
                        stat_map.append((ReportUptime, [('uptime_exp',
                            int(math.log(random.randrange(1, 100000))))]))

                    if randutils.toss():
                        stat_map.append((ReportOpSysRelease,
                            [('opsysrelease', random.choice(releases))]))

                    if randutils.tosslow():
                        stat_map.append((ReportRhbz,
                            [('rhbzbug', random.choice(bugs))]))

                    for table, cols in stat_map:
                        fn = lambda x: type(x) == table
                        for report_stat in filter(fn, current):
                            matching = True
                            for name, value in cols:
                                if getattr(report_stat, name) != value:
                                    matching = False
                            if matching:
                                report_stat.count += 1
                                break
                        else:
                            report_stat = table()
                            report_stat.report = report
                            for name, value in cols:
                                setattr(report_stat, name, value)
                            report_stat.count = 1
                            current.append(report_stat)

                self.extend(current)
                report.last_occurence = last_occ
            self.commit()
Пример #3
0
def add_report(ureport, db, utctime=None, count=1, only_check_if_known=False, return_report=False):
    if not utctime:
        utctime = datetime.datetime.utcnow()

    flip_corebt_if_necessary(ureport)

    if "component" in ureport:
        component = get_component(ureport["component"], ureport["os"], db)
    else:
        component = guess_component(ureport["installed_package"], ureport["os"], db)
    if component is None:
        raise Exception, "Unknown component."

    for frame in ureport["core_backtrace"]:
        if not "path" in frame and "executable" in ureport:
            frame["path"] = ureport["executable"]

        if ureport["type"].lower() != "kerneloops":
            frame["path"] = os.path.abspath(frame["path"])

    hash_type, hash_hash = get_report_hash(ureport, component.name)

    # Find a report with matching hash and component.
    report = (
        db.session.query(Report)
        .join(ReportBacktrace)
        .join(ReportBtHash)
        .filter((ReportBtHash.hash == hash_hash) & (ReportBtHash.type == hash_type) & (Report.component == component))
        .first()
    )

    if only_check_if_known:
        # check whether the report has a BZ associated
        # if it does not, proclaim it unknown
        if report:
            reportbz = db.session.query(ReportRhbz).filter(ReportRhbz.report_id == report.id).first()

            solution = find_ureport_kb_solution(ureport, db, opsys_id=report.component.opsys.id)

            if not reportbz and not solution:
                report = None

        if return_report:
            return report

        return bool(report)

    # Create a new report if not found.
    if not report:
        # do not process reports with empty
        # core-backtrace, except for selinux
        if ureport["type"].lower() != "selinux" and len(ureport["core_backtrace"]) < 1:
            raise Exception, "empty core_backtrace"

        report = Report()
        report.type = ureport["type"].upper()
        report.first_occurence = report.last_occurence = utctime
        report.count = count
        report.component = component
        db.session.add(report)

        report_backtrace = ReportBacktrace()
        report_backtrace.report = report
        db.session.add(report_backtrace)

        report_bthash = ReportBtHash()
        report_bthash.type = hash_type
        report_bthash.hash = hash_hash
        report_bthash.backtrace = report_backtrace
        db.session.add(report_bthash)

        # Add frames, symbols, hashes and sources.
        for frame in get_crash_thread(ureport):
            report_btframe = ReportBtFrame()
            report_btframe.backtrace = report_backtrace
            report_btframe.order = frame["frame"]
            if not "buildid" in frame:
                frame["buildid"] = None

            # Check if there was already such symbol added, but first check
            # the current session before doing a query.
            for symbolsource in (x for x in db.session.new if type(x) is SymbolSource):
                if (
                    symbolsource.build_id == frame["buildid"]
                    and symbolsource.path == frame["path"]
                    and symbolsource.offset == frame["offset"]
                ):
                    break
            else:
                symbolsource = (
                    db.session.query(SymbolSource)
                    .filter(
                        (SymbolSource.build_id == frame["buildid"])
                        & (SymbolSource.path == frame["path"])
                        & (SymbolSource.offset == frame["offset"])
                    )
                    .first()
                )

            # Create a new symbolsource if not found.
            if not symbolsource:
                symbolsource = SymbolSource()
                symbolsource.build_id = frame["buildid"]
                symbolsource.path = frame["path"]
                symbolsource.offset = frame["offset"]
                if ureport["type"].lower() == "python":
                    symbolsource.source_path = frame["path"]
                    symbolsource.line_number = frame["offset"]

                if "funchash" in frame:
                    symbolsource.hash = frame["funchash"]

                if "funcname" in frame:
                    normalized_path = get_libname(frame["path"])
                    for symbol in (x for x in db.session.new if type(x) is Symbol):
                        if symbol.name == frame["funcname"] and symbol.normalized_path == normalized_path:
                            break
                    else:
                        symbol = (
                            db.session.query(Symbol)
                            .filter((Symbol.name == frame["funcname"]) & (Symbol.normalized_path == normalized_path))
                            .first()
                        )

                    # Create a new symbol if not found.
                    if not symbol:
                        symbol = Symbol()
                        symbol.name = frame["funcname"]

                        demangled = cpp_demangle(symbol.name)
                        if demangled != symbol.name:
                            symbol.nice_name = demangled

                        symbol.normalized_path = normalized_path
                        db.session.add(symbol)

                    symbolsource.symbol = symbol

                db.session.add(symbolsource)

            report_btframe.symbolsource = symbolsource
            db.session.add(report_btframe)
    else:
        report.count += count
        if report.last_occurence < utctime:
            report.last_occurence = utctime
        elif report.first_occurence > utctime:
            report.first_occurence = utctime

        if report.problem:
            if report.problem.last_occurence < report.last_occurence:
                report.problem.last_occurence = report.last_occurence
            if report.problem.first_occurence > report.first_occurence:
                report.problem.first_occurence = report.first_occurence

    db.session.flush()
    if report.type == "KERNELOOPS":
        if not report.get_lob_fd("oops") and "oops" in ureport:
            report.save_lob("oops", ureport["oops"])

    # Update various stats.

    opsysrelease = (
        db.session.query(OpSysRelease)
        .join(OpSys)
        .filter((OpSysRelease.version == ureport["os"]["version"]) & (OpSys.name == ureport["os"]["name"]))
        .one()
    )

    arch = db.session.query(Arch).filter_by(name=ureport["architecture"]).one()

    day = utctime.date()
    week = day - datetime.timedelta(days=day.weekday())
    month = day.replace(day=1)

    stat_map = [
        (ReportArch, [("arch", arch)]),
        (ReportOpSysRelease, [("opsysrelease", opsysrelease)]),
        (ReportReason, [("reason", ureport["reason"])]),
        (ReportHistoryMonthly, [("opsysrelease", opsysrelease), ("month", month)]),
        (ReportHistoryWeekly, [("opsysrelease", opsysrelease), ("week", week)]),
        (ReportHistoryDaily, [("opsysrelease", opsysrelease), ("day", day)]),
    ]

    if "executable" in ureport:
        stat_map.append((ReportExecutable, [("path", ureport["executable"])]))

    if "uptime" in ureport:
        if ureport["uptime"] < 0.1:
            uptime_exp = -1
        else:
            uptime_exp = int(math.log(ureport["uptime"], 10))
        stat_map.append((ReportUptime, [("uptime_exp", uptime_exp)]))

    # Add the reported package (installed and running).
    stat_map.append(get_package_stat("CRASHED", ureport, ureport["os"], db))

    # Similarly add related packages.
    if "related_packages" in ureport:
        for related_package in ureport["related_packages"]:
            stat_map.append(get_package_stat("RELATED", related_package, ureport["os"], db))

    # Add selinux fields to stat_map
    if "selinux" in ureport:
        stat_map.append((ReportSelinuxMode, [("mode", ureport["selinux"]["mode"].upper())]))

        if "context" in ureport["selinux"]:
            stat_map.append((ReportSelinuxContext, [("context", ureport["selinux"]["context"])]))

        if "policy_package" in ureport["selinux"]:
            stat_map.append(
                get_package_stat(
                    "SELINUX_POLICY", {"installed_package": ureport["selinux"]["policy_package"]}, ureport["os"], db
                )
            )

    # Add kernel taint state fields to stat_map.
    if "kernel_taint_state" in ureport:
        stat_map.append((ReportKernelTaintState, [("state", ureport["kernel_taint_state"])]))

    # Create missing stats and increase counters.
    for table, cols in stat_map:
        report_stat_query = db.session.query(table).join(Report).filter(Report.id == report.id)
        for name, value in cols:
            report_stat_query = report_stat_query.filter(getattr(table, name) == value)

        report_stat = report_stat_query.first()
        if not report_stat:
            report_stat = table()
            report_stat.report = report
            for name, value in cols:
                setattr(report_stat, name, value)
            report_stat.count = 0
            db.session.add(report_stat)
        report_stat.count += count