Beispiel #1
0
    def problems(self, cmdline, db, opsys, release):
        """
        Get hot/long-term problem statistics
        """

        release_ids = get_release_ids(db, opsys, release)

        num_days = 7
        if cmdline.last:
            num_days = int(cmdline.last)

        since = datetime.datetime.now() - datetime.timedelta(days=num_days)

        hot = query_hot_problems(db, release_ids, history=self.history_type,
                                 last_date=since)

        if not cmdline.include_low_quality:
            hot = filter(lambda x: x.quality >= 0, hot)

        out = ""
        if hot:
            out += "Hot problems:\n\n"
            out += self._render_problems(hot, cmdline.count, release_ids)
            out += "\n"
            if webfaf_installed():
                out += "URL: "
                out += reverse("webfaf.problems.views.hot")
                out += "\n\n"

        lt = query_longterm_problems(db, release_ids, history=self.history_type)
        if not cmdline.include_low_quality:
            lt = filter(lambda x: x.quality >= 0, lt)

        if lt:
            out += "Long-term problems:\n\n"
            out += self._render_problems(lt, cmdline.count, release_ids)
            out += "\n"
            if webfaf_installed():
                out += "URL: "
                out += reverse("webfaf.problems.views.longterm")
                out += "\n\n"

        return out
Beispiel #2
0
    def problems(self, cmdline, db, opsys, release):
        """
        Get hot/long-term problem statistics
        """

        release_ids = get_release_ids(db, opsys, release)

        num_days = 7
        if cmdline.last:
            num_days = int(cmdline.last)

        since = datetime.datetime.now() - datetime.timedelta(days=num_days)

        hot = query_hot_problems(db,
                                 release_ids,
                                 history=self.history_type,
                                 last_date=since)

        if not cmdline.include_low_quality:
            hot = [x for x in hot if x.quality >= 0]
        hot = [p for p in hot if p.type in self.ptypes]

        out = ""
        if hot:
            out += "Hot problems:\n\n"
            out += self._render_problems(hot, cmdline.count, release_ids)
            out += "\n"
            if webfaf_installed():
                out += "URL: "
                out += reverse("problems.dashboard")
                out += "\n\n"

        lt = query_longterm_problems(db,
                                     release_ids,
                                     history=self.history_type)
        if not cmdline.include_low_quality:
            lt = [x for x in lt if x.quality >= 0]
        lt = [p for p in lt if p.type in self.ptypes]

        if lt:
            out += "Long-term problems:\n\n"
            out += self._render_problems(lt, cmdline.count, release_ids)
            out += "\n\n"

        return out
Beispiel #3
0
    def problems(self, cmdline, db, opsys, release):
        """
        Get hot/long-term problem statistics
        """

        release_ids = get_release_ids(db, opsys, release)

        num_days = 7
        if cmdline.last:
            num_days = int(cmdline.last)

        since = datetime.datetime.now() - datetime.timedelta(days=num_days)

        hot = query_hot_problems(db, release_ids, history=self.history_type,
                                 last_date=since)

        if not cmdline.include_low_quality:
            hot = [x for x in hot if x.quality >= 0]
        hot = [p for p in hot if p.type in self.ptypes]

        out = ""
        if hot:
            out += "Hot problems:\n\n"
            out += self._render_problems(hot, cmdline.count, release_ids)
            out += "\n"
            if webfaf_installed():
                out += "URL: "
                out += reverse("problems.dashboard")
                out += "\n\n"

        lt = query_longterm_problems(db, release_ids, history=self.history_type)
        if not cmdline.include_low_quality:
            lt = [x for x in lt if x.quality >= 0]
        lt = [p for p in lt if p.type in self.ptypes]

        if lt:
            out += "Long-term problems:\n\n"
            out += self._render_problems(lt, cmdline.count, release_ids)
            out += "\n\n"

        return out
Beispiel #4
0
    def run(self, cmdline, db):
        levels = tuple(10**n for n in range(7))
        fedmsg.init(name=self.fedmsg_name, environment=self.fedmsg_environment)
        if cmdline.reports:
            # Sum of counts until yesterday
            q_yesterday = (
                db.session
                .query(Report.id.label("y_report_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day < cmdline.date)
                .group_by(Report.id)
                .subquery()
                )
            # Sum of counts until today
            q_today = (
                db.session
                .query(Report.id.label("t_report_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_today"))
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day <= cmdline.date)
                .group_by(Report.id)
                .subquery()
                )
            q = (db.session.query(Report,
                                  q_today.c.sum_today,
                                  q_yesterday.c.sum_yesterday)
                 .outerjoin(q_today, Report.id == q_today.c.t_report_id)
                 .outerjoin(q_yesterday, Report.id == q_yesterday.c.y_report_id)
                 .filter(or_(Report.max_certainty.isnot(None), Report.max_certainty != 100))
                 .filter(or_(and_(q_yesterday.c.sum_yesterday == None,
                                  q_today.c.sum_today != None),
                             q_today.c.sum_today != q_yesterday.c.sum_yesterday))
                 )

            for db_report, sum_today, sum_yesterday in q.yield_per(100):
                # avoid None
                sum_yesterday = sum_yesterday or 0

                for level in levels:
                    if sum_yesterday < level and sum_today >= level:
                        self.log_info("Notifying about report #{0} level {1}"
                                      .format(db_report.id, level))
                        msg = {
                            "report_id": db_report.id,
                            "function": db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence": db_report.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": sum_today,
                            "type": db_report.type,
                            "level": level,
                        }
                        if web.webfaf_installed():
                            msg["url"] = web.reverse("reports.item",
                                                     report_id=db_report.id)
                        if db_report.problem_id:
                            msg["problem_id"] = db_report.problem_id

                        fedmsg.publish(
                            topic="report.threshold{0}".format(level),
                            modname='faf',
                            msg=msg)

        if cmdline.problems:
            # Sum of counts until yesterday
            q_yesterday = (
                db.session
                .query(Problem.id.label("y_problem_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
                .join(Report)
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day < cmdline.date)
                .group_by(Problem.id)
                .subquery()
                )
            # Sum of counts until today
            q_today = (
                db.session
                .query(Problem.id.label("t_problem_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_today"))
                .join(Report)
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day <= cmdline.date)
                .group_by(Problem.id)
                .subquery()
                )
            q = (db.session
                 .query(Problem, q_today.c.sum_today, q_yesterday.c.sum_yesterday)
                 .outerjoin(q_today, Problem.id == q_today.c.t_problem_id)
                 .outerjoin(q_yesterday, Problem.id == q_yesterday.c.y_problem_id)
                 .filter(or_(and_(q_yesterday.c.sum_yesterday == None,
                                  q_today.c.sum_today != None),
                             q_today.c.sum_today != q_yesterday.c.sum_yesterday))
                 )

            for db_problem, sum_today, sum_yesterday in q.yield_per(100):
                # avoid None
                sum_yesterday = sum_yesterday or 0

                for level in levels:
                    if sum_yesterday < level and sum_today >= level:
                        self.log_info("Notifying about problem #{0} level {1}"
                                      .format(db_problem.id, level))
                        msg = {
                            "problem_id": db_problem.id,
                            "function": db_problem.crash_function,
                            "components": db_problem.unique_component_names,
                            "first_occurrence": db_problem.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": sum_today,
                            "type": db_problem.type,
                            "level": level,
                        }
                        if web.webfaf_installed():
                            msg["url"] = web.reverse("problems.item",
                                                     problem_id=db_problem.id)
                        fedmsg.publish(
                            topic="problem.threshold{0}".format(level),
                            modname='faf',
                            msg=msg)
Beispiel #5
0
    def components(self, cmdline, db, opsys, release):
        """
        Get statistics for most crashing components
        """

        hist_table, hist_field = get_history_target(self.history_type)
        total = get_history_sum(db, opsys, release)
        comps = get_report_count_by_component(db, opsys, release)

        if cmdline.last:
            now = datetime.datetime.now()
            since = now - datetime.timedelta(days=int(cmdline.last))
            comps = comps.filter(hist_field >= since)
            total = total.filter(hist_field >= since)

        total_num = total.first()[0]

        limit = int(cmdline.count)
        limit_details = int(cmdline.detail_count)
        results = []

        for num, (comp, count) in enumerate(comps):
            if num >= limit:
                break

            if comp in self.comps_filter:
                continue

            reports = get_report_stats_by_component(db, comp, opsys, release,
                                                    self.history_type)

            if cmdline.last:
                reports = reports.filter(hist_field >= since)

            problem_ids = set()
            attached_reports = []

            for report, report_count in reports:
                if len(problem_ids) >= limit_details:
                    break
                if not report.problem:
                    continue
                if report.problem.id in problem_ids:
                    continue
                if report.quality < 0 and not cmdline.include_low_quality:
                    continue

                problem_ids.add(report.problem.id)

                problem_url = ""
                if webfaf_installed():
                    problem_url = reverse("problems.item",
                                          problem_id=report.problem.id)

                attached_reports.append((problem_url, report.bugs))

            results.append((comp, count, attached_reports))

        if not results:
            return ""

        out = "Components:\n\n"

        for num, (comp, count, reports) in enumerate(results):
            if reports:
                out += ("{0}. {1} seen {2} times ({3:.0%} of all reports)\n".
                        format(num + 1, comp, count, count / float(total_num)))

                for problem_url, bugs in reports:
                    if problem_url or bugs:
                        out += "    {0} {1}\n".format(
                            problem_url, ", ".join(map(str, bugs)))

        return out
Beispiel #6
0
    def text_overview(self, cmdline, db, opsys, release):
        release_ids = get_release_ids(db, opsys, release)

        num_days = 7
        if cmdline.last:
            num_days = int(cmdline.last)

        since = datetime.datetime.now() - datetime.timedelta(days=num_days)

        hot = query_hot_problems(db,
                                 release_ids,
                                 history=self.history_type,
                                 last_date=since)

        if not cmdline.include_low_quality:
            hot = [x for x in hot if x.quality >= 0]

        ptypes = ""
        if len(self.ptypes) != len(problemtypes):
            ptypes = " " + ", ".join(self.ptypes)
        out = "Overview of the top {0}{1} crashes over the last {2} days:\n".format(
            cmdline.count, ptypes, num_days)

        hot = [p for p in hot if p.type in self.ptypes]

        for (rank, problem) in enumerate(hot[:cmdline.count]):
            out += "#{0} {1} - {2}x\n".format(
                rank + 1, ', '.join(problem.unique_component_names),
                problem.count)

            # Reports with bugzillas for this OpSysRelease go first
            reports = sorted(
                problem.reports,
                cmp=lambda x, y: len(
                    [b for b in x.bugs if b.opsysrelease_id in release_ids]) -
                len([b for b in y.bugs if b.opsysrelease_id in release_ids]),
                reverse=True)

            if webfaf_installed():
                for report in reports[:3]:
                    out += "{0}\n".format(
                        reverse("reports.bthash_forward",
                                bthash=report.hashes[0].hash))
                    for bug in report.bugs:
                        out += "  {0}\n".format(bug.url)
            else:
                for report in reports[:3]:
                    out += "Report BT hash: {0}\n".format(
                        report.hashes[0].hash)
            if len(problem.reports) > 3:
                out += "... and {0} more.\n".format(len(problem.reports) - 3)

            if problem.tainted:
                out += "Kernel tainted.\n"

            crash_function = problem.crash_function
            if crash_function:
                out += "Crash function: {0}\n".format(crash_function)

            affected_all = []
            for report in problem.reports:
                affected_known = [
                    (affected.build.base_package_name, affected.build.epoch,
                     affected.build.version, affected.build.release)
                    for affected in get_crashed_package_for_report(
                        db, report.id)
                ]

                affected_unknown = \
                    get_crashed_unknown_package_nevr_for_report(db, report.id)

                affected_all += affected_known + affected_unknown
            affected_all = sorted(set(affected_all),
                                  cmp=lambda a, b: cmp_evr(a[1:], b[1:]),
                                  reverse=True)

            if affected_all:
                out += "Affected builds: {0}".format(", ".join([
                    "{0}-{1}:{2}-{3}".format(n, e, v, r)
                    for (n, e, v, r) in affected_all[:5]
                ]))
                if len(problem.reports) > 5:
                    out += " and {0} more.".format(len(problem.reports) - 5)
                out += "\n"

            pfix = problem.probable_fix_for_opsysrelease_ids(release_ids)
            if len(pfix) > 0:
                out += ("Problem seems to be fixed since the release of {0}\n".
                        format(pfix))
            out += "\n"

        return out
Beispiel #7
0
    def run(self, cmdline, db) -> None:
        levels = tuple(10**n for n in range(7))
        if cmdline.reports:
            # Sum of counts until yesterday
            q_yesterday = (db.session.query(
                Report.id.label("y_report_id"),
                func.sum(ReportHistoryDaily.count).label(
                    "sum_yesterday")).outerjoin(ReportHistoryDaily).filter(
                        ReportHistoryDaily.day < cmdline.date).group_by(
                            Report.id).subquery())
            # Sum of counts until today
            q_today = (db.session.query(
                Report.id.label("t_report_id"),
                func.sum(ReportHistoryDaily.count).label(
                    "sum_today")).outerjoin(ReportHistoryDaily).filter(
                        ReportHistoryDaily.day <= cmdline.date).group_by(
                            Report.id).subquery())
            q = (db.session.query(
                Report, q_today.c.sum_today,
                q_yesterday.c.sum_yesterday).outerjoin(
                    q_today, Report.id == q_today.c.t_report_id).outerjoin(
                        q_yesterday,
                        Report.id == q_yesterday.c.y_report_id).filter(
                            or_(Report.max_certainty.isnot(None),
                                Report.max_certainty != 100)).
                 filter(
                     or_(
                         and_(q_yesterday.c.sum_yesterday.is_(None),
                              q_today.c.sum_today.isnot(None)),
                         q_today.c.sum_today != q_yesterday.c.sum_yesterday)))

            for db_report, sum_today, sum_yesterday in q.yield_per(100):
                # avoid None
                sum_yesterday = sum_yesterday or 0

                for level in levels:
                    if sum_yesterday < level <= sum_today:
                        self.log_info(
                            "Notifying about report #{0} level {1}".format(
                                db_report.id, level))
                        msg_body = {
                            "report_id":
                            db_report.id,
                            "function":
                            db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence":
                            db_report.first_occurrence.strftime(DATE_FORMAT),
                            "count":
                            sum_today,
                            "type":
                            db_report.type,
                            "level":
                            level,
                        }
                        if web.webfaf_installed():
                            msg_body["url"] = web.reverse(
                                "reports.item", report_id=db_report.id)

                        if db_report.problem_id:
                            msg_body["problem_id"] = db_report.problem_id

                        try:
                            msg = FafReportMessage(
                                topic="faf.report.threshold{0}".format(level),
                                body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            self.log_warn(
                                "Fedora Messaging broker rejected message {0}: {1}"
                                .format(msg.id, e))
                        except ConnectionException as e:
                            self.log_warn(
                                "Error sending message {0}: {1}".format(
                                    msg.id, e))

        if cmdline.problems:
            # Sum of counts until yesterday
            q_yesterday = (db.session.query(
                Problem.id.label("y_problem_id"),
                func.sum(
                    ReportHistoryDaily.count).label("sum_yesterday")).join(
                        Report, Report.problem_id ==
                        Problem.id).outerjoin(ReportHistoryDaily).filter(
                            ReportHistoryDaily.day < cmdline.date).group_by(
                                Problem.id).subquery())
            # Sum of counts until today
            q_today = (db.session.query(
                Problem.id.label("t_problem_id"),
                func.sum(ReportHistoryDaily.count).label("sum_today")).join(
                    Report, Report.problem_id == Problem.id).outerjoin(
                        ReportHistoryDaily).filter(
                            ReportHistoryDaily.day <= cmdline.date).group_by(
                                Problem.id).subquery())
            q = (db.session.query(
                Problem, q_today.c.sum_today,
                q_yesterday.c.sum_yesterday).outerjoin(
                    q_today, Problem.id == q_today.c.t_problem_id).outerjoin(
                        q_yesterday,
                        Problem.id == q_yesterday.c.y_problem_id).filter(
                            or_(
                                and_(q_yesterday.c.sum_yesterday.is_(None),
                                     q_today.c.sum_today.isnot(None)),
                                q_today.c.sum_today !=
                                q_yesterday.c.sum_yesterday)))

            for db_problem, sum_today, sum_yesterday in q.yield_per(100):
                # avoid None
                sum_yesterday = sum_yesterday or 0

                for level in levels:
                    if sum_yesterday < level <= sum_today:
                        self.log_info(
                            "Notifying about problem #{0} level {1}".format(
                                db_problem.id, level))
                        msg_body = {
                            "problem_id":
                            db_problem.id,
                            "function":
                            db_problem.crash_function,
                            "components":
                            list(db_problem.unique_component_names),
                            "first_occurrence":
                            db_problem.first_occurrence.strftime(DATE_FORMAT),
                            "count":
                            sum_today,
                            "type":
                            db_problem.type,
                            "level":
                            level,
                        }
                        if web.webfaf_installed():
                            msg_body["url"] = web.reverse(
                                "problems.item", problem_id=db_problem.id)

                        try:
                            msg = FafProblemMessage(
                                topic="faf.problem.threshold{0}".format(level),
                                body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            self.log_warn(
                                "Fedora Messaging broker rejected message {0}: {1}"
                                .format(msg.id, e))
                        except ConnectionException as e:
                            self.log_warn(
                                "Error sending message {0}: {1}".format(
                                    msg.id, e))
Beispiel #8
0
    def fedmsg_report(target, value, oldvalue, initiator):
        """
        Send fedmsg notifications when Report.count reaches specified threshold.
        """
        try:
            db_report = target
            if notify_reports:
                oldcount = oldvalue
                newcount = value
                for level in levels:
                    if oldcount < level and newcount >= level:
                        logger.info(
                            "Notifying about report #{0} level {1}".format(
                                db_report.id, level))
                        msg = {
                            "report_id":
                            db_report.id,
                            "function":
                            db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence":
                            db_report.first_occurrence.strftime("%Y-%m-%d"),
                            "count":
                            newcount,
                            "type":
                            db_report.type,
                            "level":
                            level,
                        }
                        if web.webfaf_installed() and db_report.hashes:
                            msg["url"] = web.reverse(
                                "reports.bthash_forward",
                                bthash=db_report.hashes[0].hash)
                        if db_report.problem_id:
                            msg["problem_id"] = db_report.problem_id

                        fedmsg.publish(
                            topic="report.threshold{0}".format(level),
                            modname='faf',
                            msg=msg)

            if notify_problems and db_report.problem is not None:
                oldcount = db_report.problem.reports_count
                newcount = oldcount + value - oldvalue
                for level in levels:
                    if oldcount < level and newcount >= level:
                        logger.info(
                            "Notifying about problem #{0} level {1}".format(
                                db_report.problem.id, level))
                        msg = {
                            "problem_id":
                            db_report.problem.id,
                            "function":
                            db_report.problem.crash_function,
                            "components":
                            db_report.problem.unique_component_names,
                            "first_occurrence":
                            db_report.problem.first_occurrence.strftime(
                                "%Y-%m-%d"),
                            "count":
                            newcount,
                            "type":
                            db_report.type,
                            "level":
                            level,
                        }
                        if web.webfaf_installed():
                            msg["url"] = web.reverse(
                                "problems.item",
                                problem_id=db_report.problem.id)
                        fedmsg.publish(
                            topic="problem.threshold{0}".format(level),
                            modname='faf',
                            msg=msg)
        # Catch any exception. This is non-critical and mustn't break stuff
        # elsewhere.
        except Exception as e:
            logger.exception(e, exc_info=True)
Beispiel #9
0
    def fedmsg_report(target, value, oldvalue, initiator) -> None:  # pylint: disable=unused-argument
        """
        Send Fedora Messaging notifications when Report.count reaches specified threshold.
        """
        try:
            db_report = target
            if notify_reports:
                oldcount = oldvalue
                newcount = value
                for level in levels:
                    if oldcount < level <= newcount:
                        logger.info(
                            "Notifying about report #{0} level {1}".format(
                                db_report.id, level))
                        msg_body = {
                            "report_id":
                            db_report.id,
                            "function":
                            db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence":
                            db_report.first_occurrence.strftime("%Y-%m-%d"),
                            "count":
                            newcount,
                            "type":
                            db_report.type,
                            "level":
                            level,
                        }
                        if web.webfaf_installed() and db_report.hashes:
                            msg_body["url"] = web.reverse(
                                "reports.bthash_forward",
                                bthash=db_report.hashes[0].hash)

                        if db_report.problem_id:
                            msg_body["problem_id"] = db_report.problem_id

                        try:
                            msg = FafReportMessage(
                                topic="faf.report.threshold{0}".format(level),
                                body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            logger.exception(
                                "Fedora Messaging broker rejected message {0}: {1}"
                                .format(msg.id, e))
                        except ConnectionException as e:
                            logger.exception(
                                "Error sending message {0}: {1}".format(
                                    msg.id, e))

            if notify_problems and db_report.problem is not None:
                oldcount = db_report.problem.reports_count
                newcount = oldcount + value - oldvalue
                for level in levels:
                    if oldcount < level <= newcount:
                        logger.info(
                            "Notifying about problem #{0} level {1}".format(
                                db_report.problem.id, level))
                        msg_body = {
                            "problem_id":
                            db_report.problem.id,
                            "function":
                            db_report.problem.crash_function,
                            "components":
                            list(db_report.problem.unique_component_names),
                            "first_occurrence":
                            db_report.problem.first_occurrence.strftime(
                                "%Y-%m-%d"),
                            "count":
                            newcount,
                            "type":
                            db_report.type,
                            "level":
                            level,
                        }
                        if web.webfaf_installed():
                            msg_body["url"] = web.reverse(
                                "problems.item",
                                problem_id=db_report.problem.id)

                        try:
                            msg = FafProblemMessage(
                                topic="faf.problem.threshold{0}".format(level),
                                body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            logger.exception(
                                "Fedora Messaging broker rejected message {0}: {1}"
                                .format(msg.id, e))
                        except ConnectionException as e:
                            logger.exception(
                                "Error sending message {0}: {1}".format(
                                    msg.id, e))

        # Catch any exception. This is non-critical and mustn't break stuff
        # elsewhere.
        except Exception as e:  # pylint: disable=broad-except
            logger.exception(e, exc_info=True)
Beispiel #10
0
    def fedmsg_report(target, value, oldvalue, initiator): # pylint: disable=unused-argument
        """
        Send Fedora Messaging notifications when Report.count reaches specified threshold.
        """
        try:
            db_report = target
            if notify_reports:
                oldcount = oldvalue
                newcount = value
                for level in levels:
                    if oldcount < level <= newcount:
                        logger.info("Notifying about report #{0} level {1}"
                                    .format(db_report.id, level))
                        msg_body = {
                            "report_id": db_report.id,
                            "function": db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence": db_report.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": newcount,
                            "type": db_report.type,
                            "level": level,
                        }
                        if web.webfaf_installed() and db_report.hashes:
                            msg_body["url"] = web.reverse("reports.bthash_forward",
                                                          bthash=db_report.hashes[0].hash)

                        if db_report.problem_id:
                            msg_body["problem_id"] = db_report.problem_id

                        try:
                            msg = FafReportMessage(topic="faf.report.threshold{0}".format(level),
                                                   body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            logger.exception("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
                        except ConnectionException as e:
                            logger.exception("Error sending message {0}: {1}".format(msg.id, e))

            if notify_problems and db_report.problem is not None:
                oldcount = db_report.problem.reports_count
                newcount = oldcount + value - oldvalue
                for level in levels:
                    if oldcount < level <= newcount:
                        logger.info("Notifying about problem #{0} level {1}"
                                    .format(db_report.problem.id, level))
                        msg_body = {
                            "problem_id": db_report.problem.id,
                            "function": db_report.problem.crash_function,
                            "components": list(db_report.problem.unique_component_names),
                            "first_occurrence": db_report.problem.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": newcount,
                            "type": db_report.type,
                            "level": level,
                        }
                        if web.webfaf_installed():
                            msg_body["url"] = web.reverse("problems.item", problem_id=db_report.problem.id)

                        try:
                            msg = FafProblemMessage(topic="faf.problem.threshold{0}".format(level),
                                                    body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            logger.exception("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
                        except ConnectionException as e:
                            logger.exception("Error sending message {0}: {1}".format(msg.id, e))

        # Catch any exception. This is non-critical and mustn't break stuff
        # elsewhere.
        except Exception as e: # pylint: disable=broad-except
            logger.exception(e, exc_info=True)
Beispiel #11
0
    def components(self, cmdline, db, opsys, release):
        """
        Get statistics for most crashing components
        """

        hist_table, hist_field = get_history_target(self.history_type)
        total = get_history_sum(db, opsys, release)
        comps = get_report_count_by_component(db, opsys, release)

        if cmdline.last:
            now = datetime.datetime.now()
            since = now - datetime.timedelta(days=int(cmdline.last))
            comps = comps.filter(hist_field >= since)
            total = total.filter(hist_field >= since)

        total_num = total.first()[0]

        limit = int(cmdline.count)
        limit_details = int(cmdline.detail_count)
        results = []

        for num, (comp, count) in enumerate(comps):
            if num >= limit:
                break

            if comp in self.comps_filter:
                continue

            reports = get_report_stats_by_component(db, comp, opsys,
                                                    release,
                                                    self.history_type)

            if cmdline.last:
                reports = reports.filter(hist_field >= since)

            problem_ids = set()
            attached_reports = []

            for report, report_count in reports:
                if len(problem_ids) >= limit_details:
                    break
                if not report.problem:
                    continue
                if report.problem.id in problem_ids:
                    continue
                if report.quality < 0 and not cmdline.include_low_quality:
                    continue

                problem_ids.add(report.problem.id)

                problem_url = ""
                if webfaf_installed():
                    problem_url = reverse("problems.item",
                                          problem_id=report.problem.id)

                attached_reports.append((problem_url, report.bugs))

            results.append((comp, count, attached_reports))

        if not results:
            return ""

        out = "Components:\n\n"

        for num, (comp, count, reports) in enumerate(results):
            if reports:
                out += ("{0}. {1} seen {2} times ({3:.0%} of all reports)\n"
                        .format(num + 1, comp, count, count / float(total_num)))

                for problem_url, bugs in reports:
                    if problem_url or bugs:
                        out += "    {0} {1}\n".format(
                            problem_url, ", ".join(map(str, bugs)))

        return out
Beispiel #12
0
    def text_overview(self, cmdline, db, opsys, release):
        release_ids = get_release_ids(db, opsys, release)

        num_days = 7
        if cmdline.last:
            num_days = int(cmdline.last)

        since = datetime.datetime.now() - datetime.timedelta(days=num_days)

        hot = query_hot_problems(db, release_ids, history=self.history_type,
                                 last_date=since)

        if not cmdline.include_low_quality:
            hot = [x for x in hot if x.quality >= 0]

        ptypes = ""
        if len(self.ptypes) != len(problemtypes):
            ptypes = " "+", ".join(self.ptypes)
        out = "Overview of the top {0}{1} crashes over the last {2} days:\n".format(
            cmdline.count, ptypes, num_days)

        hot = [p for p in hot if p.type in self.ptypes]

        for (rank, problem) in enumerate(hot[:cmdline.count]):
            out += "#{0} {1} - {2}x\n".format(
                rank+1,
                ', '.join(problem.unique_component_names),
                problem.count)

            # Reports with bugzillas for this OpSysRelease go first
            reports = sorted(problem.reports,
                             cmp=lambda x, y: len([b for b in x.bugs if b.opsysrelease_id in release_ids])
                             - len([b for b in y.bugs if b.opsysrelease_id in release_ids]), reverse=True)

            if webfaf_installed():
                for report in reports[:3]:
                    out += "{0}\n".format(reverse("reports.bthash_forward",
                                                  bthash=report.hashes[0].hash))
                    for bug in report.bugs:
                        out += "  {0}\n".format(bug.url)
            else:
                for report in reports[:3]:
                    out += "Report BT hash: {0}\n".format(report.hashes[0].hash)
            if len(problem.reports) > 3:
                out += "... and {0} more.\n".format(len(problem.reports)-3)

            if problem.tainted:
                out += "Kernel tainted.\n"

            crash_function = problem.crash_function
            if crash_function:
                out += "Crash function: {0}\n".format(crash_function)

            affected_all = []
            for report in problem.reports:
                affected_known = [
                    (affected.build.base_package_name,
                     affected.build.epoch,
                     affected.build.version,
                     affected.build.release) for affected in
                    get_crashed_package_for_report(db, report.id)]

                affected_unknown = \
                    get_crashed_unknown_package_nevr_for_report(db, report.id)

                affected_all += affected_known + affected_unknown
            affected_all = sorted(set(affected_all),
                                  cmp=lambda a, b: cmp_evr(a[1:], b[1:]),
                                  reverse=True)

            if affected_all:
                out += "Affected builds: {0}".format(", ".join(
                    ["{0}-{1}:{2}-{3}".format(n, e, v, r)
                     for (n, e, v, r) in affected_all[:5]]))
                if len(problem.reports) > 5:
                    out += " and {0} more.".format(len(problem.reports)-5)
                out += "\n"

            pfix = problem.probable_fix_for_opsysrelease_ids(release_ids)
            if pfix:
                out += ("Problem seems to be fixed since the release of {0}\n"
                        .format(pfix))
            out += "\n"

        return out
Beispiel #13
0
    def text_overview(self, cmdline, db, opsys, release):
        release_ids = get_release_ids(db, opsys, release)

        num_days = 7
        if cmdline.last:
            num_days = int(cmdline.last)

        since = datetime.datetime.now() - datetime.timedelta(days=num_days)

        hot = query_hot_problems(db, release_ids, history=self.history_type,
                                 last_date=since)

        if not cmdline.include_low_quality:
            hot = filter(lambda x: x.quality >= 0, hot)

        out = "Overview of the top {0} crashes over the last {1} days:\n".format(
            cmdline.count, num_days)

        for (rank, problem) in enumerate(hot[:cmdline.count]):
            out += "#{0} {1} - {2}x\n".format(
                rank+1,
                ', '.join(problem.unique_component_names),
                problem.count)
            if webfaf_installed():
                for report in problem.reports:
                    out += "{0}\n".format(reverse("webfaf.reports.views.bthash_forward",
                                          args=[report.hashes[0].hash]))
            else:
                for report in problem.reports:
                    out += "Report BT hash: {0}\n".format(report.hashes[0].hash)

            if problem.tainted:
                out += "Kernel tainted.\n"

            crash_function = problem.crash_function
            if crash_function:
                out += "Crash function: {0}\n".format(crash_function)

            affected_all = []
            for report in problem.reports:
                affected_known = [
                    (affected.build.base_package_name,
                     affected.build.epoch,
                     affected.build.version,
                     affected.build.release) for affected in
                    get_crashed_package_for_report(db, report.id)]

                affected_unknown = \
                    get_crashed_unknown_package_nevr_for_report(db, report.id)

                affected_all += affected_known + affected_unknown
            affected_all = sorted(set(affected_all), cmp=lambda a, b: cmp_evr(a[1:], b[1:]))

            if affected_all:
                out += "Affected builds: {0}\n".format(", ".join(
                    ["{0}-{1}:{2}-{3}".format(n, e, v, r)
                     for (n, e, v, r) in affected_all]))

            pfix = problem.probable_fix_for_opsysrelease_ids(release_ids)
            if len(pfix) > 0:
                out += ("Problem seems to be fixed since the release of {0}\n"
                        .format(pfix))
            out += "\n"

        return out
Beispiel #14
0
    def fedmsg_report(target, value, oldvalue, initiator):
        """
        Send fedmsg notifications when Report.count reaches specified threshold.
        """
        try:
            db_report = target
            if notify_reports:
                oldcount = oldvalue
                newcount = value
                for level in levels:
                    if oldcount < level and newcount >= level:
                        logger.info("Notifying about report #{0} level {1}"
                                    .format(db_report.id, level))
                        msg = {
                            "report_id": db_report.id,
                            "function": db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence": db_report.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": newcount,
                            "type": db_report.type,
                            "level": level,
                        }
                        if web.webfaf_installed() and db_report.hashes:
                            msg["url"] = web.reverse("reports.bthash_forward",
                                                     bthash=db_report.hashes[0].hash)
                        if db_report.problem_id:
                            msg["problem_id"] = db_report.problem_id

                        fedmsg.publish(
                            topic="report.threshold{0}".format(level),
                            modname='faf',
                            msg=msg)

            if notify_problems and db_report.problem is not None:
                oldcount = db_report.problem.reports_count
                newcount = oldcount + value - oldvalue
                for level in levels:
                    if oldcount < level and newcount >= level:
                        logger.info("Notifying about problem #{0} level {1}"
                                    .format(db_report.problem.id, level))
                        msg = {
                            "problem_id": db_report.problem.id,
                            "function": db_report.problem.crash_function,
                            "components": db_report.problem.unique_component_names,
                            "first_occurrence": db_report.problem.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": newcount,
                            "type": db_report.type,
                            "level": level,
                        }
                        if web.webfaf_installed():
                            msg["url"] = web.reverse("problems.item",
                                                     problem_id=db_report.problem.id)
                        fedmsg.publish(
                            topic="problem.threshold{0}".format(level),
                            modname='faf',
                            msg=msg)
        # Catch any exception. This is non-critical and mustn't break stuff
        # elsewhere.
        except Exception as e:
            logger.exception(e, exc_info=True)
Beispiel #15
0
    def run(self, cmdline, db):
        levels = tuple(10**n for n in range(7))
        if cmdline.reports:
            # Sum of counts until yesterday
            q_yesterday = (
                db.session
                .query(Report.id.label("y_report_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day < cmdline.date)
                .group_by(Report.id)
                .subquery()
                )
            # Sum of counts until today
            q_today = (
                db.session
                .query(Report.id.label("t_report_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_today"))
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day <= cmdline.date)
                .group_by(Report.id)
                .subquery()
                )
            q = (db.session.query(Report,
                                  q_today.c.sum_today,
                                  q_yesterday.c.sum_yesterday)
                 .outerjoin(q_today, Report.id == q_today.c.t_report_id)
                 .outerjoin(q_yesterday, Report.id == q_yesterday.c.y_report_id)
                 .filter(or_(Report.max_certainty.isnot(None), Report.max_certainty != 100))
                 .filter(or_(and_(q_yesterday.c.sum_yesterday.is_(None),
                                  q_today.c.sum_today.isnot(None)),
                             q_today.c.sum_today != q_yesterday.c.sum_yesterday))
                )

            for db_report, sum_today, sum_yesterday in q.yield_per(100):
                # avoid None
                sum_yesterday = sum_yesterday or 0

                for level in levels:
                    if sum_yesterday < level <= sum_today:
                        self.log_info("Notifying about report #{0} level {1}"
                                      .format(db_report.id, level))
                        msg_body = {
                            "report_id": db_report.id,
                            "function": db_report.crash_function,
                            "components": [db_report.component.name],
                            "first_occurrence": db_report.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": sum_today,
                            "type": db_report.type,
                            "level": level,
                        }
                        if web.webfaf_installed():
                            msg_body["url"] = web.reverse("reports.item",
                                                          report_id=db_report.id)

                        if db_report.problem_id:
                            msg_body["problem_id"] = db_report.problem_id

                        try:
                            msg = FafReportMessage(topic="faf.report.threshold{0}".format(level),
                                                   body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            self.log_warn("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
                        except ConnectionException as e:
                            self.log_warn("Error sending message {0}: {1}".format(msg.id, e))

        if cmdline.problems:
            # Sum of counts until yesterday
            q_yesterday = (
                db.session
                .query(Problem.id.label("y_problem_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
                .join(Report, Report.problem_id == Problem.id)
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day < cmdline.date)
                .group_by(Problem.id)
                .subquery()
                )
            # Sum of counts until today
            q_today = (
                db.session
                .query(Problem.id.label("t_problem_id"),
                       func.sum(ReportHistoryDaily.count).label("sum_today"))
                .join(Report, Report.problem_id == Problem.id)
                .outerjoin(ReportHistoryDaily)
                .filter(ReportHistoryDaily.day <= cmdline.date)
                .group_by(Problem.id)
                .subquery()
                )
            q = (db.session
                 .query(Problem, q_today.c.sum_today, q_yesterday.c.sum_yesterday)
                 .outerjoin(q_today, Problem.id == q_today.c.t_problem_id)
                 .outerjoin(q_yesterday, Problem.id == q_yesterday.c.y_problem_id)
                 .filter(or_(and_(q_yesterday.c.sum_yesterday.is_(None),
                                  q_today.c.sum_today.isnot(None)),
                             q_today.c.sum_today != q_yesterday.c.sum_yesterday))
                )

            for db_problem, sum_today, sum_yesterday in q.yield_per(100):
                # avoid None
                sum_yesterday = sum_yesterday or 0

                for level in levels:
                    if sum_yesterday < level <= sum_today:
                        self.log_info("Notifying about problem #{0} level {1}"
                                      .format(db_problem.id, level))
                        msg_body = {
                            "problem_id": db_problem.id,
                            "function": db_problem.crash_function,
                            "components": list(db_problem.unique_component_names),
                            "first_occurrence": db_problem.first_occurrence
                                                .strftime("%Y-%m-%d"),
                            "count": sum_today,
                            "type": db_problem.type,
                            "level": level,
                        }
                        if web.webfaf_installed():
                            msg_body["url"] = web.reverse("problems.item",
                                                          problem_id=db_problem.id)

                        try:
                            msg = FafProblemMessage(topic="faf.problem.threshold{0}".format(level),
                                                    body=msg_body)
                            publish(msg)
                        except PublishReturned as e:
                            self.log_warn("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
                        except ConnectionException as e:
                            self.log_warn("Error sending message {0}: {1}".format(msg.id, e))