def problems(self, cmdline, db, opsys, release):
"""
Get hot/long-term problem statistics
"""
release_ids = get_release_ids(db, opsys, release)
num_days = 7
if cmdline.last:
num_days = int(cmdline.last)
since = datetime.datetime.now() - datetime.timedelta(days=num_days)
hot = query_hot_problems(db, release_ids, history=self.history_type,
last_date=since)
if not cmdline.include_low_quality:
hot = filter(lambda x: x.quality >= 0, hot)
out = ""
if hot:
out += "Hot problems:\n\n"
out += self._render_problems(hot, cmdline.count, release_ids)
out += "\n"
if webfaf_installed():
out += "URL: "
out += reverse("webfaf.problems.views.hot")
out += "\n\n"
lt = query_longterm_problems(db, release_ids, history=self.history_type)
if not cmdline.include_low_quality:
lt = filter(lambda x: x.quality >= 0, lt)
if lt:
out += "Long-term problems:\n\n"
out += self._render_problems(lt, cmdline.count, release_ids)
out += "\n"
if webfaf_installed():
out += "URL: "
out += reverse("webfaf.problems.views.longterm")
out += "\n\n"
return out
def problems(self, cmdline, db, opsys, release):
"""
Get hot/long-term problem statistics
"""
release_ids = get_release_ids(db, opsys, release)
num_days = 7
if cmdline.last:
num_days = int(cmdline.last)
since = datetime.datetime.now() - datetime.timedelta(days=num_days)
hot = query_hot_problems(db,
release_ids,
history=self.history_type,
last_date=since)
if not cmdline.include_low_quality:
hot = [x for x in hot if x.quality >= 0]
hot = [p for p in hot if p.type in self.ptypes]
out = ""
if hot:
out += "Hot problems:\n\n"
out += self._render_problems(hot, cmdline.count, release_ids)
out += "\n"
if webfaf_installed():
out += "URL: "
out += reverse("problems.dashboard")
out += "\n\n"
lt = query_longterm_problems(db,
release_ids,
history=self.history_type)
if not cmdline.include_low_quality:
lt = [x for x in lt if x.quality >= 0]
lt = [p for p in lt if p.type in self.ptypes]
if lt:
out += "Long-term problems:\n\n"
out += self._render_problems(lt, cmdline.count, release_ids)
out += "\n\n"
return out
def problems(self, cmdline, db, opsys, release):
"""
Get hot/long-term problem statistics
"""
release_ids = get_release_ids(db, opsys, release)
num_days = 7
if cmdline.last:
num_days = int(cmdline.last)
since = datetime.datetime.now() - datetime.timedelta(days=num_days)
hot = query_hot_problems(db, release_ids, history=self.history_type,
last_date=since)
if not cmdline.include_low_quality:
hot = [x for x in hot if x.quality >= 0]
hot = [p for p in hot if p.type in self.ptypes]
out = ""
if hot:
out += "Hot problems:\n\n"
out += self._render_problems(hot, cmdline.count, release_ids)
out += "\n"
if webfaf_installed():
out += "URL: "
out += reverse("problems.dashboard")
out += "\n\n"
lt = query_longterm_problems(db, release_ids, history=self.history_type)
if not cmdline.include_low_quality:
lt = [x for x in lt if x.quality >= 0]
lt = [p for p in lt if p.type in self.ptypes]
if lt:
out += "Long-term problems:\n\n"
out += self._render_problems(lt, cmdline.count, release_ids)
out += "\n\n"
return out
def run(self, cmdline, db):
levels = tuple(10**n for n in range(7))
fedmsg.init(name=self.fedmsg_name, environment=self.fedmsg_environment)
if cmdline.reports:
# Sum of counts until yesterday
q_yesterday = (
db.session
.query(Report.id.label("y_report_id"),
func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day < cmdline.date)
.group_by(Report.id)
.subquery()
)
# Sum of counts until today
q_today = (
db.session
.query(Report.id.label("t_report_id"),
func.sum(ReportHistoryDaily.count).label("sum_today"))
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day <= cmdline.date)
.group_by(Report.id)
.subquery()
)
q = (db.session.query(Report,
q_today.c.sum_today,
q_yesterday.c.sum_yesterday)
.outerjoin(q_today, Report.id == q_today.c.t_report_id)
.outerjoin(q_yesterday, Report.id == q_yesterday.c.y_report_id)
.filter(or_(Report.max_certainty.isnot(None), Report.max_certainty != 100))
.filter(or_(and_(q_yesterday.c.sum_yesterday == None,
q_today.c.sum_today != None),
q_today.c.sum_today != q_yesterday.c.sum_yesterday))
)
for db_report, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level and sum_today >= level:
self.log_info("Notifying about report #{0} level {1}"
.format(db_report.id, level))
msg = {
"report_id": db_report.id,
"function": db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence": db_report.first_occurrence
.strftime("%Y-%m-%d"),
"count": sum_today,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed():
msg["url"] = web.reverse("reports.item",
report_id=db_report.id)
if db_report.problem_id:
msg["problem_id"] = db_report.problem_id
fedmsg.publish(
topic="report.threshold{0}".format(level),
modname='faf',
msg=msg)
if cmdline.problems:
# Sum of counts until yesterday
q_yesterday = (
db.session
.query(Problem.id.label("y_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
.join(Report)
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day < cmdline.date)
.group_by(Problem.id)
.subquery()
)
# Sum of counts until today
q_today = (
db.session
.query(Problem.id.label("t_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_today"))
.join(Report)
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day <= cmdline.date)
.group_by(Problem.id)
.subquery()
)
q = (db.session
.query(Problem, q_today.c.sum_today, q_yesterday.c.sum_yesterday)
.outerjoin(q_today, Problem.id == q_today.c.t_problem_id)
.outerjoin(q_yesterday, Problem.id == q_yesterday.c.y_problem_id)
.filter(or_(and_(q_yesterday.c.sum_yesterday == None,
q_today.c.sum_today != None),
q_today.c.sum_today != q_yesterday.c.sum_yesterday))
)
for db_problem, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level and sum_today >= level:
self.log_info("Notifying about problem #{0} level {1}"
.format(db_problem.id, level))
msg = {
"problem_id": db_problem.id,
"function": db_problem.crash_function,
"components": db_problem.unique_component_names,
"first_occurrence": db_problem.first_occurrence
.strftime("%Y-%m-%d"),
"count": sum_today,
"type": db_problem.type,
"level": level,
}
if web.webfaf_installed():
msg["url"] = web.reverse("problems.item",
problem_id=db_problem.id)
fedmsg.publish(
topic="problem.threshold{0}".format(level),
modname='faf',
msg=msg)
def components(self, cmdline, db, opsys, release):
"""
Get statistics for most crashing components
"""
hist_table, hist_field = get_history_target(self.history_type)
total = get_history_sum(db, opsys, release)
comps = get_report_count_by_component(db, opsys, release)
if cmdline.last:
now = datetime.datetime.now()
since = now - datetime.timedelta(days=int(cmdline.last))
comps = comps.filter(hist_field >= since)
total = total.filter(hist_field >= since)
total_num = total.first()[0]
limit = int(cmdline.count)
limit_details = int(cmdline.detail_count)
results = []
for num, (comp, count) in enumerate(comps):
if num >= limit:
break
if comp in self.comps_filter:
continue
reports = get_report_stats_by_component(db, comp, opsys, release,
self.history_type)
if cmdline.last:
reports = reports.filter(hist_field >= since)
problem_ids = set()
attached_reports = []
for report, report_count in reports:
if len(problem_ids) >= limit_details:
break
if not report.problem:
continue
if report.problem.id in problem_ids:
continue
if report.quality < 0 and not cmdline.include_low_quality:
continue
problem_ids.add(report.problem.id)
problem_url = ""
if webfaf_installed():
problem_url = reverse("problems.item",
problem_id=report.problem.id)
attached_reports.append((problem_url, report.bugs))
results.append((comp, count, attached_reports))
if not results:
return ""
out = "Components:\n\n"
for num, (comp, count, reports) in enumerate(results):
if reports:
out += ("{0}. {1} seen {2} times ({3:.0%} of all reports)\n".
format(num + 1, comp, count, count / float(total_num)))
for problem_url, bugs in reports:
if problem_url or bugs:
out += " {0} {1}\n".format(
problem_url, ", ".join(map(str, bugs)))
return out
def text_overview(self, cmdline, db, opsys, release):
release_ids = get_release_ids(db, opsys, release)
num_days = 7
if cmdline.last:
num_days = int(cmdline.last)
since = datetime.datetime.now() - datetime.timedelta(days=num_days)
hot = query_hot_problems(db,
release_ids,
history=self.history_type,
last_date=since)
if not cmdline.include_low_quality:
hot = [x for x in hot if x.quality >= 0]
ptypes = ""
if len(self.ptypes) != len(problemtypes):
ptypes = " " + ", ".join(self.ptypes)
out = "Overview of the top {0}{1} crashes over the last {2} days:\n".format(
cmdline.count, ptypes, num_days)
hot = [p for p in hot if p.type in self.ptypes]
for (rank, problem) in enumerate(hot[:cmdline.count]):
out += "#{0} {1} - {2}x\n".format(
rank + 1, ', '.join(problem.unique_component_names),
problem.count)
# Reports with bugzillas for this OpSysRelease go first
reports = sorted(
problem.reports,
cmp=lambda x, y: len(
[b for b in x.bugs if b.opsysrelease_id in release_ids]) -
len([b for b in y.bugs if b.opsysrelease_id in release_ids]),
reverse=True)
if webfaf_installed():
for report in reports[:3]:
out += "{0}\n".format(
reverse("reports.bthash_forward",
bthash=report.hashes[0].hash))
for bug in report.bugs:
out += " {0}\n".format(bug.url)
else:
for report in reports[:3]:
out += "Report BT hash: {0}\n".format(
report.hashes[0].hash)
if len(problem.reports) > 3:
out += "... and {0} more.\n".format(len(problem.reports) - 3)
if problem.tainted:
out += "Kernel tainted.\n"
crash_function = problem.crash_function
if crash_function:
out += "Crash function: {0}\n".format(crash_function)
affected_all = []
for report in problem.reports:
affected_known = [
(affected.build.base_package_name, affected.build.epoch,
affected.build.version, affected.build.release)
for affected in get_crashed_package_for_report(
db, report.id)
]
affected_unknown = \
get_crashed_unknown_package_nevr_for_report(db, report.id)
affected_all += affected_known + affected_unknown
affected_all = sorted(set(affected_all),
cmp=lambda a, b: cmp_evr(a[1:], b[1:]),
reverse=True)
if affected_all:
out += "Affected builds: {0}".format(", ".join([
"{0}-{1}:{2}-{3}".format(n, e, v, r)
for (n, e, v, r) in affected_all[:5]
]))
if len(problem.reports) > 5:
out += " and {0} more.".format(len(problem.reports) - 5)
out += "\n"
pfix = problem.probable_fix_for_opsysrelease_ids(release_ids)
if len(pfix) > 0:
out += ("Problem seems to be fixed since the release of {0}\n".
format(pfix))
out += "\n"
return out
def run(self, cmdline, db) -> None:
levels = tuple(10**n for n in range(7))
if cmdline.reports:
# Sum of counts until yesterday
q_yesterday = (db.session.query(
Report.id.label("y_report_id"),
func.sum(ReportHistoryDaily.count).label(
"sum_yesterday")).outerjoin(ReportHistoryDaily).filter(
ReportHistoryDaily.day < cmdline.date).group_by(
Report.id).subquery())
# Sum of counts until today
q_today = (db.session.query(
Report.id.label("t_report_id"),
func.sum(ReportHistoryDaily.count).label(
"sum_today")).outerjoin(ReportHistoryDaily).filter(
ReportHistoryDaily.day <= cmdline.date).group_by(
Report.id).subquery())
q = (db.session.query(
Report, q_today.c.sum_today,
q_yesterday.c.sum_yesterday).outerjoin(
q_today, Report.id == q_today.c.t_report_id).outerjoin(
q_yesterday,
Report.id == q_yesterday.c.y_report_id).filter(
or_(Report.max_certainty.isnot(None),
Report.max_certainty != 100)).
filter(
or_(
and_(q_yesterday.c.sum_yesterday.is_(None),
q_today.c.sum_today.isnot(None)),
q_today.c.sum_today != q_yesterday.c.sum_yesterday)))
for db_report, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level <= sum_today:
self.log_info(
"Notifying about report #{0} level {1}".format(
db_report.id, level))
msg_body = {
"report_id":
db_report.id,
"function":
db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence":
db_report.first_occurrence.strftime(DATE_FORMAT),
"count":
sum_today,
"type":
db_report.type,
"level":
level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse(
"reports.item", report_id=db_report.id)
if db_report.problem_id:
msg_body["problem_id"] = db_report.problem_id
try:
msg = FafReportMessage(
topic="faf.report.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
self.log_warn(
"Fedora Messaging broker rejected message {0}: {1}"
.format(msg.id, e))
except ConnectionException as e:
self.log_warn(
"Error sending message {0}: {1}".format(
msg.id, e))
if cmdline.problems:
# Sum of counts until yesterday
q_yesterday = (db.session.query(
Problem.id.label("y_problem_id"),
func.sum(
ReportHistoryDaily.count).label("sum_yesterday")).join(
Report, Report.problem_id ==
Problem.id).outerjoin(ReportHistoryDaily).filter(
ReportHistoryDaily.day < cmdline.date).group_by(
Problem.id).subquery())
# Sum of counts until today
q_today = (db.session.query(
Problem.id.label("t_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_today")).join(
Report, Report.problem_id == Problem.id).outerjoin(
ReportHistoryDaily).filter(
ReportHistoryDaily.day <= cmdline.date).group_by(
Problem.id).subquery())
q = (db.session.query(
Problem, q_today.c.sum_today,
q_yesterday.c.sum_yesterday).outerjoin(
q_today, Problem.id == q_today.c.t_problem_id).outerjoin(
q_yesterday,
Problem.id == q_yesterday.c.y_problem_id).filter(
or_(
and_(q_yesterday.c.sum_yesterday.is_(None),
q_today.c.sum_today.isnot(None)),
q_today.c.sum_today !=
q_yesterday.c.sum_yesterday)))
for db_problem, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level <= sum_today:
self.log_info(
"Notifying about problem #{0} level {1}".format(
db_problem.id, level))
msg_body = {
"problem_id":
db_problem.id,
"function":
db_problem.crash_function,
"components":
list(db_problem.unique_component_names),
"first_occurrence":
db_problem.first_occurrence.strftime(DATE_FORMAT),
"count":
sum_today,
"type":
db_problem.type,
"level":
level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse(
"problems.item", problem_id=db_problem.id)
try:
msg = FafProblemMessage(
topic="faf.problem.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
self.log_warn(
"Fedora Messaging broker rejected message {0}: {1}"
.format(msg.id, e))
except ConnectionException as e:
self.log_warn(
"Error sending message {0}: {1}".format(
msg.id, e))
def fedmsg_report(target, value, oldvalue, initiator):
"""
Send fedmsg notifications when Report.count reaches specified threshold.
"""
try:
db_report = target
if notify_reports:
oldcount = oldvalue
newcount = value
for level in levels:
if oldcount < level and newcount >= level:
logger.info(
"Notifying about report #{0} level {1}".format(
db_report.id, level))
msg = {
"report_id":
db_report.id,
"function":
db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence":
db_report.first_occurrence.strftime("%Y-%m-%d"),
"count":
newcount,
"type":
db_report.type,
"level":
level,
}
if web.webfaf_installed() and db_report.hashes:
msg["url"] = web.reverse(
"reports.bthash_forward",
bthash=db_report.hashes[0].hash)
if db_report.problem_id:
msg["problem_id"] = db_report.problem_id
fedmsg.publish(
topic="report.threshold{0}".format(level),
modname='faf',
msg=msg)
if notify_problems and db_report.problem is not None:
oldcount = db_report.problem.reports_count
newcount = oldcount + value - oldvalue
for level in levels:
if oldcount < level and newcount >= level:
logger.info(
"Notifying about problem #{0} level {1}".format(
db_report.problem.id, level))
msg = {
"problem_id":
db_report.problem.id,
"function":
db_report.problem.crash_function,
"components":
db_report.problem.unique_component_names,
"first_occurrence":
db_report.problem.first_occurrence.strftime(
"%Y-%m-%d"),
"count":
newcount,
"type":
db_report.type,
"level":
level,
}
if web.webfaf_installed():
msg["url"] = web.reverse(
"problems.item",
problem_id=db_report.problem.id)
fedmsg.publish(
topic="problem.threshold{0}".format(level),
modname='faf',
msg=msg)
# Catch any exception. This is non-critical and mustn't break stuff
# elsewhere.
except Exception as e:
logger.exception(e, exc_info=True)
def fedmsg_report(target, value, oldvalue, initiator) -> None: # pylint: disable=unused-argument
"""
Send Fedora Messaging notifications when Report.count reaches specified threshold.
"""
try:
db_report = target
if notify_reports:
oldcount = oldvalue
newcount = value
for level in levels:
if oldcount < level <= newcount:
logger.info(
"Notifying about report #{0} level {1}".format(
db_report.id, level))
msg_body = {
"report_id":
db_report.id,
"function":
db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence":
db_report.first_occurrence.strftime("%Y-%m-%d"),
"count":
newcount,
"type":
db_report.type,
"level":
level,
}
if web.webfaf_installed() and db_report.hashes:
msg_body["url"] = web.reverse(
"reports.bthash_forward",
bthash=db_report.hashes[0].hash)
if db_report.problem_id:
msg_body["problem_id"] = db_report.problem_id
try:
msg = FafReportMessage(
topic="faf.report.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
logger.exception(
"Fedora Messaging broker rejected message {0}: {1}"
.format(msg.id, e))
except ConnectionException as e:
logger.exception(
"Error sending message {0}: {1}".format(
msg.id, e))
if notify_problems and db_report.problem is not None:
oldcount = db_report.problem.reports_count
newcount = oldcount + value - oldvalue
for level in levels:
if oldcount < level <= newcount:
logger.info(
"Notifying about problem #{0} level {1}".format(
db_report.problem.id, level))
msg_body = {
"problem_id":
db_report.problem.id,
"function":
db_report.problem.crash_function,
"components":
list(db_report.problem.unique_component_names),
"first_occurrence":
db_report.problem.first_occurrence.strftime(
"%Y-%m-%d"),
"count":
newcount,
"type":
db_report.type,
"level":
level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse(
"problems.item",
problem_id=db_report.problem.id)
try:
msg = FafProblemMessage(
topic="faf.problem.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
logger.exception(
"Fedora Messaging broker rejected message {0}: {1}"
.format(msg.id, e))
except ConnectionException as e:
logger.exception(
"Error sending message {0}: {1}".format(
msg.id, e))
# Catch any exception. This is non-critical and mustn't break stuff
# elsewhere.
except Exception as e: # pylint: disable=broad-except
logger.exception(e, exc_info=True)
def fedmsg_report(target, value, oldvalue, initiator): # pylint: disable=unused-argument
"""
Send Fedora Messaging notifications when Report.count reaches specified threshold.
"""
try:
db_report = target
if notify_reports:
oldcount = oldvalue
newcount = value
for level in levels:
if oldcount < level <= newcount:
logger.info("Notifying about report #{0} level {1}"
.format(db_report.id, level))
msg_body = {
"report_id": db_report.id,
"function": db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence": db_report.first_occurrence
.strftime("%Y-%m-%d"),
"count": newcount,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed() and db_report.hashes:
msg_body["url"] = web.reverse("reports.bthash_forward",
bthash=db_report.hashes[0].hash)
if db_report.problem_id:
msg_body["problem_id"] = db_report.problem_id
try:
msg = FafReportMessage(topic="faf.report.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
logger.exception("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
logger.exception("Error sending message {0}: {1}".format(msg.id, e))
if notify_problems and db_report.problem is not None:
oldcount = db_report.problem.reports_count
newcount = oldcount + value - oldvalue
for level in levels:
if oldcount < level <= newcount:
logger.info("Notifying about problem #{0} level {1}"
.format(db_report.problem.id, level))
msg_body = {
"problem_id": db_report.problem.id,
"function": db_report.problem.crash_function,
"components": list(db_report.problem.unique_component_names),
"first_occurrence": db_report.problem.first_occurrence
.strftime("%Y-%m-%d"),
"count": newcount,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse("problems.item", problem_id=db_report.problem.id)
try:
msg = FafProblemMessage(topic="faf.problem.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
logger.exception("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
logger.exception("Error sending message {0}: {1}".format(msg.id, e))
# Catch any exception. This is non-critical and mustn't break stuff
# elsewhere.
except Exception as e: # pylint: disable=broad-except
logger.exception(e, exc_info=True)
def components(self, cmdline, db, opsys, release):
"""
Get statistics for most crashing components
"""
hist_table, hist_field = get_history_target(self.history_type)
total = get_history_sum(db, opsys, release)
comps = get_report_count_by_component(db, opsys, release)
if cmdline.last:
now = datetime.datetime.now()
since = now - datetime.timedelta(days=int(cmdline.last))
comps = comps.filter(hist_field >= since)
total = total.filter(hist_field >= since)
total_num = total.first()[0]
limit = int(cmdline.count)
limit_details = int(cmdline.detail_count)
results = []
for num, (comp, count) in enumerate(comps):
if num >= limit:
break
if comp in self.comps_filter:
continue
reports = get_report_stats_by_component(db, comp, opsys,
release,
self.history_type)
if cmdline.last:
reports = reports.filter(hist_field >= since)
problem_ids = set()
attached_reports = []
for report, report_count in reports:
if len(problem_ids) >= limit_details:
break
if not report.problem:
continue
if report.problem.id in problem_ids:
continue
if report.quality < 0 and not cmdline.include_low_quality:
continue
problem_ids.add(report.problem.id)
problem_url = ""
if webfaf_installed():
problem_url = reverse("problems.item",
problem_id=report.problem.id)
attached_reports.append((problem_url, report.bugs))
results.append((comp, count, attached_reports))
if not results:
return ""
out = "Components:\n\n"
for num, (comp, count, reports) in enumerate(results):
if reports:
out += ("{0}. {1} seen {2} times ({3:.0%} of all reports)\n"
.format(num + 1, comp, count, count / float(total_num)))
for problem_url, bugs in reports:
if problem_url or bugs:
out += " {0} {1}\n".format(
problem_url, ", ".join(map(str, bugs)))
return out
def text_overview(self, cmdline, db, opsys, release):
release_ids = get_release_ids(db, opsys, release)
num_days = 7
if cmdline.last:
num_days = int(cmdline.last)
since = datetime.datetime.now() - datetime.timedelta(days=num_days)
hot = query_hot_problems(db, release_ids, history=self.history_type,
last_date=since)
if not cmdline.include_low_quality:
hot = [x for x in hot if x.quality >= 0]
ptypes = ""
if len(self.ptypes) != len(problemtypes):
ptypes = " "+", ".join(self.ptypes)
out = "Overview of the top {0}{1} crashes over the last {2} days:\n".format(
cmdline.count, ptypes, num_days)
hot = [p for p in hot if p.type in self.ptypes]
for (rank, problem) in enumerate(hot[:cmdline.count]):
out += "#{0} {1} - {2}x\n".format(
rank+1,
', '.join(problem.unique_component_names),
problem.count)
# Reports with bugzillas for this OpSysRelease go first
reports = sorted(problem.reports,
cmp=lambda x, y: len([b for b in x.bugs if b.opsysrelease_id in release_ids])
- len([b for b in y.bugs if b.opsysrelease_id in release_ids]), reverse=True)
if webfaf_installed():
for report in reports[:3]:
out += "{0}\n".format(reverse("reports.bthash_forward",
bthash=report.hashes[0].hash))
for bug in report.bugs:
out += " {0}\n".format(bug.url)
else:
for report in reports[:3]:
out += "Report BT hash: {0}\n".format(report.hashes[0].hash)
if len(problem.reports) > 3:
out += "... and {0} more.\n".format(len(problem.reports)-3)
if problem.tainted:
out += "Kernel tainted.\n"
crash_function = problem.crash_function
if crash_function:
out += "Crash function: {0}\n".format(crash_function)
affected_all = []
for report in problem.reports:
affected_known = [
(affected.build.base_package_name,
affected.build.epoch,
affected.build.version,
affected.build.release) for affected in
get_crashed_package_for_report(db, report.id)]
affected_unknown = \
get_crashed_unknown_package_nevr_for_report(db, report.id)
affected_all += affected_known + affected_unknown
affected_all = sorted(set(affected_all),
cmp=lambda a, b: cmp_evr(a[1:], b[1:]),
reverse=True)
if affected_all:
out += "Affected builds: {0}".format(", ".join(
["{0}-{1}:{2}-{3}".format(n, e, v, r)
for (n, e, v, r) in affected_all[:5]]))
if len(problem.reports) > 5:
out += " and {0} more.".format(len(problem.reports)-5)
out += "\n"
pfix = problem.probable_fix_for_opsysrelease_ids(release_ids)
if pfix:
out += ("Problem seems to be fixed since the release of {0}\n"
.format(pfix))
out += "\n"
return out
def text_overview(self, cmdline, db, opsys, release):
release_ids = get_release_ids(db, opsys, release)
num_days = 7
if cmdline.last:
num_days = int(cmdline.last)
since = datetime.datetime.now() - datetime.timedelta(days=num_days)
hot = query_hot_problems(db, release_ids, history=self.history_type,
last_date=since)
if not cmdline.include_low_quality:
hot = filter(lambda x: x.quality >= 0, hot)
out = "Overview of the top {0} crashes over the last {1} days:\n".format(
cmdline.count, num_days)
for (rank, problem) in enumerate(hot[:cmdline.count]):
out += "#{0} {1} - {2}x\n".format(
rank+1,
', '.join(problem.unique_component_names),
problem.count)
if webfaf_installed():
for report in problem.reports:
out += "{0}\n".format(reverse("webfaf.reports.views.bthash_forward",
args=[report.hashes[0].hash]))
else:
for report in problem.reports:
out += "Report BT hash: {0}\n".format(report.hashes[0].hash)
if problem.tainted:
out += "Kernel tainted.\n"
crash_function = problem.crash_function
if crash_function:
out += "Crash function: {0}\n".format(crash_function)
affected_all = []
for report in problem.reports:
affected_known = [
(affected.build.base_package_name,
affected.build.epoch,
affected.build.version,
affected.build.release) for affected in
get_crashed_package_for_report(db, report.id)]
affected_unknown = \
get_crashed_unknown_package_nevr_for_report(db, report.id)
affected_all += affected_known + affected_unknown
affected_all = sorted(set(affected_all), cmp=lambda a, b: cmp_evr(a[1:], b[1:]))
if affected_all:
out += "Affected builds: {0}\n".format(", ".join(
["{0}-{1}:{2}-{3}".format(n, e, v, r)
for (n, e, v, r) in affected_all]))
pfix = problem.probable_fix_for_opsysrelease_ids(release_ids)
if len(pfix) > 0:
out += ("Problem seems to be fixed since the release of {0}\n"
.format(pfix))
out += "\n"
return out
def fedmsg_report(target, value, oldvalue, initiator):
"""
Send fedmsg notifications when Report.count reaches specified threshold.
"""
try:
db_report = target
if notify_reports:
oldcount = oldvalue
newcount = value
for level in levels:
if oldcount < level and newcount >= level:
logger.info("Notifying about report #{0} level {1}"
.format(db_report.id, level))
msg = {
"report_id": db_report.id,
"function": db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence": db_report.first_occurrence
.strftime("%Y-%m-%d"),
"count": newcount,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed() and db_report.hashes:
msg["url"] = web.reverse("reports.bthash_forward",
bthash=db_report.hashes[0].hash)
if db_report.problem_id:
msg["problem_id"] = db_report.problem_id
fedmsg.publish(
topic="report.threshold{0}".format(level),
modname='faf',
msg=msg)
if notify_problems and db_report.problem is not None:
oldcount = db_report.problem.reports_count
newcount = oldcount + value - oldvalue
for level in levels:
if oldcount < level and newcount >= level:
logger.info("Notifying about problem #{0} level {1}"
.format(db_report.problem.id, level))
msg = {
"problem_id": db_report.problem.id,
"function": db_report.problem.crash_function,
"components": db_report.problem.unique_component_names,
"first_occurrence": db_report.problem.first_occurrence
.strftime("%Y-%m-%d"),
"count": newcount,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed():
msg["url"] = web.reverse("problems.item",
problem_id=db_report.problem.id)
fedmsg.publish(
topic="problem.threshold{0}".format(level),
modname='faf',
msg=msg)
# Catch any exception. This is non-critical and mustn't break stuff
# elsewhere.
except Exception as e:
logger.exception(e, exc_info=True)
def run(self, cmdline, db):
levels = tuple(10**n for n in range(7))
if cmdline.reports:
# Sum of counts until yesterday
q_yesterday = (
db.session
.query(Report.id.label("y_report_id"),
func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day < cmdline.date)
.group_by(Report.id)
.subquery()
)
# Sum of counts until today
q_today = (
db.session
.query(Report.id.label("t_report_id"),
func.sum(ReportHistoryDaily.count).label("sum_today"))
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day <= cmdline.date)
.group_by(Report.id)
.subquery()
)
q = (db.session.query(Report,
q_today.c.sum_today,
q_yesterday.c.sum_yesterday)
.outerjoin(q_today, Report.id == q_today.c.t_report_id)
.outerjoin(q_yesterday, Report.id == q_yesterday.c.y_report_id)
.filter(or_(Report.max_certainty.isnot(None), Report.max_certainty != 100))
.filter(or_(and_(q_yesterday.c.sum_yesterday.is_(None),
q_today.c.sum_today.isnot(None)),
q_today.c.sum_today != q_yesterday.c.sum_yesterday))
)
for db_report, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level <= sum_today:
self.log_info("Notifying about report #{0} level {1}"
.format(db_report.id, level))
msg_body = {
"report_id": db_report.id,
"function": db_report.crash_function,
"components": [db_report.component.name],
"first_occurrence": db_report.first_occurrence
.strftime("%Y-%m-%d"),
"count": sum_today,
"type": db_report.type,
"level": level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse("reports.item",
report_id=db_report.id)
if db_report.problem_id:
msg_body["problem_id"] = db_report.problem_id
try:
msg = FafReportMessage(topic="faf.report.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
self.log_warn("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
self.log_warn("Error sending message {0}: {1}".format(msg.id, e))
if cmdline.problems:
# Sum of counts until yesterday
q_yesterday = (
db.session
.query(Problem.id.label("y_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_yesterday"))
.join(Report, Report.problem_id == Problem.id)
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day < cmdline.date)
.group_by(Problem.id)
.subquery()
)
# Sum of counts until today
q_today = (
db.session
.query(Problem.id.label("t_problem_id"),
func.sum(ReportHistoryDaily.count).label("sum_today"))
.join(Report, Report.problem_id == Problem.id)
.outerjoin(ReportHistoryDaily)
.filter(ReportHistoryDaily.day <= cmdline.date)
.group_by(Problem.id)
.subquery()
)
q = (db.session
.query(Problem, q_today.c.sum_today, q_yesterday.c.sum_yesterday)
.outerjoin(q_today, Problem.id == q_today.c.t_problem_id)
.outerjoin(q_yesterday, Problem.id == q_yesterday.c.y_problem_id)
.filter(or_(and_(q_yesterday.c.sum_yesterday.is_(None),
q_today.c.sum_today.isnot(None)),
q_today.c.sum_today != q_yesterday.c.sum_yesterday))
)
for db_problem, sum_today, sum_yesterday in q.yield_per(100):
# avoid None
sum_yesterday = sum_yesterday or 0
for level in levels:
if sum_yesterday < level <= sum_today:
self.log_info("Notifying about problem #{0} level {1}"
.format(db_problem.id, level))
msg_body = {
"problem_id": db_problem.id,
"function": db_problem.crash_function,
"components": list(db_problem.unique_component_names),
"first_occurrence": db_problem.first_occurrence
.strftime("%Y-%m-%d"),
"count": sum_today,
"type": db_problem.type,
"level": level,
}
if web.webfaf_installed():
msg_body["url"] = web.reverse("problems.item",
problem_id=db_problem.id)
try:
msg = FafProblemMessage(topic="faf.problem.threshold{0}".format(level),
body=msg_body)
publish(msg)
except PublishReturned as e:
self.log_warn("Fedora Messaging broker rejected message {0}: {1}".format(msg.id, e))
except ConnectionException as e:
self.log_warn("Error sending message {0}: {1}".format(msg.id, e))