def is_bad(r):
if settings.param.sustained_median.trigger < r.result.confidence:
test_param = nvl(settings.param.test[literal_field(r.Talos.Test.name)], settings.param.suite[literal_field(r.Talos.Test.suite)])
if test_param == None:
return True
if test_param.disable:
return False
if test_param.better == "higher":
diff = -r.diff
elif test_param.better == "lower":
diff = r.diff
else:
diff = abs(r.diff) # DEFAULT = ANY DIRECTION IS BAD
if test_param.min_regression:
if unicode(test_param.min_regression.strip()[-1]) == "%":
min_diff = Math.abs(r.past_stats.mean * float(test_param.min_regression.strip()[:-1]) / 100.0)
else:
min_diff = Math.abs(float(test_param.min_regression))
else:
min_diff = Math.abs(r.past_stats.mean * 0.01)
if diff > min_diff:
return True
return False
def failure_rate(previous_results, current_results, real_rate=REAL_REGRESSION_RATE, confidence=CONFIDENCE):
if confidence is None:
#USED FOR TESTING
for page_name, results in previous_results.items():
results["failure_probability"] = float(results["total_fail"]) / (results["total_pass"] + results["total_fail"])
else:
for page_name, results in previous_results.items():
results["failure_probability"] = confident_fail_rate(results["total_fail"], results["total_pass"], confidence)
result = real_rate
for page_name, results in current_results.items():
p = previous_results[page_name]["failure_probability"]
for i in range(results["total_fail"]):
result = Math.bayesian_add(result, 1 - p)
for i in range(results["total_pass"]):
result = Math.bayesian_add(result, p)
return result
def significant_difference(a, b):
try:
if a in (0.0, 1.0) or b in (0.0, 1.0):
return True
if a / b < (1 - SIGNIFICANT) or (1 + SIGNIFICANT) < a / b:
return True
b_diff = Math.bayesian_subtract(a, b)
if 0.3 < b_diff < 0.7:
return False
return True
except Exception, e:
Log.error("Problem", e)
def is_bad(r):
if settings.param.sustained_median.trigger < r.result.confidence:
test_param = settings.param.test[literal_field(r.B2G.Test.name)]
if test_param == None:
return True
if test_param.better == "higher":
diff = -r.diff
else:
diff = r.diff
if unicode(test_param.min_regression.strip()[-1]) == "%":
min_diff = Math.abs(r.past_stats.mean * float(test_param.min_regression.strip()[:-1]) / 100.0)
else:
min_diff = Math.abs(float(test_param.min_regression))
if diff > min_diff:
return True
return False
def __init__(self, db):
self.now = datetime.utcnow() - timedelta(seconds=1)
self.recent_past = self.now - timedelta(hours=1)
self.far_past = self.now - timedelta(days=2)
self.db = db
# self.uid=None
self.series = 0
self.reason = "used for testing 1"
self.high_severity = 0.7
self.high_confidence = 0.9
self.important = Math.bayesian_add(self.high_severity, self.high_confidence)
self.low_severity = 0.5
self.low_confidence = 0.7
def send_alerts(settings, db):
"""
BLINDLY SENDS ALERTS FROM THE ALERTS TABLE, ASSUMING ALL HAVE THE SAME STRUCTURE.
"""
debug = settings.param.debug
db.debug = debug
#TODO: REMOVE, LEAVE IN DB
if db.debug:
db.execute("update reasons set email_subject={{subject}}, email_template={{template}} where code={{reason}}", {
"template": CNV.object2JSON(TEMPLATE),
"subject": CNV.object2JSON(SUBJECT),
"reason": REASON
})
db.flush()
try:
new_alerts = db.query("""
SELECT
a.id alert_id,
a.reason,
r.description,
a.details,
a.severity,
a.confidence,
a.revision,
r.email_template,
r.email_subject
FROM
alerts a
JOIN
reasons r on r.code = a.reason
WHERE
a.last_sent IS NULL AND
a.status <> 'obsolete' AND
math.bayesian_add(a.severity, a.confidence) > {{alert_limit}} AND
a.solution IS NULL AND
a.reason in {{reasons}} AND
a.create_time > {{min_time}}
ORDER BY
math.bayesian_add(a.severity, a.confidence) DESC,
json.number(left(details, 65000), "diff_percent") DESC
LIMIT
10
""", {
"last_sent": datetime.utcnow() - RESEND_AFTER,
"alert_limit": ALERT_LIMIT - EPSILON,
"min_time": datetime.utcnow()-LOOK_BACK,
"reasons": SQL("("+", ".join(db.quote_value(v) for v in SEND_REASONS)+")")
})
if not new_alerts:
if debug:
Log.note("Nothing important to email")
return
for alert in new_alerts:
#poor souls that signed up for emails
listeners = ";".join(db.query("SELECT email FROM listeners WHERE reason={{reason}}", {"reason": alert.reason}).email)
body = [HEADER]
if alert.confidence >= 1:
alert.confidence = 0.999999
alert.details = CNV.JSON2object(alert.details)
try:
alert.revision = CNV.JSON2object(alert.revision)
except Exception, e:
pass
alert.score = str(-log(1.0-Math.bayesian_add(alert.severity, alert.confidence), 10)) #SHOW NUMBER OF NINES
alert.details.url = alert.details.page_url
example = alert.details.example
for e in alert.details.tests.example + [example]:
if e.push_date_min:
e.push_date_max = (2 * e.push_date) - e.push_date_min
e.date_range = (datetime.utcnow()-CNV.milli2datetime(e.push_date_min)).total_seconds()/(24*60*60) #REQUIRED FOR DATAZILLA B2G CHART REFERENCE
e.date_range = nvl(nvl(*[v for v in (7, 30, 60) if v > e.date_range]), 90) #PICK FIRST v > CURRENT VALUE
subject = expand_template(CNV.JSON2object(alert.email_subject), alert)
body.append(expand_template(CNV.JSON2object(alert.email_template), alert))
body = "".join(body)+FOOTER
if debug:
Log.note("EMAIL: {{email}}", {"email": body})
if len(body) > MAX_EMAIL_LENGTH:
Log.note("Truncated the email body")
suffix = "... (has been truncated)"
body = body[0:MAX_EMAIL_LENGTH - len(suffix)] + suffix #keep it reasonable
db.call("mail.send", (
listeners, #to
subject,
body, #body
None
))
#I HOPE I CAN SEND ARRAYS OF NUMBERS
db.execute(
"UPDATE alerts SET last_sent={{time}} WHERE {{where}}", {
"time": datetime.utcnow(),
"where": esfilter2sqlwhere(db, {"terms": {"id": Q.select(new_alerts, "alert_id")}})
})
except Exception, e:
Log.error("Could not send alerts", e)
from math import log from dzAlerts.daemons import b2g_alert_revision, talos_alert_revision from dzAlerts.daemons.email_send import email_send from dzAlerts.daemons.talos_alert_revision import TEMPLATE, SUBJECT, REASON from dzAlerts.util.cnv import CNV from dzAlerts.util.env import startup from dzAlerts.util.queries import Q from dzAlerts.util.queries.db_query import esfilter2sqlwhere from dzAlerts.util.strings import expand_template from dzAlerts.util.maths import Math from dzAlerts.util.env.logs import Log from dzAlerts.util.sql.db import DB, SQL from dzAlerts.util.struct import nvl from dzAlerts.util.env.emailer import Emailer ALERT_LIMIT = Math.bayesian_add(0.90, 0.70) #SIMPLE severity*confidence LIMIT (FOR NOW) HEADER = "<h3>Performance Regression Alert</h3>" FOOTER = "<hr><a style='font-size:70%' href='https://wiki.mozilla.org/FirefoxOS/Performance/Investigating_Alerts'>Understanding this alert</a>" #TBPL link: https://tbpl.mozilla.org/?rev=c3598b276048 #TBPL test results: https://tbpl.mozilla.org/?tree=Mozilla-Inbound&rev=c9429cf294af #HG: https://hg.mozilla.org/mozilla-central/rev/330feedee4f1 #BZ: https://bugzilla.mozilla.org/show_bug.cgi?id=702559 #DZ: https://datazilla.mozilla.org/talos/summary/Mozilla-Inbound/897654df47b6?product=Firefox&branch_version=23.0a1 # "product":v.product, # "branch":v.branch, # "branch_version":v.branch_version, # "revision":v.revision SEPARATOR = "<hr>\n" RESEND_AFTER = timedelta(days=7) LOOK_BACK = timedelta(days=30)
