def is_bad(r):
if settings.param.sustained_median.trigger < r.result.confidence:
test_param = nvl(settings.param.test[literal_field(r.Talos.Test.name)], settings.param.suite[literal_field(r.Talos.Test.suite)])
if test_param == None:
return True
if test_param.disable:
return False
if test_param.better == "higher":
diff = -r.diff
elif test_param.better == "lower":
diff = r.diff
else:
diff = abs(r.diff) # DEFAULT = ANY DIRECTION IS BAD
if test_param.min_regression:
if unicode(test_param.min_regression.strip()[-1]) == "%":
min_diff = Math.abs(r.past_stats.mean * float(test_param.min_regression.strip()[:-1]) / 100.0)
else:
min_diff = Math.abs(float(test_param.min_regression))
else:
min_diff = Math.abs(r.past_stats.mean * 0.01)
if diff > min_diff:
return True
return False
def main():
settings = startup.read_settings()
Log.start(settings.debug)
try:
for repo in settings.param.repos:
with DB(settings.database) as db:
try:
pull_repo(repo)
#GET LATEST DATE
existing_range = db.query("""
SELECT
max(`date`) `max`,
min(`date`) `min`,
min(revision) min_rev,
max(revision) max_rev
FROM
changesets
WHERE
repo={{repo}}
""", {"repo": repo.name})[0]
ranges = struct.wrap([
{"min": nvl(existing_range.max, CNV.milli2datetime(0)) + timedelta(0, 1)},
{"max": existing_range.min}
])
for r in ranges:
for g, docs in Q.groupby(get_changesets(date_range=r, repo=repo), size=100):
for doc in docs:
doc.file_changes = None
doc.file_adds = None
doc.file_dels = None
doc.description = doc.description[0:16000]
db.insert_list("changesets", docs)
db.flush()
missing_revisions = sql.find_holes(db, "changesets", "revision", {"term":{"repo":repo.name}}, {"min": 0, "max": existing_range.max_rev + 1})
for _range in missing_revisions:
for g, docs in Q.groupby(get_changesets(revision_range=_range, repo=repo), size=100):
for doc in docs:
doc.file_changes = None
doc.file_adds = None
doc.file_dels = None
doc.description = doc.description[0:16000]
db.insert_list("changesets", docs)
db.flush()
except Exception, e:
Log.warning("Failure to pull from {{repo.name}}", {"repo":repo}, e)
finally:
Log.stop()
def main():
settings = startup.read_settings()
Log.start(settings.debug)
try:
Log.note("Running email using schema {{schema}}", {"schema": settings.perftest.schema})
with DB(settings.alerts) as db:
email_send(
db=db,
emailer=Emailer(settings.email),
debug=nvl(settings.debug, False)
)
except Exception, e:
Log.warning("Failure to send emails", cause=e)
def send_alerts(settings, db):
"""
BLINDLY SENDS ALERTS FROM THE ALERTS TABLE, ASSUMING ALL HAVE THE SAME STRUCTURE.
"""
debug = settings.param.debug
db.debug = debug
#TODO: REMOVE, LEAVE IN DB
if db.debug:
db.execute("update reasons set email_subject={{subject}}, email_template={{template}} where code={{reason}}", {
"template": CNV.object2JSON(TEMPLATE),
"subject": CNV.object2JSON(SUBJECT),
"reason": REASON
})
db.flush()
try:
new_alerts = db.query("""
SELECT
a.id alert_id,
a.reason,
r.description,
a.details,
a.severity,
a.confidence,
a.revision,
r.email_template,
r.email_subject
FROM
alerts a
JOIN
reasons r on r.code = a.reason
WHERE
a.last_sent IS NULL AND
a.status <> 'obsolete' AND
math.bayesian_add(a.severity, a.confidence) > {{alert_limit}} AND
a.solution IS NULL AND
a.reason in {{reasons}} AND
a.create_time > {{min_time}}
ORDER BY
math.bayesian_add(a.severity, a.confidence) DESC,
json.number(left(details, 65000), "diff_percent") DESC
LIMIT
10
""", {
"last_sent": datetime.utcnow() - RESEND_AFTER,
"alert_limit": ALERT_LIMIT - EPSILON,
"min_time": datetime.utcnow()-LOOK_BACK,
"reasons": SQL("("+", ".join(db.quote_value(v) for v in SEND_REASONS)+")")
})
if not new_alerts:
if debug:
Log.note("Nothing important to email")
return
for alert in new_alerts:
#poor souls that signed up for emails
listeners = ";".join(db.query("SELECT email FROM listeners WHERE reason={{reason}}", {"reason": alert.reason}).email)
body = [HEADER]
if alert.confidence >= 1:
alert.confidence = 0.999999
alert.details = CNV.JSON2object(alert.details)
try:
alert.revision = CNV.JSON2object(alert.revision)
except Exception, e:
pass
alert.score = str(-log(1.0-Math.bayesian_add(alert.severity, alert.confidence), 10)) #SHOW NUMBER OF NINES
alert.details.url = alert.details.page_url
example = alert.details.example
for e in alert.details.tests.example + [example]:
if e.push_date_min:
e.push_date_max = (2 * e.push_date) - e.push_date_min
e.date_range = (datetime.utcnow()-CNV.milli2datetime(e.push_date_min)).total_seconds()/(24*60*60) #REQUIRED FOR DATAZILLA B2G CHART REFERENCE
e.date_range = nvl(nvl(*[v for v in (7, 30, 60) if v > e.date_range]), 90) #PICK FIRST v > CURRENT VALUE
subject = expand_template(CNV.JSON2object(alert.email_subject), alert)
body.append(expand_template(CNV.JSON2object(alert.email_template), alert))
body = "".join(body)+FOOTER
if debug:
Log.note("EMAIL: {{email}}", {"email": body})
if len(body) > MAX_EMAIL_LENGTH:
Log.note("Truncated the email body")
suffix = "... (has been truncated)"
body = body[0:MAX_EMAIL_LENGTH - len(suffix)] + suffix #keep it reasonable
db.call("mail.send", (
listeners, #to
subject,
body, #body
None
))
#I HOPE I CAN SEND ARRAYS OF NUMBERS
db.execute(
"UPDATE alerts SET last_sent={{time}} WHERE {{where}}", {
"time": datetime.utcnow(),
"where": esfilter2sqlwhere(db, {"terms": {"id": Q.select(new_alerts, "alert_id")}})
})
except Exception, e:
Log.error("Could not send alerts", e)
def alert_sustained_median(settings, qb, alerts_db):
"""
find single points that deviate from the trend
"""
# OBJECTSTORE = settings.objectstore.schema + ".objectstore"
# TDAD = settings.perftest.schema + ".test_data_all_dimensions"
TDAD = settings.query["from"]
PUSH_DATE = "datazilla.date_loaded"
debug = nvl(settings.param.debug, DEBUG)
query = settings.query
def is_bad(r):
if settings.param.sustained_median.trigger < r.result.confidence:
test_param = nvl(settings.param.test[literal_field(r.Talos.Test.name)], settings.param.suite[literal_field(r.Talos.Test.suite)])
if test_param == None:
return True
if test_param.disable:
return False
if test_param.better == "higher":
diff = -r.diff
elif test_param.better == "lower":
diff = r.diff
else:
diff = abs(r.diff) # DEFAULT = ANY DIRECTION IS BAD
if test_param.min_regression:
if unicode(test_param.min_regression.strip()[-1]) == "%":
min_diff = Math.abs(r.past_stats.mean * float(test_param.min_regression.strip()[:-1]) / 100.0)
else:
min_diff = Math.abs(float(test_param.min_regression))
else:
min_diff = Math.abs(r.past_stats.mean * 0.01)
if diff > min_diff:
return True
return False
with Timer("pull combinations"):
disabled_suites = [s for s, p in settings.param.suite.items() if p.disable]
disabled_tests = [t for t, p in settings.param.test.items() if p.disable]
temp = Query({
"from": TDAD,
"select": {"name": "min_push_date", "value": PUSH_DATE, "aggregate": "min"},
"edges": query.edges,
"where": {"and": [
True if settings.args.restart else {"missing": {"field": settings.param.mark_complete}},
{"exists": {"field": "result.test_name"}},
{"range": {PUSH_DATE: {"gte": OLDEST_TS}}},
{"not": {"terms": {"Talos.Test.suite": disabled_suites}}},
{"not": {"terms": {"Talos.Test.name": disabled_tests}}}
# {"term": {"testrun.suite": "cart"}},
# {"term": {"result.test_name": "1-customize-enter.error.TART"}},
# {"term": {"test_machine.osversion": "OS X 10.8"}}
#FOR DEBUGGING SPECIFIC SERIES
# {"term": {"test_machine.type": "hamachi"}},
# {"term": {"test_machine.platform": "Gonk"}},
# {"term": {"test_machine.os": "Firefox OS"}},
# {"term": {"test_build.branch": "master"}},
# {"term": {"testrun.suite": "communications/ftu"}},
# {"term": {"result.test_name": "startup_time"}}
]},
"limit": nvl(settings.param.combo_limit, 1000)
}, qb)
new_test_points = qb.query(temp)
#BRING IN ALL NEEDED DATA
if debug:
Log.note("Pull all data for {{num}} groups:\n{{groups.name}}", {
"num": len(new_test_points),
"groups": query.edges
})
# all_min_date = Null
all_touched = set()
evaled_tests = set()
alerts = [] # PUT ALL THE EXCEPTION ITEMS HERE
for g, test_points in Q.groupby(new_test_points, query.edges):
if not test_points.min_push_date:
continue
try:
if settings.args.restart:
first_sample = OLDEST_TS
else:
first_sample = MAX(MIN(test_points.min_push_date), OLDEST_TS)
# FOR THIS g, HOW FAR BACK IN TIME MUST WE GO TO COVER OUR WINDOW_SIZE?
first_in_window = qb.query({
"select": {"name": "min_date", "value": "push_date", "aggregate": "min"},
"from": {
"from": TDAD,
"select": {"name": "push_date", "value": PUSH_DATE},
"where": {"and": [
{"term": g},
{"range": {PUSH_DATE: {"lt": first_sample}}}
]},
"sort": {"field": PUSH_DATE, "sort": -1},
"limit": settings.param.sustained_median.window_size * 2
}
})
if len(first_in_window) > settings.param.sustained_median.window_size * 2:
do_all = False
else:
do_all = True
min_date = MIN(first_sample, first_in_window.min_date)
#LOAD TEST RESULTS FROM DATABASE
test_results = qb.query({
"from": {
"from": "talos",
"select": [{"name": "push_date", "value": PUSH_DATE}] +
query.select +
query.edges,
"where": {"and": [
{"term": g},
{"range": {PUSH_DATE: {"gte": min_date}}}
]},
},
"sort": "push_date"
})
Log.note("{{num}} test results found for {{group}} dating back no further than {{start_date}}", {
"num": len(test_results),
"group": g,
"start_date": CNV.milli2datetime(min_date)
})
if debug:
Log.note("Find sustained_median exceptions")
#APPLY WINDOW FUNCTIONS
stats = Q.run({
"from": {
"from": test_results,
"where": {"exists": {"field": "value"}}
},
"window": [
{
# WE DO NOT WANT TO CONSIDER THE POINTS BEFORE FULL WINDOW SIZE
"name": "ignored",
"value": lambda r, i: False if do_all or i > settings.param.sustained_median.window_size else True
}, {
# SO WE CAN SHOW A DATAZILLA WINDOW
"name": "push_date_min",
"value": lambda r: r.push_date,
"sort": "push_date",
"aggregate": windows.Min,
"range": {"min": -settings.param.sustained_median.window_size, "max": 0}
}, {
# SO WE CAN SHOW A DATAZILLA WINDOW
"name": "push_date_max",
"value": lambda r: r.push_date,
"sort": "push_date",
"aggregate": windows.Max,
"range": {"min": 0, "max": settings.param.sustained_median.window_size}
}, {
"name": "past_revision",
"value": lambda r, i, rows: rows[i - 1].Talos.Revision,
"sort": "push_date"
}, {
"name": "past_stats",
"value": lambda r: r.value,
"sort": "push_date",
"aggregate": windows.Stats(middle=0.60),
"range": {"min": -settings.param.sustained_median.window_size, "max": 0}
}, {
"name": "future_stats",
"value": lambda r: r.value,
"sort": "push_date",
"aggregate": windows.Stats(middle=0.60),
"range": {"min": 0, "max": settings.param.sustained_median.window_size}
}, {
"name": "result",
"value": lambda r, i, rows: median_test(
rows[-settings.param.sustained_median.window_size + i:i:].value,
rows[i:settings.param.sustained_median.window_size + i:].value,
interpolate=False
),
"sort": "push_date"
}, {
"name": "diff",
"value": lambda r: r.future_stats.mean - r.past_stats.mean
}, {
"name": "diff_percent",
"value": lambda r: (r.future_stats.mean - r.past_stats.mean) / r.past_stats.mean
}, {
"name": "is_diff",
"value": is_bad
}, {
#USE THIS TO FILL CONFIDENCE HOLES
#WE CAN MARK IT is_diff KNOWING THERE IS A HIGHER CONFIDENCE
"name": "future_is_diff",
"value": lambda r, i, rows: rows[i - 1].is_diff and r.result.confidence < rows[i - 1].result.confidence,
"sort": "push_date"
}, {
#WE CAN MARK IT is_diff KNOWING THERE IS A HIGHER CONFIDENCE
"name": "past_is_diff",
"value": lambda r, i, rows: rows[i - 1].is_diff and r.result.confidence < rows[i - 1].result.confidence,
"sort": {"value": "push_date", "sort": -1}
},
]
})
#PICK THE BEST SCORE FOR EACH is_diff==True REGION
for g2, data in Q.groupby(stats, "is_diff", contiguous=True):
if g2.is_diff:
best = Q.sort(data, ["result.confidence", "diff"]).last()
best["pass"] = True
all_touched.update(Q.select(test_results, ["test_run_id", "Talos.Test"]))
# TESTS THAT HAVE BEEN (RE)EVALUATED GIVEN THE NEW INFORMATION
evaled_tests.update(Q.run({
"from": test_results,
"select": ["test_run_id", "Talos.Test"],
"where": {"term": {"ignored": False}}
}))
File("test_values.txt").write(CNV.list2tab(Q.select(stats, [
{"name": "push_date", "value": lambda x: CNV.datetime2string(CNV.milli2datetime(x.push_date), "%d-%b-%Y %H:%M:%S")},
"value",
{"name": "revision", "value": "Talos.Revision"},
{"name": "confidence", "value": "result.confidence"},
"pass"
])))
#TESTS THAT HAVE SHOWN THEMSELVES TO BE EXCEPTIONAL
new_exceptions = Q.filter(stats, {"term": {"pass": True}})
for v in new_exceptions:
if v.ignored:
continue
alert = Struct(
status="new",
create_time=CNV.milli2datetime(v.push_date),
tdad_id={"test_run_id": v.test_run_id, "Talos": {"Test": v.Talos.Test}},
reason=REASON,
revision=v.Talos.Revision,
details=v,
severity=SEVERITY,
confidence=v.result.confidence
)
alerts.append(alert)
if debug:
Log.note("{{num}} new exceptions found", {"num": len(new_exceptions)})
except Exception, e:
Log.warning("Problem with alert identification, continue to log existing alerts and stop cleanly", e)
"num_new": len(new_alerts),
"num_change": len(changed_alerts),
"num_delete": len(obsolete_alerts)
})
if new_alerts:
for a in new_alerts:
a.id = SQL("util.newid()")
a.last_updated = NOW
try:
alerts_db.insert_list("alerts", new_alerts)
except Exception, e:
Log.error("problem with insert", e)
for curr in changed_alerts:
if len(nvl(curr.solution, "").strip()) != 0:
continue # DO NOT TOUCH SOLVED ALERTS
a = found_alerts[(curr.tdad_id, )]
if a == None:
Log.error("Programmer error, changed_alerts must have {{key_value}}", {"key_value": curr.tdad.id})
if significant_difference(curr.severity, a.severity) or \
significant_difference(curr.confidence, a.confidence) or \
curr.reason != a.reason:
curr.last_updated = NOW
alerts_db.update("alerts", {"id": curr.id}, a)
#OBSOLETE THE ALERTS THAT ARE NO LONGER VALID
if obsolete_alerts:
def page_threshold_limit(db, debug):
"""
simplest of rules to test the dataflow from test_run, to alert, to email
may prove slightly useful also!
#point out any pages that are breaking human-set threshold limits
"""
db.debug = debug
try:
# CALCULATE HOW FAR BACK TO LOOK
lasttime = db.query("SELECT last_run, description FROM reasons WHERE code={{type}}", {"type": REASON})[0]
lasttime = nvl(lasttime.last_run, datetime.utcnow())
min_date = lasttime + LOOK_BACK
# FIND ALL PAGES THAT HAVE LIMITS TO TEST
# BRING BACK ONES THAT BREAK LIMITS
# BUT DO NOT ALREADY HAVE AN ALERTS EXISTING
pages = db.query(
"""
SELECT
t.id tdad_id,
t.n_replicates,
t.mean,
t.std,
h.threshold,
h.severity,
h.reason,
m.id alert_id
FROM
alert_page_thresholds h
JOIN
test_data_all_dimensions t ON t.page_id=h.page
LEFT JOIN
alerts m on m.tdad_id=t.test_run_id AND m.reason={{type}}
WHERE
h.threshold<t.mean AND
t.push_date>{{min_date}} AND
(m.id IS NULL OR m.status='obsol11ete')
""",
{"type": REASON, "min_date": min_date},
)
# FOR EACH PAGE THAT BREAKS LIMITS
for page in pages:
if page.alert_id != None:
break
alert = {
"id": SQL("util.newid()"),
"status": "new",
"create_time": datetime.utcnow(),
"last_updated": datetime.utcnow(),
"tdad_id": page.tdad_id,
"reason": REASON,
"details": CNV.object2JSON(
{"expected": float(page.threshold), "actual": float(page.mean), "reason": page.reason}
),
"severity": page.severity,
"confidence": 1.0 # USING NORMAL DIST ASSUMPTION WE CAN ADJUST
# CONFIDENCE EVEN BEFORE THRESHOLD IS HIT!
# FOR NOW WE KEEP IT SIMPLE
}
db.insert("alerts", alert)
for page in pages:
if page.alert_id == None:
break
db.update("alerts", None) # ERROR FOR NOW
# OBSOLETE THE ALERTS THAT SHOULD NO LONGER GET SENT
obsolete = db.query(
"""
SELECT
m.id,
m.tdad_id
FROM
alerts m
JOIN
test_data_all_dimensions t ON m.tdad_id=t.id
JOIN
alert_page_thresholds h on t.page_id=h.page
WHERE
m.reason={{reason}} AND
h.threshold>=t.mean AND
t.push_date>{{time}}
""",
{"reason": REASON, "time": min_date},
)
if obsolete:
db.execute(
"UPDATE alerts SET status='obsolete' WHERE {{where}}",
{"where": esfilter2sqlwhere(db, {"terms": {"id": Q.select(obsolete, "id")}})},
)
db.execute(
"UPDATE reasons SET last_run={{now}} WHERE code={{reason}}", {"now": datetime.utcnow(), "reason": REASON}
)
except Exception, e:
Log.error("Could not perform threshold comparisons", e)
def talos_alert_revision(settings):
assert settings.alerts != None
settings.db.debug = settings.param.debug
with DB(settings.alerts) as db:
with ESQuery(ElasticSearch(settings.query["from"])) as esq:
dbq = DBQuery(db)
esq.addDimension(CNV.JSON2object(File(settings.dimension.filename).read()))
#TODO: REMOVE, LEAVE IN DB
if db.debug:
db.execute("update reasons set email_subject={{subject}}, email_template={{template}} where code={{reason}}", {
"template": CNV.object2JSON(TEMPLATE),
"subject": CNV.object2JSON(SUBJECT),
"reason": REASON
})
db.flush()
#EXISTING SUSTAINED EXCEPTIONS
existing_sustained_alerts = dbq.query({
"from": "alerts",
"select": "*",
"where": {"and": [
{"term": {"reason": talos_sustained_median.REASON}},
{"not": {"term": {"status": "obsolete"}}},
{"range": {"create_time": {"gte": NOW - LOOK_BACK}}}
]}
})
tests = Q.index(existing_sustained_alerts, ["revision", "details.Talos.Test"])
#EXISTING REVISION-LEVEL ALERTS
old_alerts = dbq.query({
"from": "alerts",
"select": "*",
"where": {"and": [
{"term": {"reason": REASON}},
{"or": [
{"terms": {"revision": set(existing_sustained_alerts.revision)}},
{"term": {"reason": talos_sustained_median.REASON}},
{"term": {"status": "obsolete"}},
{"range": {"create_time": {"gte": NOW - LOOK_BACK}}}
]}
]}
})
old_alerts = Q.unique_index(old_alerts, "revision")
#SUMMARIZE
known_alerts = StructList()
for revision in set(existing_sustained_alerts.revision):
#FIND TOTAL TDAD FOR EACH INTERESTING REVISION
total_tests = esq.query({
"from": "talos",
"select": {"name": "count", "aggregate": "count"},
"where": {"and":[
{"terms": {"Talos.Revision": revision}}
]}
})
total_exceptions = tests[(revision, )] # FILTER BY revision
parts = StructList()
for g, exceptions in Q.groupby(total_exceptions, ["details.Talos.Test"]):
worst_in_test = Q.sort(exceptions, ["confidence", "details.diff_percent"]).last()
example = worst_in_test.details
# ADD SOME DATAZILLA SPECIFIC URL PARAMETERS
branch = example.Talos.Branch.replace("-Non-PGO", "")
example.tbpl.url.branch = TBPL_PATH.get(branch, branch)
example.mercurial.url.branch = MECURIAL_PATH.get(branch, branch)
example.datazilla.url.branch = example.Talos.Branch #+ ("" if worst_in_test.Talos.Branch.pgo else "-Non-PGO")
example.datazilla.url.x86 = "true" if example.Talos.Platform == "x86" else "false"
example.datazilla.url.x86_64 = "true" if example.Talos.Platform == "x86_64" else "false"
example.datazilla.url.stop = nvl(example.push_date_max, (2*example.push_date) - example.push_date_min)
num_except = len(exceptions)
if num_except == 0:
continue
part = {
"test": g.details.Talos.Test,
"num_exceptions": num_except,
"num_tests": total_tests,
"confidence": worst_in_test.confidence,
"example": example
}
parts.append(part)
parts = Q.sort(parts, [{"field": "confidence", "sort": -1}])
worst_in_revision = parts[0].example
known_alerts.append({
"status": "new",
"create_time": CNV.milli2datetime(worst_in_revision.push_date),
"reason": REASON,
"revision": revision,
"tdad_id": revision,
"details": {
"revision": revision,
"total_tests": total_tests,
"total_exceptions": len(total_exceptions),
"tests": parts,
"example": worst_in_revision
},
"severity": SEVERITY,
"confidence": worst_in_revision.result.confidence
})
known_alerts = Q.unique_index(known_alerts, "revision")
#NEW ALERTS, JUST INSERT
new_alerts = known_alerts - old_alerts
if new_alerts:
for revision in new_alerts:
revision.id = SQL("util.newid()")
revision.last_updated = NOW
db.insert_list("alerts", new_alerts)
#SHOW SUSTAINED ALERTS ARE COVERED
db.execute("""
INSERT INTO hierarchy (parent, child)
SELECT
r.id parent,
p.id child
FROM
alerts p
LEFT JOIN
hierarchy h on h.child=p.id
LEFT JOIN
alerts r on r.revision=p.revision AND r.reason={{parent_reason}}
WHERE
{{where}}
""", {
"where": esfilter2sqlwhere(db, {"and": [
{"term": {"p.reason": talos_sustained_median.REASON}},
{"terms": {"p.revision": Q.select(existing_sustained_alerts, "revision")}},
{"missing": "h.parent"}
]}),
"parent_reason": REASON
})
#CURRENT ALERTS, UPDATE IF DIFFERENT
for known_alert in known_alerts & old_alerts:
if len(nvl(known_alert.solution, "").strip()) != 0:
continue # DO NOT TOUCH SOLVED ALERTS
old_alert = old_alerts[known_alert]
if old_alert.status == 'obsolete' or significant_difference(known_alert.severity, old_alert.severity) or significant_difference(known_alert.confidence, old_alert.confidence):
known_alert.last_updated = NOW
db.update("alerts", {"id": old_alert.id}, known_alert)
#OLD ALERTS, OBSOLETE
for old_alert in old_alerts - known_alerts:
if old_alert.status == 'obsolete':
continue
db.update("alerts", {"id": old_alert.id}, {"status": "obsolete", "last_updated": NOW, "details":None})
