def main():
settings = startup.read_settings()
Log.start(settings.debug)
try:
for repo in settings.param.repos:
with DB(settings.database) as db:
try:
pull_repo(repo)
#GET LATEST DATE
existing_range = db.query("""
SELECT
max(`date`) `max`,
min(`date`) `min`,
min(revision) min_rev,
max(revision) max_rev
FROM
changesets
WHERE
repo={{repo}}
""", {"repo": repo.name})[0]
ranges = struct.wrap([
{"min": nvl(existing_range.max, CNV.milli2datetime(0)) + timedelta(0, 1)},
{"max": existing_range.min}
])
for r in ranges:
for g, docs in Q.groupby(get_changesets(date_range=r, repo=repo), size=100):
for doc in docs:
doc.file_changes = None
doc.file_adds = None
doc.file_dels = None
doc.description = doc.description[0:16000]
db.insert_list("changesets", docs)
db.flush()
missing_revisions = sql.find_holes(db, "changesets", "revision", {"term":{"repo":repo.name}}, {"min": 0, "max": existing_range.max_rev + 1})
for _range in missing_revisions:
for g, docs in Q.groupby(get_changesets(revision_range=_range, repo=repo), size=100):
for doc in docs:
doc.file_changes = None
doc.file_adds = None
doc.file_dels = None
doc.description = doc.description[0:16000]
db.insert_list("changesets", docs)
db.flush()
except Exception, e:
Log.warning("Failure to pull from {{repo.name}}", {"repo":repo}, e)
finally:
Log.stop()
def email_send(db, emailer, debug):
db.debug = debug
##VERIFY self SHOULD BE THE ONE PERFORMING OPS (TO PREVENT MULTIPLE INSTANCES NEEDLESSLY RUNNING)
try:
## EXIT EARLY IF THERE ARE NO EMAILS TO SEND
has_mail = db.query("SELECT max(new_mail) new_mail FROM mail.notify")
if has_mail[0]["new_mail"] == 0:
Log.note("No emails to send")
return
## GET LIST OF MAILS TO SEND
emails = db.query("""
SELECT
c.id,
group_concat(d.deliver_to SEPARATOR ',') `to`,
c.subject,
c.body
FROM
mail.content c
LEFT JOIN
mail.delivery d ON d.content=c.id
WHERE
d.content IS NOT NULL AND
c.date_sent IS NULL
GROUP BY
c.id
""")
## SEND MAILS
not_done = 0 ##SET TO ONE IF THERE ARE MAIL FAILURES, AND THERE ARE MAILS STILL LEFT TO SEND
num_done = 0
for email in emails:
try:
emailer.send_email(
to_addrs=email.to.split(','),
subject=email.subject,
html_data=email.body
)
db.execute("UPDATE mail.content SET date_sent={{now}} WHERE id={{id}}", {"id": email.id, "now": datetime.utcnow()})
db.flush()
num_done += len(email.to.split(','))
except Exception, e:
Log.warning("Problem sending email", e)
not_done = 1
db.execute("UPDATE mail.notify SET new_mail={{not_done}}", {"not_done": not_done})
Log.note(str(num_done) + " emails have been sent")
def main():
settings = startup.read_settings()
Log.start(settings.debug)
try:
Log.note("Running email using schema {{schema}}", {"schema": settings.perftest.schema})
with DB(settings.alerts) as db:
email_send(
db=db,
emailer=Emailer(settings.email),
debug=nvl(settings.debug, False)
)
except Exception, e:
Log.warning("Failure to send emails", cause=e)
def main():
try:
settings = startup.read_settings()
Log.start(settings.debug)
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/util.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/debug.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/cnv.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/string.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/math.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/json.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/util/mail.sql")
DB.execute_file(settings.database, settings.sql.rstrip("/")+"/alerts.sql")
Log.note("DB setup complete")
except Exception, e:
Log.warning("Failure to setup DB", cause=e)
def main():
settings = startup.read_settings(defs=[{
"name": ["--restart", "--reset", "--redo"],
"help": "use this to recalc alerts",
"action": "store_true",
"dest": "restart"
}])
Log.start(settings.debug)
try:
with startup.SingleInstance(flavor_id=settings.args.filename):
Log.note("Finding exceptions in index {{index_name}}", {"index_name": settings.query["from"].name})
with ESQuery(ElasticSearch(settings.query["from"])) as qb:
qb.addDimension(CNV.JSON2object(File(settings.dimension.filename).read()))
with DB(settings.alerts) as alerts_db:
alert_sustained_median(
settings,
qb,
alerts_db
)
except Exception, e:
Log.warning("Failure to find sustained_median exceptions", e)
#I HOPE I CAN SEND ARRAYS OF NUMBERS
db.execute(
"UPDATE alerts SET last_sent={{time}} WHERE {{where}}", {
"time": datetime.utcnow(),
"where": esfilter2sqlwhere(db, {"terms": {"id": Q.select(new_alerts, "alert_id")}})
})
except Exception, e:
Log.error("Could not send alerts", e)
if __name__ == '__main__':
settings = startup.read_settings()
Log.start(settings.debug)
try:
Log.note("Running alerts off of schema {{schema}}", {"schema": settings.perftest.schema})
with DB(settings.alerts) as db:
send_alerts(
settings=settings,
db=db
)
except Exception, e:
Log.warning("Failure to run alerts", cause=e)
finally:
Log.stop()
def alert_sustained_median(settings, qb, alerts_db):
"""
find single points that deviate from the trend
"""
# OBJECTSTORE = settings.objectstore.schema + ".objectstore"
# TDAD = settings.perftest.schema + ".test_data_all_dimensions"
TDAD = settings.query["from"]
PUSH_DATE = "datazilla.date_loaded"
debug = nvl(settings.param.debug, DEBUG)
query = settings.query
def is_bad(r):
if settings.param.sustained_median.trigger < r.result.confidence:
test_param = nvl(settings.param.test[literal_field(r.Talos.Test.name)], settings.param.suite[literal_field(r.Talos.Test.suite)])
if test_param == None:
return True
if test_param.disable:
return False
if test_param.better == "higher":
diff = -r.diff
elif test_param.better == "lower":
diff = r.diff
else:
diff = abs(r.diff) # DEFAULT = ANY DIRECTION IS BAD
if test_param.min_regression:
if unicode(test_param.min_regression.strip()[-1]) == "%":
min_diff = Math.abs(r.past_stats.mean * float(test_param.min_regression.strip()[:-1]) / 100.0)
else:
min_diff = Math.abs(float(test_param.min_regression))
else:
min_diff = Math.abs(r.past_stats.mean * 0.01)
if diff > min_diff:
return True
return False
with Timer("pull combinations"):
disabled_suites = [s for s, p in settings.param.suite.items() if p.disable]
disabled_tests = [t for t, p in settings.param.test.items() if p.disable]
temp = Query({
"from": TDAD,
"select": {"name": "min_push_date", "value": PUSH_DATE, "aggregate": "min"},
"edges": query.edges,
"where": {"and": [
True if settings.args.restart else {"missing": {"field": settings.param.mark_complete}},
{"exists": {"field": "result.test_name"}},
{"range": {PUSH_DATE: {"gte": OLDEST_TS}}},
{"not": {"terms": {"Talos.Test.suite": disabled_suites}}},
{"not": {"terms": {"Talos.Test.name": disabled_tests}}}
# {"term": {"testrun.suite": "cart"}},
# {"term": {"result.test_name": "1-customize-enter.error.TART"}},
# {"term": {"test_machine.osversion": "OS X 10.8"}}
#FOR DEBUGGING SPECIFIC SERIES
# {"term": {"test_machine.type": "hamachi"}},
# {"term": {"test_machine.platform": "Gonk"}},
# {"term": {"test_machine.os": "Firefox OS"}},
# {"term": {"test_build.branch": "master"}},
# {"term": {"testrun.suite": "communications/ftu"}},
# {"term": {"result.test_name": "startup_time"}}
]},
"limit": nvl(settings.param.combo_limit, 1000)
}, qb)
new_test_points = qb.query(temp)
#BRING IN ALL NEEDED DATA
if debug:
Log.note("Pull all data for {{num}} groups:\n{{groups.name}}", {
"num": len(new_test_points),
"groups": query.edges
})
# all_min_date = Null
all_touched = set()
evaled_tests = set()
alerts = [] # PUT ALL THE EXCEPTION ITEMS HERE
for g, test_points in Q.groupby(new_test_points, query.edges):
if not test_points.min_push_date:
continue
try:
if settings.args.restart:
first_sample = OLDEST_TS
else:
first_sample = MAX(MIN(test_points.min_push_date), OLDEST_TS)
# FOR THIS g, HOW FAR BACK IN TIME MUST WE GO TO COVER OUR WINDOW_SIZE?
first_in_window = qb.query({
"select": {"name": "min_date", "value": "push_date", "aggregate": "min"},
"from": {
"from": TDAD,
"select": {"name": "push_date", "value": PUSH_DATE},
"where": {"and": [
{"term": g},
{"range": {PUSH_DATE: {"lt": first_sample}}}
]},
"sort": {"field": PUSH_DATE, "sort": -1},
"limit": settings.param.sustained_median.window_size * 2
}
})
if len(first_in_window) > settings.param.sustained_median.window_size * 2:
do_all = False
else:
do_all = True
min_date = MIN(first_sample, first_in_window.min_date)
#LOAD TEST RESULTS FROM DATABASE
test_results = qb.query({
"from": {
"from": "talos",
"select": [{"name": "push_date", "value": PUSH_DATE}] +
query.select +
query.edges,
"where": {"and": [
{"term": g},
{"range": {PUSH_DATE: {"gte": min_date}}}
]},
},
"sort": "push_date"
})
Log.note("{{num}} test results found for {{group}} dating back no further than {{start_date}}", {
"num": len(test_results),
"group": g,
"start_date": CNV.milli2datetime(min_date)
})
if debug:
Log.note("Find sustained_median exceptions")
#APPLY WINDOW FUNCTIONS
stats = Q.run({
"from": {
"from": test_results,
"where": {"exists": {"field": "value"}}
},
"window": [
{
# WE DO NOT WANT TO CONSIDER THE POINTS BEFORE FULL WINDOW SIZE
"name": "ignored",
"value": lambda r, i: False if do_all or i > settings.param.sustained_median.window_size else True
}, {
# SO WE CAN SHOW A DATAZILLA WINDOW
"name": "push_date_min",
"value": lambda r: r.push_date,
"sort": "push_date",
"aggregate": windows.Min,
"range": {"min": -settings.param.sustained_median.window_size, "max": 0}
}, {
# SO WE CAN SHOW A DATAZILLA WINDOW
"name": "push_date_max",
"value": lambda r: r.push_date,
"sort": "push_date",
"aggregate": windows.Max,
"range": {"min": 0, "max": settings.param.sustained_median.window_size}
}, {
"name": "past_revision",
"value": lambda r, i, rows: rows[i - 1].Talos.Revision,
"sort": "push_date"
}, {
"name": "past_stats",
"value": lambda r: r.value,
"sort": "push_date",
"aggregate": windows.Stats(middle=0.60),
"range": {"min": -settings.param.sustained_median.window_size, "max": 0}
}, {
"name": "future_stats",
"value": lambda r: r.value,
"sort": "push_date",
"aggregate": windows.Stats(middle=0.60),
"range": {"min": 0, "max": settings.param.sustained_median.window_size}
}, {
"name": "result",
"value": lambda r, i, rows: median_test(
rows[-settings.param.sustained_median.window_size + i:i:].value,
rows[i:settings.param.sustained_median.window_size + i:].value,
interpolate=False
),
"sort": "push_date"
}, {
"name": "diff",
"value": lambda r: r.future_stats.mean - r.past_stats.mean
}, {
"name": "diff_percent",
"value": lambda r: (r.future_stats.mean - r.past_stats.mean) / r.past_stats.mean
}, {
"name": "is_diff",
"value": is_bad
}, {
#USE THIS TO FILL CONFIDENCE HOLES
#WE CAN MARK IT is_diff KNOWING THERE IS A HIGHER CONFIDENCE
"name": "future_is_diff",
"value": lambda r, i, rows: rows[i - 1].is_diff and r.result.confidence < rows[i - 1].result.confidence,
"sort": "push_date"
}, {
#WE CAN MARK IT is_diff KNOWING THERE IS A HIGHER CONFIDENCE
"name": "past_is_diff",
"value": lambda r, i, rows: rows[i - 1].is_diff and r.result.confidence < rows[i - 1].result.confidence,
"sort": {"value": "push_date", "sort": -1}
},
]
})
#PICK THE BEST SCORE FOR EACH is_diff==True REGION
for g2, data in Q.groupby(stats, "is_diff", contiguous=True):
if g2.is_diff:
best = Q.sort(data, ["result.confidence", "diff"]).last()
best["pass"] = True
all_touched.update(Q.select(test_results, ["test_run_id", "Talos.Test"]))
# TESTS THAT HAVE BEEN (RE)EVALUATED GIVEN THE NEW INFORMATION
evaled_tests.update(Q.run({
"from": test_results,
"select": ["test_run_id", "Talos.Test"],
"where": {"term": {"ignored": False}}
}))
File("test_values.txt").write(CNV.list2tab(Q.select(stats, [
{"name": "push_date", "value": lambda x: CNV.datetime2string(CNV.milli2datetime(x.push_date), "%d-%b-%Y %H:%M:%S")},
"value",
{"name": "revision", "value": "Talos.Revision"},
{"name": "confidence", "value": "result.confidence"},
"pass"
])))
#TESTS THAT HAVE SHOWN THEMSELVES TO BE EXCEPTIONAL
new_exceptions = Q.filter(stats, {"term": {"pass": True}})
for v in new_exceptions:
if v.ignored:
continue
alert = Struct(
status="new",
create_time=CNV.milli2datetime(v.push_date),
tdad_id={"test_run_id": v.test_run_id, "Talos": {"Test": v.Talos.Test}},
reason=REASON,
revision=v.Talos.Revision,
details=v,
severity=SEVERITY,
confidence=v.result.confidence
)
alerts.append(alert)
if debug:
Log.note("{{num}} new exceptions found", {"num": len(new_exceptions)})
except Exception, e:
Log.warning("Problem with alert identification, continue to log existing alerts and stop cleanly", e)
def __exit__(self, type, value, traceback):
try:
self.server.quit()
except Exception, e:
Log.warning("Problem with smtp server quit(), ignoring problem", e)
