def test_single_file_list(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.post_json(
url,
json={
"meta": {
"format": "list"
},
"from": "files",
"where": {
"and": [{
"eq": {
"revision": "29dcc9cb77c372c97681a47496488ec6c623915d"
}
}, {
"in": {
"path": ["gfx/thebes/gfxFontVariations.h"]
}
}, {
"eq": {
"branch": "mozilla-central"
}
}]
}
})
list_response = response.data
tuids = list_response[0].tuids
assert len(tuids) == 41 # 41 lines expected
assert len(set(tuids)) == 41 # tuids much be unique
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
some_failures = http.post_json("http://activedata.allizom.org/query", data={
"from": "unittest",
"select": [
{"name": "branch", "value": "build.branch"},
{"name": "revision", "value": "build.revision12"},
{"name": "suite", "value": "run.suite"},
{"name": "chunk", "value": "run.chunk"},
{"name": "test", "value": "result.test"}
],
"where": {"and": [
{"eq": {"result.ok": False}},
{"gt": {"run.timestamp": Date.today() - WEEK}},
{"missing": "treeherder.job.note"}
]},
"format": "list",
"limit": 10
})
th = TreeHerder(settings={})
# th.get_job_classification("mozilla-inbound", "7380457b8ba0")
for f in some_failures.data:
th.get_job_classification(f.branch, f.revision)
except Exception, e:
Log.error("Problem with etl", e)
def test_big_result_works(self):
result = http.post_json(global_settings.testing.query,
data={
"from": "unittest",
"where": {
"and": [{
"gte": {
"run.timestamp":
Date.today() - DAY
}
}, {
"lt": {
"run.timestamp": Date.today()
}
}, {
"eq": {
"result.ok": False
}
}]
},
"format": "list",
"limit": 10000
})
if result.template:
result = Except.new_instance(result)
Log.error("problem with call", cause=result)
Log.note("Got {{num}} test failures", num=len(result.data))
def test_index_wo_name(self):
data = {
"name": "The Parent Trap",
"released": "29 July` 1998",
"imdb": "http://www.imdb.com/title/tt0120783/",
"rating": "PG",
"director": {
"name": "Nancy Meyers",
"dob": "December 8, 1949"
}
}
container = self.utils._es_cluster.get_or_create_index(
index=TEST_TABLE, kwargs=self.utils._es_test_settings)
try:
self.utils._es_cluster.delete_index(container.settings.index)
except Exception:
pass
container = self.utils._es_cluster.get_or_create_index(
index=TEST_TABLE, kwargs=self.utils._es_test_settings)
container.add({"value": data})
container.refresh()
result = http.post_json(url=self.utils.service_url,
json={
"format": "list",
"from": container.settings.index
})
self.assertEqual(result.data, [data])
def find_some_work(th):
# th.get_markup("fx-team", "036f62007472", "B8kS5IJ5Rom8l-kcSIRIlA")
# th.get_markup("mozilla-inbound", "971c1ee26cad", "fNuzNmZxS6m3i_p9jDh8iA")
# GET SOME TASKS
result = http.post_json(url="http://activedata.allizom.org/query",
data={
"from":
"task",
"select":
["build.branch", "build.revision", "task.id"],
"where": {
"and": [{
"gt": {
"task.run.start_time":
(Date.today() - DAY).unix
}
}, {
"exists": "build.revision"
}, {
"exists": "build.branch"
}]
},
"format":
"list"
})
# TRY TO GET THEM OUT OF OUR CACHE
for r in result.data:
Log.note("look for task {{task_id}}", task_id=r.task.id)
th.get_markup(r.build.branch, r.build.revision, r.task.id)
def main():
try:
settings = startup.read_settings()
constants.set(settings.constants)
Log.start(settings.debug)
some_failures = http.post_json("http://activedata.allizom.org/query",
data={
"from":
"unittest",
"select": [{
"name": "branch",
"value": "build.branch"
}, {
"name":
"revision",
"value":
"build.revision12"
}, {
"name": "suite",
"value": "run.suite"
}, {
"name": "chunk",
"value": "run.chunk"
}, {
"name": "test",
"value": "result.test"
}],
"where": {
"and": [{
"eq": {
"result.ok": False
}
}, {
"gt": {
"run.timestamp":
Date.today() - WEEK
}
}, {
"missing":
"treeherder.job.note"
}]
},
"format":
"list",
"limit":
10
})
th = TreeHerder(settings={})
# th.get_job_classification("mozilla-inbound", "7380457b8ba0")
for f in some_failures.data:
th.get_job_classification(f.branch, f.revision)
except Exception, e:
Log.error("Problem with etl", e)
def test_zero_files(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.post_json(url, json={
"from": "files",
"where": {"and": [
{"eq": {"revision": "29dcc9cb77c372c97681a47496488ec6c623915d"}},
{"in": {"path": []}},
{"eq": {"branch": "mozilla-central"}}
]}
})
assert len(response.data) == 0
def move_shards(settings):
path = settings.elasticsearch.host + ":" + unicode(
settings.elasticsearch.port)
command = [{
"move": {
"index": "unittest20150803_045709",
"shard": 20,
"from_node": "primary",
"to_node": "tertiary"
}
}]
result = http.post_json(path + "/_cluster/reroute",
data={"commands": command})
Log.note("result {{result}}", result=result)
def test_single_file_list(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.post_json(url, json={
"meta": {"format": "list"},
"from": "files",
"where": {"and": [
{"eq": {"revision": "29dcc9cb77c372c97681a47496488ec6c623915d"}},
{"in": {"path": ["gfx/thebes/gfxFontVariations.h"]}},
{"eq": {"branch": "mozilla-central"}}
]}
})
list_response = response.data
tuids = list_response[0].tuids
assert len(tuids) == 41 # 41 lines expected
assert len(set(tuids)) == 41 # tuids much be unique
def one_request(request, please_stop):
and_op = request.where['and']
files = []
for a in and_op:
if a['in'].path:
files = a['in'].path
elif a.eq.path:
files = [a.eq.path]
with Timer("Make TUID request from {{timestamp|datetime}}", {"timestamp": request.meta.request_time}):
try:
result = http.post_json(
"http://localhost:5000/tuid",
json=request,
timeout=30
)
if result is None or len(result.data) != len(files):
Log.note("incomplete response for {{thread}}", thread=Thread.current().name)
except Exception as e:
Log.warning("Request failure", cause=e)
def one_request(request, please_stop):
and_op = request.where['and']
files = []
for a in and_op:
if a['in'].path:
files = a['in'].path
elif a.eq.path:
files = [a.eq.path]
with Timer("Make TUID request from {{timestamp|datetime}}",
{"timestamp": request.meta.request_time}):
try:
result = http.post_json("http://localhost:5000/tuid",
json=request,
timeout=30)
if result is None or len(result.data) != len(files):
Log.note("incomplete response for {{thread}}",
thread=Thread.current().name)
except Exception as e:
Log.warning("Request failure", cause=e)
def find_some_work(th):
# th.get_markup("fx-team", "036f62007472", "B8kS5IJ5Rom8l-kcSIRIlA")
# th.get_markup("mozilla-inbound", "971c1ee26cad", "fNuzNmZxS6m3i_p9jDh8iA")
# GET SOME TASKS
result = http.post_json(url="http://activedata.allizom.org/query", data={
"from": "task",
"select": ["build.branch", "build.revision", "task.id"],
"where": {"and": [
{"gt": {"task.run.start_time": (Date.today() - DAY).unix}},
{"exists": "build.revision"},
{"exists": "build.branch"}
]},
"format": "list"
})
# TRY TO GET THEM OUT OF OUR CACHE
for r in result.data:
Log.note("look for task {{task_id}}", task_id=r.task.id)
th.get_markup(r.build.branch, r.build.revision, r.task.id)
def loop_all_days(destination, please_stop):
try:
today = Date.today()
# WHICH DAYS DO WE NEED TO CALCULATE
# ALL BUILD DATES WITH WITH ETL TIMESTAMP OF A WEEK AGO
# ALL BUILD DATES THAT HAVE NOT BEEN PROCESSED YET
build_dates = http.post_json(config.source.url, json={
"from": "unittest",
"edges": [
{
"name": "date",
"value": "build.date",
"allowNulls": False,
"domain": {
"type": "time",
"min": "today-week",
"max": "eod",
"interval": "day"
}
}
],
"where": {"gte": {"etl.timestamp": (today - WEEK).unix}},
"sort": {"value": "build.date", "sort": -1},
"limit": 14,
"format": "list"
})
build_dates.data = jx.sort(build_dates.data, {"value": "date", "sort": -1})
for d in build_dates.data:
if please_stop:
return
agg(Date(d.date), destination, please_stop=please_stop)
finally:
please_stop.go()
def test_zero_files(config, app):
url = "http://localhost:" + text_type(config.flask.port) + "/tuid"
response = http.post_json(
url,
json={
"from": "files",
"where": {
"and": [{
"eq": {
"revision": "29dcc9cb77c372c97681a47496488ec6c623915d"
}
}, {
"in": {
"path": []
}
}, {
"eq": {
"branch": "mozilla-central"
}
}]
}
})
assert len(response.data) == 0
def get_tuids(self, branch, revision, files):
"""
GET TUIDS FROM ENDPOINT, AND STORE IN DB
:param branch: BRANCH TO FIND THE REVISION/FILE
:param revision: THE REVISION NUNMBER
:param files: THE FULL PATHS TO THE FILES
:return: MAP FROM FILENAME TO TUID LIST
"""
# SCRUB INPUTS
revision = revision[:12]
files = [file.lstrip('/') for file in files]
with Timer(
"ask tuid service for {{num}} files at {{revision|left(12)}}",
{
"num": len(files),
"revision": revision
},
silent=not self.enabled):
response = self.db.query(
"SELECT file, tuids FROM tuid WHERE revision=" +
quote_value(revision) + " AND file IN " + quote_list(files))
found = {file: json2value(tuids) for file, tuids in response.data}
try:
remaining = set(files) - set(found.keys())
new_response = None
if remaining:
request = wrap({
"from": "files",
"where": {
"and": [{
"eq": {
"revision": revision
}
}, {
"in": {
"path": remaining
}
}, {
"eq": {
"branch": branch
}
}]
},
"branch": branch,
"meta": {
"format": "list",
"request_time": Date.now()
}
})
if self.push_queue is not None:
if DEBUG:
Log.note(
"record tuid request to SQS: {{timestamp}}",
timestamp=request.meta.request_time)
self.push_queue.add(request)
else:
if DEBUG:
Log.note("no recorded tuid request")
if not self.enabled:
return found
new_response = http.post_json(self.endpoint,
json=request,
timeout=self.timeout)
with self.db.transaction() as transaction:
command = "INSERT INTO tuid (revision, file, tuids) VALUES " + sql_list(
quote_list((revision, r.path, value2json(r.tuids)))
for r in new_response.data if r.tuids != None)
if not command.endswith(" VALUES "):
transaction.execute(command)
self.num_bad_requests = 0
found.update(
{r.path: r.tuids
for r in new_response.data} if new_response else {})
return found
except Exception as e:
self.num_bad_requests += 1
Till(seconds=SLEEP_ON_ERROR).wait()
if self.enabled and self.num_bad_requests >= 3:
self.enabled = False
Log.error("TUID service has problems.", cause=e)
return found
def loop(source, coverage_summary_index, settings, please_stop):
try:
cluster = elasticsearch.Cluster(source)
aliases = cluster.get_aliases()
candidates = []
for pairs in aliases:
if pairs.alias == source.index:
candidates.append(pairs.index)
candidates = jx.sort(candidates, {".": "desc"})
for index_name in candidates:
coverage_index = elasticsearch.Index(index=index_name, read_only=False, settings=source)
push_date_filter = unicode2Date(coverage_index.settings.index[-15::], elasticsearch.INDEX_DATE_FORMAT)
while not please_stop:
# IDENTIFY NEW WORK
Log.note("Working on index {{index}}", index=index_name)
coverage_index.refresh()
todo = http.post_json(settings.url, json={
"from": "coverage",
"groupby": ["source.file.name", "build.revision12"],
"where": {"and": [
{"missing": "source.method.name"},
{"missing": "source.file.min_line_siblings"},
{"gte": {"repo.push.date": push_date_filter}}
]},
"format": "list",
"limit": coalesce(settings.batch_size, 100)
})
if not todo.data:
break
queue = Queue("pending source files to review")
queue.extend(todo.data[0:coalesce(settings.batch_size, 100):])
threads = [
Thread.run(
"processor" + unicode(i),
process_batch,
queue,
coverage_index,
coverage_summary_index,
settings,
please_stop=please_stop
)
for i in range(NUM_THREAD)
]
# ADD STOP MESSAGE
queue.add(Thread.STOP)
# WAIT FOR THEM TO COMPLETE
for t in threads:
t.join()
please_stop.go()
return
except Exception, e:
Log.warning("Problem processing", cause=e)
def process_batch(todo, coverage_index, coverage_summary_index, settings, please_stop):
for not_summarized in todo:
if please_stop:
return True
# IS THERE MORE THAN ONE COVERAGE FILE FOR THIS REVISION?
Log.note("Find dups for file {{file}}", file=not_summarized.source.file.name)
dups = http.post_json(settings.url, json={
"from": "coverage",
"select": [
{"name": "max_id", "value": "etl.source.id", "aggregate": "max"},
{"name": "min_id", "value": "etl.source.id", "aggregate": "min"}
],
"where": {"and": [
{"missing": "source.method.name"},
{"eq": {
"source.file.name": not_summarized.source.file.name,
"build.revision12": not_summarized.build.revision12
}},
]},
"groupby": [
"test.url"
],
"limit": 100000,
"format": "list"
})
dups_found = False
for d in dups.data:
if d.max_id != d.min_id:
dups_found = True
Log.note(
"removing dups {{details|json}}\n{{dups|json|indent}}",
details={
"id": int(d.max_id),
"test": d.test.url,
"source": not_summarized.source.file.name,
"revision": not_summarized.build.revision12
}
)
# FIND ALL INDEXES
all_indexes = [
p.index
for p in coverage_index.cluster.get_aliases()
if p.alias == coverage_index.settings.alias
]
for index_name in all_indexes:
elasticsearch.Index(index=index_name, read_only=False, cluster=coverage_index.cluster).delete_record({"and": [
{"not": {"term": {"etl.source.id": int(d.max_id)}}},
{"term": {"test.url": d.test.url}},
{"term": {"source.file.name": not_summarized.source.file.name}},
{"term": {"build.revision12": not_summarized.build.revision12}}
]})
if dups_found:
continue
# LIST ALL TESTS THAT COVER THIS FILE, AND THE LINES COVERED
test_count = http.post_json(settings.url, json={
"from": "coverage.source.file.covered",
"where": {"and": [
{"missing": "source.method.name"},
{"eq": {
"source.file.name": not_summarized.source.file.name,
"build.revision12": not_summarized.build.revision12
}},
]},
"groupby": [
"test.url",
"line"
],
"limit": 100000,
"format": "list"
})
all_tests_covering_file = UNION(test_count.data.get("test.url"))
num_tests = len(all_tests_covering_file)
max_siblings = num_tests - 1
Log.note(
"{{filename}} rev {{revision}} is covered by {{num}} tests",
filename=not_summarized.source.file.name,
num=num_tests,
revision=not_summarized.build.revision12
)
line_summary = list(
(k, unwrap(wrap(list(v)).get("test.url")))
for k, v in jx.groupby(test_count.data, keys="line")
)
# PULL THE RAW RECORD FOR MODIFICATION
file_level_coverage_records = http.post_json(settings.url, json={
"from": "coverage",
"where": {"and": [
{"missing": "source.method.name"},
{"in": {"test.url": all_tests_covering_file}},
{"eq": {
"source.file.name": not_summarized.source.file.name,
"build.revision12": not_summarized.build.revision12
}}
]},
"limit": 100000,
"format": "list"
})
for test_name in all_tests_covering_file:
siblings = [len(test_names)-1 for g, test_names in line_summary if test_name in test_names]
min_siblings = MIN(siblings)
coverage_candidates = jx.filter(file_level_coverage_records.data, lambda row, rownum, rows: row.test.url == test_name)
if coverage_candidates:
if len(coverage_candidates) > 1 and any(coverage_candidates[0]._id != c._id for c in coverage_candidates):
Log.warning(
"Duplicate coverage\n{{cov|json|indent}}",
cov=[{"_id": c._id, "run": c.run, "test": c.test} for c in coverage_candidates]
)
# MORE THAN ONE COVERAGE CANDIDATE CAN HAPPEN WHEN THE SAME TEST IS IN TWO DIFFERENT CHUNKS OF THE SAME SUITE
for coverage_record in coverage_candidates:
coverage_record.source.file.max_test_siblings = max_siblings
coverage_record.source.file.min_line_siblings = min_siblings
coverage_record.source.file.score = (max_siblings - min_siblings) / (max_siblings + min_siblings + 1)
else:
example = http.post_json(settings.url, json={
"from": "coverage",
"where": {"eq": {
"test.url": test_name,
"source.file.name": not_summarized.source.file.name,
"build.revision12": not_summarized.build.revision12
}},
"limit": 1,
"format": "list"
})
Log.warning(
"{{test|quote}} rev {{revision}} appears to have no coverage for {{file|quote}}!\n{{example|json|indent}}",
test=test_name,
file=not_summarized.source.file.name,
revision=not_summarized.build.revision12,
example=example.data[0]
)
bad_example = [d for d in file_level_coverage_records.data if d["source.file.min_line_siblings"] == None]
if bad_example:
Log.warning("expecting all records to have summary. Example:\n{{example}}", example=bad_example[0])
rows = [{"id": d._id, "value": d} for d in file_level_coverage_records.data]
coverage_summary_index.extend(rows)
coverage_index.extend(rows)
all_test_summary = []
for g, records in jx.groupby(file_level_coverage_records.data, "source.file.name"):
cov = UNION(records.source.file.covered)
uncov = UNION(records.source.file.uncovered)
coverage = {
"_id": "|".join([records[0].build.revision12, g["source.file.name"]]), # SOMETHING UNIQUE, IN CASE WE RECALCULATE
"source": {
"file": {
"name": g["source.file.name"],
"is_file": True,
"covered": jx.sort(cov, "line"),
"uncovered": jx.sort(uncov),
"total_covered": len(cov),
"total_uncovered": len(uncov),
"min_line_siblings": 0 # PLACEHOLDER TO INDICATE DONE
}
},
"build": records[0].build,
"repo": records[0].repo,
"run": records[0].run,
"etl": {"timestamp": Date.now()}
}
all_test_summary.append(coverage)
sum_rows = [{"id": d["_id"], "value": d} for d in all_test_summary]
coverage_summary_index.extend(sum_rows)
if DEBUG:
coverage_index.refresh()
todo = http.post_json(settings.url, json={
"from": "coverage",
"where": {"and": [
{"missing": "source.method.name"},
{"missing": "source.file.min_line_siblings"},
{"eq": {"source.file.name": not_summarized.source.file.name}},
{"eq": {"build.revision12": not_summarized.build.revision12}}
]},
"format": "list",
"limit": 10
})
if todo.data:
Log.error("Failure to update")
def queue_consumer(client, pull_queue, please_stop=None, kwargs=None):
queue = aws.Queue(pull_queue)
client = TuidClient(client)
try_revs = {}
test_try_revs = True
#while len(queue) > 0:
# request = queue.pop(till=please_stop)
# if request:
# Log.note("Popping request from {{time}}", time=request.meta.request_time)
# queue.commit()
while not please_stop:
request = queue.pop(till=please_stop)
if please_stop:
break
if not request:
Log.note("Nothing in queue, pausing for 5 seconds...")
(please_stop | Till(seconds=5)).wait()
continue
Log.note("Found something in queue")
repo = 'mozilla-central'
and_op = request.where['and']
revision = None
files = None
for a in and_op:
if a.eq.revision:
revision = a.eq.revision
elif a['in'].path:
files = a['in'].path
elif a.eq.path:
files = [a.eq.path]
if len(files) <= 0:
Log.warning("No files in the given request: {{request}}",
request=request)
continue
if revision[:12] in try_revs and not test_try_revs:
Log.warning(
"Revision {{cset}} does not exist in the {{branch}} branch",
cset=revision[:12],
branch='mozilla-central')
queue.commit()
continue
clog_url = HG_URL / 'mozilla-central' / 'json-log' / revision[:12]
clog_obj = http.get_json(clog_url, retry=RETRY)
if isinstance(clog_obj, (text_type, str)):
Log.warning(
"Revision {{cset}} does not exist in the {{branch}} branch",
cset=revision[:12],
branch='mozilla-central')
try_revs[revision[:12]] = True
if not test_try_revs:
queue.commit()
continue
else:
json_rev_url = 'https://hg.mozilla.org/try/json-rev/' + revision[:
12]
clog_obj = http.get_json(json_rev_url, retry=RETRY)
if 'phase' not in clog_obj:
Log.warning(
"Revision {{cset}} does not exist in the try branch",
cset=revision[:12],
branch='mozilla-central')
queue.commit()
continue
if clog_obj['phase'] == 'draft':
repo = 'try'
else:
Log.note("Revision {{cset}} exists on mozilla-central.",
cset=revision[:12])
request.branch = repo
with Timer("Make TUID request from {{timestamp|date}}",
{"timestamp": request.meta.request_time}):
client.enabled = True # ENSURE THE REQUEST IS MADE
result = http.post_json("http://localhost:5000/tuid",
json=request,
timeout=10000)
if not client.enabled:
Log.note("pausing consumer for {{num}}sec",
num=PAUSE_ON_FAILURE)
Till(seconds=PAUSE_ON_FAILURE).wait()
if result is None or len(result.data) != len(files):
Log.warning("expecting response for every file requested")
queue.commit()
def get_tuids(self, branch, revision, files):
"""
GET TUIDS FROM ENDPOINT, AND STORE IN DB
:param branch: BRANCH TO FIND THE REVISION/FILE
:param revision: THE REVISION NUNMBER
:param files: THE FULL PATHS TO THE FILES
:return: MAP FROM FILENAME TO TUID LIST
"""
# SCRUB INPUTS
revision = revision[:12]
files = [file.lstrip('/') for file in files]
with Timer(
"ask tuid service for {{num}} files at {{revision|left(12)}}",
{"num": len(files), "revision": revision},
silent=not self.enabled
):
response = self.db.query(
"SELECT file, tuids FROM tuid WHERE revision=" + quote_value(revision) +
" AND file IN " + quote_list(files)
)
found = {file: json2value(tuids) for file, tuids in response.data}
try:
remaining = set(files) - set(found.keys())
new_response = None
if remaining:
request = wrap({
"from": "files",
"where": {"and": [
{"eq": {"revision": revision}},
{"in": {"path": remaining}},
{"eq": {"branch": branch}}
]},
"branch": branch,
"meta": {
"format": "list",
"request_time": Date.now()
}
})
if self.push_queue is not None:
if DEBUG:
Log.note("record tuid request to SQS: {{timestamp}}", timestamp=request.meta.request_time)
self.push_queue.add(request)
else:
if DEBUG:
Log.note("no recorded tuid request")
if not self.enabled:
return found
new_response = http.post_json(
self.endpoint,
json=request,
timeout=self.timeout
)
with self.db.transaction() as transaction:
command = "INSERT INTO tuid (revision, file, tuids) VALUES " + sql_list(
quote_list((revision, r.path, value2json(r.tuids)))
for r in new_response.data
if r.tuids != None
)
if not command.endswith(" VALUES "):
transaction.execute(command)
self.num_bad_requests = 0
found.update({r.path: r.tuids for r in new_response.data} if new_response else {})
return found
except Exception as e:
self.num_bad_requests += 1
Till(seconds=SLEEP_ON_ERROR).wait()
if self.enabled and self.num_bad_requests >= 3:
self.enabled = False
Log.error("TUID service has problems.", cause=e)
return found
def agg(today, destination, debug_filter=None, please_stop=None):
"""
:param today: The day we are performing the calculation for
:param destination: The ES index where we put the results
:param debug_filter: Some extra limitation to go faster, and focus, for testing
:param please_stop: Signal for stopping early
:return: nothing
"""
# GET LIST OF ALL TESTS, BY PLATFORM, TYPE, SUITE
for suite in SUITES:
domain = {"and": [
{"prefix": {"run.suite": suite}},
{"gt": {"build.date": (today - 3 * DAY).unix}},
{"lt": {"build.date": (today + 4 * DAY).unix}},
{"exists": "build.platform"},
{"not": {"in": {"build.platform": EXCLUDE_PLATFORMS}}},
{"not": {"in": {"build.branch": EXCLUDE_BRANCHES}}}
]}
if debug_filter:
domain['and'].append(debug_filter)
_ = convert.value2json("\"\"")
# WE CAN NOT PULL ALL TESTS, THERE ARE TOO MANY, SO DO ONE SUITE AT A TIME
Log.note("Get summary of failures in {{suite}} for date {{date}}", suite=suite, date=today)
suite_summary = http.post_json(config.source.url, json={
"from": "unittest",
"groupby": [
{"name": "test", "value": "result.test"}
],
"where": {"and": [
domain,
{"eq": {"result.ok": False}}
]},
"format": "list",
"limit": 100000
})
often_fail = jx.filter(suite_summary.data, {"gt": {"count": 1}})
for g, tests in jx.groupby(often_fail, size=100):
tests = wrap(tests)
if please_stop:
return
Log.note("Collect stats on {{num}} tests", num=len(tests))
tests_summary = http.post_json(config.source.url, json={
"from": "unittest",
"groupby": [
"run.suite",
{"name": "test", "value": "result.test"},
"build.platform",
"build.product",
"build.type",
"run.type"
],
"select": [
{
"name": "date_fails",
"value": {
"mult": [
{"div": [{"sub": {"build.date": today + 0.5 * DAY}}, DAY.seconds]},
{"when": "result.ok", "then": 0, "else": 1}
]
},
"aggregate": "stats"
},
{
"name": "date",
"value": {"div": [{"sub": {"build.date": today + 0.5 * DAY}}, DAY.seconds]},
"aggregate": "stats"
},
{
"name": "fails",
"value": {"when": "result.ok", "then": 0, "else": 1},
"aggregate": "stats"
}
],
"where": {"and": [
domain,
{"in": {"result.test": tests}}
]},
"format": "list",
"limit": 100000
})
# FOR EACH TEST, CALCULATE THE "RECENTLY BAD" STATISTIC (linear regression slope)
# THIS IS ONLY A ROUGH CALC FOR TESTING THE UI
for t in tests_summary.data:
try:
t._id = "-".join([
coalesce(t.build.product, ""),
t.build.platform,
coalesce(t.build.type, ""),
coalesce(t.run.type, ""),
t.run.suite,
t.test,
unicode(today.unix)
])
except Exception, e:
Log.error("text join problem", cause=e)
t.timestamp = today
t.average = t.fails.avg
if t.date.var == 0:
t.slope = 0
else:
t.slope = (t.date_fails.avg - t.date.avg * t.fails.avg) / t.date.var
t.etl.timestamp = Date.now()
# PUSH STATS TO ES
docs = [{"id": t._id, "value": t} for t in tests_summary.data if t.fails.sum > 0]
Log.note("Adding {{num}} test summaries", num=len(docs))
destination.extend(docs)
