def query(self, command):
"""
WILL STALL CALLING THREAD UNTIL THE command IS COMPLETED
:param command: COMMAND FOR SQLITE
:return: list OF RESULTS
"""
signal = Signal()
result = Dict()
self.queue.add((command, result, signal, None))
signal.wait_for_go()
if result.exception:
Log.error("Problem with Sqlite call", cause=result.exception)
return result
def query(self, command):
"""
WILL BLOCK CALLING THREAD UNTIL THE command IS COMPLETED
:param command: COMMAND FOR SQLITE
:return: list OF RESULTS
"""
signal = Signal()
result = Dict()
self.queue.add((command, result, signal, None))
signal.wait_for_go()
if result.exception:
Log.error("Problem with Sqlite call", cause=result.exception)
return result
def _get_job_results_from_th(self, branch, revision):
output = []
with self.locker:
waiting_threads = self.pending.get((branch, revision))
if waiting_threads is None:
sig = None
waiting_threads = self.pending[(branch, revision)] = [output]
else:
sig = Signal()
waiting_threads.append(Signal())
if sig is not None:
Log.note("Holding thread for {{branch}}/{{revision}}",
branch=branch,
revision=revision)
sig.wait_for_go()
return waiting_threads[0]
try:
results = DictList()
while True:
response = self._rate_limited_get_json(
expand_template(RESULT_SET_URL, {
"branch": branch,
"revision": revision[0:12:]
}))
results.extend(response.results)
if len(response.results) != 1000:
break
for g, repo_ids in jx.groupby(results.id, size=10):
jobs = DictList()
with Timer("Get {{num}} jobs", {"num": len(repo_ids)},
debug=DEBUG):
while True:
response = self._rate_limited_get_json(
expand_template(
JOBS_URL, {
"branch":
branch,
"offset":
len(jobs),
"result_set_id":
",".join(map(unicode, repo_ids))
}))
jobs.extend(response.results)
if len(response.results) != 2000:
break
with Timer("Get (up to {{num}}) details from TH",
{"num": len(jobs)},
debug=DEBUG):
details = []
for _, ids in jx.groupby(jobs.id, size=40):
details.extend(
self._rate_limited_get_json(url=expand_template(
DETAILS_URL, {
"branch": branch,
"job_id": ",".join(map(unicode, ids))
}),
retry={
"times": 3
}).results)
details = {
k.job_guid: list(v)
for k, v in jx.groupby(details, "job_guid")
}
with Timer("Get (up to {{num}}) stars from TH",
{"num": len(jobs)},
debug=DEBUG):
stars = []
for _, ids in jx.groupby(jobs.id, size=40):
response = self._rate_limited_get_json(
expand_template(
JOB_BUG_MAP, {
"branch": branch,
"job_id": "&job_id=".join(map(
unicode, ids))
}))
stars.extend(response),
stars = {
k.job_id: list(v)
for k, v in jx.groupby(stars, "job_id")
}
with Timer("Get notes from TH", debug=DEBUG):
notes = []
for jid in set([
j.id
for j in jobs if j.failure_classification_id != 1
] + stars.keys()):
response = self._rate_limited_get_json(
expand_template(NOTES_URL, {
"branch": branch,
"job_id": unicode(jid)
}))
notes.extend(response),
notes = {
k.job_id: list(v)
for k, v in jx.groupby(notes, "job_id")
}
for j in jobs:
output.append(
self._normalize_job_result(branch, revision, j,
details, notes, stars))
if output:
with Timer("Write to ES cache", debug=DEBUG):
self.cache.extend(
{
"id": "-".join([c.repo.branch,
unicode(c.job.id)]),
"value": c
} for c in output)
try:
self.cache.flush()
except Exception, e:
Log.warning("problem flushing. nevermind.", cause=e)
finally:
with self.locker:
for p in waiting_threads[1:]:
if DEBUG:
Log.note(
"releasing thread for {{branch}}/{{revision}}",
branch=branch,
revision=revision)
p.go()
self.pending[(branch, revision)] = None
return output
def _get_job_results_from_th(self, branch, revision):
output = []
with self.locker:
waiting_threads = self.pending.get((branch, revision))
if waiting_threads is None:
sig = None
waiting_threads = self.pending[(branch, revision)] = [output]
else:
sig = Signal()
waiting_threads.append(Signal())
if sig is not None:
Log.note("Holding thread for {{branch}}/{{revision}}", branch=branch, revision=revision)
sig.wait_for_go()
return waiting_threads[0]
try:
results = DictList()
while True:
response = self._rate_limited_get_json(expand_template(RESULT_SET_URL, {"branch": branch, "revision": revision[0:12:]}))
results.extend(response.results)
if len(response.results) != 1000:
break
for g, repo_ids in jx.groupby(results.id, size=10):
jobs = DictList()
with Timer("Get {{num}} jobs", {"num": len(repo_ids)}, debug=DEBUG):
while True:
response = self._rate_limited_get_json(expand_template(JOBS_URL, {"branch": branch, "offset": len(jobs), "result_set_id": ",".join(map(unicode, repo_ids))}))
jobs.extend(response.results)
if len(response.results) != 2000:
break
with Timer("Get (up to {{num}}) details from TH", {"num": len(jobs)}, debug=DEBUG):
details = []
for _, ids in jx.groupby(jobs.id, size=40):
details.extend(self._rate_limited_get_json(
url=expand_template(DETAILS_URL, {"branch": branch, "job_id": ",".join(map(unicode, ids))}),
retry={"times": 3}
).results)
details = {k.job_guid: list(v) for k, v in jx.groupby(details, "job_guid")}
with Timer("Get (up to {{num}}) stars from TH", {"num": len(jobs)}, debug=DEBUG):
stars = []
for _, ids in jx.groupby(jobs.id, size=40):
response = self._rate_limited_get_json(expand_template(JOB_BUG_MAP, {"branch": branch, "job_id": "&job_id=".join(map(unicode, ids))}))
stars.extend(response),
stars = {k.job_id: list(v) for k, v in jx.groupby(stars, "job_id")}
with Timer("Get notes from TH", debug=DEBUG):
notes = []
for jid in set([j.id for j in jobs if j.failure_classification_id != 1] + stars.keys()):
response = self._rate_limited_get_json(expand_template(NOTES_URL, {"branch": branch, "job_id": unicode(jid)}))
notes.extend(response),
notes = {k.job_id: list(v) for k, v in jx.groupby(notes, "job_id")}
for j in jobs:
output.append(self._normalize_job_result(branch, revision, j, details, notes, stars))
if output:
with Timer("Write to ES cache", debug=DEBUG):
self.cache.extend({"id": "-".join([c.repo.branch, unicode(c.job.id)]), "value": c} for c in output)
try:
self.cache.flush()
except Exception, e:
Log.warning("problem flushing. nevermind.", cause=e)
finally:
with self.locker:
for p in waiting_threads[1:]:
if DEBUG:
Log.note("releasing thread for {{branch}}/{{revision}}", branch=branch, revision=revision)
p.go()
self.pending[(branch, revision)] = None
return output
