def crash_signature_matcher(text_log_error):
"""
Query for TextLogErrorMatches with the same crash signature.
Produces two queries, first checking if the same test produces matches
and secondly checking without the same test but lowering the produced
scores.
"""
failure_line = text_log_error.metadata.failure_line
if (failure_line.action != "crash" or failure_line.signature is None
or failure_line.signature == "None"):
return
f = {
'text_log_error___metadata__failure_line__action':
'crash',
'text_log_error___metadata__failure_line__signature':
failure_line.signature,
}
qwargs = (Q(text_log_error___metadata__best_classification=None)
& (Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job)))
qs = (TextLogErrorMatch.objects.filter(**f).exclude(qwargs).select_related(
'text_log_error',
'text_log_error___metadata').order_by('-score', '-classified_failure'))
size = 20000
time_budget = 500
# See if we can get any matches when filtering by the same test
first_attempt = qs.filter(
text_log_error___metadata__failure_line__test=failure_line.test)
chunks = chunked_qs_reverse(first_attempt, chunk_size=size)
scored_matches = chain.from_iterable(
time_boxed(score_matches, chunks, time_budget))
if scored_matches:
return scored_matches
# try again without filtering to the test but applying a .8 score multiplyer
chunks = chunked_qs_reverse(qs, chunk_size=size)
scored_matches = chain.from_iterable(
time_boxed(
score_matches,
chunks,
time_budget,
score_multiplier=(8, 10),
))
return scored_matches
def crash_signature_matcher(text_log_error):
"""
Query for TextLogErrorMatches with the same crash signature.
Produces two queries, first checking if the same test produces matches
and secondly checking without the same test but lowering the produced
scores.
"""
failure_line = text_log_error.metadata.failure_line
if (failure_line.action != "crash" or
failure_line.signature is None or
failure_line.signature == "None"):
return
f = {
'text_log_error___metadata__failure_line__action': 'crash',
'text_log_error___metadata__failure_line__signature': failure_line.signature,
}
qwargs = (
Q(text_log_error___metadata__best_classification=None)
& (Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job))
)
qs = (TextLogErrorMatch.objects.filter(**f)
.exclude(qwargs)
.select_related('text_log_error', 'text_log_error___metadata')
.order_by('-score', '-classified_failure'))
size = 20000
time_budget = 500
# See if we can get any matches when filtering by the same test
first_attempt = qs.filter(text_log_error___metadata__failure_line__test=failure_line.test)
chunks = chunked_qs_reverse(first_attempt, chunk_size=size)
scored_matches = chain.from_iterable(time_boxed(score_matches, chunks, time_budget))
if scored_matches:
return scored_matches
# try again without filtering to the test but applying a .8 score multiplyer
chunks = chunked_qs_reverse(qs, chunk_size=size)
scored_matches = chain.from_iterable(time_boxed(
score_matches,
chunks,
time_budget,
score_multiplier=(8, 10),
))
return scored_matches
def precise_matcher(text_log_error):
"""Query for TextLogErrorMatches identical to matches of the given TextLogError."""
failure_line = text_log_error.metadata.failure_line
logger.debug("Looking for test match in failure %d", failure_line.id)
if failure_line.action != "test_result" or failure_line.message is None:
return
f = {
'text_log_error___metadata__failure_line__action': 'test_result',
'text_log_error___metadata__failure_line__test': failure_line.test,
'text_log_error___metadata__failure_line__subtest':
failure_line.subtest,
'text_log_error___metadata__failure_line__status': failure_line.status,
'text_log_error___metadata__failure_line__expected':
failure_line.expected,
'text_log_error___metadata__failure_line__message':
failure_line.message
}
qwargs = (Q(text_log_error___metadata__best_classification=None)
& (Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job)))
qs = (TextLogErrorMatch.objects.filter(**f).exclude(qwargs).order_by(
'-score', '-classified_failure'))
if not qs:
return
# chunk through the QuerySet because it could potentially be very large
# time bound each call to the scoring function to avoid job timeouts
# returns an iterable of (score, classified_failure_id) tuples
chunks = chunked_qs_reverse(qs, chunk_size=20000)
return chain.from_iterable(
time_boxed(score_matches, chunks, time_budget=500))
def test_chunked_qs_reverse(test_job):
"""
Test `chunked_qs_reverse` function
Specifically checks the length of chunks and their items don't overlap.
"""
# create 25 failure lines
create_failure_lines(test_job, [(test_line, {}) for i in range(25)])
qs = FailureLine.objects.all()
chunks = list(chunked_qs_reverse(qs, chunk_size=5))
one = chunks[0]
two = chunks[1]
five = chunks[4]
assert len(one) == 5
assert one[0].id == 25
assert one[4].id == 21
assert len(two) == 5
assert two[0].id == 20
assert two[4].id == 16
assert len(five) == 5
assert five[0].id == 5
assert five[4].id == 1
def precise_matcher(text_log_error):
"""Query for TextLogErrorMatches identical to matches of the given TextLogError."""
failure_line = text_log_error.metadata.failure_line
logger.debug("Looking for test match in failure %d", failure_line.id)
if failure_line.action != "test_result" or failure_line.message is None:
return
f = {
'text_log_error___metadata__failure_line__action': 'test_result',
'text_log_error___metadata__failure_line__test': failure_line.test,
'text_log_error___metadata__failure_line__subtest': failure_line.subtest,
'text_log_error___metadata__failure_line__status': failure_line.status,
'text_log_error___metadata__failure_line__expected': failure_line.expected,
'text_log_error___metadata__failure_line__message': failure_line.message
}
qwargs = (
Q(text_log_error___metadata__best_classification=None)
& (Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job))
)
qs = (TextLogErrorMatch.objects.filter(**f)
.exclude(qwargs)
.order_by('-score', '-classified_failure'))
if not qs:
return
# chunk through the QuerySet because it could potentially be very large
# time bound each call to the scoring function to avoid job timeouts
# returns an iterable of (score, classified_failure_id) tuples
chunks = chunked_qs_reverse(qs, chunk_size=20000)
return chain.from_iterable(time_boxed(score_matches, chunks, time_budget=500))
def query_best(self, text_log_error):
"""Query for TextLogErrorMatches identical to matches of the given TextLogError."""
failure_line = text_log_error.metadata.failure_line
logger.debug("Looking for test match in failure %d", failure_line.id)
if failure_line.action != "test_result" or failure_line.message is None:
return
f = {
'text_log_error___metadata__failure_line__action': 'test_result',
'text_log_error___metadata__failure_line__test': failure_line.test,
'text_log_error___metadata__failure_line__subtest': failure_line.subtest,
'text_log_error___metadata__failure_line__status': failure_line.status,
'text_log_error___metadata__failure_line__expected': failure_line.expected,
'text_log_error___metadata__failure_line__message': failure_line.message
}
qwargs = (
Q(text_log_error___metadata__best_classification=None)
& (Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job))
)
qs = (TextLogErrorMatch.objects.filter(**f)
.exclude(qwargs)
.order_by('-score', '-classified_failure'))
if not qs:
return
chunks = chunked_qs_reverse(qs, chunk_size=20000)
matches = chain.from_iterable(time_boxed(score_matches, chunks, time_budget=500))
return score_by_classified_fail_id(matches)
def query_best(self, text_log_error):
"""Query for TextLogErrorMatches identical to matches of the given TextLogError."""
failure_line = text_log_error.metadata.failure_line
logger.debug("Looking for test match in failure %d", failure_line.id)
if failure_line.action != "test_result" or failure_line.message is None:
return
f = {
'text_log_error___metadata__failure_line__action': 'test_result',
'text_log_error___metadata__failure_line__test': failure_line.test,
'text_log_error___metadata__failure_line__subtest': failure_line.subtest,
'text_log_error___metadata__failure_line__status': failure_line.status,
'text_log_error___metadata__failure_line__expected': failure_line.expected,
'text_log_error___metadata__failure_line__message': failure_line.message
}
qwargs = (
Q(text_log_error___metadata__best_classification=None)
& (Q(text_log_error___metadata__best_is_verified=True)
| Q(text_log_error__step__job=text_log_error.step.job))
)
qs = (TextLogErrorMatch.objects.filter(**f)
.exclude(qwargs)
.order_by('-score', '-classified_failure'))
if not qs:
return
chunks = chunked_qs_reverse(qs, chunk_size=20000)
matches = chain.from_iterable(time_boxed(score_matches, chunks, time_budget=500))
return score_by_classified_fail_id(matches)
def test_chunked_qs_reverse_with_empty_qs():
chunks = list(chunked_qs_reverse(FailureLine.objects.none()))
assert len(chunks) == 0