def _crossreference(job):
if TextLogSummary.objects.filter(job_guid=job.guid).exists():
logger.info("crossreference_error_lines already ran for job %s" %
job.id)
return
failure_lines = FailureLine.objects.filter(job_guid=job.guid)
text_log_errors = TextLogError.objects.filter(
step__job=job).order_by('line_number')
# If we don't have both failure lines and text log errors nothing will happen
# so return early
if not (failure_lines.exists() and text_log_errors.exists()):
return False
summary = TextLogSummary.objects.create(job_guid=job.guid,
repository=job.repository)
match_iter = structured_iterator(failure_lines)
failure_line, regexp = match_iter.next()
summary_lines = []
# For each error in the text log, try to match the next unmatched
# structured log line
for error in text_log_errors:
if regexp and regexp.match(error.line.strip()):
logger.debug("Matched '%s'" % (error.line, ))
summary_lines.append(
TextLogSummaryLine(summary=summary,
line_number=error.line_number,
failure_line=failure_line))
TextLogErrorMetadata.objects.create(text_log_error=error,
failure_line=failure_line)
failure_line, regexp = match_iter.next()
else:
logger.debug("Failed to match '%s'" % (error.line, ))
summary_lines.append(
TextLogSummaryLine(summary=summary,
line_number=error.line_number,
failure_line=None))
TextLogSummaryLine.objects.bulk_create(summary_lines)
# We should have exhausted all structured lines
for leftover in match_iter:
# We can have a line without a pattern at the end if the log is truncated
if leftover[1] is None:
break
logger.error(
"Failed to match structured line '%s' to an unstructured line" %
(leftover[1].pattern, ))
return bool(summary_lines)
def crossreference_error_lines(self, repository, job_guid, failure_lines,
text_log_errors, bug_suggestions):
"""Populate the TextLogSummary and TextLogSummaryLine tables for a
job. Specifically this function tries to match the
unstructured error lines with the corresponding structured error lines, relying on
the fact that serialization of mozlog (and hence errorsummary files) is determinisic
so we can reserialize each structured error line and perform an in-order textual
match.
:param repository: Repository containing the job
:param job_guid: guid for the job being crossreferenced
:param failure_lines: List of FailureLine objects for this job
:param text_log_summary: text_log_summary artifact for this job
:param bug_suggestions: Bug suggestions artifact for this job"""
summary, _ = TextLogSummary.objects.get_or_create(job_guid=job_guid,
repository=repository)
summary.bug_suggestions_artifact_id = bug_suggestions["id"]
summary.save()
match_iter = structured_iterator(failure_lines)
failure_line, regexp = match_iter.next()
summary_lines = []
# For each error in the text log, try to match the next unmatched
# structured log line
for error in text_log_errors:
if regexp and regexp.match(error.line.strip()):
logger.debug("Matched '%s'" % (error.line,))
summary_lines.append(TextLogSummaryLine(
summary=summary,
line_number=error.line_number,
failure_line=failure_line))
failure_line, regexp = match_iter.next()
else:
logger.debug("Failed to match '%s'" % (error.line,))
summary_lines.append(TextLogSummaryLine(
summary=summary,
line_number=error.line_number,
failure_line=None))
TextLogSummaryLine.objects.bulk_create(summary_lines)
# We should have exhausted all structured lines
for leftover in match_iter:
# We can have a line without a pattern at the end if the log is truncated
if leftover[1] is None:
break
logger.error("Failed to match structured line '%s' to an unstructured line" % (leftover[1].pattern,))
def text_summary_lines(test_job, failure_lines, text_log_error_lines):
from treeherder.model.models import TextLogSummary, TextLogSummaryLine
summary = TextLogSummary(job_guid=test_job.guid,
repository=test_job.repository)
summary.save()
summary_lines = []
for line in failure_lines:
summary_line = TextLogSummaryLine(summary=summary,
line_number=line.line,
failure_line=line)
summary_line.save()
summary_lines.append(summary_line)
return summary_lines
def text_summary_lines(test_job, failure_lines, text_log_error_lines):
from treeherder.model.models import TextLogSummary, TextLogSummaryLine
summary = TextLogSummary(
job_guid=test_job.guid,
repository=test_job.repository
)
summary.save()
summary_lines = []
for line in failure_lines:
summary_line = TextLogSummaryLine(
summary=summary,
line_number=line.line,
failure_line=line)
summary_line.save()
summary_lines.append(summary_line)
return summary_lines
def text_summary_lines(jm, failure_lines, test_repository, artifacts):
from treeherder.model.models import TextLogSummary, TextLogSummaryLine
job = jm.get_job(1)[0]
summary = TextLogSummary(job_guid=job["job_guid"],
repository=test_repository,
text_log_summary_artifact_id=artifacts[0]["id"],
bug_suggestions_artifact_id=artifacts[1]["id"])
summary.save()
summary_lines = []
for line in failure_lines:
summary_line = TextLogSummaryLine(summary=summary,
line_number=line.line,
failure_line=line)
summary_line.save()
summary_lines.append(summary_line)
return summary_lines
def text_summary_lines(jm, failure_lines, test_repository, artifacts):
from treeherder.model.models import TextLogSummary, TextLogSummaryLine
job = jm.get_job(1)[0]
summary = TextLogSummary(
job_guid=job["job_guid"],
repository=test_repository,
text_log_summary_artifact_id=artifacts[0]["id"],
bug_suggestions_artifact_id=artifacts[1]["id"]
)
summary.save()
summary_lines = []
for line in failure_lines:
summary_line = TextLogSummaryLine(
summary=summary,
line_number=line.line,
failure_line=line)
summary_line.save()
summary_lines.append(summary_line)
return summary_lines