def _schedule_log_parsing(job, job_logs, result):
"""Kick off the initial task that parses the log data.
log_data is a list of job log objects and the result for that job
"""
# importing here to avoid an import loop
from treeherder.log_parser.tasks import parse_logs
task_types = {"errorsummary_json", "buildbot_text", "builds-4h"}
job_log_ids = []
for job_log in job_logs:
# a log can be submitted already parsed. So only schedule
# a parsing task if it's ``pending``
# the submitter is then responsible for submitting the
# text_log_summary artifact
if job_log.status != JobLog.PENDING:
continue
# if this is not a known type of log, abort parse
if job_log.name not in task_types:
continue
job_log_ids.append(job_log.id)
if result != 'success':
priority = 'failures'
else:
priority = "normal"
parse_logs.apply_async(routing_key="log_parser.%s" % priority,
args=[job.id, job_log_ids, priority])
def _schedule_log_parsing(job, job_logs, result, repository):
"""Kick off the initial task that parses the log data.
log_data is a list of job log objects and the result for that job
"""
# importing here to avoid an import loop
from treeherder.log_parser.tasks import parse_logs
task_types = {"errorsummary_json", "live_backing_log"}
sheriffed_repos = {
"autoland",
"mozilla-central",
"mozilla-beta",
"mozilla-release",
"mozilla-esr78",
"mozilla-esr91",
}
job_log_ids = []
for job_log in job_logs:
# a log can be submitted already parsed. So only schedule
# a parsing task if it's ``pending``
# the submitter is then responsible for submitting the
# text_log_summary artifact
if job_log.status != JobLog.PENDING:
continue
# if this is not a known type of log, abort parse
if job_log.name not in task_types:
continue
job_log_ids.append(job_log.id)
# TODO: Replace the use of different queues for failures vs not with the
# RabbitMQ priority feature (since the idea behind separate queues was
# only to ensure failures are dealt with first if there is a backlog).
if result != 'success':
if job_log.name == "errorsummary_json":
queue = "log_parser_fail_json"
priority = "failures"
else:
queue = "log_parser_fail_raw"
priority = "failures"
if repository.name in sheriffed_repos:
queue += "_sheriffed"
else:
queue += "_unsheriffed"
else:
queue = 'log_parser'
priority = "normal"
parse_logs.apply_async(queue=queue,
args=[job.id, [job_log.id], priority])
def _schedule_log_parsing(job, job_logs, result):
"""Kick off the initial task that parses the log data.
log_data is a list of job log objects and the result for that job
"""
# importing here to avoid an import loop
from treeherder.log_parser.tasks import parse_logs
task_types = {
"errorsummary_json",
"buildbot_text",
"builds-4h"
}
job_log_ids = []
for job_log in job_logs:
# a log can be submitted already parsed. So only schedule
# a parsing task if it's ``pending``
# the submitter is then responsible for submitting the
# text_log_summary artifact
if job_log.status != JobLog.PENDING:
continue
# if this is not a known type of log, abort parse
if job_log.name not in task_types:
continue
job_log_ids.append(job_log.id)
# TODO: Replace the use of different queues for failures vs not with the
# RabbitMQ priority feature (since the idea behind separate queues was
# only to ensure failures are dealt with first if there is a backlog).
if result != 'success':
queue = 'log_parser_fail'
priority = 'failures'
else:
queue = 'log_parser'
priority = "normal"
parse_logs.apply_async(queue=queue,
args=[job.id, job_log_ids, priority])
def _schedule_log_parsing(job, job_logs, result):
"""Kick off the initial task that parses the log data.
log_data is a list of job log objects and the result for that job
"""
# importing here to avoid an import loop
from treeherder.log_parser.tasks import parse_logs
task_types = {
"errorsummary_json",
"buildbot_text",
"builds-4h"
}
job_log_ids = []
for job_log in job_logs:
# a log can be submitted already parsed. So only schedule
# a parsing task if it's ``pending``
# the submitter is then responsible for submitting the
# text_log_summary artifact
if job_log.status != JobLog.PENDING:
continue
# if this is not a known type of log, abort parse
if job_log.name not in task_types:
continue
job_log_ids.append(job_log.id)
if result != 'success':
priority = 'failures'
else:
priority = "normal"
parse_logs.apply_async(routing_key="log_parser.%s" % priority,
args=[job.id, job_log_ids, priority])
