def parse_log(project, job_id, result_set_id, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
pattern_obj = re.compile('\d+:\d+:\d+\s+')
jm = JobsModel(project=project)
rdm = RefDataManager()
open_bugs_cache = {}
closed_bugs_cache = {}
status_publisher = JobStatusPublisher(settings.BROKER_URL)
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
try:
# return the resultset with the job id to identify if the UI wants
# to fetch the whole thing.
resultset = jm.get_result_set_by_id(result_set_id=result_set_id)[0]
del(resultset["active_status"])
del(resultset["revision_hash"])
log_references = jm.get_log_references(job_id)
# we may have many log references per job
for log in log_references:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(
log['url'],
check_errors=check_errors,
)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_id, name, 'json', json.dumps(artifact)))
if check_errors:
# I'll try to begin with a full_text search on the entire row
all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
open_bugs_suggestions = {}
closed_bugs_suggestions = {}
for err in all_errors:
# remove timestamp
clean_line = pattern_obj.sub('', err['line'])
if clean_line not in open_bugs_cache:
open_bugs_cache[clean_line] = rdm.get_suggested_bugs(
clean_line)
if clean_line not in closed_bugs_cache:
closed_bugs_cache[clean_line] = rdm.get_suggested_bugs(
clean_line, open_bugs=False)
open_bugs_suggestions[ err['line'] ] = open_bugs_cache[clean_line]
closed_bugs_suggestions[ err['line'] ] = closed_bugs_cache[clean_line]
artifact_list.append((job_id, 'Open bugs', 'json', json.dumps(open_bugs_suggestions)))
artifact_list.append((job_id, 'Closed bugs', 'json', json.dumps(closed_bugs_suggestions)))
# store the artifacts generated
jm.store_job_artifact(artifact_list)
status_publisher.publish(job_id, resultset, project, 'processed')
if check_errors:
failure_publisher.publish(job_id, project)
finally:
rdm.disconnect()
jm.disconnect()
status_publisher.disconnect()
failure_publisher.disconnect()
def parse_log(project, log_url, job_guid, resultset, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
mozharness_pattern = re.compile(
'^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
)
bugs_cache = {'open': {}, 'closed': {}}
bug_suggestions = {'open': {}, 'closed': {}}
status_publisher = JobStatusPublisher(settings.BROKER_URL)
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
try:
# return the resultset with the job id to identify if the UI wants
# to fetch the whole thing.
bugscache_uri = '{0}{1}'.format(
settings.API_HOSTNAME,
reverse("bugscache-list")
)
credentials = OAuthCredentials.get_credentials(project)
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(
log_url,
check_errors=check_errors,
)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
for err in all_errors:
# remove the mozharness prefix
clean_line = mozharness_pattern.sub('', err['line']).strip()
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
# collect open and closed bugs suggestions
for status in ('open', 'closed'):
if not search_term:
bug_suggestions[status][clean_line] = []
continue
if search_term not in bugs_cache[status]:
# retrieve the list of suggestions from the api
bugs_cache[status][search_term] = get_bugs_for_search_term(
search_term,
status,
bugscache_uri
)
# no suggestions, try to use the crash signature as search term
if not bugs_cache[status][search_term]:
crash_signature = get_crash_signature(search_term)
if crash_signature:
bugs_cache[status][search_term] = get_bugs_for_search_term(
search_term,
status,
bugscache_uri
)
bug_suggestions[status][clean_line] = bugs_cache[status][search_term]
artifact_list.append((job_guid, 'Open bugs', 'json', json.dumps(bug_suggestions['open'])))
artifact_list.append((job_guid, 'Closed bugs', 'json', json.dumps(bug_suggestions['closed'])))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
req.send(tac)
status_publisher.publish(job_guid, resultset, project, 'processed')
if check_errors:
failure_publisher.publish(job_guid, project)
finally:
status_publisher.disconnect()
failure_publisher.disconnect()