def create_artifacts(project, data):
artifacts = ArtifactsModel.serialize_artifact_json_blobs(data)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(artifacts)
# If a ``text_log_summary`` and ``Bug suggestions`` artifact are
# posted here together, for the same ``job_guid``, then just load
# them. This is how it is done internally in our log parser
# so there is no delay in creation and the bug suggestions show
# as soon as the log is parsed.
#
# If a ``text_log_summary`` is posted WITHOUT an accompanying
# ``Bug suggestions`` artifact, then schedule to create it
# asynchronously so that this api does not take too long.
tls_list = get_artifacts_that_need_bug_suggestions(artifacts)
# tls_list will contain all ``text_log_summary`` artifacts that
# do NOT have an accompanying ``Bug suggestions`` artifact in this
# current list of artifacts. If it's empty, then we don't need
# to schedule anything.
if tls_list:
populate_error_summary.apply_async(args=[project, tls_list],
routing_key='error_summary')
def create(self, request, project):
artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA)
job_guids = [x['job_guid'] for x in artifacts]
with JobsModel(project) as jobs_model, ArtifactsModel(project) as artifacts_model:
job_id_lookup = jobs_model.get_job_ids_by_guid(job_guids)
artifacts_model.load_job_artifacts(artifacts, job_id_lookup)
# If a ``text_log_summary`` and ``Bug suggestions`` artifact are
# posted here together, for the same ``job_guid``, then just load
# them. This is how it is done internally in our log parser
# so there is no delay in creation and the bug suggestions show
# as soon as the log is parsed.
#
# If a ``text_log_summary`` is posted WITHOUT an accompanying
# ``Bug suggestions`` artifact, then schedule to create it
# asynchronously so that this api does not take too long.
tls_list = get_artifacts_that_need_bug_suggestions(artifacts)
# tls_list will contain all ``text_log_summary`` artifacts that
# do NOT have an accompanying ``Bug suggestions`` artifact in this
# current list of artifacts. If it's empty, then we don't need
# to schedule anything.
if tls_list:
populate_error_summary.apply_async(
args=[project, tls_list, job_id_lookup],
routing_key='error_summary'
)
return Response({'message': 'Artifacts stored successfully'})
def create(self, request, project):
artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA)
job_guids = [x['job_guid'] for x in artifacts]
with JobsModel(project) as jobs_model, ArtifactsModel(
project) as artifacts_model:
job_id_lookup = jobs_model.get_job_ids_by_guid(job_guids)
artifacts_model.load_job_artifacts(artifacts, job_id_lookup)
# If a ``text_log_summary`` and ``Bug suggestions`` artifact are
# posted here together, for the same ``job_guid``, then just load
# them. This is how it is done internally in our log parser
# so there is no delay in creation and the bug suggestions show
# as soon as the log is parsed.
#
# If a ``text_log_summary`` is posted WITHOUT an accompanying
# ``Bug suggestions`` artifact, then schedule to create it
# asynchronously so that this api does not take too long.
tls_list = get_artifacts_that_need_bug_suggestions(artifacts)
# tls_list will contain all ``text_log_summary`` artifacts that
# do NOT have an accompanying ``Bug suggestions`` artifact in this
# current list of artifacts. If it's empty, then we don't need
# to schedule anything.
if tls_list:
populate_error_summary.apply_async(
args=[project, tls_list, job_id_lookup],
routing_key='error_summary')
return Response({'message': 'Artifacts stored successfully'})
def create(self, request, project):
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
request.data)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
return Response({'message': 'Artifacts stored successfully'})
def create(self, request, project):
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
request.data)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
return Response({'message': 'Artifacts stored successfully'})
def post_log_artifacts(project, job_guid, job_log):
"""Post a list of artifacts to a job."""
log_url = job_log.url
log_description = "%s %s (%s)" % (project, job_guid, log_url)
logger.debug("Downloading/parsing log for %s", log_description)
try:
artifact_list = extract_text_log_artifacts(project, log_url, job_guid)
except Exception as e:
job_log.update_status(JobLog.FAILED)
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
log_description, e)
return
if isinstance(e, urllib2.URLError):
# possibly recoverable http error (e.g. problems on our end)
logger.error("Failed to download log for %s: %s", log_description,
e)
else:
# parse error or other unrecoverable error
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
raise
try:
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
artifact_list)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
job_log.update_status(JobLog.PARSED)
logger.debug("Stored artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to store parsed artifact for %s: %s",
log_description, e)
raise
def post_log_artifacts(job_log):
"""Post a list of artifacts to a job."""
logger.debug("Downloading/parsing log for log %s", job_log.id)
try:
artifact_list = extract_text_log_artifacts(job_log)
except Exception as e:
job_log.update_status(JobLog.FAILED)
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s", job_log.id, e)
return
if isinstance(e, urllib2.URLError):
# possibly recoverable http error (e.g. problems on our end)
logger.error("Failed to download log for %s: %s", job_log.id, e)
else:
# parse error or other unrecoverable error
logger.error("Failed to download/parse log for %s: %s", job_log.id, e)
raise
try:
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
artifact_list)
project = job_log.job.repository.name
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
job_log.update_status(JobLog.PARSED)
logger.debug("Stored artifact for %s %s", project, job_log.job.id)
except Exception as e:
logger.error("Failed to store parsed artifact for %s: %s", job_log.id, e)
raise