def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb): """Post a list of artifacts to a job.""" def _retry(e): # Initially retry after 1 minute, then for each subsequent retry # lengthen the retry time by another minute. retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60) # .retry() raises a RetryTaskError exception, # so nothing after this function will be executed log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url']) logger.debug("Downloading/parsing log for %s", log_description) credentials = OAuthCredentials.get_credentials(project) auth = TreeherderAuth(credentials.get('consumer_key'), credentials.get('consumer_secret'), project) client = TreeherderClient(protocol=settings.TREEHERDER_REQUEST_PROTOCOL, host=settings.TREEHERDER_REQUEST_HOST, auth=auth) try: artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid) except Exception as e: client.update_parse_status(project, job_log_url['id'], 'failed') # unrecoverable http error (doesn't exist or permission denied) # (apparently this can happen somewhat often with taskcluster if # the job fails, so just warn about it -- see # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248) if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.warning("Unable to retrieve log for %s: %s", log_description, e) return # possibly recoverable http error (e.g. problems on our end) elif isinstance(e, urllib2.URLError): logger.error("Failed to download log for %s: %s", log_description, e) _retry(e) # parse error or other unrecoverable error else: logger.error("Failed to download/parse log for %s: %s", log_description, e) # re-raise exception if we're not retrying, so new relic sees the # error raise # store the artifacts generated tac = TreeherderArtifactCollection() for artifact in artifact_list: ta = tac.get_artifact(artifact) tac.add(ta) try: client.post_collection(project, tac) client.update_parse_status(project, job_log_url['id'], 'parsed') logger.debug("Finished posting artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to upload parsed artifact for %s: %s", log_description, e) _retry(e)
def test_update_parse_status_nonexistent_id(test_project, mock_post_json): """ Attempting to update the parse status for a non-existent log should return a 404. """ client = TreeherderClient(protocol='http', host='localhost') non_existent_id = 9999999 with pytest.raises(AppError) as e: client.update_parse_status(test_project, non_existent_id, 'parsed') assert "404 NOT FOUND" in str(e.value)
def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb, check_errors=False): """Post a list of artifacts to a job.""" def _retry(e): # Initially retry after 1 minute, then for each subsequent retry # lengthen the retry time by another minute. retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60) # .retry() raises a RetryTaskError exception, # so nothing after this function will be executed log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url']) logger.debug("Downloading/parsing log for %s", log_description) credentials = OAuthCredentials.get_credentials(project) client = TreeherderClient( protocol=settings.TREEHERDER_REQUEST_PROTOCOL, host=settings.TREEHERDER_REQUEST_HOST ) try: artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid, check_errors) except Exception as e: client.update_parse_status(project, credentials.get('consumer_key'), credentials.get('consumer_secret'), job_log_url['id'], 'failed') if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.debug("Unable to retrieve log for %s: %s", log_description, e) return logger.error("Failed to download/parse log for %s: %s", log_description, e) _retry(e) # store the artifacts generated tac = TreeherderArtifactCollection() for artifact in artifact_list: ta = tac.get_artifact(artifact) tac.add(ta) try: client.post_collection(project, credentials.get('consumer_key'), credentials.get('consumer_secret'), tac) client.update_parse_status(project, credentials.get('consumer_key'), credentials.get('consumer_secret'), job_log_url['id'], 'parsed') logger.debug("Finished posting artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to upload parsed artifact for %s: %s", log_description, e) _retry(e)
def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb): """Post a list of artifacts to a job.""" def _retry(e): # Initially retry after 1 minute, then for each subsequent retry # lengthen the retry time by another minute. retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60) # .retry() raises a RetryTaskError exception, # so nothing after this function will be executed log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url']) logger.debug("Downloading/parsing log for %s", log_description) credentials = OAuthCredentials.get_credentials(project) auth = TreeherderAuth(credentials.get('consumer_key'), credentials.get('consumer_secret'), project) client = TreeherderClient( protocol=settings.TREEHERDER_REQUEST_PROTOCOL, host=settings.TREEHERDER_REQUEST_HOST, auth=auth ) try: artifact_list = extract_artifacts_cb(project, job_log_url['url'], job_guid) except Exception as e: client.update_parse_status(project, job_log_url['id'], 'failed') # unrecoverable http error (doesn't exist or permission denied) # (apparently this can happen somewhat often with taskcluster if # the job fails, so just warn about it -- see # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248) if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.warning("Unable to retrieve log for %s: %s", log_description, e) return # possibly recoverable http error (e.g. problems on our end) elif isinstance(e, urllib2.URLError): logger.error("Failed to download log for %s: %s", log_description, e) _retry(e) # parse error or other unrecoverable error else: logger.error("Failed to download/parse log for %s: %s", log_description, e) # re-raise exception if we're not retrying, so new relic sees the # error raise # store the artifacts generated tac = TreeherderArtifactCollection() for artifact in artifact_list: ta = tac.get_artifact(artifact) tac.add(ta) try: client.post_collection(project, tac) client.update_parse_status(project, job_log_url['id'], 'parsed') logger.debug("Finished posting artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to upload parsed artifact for %s: %s", log_description, e) _retry(e)