def test_send_artifact_collection(self): """Can add a artifact collections to a TreeherderRequest.""" tac = TreeherderArtifactCollection() for artifact in self.artifact_data: tac.add(tac.get_artifact(artifact)) client = TreeherderClient( server_url='http://host', client_id='client-abc', secret='secret123', ) def request_callback(request): # Check that the expected content was POSTed. posted_json = json.loads(request.body) self.assertEqual(posted_json, tac.get_collection_data()) return (200, {}, '{"message": "Artifacts stored successfully"}') url = client._get_endpoint_url(tac.endpoint_base, project='project') responses.add_callback(responses.POST, url, match_querystring=True, callback=request_callback, content_type='application/json') client.post_collection('project', tac)
def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb): """Post a list of artifacts to a job.""" def _retry(e): # Initially retry after 1 minute, then for each subsequent retry # lengthen the retry time by another minute. retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60) # .retry() raises a RetryTaskError exception, # so nothing after this function will be executed log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url']) logger.debug("Downloading/parsing log for %s", log_description) credentials = OAuthCredentials.get_credentials(project) auth = TreeherderAuth(credentials.get('consumer_key'), credentials.get('consumer_secret'), project) client = TreeherderClient(protocol=settings.TREEHERDER_REQUEST_PROTOCOL, host=settings.TREEHERDER_REQUEST_HOST, auth=auth) try: artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid) except Exception as e: client.update_parse_status(project, job_log_url['id'], 'failed') # unrecoverable http error (doesn't exist or permission denied) # (apparently this can happen somewhat often with taskcluster if # the job fails, so just warn about it -- see # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248) if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.warning("Unable to retrieve log for %s: %s", log_description, e) return # possibly recoverable http error (e.g. problems on our end) elif isinstance(e, urllib2.URLError): logger.error("Failed to download log for %s: %s", log_description, e) _retry(e) # parse error or other unrecoverable error else: logger.error("Failed to download/parse log for %s: %s", log_description, e) # re-raise exception if we're not retrying, so new relic sees the # error raise # store the artifacts generated tac = TreeherderArtifactCollection() for artifact in artifact_list: ta = tac.get_artifact(artifact) tac.add(ta) try: client.post_collection(project, tac) client.update_parse_status(project, job_log_url['id'], 'parsed') logger.debug("Finished posting artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to upload parsed artifact for %s: %s", log_description, e) _retry(e)
def test_send_artifact_collection(self, mock_post): """Can add a artifact collections to a TreeherderRequest.""" mock_post.return_value = self._expected_response_return_object() tac = TreeherderArtifactCollection() for artifact in self.artifact_data: tac.add(tac.get_artifact(artifact)) client = TreeherderClient( protocol='http', host='host', client_id='client-abc', secret='secret123', ) client.post_collection('project', tac) path, resp = mock_post.call_args self.assertEqual(mock_post.call_count, 1) self.assertEqual( tac.get_collection_data(), resp['json'] )
def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb): """Post a list of artifacts to a job.""" def _retry(e): # Initially retry after 1 minute, then for each subsequent retry # lengthen the retry time by another minute. retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60) # .retry() raises a RetryTaskError exception, # so nothing after this function will be executed log_description = "%s %s (%s)" % (project, job_guid, job_log_url) logger.debug("Downloading/parsing log for %s", log_description) job_log = JobLog.objects.get(job__guid=job_guid, url=job_log_url) credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID) client = TreeherderClient( protocol=settings.TREEHERDER_REQUEST_PROTOCOL, host=settings.TREEHERDER_REQUEST_HOST, client_id=credentials.client_id, secret=str(credentials.secret), ) try: artifact_list = extract_artifacts_cb(project, job_log_url, job_guid) except Exception as e: job_log.update_status(JobLog.FAILED) # unrecoverable http error (doesn't exist or permission denied) # (apparently this can happen somewhat often with taskcluster if # the job fails, so just warn about it -- see # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248) if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.warning("Unable to retrieve log for %s: %s", log_description, e) return # possibly recoverable http error (e.g. problems on our end) elif isinstance(e, urllib2.URLError): logger.error("Failed to download log for %s: %s", log_description, e) _retry(e) # parse error or other unrecoverable error else: logger.error("Failed to download/parse log for %s: %s", log_description, e) # re-raise exception if we're not retrying, so new relic sees the # error raise # store the artifacts generated tac = TreeherderArtifactCollection() for artifact in artifact_list: ta = tac.get_artifact(artifact) tac.add(ta) try: client.post_collection(project, tac) job_log.update_status(JobLog.PARSED) logger.debug("Finished posting artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to upload parsed artifact for %s: %s", log_description, e) _retry(e)
def test_artifact_collection(self): """Confirm the collection matches the sample data""" tac = TreeherderArtifactCollection() for artifact in self.artifact_data: ta = TreeherderArtifact(artifact) tac.add(ta) self.assertTrue(len(self.artifact_data) == len(tac.data))
def test_chunk_endpoint_base(self): """Confirm the endpoint_base of the chunks is the same as the original""" tac = TreeherderArtifactCollection() for artifact in self.artifact_data: ta = TreeherderArtifact(artifact) tac.add(ta) for chunk in tac.get_chunks(3): assert tac.endpoint_base == chunk.endpoint_base
def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb, check_errors=False): """Post a list of artifacts to a job.""" def _retry(e): # Initially retry after 1 minute, then for each subsequent retry # lengthen the retry time by another minute. retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60) # .retry() raises a RetryTaskError exception, # so nothing after this function will be executed log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url']) logger.debug("Downloading/parsing log for %s", log_description) credentials = OAuthCredentials.get_credentials(project) client = TreeherderClient( protocol=settings.TREEHERDER_REQUEST_PROTOCOL, host=settings.TREEHERDER_REQUEST_HOST ) try: artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid, check_errors) except Exception as e: client.update_parse_status(project, credentials.get('consumer_key'), credentials.get('consumer_secret'), job_log_url['id'], 'failed') if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.debug("Unable to retrieve log for %s: %s", log_description, e) return logger.error("Failed to download/parse log for %s: %s", log_description, e) _retry(e) # store the artifacts generated tac = TreeherderArtifactCollection() for artifact in artifact_list: ta = tac.get_artifact(artifact) tac.add(ta) try: client.post_collection(project, credentials.get('consumer_key'), credentials.get('consumer_secret'), tac) client.update_parse_status(project, credentials.get('consumer_key'), credentials.get('consumer_secret'), job_log_url['id'], 'parsed') logger.debug("Finished posting artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to upload parsed artifact for %s: %s", log_description, e) _retry(e)
def test_send_artifact_collection(self, mock_post): """Can add a artifact collections to a TreeherderRequest.""" mock_post.return_value = self._expected_response_return_object() tac = TreeherderArtifactCollection() for artifact in self.artifact_data: tac.add(tac.get_artifact(artifact)) client = TreeherderClient(protocol="http", host="host") auth = TreeherderAuth("key", "secret", "project") client.post_collection("project", tac, auth=auth) path, resp = mock_post.call_args self.assertEqual(mock_post.call_count, 1) self.assertEqual(tac.get_collection_data(), resp["json"])
def test_send_artifact_collection(self): """Can add a artifact collections to a TreeherderRequest.""" tac = TreeherderArtifactCollection() for artifact in self.artifact_data: tac.add(tac.get_artifact(artifact)) client = TreeherderClient(protocol="http", host="host", client_id="client-abc", secret="secret123") def request_callback(request): # Check that the expected content was POSTed. posted_json = json.loads(request.body) self.assertEqual(posted_json, tac.get_collection_data()) return (200, {}, '{"message": "Artifacts stored successfully"}') url = client._get_project_uri("project", tac.endpoint_base) responses.add_callback( responses.POST, url, match_querystring=True, callback=request_callback, content_type="application/json" ) client.post_collection("project", tac)
def test_collection_chunking(self): tac = TreeherderArtifactCollection() for artifact in self.artifact_data: ta = TreeherderArtifact(artifact) tac.add(ta) # reconstruct the chunks and make sure we have the same data rebuilt_data = [] chunk_num = 0 for chunk in tac.get_chunks(3): chunk_data = chunk.get_collection_data() rebuilt_data.extend(chunk_data) chunk_num += 1 # the last one should be the "remainder" in an uneven size if chunk_num == 4: assert len(chunk_data) == 1 else: assert len(chunk_data) == 3 assert rebuilt_data == tac.get_collection_data()
def test_artifact_sample_data(self): """Test all add methods for building an artifact""" tac = TreeherderArtifactCollection() for artifact in self.artifact_data: ta = TreeherderArtifact() ta.add_blob(artifact['blob']) ta.add_job_guid(artifact['job_guid']) ta.add_name(artifact['name']) ta.add_type(artifact['type']) self.compare_structs(ta.data, artifact) tac.add(ta) # confirm we get the same thing if we initialize from # a resultset dict ta_struct = TreeherderArtifact(artifact) self.compare_structs(ta_struct.data, artifact)
def test_send_artifact_collection(self, mock_send): """Can add a artifact collections to a TreeherderRequest.""" tac = TreeherderArtifactCollection() for artifact in self.artifact_data: tac.add(tac.get_artifact(artifact)) req = TreeherderRequest( protocol='http', host='host', project='project', oauth_key='key', oauth_secret='secret', ) req.post(tac) self.assertEqual(mock_send.call_count, 1) self.assertEqual( tac.to_json(), mock_send.call_args_list[0][1]["data"] )
def test_send_artifact_collection(self, mock_post): """Can add a artifact collections to a TreeherderRequest.""" mock_post.return_value = self._expected_response_return_object() tac = TreeherderArtifactCollection() for artifact in self.artifact_data: tac.add(tac.get_artifact(artifact)) client = TreeherderClient( protocol='http', host='host', ) client.post_collection('project', 'key', 'secret', tac) path, resp = mock_post.call_args self.assertEqual(mock_post.call_count, 1) self.assertEqual(tac.to_json(), resp['data'])