def test_send_artifact_collection(self):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tac.get_collection_data())
return (200, {}, '{"message": "Artifacts stored successfully"}')
url = client._get_endpoint_url(tac.endpoint_base, project='project')
responses.add_callback(responses.POST,
url,
match_querystring=True,
callback=request_callback,
content_type='application/json')
client.post_collection('project', tac)
def post_log_artifacts(project, job_guid, job_log_url, retry_task,
extract_artifacts_cb):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e,
countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials.get('consumer_key'),
credentials.get('consumer_secret'), project)
client = TreeherderClient(protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
auth=auth)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid)
except Exception as e:
client.update_parse_status(project, job_log_url['id'], 'failed')
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
log_description, e)
return
# possibly recoverable http error (e.g. problems on our end)
elif isinstance(e, urllib2.URLError):
logger.error("Failed to download log for %s: %s", log_description,
e)
_retry(e)
# parse error or other unrecoverable error
else:
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
# re-raise exception if we're not retrying, so new relic sees the
# error
raise
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, tac)
client.update_parse_status(project, job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s",
log_description, e)
_retry(e)
def test_send_artifact_collection(self, mock_post):
"""Can add a artifact collections to a TreeherderRequest."""
mock_post.return_value = self._expected_response_return_object()
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(
protocol='http',
host='host',
client_id='client-abc',
secret='secret123',
)
client.post_collection('project', tac)
path, resp = mock_post.call_args
self.assertEqual(mock_post.call_count, 1)
self.assertEqual(
tac.get_collection_data(),
resp['json']
)
def post_log_artifacts(project, job_guid, job_log_url, retry_task, extract_artifacts_cb):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url)
logger.debug("Downloading/parsing log for %s", log_description)
job_log = JobLog.objects.get(job__guid=job_guid, url=job_log_url)
credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)
client = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
client_id=credentials.client_id,
secret=str(credentials.secret),
)
try:
artifact_list = extract_artifacts_cb(project, job_log_url, job_guid)
except Exception as e:
job_log.update_status(JobLog.FAILED)
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s", log_description, e)
return
# possibly recoverable http error (e.g. problems on our end)
elif isinstance(e, urllib2.URLError):
logger.error("Failed to download log for %s: %s", log_description, e)
_retry(e)
# parse error or other unrecoverable error
else:
logger.error("Failed to download/parse log for %s: %s", log_description, e)
# re-raise exception if we're not retrying, so new relic sees the
# error
raise
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, tac)
job_log.update_status(JobLog.PARSED)
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def test_artifact_collection(self):
"""Confirm the collection matches the sample data"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
self.assertTrue(len(self.artifact_data) == len(tac.data))
def test_artifact_collection(self):
"""Confirm the collection matches the sample data"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
self.assertTrue(len(self.artifact_data) == len(tac.data))
def test_chunk_endpoint_base(self):
"""Confirm the endpoint_base of the chunks is the same as the original"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
for chunk in tac.get_chunks(3):
assert tac.endpoint_base == chunk.endpoint_base
def test_chunk_endpoint_base(self):
"""Confirm the endpoint_base of the chunks is the same as the original"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
for chunk in tac.get_chunks(3):
assert tac.endpoint_base == chunk.endpoint_base
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
client = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
client.update_parse_status(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
job_log_url['id'], 'failed')
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.debug("Unable to retrieve log for %s: %s", log_description, e)
return
logger.error("Failed to download/parse log for %s: %s", log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
tac)
client.update_parse_status(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def test_send_artifact_collection(self, mock_post):
"""Can add a artifact collections to a TreeherderRequest."""
mock_post.return_value = self._expected_response_return_object()
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(protocol="http", host="host")
auth = TreeherderAuth("key", "secret", "project")
client.post_collection("project", tac, auth=auth)
path, resp = mock_post.call_args
self.assertEqual(mock_post.call_count, 1)
self.assertEqual(tac.get_collection_data(), resp["json"])
def test_send_artifact_collection(self):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(protocol="http", host="host", client_id="client-abc", secret="secret123")
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tac.get_collection_data())
return (200, {}, '{"message": "Artifacts stored successfully"}')
url = client._get_project_uri("project", tac.endpoint_base)
responses.add_callback(
responses.POST, url, match_querystring=True, callback=request_callback, content_type="application/json"
)
client.post_collection("project", tac)
def test_collection_chunking(self):
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
# reconstruct the chunks and make sure we have the same data
rebuilt_data = []
chunk_num = 0
for chunk in tac.get_chunks(3):
chunk_data = chunk.get_collection_data()
rebuilt_data.extend(chunk_data)
chunk_num += 1
# the last one should be the "remainder" in an uneven size
if chunk_num == 4:
assert len(chunk_data) == 1
else:
assert len(chunk_data) == 3
assert rebuilt_data == tac.get_collection_data()
def test_artifact_sample_data(self):
"""Test all add methods for building an artifact"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact()
ta.add_blob(artifact['blob'])
ta.add_job_guid(artifact['job_guid'])
ta.add_name(artifact['name'])
ta.add_type(artifact['type'])
self.compare_structs(ta.data, artifact)
tac.add(ta)
# confirm we get the same thing if we initialize from
# a resultset dict
ta_struct = TreeherderArtifact(artifact)
self.compare_structs(ta_struct.data, artifact)
def test_artifact_sample_data(self):
"""Test all add methods for building an artifact"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact()
ta.add_blob(artifact['blob'])
ta.add_job_guid(artifact['job_guid'])
ta.add_name(artifact['name'])
ta.add_type(artifact['type'])
self.compare_structs(ta.data, artifact)
tac.add(ta)
# confirm we get the same thing if we initialize from
# a resultset dict
ta_struct = TreeherderArtifact(artifact)
self.compare_structs(ta_struct.data, artifact)
def test_send_artifact_collection(self):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tac.get_collection_data())
return (200, {}, '{"message": "Artifacts stored successfully"}')
url = client._get_endpoint_url(tac.endpoint_base, project='project')
responses.add_callback(responses.POST, url, match_querystring=True,
callback=request_callback, content_type='application/json')
client.post_collection('project', tac)
def test_send_artifact_collection(self, mock_send):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
req = TreeherderRequest(
protocol='http',
host='host',
project='project',
oauth_key='key',
oauth_secret='secret',
)
req.post(tac)
self.assertEqual(mock_send.call_count, 1)
self.assertEqual(
tac.to_json(),
mock_send.call_args_list[0][1]["data"]
)
def test_send_artifact_collection(self, mock_post):
"""Can add a artifact collections to a TreeherderRequest."""
mock_post.return_value = self._expected_response_return_object()
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(
protocol='http',
host='host',
)
client.post_collection('project', 'key', 'secret', tac)
path, resp = mock_post.call_args
self.assertEqual(mock_post.call_count, 1)
self.assertEqual(tac.to_json(), resp['data'])
def test_collection_chunking(self):
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
# reconstruct the chunks and make sure we have the same data
rebuilt_data = []
chunk_num = 0
for chunk in tac.get_chunks(3):
chunk_data = chunk.get_collection_data()
rebuilt_data.extend(chunk_data)
chunk_num += 1
# the last one should be the "remainder" in an uneven size
if chunk_num == 4:
assert len(chunk_data) == 1
else:
assert len(chunk_data) == 3
assert rebuilt_data == tac.get_collection_data()
