def test_send_with_oauth(self, mock_post, mock_time, mock_generate_nonce):
"""Tests that oauth data is sent to server"""
mock_time.return_value = 1342229050
mock_generate_nonce.return_value = "46810593"
mock_post.return_value = self._expected_response_return_object()
client = TreeherderClient(
protocol='http',
host='host',
)
tjc = TreeherderJobCollection()
for job in self.job_data:
tjc.add(tjc.get_job(job))
break
client.post_collection('project', 'key', 'secret', tjc)
self.assertEqual(mock_post.call_count, 1)
path, resp = mock_post.call_args
self.assertEqual(
path[0],
"http://host/api/project/project/objectstore/?oauth_body_hash=IKbDoi5GvTRaqjRTCDyKIN5wWiY%3D&oauth_nonce=46810593&oauth_timestamp=1342229050&oauth_consumer_key=key&oauth_signature_method=HMAC-SHA1&oauth_version=1.0&oauth_token=&user=project&oauth_signature=uq%2BrkJCRPyPUdXExSasm25ab8m4%3D"
)
def post_treeherder_collections(th_collections, chunk_size=1):
errors = []
cli = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
)
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
logger.info("collection loading request for project {0}: {1}".format(
project, th_collections[project].endpoint_base))
collection_chunks = th_collections[project].get_chunks(chunk_size)
for collection in collection_chunks:
try:
cli.post_collection(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
collection)
except Exception:
errors.append({
"project": project,
"url": th_collections[project].endpoint_base,
"message": traceback.format_exc()
})
if errors:
raise CollectionNotLoadedException(errors)
def test_post_job_collection(self):
"""Can add a treeherder collections to a TreeherderRequest."""
tjc = TreeherderJobCollection()
for job in self.job_data:
tjc.add(tjc.get_job(job))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tjc.get_collection_data())
return (200, {}, '{"message": "Job successfully updated"}')
url = client._get_endpoint_url(tjc.endpoint_base, project='project')
responses.add_callback(responses.POST,
url,
match_querystring=True,
callback=request_callback,
content_type='application/json')
client.post_collection('project', tjc)
def test_send_artifact_collection(self):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, tac.get_collection_data())
return (200, {}, '{"message": "Artifacts stored successfully"}')
url = client._get_endpoint_url(tac.endpoint_base, project='project')
responses.add_callback(responses.POST,
url,
match_querystring=True,
callback=request_callback,
content_type='application/json')
client.post_collection('project', tac)
def post_treeherder_collections(th_collections, chunk_size=1):
errors = []
credentials = Credentials.objects.get(client_id=settings.ETL_CLIENT_ID)
cli = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
client_id=credentials.client_id,
secret=str(credentials.secret),
)
for project in th_collections:
logger.info("collection loading request for project {0}: {1}".format(
project, th_collections[project].endpoint_base))
collection_chunks = th_collections[project].get_chunks(chunk_size)
for collection in collection_chunks:
try:
cli.post_collection(project, collection)
except Exception:
errors.append({
"project": project,
"url": th_collections[project].endpoint_base,
"message": traceback.format_exc()
})
if errors:
raise CollectionNotLoadedException(errors)
def post_log_artifacts(project, job_guid, job_log_url, retry_task,
extract_artifacts_cb):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e,
countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials.get('consumer_key'),
credentials.get('consumer_secret'), project)
client = TreeherderClient(protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
auth=auth)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid)
except Exception as e:
client.update_parse_status(project, job_log_url['id'], 'failed')
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
log_description, e)
return
# possibly recoverable http error (e.g. problems on our end)
elif isinstance(e, urllib2.URLError):
logger.error("Failed to download log for %s: %s", log_description,
e)
_retry(e)
# parse error or other unrecoverable error
else:
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
# re-raise exception if we're not retrying, so new relic sees the
# error
raise
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, tac)
client.update_parse_status(project, job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s",
log_description, e)
_retry(e)
def test_send_result_collection(self):
"""Can add a treeherder collections to a TreeherderRequest."""
trc = TreeherderResultSetCollection()
for resultset in self.resultset_data:
trc.add(trc.get_resultset(resultset))
client = TreeherderClient(
server_url='http://host',
client_id='client-abc',
secret='secret123',
)
def request_callback(request):
# Check that the expected content was POSTed.
posted_json = json.loads(request.body)
self.assertEqual(posted_json, trc.get_collection_data())
return (
200, {},
'{"message": "well-formed JSON stored", "resultsets": [123, 456]}'
)
url = client._get_endpoint_url(trc.endpoint_base, project='project')
responses.add_callback(responses.POST,
url,
match_querystring=True,
callback=request_callback,
content_type='application/json')
client.post_collection('project', trc)
def post_collection(
project, th_collection, status=None, expect_errors=False,
consumer_key=None, consumer_secret=None):
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
# The only time the credentials should be overridden are when
# a client needs to test authentication failure confirmation
consumer_key = consumer_key or credentials['consumer_key']
consumer_secret = consumer_secret or credentials['consumer_secret']
auth = TreeherderAuth(consumer_key, consumer_secret, project)
client = TreeherderClient(protocol='http', host='localhost', auth=auth)
uri = client._get_project_uri(project, th_collection.endpoint_base)
req = Request('POST', uri,
json=th_collection.get_collection_data(),
auth=auth)
prepped_request = req.prepare()
response = TestApp(application).post_json(
prepped_request.url,
params=th_collection.get_collection_data(),
status=status
)
return response
def test_update_parse_status_nonexistent_id(test_project, mock_post_json):
"""
Attempting to update the parse status for a non-existent log should return a 404.
"""
client = TreeherderClient(protocol='http', host='localhost')
non_existent_id = 9999999
with pytest.raises(AppError) as e:
client.update_parse_status(test_project, non_existent_id, 'parsed')
assert "404 NOT FOUND" in str(e.value)
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
client = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
client.update_parse_status(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
job_log_url['id'], 'failed')
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.debug("Unable to retrieve log for %s: %s", log_description, e)
return
logger.error("Failed to download/parse log for %s: %s", log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
tac)
client.update_parse_status(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def test_hawkauth_setup(self):
"""Test that HawkAuth is correctly set up from the `client_id` and `secret` params."""
client = TreeherderClient(
client_id='client-abc',
secret='secret123',
)
auth = client.auth
assert isinstance(auth, HawkAuth)
expected_credentials = {
'id': 'client-abc',
'key': 'secret123',
'algorithm': 'sha256'
}
self.assertEqual(auth.credentials, expected_credentials)
def test_get_job(self, mock_get):
mock_get.return_value = self._get_mock_response({
"meta": {
"count": 3,
"repository": "mozilla-inbound",
"offset": 0
},
"results":
self.JOB_RESULTS
})
tdc = TreeherderClient()
jobs = tdc.get_jobs("mozilla-inbound")
self.assertEqual(len(jobs), 3)
self.assertEqual(jobs, self.JOB_RESULTS)
def test_post_job_collection(self, mock_post):
"""Can add a treeherder collections to a TreeherderRequest."""
mock_post.return_value = self._expected_response_return_object()
tjc = TreeherderJobCollection()
for job in self.job_data:
tjc.add(tjc.get_job(job))
client = TreeherderClient(
protocol='http',
host='host',
)
client.post_collection('project', 'key', 'secret', tjc)
path, resp = mock_post.call_args
self.assertEqual(mock_post.call_count, 1)
self.assertEqual(tjc.to_json(), resp['data'])
def test_get_results(self):
tdc = TreeherderClient()
url = tdc._get_endpoint_url(tdc.RESULTSET_ENDPOINT,
project='mozilla-inbound')
content = {
"meta": {
"count": 3,
"repository": "mozilla-inbound",
"offset": 0
},
"results": self.RESULTSETS
}
responses.add(responses.GET,
url,
json=content,
match_querystring=True,
status=200)
resultsets = tdc.get_resultsets("mozilla-inbound")
self.assertEqual(len(resultsets), 3)
self.assertEqual(resultsets, self.RESULTSETS)
def test_send_artifact_collection(self, mock_post):
"""Can add a artifact collections to a TreeherderRequest."""
mock_post.return_value = self._expected_response_return_object()
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
client = TreeherderClient(
protocol='http',
host='host',
)
client.post_collection('project', 'key', 'secret', tac)
path, resp = mock_post.call_args
self.assertEqual(mock_post.call_count, 1)
self.assertEqual(tac.to_json(), resp['data'])
def post_collection(project,
th_collection,
status=None,
expect_errors=False,
consumer_key=None,
consumer_secret=None):
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
# The only time the credentials should be overridden are when
# a client needs to test authentication failure confirmation
if consumer_key:
credentials['consumer_key'] = consumer_key
if consumer_secret:
credentials['consumer_secret'] = consumer_secret
cli = TreeherderClient(
protocol='http',
host='localhost',
)
jsondata = th_collection.to_json()
signed_uri = cli._get_project_uri(
project,
th_collection.endpoint_base,
data=jsondata,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret'],
method='POST')
response = TestApp(application).post_json(
str(signed_uri),
params=th_collection.get_collection_data(),
status=status)
return response
def test_send_result_collection(self, mock_post):
"""Can add a treeherder collections to a TreeherderRequest."""
mock_post.return_value = self._expected_response_return_object()
trc = TreeherderResultSetCollection()
for resultset in self.resultset_data:
trc.add(trc.get_resultset(resultset))
client = TreeherderClient(
protocol='http',
host='host',
client_id='client-abc',
secret='secret123',
)
client.post_collection('project', trc)
path, resp = mock_post.call_args
self.assertEqual(mock_post.call_count, 1)
self.assertEqual(trc.get_collection_data(), resp['json'])
def handle(self, *args, **options):
server_params = urlparse(options['server'])
c = TreeherderClient(protocol=server_params.scheme,
host=server_params.netloc)
# options / option collection hashes
for (uuid, props) in c.get_option_collection_hash().iteritems():
for prop in props:
option, _ = Option.objects.get_or_create(name=prop['name'])
OptionCollection.objects.get_or_create(
option_collection_hash=uuid, option=option)
# machine platforms
for machine_platform in c.get_machine_platforms():
MachinePlatform.objects.get_or_create(
os_name=machine_platform['os_name'],
platform=machine_platform['platform'],
architecture=machine_platform['architecture'],
defaults={'active_status': machine_platform['active_status']})
# machine
for machine in c.get_machines():
Machine.objects.get_or_create(id=machine['id'],
name=machine['name'],
defaults={
'first_timestamp':
machine['first_timestamp'],
'last_timestamp':
machine['last_timestamp'],
'active_status':
machine['active_status']
})
# job group
for job_group in c.get_job_groups():
JobGroup.objects.get_or_create(id=job_group['id'],
symbol=job_group['symbol'],
name=job_group['name'],
defaults={
'description':
job_group['description'],
'active_status':
job_group['active_status']
})
# job type
for job_type in c.get_job_types():
jgroup = JobGroup.objects.get(id=job_type['job_group'])
JobType.objects.get_or_create(id=job_type['id'],
symbol=job_type['symbol'],
name=job_type['name'],
defaults={
'job_group':
jgroup,
'description':
job_type['description'],
'active_status':
job_type['active_status']
})
# product
for product in c.get_products():
Product.objects.get_or_create(id=product['id'],
name=product['name'],
defaults={
'description':
product['description'],
'active_status':
product['active_status']
})
# failure classification
for failure_classification in c.get_failure_classifications():
FailureClassification.objects.get_or_create(
id=failure_classification['id'],
name=failure_classification['name'],
defaults={
'description': failure_classification['description'],
'active_status': failure_classification['active_status']
})
# build platform
for build_platform in c.get_build_platforms():
BuildPlatform.objects.get_or_create(
id=build_platform['id'],
os_name=build_platform['os_name'],
defaults={
'platform': build_platform['platform'],
'architecture': build_platform['architecture'],
'active_status': build_platform['active_status']
})
# repository and repository group
for repository in c.get_repositories():
rgroup, _ = RepositoryGroup.objects.get_or_create(
name=repository['repository_group']['name'],
description=repository['repository_group']['description'])
Repository.objects.get_or_create(id=repository['id'],
repository_group=rgroup,
name=repository['name'],
dvcs_type=repository['dvcs_type'],
url=repository['url'],
defaults={
'codebase':
repository['codebase'],
'description':
repository['description'],
'active_status':
repository['active_status']
})
def handle(self, *args, **options):
c = TreeherderClient(server_url=options['server'])
# options / option collection hashes
for (uuid, props) in c.get_option_collection_hash().iteritems():
for prop in props:
option, _ = Option.objects.get_or_create(name=prop['name'])
OptionCollection.objects.get_or_create(
option_collection_hash=uuid,
option=option)
# machine platforms
for machine_platform in c.get_machine_platforms():
MachinePlatform.objects.get_or_create(
os_name=machine_platform['os_name'],
platform=machine_platform['platform'],
architecture=machine_platform['architecture'])
# machine
for machine in c.get_machines():
Machine.objects.get_or_create(
id=machine['id'],
name=machine['name'],
defaults={
'first_timestamp': machine['first_timestamp'],
'last_timestamp': machine['last_timestamp']
})
# job group
for job_group in c.get_job_groups():
JobGroup.objects.get_or_create(
id=job_group['id'],
symbol=job_group['symbol'],
name=job_group['name'],
defaults={
'description': job_group['description']
})
# job type
for job_type in c.get_job_types():
try:
jgroup = JobGroup.objects.get(id=job_type['job_group'])
JobType.objects.get_or_create(
id=job_type['id'],
symbol=job_type['symbol'],
name=job_type['name'],
defaults={
'job_group': jgroup,
'description': job_type['description']
})
except JobGroup.DoesNotExist:
# ignore job types whose job group does not exist
self.stderr.write("WARNING: Job type '{}' ({}) references a "
"job group ({}) which does not have an "
"id".format(job_type['symbol'], job_type['name'],
job_type['job_group']))
# product
for product in c.get_products():
Product.objects.get_or_create(
id=product['id'],
name=product['name'],
defaults={
'description': product['description']
})
# failure classification
for failure_classification in c.get_failure_classifications():
FailureClassification.objects.get_or_create(
id=failure_classification['id'],
name=failure_classification['name'],
defaults={
'description': failure_classification['description']
})
# build platform
for build_platform in c.get_build_platforms():
BuildPlatform.objects.get_or_create(
id=build_platform['id'],
os_name=build_platform['os_name'],
defaults={
'platform': build_platform['platform'],
'architecture': build_platform['architecture']
})
# repository and repository group
for repository in c.get_repositories():
rgroup, _ = RepositoryGroup.objects.get_or_create(
name=repository['repository_group']['name'],
description=repository['repository_group']['description']
)
Repository.objects.get_or_create(
id=repository['id'],
repository_group=rgroup,
name=repository['name'],
dvcs_type=repository['dvcs_type'],
url=repository['url'],
defaults={
'codebase': repository['codebase'],
'description': repository['description'],
'active_status': repository['active_status']
})
def post_collection(project, th_collection):
client = TreeherderClient(server_url='http://localhost')
return client.post_collection(project, th_collection)
def post_collection(project, th_collection):
client = TreeherderClient(protocol='http', host='localhost')
return client.post_collection(project, th_collection)
