def _post_json_data(url, data):
th_collection = data[jm.project]
OAuthCredentials.set_credentials( SampleData.get_credentials() )
credentials = OAuthCredentials.get_credentials(jm.project)
tr = TreeherderRequest(
protocol='http',
host='localhost',
project=jm.project,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret']
)
signed_uri = tr.oauth_client.get_signed_uri(
th_collection.to_json(),
tr.get_uri(th_collection.endpoint_base),
"POST"
)
response = TestApp(application).post_json(
str(signed_uri), params=th_collection.get_collection_data()
)
response.getcode = lambda: response.status_int
return response
def post_collection(project, th_collection, status=None, expect_errors=False, consumer_key=None, consumer_secret=None):
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
# The only time the credentials should be overridden are when
# a client needs to test authentication failure confirmation
if consumer_key:
credentials["consumer_key"] = consumer_key
if consumer_secret:
credentials["consumer_secret"] = consumer_secret
tr = TreeherderRequest(
protocol="http",
host="localhost",
project=project,
oauth_key=credentials["consumer_key"],
oauth_secret=credentials["consumer_secret"],
)
signed_uri = tr.oauth_client.get_signed_uri(
th_collection.to_json(), tr.get_uri(th_collection.endpoint_base), "POST"
)
response = TestApp(application).post_json(
str(signed_uri), params=th_collection.get_collection_data(), status=status
)
return response
def post_collection(
project, th_collection, status=None, expect_errors=False,
consumer_key=None, consumer_secret=None):
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
# The only time the credentials should be overridden are when
# a client needs to test authentication failure confirmation
if consumer_key:
credentials['consumer_key'] = consumer_key
if consumer_secret:
credentials['consumer_secret'] = consumer_secret
cli = TreeherderClient(
protocol='http',
host='localhost',
)
jsondata = th_collection.to_json()
signed_uri = cli._get_uri(project, th_collection.endpoint_base,
data=jsondata,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret'],
method='POST')
response = TestApp(application).post_json(
str(signed_uri), params=th_collection.get_collection_data(),
status=status
)
return response
def post_job_data(project, uri, data, status=None, expect_errors=False):
# Since the uri is passed in it's not generated by the
# treeherder request or collection and is missing the protocol
# and host. Add those missing elements here.
uri = 'http://localhost{0}'.format(uri)
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
tr = TreeherderRequest(protocol='http',
host='localhost',
project=project,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret'])
signed_uri = tr.get_signed_uri(json.dumps(data), uri)
response = TestApp(application).post_json(str(signed_uri),
params=data,
status=status,
expect_errors=expect_errors)
return response
def post_collection(
project, th_collection, status=None, expect_errors=False,
consumer_key=None, consumer_secret=None):
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
# The only time the credentials should be overridden are when
# a client needs to test authentication failure confirmation
consumer_key = consumer_key or credentials['consumer_key']
consumer_secret = consumer_secret or credentials['consumer_secret']
auth = TreeherderAuth(consumer_key, consumer_secret, project)
client = TreeherderClient(protocol='http', host='localhost', auth=auth)
uri = client._get_project_uri(project, th_collection.endpoint_base)
req = Request('POST', uri,
json=th_collection.get_collection_data(),
auth=auth)
prepped_request = req.prepare()
response = TestApp(application).post_json(
prepped_request.url,
params=th_collection.get_collection_data(),
status=status
)
return response
def post_job_data(
project, uri, data, status=None, expect_errors=False):
# Since the uri is passed in it's not generated by the
# treeherder request or collection and is missing the protocol
# and host. Add those missing elements here.
uri = 'http://localhost{0}'.format(uri)
# Set the credentials
OAuthCredentials.set_credentials( SampleData.get_credentials() )
credentials = OAuthCredentials.get_credentials(project)
tr = TreeherderRequest(
protocol='http',
host='localhost',
project=project,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret']
)
signed_uri = tr.get_signed_uri(
json.dumps(data), uri
)
response = TestApp(application).post_json(
str(signed_uri), params=data, status=status,
expect_errors=expect_errors
)
return response
def authenticate(self, request):
if not self.auth_detected(request):
return None
user = request.resolver_match.kwargs.get('project') or request.query_params['user']
project_credentials = OAuthCredentials.get_credentials(user)
if not project_credentials:
raise exceptions.ValidationError(
'project {0} has no OAuth credentials'.format(user)
)
parameters = OAuthCredentials.get_parameters(request.query_params)
oauth_consumer_key = parameters['oauth_consumer_key']
if oauth_consumer_key != project_credentials['consumer_key']:
raise exceptions.AuthenticationFailed(
'oauth_consumer_key does not match credentials for project {0}'.format(user)
)
uri = '{0}://{1}{2}'.format(
settings.TREEHERDER_REQUEST_PROTOCOL,
request.get_host(),
request.path
)
# Construct the OAuth request based on the django request object
json_renderer = JSONRenderer()
req_obj = oauth.Request(
method=request.method,
url=uri,
parameters=parameters,
body=json_renderer.render(request.data),
)
server = oauth.Server()
token = oauth.Token(key='', secret='')
# Get the consumer object
cons_obj = oauth.Consumer(
oauth_consumer_key,
project_credentials['consumer_secret']
)
# Set the signature method
server.add_signature_method(oauth.SignatureMethod_HMAC_SHA1())
try:
# verify oauth django request and consumer object match
server.verify_request(req_obj, cons_obj, token)
except oauth.Error:
raise exceptions.AuthenticationFailed(
'Client authentication failed for project {0}'.format(user)
)
request.legacy_oauth_authenticated = True
return (DummyUser(), None)
def test_artifact_create_text_log_summary_and_bug_suggestions(
webapp, test_project, eleven_jobs_processed,
mock_post_collection, mock_error_summary,
sample_data):
"""
test submitting text_log_summary and Bug suggestions artifacts
This should NOT generate a Bug suggestions artifact, just post the one
submitted.
"""
credentials = OAuthCredentials.get_credentials(test_project)
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
tls = sample_data.text_log_summary
bs_blob = ["flim", "flam"]
tac = client.TreeherderArtifactCollection()
tac.add(client.TreeherderArtifact({
'type': 'json',
'name': 'text_log_summary',
'blob': json.dumps(tls['blob']),
'job_guid': job['job_guid']
}))
tac.add(client.TreeherderArtifact({
'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps(bs_blob),
'job_guid': job['job_guid']
}))
cli = client.TreeherderClient(protocol='http', host='localhost')
credentials = OAuthCredentials.get_credentials(test_project)
cli.post_collection(test_project, credentials['consumer_key'],
credentials['consumer_secret'], tac)
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
'job_id': {('=', job["id"])}
})
assert len(artifacts) == 2
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0]
assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
assert bs_blob == act_bs_obj
def post_log_artifacts(project, job_guid, job_log_url, retry_task,
extract_artifacts_cb):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e,
countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials.get('consumer_key'),
credentials.get('consumer_secret'), project)
client = TreeherderClient(protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
auth=auth)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid)
except Exception as e:
client.update_parse_status(project, job_log_url['id'], 'failed')
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
log_description, e)
return
# possibly recoverable http error (e.g. problems on our end)
elif isinstance(e, urllib2.URLError):
logger.error("Failed to download log for %s: %s", log_description,
e)
_retry(e)
# parse error or other unrecoverable error
else:
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
# re-raise exception if we're not retrying, so new relic sees the
# error
raise
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, tac)
client.update_parse_status(project, job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s",
log_description, e)
_retry(e)
def post_treeherder_collections(th_collections, chunk_size=1):
errors = []
cli = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
)
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials.get('consumer_key'),
credentials.get('consumer_secret'),
project)
logger.info(
"collection loading request for project {0}: {1}".format(
project,
th_collections[project].endpoint_base))
collection_chunks = th_collections[project].get_chunks(chunk_size)
for collection in collection_chunks:
try:
cli.post_collection(project, collection, auth=auth)
except Exception:
errors.append({
"project": project,
"url": th_collections[project].endpoint_base,
"message": traceback.format_exc()
})
if errors:
raise CollectionNotLoadedException(errors)
def post_treeherder_collections(th_collections):
errors = []
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
th_request = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None)
)
logger.info(
"collection loading request: {0}".format(
th_request.get_uri(th_collections[project].endpoint_base)))
response = th_request.post(th_collections[project])
if not response or response.status_code != 200:
errors.append({
"project": project,
"url": th_collections[project].endpoint_base,
"message": response.text
})
if errors:
raise CollectionNotLoadedException(errors)
def load(self, th_collections):
errors = []
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
th_request = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None)
)
logger.info(
"collection loading request: {0}".format(
th_request.get_uri(th_collections[project].endpoint_base)))
response = th_request.post(th_collections[project])
if not response or response.status != 200:
errors.append({
"project": project,
"url": th_collections[project].endpoint_base,
"message": response.read()
})
if errors:
raise CollectionNotLoadedException(errors)
def post_treeherder_collections(th_collections, chunk_size=1):
errors = []
cli = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
)
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
logger.info("collection loading request for project {0}: {1}".format(
project, th_collections[project].endpoint_base))
collection_chunks = th_collections[project].get_chunks(chunk_size)
for collection in collection_chunks:
try:
cli.post_collection(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
collection)
except Exception:
errors.append({
"project": project,
"url": th_collections[project].endpoint_base,
"message": traceback.format_exc()
})
if errors:
raise CollectionNotLoadedException(errors)
def authenticate(self, request):
if not self.auth_detected(request):
return None
user = request.resolver_match.kwargs.get(
'project') or request.query_params['user']
project_credentials = OAuthCredentials.get_credentials(user)
if not project_credentials:
raise exceptions.ValidationError(
'project {0} has no OAuth credentials'.format(user))
parameters = OAuthCredentials.get_parameters(request.query_params)
oauth_consumer_key = parameters['oauth_consumer_key']
if oauth_consumer_key != project_credentials['consumer_key']:
raise exceptions.AuthenticationFailed(
'oauth_consumer_key does not match credentials for project {0}'
.format(user))
uri = '{0}://{1}{2}'.format(settings.TREEHERDER_REQUEST_PROTOCOL,
request.get_host(), request.path)
# Construct the OAuth request based on the django request object
json_renderer = JSONRenderer()
req_obj = oauth.Request(
method=request.method,
url=uri,
parameters=parameters,
body=json_renderer.render(request.DATA),
)
server = oauth.Server()
token = oauth.Token(key='', secret='')
# Get the consumer object
cons_obj = oauth.Consumer(oauth_consumer_key,
project_credentials['consumer_secret'])
# Set the signature method
server.add_signature_method(oauth.SignatureMethod_HMAC_SHA1())
try:
# verify oauth django request and consumer object match
server.verify_request(req_obj, cons_obj, token)
except oauth.Error:
raise exceptions.AuthenticationFailed(
'Client authentication failed for project {0}'.format(user))
request.legacy_oauth_authenticated = True
return (DummyUser(), None)
def do_post_collection(project, collection):
# assume if there were no exceptions we're ok
cli = client.TreeherderClient(protocol='http', host='localhost')
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials['consumer_key'],
credentials['consumer_secret'], project)
cli.post_collection(project, collection, auth=auth)
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e,
countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid,
check_errors)
except Exception as e:
update_parse_status(req, job_log_url, 'failed')
if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug("Log not found for %s", log_description)
return
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
try:
req.post(tac)
update_parse_status(req, job_log_url, 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s",
log_description, e)
_retry(e)
def do_post_collection(project, collection):
# assume if there were no exceptions we're ok
cli = client.TreeherderClient(protocol='http', host='localhost')
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials['consumer_key'],
credentials['consumer_secret'],
project)
cli.post_collection(project, collection, auth=auth)
def _send(th_request, th_collection):
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(jm.project)
th_request.oauth_key = credentials['consumer_key']
th_request.oauth_secret = credentials['consumer_secret']
signed_uri = th_request.get_signed_uri(
th_collection.to_json(), th_request.get_uri(th_collection)
)
response = TestApp(application).post_json(
str(signed_uri), params=th_collection.get_collection_data()
)
response.getcode = lambda: response.status_int
return response
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
update_parse_status(req, job_log_url, 'failed')
if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug("Log not found for %s", log_description)
return
logger.error("Failed to download/parse log for %s: %s", log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
try:
req.post(tac)
update_parse_status(req, job_log_url, 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
client = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
client.update_parse_status(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
job_log_url['id'], 'failed')
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.debug("Unable to retrieve log for %s: %s", log_description, e)
return
logger.error("Failed to download/parse log for %s: %s", log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
tac)
client.update_parse_status(project, credentials.get('consumer_key'),
credentials.get('consumer_secret'),
job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def _send(th_request, endpoint, method=None, data=None):
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(jm.project)
th_request.oauth_key = credentials['consumer_key']
th_request.oauth_secret = credentials['consumer_secret']
if data and not isinstance(data, str):
data = json.dumps(data)
signed_uri = th_request.oauth_client.get_signed_uri(
data, th_request.get_uri(endpoint), method)
response = getattr(TestApp(application),
method.lower())(str(signed_uri),
params=data,
content_type='application/json')
response.getcode = lambda: response.status_int
return response
def post_collection(project,
th_collection,
status=None,
expect_errors=False,
consumer_key=None,
consumer_secret=None):
# Set the credentials
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(project)
# The only time the credentials should be overridden are when
# a client needs to test authentication failure confirmation
if consumer_key:
credentials['consumer_key'] = consumer_key
if consumer_secret:
credentials['consumer_secret'] = consumer_secret
cli = TreeherderClient(
protocol='http',
host='localhost',
)
jsondata = th_collection.to_json()
signed_uri = cli._get_project_uri(
project,
th_collection.endpoint_base,
data=jsondata,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret'],
method='POST')
response = TestApp(application).post_json(
str(signed_uri),
params=th_collection.get_collection_data(),
status=status)
return response
def test_artifact_create_text_log_summary(webapp, test_project,
eleven_jobs_processed,
mock_post_collection,
mock_error_summary, sample_data):
"""
test submitting a text_log_summary artifact which auto-generates bug suggestions
"""
credentials = OAuthCredentials.get_credentials(test_project)
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
tls = sample_data.text_log_summary
tac = client.TreeherderArtifactCollection()
ta = client.TreeherderArtifact({
'type': 'json',
'name': 'text_log_summary',
'blob': json.dumps(tls['blob']),
'job_guid': job['job_guid']
})
tac.add(ta)
cli = client.TreeherderClient(protocol='http', host='localhost')
credentials = OAuthCredentials.get_credentials(test_project)
cli.post_collection(test_project, credentials['consumer_key'],
credentials['consumer_secret'], tac)
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(
0, 10, conditions={'job_id': {('=', job["id"])}})
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [
x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'
][0]
assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
assert mock_error_summary == act_bs_obj
def _send(th_request, endpoint, method=None, data=None):
OAuthCredentials.set_credentials(SampleData.get_credentials())
credentials = OAuthCredentials.get_credentials(jm.project)
th_request.oauth_key = credentials['consumer_key']
th_request.oauth_secret = credentials['consumer_secret']
if data and not isinstance(data, str):
data = json.dumps(data)
signed_uri = th_request.oauth_client.get_signed_uri(
data, th_request.get_uri(endpoint), method
)
response = getattr(TestApp(application), method.lower())(
str(signed_uri),
params=data,
content_type='application/json'
)
response.getcode = lambda: response.status_int
return response
def test_artifact_create_text_log_summary(webapp, test_project, eleven_jobs_processed,
mock_post_collection, mock_error_summary,
sample_data):
"""
test submitting a text_log_summary artifact which auto-generates bug suggestions
"""
credentials = OAuthCredentials.get_credentials(test_project)
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
tls = sample_data.text_log_summary
tac = client.TreeherderArtifactCollection()
ta = client.TreeherderArtifact({
'type': 'json',
'name': 'text_log_summary',
'blob': json.dumps(tls['blob']),
'job_guid': job['job_guid']
})
tac.add(ta)
cli = client.TreeherderClient(protocol='http', host='localhost')
credentials = OAuthCredentials.get_credentials(test_project)
cli.post_collection(test_project, credentials['consumer_key'],
credentials['consumer_secret'], tac)
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
'job_id': {('=', job["id"])}
})
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0]
assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
assert mock_error_summary == act_bs_obj
def test_artifact_create_text_log_summary_and_bug_suggestions(
webapp, test_project, eleven_jobs_stored, mock_post_json,
mock_error_summary, sample_data):
"""
test submitting text_log_summary and Bug suggestions artifacts
This should NOT generate a Bug suggestions artifact, just post the one
submitted.
"""
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
tls = sample_data.text_log_summary
bs_blob = ["flim", "flam"]
tac = client.TreeherderArtifactCollection()
tac.add(
client.TreeherderArtifact({
'type': 'json',
'name': 'text_log_summary',
'blob': json.dumps(tls['blob']),
'job_guid': job['job_guid']
}))
tac.add(
client.TreeherderArtifact({
'type': 'json',
'name': 'Bug suggestions',
'blob': bs_blob,
'job_guid': job['job_guid']
}))
credentials = OAuthCredentials.get_credentials(test_project)
auth = TreeherderAuth(credentials['consumer_key'],
credentials['consumer_secret'], test_project)
cli = client.TreeherderClient(protocol='http', host='localhost', auth=auth)
cli.post_collection(test_project, tac)
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(
0, 10, conditions={'job_id': {('=', job["id"])}})
assert len(artifacts) == 2
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [
x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'
][0]
assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
assert bs_blob == act_bs_obj
def test_post_job_with_parsed_log(test_project, result_set_stored,
mock_send_request):
"""
test submitting a job with a pre-parsed log gets the right job_log_url
parse_status value.
"""
credentials = OAuthCredentials.get_credentials(test_project)
tjc = client.TreeherderJobCollection()
tj = client.TreeherderJob({
'project': test_project,
'revision_hash': result_set_stored[0]['revision_hash'],
'job': {
'job_guid': 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33',
'state': 'completed',
'log_references': [{
'url': 'http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/...',
'name': 'builbot_text',
'parse_status': 'parsed'
}]
}
})
tjc.add(tj)
req = client.TreeherderRequest(
protocol='http',
host='localhost',
project=test_project,
oauth_key=credentials['consumer_key'],
oauth_secret=credentials['consumer_secret']
)
# Post the request to treeherder
resp = req.post(tjc)
assert resp.status_int == 200
assert resp.body == '{"message": "well-formed JSON stored"}'
with JobsModel(test_project) as jm:
jm.process_objects(10)
job_ids = [x['id'] for x in jm.get_job_list(0, 20)]
job_log_list = jm.get_job_log_url_list(job_ids)
assert len(job_log_list) == 1
assert job_log_list[0]['parse_status'] == 'parsed'
def load(self, th_collections):
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
th_request = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None))
response = th_request.post(th_collections[project])
if not response or response.status != 200:
message = response.read()
logger.error("collection loading failed: {0}".format(message))
def load(self, th_collections):
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
th_request = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None)
)
response = th_request.post(th_collections[project])
if not response or response.status != 200:
message = response.read()
logger.error("collection loading failed: {0}".format(message))
def test_two_legged_oauth_project_via_user(monkeypatch, jm, set_oauth_credentials):
view = AuthenticatedView.as_view()
credentials = OAuthCredentials.get_credentials(jm.project)
test_url = '/api/'
request = factory.get(test_url, {
'oauth_body_hash': '',
'oauth_consumer_key': credentials['consumer_key'],
'oauth_nonce': '',
'oauth_signature_method': '',
'oauth_timestamp': '',
'oauth_token': '',
'oauth_version': '',
'oauth_signature': '',
'user': jm.project
})
monkeypatch.setattr(oauth.Server, 'verify_request', lambda *x, **y: True)
request.resolver_match = resolve(test_url)
response = view(request)
assert response.data == {'authenticated': True}
def load(self, th_collections):
for project in th_collections:
credentials = OAuthCredentials.get_credentials(project)
th_request = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None)
)
logger.info(
"collection loading request: {0}".format(
th_request.get_uri(th_collections[project].endpoint_base)))
response = th_request.post(th_collections[project])
if not response or response.status != 200:
message = response.read()
logger.error('[{0}]Error posting data to {1} : {2}'.format(
project, th_collections[project].endpoint_base, message))
def test_two_legged_oauth_project_via_user(monkeypatch, jm,
set_oauth_credentials):
view = AuthenticatedView.as_view()
credentials = OAuthCredentials.get_credentials(jm.project)
test_url = '/api/'
request = factory.get(
test_url, {
'oauth_body_hash': '',
'oauth_consumer_key': credentials['consumer_key'],
'oauth_nonce': '',
'oauth_signature_method': '',
'oauth_timestamp': '',
'oauth_token': '',
'oauth_version': '',
'oauth_signature': '',
'user': jm.project
})
monkeypatch.setattr(oauth.Server, 'verify_request', lambda *x, **y: True)
request.resolver_match = resolve(test_url)
response = view(request)
assert response.data == {'authenticated': True}
def wrap_oauth(cls, *args, **kwargs):
# First argument must be request object
request = args[0]
# Get the project keyword argumet
project = kwargs.get('project', None)
# Get the project credentials
project_credentials = OAuthCredentials.get_credentials(project)
if not project_credentials:
msg = {
'response': "invalid_request",
'detail':
"project, {0}, has no OAuth credentials".format(project)
}
return Response(msg, 500)
parameters = OAuthCredentials.get_parameters(request.QUERY_PARAMS)
oauth_body_hash = parameters.get('oauth_body_hash', None)
oauth_signature = parameters.get('oauth_signature', None)
oauth_consumer_key = parameters.get('oauth_consumer_key', None)
oauth_token = parameters.get('oauth_token', None)
if not oauth_body_hash or not oauth_signature or not oauth_consumer_key:
msg = {
'response': "invalid_request",
'detail': "Required oauth parameters not provided in the uri"
}
return Response(msg, 500)
if oauth_consumer_key != project_credentials['consumer_key']:
msg = {
'response':
"access_denied",
'detail':
"oauth_consumer_key does not match project, {0}, credentials".
format(project)
}
return Response(msg, 403)
uri = '{0}://{1}{2}'.format(settings.TREEHERDER_REQUEST_PROTOCOL,
request.get_host(), request.path)
#Construct the OAuth request based on the django request object
req_obj = oauth.Request(
method=request.method,
url=uri,
parameters=parameters,
body=json.dumps(request.DATA),
)
server = oauth.Server()
token = oauth.Token(key='', secret='')
#Get the consumer object
cons_obj = oauth.Consumer(oauth_consumer_key,
project_credentials['consumer_secret'])
#Set the signature method
server.add_signature_method(oauth.SignatureMethod_HMAC_SHA1())
try:
#verify oauth django request and consumer object match
server.verify_request(req_obj, cons_obj, token)
except oauth.Error:
msg = {
'response':
"invalid_client",
'detail':
"Client authentication failed for project, {0}".format(project)
}
return Response(msg, 403)
return func(request, *args, **kwargs)
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
auth = TreeherderAuth(credentials.get('consumer_key'),
credentials.get('consumer_secret'),
project)
client = TreeherderClient(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
auth=auth
)
try:
artifact_list = extract_artifacts_cb(project, job_log_url['url'],
job_guid)
except Exception as e:
client.update_parse_status(project, job_log_url['id'], 'failed')
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
log_description, e)
return
# possibly recoverable http error (e.g. problems on our end)
elif isinstance(e, urllib2.URLError):
logger.error("Failed to download log for %s: %s",
log_description, e)
_retry(e)
# parse error or other unrecoverable error
else:
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
# re-raise exception if we're not retrying, so new relic sees the
# error
raise
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact(artifact)
tac.add(ta)
try:
client.post_collection(project, tac)
client.update_parse_status(project, job_log_url['id'], 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def set_oauth_credentials():
OAuthCredentials.set_credentials(SampleData.get_credentials())
def set_oauth_credentials():
OAuthCredentials.set_credentials(SampleData.get_credentials())
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
# if parse_status is not available, consider it pending
parse_status = job_log_url.get("parse_status", "pending")
# don't parse a log if it's already been parsed
if parse_status == "parsed":
return
try:
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id'])
artifact_list = extract_log_artifacts(job_log_url['url'], job_guid,
check_errors)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
})
except Exception, e:
# send an update to job_log_url
#the job_log_url status changes from pending/running to failed
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
})
# for every retry, set the countdown to 10 minutes
# .retry() raises a RetryTaskError exception,
# so nothing below this line will be executed.
parse_log.retry(exc=e, countdown=10 * 60)
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
credentials = OAuthCredentials.get_credentials(project)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id']
)
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
logger.error("Failed to download/parse log for %s: %s", log_description, e)
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
}
)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
try:
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
}
)
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(job_log_url['id'])
try:
log_url = job_log_url['url']
bug_suggestions = []
bugscache_uri = '{0}{1}'.format(
settings.API_HOSTNAME,
reverse("bugscache-list")
)
terms_requested = {}
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(log_url,
check_errors=check_errors)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_guid, name, 'json',
json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts.get(
'Structured Log', {}
).get(
'step_data', {}
).get(
'all_errors', [] )
for err in all_errors:
# remove the mozharness prefix
clean_line = get_mozharness_substring(err['line'])
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
bugs = dict(open_recent=[], all_others=[])
# collect open recent and all other bugs suggestions
if search_term:
if not search_term in terms_requested:
# retrieve the list of suggestions from the api
bugs = get_bugs_for_search_term(
search_term,
bugscache_uri
)
terms_requested[search_term] = bugs
else:
bugs = terms_requested[search_term]
if not bugs or not (bugs['open_recent']
or bugs['all_others']):
# no suggestions, try to use
# the crash signature as search term
crash_signature = get_crash_signature(clean_line)
if crash_signature:
if not crash_signature in terms_requested:
bugs = get_bugs_for_search_term(
crash_signature,
bugscache_uri
)
terms_requested[crash_signature] = bugs
else:
bugs = terms_requested[crash_signature]
bug_suggestions.append({
"search": clean_line,
"bugs": bugs
})
artifact_list.append((job_guid, 'Bug suggestions', 'json', json.dumps(bug_suggestions)))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'parsed'
req.send(
update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
}
)
except Exception, e:
parse_log.retry(exc=e)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'failed'
req.send(
update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
}
)
# re raise the exception to leave a trace in the log
raise
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id'])
try:
log_url = job_log_url['url']
bug_suggestions = []
bugscache_uri = '{0}{1}'.format(settings.API_HOSTNAME,
reverse("bugscache-list"))
terms_requested = {}
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(log_url,
check_errors=check_errors)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append(
(job_guid, name, 'json', json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts.get(
'Structured Log', {}).get('step_data',
{}).get('all_errors', [])
for err in all_errors:
# remove the mozharness prefix
clean_line = get_mozharness_substring(err['line'])
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
bugs = dict(open_recent=[], all_others=[])
# collect open recent and all other bugs suggestions
if search_term:
if not search_term in terms_requested:
# retrieve the list of suggestions from the api
bugs = get_bugs_for_search_term(
search_term, bugscache_uri)
terms_requested[search_term] = bugs
else:
bugs = terms_requested[search_term]
if not bugs or not (bugs['open_recent']
or bugs['all_others']):
# no suggestions, try to use
# the crash signature as search term
crash_signature = get_crash_signature(clean_line)
if crash_signature:
if not crash_signature in terms_requested:
bugs = get_bugs_for_search_term(
crash_signature, bugscache_uri)
terms_requested[crash_signature] = bugs
else:
bugs = terms_requested[crash_signature]
bug_suggestions.append({
"search": clean_line,
"bugs": bugs
})
artifact_list.append((job_guid, 'Bug suggestions', 'json',
json.dumps(bug_suggestions)))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'parsed'
req.send(update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
})
except Exception, e:
parse_log.retry(exc=e)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'failed'
req.send(update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
})
# re raise the exception to leave a trace in the log
raise
def parse_log(project, log_url, job_guid, resultset, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
mozharness_pattern = re.compile(
'^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
)
bugs_cache = {'open': {}, 'closed': {}}
bug_suggestions = {'open': {}, 'closed': {}}
status_publisher = JobStatusPublisher(settings.BROKER_URL)
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
try:
# return the resultset with the job id to identify if the UI wants
# to fetch the whole thing.
bugscache_uri = '{0}{1}'.format(
settings.API_HOSTNAME,
reverse("bugscache-list")
)
credentials = OAuthCredentials.get_credentials(project)
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(
log_url,
check_errors=check_errors,
)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
for err in all_errors:
# remove the mozharness prefix
clean_line = mozharness_pattern.sub('', err['line']).strip()
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
# collect open and closed bugs suggestions
for status in ('open', 'closed'):
if not search_term:
bug_suggestions[status][clean_line] = []
continue
if search_term not in bugs_cache[status]:
# retrieve the list of suggestions from the api
bugs_cache[status][search_term] = get_bugs_for_search_term(
search_term,
status,
bugscache_uri
)
# no suggestions, try to use the crash signature as search term
if not bugs_cache[status][search_term]:
crash_signature = get_crash_signature(search_term)
if crash_signature:
bugs_cache[status][search_term] = get_bugs_for_search_term(
search_term,
status,
bugscache_uri
)
bug_suggestions[status][clean_line] = bugs_cache[status][search_term]
artifact_list.append((job_guid, 'Open bugs', 'json', json.dumps(bug_suggestions['open'])))
artifact_list.append((job_guid, 'Closed bugs', 'json', json.dumps(bug_suggestions['closed'])))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
req.send(tac)
status_publisher.publish(job_guid, resultset, project, 'processed')
if check_errors:
failure_publisher.publish(job_guid, project)
finally:
status_publisher.disconnect()
failure_publisher.disconnect()
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
# if parse_status is not available, consider it pending
parse_status = job_log_url.get("parse_status", "pending")
# don't parse a log if it's already been parsed
if parse_status == "parsed":
return
try:
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id'])
logger.debug("Downloading and extracting log information for guid "
"'%s' (from %s)" % (job_guid, job_log_url['url']))
artifact_list = extract_log_artifacts(job_log_url['url'], job_guid,
check_errors)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
logger.debug("Finished downloading and processing artifact for guid "
"'%s'" % job_guid)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
})
logger.debug("Finished posting artifact for guid '%s'" % job_guid)
except Exception, e:
# send an update to job_log_url
#the job_log_url status changes from pending/running to failed
logger.warn("Failed to download and/or parse artifact for guid '%s'" %
job_guid)
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
})
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
parse_log.retry(exc=e, countdown=(1 + parse_log.request.retries) * 60)
def wrap_oauth(cls, *args, **kwargs):
# First argument must be request object
request = args[0]
# Get the project keyword argumet
project = kwargs.get('project', None)
# Get the project credentials
project_credentials = OAuthCredentials.get_credentials(project)
if not project_credentials:
msg = {
'response': "invalid_request",
'detail': "project, {0}, has no OAuth credentials".format(project)
}
return Response(msg, 500)
parameters = OAuthCredentials.get_parameters(request.QUERY_PARAMS)
oauth_body_hash = parameters.get('oauth_body_hash', None)
oauth_signature = parameters.get('oauth_signature', None)
oauth_consumer_key = parameters.get('oauth_consumer_key', None)
oauth_token = parameters.get('oauth_token', None)
if not oauth_body_hash or not oauth_signature or not oauth_consumer_key:
msg = {
'response':"invalid_request",
'detail':"Required oauth parameters not provided in the uri"
}
return Response(msg, 500)
if oauth_consumer_key != project_credentials['consumer_key']:
msg = {
'response':"access_denied",
'detail':"oauth_consumer_key does not match project, {0}, credentials".format(project)
}
return Response(msg, 403)
uri = '{0}://{1}{2}'.format(
settings.TREEHERDER_REQUEST_PROTOCOL, request.get_host(),
request.path
)
#Construct the OAuth request based on the django request object
req_obj = oauth.Request(
method=request.method,
url=uri,
parameters=parameters,
body=json.dumps(request.DATA),
)
server = oauth.Server()
token = oauth.Token(key='', secret='')
#Get the consumer object
cons_obj = oauth.Consumer(
oauth_consumer_key,
project_credentials['consumer_secret']
)
#Set the signature method
server.add_signature_method(oauth.SignatureMethod_HMAC_SHA1())
try:
#verify oauth django request and consumer object match
server.verify_request(req_obj, cons_obj, token)
except oauth.Error:
msg = {
'response':"invalid_client",
'detail':"Client authentication failed for project, {0}".format(project)
}
return Response(msg, 403)
return func(request, *args, **kwargs)
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
# if parse_status is not available, consider it pending
parse_status = job_log_url.get("parse_status", "pending")
# don't parse a log if it's already been parsed
if parse_status == "parsed":
return
try:
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id']
)
artifact_list = extract_log_artifacts(job_log_url['url'],
job_guid, check_errors)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
}
)
except Exception, e:
# send an update to job_log_url
#the job_log_url status changes from pending/running to failed
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
}
)
# for every retry, set the countdown to 10 minutes
# .retry() raises a RetryTaskError exception,
# so nothing below this line will be executed.
parse_log.retry(exc=e, countdown=10*60)
