def test_artifact_sample_data(self):
"""Test all add methods for building an artifact"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact()
ta.add_blob(artifact['blob'])
ta.add_job_guid(artifact['job_guid'])
ta.add_name(artifact['name'])
ta.add_type(artifact['type'])
self.compare_structs(ta.data, artifact)
tac.add(ta)
# confirm we get the same thing if we initialize from
# a resultset dict
ta_struct = TreeherderArtifact(artifact)
self.compare_structs(ta_struct.data, artifact)
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e,
countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'], job_guid,
check_errors)
except Exception as e:
update_parse_status(req, job_log_url, 'failed')
if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug("Log not found for %s", log_description)
return
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
try:
req.post(tac)
update_parse_status(req, job_log_url, 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s",
log_description, e)
_retry(e)
def test_artifact_collection(self):
"""Confirm the collection matches the sample data"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
self.assertTrue( len(self.artifact_data) == len(tac.data) )
def test_artifact_collection(self):
"""Confirm the collection matches the sample data"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact(artifact)
tac.add(ta)
self.assertTrue(len(self.artifact_data) == len(tac.data))
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
update_parse_status(req, job_log_url, 'failed')
if isinstance(e, urllib2.HTTPError) and e.code == 404:
logger.debug("Log not found for %s", log_description)
return
logger.error("Failed to download/parse log for %s: %s", log_description, e)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
try:
req.post(tac)
update_parse_status(req, job_log_url, 'parsed')
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def test_artifact_sample_data(self):
"""Test all add methods for building an artifact"""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
ta = TreeherderArtifact()
ta.add_blob(artifact['blob'])
ta.add_job_guid(artifact['job_guid'])
ta.add_name(artifact['name'])
ta.add_type(artifact['type'])
self.compare_structs(ta.data, artifact)
tac.add(ta)
# confirm we get the same thing if we initialize from
# a resultset dict
ta_struct = TreeherderArtifact(artifact)
self.compare_structs(ta_struct.data, artifact)
def test_send_artifact_collection(self, mock_send):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
req = TreeherderRequest(
protocol='http',
host='host',
project='project',
oauth_key='key',
oauth_secret='secret',
)
req.post(tac)
self.assertEqual(mock_send.call_count, 1)
self.assertEqual(
tac.to_json(),
mock_send.call_args_list[0][1]["data"]
)
def test_send_artifact_collection(self, mock_send):
"""Can add a artifact collections to a TreeherderRequest."""
tac = TreeherderArtifactCollection()
for artifact in self.artifact_data:
tac.add(tac.get_artifact(artifact))
req = TreeherderRequest(
protocol='http',
host='host',
project='project',
oauth_key='key',
oauth_secret='secret',
)
req.post(tac)
self.assertEqual(mock_send.call_count, 1)
self.assertEqual(tac.to_json(), mock_send.call_args_list[0][1]["data"])
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id'])
try:
log_url = job_log_url['url']
bug_suggestions = []
bugscache_uri = '{0}{1}'.format(settings.API_HOSTNAME,
reverse("bugscache-list"))
terms_requested = {}
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(log_url,
check_errors=check_errors)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append(
(job_guid, name, 'json', json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts.get(
'Structured Log', {}).get('step_data',
{}).get('all_errors', [])
for err in all_errors:
# remove the mozharness prefix
clean_line = get_mozharness_substring(err['line'])
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
bugs = dict(open_recent=[], all_others=[])
# collect open recent and all other bugs suggestions
if search_term:
if not search_term in terms_requested:
# retrieve the list of suggestions from the api
bugs = get_bugs_for_search_term(
search_term, bugscache_uri)
terms_requested[search_term] = bugs
else:
bugs = terms_requested[search_term]
if not bugs or not (bugs['open_recent']
or bugs['all_others']):
# no suggestions, try to use
# the crash signature as search term
crash_signature = get_crash_signature(clean_line)
if crash_signature:
if not crash_signature in terms_requested:
bugs = get_bugs_for_search_term(
crash_signature, bugscache_uri)
terms_requested[crash_signature] = bugs
else:
bugs = terms_requested[crash_signature]
bug_suggestions.append({
"search": clean_line,
"bugs": bugs
})
artifact_list.append((job_guid, 'Bug suggestions', 'json',
json.dumps(bug_suggestions)))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'parsed'
req.send(update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
})
except Exception, e:
parse_log.retry(exc=e)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'failed'
req.send(update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
})
# re raise the exception to leave a trace in the log
raise
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
# if parse_status is not available, consider it pending
parse_status = job_log_url.get("parse_status", "pending")
# don't parse a log if it's already been parsed
if parse_status == "parsed":
return
try:
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id'])
logger.debug("Downloading and extracting log information for guid "
"'%s' (from %s)" % (job_guid, job_log_url['url']))
artifact_list = extract_log_artifacts(job_log_url['url'], job_guid,
check_errors)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
logger.debug("Finished downloading and processing artifact for guid "
"'%s'" % job_guid)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
})
logger.debug("Finished posting artifact for guid '%s'" % job_guid)
except Exception, e:
# send an update to job_log_url
#the job_log_url status changes from pending/running to failed
logger.warn("Failed to download and/or parse artifact for guid '%s'" %
job_guid)
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
})
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
parse_log.retry(exc=e, countdown=(1 + parse_log.request.retries) * 60)
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
# if parse_status is not available, consider it pending
parse_status = job_log_url.get("parse_status", "pending")
# don't parse a log if it's already been parsed
if parse_status == "parsed":
return
try:
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id']
)
artifact_list = extract_log_artifacts(job_log_url['url'],
job_guid, check_errors)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
}
)
except Exception, e:
# send an update to job_log_url
#the job_log_url status changes from pending/running to failed
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
}
)
# for every retry, set the countdown to 10 minutes
# .retry() raises a RetryTaskError exception,
# so nothing below this line will be executed.
parse_log.retry(exc=e, countdown=10*60)
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(job_log_url['id'])
try:
log_url = job_log_url['url']
bug_suggestions = []
bugscache_uri = '{0}{1}'.format(
settings.API_HOSTNAME,
reverse("bugscache-list")
)
terms_requested = {}
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(log_url,
check_errors=check_errors)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_guid, name, 'json',
json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts.get(
'Structured Log', {}
).get(
'step_data', {}
).get(
'all_errors', [] )
for err in all_errors:
# remove the mozharness prefix
clean_line = get_mozharness_substring(err['line'])
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
bugs = dict(open_recent=[], all_others=[])
# collect open recent and all other bugs suggestions
if search_term:
if not search_term in terms_requested:
# retrieve the list of suggestions from the api
bugs = get_bugs_for_search_term(
search_term,
bugscache_uri
)
terms_requested[search_term] = bugs
else:
bugs = terms_requested[search_term]
if not bugs or not (bugs['open_recent']
or bugs['all_others']):
# no suggestions, try to use
# the crash signature as search term
crash_signature = get_crash_signature(clean_line)
if crash_signature:
if not crash_signature in terms_requested:
bugs = get_bugs_for_search_term(
crash_signature,
bugscache_uri
)
terms_requested[crash_signature] = bugs
else:
bugs = terms_requested[crash_signature]
bug_suggestions.append({
"search": clean_line,
"bugs": bugs
})
artifact_list.append((job_guid, 'Bug suggestions', 'json', json.dumps(bug_suggestions)))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'parsed'
req.send(
update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
}
)
except Exception, e:
parse_log.retry(exc=e)
# send an update to job_log_url
# the job_log_url status changes
# from pending to running
current_timestamp = time.time()
status = 'failed'
req.send(
update_endpoint,
method='POST',
data={
'parse_status': status,
'parse_timestamp': current_timestamp
}
)
# re raise the exception to leave a trace in the log
raise
def post_log_artifacts(project,
job_guid,
job_log_url,
retry_task,
extract_artifacts_cb,
check_errors=False):
"""Post a list of artifacts to a job."""
def _retry(e):
# Initially retry after 1 minute, then for each subsequent retry
# lengthen the retry time by another minute.
retry_task.retry(exc=e, countdown=(1 + retry_task.request.retries) * 60)
# .retry() raises a RetryTaskError exception,
# so nothing after this function will be executed
credentials = OAuthCredentials.get_credentials(project)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id']
)
log_description = "%s %s (%s)" % (project, job_guid, job_log_url['url'])
logger.debug("Downloading/parsing log for %s", log_description)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
try:
artifact_list = extract_artifacts_cb(job_log_url['url'],
job_guid, check_errors)
except Exception as e:
logger.error("Failed to download/parse log for %s: %s", log_description, e)
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
}
)
_retry(e)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
try:
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(
update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
}
)
logger.debug("Finished posting artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to upload parsed artifact for %s: %s", log_description, e)
_retry(e)
def parse_log(project, job_log_url, job_guid, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
# if parse_status is not available, consider it pending
parse_status = job_log_url.get("parse_status", "pending")
# don't parse a log if it's already been parsed
if parse_status == "parsed":
return
try:
credentials = OAuthCredentials.get_credentials(project)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
update_endpoint = 'job-log-url/{0}/update_parse_status'.format(
job_log_url['id'])
artifact_list = extract_log_artifacts(job_log_url['url'], job_guid,
check_errors)
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req.post(tac)
# send an update to job_log_url
# the job_log_url status changes from pending to parsed
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'parsed',
'parse_timestamp': current_timestamp
})
except Exception, e:
# send an update to job_log_url
#the job_log_url status changes from pending/running to failed
current_timestamp = time.time()
req.send(update_endpoint,
method='POST',
data={
'parse_status': 'failed',
'parse_timestamp': current_timestamp
})
# for every retry, set the countdown to 10 minutes
# .retry() raises a RetryTaskError exception,
# so nothing below this line will be executed.
parse_log.retry(exc=e, countdown=10 * 60)
def parse_log(project, log_url, job_guid, resultset, check_errors=False):
"""
Call ArtifactBuilderCollection on the given job.
"""
mozharness_pattern = re.compile(
'^\d+:\d+:\d+[ ]+(?:DEBUG|INFO|WARNING|ERROR|CRITICAL|FATAL) - [ ]?'
)
bugs_cache = {'open': {}, 'closed': {}}
bug_suggestions = {'open': {}, 'closed': {}}
status_publisher = JobStatusPublisher(settings.BROKER_URL)
failure_publisher = JobFailurePublisher(settings.BROKER_URL)
try:
# return the resultset with the job id to identify if the UI wants
# to fetch the whole thing.
bugscache_uri = '{0}{1}'.format(
settings.API_HOSTNAME,
reverse("bugscache-list")
)
credentials = OAuthCredentials.get_credentials(project)
if log_url:
# parse a log given its url
artifact_bc = ArtifactBuilderCollection(
log_url,
check_errors=check_errors,
)
artifact_bc.parse()
artifact_list = []
for name, artifact in artifact_bc.artifacts.items():
artifact_list.append((job_guid, name, 'json', json.dumps(artifact)))
if check_errors:
all_errors = artifact_bc.artifacts['Structured Log']['step_data']['all_errors']
for err in all_errors:
# remove the mozharness prefix
clean_line = mozharness_pattern.sub('', err['line']).strip()
# get a meaningful search term out of the error line
search_term = get_error_search_term(clean_line)
# collect open and closed bugs suggestions
for status in ('open', 'closed'):
if not search_term:
bug_suggestions[status][clean_line] = []
continue
if search_term not in bugs_cache[status]:
# retrieve the list of suggestions from the api
bugs_cache[status][search_term] = get_bugs_for_search_term(
search_term,
status,
bugscache_uri
)
# no suggestions, try to use the crash signature as search term
if not bugs_cache[status][search_term]:
crash_signature = get_crash_signature(search_term)
if crash_signature:
bugs_cache[status][search_term] = get_bugs_for_search_term(
search_term,
status,
bugscache_uri
)
bug_suggestions[status][clean_line] = bugs_cache[status][search_term]
artifact_list.append((job_guid, 'Open bugs', 'json', json.dumps(bug_suggestions['open'])))
artifact_list.append((job_guid, 'Closed bugs', 'json', json.dumps(bug_suggestions['closed'])))
# store the artifacts generated
tac = TreeherderArtifactCollection()
for artifact in artifact_list:
ta = tac.get_artifact({
"job_guid": artifact[0],
"name": artifact[1],
"type": artifact[2],
"blob": artifact[3]
})
tac.add(ta)
req = TreeherderRequest(
protocol=settings.TREEHERDER_REQUEST_PROTOCOL,
host=settings.TREEHERDER_REQUEST_HOST,
project=project,
oauth_key=credentials.get('consumer_key', None),
oauth_secret=credentials.get('consumer_secret', None),
)
req.send(tac)
status_publisher.publish(job_guid, resultset, project, 'processed')
if check_errors:
failure_publisher.publish(job_guid, project)
finally:
status_publisher.disconnect()
failure_publisher.disconnect()
