def create(self, request, project):
artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA)
job_guids = [x['job_guid'] for x in artifacts]
with JobsModel(project) as jobs_model, ArtifactsModel(
project) as artifacts_model:
job_id_lookup = jobs_model.get_job_ids_by_guid(job_guids)
artifacts_model.load_job_artifacts(artifacts, job_id_lookup)
# If a ``text_log_summary`` and ``Bug suggestions`` artifact are
# posted here together, for the same ``job_guid``, then just load
# them. This is how it is done internally in our log parser
# so there is no delay in creation and the bug suggestions show
# as soon as the log is parsed.
#
# If a ``text_log_summary`` is posted WITHOUT an accompanying
# ``Bug suggestions`` artifact, then schedule to create it
# asynchronously so that this api does not take too long.
tls_list = get_artifacts_that_need_bug_suggestions(artifacts)
# tls_list will contain all ``text_log_summary`` artifacts that
# do NOT have an accompanying ``Bug suggestions`` artifact in this
# current list of artifacts. If it's empty, then we don't need
# to schedule anything.
if tls_list:
populate_error_summary.apply_async(
args=[project, tls_list, job_id_lookup],
routing_key='error_summary')
return Response({'message': 'Artifacts stored successfully'})
def create(self, request, project):
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
request.data)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
return Response({'message': 'Artifacts stored successfully'})
def create_artifacts(project, data):
artifacts = ArtifactsModel.serialize_artifact_json_blobs(data)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(artifacts)
# If a ``text_log_summary`` and ``Bug suggestions`` artifact are
# posted here together, for the same ``job_guid``, then just load
# them. This is how it is done internally in our log parser
# so there is no delay in creation and the bug suggestions show
# as soon as the log is parsed.
#
# If a ``text_log_summary`` is posted WITHOUT an accompanying
# ``Bug suggestions`` artifact, then schedule to create it
# asynchronously so that this api does not take too long.
tls_list = get_artifacts_that_need_bug_suggestions(artifacts)
# tls_list will contain all ``text_log_summary`` artifacts that
# do NOT have an accompanying ``Bug suggestions`` artifact in this
# current list of artifacts. If it's empty, then we don't need
# to schedule anything.
if tls_list:
populate_error_summary.apply_async(args=[project, tls_list],
routing_key='error_summary')
def test_bugzilla_comment_length_capped(test_project, eleven_jobs_stored):
"""
Test that the total number of characters in the comment is capped correctly.
"""
bug_id = 12345678
job_id = 1
who = "*****@*****.**"
# Create an error line with length equal to the max comment length.
# Once the job metadata has been added, the total comment length
# will exceed the max length, unless correctly truncated.
bug_suggestions = [{
"search": "a" * settings.BZ_MAX_COMMENT_LENGTH,
"search_terms": [],
"bugs": []
}]
bug_suggestions_placeholders = [
job_id,
'Bug suggestions',
'json',
json.dumps(bug_suggestions),
job_id,
'Bug suggestions',
]
with ArtifactsModel(test_project) as artifacts_model:
artifacts_model.store_job_artifact([bug_suggestions_placeholders])
req = BugzillaCommentRequest(test_project, job_id, bug_id, who)
req.generate_request_body()
assert len(req.body['comment']) == settings.BZ_MAX_COMMENT_LENGTH
def create(self, request, project):
artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA)
job_guids = [x['job_guid'] for x in artifacts]
with JobsModel(project) as jobs_model, ArtifactsModel(project) as artifacts_model:
job_id_lookup = jobs_model.get_job_ids_by_guid(job_guids)
artifacts_model.load_job_artifacts(artifacts, job_id_lookup)
# If a ``text_log_summary`` and ``Bug suggestions`` artifact are
# posted here together, for the same ``job_guid``, then just load
# them. This is how it is done internally in our log parser
# so there is no delay in creation and the bug suggestions show
# as soon as the log is parsed.
#
# If a ``text_log_summary`` is posted WITHOUT an accompanying
# ``Bug suggestions`` artifact, then schedule to create it
# asynchronously so that this api does not take too long.
tls_list = get_artifacts_that_need_bug_suggestions(artifacts)
# tls_list will contain all ``text_log_summary`` artifacts that
# do NOT have an accompanying ``Bug suggestions`` artifact in this
# current list of artifacts. If it's empty, then we don't need
# to schedule anything.
if tls_list:
populate_error_summary.apply_async(
args=[project, tls_list, job_id_lookup],
routing_key='error_summary'
)
return Response({'message': 'Artifacts stored successfully'})
def retrieve(self, request, project, jm, pk=None):
"""
GET method implementation for detail view
Return a single job with log_references and
artifact names and links to the artifact blobs.
"""
obj = jm.get_job(pk)
if obj:
job = obj[0]
job["resource_uri"] = reverse("jobs-detail",
kwargs={"project": jm.project, "pk": job["id"]})
job["logs"] = jm.get_log_references(pk)
# make artifact ids into uris
with ArtifactsModel(project) as artifacts_model:
artifact_refs = artifacts_model.get_job_artifact_references(pk)
job["artifacts"] = []
for art in artifact_refs:
ref = reverse("artifact-detail",
kwargs={"project": jm.project, "pk": art["id"]})
art["resource_uri"] = ref
job["artifacts"].append(art)
option_collections = jm.refdata_model.get_all_option_collections()
job["platform_option"] = get_option(job, option_collections)
return Response(job)
else:
return Response("No job with id: {0}".format(pk), 404)
def create(self, request, project):
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
request.data)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
return Response({'message': 'Artifacts stored successfully'})
def test_artifact_detail(webapp, test_project, eleven_jobs_stored, sample_artifacts, jm):
"""
test retrieving a single artifact from the artifact-detail
endpoint.
"""
job = jm.get_job_list(0, 1)[0]
with ArtifactsModel(test_project) as artifacts_model:
artifact = artifacts_model.get_job_artifact_references(job["id"])[0]
resp = webapp.get(
reverse("artifact-detail",
kwargs={"project": jm.project, "pk": int(artifact["id"])})
)
assert resp.status_int == 200
assert isinstance(resp.json, dict)
assert resp.json["id"] == artifact["id"]
assert set(resp.json.keys()) == set([
"job_id",
"blob",
"type",
"id",
"name"
])
jm.disconnect()
def test_artifact_create_text_log_summary(webapp, test_project, eleven_jobs_stored,
mock_post_json, mock_error_summary,
sample_data):
"""
test submitting a text_log_summary artifact which auto-generates bug suggestions
"""
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
tls = sample_data.text_log_summary
tac = client.TreeherderArtifactCollection()
ta = client.TreeherderArtifact({
'type': 'json',
'name': 'text_log_summary',
'blob': json.dumps(tls['blob']),
'job_guid': job['job_guid']
})
tac.add(ta)
cli = client.TreeherderClient(protocol='http', host='localhost')
cli.post_collection(test_project, tac)
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
'job_id': {('=', job["id"])}
})
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0]
assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
assert mock_error_summary == act_bs_obj
def test_load_long_job_details(test_project, eleven_jobs_stored):
# job details should still load even if excessively long,
# they'll just be truncated
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
max_field_length = JobDetail.MAX_FIELD_LENGTH
(long_title, long_value, long_url) = ('t' * (2 * max_field_length),
'v' * (2 * max_field_length),
'https://' + ('u' * (2 * max_field_length)))
ji_artifact = {
'type': 'json',
'name': 'Job Info',
'blob': json.dumps({
'job_details': [{
'title': long_title,
'value': long_value,
'url': long_url
}]
}),
'job_guid': job['job_guid']
}
with ArtifactsModel(test_project) as am:
am.load_job_artifacts([ji_artifact])
assert JobDetail.objects.count() == 1
jd = JobDetail.objects.all()[0]
assert jd.title == long_title[:max_field_length]
assert jd.value == long_value[:max_field_length]
assert jd.url == long_url[:max_field_length]
def check_artifacts(test_project,
job_guid,
parse_status,
num_artifacts,
exp_artifact_names=None,
exp_error_summary=None):
job_logs = JobLog.objects.filter(job__guid=job_guid)
assert len(job_logs) == 1
assert job_logs[0].status == parse_status
with ArtifactsModel(test_project) as artifacts_model:
job_id = Job.objects.values_list('project_specific_id').get(
guid=job_guid)
artifacts = artifacts_model.get_job_artifact_list(
0, 10, conditions={'job_id': {('=', job_id)}})
assert len(artifacts) == num_artifacts
if exp_artifact_names:
artifact_names = {x['name'] for x in artifacts}
assert set(artifact_names) == exp_artifact_names
if exp_error_summary:
act_bs_obj = [
x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'
][0]
assert exp_error_summary == act_bs_obj
def unstructured_bugs(self):
"""
Get bugs that match this line in the Bug Suggestions artifact for this job.
"""
components = self._serialized_components()
if not components:
return []
# Importing this at the top level causes circular import misery
from treeherder.model.derived import JobsModel, ArtifactsModel
with JobsModel(self.repository.name) as jm, \
ArtifactsModel(self.repository.name) as am:
job_id = jm.get_job_ids_by_guid([self.job_guid
])[self.job_guid]["id"]
bug_suggestions = am.filter_bug_suggestions(
am.bug_suggestions(job_id))
rv = []
ids_seen = set()
for item in bug_suggestions:
if all(component in item["search"] for component in components):
for suggestion in itertools.chain(item["bugs"]["open_recent"],
item["bugs"]["all_others"]):
if suggestion["id"] not in ids_seen:
ids_seen.add(suggestion["id"])
rv.append(suggestion)
return rv
def test_load_long_job_details(test_project, eleven_jobs_stored):
# job details should still load even if excessively long,
# they'll just be truncated
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
def max_length(field):
"""Get the field's max_length for the JobDetail model"""
return JobDetail._meta.get_field(field).max_length
(long_title, long_value, long_url) = ('t' * (2 * max_length("title")),
'v' * (2 * max_length("value")),
'https://' + ('u' * (2 * max_length("url"))))
ji_artifact = {
'type': 'json',
'name': 'Job Info',
'blob': json.dumps({
'job_details': [{
'title': long_title,
'value': long_value,
'url': long_url
}]
}),
'job_guid': job['job_guid']
}
with ArtifactsModel(test_project) as am:
am.load_job_artifacts([ji_artifact])
assert JobDetail.objects.count() == 1
jd = JobDetail.objects.all()[0]
assert jd.title == long_title[:max_length("title")]
assert jd.value == long_value[:max_length("value")]
assert jd.url == long_url[:max_length("url")]
def check_artifacts(test_project,
job_guid,
parse_status,
num_artifacts,
exp_artifact_names=None,
exp_error_summary=None):
with JobsModel(test_project) as jobs_model:
job_id = [
x['id'] for x in jobs_model.get_job_list(0, 20)
if x['job_guid'] == job_guid
][0]
job_log_list = jobs_model.get_job_log_url_list([job_id])
assert len(job_log_list) == 1
assert job_log_list[0]['parse_status'] == parse_status
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(
0, 10, conditions={'job_id': {('=', job_id)}})
assert len(artifacts) == num_artifacts
if exp_artifact_names:
artifact_names = {x['name'] for x in artifacts}
assert set(artifact_names) == exp_artifact_names
if exp_error_summary:
act_bs_obj = [
x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'
][0]
assert exp_error_summary == act_bs_obj
def test_load_single_artifact(test_project, eleven_jobs_processed,
mock_post_collection, mock_error_summary,
sample_data):
"""
test loading a single artifact
"""
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
bs_blob = ["flim", "flam"]
bs_artifact = {
'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps(bs_blob),
'job_guid': job['job_guid']
}
with ArtifactsModel(test_project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact],
{bs_artifact['job_guid']: job})
artifacts = artifacts_model.get_job_artifact_list(
0, 10, conditions={'job_id': {('=', job["id"])}})
assert len(artifacts) == 1
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [
x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'
][0]
assert set(artifact_names) == {'Bug suggestions'}
assert bs_blob == act_bs_obj
def load_error_summary(project, artifacts):
"""Load new bug suggestions artifacts if we generate them."""
from treeherder.model.derived import ArtifactsModel
bsa = get_error_summary_artifacts(artifacts)
if bsa:
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(bsa)
def test_update_failure_line_mark_job_with_auto_note(
eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [
line for line in failure_lines if line.job_guid == job.guid
]
bs_artifact = {
'type':
'json',
'name':
'Bug suggestions',
'blob':
json.dumps([{
"search":
"TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)
} for line in job_failure_lines]),
'job_guid':
job.guid
}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
JobNote.objects.create(job=job, failure_classification_id=7, text="note")
for failure_line in job_failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
assert jm.is_fully_verified(job.project_specific_id)
notes = JobNote.objects.filter(job=job).order_by('-created')
assert notes.count() == 2
assert notes[0].failure_classification.id == 4
assert notes[0].user == test_user
assert notes[0].text == ''
assert notes[1].failure_classification.id == 7
assert not notes[1].user
assert notes[1].text == "note"
def generate_request_body(self):
"""
Create the data structure that will be sent to Elasticsearch.
"""
with JobsModel(self.project) as jobs_model, ArtifactsModel(
self.project) as artifacts_model:
job_data = jobs_model.get_job(self.job_id)[0]
option_collection = jobs_model.refdata_model.get_all_option_collections(
)
revision_list = jobs_model.get_resultset_revisions_list(
job_data["result_set_id"])
buildapi_artifact = artifacts_model.get_job_artifact_list(
0, 1, {
'job_id': set([("=", self.job_id)]),
'name': set([("=", "buildapi")])
})
if buildapi_artifact:
buildname = buildapi_artifact[0]["blob"]["buildername"]
else:
# OrangeFactor needs a buildname to be set or it skips the failure
# classification, so we make one up for non-buildbot jobs.
buildname = 'non-buildbot %s test %s' % (
job_data["platform"], job_data["job_type_name"])
self.body = {
"buildname":
buildname,
"machinename":
job_data["machine_name"],
"os":
job_data["platform"],
# I'm using the request time date here, as start time is not
# available for pending jobs
"date":
datetime.fromtimestamp(int(
job_data["submit_timestamp"])).strftime("%Y-%m-%d"),
"type":
job_data["job_type_name"],
"buildtype":
option_collection[job_data["option_collection_hash"]]["opt"],
# Intentionally using strings for starttime, bug, timestamp for compatibility
# with TBPL's legacy output format.
"starttime":
str(job_data["start_timestamp"]),
"tree":
self.project,
"rev":
revision_list[0]["revision"],
"bug":
str(self.bug_id),
"who":
self.who,
"timestamp":
str(self.classification_timestamp),
"treeherder_job_id":
self.job_id,
}
def text_log_summary(self, request, project, jm, pk=None):
"""
Get a list of test failure lines for the job
"""
job = jm.get_job(pk)
if not job:
return Response("No job with id: {0}".format(pk),
status=HTTP_404_NOT_FOUND)
job = job[0]
summary = TextLogSummary.objects.filter(
job_guid=job['job_guid']).prefetch_related("lines").all()
if len(summary) > 1:
raise ValueError("Got multiple TextLogSummaries for the same job")
if not summary:
return Response(
"No text_log_summary generated for job with id: {0}".format(
pk),
status=HTTP_404_NOT_FOUND)
with ArtifactsModel(project) as am:
artifacts = am.get_job_artifact_list(
offset=0,
limit=2,
conditions={
"job_id": set([("=", pk)]),
"name":
set([("IN", ("Bug suggestions", "text_log_summary"))]),
"type": set([("=", "json")])
})
artifacts_by_name = {item["name"]: item for item in artifacts}
text_log_summary = artifacts_by_name.get("text_log_summary", {})
error_lines = text_log_summary.get("blob",
{}).get("step_data",
{}).get("all_errors", [])
bug_suggestions = artifacts_by_name.get("Bug suggestions",
{}).get("blob")
summary = summary[0]
text_log_summary = artifacts_by_name.get("text_log_summary", {})
lines_by_number = {
line["linenumber"]: line["line"]
for line in error_lines
}
rv = serializers.TextLogSummarySerializer(summary).data
rv["bug_suggestions"] = bug_suggestions
for line in rv["lines"]:
line["line"] = lines_by_number[line["line_number"]]
return Response(rv)
def test_update_failure_lines(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
test_repository, failure_lines,
classified_failures, test_user):
jobs = (jm.get_job(1)[0], jm.get_job(2)[0])
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
create_failure_lines(test_repository,
jobs[1]["job_guid"],
[(test_line, {}),
(test_line, {"subtest": "subtest2"})])
failure_lines = FailureLine.objects.filter(
job_guid__in=[job["job_guid"] for job in jobs]).all()
for job in jobs:
job_failure_lines = FailureLine.objects.filter(job_guid=job["job_guid"]).all()
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)} for line in
job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
body = [{"id": failure_line.id,
"best_classification": classified_failures[1].id}
for failure_line in failure_lines]
for failure_line in failure_lines:
assert failure_line.best_is_verified is False
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for failure_line in failure_lines:
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
for job in jobs:
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 1
assert notes[0]["failure_classification_id"] == 4
assert notes[0]["who"] == test_user.email
def test_load_non_ascii_textlog_errors(test_project, eleven_jobs_stored):
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
text_log_summary_artifact = {
'type':
'json',
'name':
'text_log_summary',
'blob':
json.dumps({
'step_data': {
"steps": [{
'name':
'foo',
'started':
'2016-05-10 12:44:23.103904',
'started_linenumber':
8,
'finished_linenumber':
10,
'finished':
'2016-05-10 12:44:23.104394',
'result':
'success',
'errors': [
{
# non-ascii character
"line":
'07:51:28 WARNING - \U000000c3'.encode('utf-8'),
"linenumber":
1587
},
{
# astral character (i.e. higher than ucs2)
"line":
'07:51:29 WARNING - \U0001d400'.encode('utf-8'),
"linenumber":
1588
}
]
}]
}
}),
'job_guid':
job['job_guid']
}
with ArtifactsModel(test_project) as am:
am.load_job_artifacts([text_log_summary_artifact])
assert TextLogError.objects.count() == 2
assert TextLogError.objects.get(
line_number=1587).line == '07:51:28 WARNING - \U000000c3'
assert TextLogError.objects.get(
line_number=1588).line == '07:51:29 WARNING - <U+01D400>'
def test_get_job_data(jm, test_project, sample_data, sample_resultset,
test_repository, mock_log_parser):
target_len = 10
job_data = sample_data.job_data[:target_len]
test_utils.do_job_ingestion(jm, job_data, sample_resultset)
with ArtifactsModel(test_project) as artifacts_model:
job_data = artifacts_model.get_job_signatures_from_ids(range(1, 11))
assert len(job_data) is target_len
def post_log_artifacts(project, job_guid, job_log):
"""Post a list of artifacts to a job."""
log_url = job_log.url
log_description = "%s %s (%s)" % (project, job_guid, log_url)
logger.debug("Downloading/parsing log for %s", log_description)
try:
artifact_list = extract_text_log_artifacts(project, log_url, job_guid)
except Exception as e:
job_log.update_status(JobLog.FAILED)
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s",
log_description, e)
return
if isinstance(e, urllib2.URLError):
# possibly recoverable http error (e.g. problems on our end)
logger.error("Failed to download log for %s: %s", log_description,
e)
else:
# parse error or other unrecoverable error
logger.error("Failed to download/parse log for %s: %s",
log_description, e)
raise
try:
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
artifact_list)
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
job_log.update_status(JobLog.PARSED)
logger.debug("Stored artifact for %s %s", project, job_guid)
except Exception as e:
logger.error("Failed to store parsed artifact for %s: %s",
log_description, e)
raise
def test_update_failure_line_all_ignore_mark_job(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = jm.get_job(1)[0]
job_failure_lines = [line for line in failure_lines if
line.job_guid == job["job_guid"]]
for failure_line in job_failure_lines:
failure_line.best_is_verified = False
failure_line.best_classification = None
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
assert len(jm.get_job_note_list(job['id'])) == 0
for failure_line in job_failure_lines:
assert failure_line.best_is_verified is False
body = {"best_classification": None}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 1
def retrieve(self, request, project, pk=None):
"""
retrieve a single instance of job_artifact
"""
filter = UrlQueryFilter({"id": pk})
with ArtifactsModel(project) as artifactModel:
objs = artifactModel.get_job_artifact_list(0, 1, filter.conditions)
if objs:
return Response(objs[0])
else:
return Response("job_artifact {0} not found".format(pk), 404)
def project_info(request, project):
try:
with JobsModel(project) as jobs_model, ArtifactsModel(
project) as artifacts_model:
return HttpResponse(content=json.dumps({
'max_job_id':
jobs_model.get_max_job_id(),
'max_performance_artifact_id':
artifacts_model.get_max_performance_artifact_id()
}),
content_type='application/json')
except DatasetNotFoundError:
return HttpResponseNotFound('Project does not exist')
def load_error_summary(project, job_id):
"""Load new bug suggestions artifacts if we generate them."""
job = Job.objects.get(id=job_id)
bug_suggestion_artifact = {
"job_guid": job.guid,
"name": 'Bug suggestions',
"type": 'json',
"blob": json.dumps(get_error_summary(job))
}
from treeherder.model.derived import ArtifactsModel
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts([bug_suggestion_artifact])
def test_artifact_create_text_log_summary_and_bug_suggestions(
webapp, test_project, eleven_jobs_stored, mock_post_json,
mock_error_summary, sample_data):
"""
test submitting text_log_summary and Bug suggestions artifacts
This should NOT generate a Bug suggestions artifact, just post the one
submitted.
"""
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
tls = sample_data.text_log_summary
bs_blob = ["flim", "flam"]
tac = client.TreeherderArtifactCollection()
tac.add(
client.TreeherderArtifact({
'type': 'json',
'name': 'text_log_summary',
'blob': json.dumps(tls['blob']),
'job_guid': job['job_guid']
}))
tac.add(
client.TreeherderArtifact({
'type': 'json',
'name': 'Bug suggestions',
'blob': bs_blob,
'job_guid': job['job_guid']
}))
credentials = OAuthCredentials.get_credentials(test_project)
auth = TreeherderAuth(credentials['consumer_key'],
credentials['consumer_secret'], test_project)
cli = client.TreeherderClient(protocol='http', host='localhost', auth=auth)
cli.post_collection(test_project, tac)
with ArtifactsModel(test_project) as artifacts_model:
artifacts = artifacts_model.get_job_artifact_list(
0, 10, conditions={'job_id': {('=', job["id"])}})
assert len(artifacts) == 2
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [
x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'
][0]
assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'}
assert bs_blob == act_bs_obj
def test_bugzilla_comment_request_body(test_project, eleven_jobs_stored):
"""
Test the request body is created correctly
"""
bug_id = 12345678
job_id = 1
who = "*****@*****.**"
bug_suggestions = []
bug_suggestions.append({
"search": "First error line",
"search_terms": [],
"bugs": []
})
bug_suggestions.append({
"search": "Second error line",
"search_terms": [],
"bugs": []
})
bug_suggestions_placeholders = [
job_id,
'Bug suggestions',
'json',
json.dumps(bug_suggestions),
job_id,
'Bug suggestions',
]
with ArtifactsModel(test_project) as artifacts_model:
artifacts_model.store_job_artifact([bug_suggestions_placeholders])
req = BugzillaCommentRequest(test_project, job_id, bug_id, who)
req.generate_request_body()
expected = {
'comment':
(u'log: http://local.treeherder.mozilla.org/'
u'logviewer.html#?repo=test_treeherder&job_id=1\n'
u'repository: test_treeherder\n'
u'start_time: 2013-11-13T06:39:13\n'
u'who: user[at]mozilla[dot]com\n'
u'machine: bld-linux64-ec2-132\n'
u'buildname: non-buildbot b2g-emu-jb test B2G Emulator Image Build\n'
u'revision: cdfe03e77e66\n\n'
u'First error line\n'
u'Second error line')
}
assert req.body == expected
def test_load_artifact_second_time_fails(
test_project, eleven_jobs_stored,
mock_post_json, mock_error_summary,
sample_data):
"""
test loading two of the same named artifact only gets the first one
"""
with JobsModel(test_project) as jobs_model:
job = jobs_model.get_job_list(0, 1)[0]
bs_blob = ["flim", "flam"]
bs_artifact1 = {
'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps(bs_blob),
'job_guid': job['job_guid']
}
bs_artifact2 = {
'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps(["me", "you"]),
'job_guid': job['job_guid']
}
with ArtifactsModel(test_project) as artifacts_model:
artifacts_model.load_job_artifacts(
[bs_artifact1],
{bs_artifact1['job_guid']: job}
)
artifacts_model.load_job_artifacts(
[bs_artifact2],
{bs_artifact2['job_guid']: job}
)
artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={
'job_id': {('=', job["id"])}
})
assert len(artifacts) == 1
artifact_names = {x['name'] for x in artifacts}
act_bs_obj = [x['blob'] for x in artifacts
if x['name'] == 'Bug suggestions'][0]
assert set(artifact_names) == {'Bug suggestions'}
assert bs_blob == act_bs_obj
def retrieve(self, request, project, jm, pk=None):
"""
GET method implementation for detail view
Return a single job with log_references and
artifact names and links to the artifact blobs.
"""
obj = jm.get_job(pk)
if not obj:
return Response("No job with id: {0}".format(pk),
status=HTTP_404_NOT_FOUND)
job = obj[0]
job["resource_uri"] = reverse("jobs-detail",
kwargs={
"project": jm.project,
"pk": job["id"]
})
job["logs"] = []
for (name, url) in JobLog.objects.filter(
job__repository__name=jm.project,
job__project_specific_id=job['id']).values_list('name', 'url'):
job["logs"].append({'name': name, 'url': url})
# make artifact ids into uris
with ArtifactsModel(project) as artifacts_model:
artifact_refs = artifacts_model.get_job_artifact_references(pk)
job["artifacts"] = []
for art in artifact_refs:
ref = reverse("artifact-detail",
kwargs={
"project": jm.project,
"pk": art["id"]
})
art["resource_uri"] = ref
job["artifacts"].append(art)
option_hash = job['option_collection_hash']
if option_hash:
option_collection_map = self._get_option_collection_map()
job["platform_option"] = option_collection_map[option_hash]
return Response(job)
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = jm.get_job(1)[0]
job_failure_lines = [line for line in failure_lines if
line.job_guid == job["job_guid"]]
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
jm.insert_job_note(job["id"], 7, "autoclassifier", "note", autoclassify=True)
for failure_line in job_failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 2
assert notes[0]["failure_classification_id"] == 4
assert notes[0]["who"] == test_user.email
def post_log_artifacts(job_log):
"""Post a list of artifacts to a job."""
logger.debug("Downloading/parsing log for log %s", job_log.id)
try:
artifact_list = extract_text_log_artifacts(job_log)
except Exception as e:
job_log.update_status(JobLog.FAILED)
# unrecoverable http error (doesn't exist or permission denied)
# (apparently this can happen somewhat often with taskcluster if
# the job fails, so just warn about it -- see
# https://bugzilla.mozilla.org/show_bug.cgi?id=1154248)
if isinstance(e, urllib2.HTTPError) and e.code in (403, 404):
logger.warning("Unable to retrieve log for %s: %s", job_log.id, e)
return
if isinstance(e, urllib2.URLError):
# possibly recoverable http error (e.g. problems on our end)
logger.error("Failed to download log for %s: %s", job_log.id, e)
else:
# parse error or other unrecoverable error
logger.error("Failed to download/parse log for %s: %s", job_log.id, e)
raise
try:
serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs(
artifact_list)
project = job_log.job.repository.name
with ArtifactsModel(project) as artifacts_model:
artifacts_model.load_job_artifacts(serialized_artifacts)
job_log.update_status(JobLog.PARSED)
logger.debug("Stored artifact for %s %s", project, job_log.job.id)
except Exception as e:
logger.error("Failed to store parsed artifact for %s: %s", job_log.id, e)
raise