def create(self, request, project): artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA) job_guids = [x['job_guid'] for x in artifacts] with JobsModel(project) as jobs_model, ArtifactsModel( project) as artifacts_model: job_id_lookup = jobs_model.get_job_ids_by_guid(job_guids) artifacts_model.load_job_artifacts(artifacts, job_id_lookup) # If a ``text_log_summary`` and ``Bug suggestions`` artifact are # posted here together, for the same ``job_guid``, then just load # them. This is how it is done internally in our log parser # so there is no delay in creation and the bug suggestions show # as soon as the log is parsed. # # If a ``text_log_summary`` is posted WITHOUT an accompanying # ``Bug suggestions`` artifact, then schedule to create it # asynchronously so that this api does not take too long. tls_list = get_artifacts_that_need_bug_suggestions(artifacts) # tls_list will contain all ``text_log_summary`` artifacts that # do NOT have an accompanying ``Bug suggestions`` artifact in this # current list of artifacts. If it's empty, then we don't need # to schedule anything. if tls_list: populate_error_summary.apply_async( args=[project, tls_list, job_id_lookup], routing_key='error_summary') return Response({'message': 'Artifacts stored successfully'})
def create(self, request, project): serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs( request.data) with ArtifactsModel(project) as artifacts_model: artifacts_model.load_job_artifacts(serialized_artifacts) return Response({'message': 'Artifacts stored successfully'})
def create_artifacts(project, data): artifacts = ArtifactsModel.serialize_artifact_json_blobs(data) with ArtifactsModel(project) as artifacts_model: artifacts_model.load_job_artifacts(artifacts) # If a ``text_log_summary`` and ``Bug suggestions`` artifact are # posted here together, for the same ``job_guid``, then just load # them. This is how it is done internally in our log parser # so there is no delay in creation and the bug suggestions show # as soon as the log is parsed. # # If a ``text_log_summary`` is posted WITHOUT an accompanying # ``Bug suggestions`` artifact, then schedule to create it # asynchronously so that this api does not take too long. tls_list = get_artifacts_that_need_bug_suggestions(artifacts) # tls_list will contain all ``text_log_summary`` artifacts that # do NOT have an accompanying ``Bug suggestions`` artifact in this # current list of artifacts. If it's empty, then we don't need # to schedule anything. if tls_list: populate_error_summary.apply_async(args=[project, tls_list], routing_key='error_summary')
def test_bugzilla_comment_length_capped(test_project, eleven_jobs_stored): """ Test that the total number of characters in the comment is capped correctly. """ bug_id = 12345678 job_id = 1 who = "*****@*****.**" # Create an error line with length equal to the max comment length. # Once the job metadata has been added, the total comment length # will exceed the max length, unless correctly truncated. bug_suggestions = [{ "search": "a" * settings.BZ_MAX_COMMENT_LENGTH, "search_terms": [], "bugs": [] }] bug_suggestions_placeholders = [ job_id, 'Bug suggestions', 'json', json.dumps(bug_suggestions), job_id, 'Bug suggestions', ] with ArtifactsModel(test_project) as artifacts_model: artifacts_model.store_job_artifact([bug_suggestions_placeholders]) req = BugzillaCommentRequest(test_project, job_id, bug_id, who) req.generate_request_body() assert len(req.body['comment']) == settings.BZ_MAX_COMMENT_LENGTH
def create(self, request, project): artifacts = ArtifactsModel.serialize_artifact_json_blobs(request.DATA) job_guids = [x['job_guid'] for x in artifacts] with JobsModel(project) as jobs_model, ArtifactsModel(project) as artifacts_model: job_id_lookup = jobs_model.get_job_ids_by_guid(job_guids) artifacts_model.load_job_artifacts(artifacts, job_id_lookup) # If a ``text_log_summary`` and ``Bug suggestions`` artifact are # posted here together, for the same ``job_guid``, then just load # them. This is how it is done internally in our log parser # so there is no delay in creation and the bug suggestions show # as soon as the log is parsed. # # If a ``text_log_summary`` is posted WITHOUT an accompanying # ``Bug suggestions`` artifact, then schedule to create it # asynchronously so that this api does not take too long. tls_list = get_artifacts_that_need_bug_suggestions(artifacts) # tls_list will contain all ``text_log_summary`` artifacts that # do NOT have an accompanying ``Bug suggestions`` artifact in this # current list of artifacts. If it's empty, then we don't need # to schedule anything. if tls_list: populate_error_summary.apply_async( args=[project, tls_list, job_id_lookup], routing_key='error_summary' ) return Response({'message': 'Artifacts stored successfully'})
def retrieve(self, request, project, jm, pk=None): """ GET method implementation for detail view Return a single job with log_references and artifact names and links to the artifact blobs. """ obj = jm.get_job(pk) if obj: job = obj[0] job["resource_uri"] = reverse("jobs-detail", kwargs={"project": jm.project, "pk": job["id"]}) job["logs"] = jm.get_log_references(pk) # make artifact ids into uris with ArtifactsModel(project) as artifacts_model: artifact_refs = artifacts_model.get_job_artifact_references(pk) job["artifacts"] = [] for art in artifact_refs: ref = reverse("artifact-detail", kwargs={"project": jm.project, "pk": art["id"]}) art["resource_uri"] = ref job["artifacts"].append(art) option_collections = jm.refdata_model.get_all_option_collections() job["platform_option"] = get_option(job, option_collections) return Response(job) else: return Response("No job with id: {0}".format(pk), 404)
def test_artifact_detail(webapp, test_project, eleven_jobs_stored, sample_artifacts, jm): """ test retrieving a single artifact from the artifact-detail endpoint. """ job = jm.get_job_list(0, 1)[0] with ArtifactsModel(test_project) as artifacts_model: artifact = artifacts_model.get_job_artifact_references(job["id"])[0] resp = webapp.get( reverse("artifact-detail", kwargs={"project": jm.project, "pk": int(artifact["id"])}) ) assert resp.status_int == 200 assert isinstance(resp.json, dict) assert resp.json["id"] == artifact["id"] assert set(resp.json.keys()) == set([ "job_id", "blob", "type", "id", "name" ]) jm.disconnect()
def test_artifact_create_text_log_summary(webapp, test_project, eleven_jobs_stored, mock_post_json, mock_error_summary, sample_data): """ test submitting a text_log_summary artifact which auto-generates bug suggestions """ with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] tls = sample_data.text_log_summary tac = client.TreeherderArtifactCollection() ta = client.TreeherderArtifact({ 'type': 'json', 'name': 'text_log_summary', 'blob': json.dumps(tls['blob']), 'job_guid': job['job_guid'] }) tac.add(ta) cli = client.TreeherderClient(protocol='http', host='localhost') cli.post_collection(test_project, tac) with ArtifactsModel(test_project) as artifacts_model: artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={ 'job_id': {('=', job["id"])} }) artifact_names = {x['name'] for x in artifacts} act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0] assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'} assert mock_error_summary == act_bs_obj
def test_load_long_job_details(test_project, eleven_jobs_stored): # job details should still load even if excessively long, # they'll just be truncated with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] max_field_length = JobDetail.MAX_FIELD_LENGTH (long_title, long_value, long_url) = ('t' * (2 * max_field_length), 'v' * (2 * max_field_length), 'https://' + ('u' * (2 * max_field_length))) ji_artifact = { 'type': 'json', 'name': 'Job Info', 'blob': json.dumps({ 'job_details': [{ 'title': long_title, 'value': long_value, 'url': long_url }] }), 'job_guid': job['job_guid'] } with ArtifactsModel(test_project) as am: am.load_job_artifacts([ji_artifact]) assert JobDetail.objects.count() == 1 jd = JobDetail.objects.all()[0] assert jd.title == long_title[:max_field_length] assert jd.value == long_value[:max_field_length] assert jd.url == long_url[:max_field_length]
def check_artifacts(test_project, job_guid, parse_status, num_artifacts, exp_artifact_names=None, exp_error_summary=None): job_logs = JobLog.objects.filter(job__guid=job_guid) assert len(job_logs) == 1 assert job_logs[0].status == parse_status with ArtifactsModel(test_project) as artifacts_model: job_id = Job.objects.values_list('project_specific_id').get( guid=job_guid) artifacts = artifacts_model.get_job_artifact_list( 0, 10, conditions={'job_id': {('=', job_id)}}) assert len(artifacts) == num_artifacts if exp_artifact_names: artifact_names = {x['name'] for x in artifacts} assert set(artifact_names) == exp_artifact_names if exp_error_summary: act_bs_obj = [ x['blob'] for x in artifacts if x['name'] == 'Bug suggestions' ][0] assert exp_error_summary == act_bs_obj
def unstructured_bugs(self): """ Get bugs that match this line in the Bug Suggestions artifact for this job. """ components = self._serialized_components() if not components: return [] # Importing this at the top level causes circular import misery from treeherder.model.derived import JobsModel, ArtifactsModel with JobsModel(self.repository.name) as jm, \ ArtifactsModel(self.repository.name) as am: job_id = jm.get_job_ids_by_guid([self.job_guid ])[self.job_guid]["id"] bug_suggestions = am.filter_bug_suggestions( am.bug_suggestions(job_id)) rv = [] ids_seen = set() for item in bug_suggestions: if all(component in item["search"] for component in components): for suggestion in itertools.chain(item["bugs"]["open_recent"], item["bugs"]["all_others"]): if suggestion["id"] not in ids_seen: ids_seen.add(suggestion["id"]) rv.append(suggestion) return rv
def test_load_long_job_details(test_project, eleven_jobs_stored): # job details should still load even if excessively long, # they'll just be truncated with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] def max_length(field): """Get the field's max_length for the JobDetail model""" return JobDetail._meta.get_field(field).max_length (long_title, long_value, long_url) = ('t' * (2 * max_length("title")), 'v' * (2 * max_length("value")), 'https://' + ('u' * (2 * max_length("url")))) ji_artifact = { 'type': 'json', 'name': 'Job Info', 'blob': json.dumps({ 'job_details': [{ 'title': long_title, 'value': long_value, 'url': long_url }] }), 'job_guid': job['job_guid'] } with ArtifactsModel(test_project) as am: am.load_job_artifacts([ji_artifact]) assert JobDetail.objects.count() == 1 jd = JobDetail.objects.all()[0] assert jd.title == long_title[:max_length("title")] assert jd.value == long_value[:max_length("value")] assert jd.url == long_url[:max_length("url")]
def check_artifacts(test_project, job_guid, parse_status, num_artifacts, exp_artifact_names=None, exp_error_summary=None): with JobsModel(test_project) as jobs_model: job_id = [ x['id'] for x in jobs_model.get_job_list(0, 20) if x['job_guid'] == job_guid ][0] job_log_list = jobs_model.get_job_log_url_list([job_id]) assert len(job_log_list) == 1 assert job_log_list[0]['parse_status'] == parse_status with ArtifactsModel(test_project) as artifacts_model: artifacts = artifacts_model.get_job_artifact_list( 0, 10, conditions={'job_id': {('=', job_id)}}) assert len(artifacts) == num_artifacts if exp_artifact_names: artifact_names = {x['name'] for x in artifacts} assert set(artifact_names) == exp_artifact_names if exp_error_summary: act_bs_obj = [ x['blob'] for x in artifacts if x['name'] == 'Bug suggestions' ][0] assert exp_error_summary == act_bs_obj
def test_load_single_artifact(test_project, eleven_jobs_processed, mock_post_collection, mock_error_summary, sample_data): """ test loading a single artifact """ with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] bs_blob = ["flim", "flam"] bs_artifact = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps(bs_blob), 'job_guid': job['job_guid'] } with ArtifactsModel(test_project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact], {bs_artifact['job_guid']: job}) artifacts = artifacts_model.get_job_artifact_list( 0, 10, conditions={'job_id': {('=', job["id"])}}) assert len(artifacts) == 1 artifact_names = {x['name'] for x in artifacts} act_bs_obj = [ x['blob'] for x in artifacts if x['name'] == 'Bug suggestions' ][0] assert set(artifact_names) == {'Bug suggestions'} assert bs_blob == act_bs_obj
def load_error_summary(project, artifacts): """Load new bug suggestions artifacts if we generate them.""" from treeherder.model.derived import ArtifactsModel bsa = get_error_summary_artifacts(artifacts) if bsa: with ArtifactsModel(project) as artifacts_model: artifacts_model.load_job_artifacts(bsa)
def test_update_failure_line_mark_job_with_auto_note( eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [ line for line in failure_lines if line.job_guid == job.guid ] bs_artifact = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{ "search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message) } for line in job_failure_lines]), 'job_guid': job.guid } with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) JobNote.objects.create(job=job, failure_classification_id=7, text="note") for failure_line in job_failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert jm.is_fully_verified(job.project_specific_id) notes = JobNote.objects.filter(job=job).order_by('-created') assert notes.count() == 2 assert notes[0].failure_classification.id == 4 assert notes[0].user == test_user assert notes[0].text == '' assert notes[1].failure_classification.id == 7 assert not notes[1].user assert notes[1].text == "note"
def generate_request_body(self): """ Create the data structure that will be sent to Elasticsearch. """ with JobsModel(self.project) as jobs_model, ArtifactsModel( self.project) as artifacts_model: job_data = jobs_model.get_job(self.job_id)[0] option_collection = jobs_model.refdata_model.get_all_option_collections( ) revision_list = jobs_model.get_resultset_revisions_list( job_data["result_set_id"]) buildapi_artifact = artifacts_model.get_job_artifact_list( 0, 1, { 'job_id': set([("=", self.job_id)]), 'name': set([("=", "buildapi")]) }) if buildapi_artifact: buildname = buildapi_artifact[0]["blob"]["buildername"] else: # OrangeFactor needs a buildname to be set or it skips the failure # classification, so we make one up for non-buildbot jobs. buildname = 'non-buildbot %s test %s' % ( job_data["platform"], job_data["job_type_name"]) self.body = { "buildname": buildname, "machinename": job_data["machine_name"], "os": job_data["platform"], # I'm using the request time date here, as start time is not # available for pending jobs "date": datetime.fromtimestamp(int( job_data["submit_timestamp"])).strftime("%Y-%m-%d"), "type": job_data["job_type_name"], "buildtype": option_collection[job_data["option_collection_hash"]]["opt"], # Intentionally using strings for starttime, bug, timestamp for compatibility # with TBPL's legacy output format. "starttime": str(job_data["start_timestamp"]), "tree": self.project, "rev": revision_list[0]["revision"], "bug": str(self.bug_id), "who": self.who, "timestamp": str(self.classification_timestamp), "treeherder_job_id": self.job_id, }
def text_log_summary(self, request, project, jm, pk=None): """ Get a list of test failure lines for the job """ job = jm.get_job(pk) if not job: return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND) job = job[0] summary = TextLogSummary.objects.filter( job_guid=job['job_guid']).prefetch_related("lines").all() if len(summary) > 1: raise ValueError("Got multiple TextLogSummaries for the same job") if not summary: return Response( "No text_log_summary generated for job with id: {0}".format( pk), status=HTTP_404_NOT_FOUND) with ArtifactsModel(project) as am: artifacts = am.get_job_artifact_list( offset=0, limit=2, conditions={ "job_id": set([("=", pk)]), "name": set([("IN", ("Bug suggestions", "text_log_summary"))]), "type": set([("=", "json")]) }) artifacts_by_name = {item["name"]: item for item in artifacts} text_log_summary = artifacts_by_name.get("text_log_summary", {}) error_lines = text_log_summary.get("blob", {}).get("step_data", {}).get("all_errors", []) bug_suggestions = artifacts_by_name.get("Bug suggestions", {}).get("blob") summary = summary[0] text_log_summary = artifacts_by_name.get("text_log_summary", {}) lines_by_number = { line["linenumber"]: line["line"] for line in error_lines } rv = serializers.TextLogSummarySerializer(summary).data rv["bug_suggestions"] = bug_suggestions for line in rv["lines"]: line["line"] = lines_by_number[line["line_number"]] return Response(rv)
def test_update_failure_lines(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, test_repository, failure_lines, classified_failures, test_user): jobs = (jm.get_job(1)[0], jm.get_job(2)[0]) MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) create_failure_lines(test_repository, jobs[1]["job_guid"], [(test_line, {}), (test_line, {"subtest": "subtest2"})]) failure_lines = FailureLine.objects.filter( job_guid__in=[job["job_guid"] for job in jobs]).all() for job in jobs: job_failure_lines = FailureLine.objects.filter(job_guid=job["job_guid"]).all() bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job['job_guid']} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) body = [{"id": failure_line.id, "best_classification": classified_failures[1].id} for failure_line in failure_lines] for failure_line in failure_lines: assert failure_line.best_is_verified is False resp = client.put(reverse("failure-line-list"), body, format="json") assert resp.status_code == 200 for failure_line in failure_lines: failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified for job in jobs: assert jm.is_fully_verified(job['id']) notes = jm.get_job_note_list(job['id']) assert len(notes) == 1 assert notes[0]["failure_classification_id"] == 4 assert notes[0]["who"] == test_user.email
def test_load_non_ascii_textlog_errors(test_project, eleven_jobs_stored): with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] text_log_summary_artifact = { 'type': 'json', 'name': 'text_log_summary', 'blob': json.dumps({ 'step_data': { "steps": [{ 'name': 'foo', 'started': '2016-05-10 12:44:23.103904', 'started_linenumber': 8, 'finished_linenumber': 10, 'finished': '2016-05-10 12:44:23.104394', 'result': 'success', 'errors': [ { # non-ascii character "line": '07:51:28 WARNING - \U000000c3'.encode('utf-8'), "linenumber": 1587 }, { # astral character (i.e. higher than ucs2) "line": '07:51:29 WARNING - \U0001d400'.encode('utf-8'), "linenumber": 1588 } ] }] } }), 'job_guid': job['job_guid'] } with ArtifactsModel(test_project) as am: am.load_job_artifacts([text_log_summary_artifact]) assert TextLogError.objects.count() == 2 assert TextLogError.objects.get( line_number=1587).line == '07:51:28 WARNING - \U000000c3' assert TextLogError.objects.get( line_number=1588).line == '07:51:29 WARNING - <U+01D400>'
def test_get_job_data(jm, test_project, sample_data, sample_resultset, test_repository, mock_log_parser): target_len = 10 job_data = sample_data.job_data[:target_len] test_utils.do_job_ingestion(jm, job_data, sample_resultset) with ArtifactsModel(test_project) as artifacts_model: job_data = artifacts_model.get_job_signatures_from_ids(range(1, 11)) assert len(job_data) is target_len
def post_log_artifacts(project, job_guid, job_log): """Post a list of artifacts to a job.""" log_url = job_log.url log_description = "%s %s (%s)" % (project, job_guid, log_url) logger.debug("Downloading/parsing log for %s", log_description) try: artifact_list = extract_text_log_artifacts(project, log_url, job_guid) except Exception as e: job_log.update_status(JobLog.FAILED) # unrecoverable http error (doesn't exist or permission denied) # (apparently this can happen somewhat often with taskcluster if # the job fails, so just warn about it -- see # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248) if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.warning("Unable to retrieve log for %s: %s", log_description, e) return if isinstance(e, urllib2.URLError): # possibly recoverable http error (e.g. problems on our end) logger.error("Failed to download log for %s: %s", log_description, e) else: # parse error or other unrecoverable error logger.error("Failed to download/parse log for %s: %s", log_description, e) raise try: serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs( artifact_list) with ArtifactsModel(project) as artifacts_model: artifacts_model.load_job_artifacts(serialized_artifacts) job_log.update_status(JobLog.PARSED) logger.debug("Stored artifact for %s %s", project, job_guid) except Exception as e: logger.error("Failed to store parsed artifact for %s: %s", log_description, e) raise
def test_update_failure_line_all_ignore_mark_job(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = jm.get_job(1)[0] job_failure_lines = [line for line in failure_lines if line.job_guid == job["job_guid"]] for failure_line in job_failure_lines: failure_line.best_is_verified = False failure_line.best_classification = None bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job['job_guid']} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) assert len(jm.get_job_note_list(job['id'])) == 0 for failure_line in job_failure_lines: assert failure_line.best_is_verified is False body = {"best_classification": None} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert jm.is_fully_verified(job['id']) notes = jm.get_job_note_list(job['id']) assert len(notes) == 1
def retrieve(self, request, project, pk=None): """ retrieve a single instance of job_artifact """ filter = UrlQueryFilter({"id": pk}) with ArtifactsModel(project) as artifactModel: objs = artifactModel.get_job_artifact_list(0, 1, filter.conditions) if objs: return Response(objs[0]) else: return Response("job_artifact {0} not found".format(pk), 404)
def project_info(request, project): try: with JobsModel(project) as jobs_model, ArtifactsModel( project) as artifacts_model: return HttpResponse(content=json.dumps({ 'max_job_id': jobs_model.get_max_job_id(), 'max_performance_artifact_id': artifacts_model.get_max_performance_artifact_id() }), content_type='application/json') except DatasetNotFoundError: return HttpResponseNotFound('Project does not exist')
def load_error_summary(project, job_id): """Load new bug suggestions artifacts if we generate them.""" job = Job.objects.get(id=job_id) bug_suggestion_artifact = { "job_guid": job.guid, "name": 'Bug suggestions', "type": 'json', "blob": json.dumps(get_error_summary(job)) } from treeherder.model.derived import ArtifactsModel with ArtifactsModel(project) as artifacts_model: artifacts_model.load_job_artifacts([bug_suggestion_artifact])
def test_artifact_create_text_log_summary_and_bug_suggestions( webapp, test_project, eleven_jobs_stored, mock_post_json, mock_error_summary, sample_data): """ test submitting text_log_summary and Bug suggestions artifacts This should NOT generate a Bug suggestions artifact, just post the one submitted. """ with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] tls = sample_data.text_log_summary bs_blob = ["flim", "flam"] tac = client.TreeherderArtifactCollection() tac.add( client.TreeherderArtifact({ 'type': 'json', 'name': 'text_log_summary', 'blob': json.dumps(tls['blob']), 'job_guid': job['job_guid'] })) tac.add( client.TreeherderArtifact({ 'type': 'json', 'name': 'Bug suggestions', 'blob': bs_blob, 'job_guid': job['job_guid'] })) credentials = OAuthCredentials.get_credentials(test_project) auth = TreeherderAuth(credentials['consumer_key'], credentials['consumer_secret'], test_project) cli = client.TreeherderClient(protocol='http', host='localhost', auth=auth) cli.post_collection(test_project, tac) with ArtifactsModel(test_project) as artifacts_model: artifacts = artifacts_model.get_job_artifact_list( 0, 10, conditions={'job_id': {('=', job["id"])}}) assert len(artifacts) == 2 artifact_names = {x['name'] for x in artifacts} act_bs_obj = [ x['blob'] for x in artifacts if x['name'] == 'Bug suggestions' ][0] assert set(artifact_names) == {'Bug suggestions', 'text_log_summary'} assert bs_blob == act_bs_obj
def test_bugzilla_comment_request_body(test_project, eleven_jobs_stored): """ Test the request body is created correctly """ bug_id = 12345678 job_id = 1 who = "*****@*****.**" bug_suggestions = [] bug_suggestions.append({ "search": "First error line", "search_terms": [], "bugs": [] }) bug_suggestions.append({ "search": "Second error line", "search_terms": [], "bugs": [] }) bug_suggestions_placeholders = [ job_id, 'Bug suggestions', 'json', json.dumps(bug_suggestions), job_id, 'Bug suggestions', ] with ArtifactsModel(test_project) as artifacts_model: artifacts_model.store_job_artifact([bug_suggestions_placeholders]) req = BugzillaCommentRequest(test_project, job_id, bug_id, who) req.generate_request_body() expected = { 'comment': (u'log: http://local.treeherder.mozilla.org/' u'logviewer.html#?repo=test_treeherder&job_id=1\n' u'repository: test_treeherder\n' u'start_time: 2013-11-13T06:39:13\n' u'who: user[at]mozilla[dot]com\n' u'machine: bld-linux64-ec2-132\n' u'buildname: non-buildbot b2g-emu-jb test B2G Emulator Image Build\n' u'revision: cdfe03e77e66\n\n' u'First error line\n' u'Second error line') } assert req.body == expected
def test_load_artifact_second_time_fails( test_project, eleven_jobs_stored, mock_post_json, mock_error_summary, sample_data): """ test loading two of the same named artifact only gets the first one """ with JobsModel(test_project) as jobs_model: job = jobs_model.get_job_list(0, 1)[0] bs_blob = ["flim", "flam"] bs_artifact1 = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps(bs_blob), 'job_guid': job['job_guid'] } bs_artifact2 = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps(["me", "you"]), 'job_guid': job['job_guid'] } with ArtifactsModel(test_project) as artifacts_model: artifacts_model.load_job_artifacts( [bs_artifact1], {bs_artifact1['job_guid']: job} ) artifacts_model.load_job_artifacts( [bs_artifact2], {bs_artifact2['job_guid']: job} ) artifacts = artifacts_model.get_job_artifact_list(0, 10, conditions={ 'job_id': {('=', job["id"])} }) assert len(artifacts) == 1 artifact_names = {x['name'] for x in artifacts} act_bs_obj = [x['blob'] for x in artifacts if x['name'] == 'Bug suggestions'][0] assert set(artifact_names) == {'Bug suggestions'} assert bs_blob == act_bs_obj
def retrieve(self, request, project, jm, pk=None): """ GET method implementation for detail view Return a single job with log_references and artifact names and links to the artifact blobs. """ obj = jm.get_job(pk) if not obj: return Response("No job with id: {0}".format(pk), status=HTTP_404_NOT_FOUND) job = obj[0] job["resource_uri"] = reverse("jobs-detail", kwargs={ "project": jm.project, "pk": job["id"] }) job["logs"] = [] for (name, url) in JobLog.objects.filter( job__repository__name=jm.project, job__project_specific_id=job['id']).values_list('name', 'url'): job["logs"].append({'name': name, 'url': url}) # make artifact ids into uris with ArtifactsModel(project) as artifacts_model: artifact_refs = artifacts_model.get_job_artifact_references(pk) job["artifacts"] = [] for art in artifact_refs: ref = reverse("artifact-detail", kwargs={ "project": jm.project, "pk": art["id"] }) art["resource_uri"] = ref job["artifacts"].append(art) option_hash = job['option_collection_hash'] if option_hash: option_collection_map = self._get_option_collection_map() job["platform_option"] = option_collection_map[option_hash] return Response(job)
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = jm.get_job(1)[0] job_failure_lines = [line for line in failure_lines if line.job_guid == job["job_guid"]] bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job['job_guid']} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) jm.insert_job_note(job["id"], 7, "autoclassifier", "note", autoclassify=True) for failure_line in job_failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert jm.is_fully_verified(job['id']) notes = jm.get_job_note_list(job['id']) assert len(notes) == 2 assert notes[0]["failure_classification_id"] == 4 assert notes[0]["who"] == test_user.email
def post_log_artifacts(job_log): """Post a list of artifacts to a job.""" logger.debug("Downloading/parsing log for log %s", job_log.id) try: artifact_list = extract_text_log_artifacts(job_log) except Exception as e: job_log.update_status(JobLog.FAILED) # unrecoverable http error (doesn't exist or permission denied) # (apparently this can happen somewhat often with taskcluster if # the job fails, so just warn about it -- see # https://bugzilla.mozilla.org/show_bug.cgi?id=1154248) if isinstance(e, urllib2.HTTPError) and e.code in (403, 404): logger.warning("Unable to retrieve log for %s: %s", job_log.id, e) return if isinstance(e, urllib2.URLError): # possibly recoverable http error (e.g. problems on our end) logger.error("Failed to download log for %s: %s", job_log.id, e) else: # parse error or other unrecoverable error logger.error("Failed to download/parse log for %s: %s", job_log.id, e) raise try: serialized_artifacts = ArtifactsModel.serialize_artifact_json_blobs( artifact_list) project = job_log.job.repository.name with ArtifactsModel(project) as artifacts_model: artifacts_model.load_job_artifacts(serialized_artifacts) job_log.update_status(JobLog.PARSED) logger.debug("Stored artifact for %s %s", project, job_log.job.id) except Exception as e: logger.error("Failed to store parsed artifact for %s: %s", job_log.id, e) raise