def test_update_failure_line_mark_job_with_human_note( test_job, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) JobNote.objects.create(job=test_job, failure_classification_id=4, user=test_user, text="note") for failure_line in failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert test_job.is_fully_verified() # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_error_verify_ignore_now(test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) text_log_errors, failure_lines = text_log_errors_failure_lines client = APIClient() client.force_authenticate(user=test_user) failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified is False assert 78910 not in [item.bug_number for item in classified_failures] body = {} resp = client.put( reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() error_line.metadata.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert error_line.metadata.best_classification is None assert error_line.metadata.best_is_verified
def test_update_failure_line_ignore(test_job, jm, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines client = APIClient() client.force_authenticate(user=test_user) MatcherManager.register_detector(ManualDetector) text_log_error = text_log_errors[0] failure_line = failure_lines[0] assert text_log_error.best_classification == classified_failures[0] assert text_log_error.best_is_verified is False assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False body = {"project": test_job.repository.name, "best_classification": None} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() text_log_error.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert text_log_error.best_classification is None assert text_log_error.best_is_verified
def test_update_failure_line_mark_job_with_human_note(test_job, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) JobNote.objects.create(job=test_job, failure_classification_id=4, user=test_user, text="note") for failure_line in failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert test_job.is_fully_verified() # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_failure_line_replace(eleven_jobs_stored, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) failure_line = failure_lines[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False body = {"project": jm.project, "best_classification": classified_failures[1].id} resp = client.put( reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert len(failure_line.classified_failures.all()) == 2 expected_matcher = Matcher.objects.get(name="ManualDetector") assert failure_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
def test_update_failure_line_ignore(eleven_jobs_stored, jm, failure_lines, classified_failures, test_user): client = APIClient() client.force_authenticate(user=test_user) MatcherManager.register_detector(ManualDetector) failure_line = failure_lines[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False body = {"project": jm.project, "best_classification": None} resp = client.put( reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified
def test_update_error_verify_ignore_now(client, test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) text_log_errors, failure_lines = text_log_errors_failure_lines client.force_authenticate(user=test_user) failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified is False assert 78910 not in [item.bug_number for item in classified_failures] body = {} resp = client.put( reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body) assert resp.status_code == 200 failure_line.refresh_from_db() error_line.metadata.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert error_line.metadata.best_classification is None assert error_line.metadata.best_is_verified
def test_update_failure_line_mark_job_with_auto_note( test_job, mock_autoclassify_jobs_true, test_repository, text_log_errors_failure_lines, classified_failures, client, test_user): _, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client.force_authenticate(user=test_user) JobNote.objects.create(job=test_job, failure_classification_id=7, text="note") for failure_line in failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put( reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body) assert resp.status_code == 200 assert test_job.is_fully_verified() notes = JobNote.objects.filter(job=test_job).order_by('-created') assert notes.count() == 2 assert notes[0].failure_classification.id == 4 assert notes[0].user == test_user assert notes[0].text == '' assert notes[1].failure_classification.id == 7 assert not notes[1].user assert notes[1].text == "note"
def test_update_failure_line_ignore(test_job, test_repository, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines client = APIClient() client.force_authenticate(user=test_user) MatcherManager.register_detector(ManualDetector) text_log_error = text_log_errors[0] failure_line = failure_lines[0] assert text_log_error.metadata.best_classification == classified_failures[0] assert text_log_error.metadata.best_is_verified is False assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False body = {"project": test_job.repository.name, "best_classification": None} resp = client.put( reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() text_log_error.refresh_from_db() text_log_error.metadata.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert text_log_error.metadata.best_classification is None assert text_log_error.metadata.best_is_verified
def test_update_failure_line_mark_job_with_auto_note( eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [ line for line in failure_lines if line.job_guid == job.guid ] bs_artifact = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{ "search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message) } for line in job_failure_lines]), 'job_guid': job.guid } with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) JobNote.objects.create(job=job, failure_classification_id=7, text="note") for failure_line in job_failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert jm.is_fully_verified(job.project_specific_id) notes = JobNote.objects.filter(job=job).order_by('-created') assert notes.count() == 2 assert notes[0].failure_classification.id == 4 assert notes[0].user == test_user assert notes[0].text == '' assert notes[1].failure_classification.id == 7 assert not notes[1].user assert notes[1].text == "note"
def test_update_failure_lines(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, test_repository, failure_lines, classified_failures, test_user): jobs = (jm.get_job(1)[0], jm.get_job(2)[0]) MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) create_failure_lines(test_repository, jobs[1]["job_guid"], [(test_line, {}), (test_line, {"subtest": "subtest2"})]) failure_lines = FailureLine.objects.filter( job_guid__in=[job["job_guid"] for job in jobs]).all() for job in jobs: job_failure_lines = FailureLine.objects.filter(job_guid=job["job_guid"]).all() bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job['job_guid']} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) body = [{"id": failure_line.id, "best_classification": classified_failures[1].id} for failure_line in failure_lines] for failure_line in failure_lines: assert failure_line.best_is_verified is False resp = client.put(reverse("failure-line-list"), body, format="json") assert resp.status_code == 200 for failure_line in failure_lines: failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified for job in jobs: assert jm.is_fully_verified(job['id']) notes = jm.get_job_note_list(job['id']) assert len(notes) == 1 assert notes[0]["failure_classification_id"] == 4 assert notes[0]["who"] == test_user.email
def test_update_failure_lines(mock_autoclassify_jobs_true, test_repository, text_log_errors_failure_lines, classified_failures, eleven_jobs_stored, test_user): jobs = (Job.objects.get(id=1), Job.objects.get(id=2)) MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) lines = [(test_line, {}), (test_line, {"subtest": "subtest2"})] new_failure_lines = create_failure_lines(jobs[1], lines) new_text_log_errors = create_text_log_errors(jobs[1], lines) for text_log_error, failure_line in zip(new_text_log_errors, new_failure_lines): TextLogErrorMetadata.objects.create(text_log_error=text_log_error, failure_line=failure_line) failure_lines = FailureLine.objects.filter( job_guid__in=[job.guid for job in jobs]).all() text_log_errors = TextLogError.objects.filter( step__job__in=jobs).all() for text_log_error, failure_line in zip(text_log_errors, failure_lines): assert text_log_error.metadata.best_is_verified is False assert failure_line.best_is_verified is False body = [{"id": failure_line.id, "best_classification": classified_failures[1].id} for failure_line in failure_lines] resp = client.put(reverse("failure-line-list"), body, format="json") assert resp.status_code == 200 for text_log_error, failure_line in zip(text_log_errors, failure_lines): text_log_error.refresh_from_db() text_log_error.metadata.refresh_from_db() failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert text_log_error.metadata.best_classification == classified_failures[1] assert text_log_error.metadata.best_is_verified for job in jobs: assert job.is_fully_verified() # will assert if we don't have exactly one job, which is what we want note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_failure_lines(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, test_repository, failure_lines, classified_failures, test_user): jobs = (Job.objects.get(project_specific_id=1), Job.objects.get(project_specific_id=2)) MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) create_failure_lines(test_repository, jobs[1].guid, [(test_line, {}), (test_line, {"subtest": "subtest2"})]) failure_lines = FailureLine.objects.filter( job_guid__in=[job.guid for job in jobs]).all() for job in jobs: job_failure_lines = FailureLine.objects.filter(job_guid=job.guid).all() bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job.guid} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) body = [{"id": failure_line.id, "best_classification": classified_failures[1].id} for failure_line in failure_lines] for failure_line in failure_lines: assert failure_line.best_is_verified is False resp = client.put(reverse("failure-line-list"), body, format="json") assert resp.status_code == 200 for failure_line in failure_lines: failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified for job in jobs: assert jm.is_fully_verified(job.project_specific_id) # will assert if we don't have exactly one job, which is what we want note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_failure_line_all_ignore_mark_job(test_job, mock_autoclassify_jobs_true, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job_failure_lines = [ line for line in failure_lines if line.job_guid == test_job.guid ] job_text_log_errors = [ error for error in text_log_errors if error.step.job == test_job ] for error_line, failure_line in zip(job_text_log_errors, job_failure_lines): error_line.best_is_verified = False error_line.best_classification = None failure_line.best_is_verified = False failure_line.best_classification = None assert JobNote.objects.count() == 0 for error_line, failure_line in zip(job_text_log_errors, job_failure_lines): assert error_line.best_is_verified is False assert failure_line.best_is_verified is False body = {"best_classification": None} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 error_line.refresh_from_db() failure_line.refresh_from_db() assert error_line.best_classification is None assert error_line.best_is_verified assert failure_line.best_classification is None assert failure_line.best_is_verified assert test_job.is_fully_verified() assert JobNote.objects.count() == 1
def test_update_failure_line_mark_job(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [line for line in failure_lines if line.job_guid == job.guid] classified_failures[1].bug_number = 1234 classified_failures[1].save() bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job.guid} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) for failure_line in job_failure_lines: assert failure_line.best_is_verified is False body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert jm.is_fully_verified(job.project_specific_id) # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user job_bugs = BugJobMap.objects.filter(job=job) assert job_bugs.count() == 1 assert job_bugs[0].bug_id == 1234
def test_update_failure_line_all_ignore_mark_job(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = jm.get_job(1)[0] job_failure_lines = [line for line in failure_lines if line.job_guid == job["job_guid"]] for failure_line in job_failure_lines: failure_line.best_is_verified = False failure_line.best_classification = None bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job['job_guid']} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) assert len(jm.get_job_note_list(job['id'])) == 0 for failure_line in job_failure_lines: assert failure_line.best_is_verified is False body = {"best_classification": None} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert jm.is_fully_verified(job['id']) notes = jm.get_job_note_list(job['id']) assert len(notes) == 1
def test_update_failure_line_partial_ignore_mark_job(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [line for line in failure_lines if line.job_guid == job.guid] bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job.guid} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) for i, failure_line in enumerate(job_failure_lines): assert failure_line.best_is_verified is False body = {"best_classification": None if i == 0 else classified_failures[0].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() if i == 0: assert failure_line.best_classification is None else: assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified assert jm.is_fully_verified(job.project_specific_id) # will assert if we don't have exactly one note for this job, which is # what we want note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_failure_line_mark_job(test_repository, test_job, text_log_errors_failure_lines, classified_failures, client, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client.force_authenticate(user=test_user) bug = Bugscache.objects.create(id=1234, status="NEW", modified="2014-01-01 00:00:00", summary="test") classified_failures[1].bug_number = bug.id classified_failures[1].save() for text_log_error, failure_line in zip(text_log_errors, failure_lines): assert failure_line.best_is_verified is False assert text_log_error.metadata.best_is_verified is False body = {"best_classification": classified_failures[1].id} resp = client.put( reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body) assert resp.status_code == 200 failure_line.refresh_from_db() text_log_error.refresh_from_db() text_log_error.metadata.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert text_log_error.metadata.best_classification == classified_failures[ 1] assert text_log_error.metadata.best_is_verified assert test_job.is_fully_verified() # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user job_bugs = BugJobMap.objects.filter(job=test_job) assert job_bugs.count() == 1 assert job_bugs[0].bug_id == bug.id
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [line for line in failure_lines if line.job_guid == job.guid] bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job.guid} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) JobNote.objects.create(job=job, failure_classification_id=7, text="note") for failure_line in job_failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert jm.is_fully_verified(job.project_specific_id) notes = JobNote.objects.filter(job=job).order_by('-created') assert notes.count() == 2 assert notes[0].failure_classification.id == 4 assert notes[0].user == test_user assert notes[0].text == '' assert notes[1].failure_classification.id == 7 assert not notes[1].user assert notes[1].text == "note"
def test_detect_intermittents(test_repository, activate_responses, jm, eleven_jobs_stored, failure_lines, classified_failures, retriggers): retrigger = retriggers[0] test_failure_lines = create_failure_lines(test_repository, retrigger["job_guid"], [(test_line, { "subtest": "subtest2" }), (test_line, { "status": "TIMEOUT" }), (test_line, { "expected": "ERROR" }), (test_line, { "message": "message2" })]) old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all()) # Poke some internal state so that we only use a single matcher for the test MatcherManager._matcher_funcs = {} MatcherManager.register_matcher(PreciseTestMatcher) MatcherManager._detector_funcs = {} detector = MatcherManager.register_detector(TestFailureDetector) call_command('detect_intermittents', test_repository.name, retrigger['job_guid']) assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4 matches_seen = set() failure_ids_seen = old_failure_ids for item in test_failure_lines: item.refresh_from_db() failure_matches = item.matches.all() assert len(failure_matches) == 1 match = failure_matches[0] assert match.classified_failure.id not in failure_ids_seen assert match not in matches_seen assert match.matcher == detector.db_object assert match.score == 1 assert item.best_classification == match.classified_failure assert item.best_is_verified is False matches_seen.add(match) failure_ids_seen.add(match.classified_failure.id)
def test_update_failure_line_all_ignore_mark_job(test_job, mock_autoclassify_jobs_true, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job_failure_lines = [line for line in failure_lines if line.job_guid == test_job.guid] job_text_log_errors = [error for error in text_log_errors if error.step.job == test_job] for error_line, failure_line in zip(job_text_log_errors, job_failure_lines): error_line.metadata.best_is_verified = False error_line.metadata.best_classification = None failure_line.best_is_verified = False failure_line.best_classification = None assert JobNote.objects.count() == 0 for error_line, failure_line in zip(job_text_log_errors, job_failure_lines): assert error_line.metadata.best_is_verified is False assert failure_line.best_is_verified is False body = {"best_classification": None} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 error_line.refresh_from_db() error_line.metadata.refresh_from_db() failure_line.refresh_from_db() assert failure_line.best_classification is None assert failure_line.best_is_verified assert error_line.metadata.best_classification is None assert error_line.metadata.best_is_verified assert test_job.is_fully_verified() assert JobNote.objects.count() == 1
def test_update_error_partial_ignore_mark_job(test_job, mock_autoclassify_jobs_true, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) for i, (error_line, failure_line) in enumerate(zip(text_log_errors, failure_lines)): assert error_line.metadata.best_is_verified is False assert failure_line.best_is_verified is False body = { "best_classification": None if i == 0 else classified_failures[0].id } resp = client.put(reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body, format="json") assert resp.status_code == 200 error_line.metadata.refresh_from_db() failure_line.refresh_from_db() if i == 0: assert error_line.metadata.best_classification is None assert failure_line.best_classification is None else: assert error_line.metadata.best_classification == classified_failures[ 0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified assert test_job.is_fully_verified() # will assert if we don't have exactly one note for this job, which is # what we want note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_detect_intermittents(test_job, failure_lines, classified_failures, retriggered_job): test_failure_lines = create_failure_lines(retriggered_job, [(test_line, { "subtest": "subtest2" }), (test_line, { "status": "TIMEOUT" }), (test_line, { "expected": "ERROR" }), (test_line, { "message": "message2" })]) old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all()) # Poke some internal state so that we only use a single matcher for the test MatcherManager._matcher_funcs = {} MatcherManager.register_matcher(PreciseTestMatcher) MatcherManager._detector_funcs = {} detector = MatcherManager.register_detector(TestFailureDetector) # This allows us to test that the command works and avoids the # issue that the task is a no-op for !mozilla-inbound call_command('detect_intermittents', str(test_job.id)) assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4 matches_seen = set() failure_ids_seen = old_failure_ids for item in test_failure_lines: item.refresh_from_db() failure_matches = item.matches.all() assert len(failure_matches) == 1 match = failure_matches[0] assert match.classified_failure.id not in failure_ids_seen assert match not in matches_seen assert match.matcher == detector.db_object assert match.score == 1 assert item.best_classification == match.classified_failure assert item.best_is_verified is False matches_seen.add(match) failure_ids_seen.add(match.classified_failure.id)
def test_update_failure_line_replace(test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) text_log_errors, failure_lines = text_log_errors_failure_lines failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.failure_line == failure_line assert error_line.best_classification == classified_failures[0] assert error_line.best_is_verified is False body = { "project": test_repository.name, "best_classification": classified_failures[1].id } resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() error_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert len(failure_line.classified_failures.all()) == 2 assert error_line.failure_line == failure_line assert error_line.best_classification == classified_failures[1] assert error_line.best_is_verified expected_matcher = Matcher.objects.get(name="ManualDetector") assert failure_line.matches.get( classified_failure_id=classified_failures[1].id ).matcher == expected_matcher assert error_line.matches.get(classified_failure_id=classified_failures[1]. id).matcher == expected_matcher
def test_update_failure_line_mark_job(test_repository, test_job, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) classified_failures[1].bug_number = 1234 classified_failures[1].save() for text_log_error, failure_line in zip(text_log_errors, failure_lines): assert failure_line.best_is_verified is False assert text_log_error.metadata.best_is_verified is False body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() text_log_error.refresh_from_db() text_log_error.metadata.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert text_log_error.metadata.best_classification == classified_failures[1] assert text_log_error.metadata.best_is_verified assert test_job.is_fully_verified() # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user job_bugs = BugJobMap.objects.filter(job=test_job) assert job_bugs.count() == 1 assert job_bugs[0].bug_id == 1234
def test_update_error_bug_change_cf(test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) text_log_errors, failure_lines = text_log_errors_failure_lines client = APIClient() client.force_authenticate(user=test_user) failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified is False assert 78910 not in [item.bug_number for item in classified_failures] classified_failures[1].bug_number = 78910 classified_failures[1].save() body = { "best_classification": classified_failures[0].id, "bug_number": 78910 } resp = client.put(reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body, format="json") assert resp.status_code == 200 classified_failures[1].refresh_from_db() failure_line.refresh_from_db() error_line.metadata.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_classification.bug_number == 78910 assert failure_line.best_is_verified assert error_line.metadata.best_classification == classified_failures[1] assert error_line.metadata.best_is_verified assert error_line.metadata.best_classification.bug_number == 78910 assert ClassifiedFailure.objects.count() == len(classified_failures) - 1
def test_update_failure_line_mark_job(jm, test_job, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) classified_failures[1].bug_number = 1234 classified_failures[1].save() for text_log_error, failure_line in zip(text_log_errors, failure_lines): assert failure_line.best_is_verified is False assert text_log_error.best_is_verified is False body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() text_log_error.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert text_log_error.best_classification == classified_failures[1] assert text_log_error.best_is_verified assert test_job.is_fully_verified() # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user job_bugs = BugJobMap.objects.filter(job=test_job) assert job_bugs.count() == 1 assert job_bugs[0].bug_id == 1234
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = jm.get_job(1)[0] job_failure_lines = [line for line in failure_lines if line.job_guid == job["job_guid"]] bs_artifact = {'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)} for line in job_failure_lines]), 'job_guid': job['job_guid']} with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) jm.insert_job_note(job["id"], 7, "autoclassifier", "note", autoclassify=True) for failure_line in job_failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert jm.is_fully_verified(job['id']) notes = jm.get_job_note_list(job['id']) assert len(notes) == 2 assert notes[0]["failure_classification_id"] == 4 assert notes[0]["who"] == test_user.email
def test_update_error_bug_change_cf(test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) text_log_errors, failure_lines = text_log_errors_failure_lines client = APIClient() client.force_authenticate(user=test_user) failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified is False assert 78910 not in [item.bug_number for item in classified_failures] classified_failures[1].bug_number = 78910 classified_failures[1].save() body = {"best_classification": classified_failures[0].id, "bug_number": 78910} resp = client.put( reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body, format="json") assert resp.status_code == 200 classified_failures[1].refresh_from_db() failure_line.refresh_from_db() error_line.metadata.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_classification.bug_number == 78910 assert failure_line.best_is_verified assert error_line.metadata.best_classification == classified_failures[1] assert error_line.metadata.best_is_verified assert error_line.metadata.best_classification.bug_number == 78910 assert ClassifiedFailure.objects.count() == len(classified_failures) - 1
def test_update_failure_line_replace(test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) text_log_errors, failure_lines = text_log_errors_failure_lines failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified is False body = {"project": test_repository.name, "best_classification": classified_failures[1].id} resp = client.put( reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() error_line.metadata.refresh_from_db() error_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert len(failure_line.classified_failures.all()) == 2 assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[1] assert error_line.metadata.best_is_verified expected_matcher = Matcher.objects.get(name="ManualDetector") assert failure_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher assert error_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
def test_update_failure_line_partial_ignore_mark_job(test_job, mock_autoclassify_jobs_true, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) for i, (error_line, failure_line) in enumerate(zip(text_log_errors, failure_lines)): assert error_line.metadata.best_is_verified is False assert failure_line.best_is_verified is False body = {"best_classification": None if i == 0 else classified_failures[0].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 error_line.refresh_from_db() error_line.metadata.refresh_from_db() failure_line.refresh_from_db() if i == 0: assert failure_line.best_classification is None assert error_line.metadata.best_classification is None else: assert failure_line.best_classification == classified_failures[0] assert error_line.metadata.best_classification == classified_failures[0] assert failure_line.best_is_verified assert test_job.is_fully_verified() # will assert if we don't have exactly one note for this job, which is # what we want note = JobNote.objects.get(job=test_job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_error_verify_bug(test_repository, text_log_errors_failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) text_log_errors, failure_lines = text_log_errors_failure_lines client = APIClient() client.force_authenticate(user=test_user) failure_line = failure_lines[0] error_line = text_log_errors[0] assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified is False assert error_line.metadata.failure_line == failure_line assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified is False classified_failures[0].bug_number = 1234 classified_failures[0].save() body = {"bug_number": classified_failures[0].bug_number} resp = client.put( reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() error_line.metadata.refresh_from_db() assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified assert error_line.metadata.best_classification == classified_failures[0] assert error_line.metadata.best_is_verified es_line = _TestFailureLine.get(failure_line.id, routing=failure_line.test) assert es_line.best_classification == classified_failures[0].id assert es_line.best_is_verified
def test_detect_intermittents(test_repository, activate_responses, jm, eleven_jobs_stored, failure_lines, classified_failures, retriggers): retrigger = retriggers[0] test_failure_lines = create_failure_lines(test_repository, retrigger["job_guid"], [(test_line, {"subtest": "subtest2"}), (test_line, {"status": "TIMEOUT"}), (test_line, {"expected": "ERROR"}), (test_line, {"message": "message2"})]) old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all()) # Poke some internal state so that we only use a single matcher for the test MatcherManager._matcher_funcs = {} MatcherManager.register_matcher(PreciseTestMatcher) MatcherManager._detector_funcs = {} detector = MatcherManager.register_detector(TestFailureDetector) call_command('detect_intermittents', test_repository.name, retrigger['job_guid']) assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4 matches_seen = set() failure_ids_seen = old_failure_ids for item in test_failure_lines: item.refresh_from_db() failure_matches = item.matches.all() assert len(failure_matches) == 1 match = failure_matches[0] assert match.classified_failure.id not in failure_ids_seen assert match not in matches_seen assert match.matcher == detector.db_object assert match.score == 1 assert item.best_classification == match.classified_failure assert item.best_is_verified is False matches_seen.add(match) failure_ids_seen.add(match.classified_failure.id)
def test_detect_intermittents(test_job, failure_lines, classified_failures, retriggered_job): test_failure_lines = create_failure_lines(retriggered_job, [(test_line, {"subtest": "subtest2"}), (test_line, {"status": "TIMEOUT"}), (test_line, {"expected": "ERROR"}), (test_line, {"message": "message2"})]) old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all()) # Poke some internal state so that we only use a single matcher for the test MatcherManager._matcher_funcs = {} MatcherManager.register_matcher(PreciseTestMatcher) MatcherManager._detector_funcs = {} detector = MatcherManager.register_detector(TestFailureDetector) # This allows us to test that the command works and avoids the # issue that the task is a no-op for !mozilla-inbound call_command('detect_intermittents', str(test_job.id)) assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4 matches_seen = set() failure_ids_seen = old_failure_ids for item in test_failure_lines: item.refresh_from_db() failure_matches = item.matches.all() assert len(failure_matches) == 1 match = failure_matches[0] assert match.classified_failure.id not in failure_ids_seen assert match not in matches_seen assert match.matcher == detector.db_object assert match.score == 1 assert item.best_classification == match.classified_failure assert item.best_is_verified is False matches_seen.add(match) failure_ids_seen.add(match.classified_failure.id)
def test_update_failure_line_mark_job_with_auto_note(test_job, mock_autoclassify_jobs_true, test_repository, text_log_errors_failure_lines, classified_failures, test_user): text_log_errors, failure_lines = text_log_errors_failure_lines MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) JobNote.objects.create(job=test_job, failure_classification_id=7, text="note") for failure_line in failure_lines: body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 assert test_job.is_fully_verified() notes = JobNote.objects.filter(job=test_job).order_by('-created') assert notes.count() == 2 assert notes[0].failure_classification.id == 4 assert notes[0].user == test_user assert notes[0].text == '' assert notes[1].failure_classification.id == 7 assert not notes[1].user assert notes[1].text == "note"
def test_matcher(request): from treeherder.autoclassify import detectors from treeherder.model.models import MatcherManager class TreeherderUnitTestDetector(detectors.Detector): def __call__(self, failure_lines): return True MatcherManager._detector_funcs = {} MatcherManager._matcher_funcs = {} test_matcher = MatcherManager.register_detector(TreeherderUnitTestDetector) def finalize(): MatcherManager._detector_funcs = {} MatcherManager._matcher_funcs = {} request.addfinalizer(finalize) return test_matcher
def register_matchers(*args): MatcherManager._matcher_funcs = {} for item in args: MatcherManager.register_matcher(item)
def register(): for obj in [PreciseTestMatcher]: MatcherManager.register_matcher(obj)
def register_detectors(*args): MatcherManager._detector_funcs = {} for item in args: MatcherManager.register_detector(item)
def register(): """Register matchers enabled in settings.AUTOCLASSIFY_MATCHERS.""" for obj_name in settings.AUTOCLASSIFY_MATCHERS: obj = globals()[obj_name] MatcherManager.register_matcher(obj)
def register(): for obj in [ManualDetector, TestFailureDetector]: MatcherManager.register_detector(obj)
def register(): for obj_name in settings.AUTOCLASSIFY_MATCHERS: obj = globals()[obj_name] MatcherManager.register_matcher(obj)
def test_update_failure_line_partial_ignore_mark_job( eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [ line for line in failure_lines if line.job_guid == job.guid ] bs_artifact = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{ "search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message) } for line in job_failure_lines]), 'job_guid': job.guid } with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) for i, failure_line in enumerate(job_failure_lines): assert failure_line.best_is_verified is False body = { "best_classification": None if i == 0 else classified_failures[0].id } resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() if i == 0: assert failure_line.best_classification is None else: assert failure_line.best_classification == classified_failures[0] assert failure_line.best_is_verified assert jm.is_fully_verified(job.project_specific_id) # will assert if we don't have exactly one note for this job, which is # what we want note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_failure_lines(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, test_repository, failure_lines, classified_failures, test_user): jobs = (Job.objects.get(project_specific_id=1), Job.objects.get(project_specific_id=2)) MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) create_failure_lines(test_repository, jobs[1].guid, [(test_line, {}), (test_line, { "subtest": "subtest2" })]) failure_lines = FailureLine.objects.filter( job_guid__in=[job.guid for job in jobs]).all() for job in jobs: job_failure_lines = FailureLine.objects.filter(job_guid=job.guid).all() bs_artifact = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{ "search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message) } for line in job_failure_lines]), 'job_guid': job.guid } with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) body = [{ "id": failure_line.id, "best_classification": classified_failures[1].id } for failure_line in failure_lines] for failure_line in failure_lines: assert failure_line.best_is_verified is False resp = client.put(reverse("failure-line-list"), body, format="json") assert resp.status_code == 200 for failure_line in failure_lines: failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified for job in jobs: assert jm.is_fully_verified(job.project_specific_id) # will assert if we don't have exactly one job, which is what we want note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user
def test_update_failure_line_mark_job(eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines, classified_failures, test_user): MatcherManager.register_detector(ManualDetector) client = APIClient() client.force_authenticate(user=test_user) job = Job.objects.get(project_specific_id=1) job_failure_lines = [ line for line in failure_lines if line.job_guid == job.guid ] classified_failures[1].bug_number = 1234 classified_failures[1].save() bs_artifact = { 'type': 'json', 'name': 'Bug suggestions', 'blob': json.dumps([{ "search": "TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message) } for line in job_failure_lines]), 'job_guid': job.guid } with ArtifactsModel(jm.project) as artifacts_model: artifacts_model.load_job_artifacts([bs_artifact]) for failure_line in job_failure_lines: assert failure_line.best_is_verified is False body = {"best_classification": classified_failures[1].id} resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}), body, format="json") assert resp.status_code == 200 failure_line.refresh_from_db() assert failure_line.best_classification == classified_failures[1] assert failure_line.best_is_verified assert jm.is_fully_verified(job.project_specific_id) # should only be one, will assert if that isn't the case note = JobNote.objects.get(job=job) assert note.failure_classification.id == 4 assert note.user == test_user job_bugs = BugJobMap.objects.filter(job=job) assert job_bugs.count() == 1 assert job_bugs[0].bug_id == 1234
def register(): for obj in [PreciseTestMatcher, CrashSignatureMatcher]: MatcherManager.register_matcher(obj)