def test_update_failure_line_mark_job_with_human_note(
test_job, text_log_errors_failure_lines, classified_failures,
test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
JobNote.objects.create(job=test_job,
failure_classification_id=4,
user=test_user,
text="note")
for failure_line in failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
assert test_job.is_fully_verified()
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_error_verify_ignore_now(test_repository,
text_log_errors_failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
assert 78910 not in [item.bug_number for item in classified_failures]
body = {}
resp = client.put(
reverse("text-log-error-detail", kwargs={"pk": error_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert error_line.metadata.best_classification is None
assert error_line.metadata.best_is_verified
def test_update_failure_line_ignore(test_job, jm,
text_log_errors_failure_lines,
classified_failures, test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
MatcherManager.register_detector(ManualDetector)
text_log_error = text_log_errors[0]
failure_line = failure_lines[0]
assert text_log_error.best_classification == classified_failures[0]
assert text_log_error.best_is_verified is False
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
body = {"project": test_job.repository.name, "best_classification": None}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
text_log_error.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert text_log_error.best_classification is None
assert text_log_error.best_is_verified
def test_update_failure_line_mark_job_with_human_note(test_job,
text_log_errors_failure_lines,
classified_failures, test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
JobNote.objects.create(job=test_job,
failure_classification_id=4,
user=test_user,
text="note")
for failure_line in failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
assert test_job.is_fully_verified()
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_line_replace(eleven_jobs_stored, jm, failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
body = {"project": jm.project,
"best_classification": classified_failures[1].id}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert len(failure_line.classified_failures.all()) == 2
expected_matcher = Matcher.objects.get(name="ManualDetector")
assert failure_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
def test_update_failure_line_ignore(eleven_jobs_stored, jm, failure_lines,
classified_failures, test_user):
client = APIClient()
client.force_authenticate(user=test_user)
MatcherManager.register_detector(ManualDetector)
failure_line = failure_lines[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
body = {"project": jm.project,
"best_classification": None}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
def test_update_error_verify_ignore_now(client, test_repository,
text_log_errors_failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
text_log_errors, failure_lines = text_log_errors_failure_lines
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
assert 78910 not in [item.bug_number for item in classified_failures]
body = {}
resp = client.put(
reverse("text-log-error-detail", kwargs={"pk": error_line.id}), body)
assert resp.status_code == 200
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert error_line.metadata.best_classification is None
assert error_line.metadata.best_is_verified
def test_update_failure_line_replace(eleven_jobs_stored, jm, failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
body = {"project": jm.project,
"best_classification": classified_failures[1].id}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert len(failure_line.classified_failures.all()) == 2
expected_matcher = Matcher.objects.get(name="ManualDetector")
assert failure_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
def test_update_failure_line_mark_job_with_auto_note(
test_job, mock_autoclassify_jobs_true, test_repository,
text_log_errors_failure_lines, classified_failures, client, test_user):
_, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client.force_authenticate(user=test_user)
JobNote.objects.create(job=test_job,
failure_classification_id=7,
text="note")
for failure_line in failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body)
assert resp.status_code == 200
assert test_job.is_fully_verified()
notes = JobNote.objects.filter(job=test_job).order_by('-created')
assert notes.count() == 2
assert notes[0].failure_classification.id == 4
assert notes[0].user == test_user
assert notes[0].text == ''
assert notes[1].failure_classification.id == 7
assert not notes[1].user
assert notes[1].text == "note"
def test_update_failure_line_ignore(test_job,
test_repository,
text_log_errors_failure_lines,
classified_failures, test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
MatcherManager.register_detector(ManualDetector)
text_log_error = text_log_errors[0]
failure_line = failure_lines[0]
assert text_log_error.metadata.best_classification == classified_failures[0]
assert text_log_error.metadata.best_is_verified is False
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
body = {"project": test_job.repository.name,
"best_classification": None}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
text_log_error.refresh_from_db()
text_log_error.metadata.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert text_log_error.metadata.best_classification is None
assert text_log_error.metadata.best_is_verified
def test_update_failure_line_mark_job_with_auto_note(
eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [
line for line in failure_lines if line.job_guid == job.guid
]
bs_artifact = {
'type':
'json',
'name':
'Bug suggestions',
'blob':
json.dumps([{
"search":
"TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)
} for line in job_failure_lines]),
'job_guid':
job.guid
}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
JobNote.objects.create(job=job, failure_classification_id=7, text="note")
for failure_line in job_failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
assert jm.is_fully_verified(job.project_specific_id)
notes = JobNote.objects.filter(job=job).order_by('-created')
assert notes.count() == 2
assert notes[0].failure_classification.id == 4
assert notes[0].user == test_user
assert notes[0].text == ''
assert notes[1].failure_classification.id == 7
assert not notes[1].user
assert notes[1].text == "note"
def test_update_failure_lines(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
test_repository, failure_lines,
classified_failures, test_user):
jobs = (jm.get_job(1)[0], jm.get_job(2)[0])
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
create_failure_lines(test_repository,
jobs[1]["job_guid"],
[(test_line, {}),
(test_line, {"subtest": "subtest2"})])
failure_lines = FailureLine.objects.filter(
job_guid__in=[job["job_guid"] for job in jobs]).all()
for job in jobs:
job_failure_lines = FailureLine.objects.filter(job_guid=job["job_guid"]).all()
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)} for line in
job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
body = [{"id": failure_line.id,
"best_classification": classified_failures[1].id}
for failure_line in failure_lines]
for failure_line in failure_lines:
assert failure_line.best_is_verified is False
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for failure_line in failure_lines:
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
for job in jobs:
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 1
assert notes[0]["failure_classification_id"] == 4
assert notes[0]["who"] == test_user.email
def test_update_failure_lines(mock_autoclassify_jobs_true,
test_repository,
text_log_errors_failure_lines,
classified_failures,
eleven_jobs_stored,
test_user):
jobs = (Job.objects.get(id=1), Job.objects.get(id=2))
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
lines = [(test_line, {}),
(test_line, {"subtest": "subtest2"})]
new_failure_lines = create_failure_lines(jobs[1], lines)
new_text_log_errors = create_text_log_errors(jobs[1], lines)
for text_log_error, failure_line in zip(new_text_log_errors,
new_failure_lines):
TextLogErrorMetadata.objects.create(text_log_error=text_log_error,
failure_line=failure_line)
failure_lines = FailureLine.objects.filter(
job_guid__in=[job.guid for job in jobs]).all()
text_log_errors = TextLogError.objects.filter(
step__job__in=jobs).all()
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
assert text_log_error.metadata.best_is_verified is False
assert failure_line.best_is_verified is False
body = [{"id": failure_line.id,
"best_classification": classified_failures[1].id}
for failure_line in failure_lines]
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
text_log_error.refresh_from_db()
text_log_error.metadata.refresh_from_db()
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert text_log_error.metadata.best_classification == classified_failures[1]
assert text_log_error.metadata.best_is_verified
for job in jobs:
assert job.is_fully_verified()
# will assert if we don't have exactly one job, which is what we want
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_lines(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
test_repository, failure_lines,
classified_failures, test_user):
jobs = (jm.get_job(1)[0], jm.get_job(2)[0])
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
create_failure_lines(test_repository,
jobs[1]["job_guid"],
[(test_line, {}),
(test_line, {"subtest": "subtest2"})])
failure_lines = FailureLine.objects.filter(
job_guid__in=[job["job_guid"] for job in jobs]).all()
for job in jobs:
job_failure_lines = FailureLine.objects.filter(job_guid=job["job_guid"]).all()
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)} for line in
job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
body = [{"id": failure_line.id,
"best_classification": classified_failures[1].id}
for failure_line in failure_lines]
for failure_line in failure_lines:
assert failure_line.best_is_verified is False
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for failure_line in failure_lines:
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
for job in jobs:
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 1
assert notes[0]["failure_classification_id"] == 4
assert notes[0]["who"] == test_user.email
def test_update_failure_lines(mock_autoclassify_jobs_true,
test_repository,
text_log_errors_failure_lines,
classified_failures,
eleven_jobs_stored,
test_user):
jobs = (Job.objects.get(id=1), Job.objects.get(id=2))
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
lines = [(test_line, {}),
(test_line, {"subtest": "subtest2"})]
new_failure_lines = create_failure_lines(jobs[1], lines)
new_text_log_errors = create_text_log_errors(jobs[1], lines)
for text_log_error, failure_line in zip(new_text_log_errors,
new_failure_lines):
TextLogErrorMetadata.objects.create(text_log_error=text_log_error,
failure_line=failure_line)
failure_lines = FailureLine.objects.filter(
job_guid__in=[job.guid for job in jobs]).all()
text_log_errors = TextLogError.objects.filter(
step__job__in=jobs).all()
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
assert text_log_error.metadata.best_is_verified is False
assert failure_line.best_is_verified is False
body = [{"id": failure_line.id,
"best_classification": classified_failures[1].id}
for failure_line in failure_lines]
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
text_log_error.refresh_from_db()
text_log_error.metadata.refresh_from_db()
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert text_log_error.metadata.best_classification == classified_failures[1]
assert text_log_error.metadata.best_is_verified
for job in jobs:
assert job.is_fully_verified()
# will assert if we don't have exactly one job, which is what we want
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_lines(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
test_repository, failure_lines,
classified_failures, test_user):
jobs = (Job.objects.get(project_specific_id=1),
Job.objects.get(project_specific_id=2))
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
create_failure_lines(test_repository,
jobs[1].guid,
[(test_line, {}),
(test_line, {"subtest": "subtest2"})])
failure_lines = FailureLine.objects.filter(
job_guid__in=[job.guid for job in jobs]).all()
for job in jobs:
job_failure_lines = FailureLine.objects.filter(job_guid=job.guid).all()
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)} for line in
job_failure_lines]),
'job_guid': job.guid}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
body = [{"id": failure_line.id,
"best_classification": classified_failures[1].id}
for failure_line in failure_lines]
for failure_line in failure_lines:
assert failure_line.best_is_verified is False
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for failure_line in failure_lines:
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
for job in jobs:
assert jm.is_fully_verified(job.project_specific_id)
# will assert if we don't have exactly one job, which is what we want
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_line_all_ignore_mark_job(test_job,
mock_autoclassify_jobs_true,
text_log_errors_failure_lines,
classified_failures,
test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job_failure_lines = [
line for line in failure_lines if line.job_guid == test_job.guid
]
job_text_log_errors = [
error for error in text_log_errors if error.step.job == test_job
]
for error_line, failure_line in zip(job_text_log_errors,
job_failure_lines):
error_line.best_is_verified = False
error_line.best_classification = None
failure_line.best_is_verified = False
failure_line.best_classification = None
assert JobNote.objects.count() == 0
for error_line, failure_line in zip(job_text_log_errors,
job_failure_lines):
assert error_line.best_is_verified is False
assert failure_line.best_is_verified is False
body = {"best_classification": None}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
error_line.refresh_from_db()
failure_line.refresh_from_db()
assert error_line.best_classification is None
assert error_line.best_is_verified
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert test_job.is_fully_verified()
assert JobNote.objects.count() == 1
def test_update_failure_line_mark_job(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [line for line in failure_lines if
line.job_guid == job.guid]
classified_failures[1].bug_number = 1234
classified_failures[1].save()
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job.guid}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
for failure_line in job_failure_lines:
assert failure_line.best_is_verified is False
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert jm.is_fully_verified(job.project_specific_id)
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
job_bugs = BugJobMap.objects.filter(job=job)
assert job_bugs.count() == 1
assert job_bugs[0].bug_id == 1234
def test_update_failure_line_all_ignore_mark_job(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = jm.get_job(1)[0]
job_failure_lines = [line for line in failure_lines if
line.job_guid == job["job_guid"]]
for failure_line in job_failure_lines:
failure_line.best_is_verified = False
failure_line.best_classification = None
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
assert len(jm.get_job_note_list(job['id'])) == 0
for failure_line in job_failure_lines:
assert failure_line.best_is_verified is False
body = {"best_classification": None}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 1
def test_update_failure_line_all_ignore_mark_job(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = jm.get_job(1)[0]
job_failure_lines = [line for line in failure_lines if
line.job_guid == job["job_guid"]]
for failure_line in job_failure_lines:
failure_line.best_is_verified = False
failure_line.best_classification = None
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
assert len(jm.get_job_note_list(job['id'])) == 0
for failure_line in job_failure_lines:
assert failure_line.best_is_verified is False
body = {"best_classification": None}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 1
def test_update_failure_line_partial_ignore_mark_job(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [line for line in failure_lines if
line.job_guid == job.guid]
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job.guid}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
for i, failure_line in enumerate(job_failure_lines):
assert failure_line.best_is_verified is False
body = {"best_classification": None if i == 0 else classified_failures[0].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
if i == 0:
assert failure_line.best_classification is None
else:
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified
assert jm.is_fully_verified(job.project_specific_id)
# will assert if we don't have exactly one note for this job, which is
# what we want
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_line_mark_job(test_repository, test_job,
text_log_errors_failure_lines,
classified_failures, client, test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client.force_authenticate(user=test_user)
bug = Bugscache.objects.create(id=1234,
status="NEW",
modified="2014-01-01 00:00:00",
summary="test")
classified_failures[1].bug_number = bug.id
classified_failures[1].save()
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
assert failure_line.best_is_verified is False
assert text_log_error.metadata.best_is_verified is False
body = {"best_classification": classified_failures[1].id}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body)
assert resp.status_code == 200
failure_line.refresh_from_db()
text_log_error.refresh_from_db()
text_log_error.metadata.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert text_log_error.metadata.best_classification == classified_failures[
1]
assert text_log_error.metadata.best_is_verified
assert test_job.is_fully_verified()
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
job_bugs = BugJobMap.objects.filter(job=test_job)
assert job_bugs.count() == 1
assert job_bugs[0].bug_id == bug.id
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [line for line in failure_lines if
line.job_guid == job.guid]
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job.guid}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
JobNote.objects.create(job=job,
failure_classification_id=7,
text="note")
for failure_line in job_failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
assert jm.is_fully_verified(job.project_specific_id)
notes = JobNote.objects.filter(job=job).order_by('-created')
assert notes.count() == 2
assert notes[0].failure_classification.id == 4
assert notes[0].user == test_user
assert notes[0].text == ''
assert notes[1].failure_classification.id == 7
assert not notes[1].user
assert notes[1].text == "note"
def test_detect_intermittents(test_repository, activate_responses, jm,
eleven_jobs_stored, failure_lines,
classified_failures, retriggers):
retrigger = retriggers[0]
test_failure_lines = create_failure_lines(test_repository,
retrigger["job_guid"],
[(test_line, {
"subtest": "subtest2"
}),
(test_line, {
"status": "TIMEOUT"
}),
(test_line, {
"expected": "ERROR"
}),
(test_line, {
"message": "message2"
})])
old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all())
# Poke some internal state so that we only use a single matcher for the test
MatcherManager._matcher_funcs = {}
MatcherManager.register_matcher(PreciseTestMatcher)
MatcherManager._detector_funcs = {}
detector = MatcherManager.register_detector(TestFailureDetector)
call_command('detect_intermittents', test_repository.name,
retrigger['job_guid'])
assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4
matches_seen = set()
failure_ids_seen = old_failure_ids
for item in test_failure_lines:
item.refresh_from_db()
failure_matches = item.matches.all()
assert len(failure_matches) == 1
match = failure_matches[0]
assert match.classified_failure.id not in failure_ids_seen
assert match not in matches_seen
assert match.matcher == detector.db_object
assert match.score == 1
assert item.best_classification == match.classified_failure
assert item.best_is_verified is False
matches_seen.add(match)
failure_ids_seen.add(match.classified_failure.id)
def test_update_failure_line_all_ignore_mark_job(test_job,
mock_autoclassify_jobs_true,
text_log_errors_failure_lines,
classified_failures,
test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job_failure_lines = [line for line in failure_lines if
line.job_guid == test_job.guid]
job_text_log_errors = [error for error in text_log_errors if
error.step.job == test_job]
for error_line, failure_line in zip(job_text_log_errors, job_failure_lines):
error_line.metadata.best_is_verified = False
error_line.metadata.best_classification = None
failure_line.best_is_verified = False
failure_line.best_classification = None
assert JobNote.objects.count() == 0
for error_line, failure_line in zip(job_text_log_errors, job_failure_lines):
assert error_line.metadata.best_is_verified is False
assert failure_line.best_is_verified is False
body = {"best_classification": None}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
error_line.refresh_from_db()
error_line.metadata.refresh_from_db()
failure_line.refresh_from_db()
assert failure_line.best_classification is None
assert failure_line.best_is_verified
assert error_line.metadata.best_classification is None
assert error_line.metadata.best_is_verified
assert test_job.is_fully_verified()
assert JobNote.objects.count() == 1
def test_update_error_partial_ignore_mark_job(test_job,
mock_autoclassify_jobs_true,
text_log_errors_failure_lines,
classified_failures, test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
for i, (error_line,
failure_line) in enumerate(zip(text_log_errors, failure_lines)):
assert error_line.metadata.best_is_verified is False
assert failure_line.best_is_verified is False
body = {
"best_classification":
None if i == 0 else classified_failures[0].id
}
resp = client.put(reverse("text-log-error-detail",
kwargs={"pk": error_line.id}),
body,
format="json")
assert resp.status_code == 200
error_line.metadata.refresh_from_db()
failure_line.refresh_from_db()
if i == 0:
assert error_line.metadata.best_classification is None
assert failure_line.best_classification is None
else:
assert error_line.metadata.best_classification == classified_failures[
0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified
assert test_job.is_fully_verified()
# will assert if we don't have exactly one note for this job, which is
# what we want
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_detect_intermittents(test_job, failure_lines, classified_failures,
retriggered_job):
test_failure_lines = create_failure_lines(retriggered_job,
[(test_line, {
"subtest": "subtest2"
}),
(test_line, {
"status": "TIMEOUT"
}),
(test_line, {
"expected": "ERROR"
}),
(test_line, {
"message": "message2"
})])
old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all())
# Poke some internal state so that we only use a single matcher for the test
MatcherManager._matcher_funcs = {}
MatcherManager.register_matcher(PreciseTestMatcher)
MatcherManager._detector_funcs = {}
detector = MatcherManager.register_detector(TestFailureDetector)
# This allows us to test that the command works and avoids the
# issue that the task is a no-op for !mozilla-inbound
call_command('detect_intermittents', str(test_job.id))
assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4
matches_seen = set()
failure_ids_seen = old_failure_ids
for item in test_failure_lines:
item.refresh_from_db()
failure_matches = item.matches.all()
assert len(failure_matches) == 1
match = failure_matches[0]
assert match.classified_failure.id not in failure_ids_seen
assert match not in matches_seen
assert match.matcher == detector.db_object
assert match.score == 1
assert item.best_classification == match.classified_failure
assert item.best_is_verified is False
matches_seen.add(match)
failure_ids_seen.add(match.classified_failure.id)
def test_update_failure_line_replace(test_repository,
text_log_errors_failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
text_log_errors, failure_lines = text_log_errors_failure_lines
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.failure_line == failure_line
assert error_line.best_classification == classified_failures[0]
assert error_line.best_is_verified is False
body = {
"project": test_repository.name,
"best_classification": classified_failures[1].id
}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
error_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert len(failure_line.classified_failures.all()) == 2
assert error_line.failure_line == failure_line
assert error_line.best_classification == classified_failures[1]
assert error_line.best_is_verified
expected_matcher = Matcher.objects.get(name="ManualDetector")
assert failure_line.matches.get(
classified_failure_id=classified_failures[1].id
).matcher == expected_matcher
assert error_line.matches.get(classified_failure_id=classified_failures[1].
id).matcher == expected_matcher
def test_update_failure_line_mark_job(test_repository, test_job,
text_log_errors_failure_lines,
classified_failures,
test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
classified_failures[1].bug_number = 1234
classified_failures[1].save()
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
assert failure_line.best_is_verified is False
assert text_log_error.metadata.best_is_verified is False
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
text_log_error.refresh_from_db()
text_log_error.metadata.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert text_log_error.metadata.best_classification == classified_failures[1]
assert text_log_error.metadata.best_is_verified
assert test_job.is_fully_verified()
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
job_bugs = BugJobMap.objects.filter(job=test_job)
assert job_bugs.count() == 1
assert job_bugs[0].bug_id == 1234
def test_update_error_bug_change_cf(test_repository,
text_log_errors_failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
assert 78910 not in [item.bug_number for item in classified_failures]
classified_failures[1].bug_number = 78910
classified_failures[1].save()
body = {
"best_classification": classified_failures[0].id,
"bug_number": 78910
}
resp = client.put(reverse("text-log-error-detail",
kwargs={"pk": error_line.id}),
body,
format="json")
assert resp.status_code == 200
classified_failures[1].refresh_from_db()
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_classification.bug_number == 78910
assert failure_line.best_is_verified
assert error_line.metadata.best_classification == classified_failures[1]
assert error_line.metadata.best_is_verified
assert error_line.metadata.best_classification.bug_number == 78910
assert ClassifiedFailure.objects.count() == len(classified_failures) - 1
def test_update_failure_line_mark_job(jm, test_job,
text_log_errors_failure_lines,
classified_failures, test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
classified_failures[1].bug_number = 1234
classified_failures[1].save()
for text_log_error, failure_line in zip(text_log_errors, failure_lines):
assert failure_line.best_is_verified is False
assert text_log_error.best_is_verified is False
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
text_log_error.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert text_log_error.best_classification == classified_failures[1]
assert text_log_error.best_is_verified
assert test_job.is_fully_verified()
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
job_bugs = BugJobMap.objects.filter(job=test_job)
assert job_bugs.count() == 1
assert job_bugs[0].bug_id == 1234
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = jm.get_job(1)[0]
job_failure_lines = [line for line in failure_lines if
line.job_guid == job["job_guid"]]
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
jm.insert_job_note(job["id"], 7, "autoclassifier", "note", autoclassify=True)
for failure_line in job_failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 2
assert notes[0]["failure_classification_id"] == 4
assert notes[0]["who"] == test_user.email
def test_update_failure_line_mark_job_with_auto_note(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = jm.get_job(1)[0]
job_failure_lines = [line for line in failure_lines if
line.job_guid == job["job_guid"]]
bs_artifact = {'type': 'json',
'name': 'Bug suggestions',
'blob': json.dumps([{"search": "TEST-UNEXPECTED-%s %s" %
(line.status.upper(), line.message)}
for line in job_failure_lines]),
'job_guid': job['job_guid']}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
jm.insert_job_note(job["id"], 7, "autoclassifier", "note", autoclassify=True)
for failure_line in job_failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
assert jm.is_fully_verified(job['id'])
notes = jm.get_job_note_list(job['id'])
assert len(notes) == 2
assert notes[0]["failure_classification_id"] == 4
assert notes[0]["who"] == test_user.email
def test_update_error_bug_change_cf(test_repository,
text_log_errors_failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
assert 78910 not in [item.bug_number for item in classified_failures]
classified_failures[1].bug_number = 78910
classified_failures[1].save()
body = {"best_classification": classified_failures[0].id,
"bug_number": 78910}
resp = client.put(
reverse("text-log-error-detail", kwargs={"pk": error_line.id}),
body, format="json")
assert resp.status_code == 200
classified_failures[1].refresh_from_db()
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_classification.bug_number == 78910
assert failure_line.best_is_verified
assert error_line.metadata.best_classification == classified_failures[1]
assert error_line.metadata.best_is_verified
assert error_line.metadata.best_classification.bug_number == 78910
assert ClassifiedFailure.objects.count() == len(classified_failures) - 1
def test_update_failure_line_replace(test_repository,
text_log_errors_failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
text_log_errors, failure_lines = text_log_errors_failure_lines
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
body = {"project": test_repository.name,
"best_classification": classified_failures[1].id}
resp = client.put(
reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
error_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert len(failure_line.classified_failures.all()) == 2
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[1]
assert error_line.metadata.best_is_verified
expected_matcher = Matcher.objects.get(name="ManualDetector")
assert failure_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
assert error_line.matches.get(classified_failure_id=classified_failures[1].id).matcher == expected_matcher
def test_update_failure_line_partial_ignore_mark_job(test_job,
mock_autoclassify_jobs_true,
text_log_errors_failure_lines,
classified_failures,
test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
for i, (error_line, failure_line) in enumerate(zip(text_log_errors, failure_lines)):
assert error_line.metadata.best_is_verified is False
assert failure_line.best_is_verified is False
body = {"best_classification": None if i == 0 else classified_failures[0].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
error_line.refresh_from_db()
error_line.metadata.refresh_from_db()
failure_line.refresh_from_db()
if i == 0:
assert failure_line.best_classification is None
assert error_line.metadata.best_classification is None
else:
assert failure_line.best_classification == classified_failures[0]
assert error_line.metadata.best_classification == classified_failures[0]
assert failure_line.best_is_verified
assert test_job.is_fully_verified()
# will assert if we don't have exactly one note for this job, which is
# what we want
note = JobNote.objects.get(job=test_job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_error_verify_bug(test_repository,
text_log_errors_failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
classified_failures[0].bug_number = 1234
classified_failures[0].save()
body = {"bug_number": classified_failures[0].bug_number}
resp = client.put(
reverse("text-log-error-detail", kwargs={"pk": error_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified
es_line = _TestFailureLine.get(failure_line.id, routing=failure_line.test)
assert es_line.best_classification == classified_failures[0].id
assert es_line.best_is_verified
def test_update_error_verify_bug(test_repository,
text_log_errors_failure_lines,
classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
text_log_errors, failure_lines = text_log_errors_failure_lines
client = APIClient()
client.force_authenticate(user=test_user)
failure_line = failure_lines[0]
error_line = text_log_errors[0]
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified is False
assert error_line.metadata.failure_line == failure_line
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified is False
classified_failures[0].bug_number = 1234
classified_failures[0].save()
body = {"bug_number": classified_failures[0].bug_number}
resp = client.put(
reverse("text-log-error-detail", kwargs={"pk": error_line.id}),
body, format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
error_line.metadata.refresh_from_db()
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified
assert error_line.metadata.best_classification == classified_failures[0]
assert error_line.metadata.best_is_verified
es_line = _TestFailureLine.get(failure_line.id, routing=failure_line.test)
assert es_line.best_classification == classified_failures[0].id
assert es_line.best_is_verified
def test_detect_intermittents(test_repository, activate_responses, jm, eleven_jobs_stored,
failure_lines, classified_failures, retriggers):
retrigger = retriggers[0]
test_failure_lines = create_failure_lines(test_repository,
retrigger["job_guid"],
[(test_line, {"subtest": "subtest2"}),
(test_line, {"status": "TIMEOUT"}),
(test_line, {"expected": "ERROR"}),
(test_line, {"message": "message2"})])
old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all())
# Poke some internal state so that we only use a single matcher for the test
MatcherManager._matcher_funcs = {}
MatcherManager.register_matcher(PreciseTestMatcher)
MatcherManager._detector_funcs = {}
detector = MatcherManager.register_detector(TestFailureDetector)
call_command('detect_intermittents', test_repository.name, retrigger['job_guid'])
assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4
matches_seen = set()
failure_ids_seen = old_failure_ids
for item in test_failure_lines:
item.refresh_from_db()
failure_matches = item.matches.all()
assert len(failure_matches) == 1
match = failure_matches[0]
assert match.classified_failure.id not in failure_ids_seen
assert match not in matches_seen
assert match.matcher == detector.db_object
assert match.score == 1
assert item.best_classification == match.classified_failure
assert item.best_is_verified is False
matches_seen.add(match)
failure_ids_seen.add(match.classified_failure.id)
def test_detect_intermittents(test_job, failure_lines, classified_failures,
retriggered_job):
test_failure_lines = create_failure_lines(retriggered_job,
[(test_line, {"subtest": "subtest2"}),
(test_line, {"status": "TIMEOUT"}),
(test_line, {"expected": "ERROR"}),
(test_line, {"message": "message2"})])
old_failure_ids = set(item.id for item in ClassifiedFailure.objects.all())
# Poke some internal state so that we only use a single matcher for the test
MatcherManager._matcher_funcs = {}
MatcherManager.register_matcher(PreciseTestMatcher)
MatcherManager._detector_funcs = {}
detector = MatcherManager.register_detector(TestFailureDetector)
# This allows us to test that the command works and avoids the
# issue that the task is a no-op for !mozilla-inbound
call_command('detect_intermittents', str(test_job.id))
assert ClassifiedFailure.objects.count() == len(old_failure_ids) + 4
matches_seen = set()
failure_ids_seen = old_failure_ids
for item in test_failure_lines:
item.refresh_from_db()
failure_matches = item.matches.all()
assert len(failure_matches) == 1
match = failure_matches[0]
assert match.classified_failure.id not in failure_ids_seen
assert match not in matches_seen
assert match.matcher == detector.db_object
assert match.score == 1
assert item.best_classification == match.classified_failure
assert item.best_is_verified is False
matches_seen.add(match)
failure_ids_seen.add(match.classified_failure.id)
def test_update_failure_line_mark_job_with_auto_note(test_job,
mock_autoclassify_jobs_true,
test_repository,
text_log_errors_failure_lines,
classified_failures,
test_user):
text_log_errors, failure_lines = text_log_errors_failure_lines
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
JobNote.objects.create(job=test_job,
failure_classification_id=7,
text="note")
for failure_line in failure_lines:
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail", kwargs={"pk": failure_line.id}),
body, format="json")
assert resp.status_code == 200
assert test_job.is_fully_verified()
notes = JobNote.objects.filter(job=test_job).order_by('-created')
assert notes.count() == 2
assert notes[0].failure_classification.id == 4
assert notes[0].user == test_user
assert notes[0].text == ''
assert notes[1].failure_classification.id == 7
assert not notes[1].user
assert notes[1].text == "note"
def test_matcher(request):
from treeherder.autoclassify import detectors
from treeherder.model.models import MatcherManager
class TreeherderUnitTestDetector(detectors.Detector):
def __call__(self, failure_lines):
return True
MatcherManager._detector_funcs = {}
MatcherManager._matcher_funcs = {}
test_matcher = MatcherManager.register_detector(TreeherderUnitTestDetector)
def finalize():
MatcherManager._detector_funcs = {}
MatcherManager._matcher_funcs = {}
request.addfinalizer(finalize)
return test_matcher
def test_matcher(request):
from treeherder.autoclassify import detectors
from treeherder.model.models import MatcherManager
class TreeherderUnitTestDetector(detectors.Detector):
def __call__(self, failure_lines):
return True
MatcherManager._detector_funcs = {}
MatcherManager._matcher_funcs = {}
test_matcher = MatcherManager.register_detector(TreeherderUnitTestDetector)
def finalize():
MatcherManager._detector_funcs = {}
MatcherManager._matcher_funcs = {}
request.addfinalizer(finalize)
return test_matcher
def register_matchers(*args):
MatcherManager._matcher_funcs = {}
for item in args:
MatcherManager.register_matcher(item)
def register():
for obj in [PreciseTestMatcher]:
MatcherManager.register_matcher(obj)
def register_detectors(*args):
MatcherManager._detector_funcs = {}
for item in args:
MatcherManager.register_detector(item)
def register():
"""Register matchers enabled in settings.AUTOCLASSIFY_MATCHERS."""
for obj_name in settings.AUTOCLASSIFY_MATCHERS:
obj = globals()[obj_name]
MatcherManager.register_matcher(obj)
def register_matchers(*args):
MatcherManager._matcher_funcs = {}
for item in args:
MatcherManager.register_matcher(item)
def register():
"""Register matchers enabled in settings.AUTOCLASSIFY_MATCHERS."""
for obj_name in settings.AUTOCLASSIFY_MATCHERS:
obj = globals()[obj_name]
MatcherManager.register_matcher(obj)
def register():
for obj in [ManualDetector, TestFailureDetector]:
MatcherManager.register_detector(obj)
def register():
for obj_name in settings.AUTOCLASSIFY_MATCHERS:
obj = globals()[obj_name]
MatcherManager.register_matcher(obj)
def test_update_failure_line_partial_ignore_mark_job(
eleven_jobs_stored, mock_autoclassify_jobs_true, jm, failure_lines,
classified_failures, test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [
line for line in failure_lines if line.job_guid == job.guid
]
bs_artifact = {
'type':
'json',
'name':
'Bug suggestions',
'blob':
json.dumps([{
"search":
"TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)
} for line in job_failure_lines]),
'job_guid':
job.guid
}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
for i, failure_line in enumerate(job_failure_lines):
assert failure_line.best_is_verified is False
body = {
"best_classification":
None if i == 0 else classified_failures[0].id
}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
if i == 0:
assert failure_line.best_classification is None
else:
assert failure_line.best_classification == classified_failures[0]
assert failure_line.best_is_verified
assert jm.is_fully_verified(job.project_specific_id)
# will assert if we don't have exactly one note for this job, which is
# what we want
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_lines(eleven_jobs_stored, mock_autoclassify_jobs_true,
jm, test_repository, failure_lines,
classified_failures, test_user):
jobs = (Job.objects.get(project_specific_id=1),
Job.objects.get(project_specific_id=2))
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
create_failure_lines(test_repository, jobs[1].guid,
[(test_line, {}), (test_line, {
"subtest": "subtest2"
})])
failure_lines = FailureLine.objects.filter(
job_guid__in=[job.guid for job in jobs]).all()
for job in jobs:
job_failure_lines = FailureLine.objects.filter(job_guid=job.guid).all()
bs_artifact = {
'type':
'json',
'name':
'Bug suggestions',
'blob':
json.dumps([{
"search":
"TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)
} for line in job_failure_lines]),
'job_guid':
job.guid
}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
body = [{
"id": failure_line.id,
"best_classification": classified_failures[1].id
} for failure_line in failure_lines]
for failure_line in failure_lines:
assert failure_line.best_is_verified is False
resp = client.put(reverse("failure-line-list"), body, format="json")
assert resp.status_code == 200
for failure_line in failure_lines:
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
for job in jobs:
assert jm.is_fully_verified(job.project_specific_id)
# will assert if we don't have exactly one job, which is what we want
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
def test_update_failure_line_mark_job(eleven_jobs_stored,
mock_autoclassify_jobs_true, jm,
failure_lines, classified_failures,
test_user):
MatcherManager.register_detector(ManualDetector)
client = APIClient()
client.force_authenticate(user=test_user)
job = Job.objects.get(project_specific_id=1)
job_failure_lines = [
line for line in failure_lines if line.job_guid == job.guid
]
classified_failures[1].bug_number = 1234
classified_failures[1].save()
bs_artifact = {
'type':
'json',
'name':
'Bug suggestions',
'blob':
json.dumps([{
"search":
"TEST-UNEXPECTED-%s %s" % (line.status.upper(), line.message)
} for line in job_failure_lines]),
'job_guid':
job.guid
}
with ArtifactsModel(jm.project) as artifacts_model:
artifacts_model.load_job_artifacts([bs_artifact])
for failure_line in job_failure_lines:
assert failure_line.best_is_verified is False
body = {"best_classification": classified_failures[1].id}
resp = client.put(reverse("failure-line-detail",
kwargs={"pk": failure_line.id}),
body,
format="json")
assert resp.status_code == 200
failure_line.refresh_from_db()
assert failure_line.best_classification == classified_failures[1]
assert failure_line.best_is_verified
assert jm.is_fully_verified(job.project_specific_id)
# should only be one, will assert if that isn't the case
note = JobNote.objects.get(job=job)
assert note.failure_classification.id == 4
assert note.user == test_user
job_bugs = BugJobMap.objects.filter(job=job)
assert job_bugs.count() == 1
assert job_bugs[0].bug_id == 1234
def register_detectors(*args):
MatcherManager._detector_funcs = {}
for item in args:
MatcherManager.register_detector(item)
def register():
for obj in [ManualDetector, TestFailureDetector]:
MatcherManager.register_detector(obj)
def register():
for obj in [PreciseTestMatcher, CrashSignatureMatcher]:
MatcherManager.register_matcher(obj)
