def test_classify_good_push_only_intermittent_failures(monkeypatch):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
test_selection_data = {"groups": {"group1": 0.7, "group2": 0.3}}
likely_regressions = {"group3", "group4"}
are_cross_config = [False for i in range(0, len(GROUP_SUMMARIES_DEFAULT))]
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
are_cross_config,
)
assert push.classify() == (
PushStatus.GOOD,
Regressions(
real={},
# All groups aren't cross config failures and were either selected by bugbug
# with low confidence or not at all (no confidence)
intermittent={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
unknown={},
),
)
def test_classify_almost_bad_push(monkeypatch, test_selection_data,
likely_regressions, are_cross_config):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
are_cross_config,
)
assert push.classify() == (
PushStatus.UNKNOWN,
Regressions(
real={},
intermittent={},
unknown={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
),
)
def test_classify_almost_good_push(monkeypatch, test_selection_data,
are_cross_config):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
generate_mocks(
monkeypatch,
push,
test_selection_data,
set(),
are_cross_config,
)
assert push.classify(
unknown_from_regressions=False,
consistent_failures_counts=None,
consider_children_pushes_configs=False,
) == (
PushStatus.UNKNOWN,
Regressions(
real={},
intermittent={},
unknown={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
),
)
def test_classify_bad_push_some_real_failures(monkeypatch):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
test_selection_data = {
"groups": {
"group1": 0.99,
"group2": 0.95,
"group3": 0.91
}
}
likely_regressions = {"group1", "group2", "group3"}
are_cross_config = [
False if i % 2 else True
for i in range(0, len(GROUP_SUMMARIES_DEFAULT))
]
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
set(),
are_cross_config,
)
assert push.classify(
unknown_from_regressions=False, consider_children_pushes_configs=False
) == (
PushStatus.BAD,
Regressions(
# group1 & group3 were both selected by bugbug with high confidence, likely to regress
# and are cross config failures
real={
"group1": make_tasks("group1"),
"group3": make_tasks("group3")
},
# group4 isn't a cross config failure and was not selected by bugbug (no confidence)
intermittent={"group4": make_tasks("group4")},
# group2 isn't a cross config failure but was selected with high confidence by bugbug
# group5 is a cross config failure but was not selected by bugbug nor likely to regress
unknown={
"group2": make_tasks("group2"),
"group5": make_tasks("group5")
},
),
ToRetriggerOrBackfill(
real_retrigger={"group2": make_tasks("group2")},
intermittent_retrigger={"group5": make_tasks("group5")},
backfill={},
),
)
def test_classify_almost_bad_push(monkeypatch, test_selection_data,
likely_regressions, are_cross_config,
to_retrigger):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
set(),
are_cross_config,
)
to_retrigger_or_backill = {
"real_retrigger": {},
"intermittent_retrigger": {},
"backfill": {},
}
for key, groups in to_retrigger.items():
to_retrigger[key] = {group: make_tasks(group) for group in groups}
to_retrigger_or_backill.update(to_retrigger)
assert push.classify(
unknown_from_regressions=False,
consistent_failures_counts=None,
consider_children_pushes_configs=False,
) == (
PushStatus.UNKNOWN,
Regressions(
real={},
intermittent={},
unknown={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
),
ToRetriggerOrBackfill(**to_retrigger_or_backill),
)
def test_classification_evolution(create_push, responses, previous, current,
email_content):
# Setup dummy regressions with real failures
regressions = Regressions(
real={
"group1": [TestTask(id="taskIdXXX", label="random-test-task")],
"group2": [],
},
intermittent={},
unknown={},
)
# Create a random push
push = create_push()
# Mock Taskcluster email notification service
responses.add(
responses.POST,
"https://community-tc.services.mozilla.com/api/notify/v1/email",
json={},
status=200,
)
# Mock Taskcluster Matrix notification service
responses.add(
responses.POST,
"https://community-tc.services.mozilla.com/api/notify/v1/matrix",
json={},
status=200,
)
# Run the notification code from mozci push classify
cmd = ClassifyCommand()
cmd.name = "classify"
cmd.branch = "unittest"
cmd.send_notifications(
emails=["*****@*****.**"],
matrix_room="!tEsTmAtRIxRooM:mozilla.org",
push=push,
previous=previous,
current=current,
regressions=regressions,
)
if email_content:
# Check an email and a matrix notification were correctly sent
assert len(responses.calls) == 2
email_call = responses.calls[0]
assert json.loads(email_call.request.body) == {
"address": "*****@*****.**",
"subject": "Mozci | Push status evolution 1 rev1",
"content": email_content,
}
matrix_call = responses.calls[1]
assert json.loads(matrix_call.request.body) == {
"roomId": "!tEsTmAtRIxRooM:mozilla.org",
"body": email_content,
"formattedBody": markdown2.markdown(email_content),
"format": "org.matrix.custom.html",
"msgtype": "m.text",
}
else:
# Check no email was sent
assert len(responses.calls) == 0
data = push.get_test_selection_data()
assert data == SCHEDULES_EXTRACT
assert len(responses.calls) == 3
assert [(call.request.method, call.request.url)
for call in responses.calls] == [
("GET", task_url),
("GET", cache_url),
("GET", url),
]
@pytest.mark.parametrize(
"classify_regressions_return_value, expected_result",
[
(Regressions(real={"group1": []}, intermittent={},
unknown={}), PushStatus.BAD),
(
Regressions(
real={"group1": []}, intermittent={"group2": []}, unknown={}),
PushStatus.BAD,
),
(
Regressions(
real={"group1": []}, intermittent={}, unknown={"group2": []}),
PushStatus.BAD,
),
(Regressions(real={}, intermittent={}, unknown={}), PushStatus.GOOD),
(
Regressions(real={}, intermittent={"group1": []}, unknown={}),
PushStatus.GOOD,
),
data = push.get_test_selection_data()
assert data == SCHEDULES_EXTRACT
assert len(responses.calls) == 3
assert [(call.request.method, call.request.url)
for call in responses.calls] == [
("GET", task_url),
("GET", cache_url),
("GET", url),
]
@pytest.mark.parametrize(
"classify_regressions_return_value, expected_result",
[
(Regressions(real={"group1": []}, intermittent={},
unknown={}), PushStatus.BAD),
(
Regressions(
real={"group1": []}, intermittent={"group2": []}, unknown={}),
PushStatus.BAD,
),
(
Regressions(
real={"group1": []}, intermittent={}, unknown={"group2": []}),
PushStatus.BAD,
),
(Regressions(real={}, intermittent={}, unknown={}), PushStatus.GOOD),
(
Regressions(real={}, intermittent={"group1": []}, unknown={}),
PushStatus.GOOD,
),
