def test_classify_almost_bad_push(monkeypatch, test_selection_data,
likely_regressions, are_cross_config):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
are_cross_config,
)
assert push.classify() == (
PushStatus.UNKNOWN,
Regressions(
real={},
intermittent={},
unknown={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
),
)
def test_classify_almost_good_push(monkeypatch, test_selection_data,
are_cross_config):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
generate_mocks(
monkeypatch,
push,
test_selection_data,
set(),
are_cross_config,
)
assert push.classify(
unknown_from_regressions=False,
consistent_failures_counts=None,
consider_children_pushes_configs=False,
) == (
PushStatus.UNKNOWN,
Regressions(
real={},
intermittent={},
unknown={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
),
)
def test_classify_good_push_only_intermittent_failures(monkeypatch):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
test_selection_data = {"groups": {"group1": 0.7, "group2": 0.3}}
likely_regressions = {"group3", "group4"}
are_cross_config = [False for i in range(0, len(GROUP_SUMMARIES_DEFAULT))]
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
are_cross_config,
)
assert push.classify() == (
PushStatus.GOOD,
Regressions(
real={},
# All groups aren't cross config failures and were either selected by bugbug
# with low confidence or not at all (no confidence)
intermittent={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
unknown={},
),
)
def test_classify(monkeypatch, classify_regressions_return_value,
expected_result):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
def mock_return(self, *args, **kwargs):
return classify_regressions_return_value
monkeypatch.setattr(Push, "classify_regressions", mock_return)
assert push.classify()[0] == expected_result
def test_classify_bad_push_some_real_failures(monkeypatch):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
test_selection_data = {
"groups": {
"group1": 0.99,
"group2": 0.95,
"group3": 0.91
}
}
likely_regressions = {"group1", "group2", "group3"}
are_cross_config = [
False if i % 2 else True
for i in range(0, len(GROUP_SUMMARIES_DEFAULT))
]
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
set(),
are_cross_config,
)
assert push.classify(
unknown_from_regressions=False, consider_children_pushes_configs=False
) == (
PushStatus.BAD,
Regressions(
# group1 & group3 were both selected by bugbug with high confidence, likely to regress
# and are cross config failures
real={
"group1": make_tasks("group1"),
"group3": make_tasks("group3")
},
# group4 isn't a cross config failure and was not selected by bugbug (no confidence)
intermittent={"group4": make_tasks("group4")},
# group2 isn't a cross config failure but was selected with high confidence by bugbug
# group5 is a cross config failure but was not selected by bugbug nor likely to regress
unknown={
"group2": make_tasks("group2"),
"group5": make_tasks("group5")
},
),
ToRetriggerOrBackfill(
real_retrigger={"group2": make_tasks("group2")},
intermittent_retrigger={"group5": make_tasks("group5")},
backfill={},
),
)
def test_classify(monkeypatch, classify_regressions_return_value,
expected_result):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
def mock_return(self, *args, **kwargs):
return classify_regressions_return_value, ToRetriggerOrBackfill(
real_retrigger={},
intermittent_retrigger={},
backfill={},
)
monkeypatch.setattr(Push, "classify_regressions", mock_return)
assert push.classify()[0] == expected_result
def test_classify_almost_bad_push(monkeypatch, test_selection_data,
likely_regressions, are_cross_config,
to_retrigger):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
generate_mocks(
monkeypatch,
push,
test_selection_data,
likely_regressions,
set(),
are_cross_config,
)
to_retrigger_or_backill = {
"real_retrigger": {},
"intermittent_retrigger": {},
"backfill": {},
}
for key, groups in to_retrigger.items():
to_retrigger[key] = {group: make_tasks(group) for group in groups}
to_retrigger_or_backill.update(to_retrigger)
assert push.classify(
unknown_from_regressions=False,
consistent_failures_counts=None,
consider_children_pushes_configs=False,
) == (
PushStatus.UNKNOWN,
Regressions(
real={},
intermittent={},
unknown={
"group1": make_tasks("group1"),
"group2": make_tasks("group2"),
"group3": make_tasks("group3"),
"group4": make_tasks("group4"),
"group5": make_tasks("group5"),
},
),
ToRetriggerOrBackfill(**to_retrigger_or_backill),
)
def run_combinations_for_push(push):
push = Push(push["rev"], branch=push["branch"])
push_dir = f"{BASE_OUTPUT_DIR}/{push.id}"
if not os.path.exists(push_dir):
os.makedirs(push_dir)
csv_rows = []
for parameters in PARAMETERS_COMBINATIONS:
run_id = uuid.uuid4()
start = time.time()
try:
classification, regressions = push.classify(**parameters)
end = time.time()
# Only save results to a JSON file if the execution was successful
classification_name = classification.name
create_json_file(push, run_id, classification_name, regressions)
except Exception as e:
end = time.time()
classification_name = "SYSTEM_ERROR"
logger.error(
f"An error occurred during the classification of push {push.push_uuid}: {e}"
)
csv_rows.append(
{
"run_uuid": run_id,
"push_uuid": push.push_uuid,
**parameters,
"classification": classification_name,
"time_spent": round(end - start, 3),
"now": datetime.datetime.now(),
}
)
return csv_rows
def test_classify_retrigger_unknown_tasks(responses, monkeypatch,
classify_regressions_return_value,
expected_result):
rev = "a" * 40
branch = "autoland"
push = Push(rev, branch)
def mock_return(self, *args, **kwargs):
return classify_regressions_return_value
monkeypatch.setattr(Push, "classify_regressions", mock_return)
responses.add(
responses.PUT,
"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/JHT2zBEmRvKeXTAs0_PXYQ",
json={
"payload": {},
"tags": {
"retrigger": "true",
"label": "test_retrigger"
}
},
status=200,
)
responses.add(
responses.GET,
"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/NjJqN07WQ9Cs6HvVLUJXnw",
json={
"payload": {},
"tags": {
"retrigger": "true",
"label": "test-taskNjJqN07WQ9Cs6HvVLUJXnw"
},
},
status=200,
)
create_new_task_url_matcher = re.compile(
r"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/*")
responses.add(
responses.PUT,
create_new_task_url_matcher,
json={
"payload": {},
"tags": {
"retrigger": "true",
"label": "test-taskNjJqN07WQ9Cs6HvVLUJXnw"
},
},
status=200,
)
_, regressions = push.classify()
for _, tasks in regressions.unknown.items():
retrigger(tasks=tasks, repeat_retrigger=1)
# verify last call was to create a new task i.e a retrigger
assert (len(responses.calls) != 0
and responses.calls[-1].request.method == "PUT") == expected_result
assert (len(responses.calls) != 0 and bool(
create_new_task_url_matcher.match(
responses.calls[-1].request.url))) == expected_result
if len(responses.calls) < 0:
# assert that the new task created has the same label as the original task
last_request_body = json.loads(responses.calls[-1].request.body)
assert (last_request_body["tags"]["label"] ==
"test-taskNjJqN07WQ9Cs6HvVLUJXnw") == expected_result
