def test_classify_good_push_only_intermittent_failures(monkeypatch): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) test_selection_data = {"groups": {"group1": 0.7, "group2": 0.3}} likely_regressions = {"group3", "group4"} are_cross_config = [False for i in range(0, len(GROUP_SUMMARIES_DEFAULT))] generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, are_cross_config, ) assert push.classify() == ( PushStatus.GOOD, Regressions( real={}, # All groups aren't cross config failures and were either selected by bugbug # with low confidence or not at all (no confidence) intermittent={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, unknown={}, ), )
def test_classify_almost_bad_push(monkeypatch, test_selection_data, likely_regressions, are_cross_config): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, are_cross_config, ) assert push.classify() == ( PushStatus.UNKNOWN, Regressions( real={}, intermittent={}, unknown={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, ), )
def test_classify_almost_good_push(monkeypatch, test_selection_data, are_cross_config): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) generate_mocks( monkeypatch, push, test_selection_data, set(), are_cross_config, ) assert push.classify( unknown_from_regressions=False, consistent_failures_counts=None, consider_children_pushes_configs=False, ) == ( PushStatus.UNKNOWN, Regressions( real={}, intermittent={}, unknown={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, ), )
def test_classify_bad_push_some_real_failures(monkeypatch): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) test_selection_data = { "groups": { "group1": 0.99, "group2": 0.95, "group3": 0.91 } } likely_regressions = {"group1", "group2", "group3"} are_cross_config = [ False if i % 2 else True for i in range(0, len(GROUP_SUMMARIES_DEFAULT)) ] generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, set(), are_cross_config, ) assert push.classify( unknown_from_regressions=False, consider_children_pushes_configs=False ) == ( PushStatus.BAD, Regressions( # group1 & group3 were both selected by bugbug with high confidence, likely to regress # and are cross config failures real={ "group1": make_tasks("group1"), "group3": make_tasks("group3") }, # group4 isn't a cross config failure and was not selected by bugbug (no confidence) intermittent={"group4": make_tasks("group4")}, # group2 isn't a cross config failure but was selected with high confidence by bugbug # group5 is a cross config failure but was not selected by bugbug nor likely to regress unknown={ "group2": make_tasks("group2"), "group5": make_tasks("group5") }, ), ToRetriggerOrBackfill( real_retrigger={"group2": make_tasks("group2")}, intermittent_retrigger={"group5": make_tasks("group5")}, backfill={}, ), )
def test_classify_almost_bad_push(monkeypatch, test_selection_data, likely_regressions, are_cross_config, to_retrigger): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, set(), are_cross_config, ) to_retrigger_or_backill = { "real_retrigger": {}, "intermittent_retrigger": {}, "backfill": {}, } for key, groups in to_retrigger.items(): to_retrigger[key] = {group: make_tasks(group) for group in groups} to_retrigger_or_backill.update(to_retrigger) assert push.classify( unknown_from_regressions=False, consistent_failures_counts=None, consider_children_pushes_configs=False, ) == ( PushStatus.UNKNOWN, Regressions( real={}, intermittent={}, unknown={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, ), ToRetriggerOrBackfill(**to_retrigger_or_backill), )
def test_classification_evolution(create_push, responses, previous, current, email_content): # Setup dummy regressions with real failures regressions = Regressions( real={ "group1": [TestTask(id="taskIdXXX", label="random-test-task")], "group2": [], }, intermittent={}, unknown={}, ) # Create a random push push = create_push() # Mock Taskcluster email notification service responses.add( responses.POST, "https://community-tc.services.mozilla.com/api/notify/v1/email", json={}, status=200, ) # Mock Taskcluster Matrix notification service responses.add( responses.POST, "https://community-tc.services.mozilla.com/api/notify/v1/matrix", json={}, status=200, ) # Run the notification code from mozci push classify cmd = ClassifyCommand() cmd.name = "classify" cmd.branch = "unittest" cmd.send_notifications( emails=["*****@*****.**"], matrix_room="!tEsTmAtRIxRooM:mozilla.org", push=push, previous=previous, current=current, regressions=regressions, ) if email_content: # Check an email and a matrix notification were correctly sent assert len(responses.calls) == 2 email_call = responses.calls[0] assert json.loads(email_call.request.body) == { "address": "*****@*****.**", "subject": "Mozci | Push status evolution 1 rev1", "content": email_content, } matrix_call = responses.calls[1] assert json.loads(matrix_call.request.body) == { "roomId": "!tEsTmAtRIxRooM:mozilla.org", "body": email_content, "formattedBody": markdown2.markdown(email_content), "format": "org.matrix.custom.html", "msgtype": "m.text", } else: # Check no email was sent assert len(responses.calls) == 0
data = push.get_test_selection_data() assert data == SCHEDULES_EXTRACT assert len(responses.calls) == 3 assert [(call.request.method, call.request.url) for call in responses.calls] == [ ("GET", task_url), ("GET", cache_url), ("GET", url), ] @pytest.mark.parametrize( "classify_regressions_return_value, expected_result", [ (Regressions(real={"group1": []}, intermittent={}, unknown={}), PushStatus.BAD), ( Regressions( real={"group1": []}, intermittent={"group2": []}, unknown={}), PushStatus.BAD, ), ( Regressions( real={"group1": []}, intermittent={}, unknown={"group2": []}), PushStatus.BAD, ), (Regressions(real={}, intermittent={}, unknown={}), PushStatus.GOOD), ( Regressions(real={}, intermittent={"group1": []}, unknown={}), PushStatus.GOOD, ),
data = push.get_test_selection_data() assert data == SCHEDULES_EXTRACT assert len(responses.calls) == 3 assert [(call.request.method, call.request.url) for call in responses.calls] == [ ("GET", task_url), ("GET", cache_url), ("GET", url), ] @pytest.mark.parametrize( "classify_regressions_return_value, expected_result", [ (Regressions(real={"group1": []}, intermittent={}, unknown={}), PushStatus.BAD), ( Regressions( real={"group1": []}, intermittent={"group2": []}, unknown={}), PushStatus.BAD, ), ( Regressions( real={"group1": []}, intermittent={}, unknown={"group2": []}), PushStatus.BAD, ), (Regressions(real={}, intermittent={}, unknown={}), PushStatus.GOOD), ( Regressions(real={}, intermittent={"group1": []}, unknown={}), PushStatus.GOOD, ),