def test_classify_almost_bad_push(monkeypatch, test_selection_data, likely_regressions, are_cross_config): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, are_cross_config, ) assert push.classify() == ( PushStatus.UNKNOWN, Regressions( real={}, intermittent={}, unknown={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, ), )
def test_classify_almost_good_push(monkeypatch, test_selection_data, are_cross_config): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) generate_mocks( monkeypatch, push, test_selection_data, set(), are_cross_config, ) assert push.classify( unknown_from_regressions=False, consistent_failures_counts=None, consider_children_pushes_configs=False, ) == ( PushStatus.UNKNOWN, Regressions( real={}, intermittent={}, unknown={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, ), )
def test_classify_good_push_only_intermittent_failures(monkeypatch): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) test_selection_data = {"groups": {"group1": 0.7, "group2": 0.3}} likely_regressions = {"group3", "group4"} are_cross_config = [False for i in range(0, len(GROUP_SUMMARIES_DEFAULT))] generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, are_cross_config, ) assert push.classify() == ( PushStatus.GOOD, Regressions( real={}, # All groups aren't cross config failures and were either selected by bugbug # with low confidence or not at all (no confidence) intermittent={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, unknown={}, ), )
def test_classify(monkeypatch, classify_regressions_return_value, expected_result): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) def mock_return(self, *args, **kwargs): return classify_regressions_return_value monkeypatch.setattr(Push, "classify_regressions", mock_return) assert push.classify()[0] == expected_result
def test_classify_bad_push_some_real_failures(monkeypatch): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) test_selection_data = { "groups": { "group1": 0.99, "group2": 0.95, "group3": 0.91 } } likely_regressions = {"group1", "group2", "group3"} are_cross_config = [ False if i % 2 else True for i in range(0, len(GROUP_SUMMARIES_DEFAULT)) ] generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, set(), are_cross_config, ) assert push.classify( unknown_from_regressions=False, consider_children_pushes_configs=False ) == ( PushStatus.BAD, Regressions( # group1 & group3 were both selected by bugbug with high confidence, likely to regress # and are cross config failures real={ "group1": make_tasks("group1"), "group3": make_tasks("group3") }, # group4 isn't a cross config failure and was not selected by bugbug (no confidence) intermittent={"group4": make_tasks("group4")}, # group2 isn't a cross config failure but was selected with high confidence by bugbug # group5 is a cross config failure but was not selected by bugbug nor likely to regress unknown={ "group2": make_tasks("group2"), "group5": make_tasks("group5") }, ), ToRetriggerOrBackfill( real_retrigger={"group2": make_tasks("group2")}, intermittent_retrigger={"group5": make_tasks("group5")}, backfill={}, ), )
def test_classify(monkeypatch, classify_regressions_return_value, expected_result): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) def mock_return(self, *args, **kwargs): return classify_regressions_return_value, ToRetriggerOrBackfill( real_retrigger={}, intermittent_retrigger={}, backfill={}, ) monkeypatch.setattr(Push, "classify_regressions", mock_return) assert push.classify()[0] == expected_result
def test_classify_almost_bad_push(monkeypatch, test_selection_data, likely_regressions, are_cross_config, to_retrigger): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) generate_mocks( monkeypatch, push, test_selection_data, likely_regressions, set(), are_cross_config, ) to_retrigger_or_backill = { "real_retrigger": {}, "intermittent_retrigger": {}, "backfill": {}, } for key, groups in to_retrigger.items(): to_retrigger[key] = {group: make_tasks(group) for group in groups} to_retrigger_or_backill.update(to_retrigger) assert push.classify( unknown_from_regressions=False, consistent_failures_counts=None, consider_children_pushes_configs=False, ) == ( PushStatus.UNKNOWN, Regressions( real={}, intermittent={}, unknown={ "group1": make_tasks("group1"), "group2": make_tasks("group2"), "group3": make_tasks("group3"), "group4": make_tasks("group4"), "group5": make_tasks("group5"), }, ), ToRetriggerOrBackfill(**to_retrigger_or_backill), )
def run_combinations_for_push(push): push = Push(push["rev"], branch=push["branch"]) push_dir = f"{BASE_OUTPUT_DIR}/{push.id}" if not os.path.exists(push_dir): os.makedirs(push_dir) csv_rows = [] for parameters in PARAMETERS_COMBINATIONS: run_id = uuid.uuid4() start = time.time() try: classification, regressions = push.classify(**parameters) end = time.time() # Only save results to a JSON file if the execution was successful classification_name = classification.name create_json_file(push, run_id, classification_name, regressions) except Exception as e: end = time.time() classification_name = "SYSTEM_ERROR" logger.error( f"An error occurred during the classification of push {push.push_uuid}: {e}" ) csv_rows.append( { "run_uuid": run_id, "push_uuid": push.push_uuid, **parameters, "classification": classification_name, "time_spent": round(end - start, 3), "now": datetime.datetime.now(), } ) return csv_rows
def test_classify_retrigger_unknown_tasks(responses, monkeypatch, classify_regressions_return_value, expected_result): rev = "a" * 40 branch = "autoland" push = Push(rev, branch) def mock_return(self, *args, **kwargs): return classify_regressions_return_value monkeypatch.setattr(Push, "classify_regressions", mock_return) responses.add( responses.PUT, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/JHT2zBEmRvKeXTAs0_PXYQ", json={ "payload": {}, "tags": { "retrigger": "true", "label": "test_retrigger" } }, status=200, ) responses.add( responses.GET, "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/NjJqN07WQ9Cs6HvVLUJXnw", json={ "payload": {}, "tags": { "retrigger": "true", "label": "test-taskNjJqN07WQ9Cs6HvVLUJXnw" }, }, status=200, ) create_new_task_url_matcher = re.compile( r"https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/*") responses.add( responses.PUT, create_new_task_url_matcher, json={ "payload": {}, "tags": { "retrigger": "true", "label": "test-taskNjJqN07WQ9Cs6HvVLUJXnw" }, }, status=200, ) _, regressions = push.classify() for _, tasks in regressions.unknown.items(): retrigger(tasks=tasks, repeat_retrigger=1) # verify last call was to create a new task i.e a retrigger assert (len(responses.calls) != 0 and responses.calls[-1].request.method == "PUT") == expected_result assert (len(responses.calls) != 0 and bool( create_new_task_url_matcher.match( responses.calls[-1].request.url))) == expected_result if len(responses.calls) < 0: # assert that the new task created has the same label as the original task last_request_body = json.loads(responses.calls[-1].request.body) assert (last_request_body["tags"]["label"] == "test-taskNjJqN07WQ9Cs6HvVLUJXnw") == expected_result