def test_group_translation(responses): branch = "integration/autoland", rev = "abcdef" query = "/push/{}/{}/schedules".format(branch, rev) url = BUGBUG_BASE_URL + query responses.add( responses.GET, url, json={ "groups": { "dom/indexedDB": 1, "testing/web-platform/tests/IndexedDB": 1, "testing/web-platform/mozilla/tests/IndexedDB": 1, } }, status=200, ) assert len(push_schedules) == 0 data = push_schedules(branch, rev) print(data) assert sorted(data["groups"]) == [ "/IndexedDB", "/_mozilla/IndexedDB", "dom/indexedDB", ] assert len(push_schedules) == 1 # Value is memoized. responses.reset() push_schedules(branch, rev) assert len(push_schedules) == 1
def get_manifests(self, suite, mozinfo): manifests = super(BugbugLoader, self).get_manifests(suite, mozinfo) # Don't prune any manifests if we're on a backstop push or there was a timeout. if self.params["backstop"] or self.timedout: return manifests try: data = push_schedules(self.params["project"], self.params["head_rev"]) except BugbugTimeoutException: logger.warning( "Timed out waiting for bugbug, loading all test manifests.") self.timedout = True return self.get_manifests(suite, mozinfo) bugbug_manifests = { m for m, c in data.get("groups", {}).items() if c >= self.CONFIDENCE_THRESHOLD } manifests["active"] = list(set(manifests["active"]) & bugbug_manifests) manifests["skipped"] = list( set(manifests["skipped"]) & bugbug_manifests) return manifests
def should_remove_task(self, task, params, importance): if params['project'] not in ("autoland", "try"): return False branch = urlsplit(params['head_repository']).path.strip('/') rev = params['head_rev'] if self.timedout: return seta.is_low_value_task(task.label, params['project']) try: data = push_schedules(branch, rev) except BugbugTimeoutException: if not self.fallback: raise self.timedout = True return self.should_remove_task(task, params, importance) key = "reduced_tasks" if self.use_reduced_tasks else "tasks" tasks = set( task for task, confidence in data.get(key, {}).items() if confidence >= self.confidence_threshold ) test_manifests = task.attributes.get('test_manifests') if test_manifests is None or self.use_reduced_tasks: if data.get("known_tasks") and task.label not in data["known_tasks"]: return False if task.label not in tasks: return True return False # If a task contains more than one group, use the max confidence. groups = data.get("groups", {}) confidences = [c for g, c in groups.items() if g in test_manifests] if not confidences or max(confidences) < self.confidence_threshold: return True # Store group importance so future optimizers can access it. for manifest in test_manifests: if manifest not in groups: continue confidence = groups[manifest] if confidence >= CT_HIGH: importance[manifest] = 'high' elif confidence >= CT_MEDIUM: importance[manifest] = 'medium' elif confidence >= CT_LOW: importance[manifest] = 'low' else: importance[manifest] = 'lowest' return False
def get_manifests(self, suite, mozinfo): manifests = super(BugbugLoader, self).get_manifests(suite, mozinfo) data = push_schedules(self.params['project'], self.params['head_rev']) bugbug_manifests = { m for m, c in data.get('groups', {}).items() if c >= self.CONFIDENCE_THRESHOLD } manifests['active'] = list(set(manifests['active']) & bugbug_manifests) return manifests
def test_only_important_manifests(params, full_task_graph, filter_tasks): data = push_schedules(params["project"], params["head_rev"]) important_manifests = { m for m, c in data.get("groups", {}).items() if c >= BugbugLoader.CONFIDENCE_THRESHOLD } # Ensure we don't schedule unimportant manifests. for task in filter_tasks(full_task_graph, lambda t: t.kind == "test"): attr = task.attributes.get if attr("unittest_suite") in CHUNK_SUITES_BLACKLIST: assert not attr("test_manifests") else: unimportant = [ t for t in attr("test_manifests") if t not in important_manifests ] assert unimportant == []
def test_only_important_manifests(params, full_task_graph, filter_tasks): data = push_schedules(params["project"], params["head_rev"]) important_manifests = { m for m, c in data.get("groups", {}).items() if c >= BugbugLoader.CONFIDENCE_THRESHOLD } # Ensure we don't schedule unimportant manifests. for task in filter_tasks(full_task_graph, lambda t: t.kind == "test"): attr = task.attributes.get if "test_manifests" in task.attributes: unimportant = [ t for t in attr("test_manifests") if t not in important_manifests ] # Manifest scheduling is disabled for mochitest-ally. if attr("unittest_suite") == "mochitest-a11y": assert len(unimportant) > 0 else: assert unimportant == []
def should_remove_task(self, task, params, importance): project = params["project"] if project not in ("autoland", "try"): return False current_push_id = int(params["pushlog_id"]) branch = urlsplit(params["head_repository"]).path.strip("/") rev = params["head_rev"] if self.timedout: return registry[self.fallback].should_remove_task( task, params, importance) data = {} start_push_id = current_push_id - self.num_pushes + 1 if self.num_pushes != 1: push_data = get_push_data(params["head_repository"], project, start_push_id, current_push_id - 1) for push_id in range(start_push_id, current_push_id + 1): if push_id == current_push_id: rev = params["head_rev"] else: rev = push_data[push_id]["changesets"][-1] try: new_data = push_schedules(branch, rev) merge_bugbug_replies(data, new_data) except BugbugTimeoutException: if not self.fallback: raise self.timedout = True return self.should_remove_task(task, params, importance) key = "reduced_tasks" if self.use_reduced_tasks else "tasks" tasks = set(task for task, confidence in data.get(key, {}).items() if confidence >= self.confidence_threshold) test_manifests = task.attributes.get("test_manifests") if test_manifests is None or self.tasks_only: if data.get( "known_tasks") and task.label not in data["known_tasks"]: return False if task.label not in tasks: return True return False # If a task contains more than one group, use the max confidence. groups = data.get("groups", {}) confidences = [c for g, c in groups.items() if g in test_manifests] if not confidences or max(confidences) < self.confidence_threshold: return True # If the task configuration doesn't match the ones selected by bugbug for # the manifests, optimize out. if self.select_configs: selected_groups = [ g for g, c in groups.items() if g in test_manifests and c > self.confidence_threshold ] config_groups = data.get("config_groups", defaultdict(list)) # Configurations returned by bugbug are in a format such as # `test-windows10-64/opt-*-e10s`, while task labels are like # test-windows10-64-qr/opt-mochitest-browser-chrome-e10s-6. # In order to match the strings, we need to ignore the chunk number # from the task label. parts = task.label.split("-") label_without_chunk_number = "-".join( parts[:-1] if parts[-1].isdigit() else parts) if not any( fnmatch(label_without_chunk_number, config) for group in selected_groups for config in config_groups[group]): return True # Store group importance so future optimizers can access it. for manifest in test_manifests: if manifest not in groups: continue confidence = groups[manifest] if confidence >= CT_HIGH: importance[manifest] = "high" elif confidence >= CT_MEDIUM: importance[manifest] = "medium" elif confidence >= CT_LOW: importance[manifest] = "low" else: importance[manifest] = "lowest" return False
def should_remove_task(self, task, params, importance): project = params['project'] if project not in ("autoland", "try"): return False current_push_id = int(params['pushlog_id']) branch = urlsplit(params['head_repository']).path.strip('/') rev = params['head_rev'] if self.timedout: return registry[self.fallback].should_remove_task( task, params, importance) data = {} start_push_id = current_push_id - self.num_pushes + 1 if self.num_pushes != 1: push_data = get_push_data(params["head_repository"], project, start_push_id, current_push_id - 1) for push_id in range(start_push_id, current_push_id + 1): if push_id == current_push_id: rev = params["head_rev"] else: rev = push_data[push_id]["changesets"][-1] try: new_data = push_schedules(branch, rev) for key, value in new_data.items(): if isinstance(value, dict): if key not in data: data[key] = {} for name, confidence in value.items(): if name not in data[ key] or data[key][name] < confidence: data[key][name] = confidence elif isinstance(value, list): if key not in data: data[key] = set() data[key].update(value) except BugbugTimeoutException: if not self.fallback: raise self.timedout = True return self.should_remove_task(task, params, importance) key = "reduced_tasks" if self.use_reduced_tasks else "tasks" tasks = set(task for task, confidence in data.get(key, {}).items() if confidence >= self.confidence_threshold) test_manifests = task.attributes.get('test_manifests') if test_manifests is None or self.tasks_only: if data.get( "known_tasks") and task.label not in data["known_tasks"]: return False if task.label not in tasks: return True return False # If a task contains more than one group, use the max confidence. groups = data.get("groups", {}) confidences = [c for g, c in groups.items() if g in test_manifests] if not confidences or max(confidences) < self.confidence_threshold: return True # Store group importance so future optimizers can access it. for manifest in test_manifests: if manifest not in groups: continue confidence = groups[manifest] if confidence >= CT_HIGH: importance[manifest] = 'high' elif confidence >= CT_MEDIUM: importance[manifest] = 'medium' elif confidence >= CT_LOW: importance[manifest] = 'low' else: importance[manifest] = 'lowest' return False