def retrieve_pushes(self): now = datetime.datetime.today() to_date = now.strftime("%Y-%m-%d") from_date = (now - datetime.timedelta(days=DAYS_FROM_TODAY)).strftime( "%Y-%m-%d" ) pushes = make_push_objects( from_date=from_date, to_date=to_date, branch="autoland" ) return [{"rev": push.rev, "branch": push.branch} for push in pushes]
def run(args): # compute pushes in range pushes = make_push_objects( from_date=args.from_date, to_date=args.to_date, branch=args.branch ) header = [ 'Revisions', 'All Runnables', 'Regressions (possible)', 'Regressions (likely)', ] data = [ header ] num_cached = 0 for push in tqdm(pushes): key = f"push_data.{args.runnable}.{push.rev}" logger.info(f"Analyzing {push.rev} at the {args.runnable} level...") if config.cache.has(key): num_cached += 1 data.append(config.cache.get(key)) else: try: if args.runnable == "label": runnables = push.task_labels elif args.runnable == "group": runnables = push.group_summaries.keys() value = [ push.revs, list(runnables), list(push.get_possible_regressions(args.runnable)), list(push.get_likely_regressions(args.runnable)), ] data.append(value) config.cache.forever(key, value) except MissingDataError: logger.warning(f"Tasks for push {push.rev} can't be found on ActiveData") except Exception as e: traceback.print_exc() logger.info(f"{num_cached} pushes were already cached out of {len(pushes)}") return data
def classify_commands_pushes(branch: str, from_date: str, to_date: str, rev: str) -> List[Push]: if not (bool(rev) ^ bool(from_date or to_date)): raise Exception( "You must either provide a single push revision with --rev or define at least --from-date option to classify a range of pushes (note: --to-date will default to current time if not given)." ) if rev: pushes = [Push(rev, branch)] else: if not from_date: raise Exception( "You must provide at least --from-date to classify a range of pushes (note: --to-date will default to current time if not given)." ) now = datetime.datetime.now() if not to_date: to_date = datetime.datetime.strftime(now, "%Y-%m-%d") arrow_now = arrow.get(now) try: datetime.datetime.strptime(from_date, "%Y-%m-%d") except ValueError: try: from_date = arrow_now.dehumanize(from_date).format( "YYYY-MM-DD") except ValueError: raise Exception( 'Provided --from-date should be a date in yyyy-mm-dd format or a human expression like "1 days ago".' ) try: datetime.datetime.strptime(to_date, "%Y-%m-%d") except ValueError: try: to_date = arrow_now.dehumanize(to_date).format("YYYY-MM-DD") except ValueError: raise Exception( 'Provided --to-date should be a date in yyyy-mm-dd format or a human expression like "1 days ago".' ) pushes = make_push_objects(from_date=from_date, to_date=to_date, branch=branch) return pushes
def handle(self): branch = self.argument("branch") dry_run = self.option("dry-run") nb_pushes = int(self.option("nb-pushes")) self.queue = not dry_run and get_proxy_queue() or None self.line(f"Process pushes from {branch}") # List most recent pushes for push in make_push_objects(nb=nb_pushes, branch=branch): if dry_run: self.line(f"Would classify {push.branch}@{push.rev}") continue # Create a child task to classify that push task_id = self.create_task(push) self.line(f"<info>Created task {task_id}</info>")
def test_create_pushes_and_get_regressions(): """ An integration test mimicking the mozci usage done by bugbug. """ pushes = make_push_objects( from_date="today-7day", to_date="today-6day", branch="autoland", ) assert len(pushes) > 0 push = pushes[round(len(pushes) / 2)] assert len(push.task_labels) > 0 assert len(push.group_summaries) > 0 push.get_possible_regressions("label") push.get_possible_regressions("group") push.get_likely_regressions("label") push.get_likely_regressions("group")
def handle(self) -> None: branch = self.option("branch") environment = self.option("environment") matrix_room = config.get("matrix-room-id") current_task_id = os.environ.get("TASK_ID") try: nb_pushes = int(self.option("nb-pushes")) except ValueError: self.line("<error>Provided --nb-pushes should be an int.</error>") exit(1) self.line("<comment>Loading pushes...</comment>") self.pushes = make_push_objects(nb=nb_pushes, branch=branch) nb_pushes = len(self.pushes) to_notify: Dict[str, Dict[str, Any]] = {} for index, push in enumerate(self.pushes, start=1): self.line( f"<comment>Processing push {index}/{nb_pushes}: {push.push_uuid}</comment>" ) backfill_tasks = [] try: indexed_tasks = list_indexed_tasks( f"gecko.v2.{push.branch}.revision.{push.rev}.taskgraph.actions" ) except requests.exceptions.HTTPError as e: self.line( f"<error>Couldn't fetch indexed tasks on push {push.push_uuid}: {e}</error>" ) continue for indexed_task in indexed_tasks: task_id = indexed_task["taskId"] try: children_tasks = list_dependent_tasks(task_id) except requests.exceptions.HTTPError as e: self.line( f"<error>Couldn't fetch dependent tasks of indexed task {task_id} on push {push.push_uuid}: {e}</error>" ) continue for child_task in children_tasks: task_section = child_task.get("task", {}) task_action = task_section.get("tags", {}).get("action", "") # We are looking for the Treeherder symbol because Sheriffs are # only interested in backfill-tasks holding the '-bk' suffix in TH th_symbol = (task_section.get("extra", {}).get( "treeherder", {}).get("symbol", "")) status = child_task.get("status", {}) if task_action == "backfill-task" and th_symbol.endswith( "-bk"): assert status.get( "taskId" ), "Missing taskId attribute in backfill task status" label = task_section.get( "tags", {}).get("label") or task_section.get( "metadata", {}).get("name") assert ( label ), "Missing label attribute in backfill task tags or name attribute in backfill task metadata" assert status.get( "state" ), "Missing state attribute in backfill task status" backfill_tasks.append( BackfillTask(status["taskId"], label, th_symbol, status["state"])) else: logger.debug( f"Skipping non-backfill task {status.get('taskId')}" ) def group_key(task): return task.th_symbol # Sorting backfill tasks by their Treeherder symbol backfill_tasks = sorted(backfill_tasks, key=group_key) # Grouping ordered backfill tasks by their associated Treeherder symbol for th_symbol, tasks_iter in groupby(backfill_tasks, group_key): if th_symbol not in to_notify: to_notify[th_symbol] = { "newest_push": None, "backfill_tasks": set(), } # make_push_objects returns the latest pushes in chronological order from oldest to newest # We only need to store the newest Push that appeared for this Treeherder symbol to_notify[th_symbol]["newest_push"] = push # Storing all backfill tasks for this symbol across multiple pushes to_notify[th_symbol]["backfill_tasks"].update(tasks_iter) for th_symbol, data in to_notify.items(): all_backfill_tasks = data["backfill_tasks"] # Checking that all backfill tasks for this symbol are in a "final" state if not all(task.state in TASK_FINAL_STATES for task in all_backfill_tasks): logger.debug( f"Not all backfill tasks for the Treeherder symbol {th_symbol} are in a final state, not notifying now." ) continue newest_push = data["newest_push"] index_path = f"project.mozci.check-backfill.{environment}.{newest_push.branch}.{newest_push.rev}.{th_symbol}" try: find_task_id(index_path, root_url=COMMUNITY_TASKCLUSTER_ROOT_URL) except requests.exceptions.HTTPError: pass else: logger.debug( f"A notification was already sent for the backfill tasks associated to the Treeherder symbol {th_symbol}." ) continue try: parents = [ parent for parent in newest_push._iterate_parents(max_depth=20) ] except Exception as e: logger.debug( f"Failed to load the last twenty parent pushes for push {newest_push.push_uuid}, because: {e}." ) parents = None cleaned_label = re.sub(r"(-e10s|-1proc)?(-\d+)?$", "", all_backfill_tasks.pop().label) notification = NOTIFICATION_BACKFILL_GROUP_COMPLETED.format( th_symbol=th_symbol, push=newest_push, tochange=f"&tochange={newest_push.child.rev}", fromchange=f"&fromchange={parents[-1].rev}" if parents else "", searchstr=f"&searchStr={cleaned_label}", ) if not matrix_room: self.line( f"<comment>A notification should be sent for the backfill tasks associated to the Treeherder symbol {th_symbol} but no matrix room was provided in the secret.</comment>" ) logger.debug(f"The notification: {notification}") continue # Sending a notification to the Matrix channel defined in secret notify_matrix( room=matrix_room, body=notification, ) if not current_task_id: self.line( f"<comment>The current task should be indexed in {index_path} but TASK_ID environment variable isn't set.</comment>" ) continue # Populating the index with the current task to prevent sending the notification once again index_current_task( index_path, root_url=COMMUNITY_TASKCLUSTER_ROOT_URL, )
def main(): try: config = startup.read_settings() constants.set(config.constants) Log.start(config.debug) # SHUNT PYTHON LOGGING TO MAIN LOGGING capture_logging() # SHUNT ADR LOGGING TO MAIN LOGGING # https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add capture_loguru() if config.taskcluster: inject_secrets(config) @extend(Configuration) def update(self, config): """ Update the configuration object with new parameters :param config: dict of configuration """ for k, v in config.items(): if v != None: self._config[k] = v self._config["sources"] = sorted( map(os.path.expanduser, set(self._config["sources"]))) # Use the NullStore by default. This allows us to control whether # caching is enabled or not at runtime. self._config["cache"].setdefault("stores", {"null": { "driver": "null" }}) object.__setattr__(self, "cache", CustomCacheManager(self._config)) for _, store in self._config["cache"]["stores"].items(): if store.path and not store.path.endswith("/"): # REQUIRED, OTHERWISE FileStore._create_cache_directory() WILL LOOK AT PARENT DIRECTORY store.path = store.path + "/" if SHOW_S3_CACHE_HIT: s3_get = S3Store._get @extend(S3Store) def _get(self, key): with Timer("get {{key}} from S3", {"key": key}, verbose=False) as timer: output = s3_get(self, key) if output is not None: timer.verbose = True return output # UPDATE ADR CONFIGURATION with Repeat("waiting for ADR", every="10second"): adr.config.update(config.adr) # DUMMY TO TRIGGER CACHE make_push_objects(from_date=Date.today().format(), to_date=Date.now().format(), branch="autoland") outatime = Till(seconds=Duration(MAX_RUNTIME).total_seconds()) outatime.then(lambda: Log.alert("Out of time, exit early")) Schedulers(config).process(outatime) except Exception as e: Log.warning("Problem with etl! Shutting down.", cause=e) finally: Log.stop()
def process_one(self, start, end, branch, please_stop): # ASSUME PREVIOUS WORK IS DONE # UPDATE THE DATABASE STATE self.done.min = mo_math.min(end, self.done.min) self.done.max = mo_math.max(start, self.done.max) self.set_state() try: pushes = make_push_objects(from_date=start.format(), to_date=end.format(), branch=branch) except MissingDataError: return except Exception as e: raise Log.error("not expected", cause=e) Log.note( "Found {{num}} pushes on {{branch}} in ({{start}}, {{end}})", num=len(pushes), start=start, end=end, branch=branch, ) data = [] try: for push in pushes: if please_stop: break with Timer("get tasks for push {{push}}", {"push": push.id}): try: schedulers = [ label.split("shadow-scheduler-")[1] for label in push.scheduled_task_labels if "shadow-scheduler" in label ] except Exception as e: Log.warning("could not get schedulers", cause=e) schedulers = [] scheduler = [] for s in schedulers: try: scheduler.append({ "name": s, "tasks": jx.sort(push.get_shadow_scheduler_tasks(s)), }) except Exception: pass try: regressions = push.get_regressions("label").keys() except Exception as e: regressions = [] Log.warning("could not get regressions for {{push}}", push=push.id, cause=e) # RECORD THE PUSH data.append({ "push": { "id": push.id, "date": push.date, "changesets": push.revs, "backedoutby": push.backedoutby, }, "schedulers": scheduler, "regressions": [{ "label": name } for name in jx.sort(regressions)], "branch": branch, "etl": { "revision": git.get_revision(), "timestamp": Date.now(), }, }) finally: # ADD WHATEVER WE HAVE with Timer("adding {{num}} records to bigquery", {"num": len(data)}): self.destination.extend(data)
def run(args): global GECKO, logger if args.gecko_path: GECKO = args.gecko_path if GECKO and not Path(GECKO).is_dir(): if args.clone: clone_gecko() else: logger.error( f"Gecko path '{GECKO}' does not exist! Pass --clone to clone it to this location." ) sys.exit(1) schedulers = [] for s in args.strategies: logger.info(f"Creating scheduler using strategy {s}") schedulers.append(Scheduler(s)) # use what was actually scheduled as a baseline comparison schedulers.append(Scheduler('baseline')) # compute dates in range pushes = make_push_objects(from_date=args.from_date, to_date=args.to_date, branch=args.branch) total_pushes = len(pushes) logger.info(f"Found {total_pushes} pushes in specified range.") pushes_by_date = defaultdict(list) for push in pushes: date = datetime.utcfromtimestamp(push.date).strftime('%Y-%m-%d') pushes_by_date[date].append(push) if GECKO: orig_rev = hg(["log", "-r", ".", "-T", "{node}"]) logger.info(f"Found previous revision: {orig_rev}") try: i = 0 for date in sorted(pushes_by_date): pushes = pushes_by_date[date] logger.info(f"Analyzing pushes from {date} ({len(pushes)} pushes)") _hash = config.cache._hash(''.join([p.rev for p in pushes]) + ''.join([s.name for s in schedulers])) key = f"scheduler_analysis.{date}.{_hash}" if config.cache.has(key): logger.info(f"Loading results for {date} from cache") data = config.cache.get(key) for s in schedulers: s.score.update(Score(**data[s.name])) i += len(pushes) continue scores = defaultdict(Score) for push in sorted(pushes, key=lambda p: p.id): i += 1 logger.info( f"Analyzing https://treeherder.mozilla.org/#/jobs?repo=autoland&revision={push.rev} ({i}/{total_pushes})" ) # noqa if GECKO: hg(["update", push.rev]) for s in schedulers: try: scores[s.name].update(s.analyze(push)) except MissingDataError: logger.warning( f"MissingDataError: Skipping {push.rev}") config.cache.put(key, {k: v.as_dict() for k, v in scores.items()}, 43200) # 30 days finally: if GECKO: logger.info("restoring repo") hg(["update", orig_rev]) header = [ "Scheduler", "Total Tasks", "Primary Backouts", "Secondary Backouts", "Secondary Backout Rate", "Scheduler Efficiency", ] data = [] for sched in schedulers: s = sched.score data.append([ sched.name, s.tasks, s.primary_backouts, s.secondary_backouts, s.secondary_backout_rate, s.scheduler_efficiency, ]) data.sort(key=lambda x: x[-1], reverse=True) data.insert(0, header) return data
def handle(self): branch = self.argument("branch") from_date = self.option("from-date") to_date = self.option("to-date") if not (bool(self.option("rev")) ^ bool(from_date or to_date)): self.line( "<error>You must either provide a single push revision with --rev or define --from-date AND --to-date options to classify a range of pushes.</error>" ) return if self.option("rev"): pushes = [Push(self.option("rev"), branch)] else: if not from_date or not to_date: self.line( "<error>You must provide --from-date AND --to-date options to classify a range of pushes.</error>" ) return try: datetime.datetime.strptime(from_date, "%Y-%m-%d") except ValueError: self.line( "<error>Provided --from-date should be a date in yyyy-mm-dd format.</error>" ) return try: datetime.datetime.strptime(to_date, "%Y-%m-%d") except ValueError: self.line( "<error>Provided --to-date should be a date in yyyy-mm-dd format.</error>" ) return pushes = make_push_objects(from_date=from_date, to_date=to_date, branch=branch) try: medium_conf = float(self.option("medium-confidence")) except ValueError: self.line( "<error>Provided --medium-confidence should be a float.</error>" ) return try: high_conf = float(self.option("high-confidence")) except ValueError: self.line( "<error>Provided --high-confidence should be a float.</error>") return output = self.option("output") if output and not os.path.isdir(output): os.makedirs(output) self.line( "<comment>Provided --output pointed to a inexistent directory that is now created.</comment>" ) for push in pushes: try: classification, regressions = push.classify( confidence_medium=medium_conf, confidence_high=high_conf) self.line( f"Push associated with the head revision {push.rev} on " f"the branch {branch} is classified as {classification.name}" ) except Exception as e: self.line( f"<error>Couldn't classify push {push.push_uuid}: {e}.</error>" ) continue if self.option("show-intermittents"): self.line("-" * 50) self.line( "Printing tasks that should be marked as intermittent failures:" ) for task in regressions.intermittent: self.line(task) self.line("-" * 50) if output: to_save = { "push": { "id": push.push_uuid, "classification": classification.name, }, "failures": { "real": { group: [{ "task_id": task.id, "label": task.label } for task in failing_tasks] for group, failing_tasks in regressions.real.items() }, "intermittent": { group: [{ "task_id": task.id, "label": task.label } for task in failing_tasks] for group, failing_tasks in regressions.intermittent.items() }, "unknown": { group: [{ "task_id": task.id, "label": task.label } for task in failing_tasks] for group, failing_tasks in regressions.unknown.items() }, }, } filename = f"{output}/classify_output_{branch}_{push.rev}.json" with open(filename, "w") as file: json.dump(to_save, file, indent=2) self.line( f"Classification and regressions details for push {push.push_uuid} were saved in {filename} JSON file" )