def retrieve_pushes(self):
now = datetime.datetime.today()
to_date = now.strftime("%Y-%m-%d")
from_date = (now - datetime.timedelta(days=DAYS_FROM_TODAY)).strftime(
"%Y-%m-%d"
)
pushes = make_push_objects(
from_date=from_date, to_date=to_date, branch="autoland"
)
return [{"rev": push.rev, "branch": push.branch} for push in pushes]
def run(args):
# compute pushes in range
pushes = make_push_objects(
from_date=args.from_date, to_date=args.to_date, branch=args.branch
)
header = [
'Revisions',
'All Runnables',
'Regressions (possible)',
'Regressions (likely)',
]
data = [
header
]
num_cached = 0
for push in tqdm(pushes):
key = f"push_data.{args.runnable}.{push.rev}"
logger.info(f"Analyzing {push.rev} at the {args.runnable} level...")
if config.cache.has(key):
num_cached += 1
data.append(config.cache.get(key))
else:
try:
if args.runnable == "label":
runnables = push.task_labels
elif args.runnable == "group":
runnables = push.group_summaries.keys()
value = [
push.revs,
list(runnables),
list(push.get_possible_regressions(args.runnable)),
list(push.get_likely_regressions(args.runnable)),
]
data.append(value)
config.cache.forever(key, value)
except MissingDataError:
logger.warning(f"Tasks for push {push.rev} can't be found on ActiveData")
except Exception as e:
traceback.print_exc()
logger.info(f"{num_cached} pushes were already cached out of {len(pushes)}")
return data
def classify_commands_pushes(branch: str, from_date: str, to_date: str,
rev: str) -> List[Push]:
if not (bool(rev) ^ bool(from_date or to_date)):
raise Exception(
"You must either provide a single push revision with --rev or define at least --from-date option to classify a range of pushes (note: --to-date will default to current time if not given)."
)
if rev:
pushes = [Push(rev, branch)]
else:
if not from_date:
raise Exception(
"You must provide at least --from-date to classify a range of pushes (note: --to-date will default to current time if not given)."
)
now = datetime.datetime.now()
if not to_date:
to_date = datetime.datetime.strftime(now, "%Y-%m-%d")
arrow_now = arrow.get(now)
try:
datetime.datetime.strptime(from_date, "%Y-%m-%d")
except ValueError:
try:
from_date = arrow_now.dehumanize(from_date).format(
"YYYY-MM-DD")
except ValueError:
raise Exception(
'Provided --from-date should be a date in yyyy-mm-dd format or a human expression like "1 days ago".'
)
try:
datetime.datetime.strptime(to_date, "%Y-%m-%d")
except ValueError:
try:
to_date = arrow_now.dehumanize(to_date).format("YYYY-MM-DD")
except ValueError:
raise Exception(
'Provided --to-date should be a date in yyyy-mm-dd format or a human expression like "1 days ago".'
)
pushes = make_push_objects(from_date=from_date,
to_date=to_date,
branch=branch)
return pushes
def handle(self):
branch = self.argument("branch")
dry_run = self.option("dry-run")
nb_pushes = int(self.option("nb-pushes"))
self.queue = not dry_run and get_proxy_queue() or None
self.line(f"Process pushes from {branch}")
# List most recent pushes
for push in make_push_objects(nb=nb_pushes, branch=branch):
if dry_run:
self.line(f"Would classify {push.branch}@{push.rev}")
continue
# Create a child task to classify that push
task_id = self.create_task(push)
self.line(f"<info>Created task {task_id}</info>")
def test_create_pushes_and_get_regressions():
"""
An integration test mimicking the mozci usage done by bugbug.
"""
pushes = make_push_objects(
from_date="today-7day",
to_date="today-6day",
branch="autoland",
)
assert len(pushes) > 0
push = pushes[round(len(pushes) / 2)]
assert len(push.task_labels) > 0
assert len(push.group_summaries) > 0
push.get_possible_regressions("label")
push.get_possible_regressions("group")
push.get_likely_regressions("label")
push.get_likely_regressions("group")
def handle(self) -> None:
branch = self.option("branch")
environment = self.option("environment")
matrix_room = config.get("matrix-room-id")
current_task_id = os.environ.get("TASK_ID")
try:
nb_pushes = int(self.option("nb-pushes"))
except ValueError:
self.line("<error>Provided --nb-pushes should be an int.</error>")
exit(1)
self.line("<comment>Loading pushes...</comment>")
self.pushes = make_push_objects(nb=nb_pushes, branch=branch)
nb_pushes = len(self.pushes)
to_notify: Dict[str, Dict[str, Any]] = {}
for index, push in enumerate(self.pushes, start=1):
self.line(
f"<comment>Processing push {index}/{nb_pushes}: {push.push_uuid}</comment>"
)
backfill_tasks = []
try:
indexed_tasks = list_indexed_tasks(
f"gecko.v2.{push.branch}.revision.{push.rev}.taskgraph.actions"
)
except requests.exceptions.HTTPError as e:
self.line(
f"<error>Couldn't fetch indexed tasks on push {push.push_uuid}: {e}</error>"
)
continue
for indexed_task in indexed_tasks:
task_id = indexed_task["taskId"]
try:
children_tasks = list_dependent_tasks(task_id)
except requests.exceptions.HTTPError as e:
self.line(
f"<error>Couldn't fetch dependent tasks of indexed task {task_id} on push {push.push_uuid}: {e}</error>"
)
continue
for child_task in children_tasks:
task_section = child_task.get("task", {})
task_action = task_section.get("tags",
{}).get("action", "")
# We are looking for the Treeherder symbol because Sheriffs are
# only interested in backfill-tasks holding the '-bk' suffix in TH
th_symbol = (task_section.get("extra", {}).get(
"treeherder", {}).get("symbol", ""))
status = child_task.get("status", {})
if task_action == "backfill-task" and th_symbol.endswith(
"-bk"):
assert status.get(
"taskId"
), "Missing taskId attribute in backfill task status"
label = task_section.get(
"tags", {}).get("label") or task_section.get(
"metadata", {}).get("name")
assert (
label
), "Missing label attribute in backfill task tags or name attribute in backfill task metadata"
assert status.get(
"state"
), "Missing state attribute in backfill task status"
backfill_tasks.append(
BackfillTask(status["taskId"], label, th_symbol,
status["state"]))
else:
logger.debug(
f"Skipping non-backfill task {status.get('taskId')}"
)
def group_key(task):
return task.th_symbol
# Sorting backfill tasks by their Treeherder symbol
backfill_tasks = sorted(backfill_tasks, key=group_key)
# Grouping ordered backfill tasks by their associated Treeherder symbol
for th_symbol, tasks_iter in groupby(backfill_tasks, group_key):
if th_symbol not in to_notify:
to_notify[th_symbol] = {
"newest_push": None,
"backfill_tasks": set(),
}
# make_push_objects returns the latest pushes in chronological order from oldest to newest
# We only need to store the newest Push that appeared for this Treeherder symbol
to_notify[th_symbol]["newest_push"] = push
# Storing all backfill tasks for this symbol across multiple pushes
to_notify[th_symbol]["backfill_tasks"].update(tasks_iter)
for th_symbol, data in to_notify.items():
all_backfill_tasks = data["backfill_tasks"]
# Checking that all backfill tasks for this symbol are in a "final" state
if not all(task.state in TASK_FINAL_STATES
for task in all_backfill_tasks):
logger.debug(
f"Not all backfill tasks for the Treeherder symbol {th_symbol} are in a final state, not notifying now."
)
continue
newest_push = data["newest_push"]
index_path = f"project.mozci.check-backfill.{environment}.{newest_push.branch}.{newest_push.rev}.{th_symbol}"
try:
find_task_id(index_path,
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL)
except requests.exceptions.HTTPError:
pass
else:
logger.debug(
f"A notification was already sent for the backfill tasks associated to the Treeherder symbol {th_symbol}."
)
continue
try:
parents = [
parent
for parent in newest_push._iterate_parents(max_depth=20)
]
except Exception as e:
logger.debug(
f"Failed to load the last twenty parent pushes for push {newest_push.push_uuid}, because: {e}."
)
parents = None
cleaned_label = re.sub(r"(-e10s|-1proc)?(-\d+)?$", "",
all_backfill_tasks.pop().label)
notification = NOTIFICATION_BACKFILL_GROUP_COMPLETED.format(
th_symbol=th_symbol,
push=newest_push,
tochange=f"&tochange={newest_push.child.rev}",
fromchange=f"&fromchange={parents[-1].rev}" if parents else "",
searchstr=f"&searchStr={cleaned_label}",
)
if not matrix_room:
self.line(
f"<comment>A notification should be sent for the backfill tasks associated to the Treeherder symbol {th_symbol} but no matrix room was provided in the secret.</comment>"
)
logger.debug(f"The notification: {notification}")
continue
# Sending a notification to the Matrix channel defined in secret
notify_matrix(
room=matrix_room,
body=notification,
)
if not current_task_id:
self.line(
f"<comment>The current task should be indexed in {index_path} but TASK_ID environment variable isn't set.</comment>"
)
continue
# Populating the index with the current task to prevent sending the notification once again
index_current_task(
index_path,
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL,
)
def main():
try:
config = startup.read_settings()
constants.set(config.constants)
Log.start(config.debug)
# SHUNT PYTHON LOGGING TO MAIN LOGGING
capture_logging()
# SHUNT ADR LOGGING TO MAIN LOGGING
# https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add
capture_loguru()
if config.taskcluster:
inject_secrets(config)
@extend(Configuration)
def update(self, config):
"""
Update the configuration object with new parameters
:param config: dict of configuration
"""
for k, v in config.items():
if v != None:
self._config[k] = v
self._config["sources"] = sorted(
map(os.path.expanduser, set(self._config["sources"])))
# Use the NullStore by default. This allows us to control whether
# caching is enabled or not at runtime.
self._config["cache"].setdefault("stores",
{"null": {
"driver": "null"
}})
object.__setattr__(self, "cache", CustomCacheManager(self._config))
for _, store in self._config["cache"]["stores"].items():
if store.path and not store.path.endswith("/"):
# REQUIRED, OTHERWISE FileStore._create_cache_directory() WILL LOOK AT PARENT DIRECTORY
store.path = store.path + "/"
if SHOW_S3_CACHE_HIT:
s3_get = S3Store._get
@extend(S3Store)
def _get(self, key):
with Timer("get {{key}} from S3", {"key": key},
verbose=False) as timer:
output = s3_get(self, key)
if output is not None:
timer.verbose = True
return output
# UPDATE ADR CONFIGURATION
with Repeat("waiting for ADR", every="10second"):
adr.config.update(config.adr)
# DUMMY TO TRIGGER CACHE
make_push_objects(from_date=Date.today().format(),
to_date=Date.now().format(),
branch="autoland")
outatime = Till(seconds=Duration(MAX_RUNTIME).total_seconds())
outatime.then(lambda: Log.alert("Out of time, exit early"))
Schedulers(config).process(outatime)
except Exception as e:
Log.warning("Problem with etl! Shutting down.", cause=e)
finally:
Log.stop()
def process_one(self, start, end, branch, please_stop):
# ASSUME PREVIOUS WORK IS DONE
# UPDATE THE DATABASE STATE
self.done.min = mo_math.min(end, self.done.min)
self.done.max = mo_math.max(start, self.done.max)
self.set_state()
try:
pushes = make_push_objects(from_date=start.format(),
to_date=end.format(),
branch=branch)
except MissingDataError:
return
except Exception as e:
raise Log.error("not expected", cause=e)
Log.note(
"Found {{num}} pushes on {{branch}} in ({{start}}, {{end}})",
num=len(pushes),
start=start,
end=end,
branch=branch,
)
data = []
try:
for push in pushes:
if please_stop:
break
with Timer("get tasks for push {{push}}", {"push": push.id}):
try:
schedulers = [
label.split("shadow-scheduler-")[1]
for label in push.scheduled_task_labels
if "shadow-scheduler" in label
]
except Exception as e:
Log.warning("could not get schedulers", cause=e)
schedulers = []
scheduler = []
for s in schedulers:
try:
scheduler.append({
"name":
s,
"tasks":
jx.sort(push.get_shadow_scheduler_tasks(s)),
})
except Exception:
pass
try:
regressions = push.get_regressions("label").keys()
except Exception as e:
regressions = []
Log.warning("could not get regressions for {{push}}",
push=push.id,
cause=e)
# RECORD THE PUSH
data.append({
"push": {
"id": push.id,
"date": push.date,
"changesets": push.revs,
"backedoutby": push.backedoutby,
},
"schedulers":
scheduler,
"regressions": [{
"label": name
} for name in jx.sort(regressions)],
"branch":
branch,
"etl": {
"revision": git.get_revision(),
"timestamp": Date.now(),
},
})
finally:
# ADD WHATEVER WE HAVE
with Timer("adding {{num}} records to bigquery",
{"num": len(data)}):
self.destination.extend(data)
def run(args):
global GECKO, logger
if args.gecko_path:
GECKO = args.gecko_path
if GECKO and not Path(GECKO).is_dir():
if args.clone:
clone_gecko()
else:
logger.error(
f"Gecko path '{GECKO}' does not exist! Pass --clone to clone it to this location."
)
sys.exit(1)
schedulers = []
for s in args.strategies:
logger.info(f"Creating scheduler using strategy {s}")
schedulers.append(Scheduler(s))
# use what was actually scheduled as a baseline comparison
schedulers.append(Scheduler('baseline'))
# compute dates in range
pushes = make_push_objects(from_date=args.from_date,
to_date=args.to_date,
branch=args.branch)
total_pushes = len(pushes)
logger.info(f"Found {total_pushes} pushes in specified range.")
pushes_by_date = defaultdict(list)
for push in pushes:
date = datetime.utcfromtimestamp(push.date).strftime('%Y-%m-%d')
pushes_by_date[date].append(push)
if GECKO:
orig_rev = hg(["log", "-r", ".", "-T", "{node}"])
logger.info(f"Found previous revision: {orig_rev}")
try:
i = 0
for date in sorted(pushes_by_date):
pushes = pushes_by_date[date]
logger.info(f"Analyzing pushes from {date} ({len(pushes)} pushes)")
_hash = config.cache._hash(''.join([p.rev for p in pushes]) +
''.join([s.name for s in schedulers]))
key = f"scheduler_analysis.{date}.{_hash}"
if config.cache.has(key):
logger.info(f"Loading results for {date} from cache")
data = config.cache.get(key)
for s in schedulers:
s.score.update(Score(**data[s.name]))
i += len(pushes)
continue
scores = defaultdict(Score)
for push in sorted(pushes, key=lambda p: p.id):
i += 1
logger.info(
f"Analyzing https://treeherder.mozilla.org/#/jobs?repo=autoland&revision={push.rev} ({i}/{total_pushes})"
) # noqa
if GECKO:
hg(["update", push.rev])
for s in schedulers:
try:
scores[s.name].update(s.analyze(push))
except MissingDataError:
logger.warning(
f"MissingDataError: Skipping {push.rev}")
config.cache.put(key, {k: v.as_dict()
for k, v in scores.items()},
43200) # 30 days
finally:
if GECKO:
logger.info("restoring repo")
hg(["update", orig_rev])
header = [
"Scheduler",
"Total Tasks",
"Primary Backouts",
"Secondary Backouts",
"Secondary Backout Rate",
"Scheduler Efficiency",
]
data = []
for sched in schedulers:
s = sched.score
data.append([
sched.name,
s.tasks,
s.primary_backouts,
s.secondary_backouts,
s.secondary_backout_rate,
s.scheduler_efficiency,
])
data.sort(key=lambda x: x[-1], reverse=True)
data.insert(0, header)
return data
def handle(self):
branch = self.argument("branch")
from_date = self.option("from-date")
to_date = self.option("to-date")
if not (bool(self.option("rev")) ^ bool(from_date or to_date)):
self.line(
"<error>You must either provide a single push revision with --rev or define --from-date AND --to-date options to classify a range of pushes.</error>"
)
return
if self.option("rev"):
pushes = [Push(self.option("rev"), branch)]
else:
if not from_date or not to_date:
self.line(
"<error>You must provide --from-date AND --to-date options to classify a range of pushes.</error>"
)
return
try:
datetime.datetime.strptime(from_date, "%Y-%m-%d")
except ValueError:
self.line(
"<error>Provided --from-date should be a date in yyyy-mm-dd format.</error>"
)
return
try:
datetime.datetime.strptime(to_date, "%Y-%m-%d")
except ValueError:
self.line(
"<error>Provided --to-date should be a date in yyyy-mm-dd format.</error>"
)
return
pushes = make_push_objects(from_date=from_date,
to_date=to_date,
branch=branch)
try:
medium_conf = float(self.option("medium-confidence"))
except ValueError:
self.line(
"<error>Provided --medium-confidence should be a float.</error>"
)
return
try:
high_conf = float(self.option("high-confidence"))
except ValueError:
self.line(
"<error>Provided --high-confidence should be a float.</error>")
return
output = self.option("output")
if output and not os.path.isdir(output):
os.makedirs(output)
self.line(
"<comment>Provided --output pointed to a inexistent directory that is now created.</comment>"
)
for push in pushes:
try:
classification, regressions = push.classify(
confidence_medium=medium_conf, confidence_high=high_conf)
self.line(
f"Push associated with the head revision {push.rev} on "
f"the branch {branch} is classified as {classification.name}"
)
except Exception as e:
self.line(
f"<error>Couldn't classify push {push.push_uuid}: {e}.</error>"
)
continue
if self.option("show-intermittents"):
self.line("-" * 50)
self.line(
"Printing tasks that should be marked as intermittent failures:"
)
for task in regressions.intermittent:
self.line(task)
self.line("-" * 50)
if output:
to_save = {
"push": {
"id": push.push_uuid,
"classification": classification.name,
},
"failures": {
"real": {
group: [{
"task_id": task.id,
"label": task.label
} for task in failing_tasks]
for group, failing_tasks in
regressions.real.items()
},
"intermittent": {
group: [{
"task_id": task.id,
"label": task.label
} for task in failing_tasks]
for group, failing_tasks in
regressions.intermittent.items()
},
"unknown": {
group: [{
"task_id": task.id,
"label": task.label
} for task in failing_tasks]
for group, failing_tasks in
regressions.unknown.items()
},
},
}
filename = f"{output}/classify_output_{branch}_{push.rev}.json"
with open(filename, "w") as file:
json.dump(to_save, file, indent=2)
self.line(
f"Classification and regressions details for push {push.push_uuid} were saved in {filename} JSON file"
)