def retrieve_commits(self, limit):
repository.clone(self.repo_dir)
if limit:
# Mercurial revset supports negative integers starting from tip
rev_start = -limit
else:
db.download(repository.COMMITS_DB, support_files_too=True)
rev_start = 0
for commit in repository.get_commits():
rev_start = f"children({commit['node']})"
with hglib.open(self.repo_dir) as hg:
revs = repository.get_revs(hg, rev_start)
chunk_size = 70000
for i in range(0, len(revs), chunk_size):
repository.download_commits(self.repo_dir,
revs=revs[i:(i + chunk_size)])
logger.info("commit data extracted from repository")
# Some commits that were already in the DB from the previous run might need
# to be updated (e.g. coverage information).
repository.update_commits()
zstd_compress(repository.COMMITS_DB)
create_tar_zst(os.path.join("data", repository.COMMIT_EXPERIENCES_DB))
def upload_adr_cache(self):
cache_path = os.path.splitext(ADR_CACHE_DB)[0]
assert os.path.abspath(adr.config["cache"]["stores"]["file"]
["path"]) == os.path.abspath(cache_path)
create_tar_zst(f"{ADR_CACHE_DB}.zst")
db.upload(ADR_CACHE_DB)
def retrieve_commits(self, limit):
repository.clone(self.repo_dir)
if limit:
# Mercurial revset supports negative integers starting from tip
rev_start = -limit
else:
db.download(repository.COMMITS_DB, support_files_too=True)
rev_start = 0
for commit in repository.get_commits():
rev_start = f"children({commit['node']})"
repository.download_commits(self.repo_dir, rev_start=rev_start)
logger.info("commit data extracted from repository")
zstd_compress(repository.COMMITS_DB)
create_tar_zst(os.path.join("data", repository.COMMIT_EXPERIENCES_DB))
def generate_test_scheduling_history(self, granularity: str,
training_months: int) -> None:
if granularity != "config_group":
# Get the commits DB.
assert db.download(repository.COMMITS_DB)
HISTORY_DATE_START = datetime.now() - relativedelta(
months=training_months)
if granularity == "label":
test_scheduling_db = test_scheduling.TEST_LABEL_SCHEDULING_DB
past_failures_db = os.path.join(
"data", test_scheduling.PAST_FAILURES_LABEL_DB)
failing_together_db = os.path.join(
"data", test_scheduling.FAILING_TOGETHER_LABEL_DB)
elif granularity == "group":
test_scheduling_db = test_scheduling.TEST_GROUP_SCHEDULING_DB
past_failures_db = os.path.join(
"data", test_scheduling.PAST_FAILURES_GROUP_DB)
touched_together_db = os.path.join(
"data", test_scheduling.TOUCHED_TOGETHER_DB)
elif granularity == "config_group":
test_scheduling_db = test_scheduling.TEST_CONFIG_GROUP_SCHEDULING_DB
past_failures_db = os.path.join(
"data", test_scheduling.PAST_FAILURES_CONFIG_GROUP_DB)
failing_together_db = os.path.join(
"data", test_scheduling.FAILING_TOGETHER_CONFIG_GROUP_DB)
push_data_iter, push_data_count, all_runnables = test_scheduling.get_push_data(
granularity)
if granularity in ("label", "config_group"):
test_scheduling.generate_failing_together_probabilities(
granularity, push_data_iter(), push_data_count)
def generate_all_data() -> Generator[Dict[str, Any], None, None]:
past_failures = test_scheduling.get_past_failures(
granularity, False)
push_num = past_failures[
"push_num"] if "push_num" in past_failures else 0
commit_map = {}
for commit_data in tqdm(repository.get_commits()):
commit_map[commit_data["node"]] = commit_data
# Store all runnables in the past_failures DB so it can be used in the evaluation phase.
past_failures["all_runnables"] = all_runnables
# XXX: Should we recreate the DB from scratch if the previous all_runnables are not the
# same as the current ones?
saved_nodes = set()
skipped_no_commits = 0
skipped_too_big_commits = 0
skipped_no_runnables = 0
if granularity in ("group", "config_group"):
update_touched_together_gen = test_scheduling.update_touched_together(
)
next(update_touched_together_gen)
for (
i,
(
revisions,
fix_revision,
push_runnables,
possible_regressions,
likely_regressions,
),
) in enumerate(tqdm(push_data_iter(), total=push_data_count)):
push_num += 1
# XXX: Some commits are skipped in the repository mining, e.g. merges and backouts. Maybe we should not skip them.
commits = tuple(
commit_map.pop(revision) for revision in revisions
if revision in commit_map)
if len(commits) == 0:
skipped_no_commits += 1
continue
# Skip wptsync commits, since they are not like normal pushes made by developers.
if any(repository.is_wptsync(commit) for commit in commits):
continue
merged_commits = commit_features.merge_commits(commits)
# XXX: For now, skip commits which are too large.
# In the future we can either:
# - Improve shelve perf and go back to consider all files;
# - Consider only files which appear with a given frequency, like the "files" feature in commit_features;
# - Keep a limit of number of files.
if len(merged_commits["files"]) > 50:
skipped_too_big_commits += 1
continue
# If we considered all_runnables, we'd generate a huge amount of data.
# We consider only the runnables which run in this push, and the possible and likely regressions
# from this push. We can't consider all runnables because we can't be sure that a task that didn't
# run on a push would have been successful.
runnables_to_consider = list(
set(push_runnables + possible_regressions +
likely_regressions))
if len(runnables_to_consider) == 0:
skipped_no_runnables += 1
continue
# Sync DB every 250 pushes, so we cleanup the shelve cache (we'd run OOM otherwise!).
if i % 250 == 0:
past_failures.sync()
pushdate = dateutil.parser.parse(merged_commits["pushdate"])
if granularity in ("group", "config_group"):
update_touched_together_gen.send(commits[0]["node"])
result_data = []
for data in test_scheduling.generate_data(
granularity,
past_failures,
merged_commits,
push_num,
runnables_to_consider,
possible_regressions,
likely_regressions,
):
if pushdate > HISTORY_DATE_START:
result_data.append(data)
if pushdate > HISTORY_DATE_START:
saved_nodes.add(i)
yield {
"revs": revisions,
"data": result_data,
}
if granularity == "group":
try:
update_touched_together_gen.send(None)
except StopIteration:
pass
logger.info(f"saved push data nodes: {len(saved_nodes)}")
logger.info(f"skipped {skipped_no_commits} (no commits in our DB)")
logger.info(f"skipped {skipped_too_big_commits} (too big commits)")
logger.info(
f"skipped {skipped_no_runnables} (no interesting runnables)")
past_failures["push_num"] = push_num
past_failures.close()
# For the config/group granularity, we are only interested in the failing together DB.
if granularity != "config_group":
db.append(test_scheduling_db, generate_all_data())
zstd_compress(test_scheduling_db)
create_tar_zst(past_failures_db)
if granularity == "group":
create_tar_zst(touched_together_db)
if granularity in ("label", "config_group"):
create_tar_zst(failing_together_db)
def generate_test_scheduling_history(self, granularity):
push_data_path = f"push_data_{granularity}.json"
updated = download_check_etag(
test_scheduling.PUSH_DATA_URL.format(granularity=granularity))
if updated:
zstd_decompress(push_data_path)
os.remove(f"{push_data_path}.zst")
assert os.path.exists(
push_data_path), "Decompressed push data file exists"
# Get the commits DB.
assert db.download(repository.COMMITS_DB)
HISTORY_DATE_START = datetime.now() - relativedelta(
months=TRAINING_MONTHS[granularity])
if granularity == "label":
test_scheduling_db = test_scheduling.TEST_LABEL_SCHEDULING_DB
past_failures_db = os.path.join(
"data", test_scheduling.PAST_FAILURES_LABEL_DB)
failing_together_db = os.path.join(
"data", test_scheduling.FAILING_TOGETHER_LABEL_DB)
elif granularity == "group":
test_scheduling_db = test_scheduling.TEST_GROUP_SCHEDULING_DB
past_failures_db = os.path.join(
"data", test_scheduling.PAST_FAILURES_GROUP_DB)
touched_together_db = os.path.join(
"data", test_scheduling.TOUCHED_TOGETHER_DB)
db.download(test_scheduling_db, support_files_too=True)
last_node = None
for revs, _ in test_scheduling.get_test_scheduling_history(
granularity):
last_node = revs[0]
def generate_failing_together_probabilities(push_data):
# TODO: we should consider the probabilities of `task1 failure -> task2 failure` and
# `task2 failure -> task1 failure` separately, as they could be different.
count_runs = collections.Counter()
count_single_failures = collections.Counter()
count_both_failures = collections.Counter()
for revisions, tasks, likely_regressions, candidate_regressions in tqdm(
push_data):
failures = set(likely_regressions + candidate_regressions)
all_tasks = list(set(tasks) | failures)
for task1, task2 in itertools.combinations(
sorted(all_tasks), 2):
count_runs[(task1, task2)] += 1
if task1 in failures:
if task2 in failures:
count_both_failures[(task1, task2)] += 1
else:
count_single_failures[(task1, task2)] += 1
elif task2 in failures:
count_single_failures[(task1, task2)] += 1
stats = {}
skipped = 0
for couple, run_count in count_runs.most_common():
failure_count = count_both_failures[couple]
support = failure_count / run_count
if support < 1 / 700:
skipped += 1
continue
if failure_count != 0:
confidence = failure_count / (
count_single_failures[couple] + failure_count)
else:
confidence = 0.0
stats[couple] = (support, confidence)
logger.info(
f"{skipped} couples skipped because their support was too low")
logger.info(
"Redundancies with the highest support and confidence:")
for couple, (support,
confidence) in sorted(stats.items(),
key=lambda k:
(-k[1][1], -k[1][0]))[:7]:
failure_count = count_both_failures[couple]
run_count = count_runs[couple]
logger.info(
f"{couple[0]} - {couple[1]} redundancy confidence {confidence}, support {support} ({failure_count} over {run_count})."
)
logger.info(
"Redundancies with the highest confidence and lowest support:")
for couple, (support,
confidence) in sorted(stats.items(),
key=lambda k:
(-k[1][1], k[1][0]))[:7]:
failure_count = count_both_failures[couple]
run_count = count_runs[couple]
logger.info(
f"{couple[0]} - {couple[1]} redundancy confidence {confidence}, support {support} ({failure_count} over {run_count})."
)
failing_together = test_scheduling.get_failing_together_db()
count_redundancies = collections.Counter()
for couple, (support, confidence) in stats.items():
if confidence == 1.0:
count_redundancies["==100%"] += 1
if confidence > 0.9:
count_redundancies[">=90%"] += 1
if confidence > 0.8:
count_redundancies[">=80%"] += 1
if confidence > 0.7:
count_redundancies[">=70%"] += 1
if confidence < 0.7:
continue
failing_together[f"{couple[0]}${couple[1]}".encode(
"utf-8")] = struct.pack("ff", support, confidence)
for percentage, count in count_redundancies.most_common():
logger.info(f"{count} with {percentage} confidence")
test_scheduling.close_failing_together_db()
def generate_all_data():
past_failures = test_scheduling.get_past_failures(granularity)
push_num = past_failures[
"push_num"] if "push_num" in past_failures else 0
# We can start once we get to the last revision we added in the previous run.
can_start = True if last_node is None else False
commit_map = {}
for commit_data in tqdm(repository.get_commits()):
if not can_start:
if last_node == commit_data["node"]:
can_start = True
continue
commit_map[commit_data["node"]] = commit_data
with open(push_data_path, "r") as f:
push_data = json.load(f)
logger.info(f"push data nodes: {len(push_data)}")
if granularity == "label":
push_data = [(
revisions,
rename_tasks(push_tasks),
rename_tasks(possible_regressions),
rename_tasks(likely_regressions),
) for revisions, push_tasks, possible_regressions,
likely_regressions in push_data]
# In the last 14 pushes, we definitely run all possible runnables.
all_runnables_set = set(
sum((push_runnables
for _, push_runnables, _, _ in push_data[-14:]), []))
# Filter runnables we don't need.
all_runnables = filter_runnables(list(all_runnables_set),
all_runnables_set, granularity)
all_runnables_set = set(all_runnables_set)
logger.info(
f"{len(all_runnables_set)} runnables run in the last 14 pushes"
)
push_data = [(
revisions,
filter_runnables(push_tasks, all_runnables_set, granularity),
filter_runnables(possible_regressions, all_runnables_set,
granularity),
filter_runnables(likely_regressions, all_runnables_set,
granularity),
) for revisions, push_tasks, possible_regressions,
likely_regressions in push_data]
if granularity == "label":
generate_failing_together_probabilities(push_data)
# Store all runnables in the past_failures DB so it can be used in the evaluation phase.
past_failures["all_runnables"] = all_runnables
# XXX: Should we recreate the DB from scratch if the previous all_runnables are not the
# same as the current ones?
saved_nodes = set()
skipped_no_commits = 0
skipped_too_big_commits = 0
skipped_no_runnables = 0
# We can start once we get to the last revision we added in the previous run.
can_start = True if last_node is None else False
if granularity == "group":
update_touched_together_gen = test_scheduling.update_touched_together(
)
next(update_touched_together_gen)
for i in tqdm(range(len(push_data))):
(
revisions,
push_runnables,
possible_regressions,
likely_regressions,
) = push_data.pop(0)
if not can_start:
if last_node == revisions[0]:
can_start = True
continue
push_num += 1
# XXX: Some commits are skipped in the repository mining, e.g. merges and backouts. Maybe we should not skip them.
commits = tuple(
commit_map.pop(revision) for revision in revisions
if revision in commit_map)
if len(commits) == 0:
skipped_no_commits += 1
continue
merged_commits = commit_features.merge_commits(commits)
# XXX: For now, skip commits which are too large.
# In the future we can either:
# - Improve shelve perf and go back to consider all files;
# - Consider only files which appear with a given frequency, like the "files" feature in commit_features;
# - Keep a limit of number of files.
if len(merged_commits["files"]) > 50:
skipped_too_big_commits += 1
continue
# If we considered all_runnables, we'd generate a huge amount of data.
# We consider only the runnables which run in this push, and the possible and likely regressions
# from this push. We can't consider all runnables because we can't be sure that a task that didn't
# run on a push would have been successful.
runnables_to_consider = list(
set(push_runnables + possible_regressions +
likely_regressions))
if len(runnables_to_consider) == 0:
skipped_no_runnables += 1
continue
# Sync DB every 250 pushes, so we cleanup the shelve cache (we'd run OOM otherwise!).
if i % 250 == 0:
past_failures.sync()
pushdate = dateutil.parser.parse(merged_commits["pushdate"])
if granularity == "group":
update_touched_together_gen.send(commits[0]["node"])
result = {
"revs": revisions,
"data": [],
}
for data in test_scheduling.generate_data(
past_failures,
merged_commits,
push_num,
runnables_to_consider,
possible_regressions,
likely_regressions,
):
if pushdate > HISTORY_DATE_START:
result["data"].append(data)
if pushdate > HISTORY_DATE_START:
saved_nodes.add(i)
yield result
if granularity == "group":
try:
update_touched_together_gen.send(None)
except StopIteration:
pass
logger.info(f"saved push data nodes: {len(saved_nodes)}")
logger.info(f"skipped {skipped_no_commits} (no commits in our DB)")
logger.info(f"skipped {skipped_too_big_commits} (too big commits)")
logger.info(
f"skipped {skipped_no_runnables} (no interesting runnables)")
past_failures["push_num"] = push_num
past_failures.close()
db.append(test_scheduling_db, generate_all_data())
zstd_compress(test_scheduling_db)
create_tar_zst(past_failures_db)
if granularity == "group":
create_tar_zst(touched_together_db)
if granularity == "label":
create_tar_zst(failing_together_db)