def get_test_time_reports_from_S3() -> List[Dict[str, Any]]:
commit_date_ts = subprocess.check_output(
['git', 'show', '-s', '--format=%ct', 'HEAD'],
encoding="ascii").strip()
commit_date = datetime.fromtimestamp(int(commit_date_ts))
day_before_commit = str(commit_date - timedelta(days=1)).split(' ')[0]
# something like git rev-list --before="2021-03-04" --max-count=10 --remotes="*origin/nightly"
nightly_commits = subprocess.check_output([
"git", "rev-list", f"--before={day_before_commit}", "--max-count=10",
"--remotes=*origin/nightly"
],
encoding="ascii").splitlines()
stripped_job = get_stripped_CI_job()
bucket = get_S3_bucket_readonly('ossci-metrics')
reports = []
commit_index = 0
while len(reports) == 0 and commit_index < len(nightly_commits):
nightly_commit = nightly_commits[commit_index]
print(f'Grabbing reports from nightly commit: {nightly_commit}')
summaries = bucket.objects.filter(
Prefix=f"test_time/{nightly_commit}/{stripped_job}")
for summary in summaries:
binary = summary.get()["Body"].read()
string = bz2.decompress(binary).decode("utf-8")
reports.append(json.loads(string))
commit_index += 1
return reports
def print_regressions(head_report: Report, *, num_prev_commits: int) -> None:
sha1 = os.environ.get("CIRCLE_SHA1", "HEAD")
base = subprocess.check_output(
["git", "merge-base", sha1, "origin/master"],
encoding="ascii",
).strip()
count_spec = f"{base}..{sha1}"
intermediate_commits = int(
subprocess.check_output(["git", "rev-list", "--count", count_spec],
encoding="ascii"))
ancestry_path = int(
subprocess.check_output(
["git", "rev-list", "--ancestry-path", "--count", count_spec],
encoding="ascii",
))
# if current commit is already on master, we need to exclude it from
# this history; otherwise we include the merge-base
commits = subprocess.check_output(
["git", "rev-list", f"--max-count={num_prev_commits+1}", base],
encoding="ascii",
).splitlines()
on_master = False
if base == sha1:
on_master = True
commits = commits[1:]
else:
commits = commits[:-1]
job = os.environ.get("CIRCLE_JOB", "")
bucket = get_S3_bucket_readonly('ossci-metrics')
index = {}
for commit in commits:
summaries = bucket.objects.filter(Prefix=f"test_time/{commit}/{job}/")
index[commit] = list(summaries)
objects: Dict[Commit, List[Report]] = {}
# should we do these in parallel?
for commit, summaries in index.items():
objects[commit] = []
for summary in summaries:
binary = summary.get()["Body"].read()
string = bz2.decompress(binary).decode("utf-8")
objects[commit].append(json.loads(string))
print()
print(regression_info(
head_sha=sha1,
head_report=head_report,
base_reports=objects,
job_name=job,
on_master=on_master,
ancestry_path=ancestry_path - 1,
other_ancestors=intermediate_commits - ancestry_path,
),
end="")
def run(raw: List[str]) -> Iterator[str]:
args = parse_args(raw)
commits = get_git_commit_history(path=args.pytorch, ref=args.ref)
bucket = get_S3_bucket_readonly('ossci-metrics')
return history_lines(
bucket=bucket,
commits=commits,
jobs=args.jobs,
filename=args.file,
suite_name=args.suite,
test_name=args.test,
delta=args.delta,
mode=args.mode,
sha_length=args.sha_length,
digits=args.digits,
)