def assemble_flaky_test_stats(
duplicated_tests_by_file: Dict[str, DuplicatedDict]) -> Any:
flaky_tests = []
workflow_id = os.environ.get("GITHUB_RUN_ID",
os.environ.get("CIRCLE_WORKFLOW_ID", None))
for file_name, suite_to_dict in duplicated_tests_by_file.items():
for suite_name, testcase_to_runs in suite_to_dict.items():
for testcase_name, list_of_runs in testcase_to_runs.items():
num_green, num_red = process_intentional_test_runs(
list_of_runs)
if num_green > 0: # Otherwise, it's likely just a failing test
flaky_tests.append({
"name": testcase_name,
"suite": suite_name,
"file": file_name,
"num_green": num_green,
"num_red": num_red,
})
if len(flaky_tests) > 0:
# write to RDS
register_rds_schema("flaky_tests", schema_from_sample(flaky_tests[0]))
rds_write("flaky_tests", flaky_tests, only_on_master=False)
# write to S3 to go to Rockset as well
import uuid
for flaky_test in flaky_tests:
flaky_test["workflow_id"] = workflow_id
key = f"flaky_tests/{workflow_id}/{uuid.uuid4()}.json"
obj = get_S3_object_from_bucket("ossci-raw-job-status", key)
obj.put(Body=json.dumps(flaky_test),
ContentType="application/json")
def assemble_flaky_test_stats(duplicated_tests_by_file: Dict[str, DuplicatedDict]) -> Any:
flaky_tests = []
for file_name, suite_to_dict in duplicated_tests_by_file.items():
for suite_name, testcase_to_runs in suite_to_dict.items():
for testcase_name, list_of_runs in testcase_to_runs.items():
num_green, num_red = process_intentional_test_runs(list_of_runs)
if num_green > 0: # Otherwise, it's likely just a failing test
flaky_tests.append({
"name": testcase_name,
"suite": suite_name,
"file": file_name,
"num_green": num_green,
"num_red": num_red
})
if len(flaky_tests) > 0:
register_rds_schema("flaky_tests", schema_from_sample(flaky_tests[0]))
rds_write("flaky_tests", flaky_tests, only_on_master=False)
for suite in file.test_suites.values():
for case in suite.test_cases.values():
if case.errored or case.failed:
failures.append({
"name":
case.name,
"suite":
suite.name,
"file":
file.name,
"status":
"failure" if case.failed else "error"
})
if len(failures) > 0:
register_rds_schema("test_failures", schema_from_sample(failures[0]))
rds_write("test_failures", failures, only_on_master=False)
def print_regressions(head_report: Report, *, num_prev_commits: int) -> None:
sha1 = os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1", "HEAD"))
base = get_base_commit(sha1)
count_spec = f"{base}..{sha1}"
intermediate_commits = int(
subprocess.check_output(["git", "rev-list", "--count", count_spec],
encoding="ascii"))
ancestry_path = int(
subprocess.check_output(
["git", "rev-list", "--ancestry-path", "--count", count_spec],
file_dir = os.environ.get(
"PYTORCH_FINAL_PACKAGE_DIR", "/home/circleci/project/final_pkgs"
)
if len(sys.argv) == 2:
file_dir = sys.argv[1]
if os.getenv("IS_GHA", "0") == "1":
sample_lib = {
"library": "abcd",
"size": 1234,
}
sample_data = {
**base_data(),
**sample_lib,
}
register_rds_schema("binary_size", schema_from_sample(sample_data))
if "-android" in os.environ.get("BUILD_ENVIRONMENT", ""):
report_android_sizes(file_dir)
else:
if os.getenv("IS_GHA", "0") == "1":
build_path = pathlib.Path("build") / "lib"
libraries = [
(path.name, os.stat(path).st_size) for path in build_path.glob("*")
]
data = []
for name, size in libraries:
if name.strip() == "":
continue
library_data = {
"library": name,
max_rss = int(result.stdout.decode().strip())
return {
"test_name": name,
"peak_memory_bytes": max_rss,
}
data = profile("torch", "pytorch")
baseline = profile("sys", "baseline")
try:
rds_write("import_stats", [data, baseline])
except Exception as e:
raise unittest.SkipTest(f"Failed to record import_stats: {e}")
if __name__ == "__main__":
if register_rds_schema and IS_IN_CI:
try:
register_rds_schema(
"import_stats",
{
"test_name": "string",
"peak_memory_bytes": "int",
"time_ms": "int",
},
)
except Exception as e:
print(f"Failed to register RDS schema: {e}")
run_tests()
"average_cache_read_hit",
"failed_distributed_compilations",
}
if __name__ == "__main__":
if os.getenv("IS_GHA", "0") == "1":
data = {}
if len(sys.argv) == 2:
with open(sys.argv[1]) as f:
lines = f.readlines()
else:
lines = list(fileinput.input())
for line in lines:
line = line.strip()
values = [x.strip() for x in line.split(" ")]
values = [x for x in values if x != ""]
if len(values) == 2:
name = get_name(values[0])
if name in STAT_NAMES:
data[name] = parse_value(values[1])
# The data from sccache is always the same so this should be fine, if it
# ever changes we will probably need to break this out so the fields
# we want are hardcoded
register_rds_schema("sccache_stats", schema_from_sample(data))
rds_write("sccache_stats", [data])
sprint("Wrote sccache stats to DB")
else:
sprint("Not in GitHub Actions, skipping")
