def assemble_flaky_test_stats( duplicated_tests_by_file: Dict[str, DuplicatedDict]) -> Any: flaky_tests = [] workflow_id = os.environ.get("GITHUB_RUN_ID", os.environ.get("CIRCLE_WORKFLOW_ID", None)) for file_name, suite_to_dict in duplicated_tests_by_file.items(): for suite_name, testcase_to_runs in suite_to_dict.items(): for testcase_name, list_of_runs in testcase_to_runs.items(): num_green, num_red = process_intentional_test_runs( list_of_runs) if num_green > 0: # Otherwise, it's likely just a failing test flaky_tests.append({ "name": testcase_name, "suite": suite_name, "file": file_name, "num_green": num_green, "num_red": num_red, }) if len(flaky_tests) > 0: # write to RDS register_rds_schema("flaky_tests", schema_from_sample(flaky_tests[0])) rds_write("flaky_tests", flaky_tests, only_on_master=False) # write to S3 to go to Rockset as well import uuid for flaky_test in flaky_tests: flaky_test["workflow_id"] = workflow_id key = f"flaky_tests/{workflow_id}/{uuid.uuid4()}.json" obj = get_S3_object_from_bucket("ossci-raw-job-status", key) obj.put(Body=json.dumps(flaky_test), ContentType="application/json")
def assemble_flaky_test_stats(duplicated_tests_by_file: Dict[str, DuplicatedDict]) -> Any: flaky_tests = [] for file_name, suite_to_dict in duplicated_tests_by_file.items(): for suite_name, testcase_to_runs in suite_to_dict.items(): for testcase_name, list_of_runs in testcase_to_runs.items(): num_green, num_red = process_intentional_test_runs(list_of_runs) if num_green > 0: # Otherwise, it's likely just a failing test flaky_tests.append({ "name": testcase_name, "suite": suite_name, "file": file_name, "num_green": num_green, "num_red": num_red }) if len(flaky_tests) > 0: register_rds_schema("flaky_tests", schema_from_sample(flaky_tests[0])) rds_write("flaky_tests", flaky_tests, only_on_master=False)
def test_peak_memory(self): def profile(module, name): command = f"import {module}; import resource; print(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)" result = subprocess.run( [sys.executable, "-c", command], stdout=subprocess.PIPE, ) max_rss = int(result.stdout.decode().strip()) return { "test_name": name, "peak_memory_bytes": max_rss, } data = profile("torch", "pytorch") baseline = profile("sys", "baseline") rds_write( "import_stats", [data, baseline] )
for case in suite.test_cases.values(): if case.errored or case.failed: failures.append({ "name": case.name, "suite": suite.name, "file": file.name, "status": "failure" if case.failed else "error" }) if len(failures) > 0: register_rds_schema("test_failures", schema_from_sample(failures[0])) rds_write("test_failures", failures, only_on_master=False) def print_regressions(head_report: Report, *, num_prev_commits: int) -> None: sha1 = os.environ.get("SHA1", os.environ.get("CIRCLE_SHA1", "HEAD")) base = get_base_commit(sha1) count_spec = f"{base}..{sha1}" intermediate_commits = int( subprocess.check_output(["git", "rev-list", "--count", count_spec], encoding="ascii")) ancestry_path = int( subprocess.check_output( ["git", "rev-list", "--ancestry-path", "--count", count_spec], encoding="ascii",
**sample_lib, } register_rds_schema("binary_size", schema_from_sample(sample_data)) if "-android" in os.environ.get("BUILD_ENVIRONMENT", ""): report_android_sizes(file_dir) else: if os.getenv("IS_GHA", "0") == "1": build_path = pathlib.Path("build") / "lib" libraries = [ (path.name, os.stat(path).st_size) for path in build_path.glob("*") ] data = [] for name, size in libraries: if name.strip() == "": continue library_data = { "library": name, "size": size, } data.append({**base_data(), **library_data}) rds_write("binary_size", data) else: print("checking dir: " + file_dir) size = get_size(file_dir) # Sending the message anyway if no size info is collected. try: send_message([build_message(size)]) except Exception: logging.exception("can't send message")
"average_cache_read_hit", "failed_distributed_compilations", } if __name__ == "__main__": if os.getenv("IS_GHA", "0") == "1": data = {} if len(sys.argv) == 2: with open(sys.argv[1]) as f: lines = f.readlines() else: lines = list(fileinput.input()) for line in lines: line = line.strip() values = [x.strip() for x in line.split(" ")] values = [x for x in values if x != ""] if len(values) == 2: name = get_name(values[0]) if name in STAT_NAMES: data[name] = parse_value(values[1]) # The data from sccache is always the same so this should be fine, if it # ever changes we will probably need to break this out so the fields # we want are hardcoded register_rds_schema("sccache_stats", schema_from_sample(data)) rds_write("sccache_stats", [data]) sprint("Wrote sccache stats to DB") else: sprint("Not in GitHub Actions, skipping")