else: pr_link = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_info.number}" task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}" docker_env += ' -e CHPC_ADD_REPORT_LINKS="<a href={}>Job (actions)</a> <a href={}>Tested commit</a>"'.format( task_url, pr_link) if 'RUN_BY_HASH_TOTAL' in os.environ: run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL')) run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) docker_env += f' -e CHPC_TEST_RUN_BY_HASH_TOTAL={run_by_hash_total} -e CHPC_TEST_RUN_BY_HASH_NUM={run_by_hash_num}' check_name_with_group = check_name + f' [{run_by_hash_num + 1}/{run_by_hash_total}]' else: check_name_with_group = check_name rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info( "Check is already finished according to github status, exiting") sys.exit(0) docker_image = get_image_with_version(reports_path, IMAGE_NAME) #with RamDrive(ramdrive_path, ramdrive_size): result_path = ramdrive_path if not os.path.exists(result_path): os.makedirs(result_path) run_command = get_run_command(result_path, result_path, repo_tests_path, pr_info.number, pr_info.sha, docker_env, docker_image)
return state, description, test_results, additional_files if __name__ == "__main__": logging.basicConfig(level=logging.INFO) stopwatch = Stopwatch() repo_path = GITHUB_WORKSPACE temp_path = os.path.join(RUNNER_TEMP, "style_check") pr_info = PRInfo() gh = Github(get_best_robot_token()) rerun_helper = RerunHelper(gh, pr_info, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) if not os.path.exists(temp_path): os.makedirs(temp_path) docker_image = get_image_with_version(temp_path, "clickhouse/style-test") s3_helper = S3Helper("https://s3.amazonaws.com") subprocess.check_output( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " f"{docker_image}", shell=True,
def main(): logging.basicConfig(level=logging.INFO) temp_path = TEMP_PATH logging.info("Reports path %s", REPORTS_PATH) if not os.path.exists(temp_path): os.makedirs(temp_path) build_check_name = sys.argv[1] needs_data = None required_builds = 0 if os.path.exists(NEEDS_DATA_PATH): with open(NEEDS_DATA_PATH, "rb") as file_handler: needs_data = json.load(file_handler) required_builds = len(needs_data) logging.info("The next builds are required: %s", ", ".join(needs_data)) gh = Github(get_best_robot_token()) pr_info = PRInfo() rerun_helper = RerunHelper(gh, pr_info, build_check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) builds_for_check = CI_CONFIG["builds_report_config"][build_check_name] required_builds = required_builds or len(builds_for_check) # Collect reports from json artifacts builds_report_map = {} for root, _, files in os.walk(REPORTS_PATH): for f in files: if f.startswith("build_urls_") and f.endswith(".json"): logging.info("Found build report json %s", f) build_name = get_build_name_from_file_name(f) if build_name in builds_for_check: with open(os.path.join(root, f), "rb") as file_handler: builds_report_map[build_name] = json.load(file_handler) else: logging.info( "Skipping report %s for build %s, it's not in our reports list", f, build_name, ) # Sort reports by config order build_reports = [ builds_report_map[build_name] for build_name in builds_for_check if build_name in builds_report_map ] some_builds_are_missing = len(build_reports) < required_builds missing_build_names = [] if some_builds_are_missing: logging.warning( "Expected to get %s build results, got only %s", required_builds, len(build_reports), ) missing_build_names = [ name for name in needs_data if not any(rep for rep in build_reports if rep["job_name"] == name) ] else: logging.info("Got exactly %s builds", len(builds_report_map)) # Group build artifacts by groups build_results = [] # type: List[BuildResult] build_artifacts = [] # build_logs = [] for build_report in build_reports: build_result, build_artifacts_url, build_logs_url = process_report(build_report) logging.info( "Got %s artifact groups for build report report", len(build_result) ) build_results.extend(build_result) build_artifacts.extend(build_artifacts_url) build_logs.extend(build_logs_url) for failed_job in missing_build_names: build_result, build_artifacts_url, build_logs_url = get_failed_report( failed_job ) build_results.extend(build_result) build_artifacts.extend(build_artifacts_url) build_logs.extend(build_logs_url) total_groups = len(build_results) logging.info("Totally got %s artifact groups", total_groups) if total_groups == 0: logging.error("No success builds, failing check") sys.exit(1) s3_helper = S3Helper("https://s3.amazonaws.com") branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master" branch_name = "master" if pr_info.number != 0: branch_name = f"PR #{pr_info.number}" branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_info.number}" commit_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commit/{pr_info.sha}" task_url = GITHUB_RUN_URL report = create_build_html_report( build_check_name, build_results, build_logs, build_artifacts, task_url, branch_url, branch_name, commit_url, ) report_path = os.path.join(temp_path, "report.html") with open(report_path, "w", encoding="utf-8") as fd: fd.write(report) logging.info("Going to upload prepared report") context_name_for_path = build_check_name.lower().replace(" ", "_") s3_path_prefix = ( str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path ) url = s3_helper.upload_build_file_to_s3( report_path, s3_path_prefix + "/report.html" ) logging.info("Report url %s", url) print(f"::notice ::Report url: {url}") # Prepare a commit status ok_groups = 0 summary_status = "success" for build_result in build_results: if build_result.status == "failure" and summary_status != "error": summary_status = "failure" if build_result.status == "error" or not build_result.status: summary_status = "error" if build_result.status == "success": ok_groups += 1 if ok_groups == 0 or some_builds_are_missing: summary_status = "error" addition = "" if some_builds_are_missing: addition = f"({len(build_reports)} of {required_builds} builds are OK)" description = f"{ok_groups}/{total_groups} artifact groups are OK {addition}" commit = get_commit(gh, pr_info.sha) commit.create_status( context=build_check_name, description=description, state=summary_status, target_url=url, ) if summary_status == "error": sys.exit(1)
if __name__ == "__main__": logging.basicConfig(level=logging.INFO) reports_path = os.getenv("REPORTS_PATH", "./reports") temp_path = os.path.join(os.getenv("TEMP_PATH", ".")) logging.info("Reports path %s", reports_path) if not os.path.exists(temp_path): os.makedirs(temp_path) build_check_name = sys.argv[1] gh = Github(get_best_robot_token()) pr_info = PRInfo(get_event()) rerun_helper = RerunHelper(gh, pr_info, build_check_name) if rerun_helper.is_already_finished_by_status(): logging.info( "Check is already finished according to github status, exiting") sys.exit(0) reports_order = CI_CONFIG["builds_report_config"][build_check_name] logging.info("My reports list %s", reports_order) build_reports_map = {} for root, dirs, files in os.walk(reports_path): for f in files: if f.startswith("build_urls_") and f.endswith('.json'): logging.info("Found build report json %s", f) build_name = get_build_name_from_file_name(f) if build_name in reports_order:
logging.basicConfig(level=logging.INFO) logging.getLogger("git_helper").setLevel(logging.DEBUG) args = parse_args() stopwatch = Stopwatch() repo_path = GITHUB_WORKSPACE temp_path = os.path.join(RUNNER_TEMP, "style_check") pr_info = PRInfo() if args.push: checkout_head(pr_info) gh = GitHub(get_best_robot_token()) rerun_helper = RerunHelper(gh, pr_info, NAME) if rerun_helper.is_already_finished_by_status(): logging.info( "Check is already finished according to github status, exiting") # Finish with the same code as previous state = rerun_helper.get_finished_status().state # type: ignore # state == "success" -> code = 0 code = int(state != "success") sys.exit(code) if not os.path.exists(temp_path): os.makedirs(temp_path) docker_image = get_image_with_version(temp_path, "clickhouse/style-test") s3_helper = S3Helper("https://s3.amazonaws.com")