def build_clickhouse(packager_cmd, logs_path): build_log_path = os.path.join(logs_path, 'build_log.log') with TeePopen(packager_cmd, build_log_path) as process: retcode = process.wait() if retcode == 0: logging.info("Built successfully") else: logging.info("Build failed") return build_log_path, retcode == 0
def build_clickhouse(packager_cmd, logs_path, build_output_path): build_log_path = os.path.join(logs_path, 'build_log.log') with TeePopen(packager_cmd, build_log_path) as process: retcode = process.wait() if os.path.exists(build_output_path): build_results = os.listdir(build_output_path) else: build_results = [] if retcode == 0: if len(build_results) != 0: logging.info("Built successfully") else: logging.info( "Success exit code, but no build artifacts => build failed" ) else: logging.info("Build failed") return build_log_path, retcode == 0 and len(build_results) > 0
def build_clickhouse(packager_cmd: str, logs_path: str, build_output_path: str) -> Tuple[str, bool]: build_log_path = os.path.join(logs_path, "build_log.log") success = False with TeePopen(packager_cmd, build_log_path) as process: retcode = process.wait() if os.path.exists(build_output_path): build_results = os.listdir(build_output_path) else: build_results = [] if retcode == 0: if len(build_results) > 0: success = True logging.info("Built successfully") else: logging.info( "Success exit code, but no build artifacts => build failed" ) else: logging.info("Build failed") return build_log_path, success
docker_image = get_image_with_version(IMAGES_PATH, 'clickhouse/codebrowser') s3_helper = S3Helper('https://s3.amazonaws.com') result_path = os.path.join(temp_path, "result_path") if not os.path.exists(result_path): os.makedirs(result_path) run_command = get_run_command(REPO_COPY, result_path, docker_image) logging.info("Going to run codebrowser: %s", run_command) run_log_path = os.path.join(temp_path, "runlog.log") with TeePopen(run_command, run_log_path) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") else: logging.info("Run failed") subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) report_path = os.path.join(result_path, "html_report") logging.info("Report path %s", report_path) s3_path_prefix = "codebrowser" html_urls = s3_helper.fast_parallel_upload_dir(report_path, s3_path_prefix, 'clickhouse-test-reports')
if args.as_root: user = "******" else: user = f"{os.geteuid()}:{os.getegid()}" run_log_path = os.path.join(test_output, "runlog.log") with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"): cmd = ( f"docker run --cap-add=SYS_PTRACE --user={user} " f"--volume='{os.getenv('SSH_AUTH_SOCK', '')}:/ssh-agent' " f"--volume={repo_path}:/repo_path --volume={test_output}:/output_path " f"-e SSH_AUTH_SOCK=/ssh-agent -e EXTRA_BUILD_ARGS='--verbose' " f"-e CLOUDFLARE_TOKEN={CLOUDFLARE_TOKEN} {docker_image}") logging.info("Running command: %s", cmd) with TeePopen(cmd, run_log_path) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") status = "success" description = "Released successfuly" else: description = "Release failed (non zero exit code)" status = "failure" logging.info("Run failed") subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) files = os.listdir(test_output) lines = [] additional_files = []
workspace, output_path, cache_path, repo_path, pr_info.number, pr_info.sha, docker_image, ) logging.info("Going to run fasttest with cmd %s", run_cmd) logs_path = os.path.join(temp_path, "fasttest-logs") if not os.path.exists(logs_path): os.makedirs(logs_path) run_log_path = os.path.join(logs_path, "runlog.log") with TeePopen(run_cmd, run_log_path, timeout=40 * 60) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") else: logging.info("Run failed") subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {cache_path}", shell=True) test_output_files = os.listdir(output_path) additional_logs = [] for f in test_output_files: additional_logs.append(os.path.join(output_path, f)) test_log_exists = (
os.makedirs(build_path) download_all_deb_packages(check_name, reports_path, build_path) my_env = get_env_for_runner(build_path, repo_path, result_path, work_path) json_path = os.path.join(work_path, 'params.json') with open(json_path, 'w', encoding='utf-8') as json_params: json_params.write(json.dumps(get_json_params_dict(check_name, pr_info, images_with_versions, run_by_hash_total, run_by_hash_num))) output_path_log = os.path.join(result_path, "main_script_log.txt") runner_path = os.path.join(repo_path, "tests/integration", "ci-runner.py") run_command = f"sudo -E {runner_path} | tee {output_path_log}" with TeePopen(run_command, output_path_log, my_env) as process: retcode = process.wait() if retcode == 0: logging.info("Run tests successfully") else: logging.info("Some tests failed") subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) state, description, test_results, additional_logs = process_results(result_path) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) s3_helper = S3Helper('https://s3.amazonaws.com') report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [output_path_log] + additional_logs, check_name_with_group, False)
popen_env.update({ "CLICKHOUSE_PERFORMANCE_COMPARISON_DATABASE_URL": f"{database_url}:9440", "CLICKHOUSE_PERFORMANCE_COMPARISON_DATABASE_USER": database_username, "CLICKHOUSE_PERFORMANCE_COMPARISON_DATABASE_USER_PASSWORD": database_password, "CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME": check_name_with_group, "CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME_PREFIX": check_name_prefix, }) run_log_path = os.path.join(temp_path, "runlog.log") with TeePopen(run_command, run_log_path, env=popen_env) as process: retcode = process.wait() if retcode == 0: logging.info("Run successfully") else: logging.info("Run failed") subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) paths = { "compare.log": os.path.join(result_path, "compare.log"), "output.7z": os.path.join(result_path, "output.7z"), "report.html":