def build_clickhouse(packager_cmd, logs_path):
build_log_path = os.path.join(logs_path, 'build_log.log')
with TeePopen(packager_cmd, build_log_path) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Built successfully")
else:
logging.info("Build failed")
return build_log_path, retcode == 0
def build_clickhouse(packager_cmd, logs_path, build_output_path):
build_log_path = os.path.join(logs_path, 'build_log.log')
with TeePopen(packager_cmd, build_log_path) as process:
retcode = process.wait()
if os.path.exists(build_output_path):
build_results = os.listdir(build_output_path)
else:
build_results = []
if retcode == 0:
if len(build_results) != 0:
logging.info("Built successfully")
else:
logging.info(
"Success exit code, but no build artifacts => build failed"
)
else:
logging.info("Build failed")
return build_log_path, retcode == 0 and len(build_results) > 0
def build_clickhouse(packager_cmd: str, logs_path: str,
build_output_path: str) -> Tuple[str, bool]:
build_log_path = os.path.join(logs_path, "build_log.log")
success = False
with TeePopen(packager_cmd, build_log_path) as process:
retcode = process.wait()
if os.path.exists(build_output_path):
build_results = os.listdir(build_output_path)
else:
build_results = []
if retcode == 0:
if len(build_results) > 0:
success = True
logging.info("Built successfully")
else:
logging.info(
"Success exit code, but no build artifacts => build failed"
)
else:
logging.info("Build failed")
return build_log_path, success
docker_image = get_image_with_version(IMAGES_PATH,
'clickhouse/codebrowser')
s3_helper = S3Helper('https://s3.amazonaws.com')
result_path = os.path.join(temp_path, "result_path")
if not os.path.exists(result_path):
os.makedirs(result_path)
run_command = get_run_command(REPO_COPY, result_path, docker_image)
logging.info("Going to run codebrowser: %s", run_command)
run_log_path = os.path.join(temp_path, "runlog.log")
with TeePopen(run_command, run_log_path) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run successfully")
else:
logging.info("Run failed")
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}",
shell=True)
report_path = os.path.join(result_path, "html_report")
logging.info("Report path %s", report_path)
s3_path_prefix = "codebrowser"
html_urls = s3_helper.fast_parallel_upload_dir(report_path, s3_path_prefix,
'clickhouse-test-reports')
if args.as_root:
user = "******"
else:
user = f"{os.geteuid()}:{os.getegid()}"
run_log_path = os.path.join(test_output, "runlog.log")
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
cmd = (
f"docker run --cap-add=SYS_PTRACE --user={user} "
f"--volume='{os.getenv('SSH_AUTH_SOCK', '')}:/ssh-agent' "
f"--volume={repo_path}:/repo_path --volume={test_output}:/output_path "
f"-e SSH_AUTH_SOCK=/ssh-agent -e EXTRA_BUILD_ARGS='--verbose' "
f"-e CLOUDFLARE_TOKEN={CLOUDFLARE_TOKEN} {docker_image}")
logging.info("Running command: %s", cmd)
with TeePopen(cmd, run_log_path) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run successfully")
status = "success"
description = "Released successfuly"
else:
description = "Release failed (non zero exit code)"
status = "failure"
logging.info("Run failed")
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}",
shell=True)
files = os.listdir(test_output)
lines = []
additional_files = []
workspace,
output_path,
cache_path,
repo_path,
pr_info.number,
pr_info.sha,
docker_image,
)
logging.info("Going to run fasttest with cmd %s", run_cmd)
logs_path = os.path.join(temp_path, "fasttest-logs")
if not os.path.exists(logs_path):
os.makedirs(logs_path)
run_log_path = os.path.join(logs_path, "runlog.log")
with TeePopen(run_cmd, run_log_path, timeout=40 * 60) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run successfully")
else:
logging.info("Run failed")
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {cache_path}", shell=True)
test_output_files = os.listdir(output_path)
additional_logs = []
for f in test_output_files:
additional_logs.append(os.path.join(output_path, f))
test_log_exists = (
os.makedirs(build_path)
download_all_deb_packages(check_name, reports_path, build_path)
my_env = get_env_for_runner(build_path, repo_path, result_path, work_path)
json_path = os.path.join(work_path, 'params.json')
with open(json_path, 'w', encoding='utf-8') as json_params:
json_params.write(json.dumps(get_json_params_dict(check_name, pr_info, images_with_versions, run_by_hash_total, run_by_hash_num)))
output_path_log = os.path.join(result_path, "main_script_log.txt")
runner_path = os.path.join(repo_path, "tests/integration", "ci-runner.py")
run_command = f"sudo -E {runner_path} | tee {output_path_log}"
with TeePopen(run_command, output_path_log, my_env) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run tests successfully")
else:
logging.info("Some tests failed")
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
state, description, test_results, additional_logs = process_results(result_path)
ch_helper = ClickHouseHelper()
mark_flaky_tests(ch_helper, check_name, test_results)
s3_helper = S3Helper('https://s3.amazonaws.com')
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [output_path_log] + additional_logs, check_name_with_group, False)
popen_env.update({
"CLICKHOUSE_PERFORMANCE_COMPARISON_DATABASE_URL":
f"{database_url}:9440",
"CLICKHOUSE_PERFORMANCE_COMPARISON_DATABASE_USER":
database_username,
"CLICKHOUSE_PERFORMANCE_COMPARISON_DATABASE_USER_PASSWORD":
database_password,
"CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME":
check_name_with_group,
"CLICKHOUSE_PERFORMANCE_COMPARISON_CHECK_NAME_PREFIX":
check_name_prefix,
})
run_log_path = os.path.join(temp_path, "runlog.log")
with TeePopen(run_command, run_log_path, env=popen_env) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run successfully")
else:
logging.info("Run failed")
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}",
shell=True)
paths = {
"compare.log":
os.path.join(result_path, "compare.log"),
"output.7z":
os.path.join(result_path, "output.7z"),
"report.html":