return cmd
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
gh = Github(get_best_robot_token())
if not os.path.exists(temp_path):
os.makedirs(temp_path)
docker_image = get_image_with_version(IMAGES_PATH,
'clickhouse/codebrowser')
s3_helper = S3Helper('https://s3.amazonaws.com')
result_path = os.path.join(temp_path, "result_path")
if not os.path.exists(result_path):
os.makedirs(result_path)
run_command = get_run_command(REPO_COPY, result_path, docker_image)
logging.info("Going to run codebrowser: %s", run_command)
run_log_path = os.path.join(temp_path, "runlog.log")
with TeePopen(run_command, run_log_path) as process:
retcode = process.wait()
if retcode == 0:
gh = Github(get_best_robot_token())
rerun_helper = RerunHelper(gh, pr_info, CHECK_NAME)
if rerun_helper.is_already_finished_by_status():
logging.info(
"Check is already finished according to github status, exiting")
sys.exit(0)
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
docker_image = get_image_with_version(reports_path, DOCKER_IMAGE)
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
download_shared_build(CHECK_NAME, reports_path, packages_path)
server_log_path = os.path.join(temp_path, "server_log")
if not os.path.exists(server_log_path):
os.makedirs(server_log_path)
result_path = os.path.join(temp_path, "result_path")
if not os.path.exists(result_path):
os.makedirs(result_path)
def main():
logging.basicConfig(level=logging.INFO)
build_check_name = sys.argv[1]
build_name = sys.argv[2]
build_config = get_build_config(build_check_name, build_name)
if not os.path.exists(TEMP_PATH):
os.makedirs(TEMP_PATH)
pr_info = PRInfo()
logging.info("Repo copy path %s", REPO_COPY)
s3_helper = S3Helper("https://s3.amazonaws.com")
version = get_version_from_repo(REPO_COPY)
release_or_pr = get_release_or_pr(pr_info, build_config, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
# If this is rerun, then we try to find already created artifacts and just
# put them as github actions artifcat (result)
build_results = get_build_results_if_exists(s3_helper, s3_path_prefix)
if build_results is not None and len(build_results) > 0:
logging.info("Some build results found %s", build_results)
build_urls = []
log_url = ""
for url in build_results:
if "build_log.log" in url:
log_url = "https://s3.amazonaws.com/clickhouse-builds/" + url.replace(
"+", "%2B").replace(" ", "%20")
else:
build_urls.append(
"https://s3.amazonaws.com/clickhouse-builds/" +
url.replace("+", "%2B").replace(" ", "%20"))
create_json_artifact(
TEMP_PATH,
build_name,
log_url,
build_urls,
build_config,
0,
len(build_urls) > 0,
)
return
image_name = get_image_name(build_config)
docker_image = get_image_with_version(IMAGES_PATH, image_name)
image_version = docker_image.version
logging.info("Got version from repo %s", version.get_version_string())
version_type = "testing"
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
version_type = "stable"
update_version_local(REPO_COPY, pr_info.sha, version, version_type)
logging.info("Updated local files with version")
logging.info("Build short name %s", build_name)
build_output_path = os.path.join(TEMP_PATH, build_name)
if not os.path.exists(build_output_path):
os.makedirs(build_output_path)
ccache_path = os.path.join(CACHES_PATH, build_name + "_ccache")
logging.info("Will try to fetch cache for our build")
get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
if not os.path.exists(ccache_path):
logging.info("cache was not fetched, will create empty dir")
os.makedirs(ccache_path)
if build_config["package_type"] == "performance" and pr_info.number != 0:
# because perf tests store some information about git commits
subprocess.check_call(
f"cd {REPO_COPY} && git fetch origin master:master", shell=True)
packager_cmd = get_packager_cmd(
build_config,
os.path.join(REPO_COPY, "docker/packager"),
build_output_path,
version.get_version_string(),
image_version,
ccache_path,
pr_info,
)
logging.info("Going to run packager with %s", packager_cmd)
build_clickhouse_log = os.path.join(TEMP_PATH, "build_log")
if not os.path.exists(build_clickhouse_log):
os.makedirs(build_clickhouse_log)
start = time.time()
log_path, success = build_clickhouse(packager_cmd, build_clickhouse_log,
build_output_path)
elapsed = int(time.time() - start)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {build_output_path}",
shell=True)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}",
shell=True)
logging.info("Build finished with %s, log path %s", success, log_path)
logging.info("Will upload cache")
upload_ccache(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
if os.path.exists(log_path):
log_url = s3_helper.upload_build_file_to_s3(
log_path, s3_path_prefix + "/" + os.path.basename(log_path))
logging.info("Log url %s", log_url)
else:
logging.info("Build log doesn't exist")
build_urls = s3_helper.upload_build_folder_to_s3(
build_output_path,
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
)
logging.info("Got build URLs %s", build_urls)
print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
print("::notice ::Log URL: {}".format(log_url))
create_json_artifact(TEMP_PATH, build_name, log_url, build_urls,
build_config, elapsed, success)
upload_master_static_binaries(pr_info, build_config, s3_helper,
build_output_path)
# Fail build job if not successeded
if not success:
sys.exit(1)
temp_path = TEMP_PATH
repo_path = REPO_COPY
gh = Github(get_best_robot_token())
pr_info = PRInfo()
rerun_helper = RerunHelper(gh, pr_info, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info(
"Check is already finished according to github status, exiting")
sys.exit(0)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
docker_image = get_image_with_version(temp_path, "clickhouse/docs-release")
test_output = os.path.join(temp_path, "docs_release_log")
if not os.path.exists(test_output):
os.makedirs(test_output)
if args.as_root:
user = "******"
else:
user = f"{os.geteuid()}:{os.getegid()}"
run_log_path = os.path.join(test_output, "runlog.log")
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
cmd = (
f"docker run --cap-add=SYS_PTRACE --user={user} "
def main():
logging.basicConfig(level=logging.INFO)
build_name = sys.argv[1]
build_config = CI_CONFIG["build_config"][build_name]
if not os.path.exists(TEMP_PATH):
os.makedirs(TEMP_PATH)
pr_info = PRInfo()
logging.info("Repo copy path %s", REPO_COPY)
s3_helper = S3Helper()
version = get_version_from_repo(git=Git(True))
release_or_pr, performance_pr = get_release_or_pr(pr_info, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
# FIXME performance
s3_performance_path = "/".join(
(performance_pr, pr_info.sha, build_name, "performance.tgz"))
# If this is rerun, then we try to find already created artifacts and just
# put them as github actions artifact (result)
check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config)
docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME)
image_version = docker_image.version
logging.info("Got version from repo %s", version.string)
official_flag = pr_info.number == 0
if "official" in build_config:
official_flag = build_config["official"]
version_type = "testing"
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
version_type = "stable"
official_flag = True
update_version_local(version, version_type)
logging.info("Updated local files with version")
logging.info("Build short name %s", build_name)
build_output_path = os.path.join(TEMP_PATH, build_name)
if not os.path.exists(build_output_path):
os.makedirs(build_output_path)
ccache_path = os.path.join(CACHES_PATH, build_name + "_ccache")
logging.info("Will try to fetch cache for our build")
try:
get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number,
TEMP_PATH)
except Exception as e:
# In case there are issues with ccache, remove the path and do not fail a build
logging.info("Failed to get ccache, building without it. Error: %s", e)
rmtree(ccache_path, ignore_errors=True)
if not os.path.exists(ccache_path):
logging.info("cache was not fetched, will create empty dir")
os.makedirs(ccache_path)
packager_cmd = get_packager_cmd(
build_config,
os.path.join(REPO_COPY, "docker/packager"),
build_output_path,
version.string,
image_version,
ccache_path,
official_flag,
)
logging.info("Going to run packager with %s", packager_cmd)
logs_path = os.path.join(TEMP_PATH, "build_log")
if not os.path.exists(logs_path):
os.makedirs(logs_path)
start = time.time()
log_path, success = build_clickhouse(packager_cmd, logs_path,
build_output_path)
elapsed = int(time.time() - start)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {build_output_path}",
shell=True)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}",
shell=True)
logging.info("Build finished with %s, log path %s", success, log_path)
# Upload the ccache first to have the least build time in case of problems
logging.info("Will upload cache")
upload_ccache(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
# FIXME performance
performance_urls = []
performance_path = os.path.join(build_output_path, "performance.tgz")
if os.path.exists(performance_path):
performance_urls.append(
s3_helper.upload_build_file_to_s3(performance_path,
s3_performance_path))
logging.info(
"Uploaded performance.tgz to %s, now delete to avoid duplication",
performance_urls[0],
)
os.remove(performance_path)
build_urls = (s3_helper.upload_build_folder_to_s3(
build_output_path,
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
) + performance_urls)
logging.info("Got build URLs %s", build_urls)
print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
if os.path.exists(log_path):
log_url = s3_helper.upload_build_file_to_s3(
log_path, s3_path_prefix + "/" + os.path.basename(log_path))
logging.info("Log url %s", log_url)
else:
logging.info("Build log doesn't exist")
print(f"::notice ::Log URL: {log_url}")
create_json_artifact(TEMP_PATH, build_name, log_url, build_urls,
build_config, elapsed, success)
upload_master_static_binaries(pr_info, build_config, s3_helper,
build_output_path)
# Fail build job if not successeded
if not success:
sys.exit(1)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
caches_path = os.getenv("CACHES_PATH", temp_path)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event)
gh = Github(get_best_robot_token())
docker_image = get_image_with_version(temp_path, 'clickhouse/fasttest')
s3_helper = S3Helper('https://s3.amazonaws.com')
workspace = os.path.join(temp_path, "fasttest-workspace")
if not os.path.exists(workspace):
os.makedirs(workspace)
output_path = os.path.join(temp_path, "fasttest-output")
if not os.path.exists(output_path):
os.makedirs(output_path)
cache_path = os.path.join(caches_path, "fasttest")
if not os.path.exists(cache_path):
os.makedirs(cache_path)
gh = Github(get_best_robot_token())
if not pr_info.has_changes_in_documentation():
logging.info("No changes in documentation")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME,
description="No changes in docs",
state="success")
sys.exit(0)
logging.info("Has changes in docs")
if not os.path.exists(temp_path):
os.makedirs(temp_path)
docker_image = get_image_with_version(temp_path, 'clickhouse/docs-check')
test_output = os.path.join(temp_path, 'docs_check_log')
if not os.path.exists(test_output):
os.makedirs(test_output)
cmd = f"docker run --cap-add=SYS_PTRACE --volume={repo_path}:/repo_path --volume={test_output}:/output_path {docker_image}"
run_log_path = os.path.join(test_output, 'runlog.log')
with open(run_log_path, 'w', encoding='utf-8') as log:
with subprocess.Popen(cmd, shell=True, stderr=log,
stdout=log) as process:
retcode = process.wait()
if retcode == 0:
logging.info("Run successfully")
if not os.path.exists(temp_path):
os.makedirs(temp_path)
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
atexit.register(update_mergeable_check, gh, pr_info, check_name)
rerun_helper = RerunHelper(gh, pr_info, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = get_image_with_version(reports_path, IMAGE_NAME)
download_unit_tests(check_name, reports_path, temp_path)
tests_binary_path = os.path.join(temp_path, "unit_tests_dbms")
os.chmod(tests_binary_path, 0o777)
test_output = os.path.join(temp_path, "test_output")
if not os.path.exists(test_output):
os.makedirs(test_output)
run_command = f"docker run --cap-add=SYS_PTRACE --volume={tests_binary_path}:/unit_tests_dbms --volume={test_output}:/test_output {docker_image}"
run_log_path = os.path.join(test_output, "runlog.log")
logging.info("Going to run func tests: %s", run_command)
atexit.register(update_mergeable_check, gh, pr_info, NAME)
rerun_helper = RerunHelper(gh, pr_info, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
# Finish with the same code as previous
state = rerun_helper.get_finished_status().state # type: ignore
# state == "success" -> code = 0
code = int(state != "success")
sys.exit(code)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
docker_image = get_image_with_version(temp_path, "clickhouse/style-test")
s3_helper = S3Helper()
cmd = (
f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE "
f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output "
f"{docker_image}"
)
logging.info("Is going to run the command: %s", cmd)
subprocess.check_call(
cmd,
shell=True,
)
if args.push:
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
if not os.path.exists(temp_path):
os.makedirs(temp_path)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r',
encoding='utf-8') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event)
gh = Github(get_best_robot_token())
docker_image = get_image_with_version(temp_path, IMAGE_NAME)
build_config = get_build_config_for_check(check_name)
print(build_config)
build_config_str = build_config_to_string(build_config)
print(build_config_str)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
for url in urls:
if url.endswith('/clickhouse'):
build_url = url
break
else:
raise Exception("Cannot binary clickhouse among build results")
check_name = sys.argv[1]
if not os.path.exists(temp_path):
os.makedirs(temp_path)
pr_info = PRInfo()
gh = Github(get_best_robot_token())
rerun_helper = RerunHelper(gh, pr_info, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info(
"Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = get_image_with_version(reports_path,
"clickhouse/stress-test")
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
download_all_deb_packages(check_name, reports_path, packages_path)
server_log_path = os.path.join(temp_path, "server_log")
if not os.path.exists(server_log_path):
os.makedirs(server_log_path)
result_path = os.path.join(temp_path, "result_path")
if not os.path.exists(result_path):
os.makedirs(result_path)
state = override_status("success", check_name, validate_bugix_check)
if args.post_commit_status == "commit_status":
commit.create_status(
context=check_name_with_group,
description=NO_CHANGES_MSG,
state=state,
)
elif args.post_commit_status == "file":
fpath = os.path.join(temp_path, "post_commit_status.tsv")
post_commit_status_to_file(
fpath, description=NO_CHANGES_MSG, state=state, report_url="null"
)
sys.exit(0)
image_name = get_image_name(check_name)
docker_image = get_image_with_version(reports_path, image_name)
repo_tests_path = os.path.join(repo_path, "tests")
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
if validate_bugix_check:
download_last_release(packages_path)
else:
download_all_deb_packages(check_name, reports_path, packages_path)
server_log_path = os.path.join(temp_path, "server_log")
if not os.path.exists(server_log_path):
os.makedirs(server_log_path)
logging.info("No changes in documentation")
commit = get_commit(gh, pr_info.sha)
commit.create_status(
context=NAME, description="No changes in docs", state="success"
)
sys.exit(0)
if pr_info.has_changes_in_documentation():
logging.info("Has changes in docs")
elif args.force:
logging.info("Check the docs because of force flag")
if not os.path.exists(temp_path):
os.makedirs(temp_path)
docker_image = get_image_with_version(temp_path, "clickhouse/docs-builder")
test_output = os.path.join(temp_path, "docs_check_log")
if not os.path.exists(test_output):
os.makedirs(test_output)
cmd = (
f"docker run --cap-add=SYS_PTRACE -e GIT_DOCS_BRANCH={args.docs_branch} "
f"--volume={repo_path}:/ClickHouse --volume={test_output}:/output_path "
f"{docker_image}"
)
run_log_path = os.path.join(test_output, "runlog.log")
logging.info("Running command: '%s'", cmd)
with TeePopen(cmd, run_log_path) as process:
log_url = ''
for url in build_results:
if 'build_log.log' in url:
log_url = 'https://s3.amazonaws.com/clickhouse-builds/' + url.replace(
'+', '%2B').replace(' ', '%20')
else:
build_urls.append(
'https://s3.amazonaws.com/clickhouse-builds/' +
url.replace('+', '%2B').replace(' ', '%20'))
create_json_artifact(temp_path, build_name, log_url, build_urls,
build_config, 0,
len(build_urls) > 0)
sys.exit(0)
image_name = get_image_name(build_config)
docker_image = get_image_with_version(IMAGES_PATH, image_name)
image_version = docker_image.version
logging.info("Got version from repo %s", version.get_version_string())
version_type = 'testing'
if 'release' in pr_info.labels or 'release-lts' in pr_info.labels:
version_type = 'stable'
update_version_local(repo_path, pr_info.sha, version, version_type)
logging.info("Updated local files with version")
logging.info("Build short name %s", build_name)
build_output_path = os.path.join(temp_path, build_name)
log_url = ''
for url in build_results:
if 'build_log.log' in url:
log_url = 'https://s3.amazonaws.com/clickhouse-builds/' + url.replace(
'+', '%2B').replace(' ', '%20')
else:
build_urls.append(
'https://s3.amazonaws.com/clickhouse-builds/' +
url.replace('+', '%2B').replace(' ', '%20'))
create_json_artifact(temp_path, build_name, log_url, build_urls,
build_config, 0,
len(build_urls) > 0)
sys.exit(0)
image_name = get_image_name(build_config)
docker_image = get_image_with_version(os.getenv("IMAGES_PATH"), image_name)
image_version = docker_image.version
logging.info("Got version from repo %s", version.get_version_string())
version_type = 'testing'
if 'release' in pr_info.labels or 'release-lts' in pr_info.labels:
version_type = 'stable'
update_version_local(repo_path, pr_info.sha, version, version_type)
logging.info("Updated local files with version")
logging.info("Build short name %s", build_name)
build_output_path = os.path.join(temp_path, build_name)
