def test_get_changed_docker_images(self):
pr_info = PRInfo(PRInfo.default_event.copy())
pr_info.changed_files = {
"docker/test/stateless",
"docker/test/base",
"docker/docs/builder",
}
images = sorted(
list(
di.get_changed_docker_images(pr_info, "/",
self.docker_images_path)))
self.maxDiff = None
expected = sorted([
di.DockerImage("docker/test/base", "clickhouse/test-base"),
di.DockerImage("docker/docs/builder", "clickhouse/docs-builder"),
di.DockerImage(
"docker/test/stateless",
"clickhouse/stateless-test",
"clickhouse/test-base",
),
di.DockerImage(
"docker/test/integration/base",
"clickhouse/integration-test",
"clickhouse/test-base",
),
di.DockerImage("docker/test/fuzzer", "clickhouse/fuzzer",
"clickhouse/test-base"),
di.DockerImage(
"docker/test/keeper-jepsen",
"clickhouse/keeper-jepsen-test",
"clickhouse/test-base",
),
di.DockerImage(
"docker/docs/check",
"clickhouse/docs-check",
"clickhouse/docs-builder",
),
di.DockerImage(
"docker/docs/release",
"clickhouse/docs-release",
"clickhouse/docs-builder",
),
di.DockerImage(
"docker/test/stateful",
"clickhouse/stateful-test",
"clickhouse/stateless-test",
),
di.DockerImage(
"docker/test/unit",
"clickhouse/unit-test",
"clickhouse/stateless-test",
),
di.DockerImage(
"docker/test/stress",
"clickhouse/stress-test",
"clickhouse/stateful-test",
),
])
self.assertEqual(images, expected)
def test_gen_version(self):
pr_info = PRInfo(PRInfo.default_event.copy())
versions, result_version = di.gen_versions(pr_info, None)
self.assertEqual(versions, ["latest", "0", "0-HEAD"])
self.assertEqual(result_version, "0-HEAD")
versions, result_version = di.gen_versions(pr_info, "suffix")
self.assertEqual(versions, ["latest-suffix", "0-suffix", "0-HEAD-suffix"])
self.assertEqual(result_version, versions)
pr_info.number = 1
versions, result_version = di.gen_versions(pr_info, None)
self.assertEqual(versions, ["1", "1-HEAD"])
self.assertEqual(result_version, "1-HEAD")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
if not os.path.exists(temp_path):
os.makedirs(temp_path)
is_flaky_check = 'flaky' in check_name
pr_info = PRInfo(get_event(), need_changed_files=is_flaky_check)
gh = Github(get_best_robot_token())
images = get_images_with_versions(temp_path, IMAGES)
images_with_versions = {i.name: i.version for i in images}
result_path = os.path.join(temp_path, "output_dir")
if not os.path.exists(result_path):
os.makedirs(result_path)
work_path = os.path.join(temp_path, "workdir")
if not os.path.exists(work_path):
os.makedirs(work_path)
build_path = os.path.join(temp_path, "build")
if not os.path.exists(build_path):
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
args = parse_args()
if args.suffix:
global NAME
NAME += f" {args.suffix}"
changed_json = os.path.join(TEMP_PATH,
f"changed_images_{args.suffix}.json")
else:
changed_json = os.path.join(TEMP_PATH, "changed_images.json")
if args.push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
encoding="utf-8",
shell=True,
)
if os.path.exists(TEMP_PATH):
shutil.rmtree(TEMP_PATH)
os.makedirs(TEMP_PATH)
images_dict = get_images_dict(GITHUB_WORKSPACE, "docker/images.json")
pr_info = PRInfo()
if args.all:
pr_info.changed_files = set(images_dict.keys())
elif args.image_path:
pr_info.changed_files = set(i for i in args.image_path)
else:
try:
pr_info.fetch_changed_files()
except TypeError:
# If the event does not contain diff, nothing will be built
pass
changed_images = get_changed_docker_images(pr_info, images_dict)
if changed_images:
logging.info("Has changed images: %s",
", ".join([im.path for im in changed_images]))
image_versions, result_version = gen_versions(pr_info, args.suffix)
result_images = {}
images_processing_result = []
for image in changed_images:
# If we are in backport PR, then pr_info.release_pr is defined
# We use it as tag to reduce rebuilding time
images_processing_result += process_image_with_parents(
image, image_versions, pr_info.release_pr, args.push)
result_images[image.repo] = result_version
if changed_images:
description = "Updated " + ",".join([im.repo for im in changed_images])
else:
description = "Nothing to update"
if len(description) >= 140:
description = description[:136] + "..."
with open(changed_json, "w", encoding="utf-8") as images_file:
json.dump(result_images, images_file)
s3_helper = S3Helper()
s3_path_prefix = (str(pr_info.number) + "/" + pr_info.sha + "/" +
NAME.lower().replace(" ", "_"))
status, test_results = process_test_results(s3_helper,
images_processing_result,
s3_path_prefix)
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results,
[], NAME)
print(f"::notice ::Report url: {url}")
print(f'::set-output name=url_output::"{url}"')
if not args.reports:
return
gh = Github(get_best_robot_token(), per_page=100)
post_commit_status(gh, pr_info.sha, NAME, description, status, url)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
ch_helper.insert_events_into(db="default",
table="checks",
events=prepared_events)
if status == "error":
sys.exit(1)
temp_path = TEMP_PATH
repo_path = REPO_COPY
reports_path = REPORTS_PATH
args = parse_args()
check_name = args.check_name
kill_timeout = args.kill_timeout
validate_bugix_check = args.validate_bugfix
flaky_check = "flaky" in check_name.lower()
run_changed_tests = flaky_check or validate_bugix_check
gh = Github(get_best_robot_token())
pr_info = PRInfo(need_changed_files=run_changed_tests)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
if validate_bugix_check and "pr-bugfix" not in pr_info.labels:
if args.post_commit_status == "file":
post_commit_status_to_file(
os.path.join(temp_path, "post_commit_status.tsv"),
"Skipped (no pr-bugfix)",
"success",
"null",
)
logging.info("Skipping '%s' (no pr-bugfix)", check_name)
sys.exit(0)
return state, description, test_results, additional_files
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
caches_path = os.getenv("CACHES_PATH", temp_path)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event)
gh = Github(get_best_robot_token())
docker_image = get_image_with_version(temp_path, 'clickhouse/fasttest')
s3_helper = S3Helper('https://s3.amazonaws.com')
workspace = os.path.join(temp_path, "fasttest-workspace")
if not os.path.exists(workspace):
os.makedirs(workspace)
output_path = os.path.join(temp_path, "fasttest-output")
if not os.path.exists(output_path):
os.makedirs(output_path)
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
args = parse_args()
if args.suffix:
global NAME
NAME += f" {args.suffix}"
changed_json = os.path.join(TEMP_PATH,
f"changed_images_{args.suffix}.json")
else:
changed_json = os.path.join(TEMP_PATH, "changed_images.json")
push = not args.no_push_images
if push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
encoding="utf-8",
shell=True,
)
if os.path.exists(TEMP_PATH):
shutil.rmtree(TEMP_PATH)
os.makedirs(TEMP_PATH)
if args.image_path:
pr_info = PRInfo()
pr_info.changed_files = set(i for i in args.image_path)
else:
pr_info = PRInfo(need_changed_files=True)
changed_images = get_changed_docker_images(pr_info, GITHUB_WORKSPACE,
"docker/images.json")
logging.info("Has changed images %s",
", ".join([im.path for im in changed_images]))
image_versions, result_version = gen_versions(pr_info, args.suffix)
result_images = {}
images_processing_result = []
for image in changed_images:
images_processing_result += process_image_with_parents(
image, image_versions, push)
result_images[image.repo] = result_version
if changed_images:
description = "Updated " + ",".join([im.repo for im in changed_images])
else:
description = "Nothing to update"
if len(description) >= 140:
description = description[:136] + "..."
with open(changed_json, "w", encoding="utf-8") as images_file:
json.dump(result_images, images_file)
s3_helper = S3Helper("https://s3.amazonaws.com")
s3_path_prefix = (str(pr_info.number) + "/" + pr_info.sha + "/" +
NAME.lower().replace(" ", "_"))
status, test_results = process_test_results(s3_helper,
images_processing_result,
s3_path_prefix)
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results,
[], NAME)
print(f"::notice ::Report url: {url}")
print(f'::set-output name=url_output::"{url}"')
if args.no_reports:
return
gh = Github(get_best_robot_token())
post_commit_status(gh, pr_info.sha, NAME, description, status, url)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
ch_helper.insert_events_into(db="gh-data",
table="checks",
events=prepared_events)
repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
temp_path = os.path.join(
os.getenv("RUNNER_TEMP", os.path.abspath("./temp")),
'docker_images_check')
dockerhub_password = get_parameter_from_ssm('dockerhub_robot_password')
if os.path.exists(temp_path):
shutil.rmtree(temp_path)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event, False, True)
changed_images, dockerhub_repo_name = get_changed_docker_images(
pr_info, repo_path, "docker/images.json")
logging.info("Has changed images %s",
', '.join([str(image[0]) for image in changed_images]))
pr_commit_version = str(pr_info.number) + '-' + pr_info.sha
versions = [str(pr_info.number), pr_commit_version]
if pr_info.number == 0:
versions.append("latest")
subprocess.check_output(
"docker login --username 'robotclickhouse' --password '{}'".format(
dockerhub_password),
shell=True)
result_images = {}
r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)",
category,
):
return "", category
if not entry:
return f"Changelog entry required for category '{category}'", category
return "", category
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
pr_info = PRInfo(need_orgs=True,
pr_event_from_api=True,
need_changed_files=True)
can_run, description, labels_state = should_run_checks_for_pr(pr_info)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
description_error, category = check_pr_description(pr_info)
pr_labels_to_add = []
pr_labels_to_remove = []
if (category in CATEGORY_TO_LABEL
and CATEGORY_TO_LABEL[category] not in pr_info.labels):
pr_labels_to_add.append(CATEGORY_TO_LABEL[category])
for label in pr_info.labels:
if (label in CATEGORY_TO_LABEL.values()
and category in CATEGORY_TO_LABEL
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'docker_images_check')
dockerhub_password = get_parameter_from_ssm('dockerhub_robot_password')
if os.path.exists(temp_path):
shutil.rmtree(temp_path)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
pr_info = PRInfo(get_event(), need_changed_files=True)
changed_images, dockerhub_repo_name = get_changed_docker_images(pr_info, repo_path, "docker/images.json")
logging.info("Has changed images %s", ', '.join([str(image[0]) for image in changed_images]))
pr_commit_version = str(pr_info.number) + '-' + pr_info.sha
versions = [str(pr_info.number), pr_commit_version]
if pr_info.number == 0:
versions.append("latest")
subprocess.check_output("docker login --username 'robotclickhouse' --password '{}'".format(dockerhub_password), shell=True)
result_images = {}
images_processing_result = []
for rel_path, image_name in changed_images:
full_path = os.path.join(repo_path, rel_path)
images_processing_result += process_single_image(versions, full_path, image_name)
result_images[image_name] = pr_commit_version
return True, "No special conditions apply"
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event, need_orgs=True)
can_run, description = should_run_checks_for_pr(pr_info)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
if not can_run:
print("::notice ::Cannot run")
commit.create_status(context=NAME,
description=description,
state="failure",
target_url=url)
sys.exit(1)
else:
if 'pr-documentation' in pr_info.labels or 'pr-doc-fix' in pr_info.labels:
commit.create_status(
context=NAME,
def filter_statuses(statuses):
"""
Squash statuses to latest state
1. context="first", state="success", update_time=1
2. context="second", state="success", update_time=2
3. context="first", stat="failure", update_time=3
=========>
1. context="second", state="success"
2. context="first", stat="failure"
"""
filt = {}
for status in sorted(statuses, key=lambda x: x.updated_at):
filt[status.context] = status
return filt
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
pr_info = PRInfo(get_event(), need_orgs=True)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
url = f"{os.getenv('GITHUB_SERVER_URL')}/{os.getenv('GITHUB_REPOSITORY')}/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
statuses = filter_statuses(list(commit.get_statuses()))
if NAME in statuses and statuses[NAME].state == "pending":
commit.create_status(context=NAME,
description="All checks finished",
state="success",
target_url=url)
)
parser.add_argument(
"--force",
action="store_true",
help="check the docs even if there no changes",
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = TEMP_PATH
repo_path = REPO_COPY
pr_info = PRInfo(need_changed_files=True)
gh = Github(get_best_robot_token())
rerun_helper = RerunHelper(gh, pr_info, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
if not pr_info.has_changes_in_documentation() and not args.force:
logging.info("No changes in documentation")
commit = get_commit(gh, pr_info.sha)
commit.create_status(
context=NAME, description="No changes in docs", state="success"
)
sys.exit(0)
def add_pr(self, pr):
self.prs.append(PRInfo(pr.created_at, pr.html_url, pr.closed_at))
self.number_of_prs += 1
if re.match(
r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)",
category,
):
return ""
if not entry:
return f"Changelog entry required for category '{category}'"
return ""
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
pr_info = PRInfo(need_orgs=True, pr_event_from_api=True)
can_run, description, labels_state = should_run_checks_for_pr(pr_info)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
description_report = check_pr_description(pr_info)[:139]
if description_report:
print("::notice ::Cannot run, description does not match the template")
logging.info("PR body doesn't match the template: (start)\n%s\n(end)",
pr_info.body)
url = (f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/"
"blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1")
commit.create_status(
context=NAME,
description=description_report,
state="failure",
if "RUN_BY_HASH_NUM" in os.environ:
run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM"))
run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL"))
check_name_with_group = (
check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]")
else:
run_by_hash_num = 0
run_by_hash_total = 0
check_name_with_group = check_name
if not os.path.exists(temp_path):
os.makedirs(temp_path)
is_flaky_check = "flaky" in check_name
pr_info = PRInfo(need_changed_files=is_flaky_check or validate_bugix_check)
if validate_bugix_check and "pr-bugfix" not in pr_info.labels:
if args.post_commit_status == "file":
post_commit_status_to_file(
os.path.join(temp_path, "post_commit_status.tsv"),
"Skipped (no pr-bugfix)",
"success",
"null",
)
logging.info("Skipping '%s' (no pr-bugfix)", check_name)
sys.exit(0)
gh = Github(get_best_robot_token())
rerun_helper = RerunHelper(gh, pr_info, check_name_with_group)
from ssh import SSHKey
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import get_commit
from rerun_helper import RerunHelper
NAME = "Docs Release (actions)"
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
temp_path = TEMP_PATH
repo_path = REPO_COPY
gh = Github(get_best_robot_token())
pr_info = PRInfo(need_changed_files=True)
rerun_helper = RerunHelper(gh, pr_info, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
docker_image = get_image_with_version(temp_path, "clickhouse/docs-release")
test_output = os.path.join(temp_path, "docs_release_log")
if not os.path.exists(test_output):
os.makedirs(test_output)
token = CLOUDFLARE_TOKEN
def main():
logging.basicConfig(level=logging.INFO)
temp_path = TEMP_PATH
logging.info("Reports path %s", REPORTS_PATH)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
build_check_name = sys.argv[1]
needs_data = None
required_builds = 0
if os.path.exists(NEEDS_DATA_PATH):
with open(NEEDS_DATA_PATH, "rb") as file_handler:
needs_data = json.load(file_handler)
required_builds = len(needs_data)
logging.info("The next builds are required: %s", ", ".join(needs_data))
gh = Github(get_best_robot_token())
pr_info = PRInfo()
rerun_helper = RerunHelper(gh, pr_info, build_check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
builds_for_check = CI_CONFIG["builds_report_config"][build_check_name]
required_builds = required_builds or len(builds_for_check)
# Collect reports from json artifacts
builds_report_map = {}
for root, _, files in os.walk(REPORTS_PATH):
for f in files:
if f.startswith("build_urls_") and f.endswith(".json"):
logging.info("Found build report json %s", f)
build_name = get_build_name_from_file_name(f)
if build_name in builds_for_check:
with open(os.path.join(root, f), "rb") as file_handler:
builds_report_map[build_name] = json.load(file_handler)
else:
logging.info(
"Skipping report %s for build %s, it's not in our reports list",
f,
build_name,
)
# Sort reports by config order
build_reports = [
builds_report_map[build_name]
for build_name in builds_for_check
if build_name in builds_report_map
]
some_builds_are_missing = len(build_reports) < required_builds
missing_build_names = []
if some_builds_are_missing:
logging.warning(
"Expected to get %s build results, got only %s",
required_builds,
len(build_reports),
)
missing_build_names = [
name
for name in needs_data
if not any(rep for rep in build_reports if rep["job_name"] == name)
]
else:
logging.info("Got exactly %s builds", len(builds_report_map))
# Group build artifacts by groups
build_results = [] # type: List[BuildResult]
build_artifacts = [] #
build_logs = []
for build_report in build_reports:
build_result, build_artifacts_url, build_logs_url = process_report(build_report)
logging.info(
"Got %s artifact groups for build report report", len(build_result)
)
build_results.extend(build_result)
build_artifacts.extend(build_artifacts_url)
build_logs.extend(build_logs_url)
for failed_job in missing_build_names:
build_result, build_artifacts_url, build_logs_url = get_failed_report(
failed_job
)
build_results.extend(build_result)
build_artifacts.extend(build_artifacts_url)
build_logs.extend(build_logs_url)
total_groups = len(build_results)
logging.info("Totally got %s artifact groups", total_groups)
if total_groups == 0:
logging.error("No success builds, failing check")
sys.exit(1)
s3_helper = S3Helper("https://s3.amazonaws.com")
branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master"
branch_name = "master"
if pr_info.number != 0:
branch_name = f"PR #{pr_info.number}"
branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_info.number}"
commit_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commit/{pr_info.sha}"
task_url = GITHUB_RUN_URL
report = create_build_html_report(
build_check_name,
build_results,
build_logs,
build_artifacts,
task_url,
branch_url,
branch_name,
commit_url,
)
report_path = os.path.join(temp_path, "report.html")
with open(report_path, "w", encoding="utf-8") as fd:
fd.write(report)
logging.info("Going to upload prepared report")
context_name_for_path = build_check_name.lower().replace(" ", "_")
s3_path_prefix = (
str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path
)
url = s3_helper.upload_build_file_to_s3(
report_path, s3_path_prefix + "/report.html"
)
logging.info("Report url %s", url)
print(f"::notice ::Report url: {url}")
# Prepare a commit status
ok_groups = 0
summary_status = "success"
for build_result in build_results:
if build_result.status == "failure" and summary_status != "error":
summary_status = "failure"
if build_result.status == "error" or not build_result.status:
summary_status = "error"
if build_result.status == "success":
ok_groups += 1
if ok_groups == 0 or some_builds_are_missing:
summary_status = "error"
addition = ""
if some_builds_are_missing:
addition = f"({len(build_reports)} of {required_builds} builds are OK)"
description = f"{ok_groups}/{total_groups} artifact groups are OK {addition}"
commit = get_commit(gh, pr_info.sha)
commit.create_status(
context=build_check_name,
description=description,
state=summary_status,
target_url=url,
)
if summary_status == "error":
sys.exit(1)
return f"docker run --network=host --volume={build_path}:/package_folder" \
f" --volume={server_log_folder}:/var/log/clickhouse-server" \
f" --volume={result_folder}:/test_output" \
f" {docker_image} >{result_folder}/{RESULT_LOG_NAME}"
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
reports_path = os.getenv("REPORTS_PATH", "./reports")
pr_info = PRInfo(get_event())
gh = Github(get_best_robot_token())
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
docker_image = get_image_with_version(reports_path, DOCKER_IMAGE)
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
if re.match(
r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)",
category,
):
return ""
if not entry:
return f"Changelog entry required for category '{category}'"
return ""
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
pr_info = PRInfo(need_orgs=True, labels_from_api=True)
can_run, description = should_run_checks_for_pr(pr_info)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
description_report = check_pr_description(pr_info)[:139]
if description_report:
print("::notice ::Cannot run, description does not match the template")
url = (f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/"
"blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1")
commit.create_status(
context=NAME,
description=description_report,
state="failure",
target_url=url,
)
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status, get_commit
NAME = "Docs Check (actions)"
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
temp_path = os.path.join(os.getenv("TEMP_PATH"))
repo_path = os.path.join(os.getenv("REPO_COPY"))
with open(os.getenv('GITHUB_EVENT_PATH'), 'r',
encoding='utf-8') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event, need_changed_files=True)
gh = Github(get_best_robot_token())
if not pr_info.has_changes_in_documentation():
logging.info("No changes in documentation")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME,
description="No changes in docs",
state="success")
sys.exit(0)
logging.info("Has changes in docs")
if not os.path.exists(temp_path):
os.makedirs(temp_path)
Squash statuses to latest state
1. context="first", state="success", update_time=1
2. context="second", state="success", update_time=2
3. context="first", stat="failure", update_time=3
=========>
1. context="second", state="success"
2. context="first", stat="failure"
"""
filt = {}
for status in sorted(statuses, key=lambda x: x.updated_at):
filt[status.context] = status
return filt
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
pr_info = PRInfo(need_orgs=True)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
url = GITHUB_RUN_URL
statuses = filter_statuses(list(commit.get_statuses()))
if NAME in statuses and statuses[NAME].state == "pending":
commit.create_status(
context=NAME,
description="All checks finished",
state="success",
target_url=url,
)
def main():
logging.basicConfig(level=logging.INFO)
build_name = sys.argv[1]
build_config = CI_CONFIG["build_config"][build_name]
if not os.path.exists(TEMP_PATH):
os.makedirs(TEMP_PATH)
pr_info = PRInfo()
logging.info("Repo copy path %s", REPO_COPY)
s3_helper = S3Helper()
version = get_version_from_repo(git=Git(True))
release_or_pr, performance_pr = get_release_or_pr(pr_info, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
# FIXME performance
s3_performance_path = "/".join(
(performance_pr, pr_info.sha, build_name, "performance.tgz"))
# If this is rerun, then we try to find already created artifacts and just
# put them as github actions artifact (result)
check_for_success_run(s3_helper, s3_path_prefix, build_name, build_config)
docker_image = get_image_with_version(IMAGES_PATH, IMAGE_NAME)
image_version = docker_image.version
logging.info("Got version from repo %s", version.string)
official_flag = pr_info.number == 0
if "official" in build_config:
official_flag = build_config["official"]
version_type = "testing"
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
version_type = "stable"
official_flag = True
update_version_local(version, version_type)
logging.info("Updated local files with version")
logging.info("Build short name %s", build_name)
build_output_path = os.path.join(TEMP_PATH, build_name)
if not os.path.exists(build_output_path):
os.makedirs(build_output_path)
ccache_path = os.path.join(CACHES_PATH, build_name + "_ccache")
logging.info("Will try to fetch cache for our build")
try:
get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number,
TEMP_PATH)
except Exception as e:
# In case there are issues with ccache, remove the path and do not fail a build
logging.info("Failed to get ccache, building without it. Error: %s", e)
rmtree(ccache_path, ignore_errors=True)
if not os.path.exists(ccache_path):
logging.info("cache was not fetched, will create empty dir")
os.makedirs(ccache_path)
packager_cmd = get_packager_cmd(
build_config,
os.path.join(REPO_COPY, "docker/packager"),
build_output_path,
version.string,
image_version,
ccache_path,
official_flag,
)
logging.info("Going to run packager with %s", packager_cmd)
logs_path = os.path.join(TEMP_PATH, "build_log")
if not os.path.exists(logs_path):
os.makedirs(logs_path)
start = time.time()
log_path, success = build_clickhouse(packager_cmd, logs_path,
build_output_path)
elapsed = int(time.time() - start)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {build_output_path}",
shell=True)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}",
shell=True)
logging.info("Build finished with %s, log path %s", success, log_path)
# Upload the ccache first to have the least build time in case of problems
logging.info("Will upload cache")
upload_ccache(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
# FIXME performance
performance_urls = []
performance_path = os.path.join(build_output_path, "performance.tgz")
if os.path.exists(performance_path):
performance_urls.append(
s3_helper.upload_build_file_to_s3(performance_path,
s3_performance_path))
logging.info(
"Uploaded performance.tgz to %s, now delete to avoid duplication",
performance_urls[0],
)
os.remove(performance_path)
build_urls = (s3_helper.upload_build_folder_to_s3(
build_output_path,
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
) + performance_urls)
logging.info("Got build URLs %s", build_urls)
print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
if os.path.exists(log_path):
log_url = s3_helper.upload_build_file_to_s3(
log_path, s3_path_prefix + "/" + os.path.basename(log_path))
logging.info("Log url %s", log_url)
else:
logging.info("Build log doesn't exist")
print(f"::notice ::Log URL: {log_url}")
create_json_artifact(TEMP_PATH, build_name, log_url, build_urls,
build_config, elapsed, success)
upload_master_static_binaries(pr_info, build_config, s3_helper,
build_output_path)
# Fail build job if not successeded
if not success:
sys.exit(1)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = TEMP_PATH
repo_path = REPO_COPY
reports_path = REPORTS_PATH
check_name = sys.argv[1]
kill_timeout = int(sys.argv[2])
flaky_check = 'flaky' in check_name.lower()
gh = Github(get_best_robot_token())
pr_info = PRInfo(need_changed_files=flaky_check)
if 'RUN_BY_HASH_NUM' in os.environ:
run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM'))
run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL'))
check_name_with_group = check_name + f' [{run_by_hash_num + 1}/{run_by_hash_total}]'
else:
run_by_hash_num = 0
run_by_hash_total = 0
check_name_with_group = check_name
rerun_helper = RerunHelper(gh, pr_info, check_name_with_group)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
args = parse_args()
push = not args.no_push_images
if push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
encoding="utf-8",
shell=True,
)
to_merge = {}
for suf in args.suffixes:
to_merge[suf] = load_images(args.path, suf)
changed_images = get_changed_images(check_sources(to_merge))
os.environ["DOCKER_CLI_EXPERIMENTAL"] = "enabled"
merged = merge_images(to_merge)
status = "success"
test_results = [] # type: List[Tuple[str, str]]
for image, versions in merged.items():
for tags in versions:
manifest, test_result = create_manifest(image, tags, push)
test_results.append((manifest, test_result))
if test_result != "OK":
status = "failure"
with open(os.path.join(args.path, "changed_images.json"), "w") as ci:
json.dump(changed_images, ci)
pr_info = PRInfo()
s3_helper = S3Helper("https://s3.amazonaws.com")
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results,
[], NAME)
print("::notice ::Report url: {}".format(url))
print('::set-output name=url_output::"{}"'.format(url))
if args.no_reports:
return
if changed_images:
description = "Updated " + ", ".join(changed_images.keys())
else:
description = "Nothing to update"
if len(description) >= 140:
description = description[:136] + "..."
gh = Github(get_best_robot_token())
post_commit_status(gh, pr_info.sha, NAME, description, status, url)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
ch_helper.insert_events_into(db="gh-data",
table="checks",
events=prepared_events)
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
kill_timeout = int(sys.argv[2])
flaky_check = 'flaky' in check_name.lower()
if not os.path.exists(temp_path):
os.makedirs(temp_path)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file:
event = json.load(event_file)
gh = Github(get_best_robot_token())
pr_info = PRInfo(event, need_changed_files=flaky_check)
tests_to_run = []
if flaky_check:
tests_to_run = get_tests_to_run(pr_info)
if not tests_to_run:
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=check_name, description='Not found changed stateless tests', state='success')
sys.exit(0)
image_name = get_image_name(check_name)
docker_image = get_image_with_version(reports_path, image_name)
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
def main():
logging.basicConfig(level=logging.INFO)
build_check_name = sys.argv[1]
build_name = sys.argv[2]
build_config = get_build_config(build_check_name, build_name)
if not os.path.exists(TEMP_PATH):
os.makedirs(TEMP_PATH)
pr_info = PRInfo()
logging.info("Repo copy path %s", REPO_COPY)
s3_helper = S3Helper("https://s3.amazonaws.com")
version = get_version_from_repo(REPO_COPY)
release_or_pr = get_release_or_pr(pr_info, build_config, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
# If this is rerun, then we try to find already created artifacts and just
# put them as github actions artifcat (result)
build_results = get_build_results_if_exists(s3_helper, s3_path_prefix)
if build_results is not None and len(build_results) > 0:
logging.info("Some build results found %s", build_results)
build_urls = []
log_url = ""
for url in build_results:
if "build_log.log" in url:
log_url = "https://s3.amazonaws.com/clickhouse-builds/" + url.replace(
"+", "%2B").replace(" ", "%20")
else:
build_urls.append(
"https://s3.amazonaws.com/clickhouse-builds/" +
url.replace("+", "%2B").replace(" ", "%20"))
create_json_artifact(
TEMP_PATH,
build_name,
log_url,
build_urls,
build_config,
0,
len(build_urls) > 0,
)
return
image_name = get_image_name(build_config)
docker_image = get_image_with_version(IMAGES_PATH, image_name)
image_version = docker_image.version
logging.info("Got version from repo %s", version.get_version_string())
version_type = "testing"
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
version_type = "stable"
update_version_local(REPO_COPY, pr_info.sha, version, version_type)
logging.info("Updated local files with version")
logging.info("Build short name %s", build_name)
build_output_path = os.path.join(TEMP_PATH, build_name)
if not os.path.exists(build_output_path):
os.makedirs(build_output_path)
ccache_path = os.path.join(CACHES_PATH, build_name + "_ccache")
logging.info("Will try to fetch cache for our build")
get_ccache_if_not_exists(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
if not os.path.exists(ccache_path):
logging.info("cache was not fetched, will create empty dir")
os.makedirs(ccache_path)
if build_config["package_type"] == "performance" and pr_info.number != 0:
# because perf tests store some information about git commits
subprocess.check_call(
f"cd {REPO_COPY} && git fetch origin master:master", shell=True)
packager_cmd = get_packager_cmd(
build_config,
os.path.join(REPO_COPY, "docker/packager"),
build_output_path,
version.get_version_string(),
image_version,
ccache_path,
pr_info,
)
logging.info("Going to run packager with %s", packager_cmd)
build_clickhouse_log = os.path.join(TEMP_PATH, "build_log")
if not os.path.exists(build_clickhouse_log):
os.makedirs(build_clickhouse_log)
start = time.time()
log_path, success = build_clickhouse(packager_cmd, build_clickhouse_log,
build_output_path)
elapsed = int(time.time() - start)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {build_output_path}",
shell=True)
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {ccache_path}",
shell=True)
logging.info("Build finished with %s, log path %s", success, log_path)
logging.info("Will upload cache")
upload_ccache(ccache_path, s3_helper, pr_info.number, TEMP_PATH)
if os.path.exists(log_path):
log_url = s3_helper.upload_build_file_to_s3(
log_path, s3_path_prefix + "/" + os.path.basename(log_path))
logging.info("Log url %s", log_url)
else:
logging.info("Build log doesn't exist")
build_urls = s3_helper.upload_build_folder_to_s3(
build_output_path,
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
)
logging.info("Got build URLs %s", build_urls)
print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
print("::notice ::Log URL: {}".format(log_url))
create_json_artifact(TEMP_PATH, build_name, log_url, build_urls,
build_config, elapsed, success)
upload_master_static_binaries(pr_info, build_config, s3_helper,
build_output_path)
# Fail build job if not successeded
if not success:
sys.exit(1)
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
makedirs(TEMP_PATH, exist_ok=True)
args = parse_args()
image = DockerImage(args.image_path, args.image_repo, False)
args.release_type = auto_release_type(args.version, args.release_type)
tags = gen_tags(args.version, args.release_type)
NAME = f"Docker image {image.repo} building check"
pr_info = None
if CI:
pr_info = PRInfo()
release_or_pr, _ = get_release_or_pr(pr_info, args.version)
args.bucket_prefix = (
f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{release_or_pr}/{pr_info.sha}"
)
if args.push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
encoding="utf-8",
shell=True,
)
NAME = f"Docker image {image.repo} build and push"
logging.info("Following tags will be created: %s", ", ".join(tags))
status = "success"
test_results = [] # type: List[Tuple[str, str]]
for os in args.os:
for tag in tags:
test_results.extend(
build_and_push_image(
image, args.push, args.bucket_prefix, os, tag, args.version
)
)
if test_results[-1][1] != "OK":
status = "failure"
pr_info = pr_info or PRInfo()
s3_helper = S3Helper()
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
print(f"::notice ::Report url: {url}")
print(f'::set-output name=url_output::"{url}"')
if not args.reports:
return
description = f"Processed tags: {', '.join(tags)}"
if len(description) >= 140:
description = description[:136] + "..."
gh = Github(get_best_robot_token(), per_page=100)
post_commit_status(gh, pr_info.sha, NAME, description, status, url)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
if status != "success":
sys.exit(1)
return f"docker run --network=host --volume={build_path}:/package_folder" \
f" --volume={server_log_folder}:/var/log/clickhouse-server" \
f" --volume={result_folder}:/test_output" \
f" {docker_image} >{result_folder}/{RESULT_LOG_NAME}"
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = TEMP_PATH
repo_path = REPO_COPY
reports_path = REPORTS_PATH
pr_info = PRInfo()
gh = Github(get_best_robot_token())
rerun_helper = RerunHelper(gh, pr_info, CHECK_NAME)
if rerun_helper.is_already_finished_by_status():
logging.info(
"Check is already finished according to github status, exiting")
sys.exit(0)
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
args = parse_args()
if args.suffix:
global NAME
NAME += f" {args.suffix}"
changed_json = os.path.join(TEMP_PATH,
f"changed_images_{args.suffix}.json")
else:
changed_json = os.path.join(TEMP_PATH, "changed_images.json")
push = not args.no_push_images
if push:
subprocess.check_output( # pylint: disable=unexpected-keyword-arg
"docker login --username 'robotclickhouse' --password-stdin",
input=get_parameter_from_ssm("dockerhub_robot_password"),
encoding="utf-8",
shell=True,
)
repo_path = GITHUB_WORKSPACE
if os.path.exists(TEMP_PATH):
shutil.rmtree(TEMP_PATH)
os.makedirs(TEMP_PATH)
if args.image_path:
pr_info = PRInfo()
pr_info.changed_files = set(i for i in args.image_path)
else:
pr_info = PRInfo(need_changed_files=True)
changed_images = get_changed_docker_images(pr_info, repo_path,
"docker/images.json")
logging.info("Has changed images %s",
", ".join([str(image[0]) for image in changed_images]))
pr_commit_version = str(pr_info.number) + "-" + pr_info.sha
# The order is important, PR number is used as cache during the build
versions = [str(pr_info.number), pr_commit_version]
result_version = pr_commit_version
if pr_info.number == 0:
# First get the latest for cache
versions.insert(0, "latest")
if args.suffix:
# We should build architecture specific images separately and merge a
# manifest lately in a different script
versions = [f"{v}-{args.suffix}" for v in versions]
# changed_images_{suffix}.json should contain all changed images
result_version = versions
result_images = {}
images_processing_result = []
for rel_path, image_name in changed_images:
full_path = os.path.join(repo_path, rel_path)
images_processing_result += process_single_image(
versions, full_path, image_name, push)
result_images[image_name] = result_version
if changed_images:
description = "Updated " + ",".join([im[1] for im in changed_images])
else:
description = "Nothing to update"
if len(description) >= 140:
description = description[:136] + "..."
with open(changed_json, "w") as images_file:
json.dump(result_images, images_file)
s3_helper = S3Helper("https://s3.amazonaws.com")
s3_path_prefix = (str(pr_info.number) + "/" + pr_info.sha + "/" +
NAME.lower().replace(" ", "_"))
status, test_results = process_test_results(s3_helper,
images_processing_result,
s3_path_prefix)
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results,
[], NAME)
print("::notice ::Report url: {}".format(url))
print('::set-output name=url_output::"{}"'.format(url))
if args.no_reports:
return
gh = Github(get_best_robot_token())
post_commit_status(gh, pr_info.sha, NAME, description, status, url)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
ch_helper.insert_events_into(db="gh-data",
table="checks",
events=prepared_events)
