def print_steps_for_failing_jobs(build_number):
build_info = get_build_info(build_number)
failing_jobs = get_failing_jobs(build_info)
incompatible_flags = list(fetch_incompatible_flags().keys())
pipeline_steps = []
for incompatible_flag in incompatible_flags:
for job in failing_jobs:
label = "%s: %s" % (incompatible_flag, job["name"])
command = list(job["command"])
command[1] = command[1] + " --incompatible_flag=" + incompatible_flag
pipeline_steps.append(create_step(label, command, job["platform"]))
print(yaml.dump({"steps": pipeline_steps}))
def print_incompatible_flags_info_box_step(incompatible_flags_map):
info_text = ["Build and test with the following incompatible flags:"]
for flag in incompatible_flags_map:
info_text.append("* **%s**: %s" % (flag, incompatible_flags_map[flag]))
if len(info_text) == 1:
return None
return create_step(
label="Incompatible flags info",
commands=[
'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
],
)
def print_disabled_projects_info_box_step():
info_text = ["Downstream testing is disabled for the following projects :sadpanda:"]
for project, config in DOWNSTREAM_PROJECTS.items():
disabled_reason = config.get("disabled_reason", None)
if disabled_reason:
info_text.append("* **%s**: %s" % (project, disabled_reason))
if len(info_text) == 1:
return None
return create_step(
label=":sadpanda:",
commands=[
'buildkite-agent annotate --append --style=info "\n' + "\n".join(info_text) + '\n"'
],
)
def print_culprit_finder_pipeline(
project_name, platform_name, good_bazel_commit, bad_bazel_commit, needs_clean
):
label = PLATFORMS[platform_name]["emoji-name"] + " Bisecting for {0}".format(project_name)
command = (
'%s culprit_finder.py runner --project_name="%s" --platform_name=%s --good_bazel_commit=%s --bad_bazel_commit=%s %s'
% (
python_binary(platform_name),
project_name,
platform_name,
good_bazel_commit,
bad_bazel_commit,
"--needs_clean" if needs_clean else "",
)
)
commands = [fetch_bazelcipy_command(), fetch_culprit_finder_py_command(), command]
pipeline_steps = []
pipeline_steps.append(create_step(label, commands, platform_name))
print(yaml.dump({"steps": pipeline_steps}))
def main(configs, http_config, file_config):
if not configs:
raise Exception(
"Bazel publish binaries pipeline configuration is empty.")
for platform in configs.copy():
if platform not in PLATFORMS:
raise Exception("Unknown platform '{}'".format(platform))
if not PLATFORMS[platform]["publish_binary"]:
del configs[platform]
if set(configs) != set(name for name, platform in PLATFORMS.items()
if platform["publish_binary"]):
raise Exception(
"Bazel publish binaries pipeline needs to build Bazel for every commit on all publish_binary-enabled platforms."
)
# Build Bazel
pipeline_steps = []
for platform in configs:
pipeline_steps.append(
bazel_build_step(platform,
"Bazel",
http_config,
file_config,
build_only=True))
pipeline_steps.append("wait")
# If all builds succeed, publish the Bazel binaries to GCS.
pipeline_steps.append(
create_step(
label="Publish Bazel Binaries",
commands=[
fetch_bazelcipy_command(),
python_binary() + " bazelci.py publish_binaries"
],
))
print(yaml.dump({"steps": pipeline_steps}))
def upload_project_pipeline_step(
project_name, git_repository, http_config, file_config, incompatible_flags
):
pipeline_command = (
'{0} bazelci.py project_pipeline --project_name="{1}" ' + "--git_repository={2}"
).format(python_binary(), project_name, git_repository)
if incompatible_flags is None:
pipeline_command += " --use_but"
else:
for flag in incompatible_flags:
pipeline_command += " --incompatible_flag=" + flag
if http_config:
pipeline_command += " --http_config=" + http_config
if file_config:
pipeline_command += " --file_config=" + file_config
pipeline_command += " | buildkite-agent pipeline upload"
return create_step(
label="Setup {0}".format(project_name),
commands=[fetch_bazelcipy_command(), pipeline_command],
)
def main(
configs,
project_name,
http_config,
file_config,
git_repository,
monitor_flaky_tests,
use_but,
incompatible_flags,
):
platform_configs = configs.get("platforms", None)
if not platform_configs:
raise Exception(
"{0} pipeline configuration is empty.".format(project_name))
pipeline_steps = []
if configs.get("buildifier"):
pipeline_steps.append(
create_docker_step("Buildifier",
image=f"gcr.io/{CLOUD_PROJECT}/buildifier"))
# In Bazel Downstream Project pipelines, git_repository and project_name must be specified,
# and we should test the project at the last green commit.
git_commit = None
if (use_but or incompatible_flags) and git_repository and project_name:
git_commit = get_last_green_commit(
git_repository, DOWNSTREAM_PROJECTS[project_name]["pipeline_slug"])
for platform in platform_configs:
step = runner_step(
platform,
project_name,
http_config,
file_config,
git_repository,
git_commit,
monitor_flaky_tests,
use_but,
incompatible_flags,
)
pipeline_steps.append(step)
pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
all_downstream_pipeline_slugs = []
for _, config in DOWNSTREAM_PROJECTS.items():
all_downstream_pipeline_slugs.append(config["pipeline_slug"])
# We don't need to update last green commit in the following cases:
# 1. This job is a github pull request
# 2. This job uses a custom built Bazel binary (In Bazel Downstream Projects pipeline)
# 3. This job doesn't run on master branch (Could be a custom build launched manually)
# 4. We don't intend to run the same job in downstream with Bazel@HEAD (eg. google-bazel-presubmit)
# 5. We are testing incompatible flags
if not (is_pull_request() or use_but
or os.getenv("BUILDKITE_BRANCH") != "master" or pipeline_slug
not in all_downstream_pipeline_slugs or incompatible_flags):
pipeline_steps.append("wait")
# If all builds succeed, update the last green commit of this project
pipeline_steps.append(
create_step(
label="Try Update Last Green Commit",
commands=[
fetch_bazelcipy_command(),
python_binary() +
" bazelci.py try_update_last_green_commit",
],
))
print(yaml.dump({"steps": pipeline_steps}))
def main(configs, http_config, file_config, test_incompatible_flags, test_disabled_projects):
if not configs:
raise Exception("Bazel downstream pipeline configuration is empty.")
if set(configs) != set(PLATFORMS):
raise Exception(
"Bazel downstream pipeline needs to build Bazel on all supported platforms (has=%s vs. want=%s)."
% (sorted(set(configs)), sorted(set(PLATFORMS)))
)
pipeline_steps = []
info_box_step = print_disabled_projects_info_box_step()
if info_box_step is not None:
pipeline_steps.append(info_box_step)
if not test_incompatible_flags:
for platform in configs:
pipeline_steps.append(
bazel_build_step(platform, "Bazel", http_config, file_config, build_only=True)
)
pipeline_steps.append("wait")
incompatible_flags = None
if test_incompatible_flags:
incompatible_flags_map = fetch_incompatible_flags()
info_box_step = print_incompatible_flags_info_box_step(incompatible_flags_map)
if info_box_step is not None:
pipeline_steps.append(info_box_step)
incompatible_flags = list(incompatible_flags_map.keys())
for project, config in DOWNSTREAM_PROJECTS.items():
disabled_reason = config.get("disabled_reason", None)
# If test_disabled_projects is true, we add configs for disabled projects.
# If test_disabled_projects is false, we add configs for not disbaled projects.
if (test_disabled_projects and disabled_reason) or (
not test_disabled_projects and not disabled_reason
):
pipeline_steps.append(
upload_project_pipeline_step(
project_name=project,
git_repository=config["git_repository"],
http_config=config.get("http_config", None),
file_config=config.get("file_config", None),
incompatible_flags=incompatible_flags,
)
)
if test_incompatible_flags:
pipeline_steps.append({"wait": "~", "continue_on_failure": "true"})
current_build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", None)
if not current_build_number:
raise Exception("Not running inside Buildkite")
pipeline_steps.append(
create_step(
label="Test failing jobs with incompatible flag separately",
commands=[
fetch_bazelcipy_command(),
fetch_incompatible_flag_verbose_failures_command(),
python_binary()
+ " incompatible_flag_verbose_failures.py --build_number=%s | buildkite-agent pipeline upload"
% current_build_number,
],
)
)
print(yaml.dump({"steps": pipeline_steps}))