def start_bisecting(project_name, platform_name, git_repo_location, commits_list, needs_clean):
left = 0
right = len(commits_list)
while left < right:
mid = (left + right) // 2
mid_commit = commits_list[mid]
print_expanded_group(":bazel: Test with Bazel built at " + mid_commit)
eprint("Remaining suspected commits are:\n")
for i in range(left, right):
eprint(commits_list[i] + "\n")
if test_with_bazel_at_commit(
project_name, platform_name, git_repo_location, mid_commit, needs_clean
):
print_collapsed_group(":bazel: Succeeded at " + mid_commit)
left = mid + 1
else:
print_collapsed_group(":bazel: Failed at " + mid_commit)
right = mid
print_expanded_group(":bazel: Bisect Result")
if right == len(commits_list):
eprint("first bad commit not found, every commit succeeded.")
else:
first_bad_commit = commits_list[right]
eprint("first bad commit is " + first_bad_commit)
os.chdir(BAZEL_REPO_DIR)
execute_command(["git", "--no-pager", "log", "-n", "1", first_bad_commit])
def execute_bazel_run(bazel_binary, platform, targets, incompatible_flags):
if not targets:
return
print_collapsed_group("Setup (Run Targets)")
for target in targets:
execute_command([bazel_binary] + common_startup_flags(platform) +
["run"] + common_build_flags(None, platform) +
(incompatible_flags or []) + [target])
def upload_test_logs(bep_file, tmpdir):
if not os.path.exists(bep_file):
return
test_logs = test_logs_to_upload(bep_file, tmpdir)
if test_logs:
cwd = os.getcwd()
try:
os.chdir(tmpdir)
test_logs = [
os.path.relpath(test_log, tmpdir) for test_log in test_logs
]
test_logs = sorted(test_logs)
print_collapsed_group(":gcloud: Uploading Test Logs")
execute_command(
["buildkite-agent", "artifact", "upload", ";".join(test_logs)])
finally:
os.chdir(cwd)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script")
subparsers = parser.add_subparsers(dest="subparsers_name")
subparsers.add_parser("culprit_finder")
runner = subparsers.add_parser("runner")
runner.add_argument("--project_name", type=str)
runner.add_argument("--platform_name", type=str)
runner.add_argument("--good_bazel_commit", type=str)
runner.add_argument("--bad_bazel_commit", type=str)
runner.add_argument("--needs_clean", type=bool, nargs="?", const=True)
args = parser.parse_args(argv)
if args.subparsers_name == "culprit_finder":
try:
project_name = os.environ["PROJECT_NAME"]
platform_name = os.environ["PLATFORM_NAME"]
good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"]
bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"]
except KeyError as e:
raise Exception("Environment variable %s must be set" % str(e))
needs_clean = False
if "NEEDS_CLEAN" in os.environ:
needs_clean = True
if project_name not in DOWNSTREAM_PROJECTS:
raise Exception(
"Project name '%s' not recognized, available projects are %s"
% (project_name, str((DOWNSTREAM_PROJECTS.keys())))
)
if platform_name not in PLATFORMS:
raise Exception(
"Platform name '%s' not recognized, available platforms are %s"
% (platform_name, str((PLATFORMS.keys())))
)
print_culprit_finder_pipeline(
project_name=project_name,
platform_name=platform_name,
good_bazel_commit=good_bazel_commit,
bad_bazel_commit=bad_bazel_commit,
needs_clean=needs_clean,
)
elif args.subparsers_name == "runner":
git_repo_location = clone_git_repository_for_project(args.project_name, args.platform_name)
print_collapsed_group("Check good bazel commit " + args.good_bazel_commit)
if not test_with_bazel_at_commit(
project_name=args.project_name,
platform_name=args.platform_name,
git_repo_location=git_repo_location,
bazel_commit=args.good_bazel_commit,
needs_clean=args.needs_clean,
):
raise Exception(
"Given good commit (%s) is not actually good, abort bisecting."
% args.good_bazel_commit
)
start_bisecting(
project_name=args.project_name,
platform_name=args.platform_name,
git_repo_location=git_repo_location,
commits_list=get_bazel_commits_between(args.good_bazel_commit, args.bad_bazel_commit),
needs_clean=args.needs_clean,
)
else:
parser.print_help()
return 2
return 0
def clone_git_repository(git_repository, platform, git_commit=None):
root = downstream_projects_root(platform)
project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1)
clone_path = os.path.join(root, project_name)
print_collapsed_group("Fetching %s sources at %s" %
(project_name, git_commit if git_commit else "HEAD"))
if not os.path.exists(clone_path):
if platform in [
"ubuntu1404", "ubuntu1604", "ubuntu1804", "rbe_ubuntu1604"
]:
execute_command([
"git", "clone", "--reference", "/var/lib/bazelbuild",
git_repository, clone_path
])
elif platform in ["macos"]:
execute_command([
"git",
"clone",
"--reference",
"/usr/local/var/bazelbuild",
git_repository,
clone_path,
])
elif platform in ["windows"]:
execute_command([
"git",
"clone",
"--reference",
"c:\\buildkite\\bazelbuild",
git_repository,
clone_path,
])
else:
execute_command(["git", "clone", git_repository, clone_path])
os.chdir(clone_path)
execute_command(["git", "remote", "set-url", "origin", git_repository])
execute_command(["git", "clean", "-fdqx"])
execute_command([
"git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"
])
execute_command(["git", "fetch", "origin"])
if git_commit:
# sync to a specific commit of this repository
execute_command(["git", "reset", git_commit, "--hard"])
else:
# sync to the latest commit of HEAD. Unlikely git pull this also works after a force push.
remote_head = (subprocess.check_output(
["git", "symbolic-ref",
"refs/remotes/origin/HEAD"]).decode("utf-8").rstrip())
execute_command(["git", "reset", remote_head, "--hard"])
execute_command(["git", "submodule", "sync", "--recursive"])
execute_command(
["git", "submodule", "update", "--init", "--recursive", "--force"])
execute_command([
"git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"
])
execute_command(["git", "clean", "-fdqx"])
execute_command([
"git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx"
])
return clone_path
def main(
config,
platform,
git_repository,
git_commit,
git_repo_location,
use_bazel_at_commit,
use_but,
save_but,
needs_clean,
build_only,
test_only,
monitor_flaky_tests,
incompatible_flags,
):
build_only = build_only or "test_targets" not in config
test_only = test_only or "build_targets" not in config
if build_only and test_only:
raise Exception(
"build_only and test_only cannot be true at the same time")
if use_bazel_at_commit and use_but:
raise Exception(
"use_bazel_at_commit cannot be set when use_but is true")
tmpdir = tempfile.mkdtemp()
sc_process = None
try:
if git_repo_location:
os.chdir(git_repo_location)
elif git_repository:
clone_git_repository(git_repository, platform, git_commit)
else:
git_repository = os.getenv("BUILDKITE_REPO")
if use_bazel_at_commit:
print_collapsed_group(":gcloud: Downloading Bazel built at " +
use_bazel_at_commit)
bazel_binary = download_bazel_binary_at_commit(
tmpdir, platform, use_bazel_at_commit)
elif use_but:
print_collapsed_group(":gcloud: Downloading Bazel Under Test")
bazel_binary = download_bazel_binary(tmpdir, platform)
else:
bazel_binary = "bazel"
print_bazel_version_info(bazel_binary, platform)
print_environment_variables_info()
if incompatible_flags:
print_expanded_group(
"Build and test with the following incompatible flags:")
for flag in incompatible_flags:
eprint(flag + "\n")
if platform == "windows":
execute_batch_commands(config.get("batch_commands", None))
else:
execute_shell_commands(config.get("shell_commands", None))
execute_bazel_run(bazel_binary, platform,
config.get("run_targets", None), incompatible_flags)
if config.get("sauce", None):
print_collapsed_group(":saucelabs: Starting Sauce Connect Proxy")
os.environ["SAUCE_USERNAME"] = "******"
os.environ["SAUCE_ACCESS_KEY"] = saucelabs_token()
os.environ["TUNNEL_IDENTIFIER"] = str(uuid.uuid4())
os.environ["BUILD_TAG"] = str(uuid.uuid4())
readyfile = os.path.join(tmpdir, "sc_is_ready")
if platform == "windows":
cmd = [
"sauce-connect.exe", "/i", os.environ["TUNNEL_IDENTIFIER"],
"/f", readyfile
]
else:
cmd = [
"sc", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f",
readyfile
]
sc_process = execute_command_background(cmd)
wait_start = time.time()
while not os.path.exists(readyfile):
if time.time() - wait_start > 30:
raise Exception(
"Sauce Connect Proxy is still not ready after 30 seconds, aborting!"
)
time.sleep(1)
print("Sauce Connect Proxy is ready, continuing...")
if needs_clean:
execute_bazel_clean(bazel_binary, platform)
if not test_only:
execute_bazel_build(
bazel_binary,
platform,
config.get("build_flags", []),
config.get("build_targets", None),
None,
incompatible_flags,
)
if save_but:
upload_bazel_binary(platform)
if not build_only:
test_bep_file = os.path.join(tmpdir, "test_bep.json")
try:
execute_bazel_test(
bazel_binary,
platform,
config.get("test_flags", []),
config.get("test_targets", None),
test_bep_file,
monitor_flaky_tests,
incompatible_flags,
)
if has_flaky_tests(test_bep_file):
if monitor_flaky_tests:
# Upload the BEP logs from Bazel builds for later analysis on flaky tests
build_id = os.getenv("BUILDKITE_BUILD_ID")
pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
execute_command([
gsutil_command(),
"cp",
test_bep_file,
"gs://bazel-buildkite-stats/flaky-tests-bep/" +
pipeline_slug + "/" + build_id + ".json",
])
finally:
upload_test_logs(test_bep_file, tmpdir)
finally:
if sc_process:
sc_process.terminate()
try:
sc_process.wait(timeout=10)
except subprocess.TimeoutExpired:
sc_process.kill()
if tmpdir:
shutil.rmtree(tmpdir)
def upload_bazel_binary(platform):
print_collapsed_group(":gcloud: Uploading Bazel Under Test")
binary_path = "bazel-bin/src/bazel"
if platform == "windows":
binary_path = r"bazel-bin\src\bazel"
execute_command(["buildkite-agent", "artifact", "upload", binary_path])
def print_environment_variables_info():
print_collapsed_group(":information_source: Environment Variables")
for key, value in os.environ.items():
eprint("%s=(%s)" % (key, value))
def execute_shell_commands(commands):
if not commands:
return
print_collapsed_group(":bash: Setup (Shell Commands)")
shell_command = "\n".join(commands)
execute_command([shell_command], shell=True)
def execute_batch_commands(commands):
if not commands:
return
print_collapsed_group(":batch: Setup (Batch Commands)")
batch_commands = "&".join(commands)
subprocess.run(batch_commands, shell=True, check=True, env=os.environ)
def print_bazel_version_info(bazel_binary, platform):
print_collapsed_group(":information_source: Bazel Info")
execute_command([bazel_binary] + common_startup_flags(platform) +
["--nomaster_bazelrc", "--bazelrc=/dev/null", "version"])
execute_command([bazel_binary] + common_startup_flags(platform) +
["--nomaster_bazelrc", "--bazelrc=/dev/null", "info"])