def start_bisecting(project_name, platform_name, git_repo_location, commits_list, needs_clean): left = 0 right = len(commits_list) while left < right: mid = (left + right) // 2 mid_commit = commits_list[mid] print_expanded_group(":bazel: Test with Bazel built at " + mid_commit) eprint("Remaining suspected commits are:\n") for i in range(left, right): eprint(commits_list[i] + "\n") if test_with_bazel_at_commit( project_name, platform_name, git_repo_location, mid_commit, needs_clean ): print_collapsed_group(":bazel: Succeeded at " + mid_commit) left = mid + 1 else: print_collapsed_group(":bazel: Failed at " + mid_commit) right = mid print_expanded_group(":bazel: Bisect Result") if right == len(commits_list): eprint("first bad commit not found, every commit succeeded.") else: first_bad_commit = commits_list[right] eprint("first bad commit is " + first_bad_commit) os.chdir(BAZEL_REPO_DIR) execute_command(["git", "--no-pager", "log", "-n", "1", first_bad_commit])
def execute_bazel_run(bazel_binary, platform, targets, incompatible_flags): if not targets: return print_collapsed_group("Setup (Run Targets)") for target in targets: execute_command([bazel_binary] + common_startup_flags(platform) + ["run"] + common_build_flags(None, platform) + (incompatible_flags or []) + [target])
def upload_test_logs(bep_file, tmpdir): if not os.path.exists(bep_file): return test_logs = test_logs_to_upload(bep_file, tmpdir) if test_logs: cwd = os.getcwd() try: os.chdir(tmpdir) test_logs = [ os.path.relpath(test_log, tmpdir) for test_log in test_logs ] test_logs = sorted(test_logs) print_collapsed_group(":gcloud: Uploading Test Logs") execute_command( ["buildkite-agent", "artifact", "upload", ";".join(test_logs)]) finally: os.chdir(cwd)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script") subparsers = parser.add_subparsers(dest="subparsers_name") subparsers.add_parser("culprit_finder") runner = subparsers.add_parser("runner") runner.add_argument("--project_name", type=str) runner.add_argument("--platform_name", type=str) runner.add_argument("--good_bazel_commit", type=str) runner.add_argument("--bad_bazel_commit", type=str) runner.add_argument("--needs_clean", type=bool, nargs="?", const=True) args = parser.parse_args(argv) if args.subparsers_name == "culprit_finder": try: project_name = os.environ["PROJECT_NAME"] platform_name = os.environ["PLATFORM_NAME"] good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"] bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"] except KeyError as e: raise Exception("Environment variable %s must be set" % str(e)) needs_clean = False if "NEEDS_CLEAN" in os.environ: needs_clean = True if project_name not in DOWNSTREAM_PROJECTS: raise Exception( "Project name '%s' not recognized, available projects are %s" % (project_name, str((DOWNSTREAM_PROJECTS.keys()))) ) if platform_name not in PLATFORMS: raise Exception( "Platform name '%s' not recognized, available platforms are %s" % (platform_name, str((PLATFORMS.keys()))) ) print_culprit_finder_pipeline( project_name=project_name, platform_name=platform_name, good_bazel_commit=good_bazel_commit, bad_bazel_commit=bad_bazel_commit, needs_clean=needs_clean, ) elif args.subparsers_name == "runner": git_repo_location = clone_git_repository_for_project(args.project_name, args.platform_name) print_collapsed_group("Check good bazel commit " + args.good_bazel_commit) if not test_with_bazel_at_commit( project_name=args.project_name, platform_name=args.platform_name, git_repo_location=git_repo_location, bazel_commit=args.good_bazel_commit, needs_clean=args.needs_clean, ): raise Exception( "Given good commit (%s) is not actually good, abort bisecting." % args.good_bazel_commit ) start_bisecting( project_name=args.project_name, platform_name=args.platform_name, git_repo_location=git_repo_location, commits_list=get_bazel_commits_between(args.good_bazel_commit, args.bad_bazel_commit), needs_clean=args.needs_clean, ) else: parser.print_help() return 2 return 0
def clone_git_repository(git_repository, platform, git_commit=None): root = downstream_projects_root(platform) project_name = re.search(r"/([^/]+)\.git$", git_repository).group(1) clone_path = os.path.join(root, project_name) print_collapsed_group("Fetching %s sources at %s" % (project_name, git_commit if git_commit else "HEAD")) if not os.path.exists(clone_path): if platform in [ "ubuntu1404", "ubuntu1604", "ubuntu1804", "rbe_ubuntu1604" ]: execute_command([ "git", "clone", "--reference", "/var/lib/bazelbuild", git_repository, clone_path ]) elif platform in ["macos"]: execute_command([ "git", "clone", "--reference", "/usr/local/var/bazelbuild", git_repository, clone_path, ]) elif platform in ["windows"]: execute_command([ "git", "clone", "--reference", "c:\\buildkite\\bazelbuild", git_repository, clone_path, ]) else: execute_command(["git", "clone", git_repository, clone_path]) os.chdir(clone_path) execute_command(["git", "remote", "set-url", "origin", git_repository]) execute_command(["git", "clean", "-fdqx"]) execute_command([ "git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx" ]) execute_command(["git", "fetch", "origin"]) if git_commit: # sync to a specific commit of this repository execute_command(["git", "reset", git_commit, "--hard"]) else: # sync to the latest commit of HEAD. Unlikely git pull this also works after a force push. remote_head = (subprocess.check_output( ["git", "symbolic-ref", "refs/remotes/origin/HEAD"]).decode("utf-8").rstrip()) execute_command(["git", "reset", remote_head, "--hard"]) execute_command(["git", "submodule", "sync", "--recursive"]) execute_command( ["git", "submodule", "update", "--init", "--recursive", "--force"]) execute_command([ "git", "submodule", "foreach", "--recursive", "git", "reset", "--hard" ]) execute_command(["git", "clean", "-fdqx"]) execute_command([ "git", "submodule", "foreach", "--recursive", "git", "clean", "-fdqx" ]) return clone_path
def main( config, platform, git_repository, git_commit, git_repo_location, use_bazel_at_commit, use_but, save_but, needs_clean, build_only, test_only, monitor_flaky_tests, incompatible_flags, ): build_only = build_only or "test_targets" not in config test_only = test_only or "build_targets" not in config if build_only and test_only: raise Exception( "build_only and test_only cannot be true at the same time") if use_bazel_at_commit and use_but: raise Exception( "use_bazel_at_commit cannot be set when use_but is true") tmpdir = tempfile.mkdtemp() sc_process = None try: if git_repo_location: os.chdir(git_repo_location) elif git_repository: clone_git_repository(git_repository, platform, git_commit) else: git_repository = os.getenv("BUILDKITE_REPO") if use_bazel_at_commit: print_collapsed_group(":gcloud: Downloading Bazel built at " + use_bazel_at_commit) bazel_binary = download_bazel_binary_at_commit( tmpdir, platform, use_bazel_at_commit) elif use_but: print_collapsed_group(":gcloud: Downloading Bazel Under Test") bazel_binary = download_bazel_binary(tmpdir, platform) else: bazel_binary = "bazel" print_bazel_version_info(bazel_binary, platform) print_environment_variables_info() if incompatible_flags: print_expanded_group( "Build and test with the following incompatible flags:") for flag in incompatible_flags: eprint(flag + "\n") if platform == "windows": execute_batch_commands(config.get("batch_commands", None)) else: execute_shell_commands(config.get("shell_commands", None)) execute_bazel_run(bazel_binary, platform, config.get("run_targets", None), incompatible_flags) if config.get("sauce", None): print_collapsed_group(":saucelabs: Starting Sauce Connect Proxy") os.environ["SAUCE_USERNAME"] = "******" os.environ["SAUCE_ACCESS_KEY"] = saucelabs_token() os.environ["TUNNEL_IDENTIFIER"] = str(uuid.uuid4()) os.environ["BUILD_TAG"] = str(uuid.uuid4()) readyfile = os.path.join(tmpdir, "sc_is_ready") if platform == "windows": cmd = [ "sauce-connect.exe", "/i", os.environ["TUNNEL_IDENTIFIER"], "/f", readyfile ] else: cmd = [ "sc", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f", readyfile ] sc_process = execute_command_background(cmd) wait_start = time.time() while not os.path.exists(readyfile): if time.time() - wait_start > 30: raise Exception( "Sauce Connect Proxy is still not ready after 30 seconds, aborting!" ) time.sleep(1) print("Sauce Connect Proxy is ready, continuing...") if needs_clean: execute_bazel_clean(bazel_binary, platform) if not test_only: execute_bazel_build( bazel_binary, platform, config.get("build_flags", []), config.get("build_targets", None), None, incompatible_flags, ) if save_but: upload_bazel_binary(platform) if not build_only: test_bep_file = os.path.join(tmpdir, "test_bep.json") try: execute_bazel_test( bazel_binary, platform, config.get("test_flags", []), config.get("test_targets", None), test_bep_file, monitor_flaky_tests, incompatible_flags, ) if has_flaky_tests(test_bep_file): if monitor_flaky_tests: # Upload the BEP logs from Bazel builds for later analysis on flaky tests build_id = os.getenv("BUILDKITE_BUILD_ID") pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG") execute_command([ gsutil_command(), "cp", test_bep_file, "gs://bazel-buildkite-stats/flaky-tests-bep/" + pipeline_slug + "/" + build_id + ".json", ]) finally: upload_test_logs(test_bep_file, tmpdir) finally: if sc_process: sc_process.terminate() try: sc_process.wait(timeout=10) except subprocess.TimeoutExpired: sc_process.kill() if tmpdir: shutil.rmtree(tmpdir)
def upload_bazel_binary(platform): print_collapsed_group(":gcloud: Uploading Bazel Under Test") binary_path = "bazel-bin/src/bazel" if platform == "windows": binary_path = r"bazel-bin\src\bazel" execute_command(["buildkite-agent", "artifact", "upload", binary_path])
def print_environment_variables_info(): print_collapsed_group(":information_source: Environment Variables") for key, value in os.environ.items(): eprint("%s=(%s)" % (key, value))
def execute_shell_commands(commands): if not commands: return print_collapsed_group(":bash: Setup (Shell Commands)") shell_command = "\n".join(commands) execute_command([shell_command], shell=True)
def execute_batch_commands(commands): if not commands: return print_collapsed_group(":batch: Setup (Batch Commands)") batch_commands = "&".join(commands) subprocess.run(batch_commands, shell=True, check=True, env=os.environ)
def print_bazel_version_info(bazel_binary, platform): print_collapsed_group(":information_source: Bazel Info") execute_command([bazel_binary] + common_startup_flags(platform) + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "version"]) execute_command([bazel_binary] + common_startup_flags(platform) + ["--nomaster_bazelrc", "--bazelrc=/dev/null", "info"])