def start_bisecting(project_name, platform_name, git_repo_location, commits_list, needs_clean, repeat_times): left = 0 right = len(commits_list) while left < right: mid = (left + right) // 2 mid_commit = commits_list[mid] bazelci.print_expanded_group(":bazel: Test with Bazel built at " + mid_commit) bazelci.eprint("Remaining suspected commits are:\n") for i in range(left, right): bazelci.eprint(commits_list[i] + "\n") if test_with_bazel_at_commit( project_name, platform_name, git_repo_location, mid_commit, needs_clean, repeat_times ): bazelci.print_collapsed_group(":bazel: Succeeded at " + mid_commit) left = mid + 1 else: bazelci.print_collapsed_group(":bazel: Failed at " + mid_commit) right = mid bazelci.print_expanded_group(":bazel: Bisect Result") if right == len(commits_list): bazelci.eprint("first bad commit not found, every commit succeeded.") else: first_bad_commit = commits_list[right] bazelci.eprint("first bad commit is " + first_bad_commit) os.chdir(BAZEL_REPO_DIR) bazelci.execute_command(["git", "--no-pager", "log", "-n", "1", first_bad_commit])
def main(argv=None): org = os.getenv("BUILDKITE_ORGANIZATION_SLUG") repo = os.getenv("BUILDKITE_REPO") settings = DOCGEN_SETTINGS.get(org, {}).get(repo) if not settings: bazelci.eprint("docgen is not enabled for '%s' org and repository %s", org, repo) return 1 bazelci.print_expanded_group( ":bazel: Building documentation from {}".format(repo)) try: bazelci.execute_command(["bazel", "build"] + DEFAULT_FLAGS + settings.build_flags + [settings.target]) except subprocess.CalledProcessError as e: bazelci.eprint("Bazel failed with exit code {}".format(e.returncode)) return e.returncode src_root = os.path.join(os.getcwd(), settings.output_dir) if settings.rewrite: bazelci.print_expanded_group( ":bazel: Rewriting links in documentation files") dest_root = os.path.join(tempfile.mkdtemp(), "site") rewrite_and_copy(src_root, dest_root, settings.rewrite) src_root = dest_root bucket = "gs://{}".format(settings.gcs_bucket) dest = get_destination(bucket, settings.gcs_subdir) bazelci.print_expanded_group( ":bazel: Uploading documentation to {}".format(dest)) try: bazelci.execute_command( ["gsutil", "-m", "rsync", "-r", "-c", "-d", src_root, dest]) bazelci.execute_command([ "gsutil", "web", "set", "-m", "index.html", "-e", "404.html", bucket ]) # TODO: does not work with 404 pages in sub directories except subprocess.CalledProcessError as e: bazelci.eprint("Upload to GCS failed with exit code {}".format( e.returncode)) return e.returncode bazelci.print_collapsed_group(":bazel: Publishing documentation URL") message = "You can find the documentation at {}".format(get_url(settings)) bazelci.execute_command([ "buildkite-agent", "annotate", "--style=info", message, "--context", "doc_url" ]) bazelci.execute_command( ["buildkite-agent", "meta-data", "set", "message", message]) return 0
def test_with_bazel_at_commit(project_name, task_name, git_repo_location, bazel_commit, needs_clean, repeat_times): http_config = bazelci.DOWNSTREAM_PROJECTS[project_name]["http_config"] for i in range(1, repeat_times + 1): if repeat_times > 1: bazelci.print_collapsed_group(":bazel: Try %s time" % i) try: return_code = bazelci.main([ "runner", "--task=" + task_name, "--http_config=" + http_config, "--git_repo_location=" + git_repo_location, "--use_bazel_at_commit=" + bazel_commit, ] + (["--needs_clean"] if needs_clean else [])) except subprocess.CalledProcessError as e: bazelci.eprint(str(e)) return False if return_code != 0: return False return True
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script") subparsers = parser.add_subparsers(dest="subparsers_name") subparsers.add_parser("culprit_finder") runner = subparsers.add_parser("runner") runner.add_argument("--project_name", type=str) runner.add_argument("--platform_name", type=str) runner.add_argument("--good_bazel_commit", type=str) runner.add_argument("--bad_bazel_commit", type=str) runner.add_argument("--needs_clean", type=bool, nargs="?", const=True) runner.add_argument("--repeat_times", type=int, default=1) args = parser.parse_args(argv) if args.subparsers_name == "culprit_finder": try: project_name = os.environ["PROJECT_NAME"] platform_name = os.environ["PLATFORM_NAME"] good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"] bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"] except KeyError as e: raise Exception("Environment variable %s must be set" % str(e)) needs_clean = False if "NEEDS_CLEAN" in os.environ: needs_clean = True repeat_times = 1 if "REPEAT_TIMES" in os.environ: repeat_times = int(os.environ["REPEAT_TIMES"]) if project_name not in DOWNSTREAM_PROJECTS: raise Exception( "Project name '%s' not recognized, available projects are %s" % (project_name, str((DOWNSTREAM_PROJECTS.keys()))) ) if platform_name not in PLATFORMS: raise Exception( "Platform name '%s' not recognized, available platforms are %s" % (platform_name, str((PLATFORMS.keys()))) ) print_culprit_finder_pipeline( project_name=project_name, platform_name=platform_name, good_bazel_commit=good_bazel_commit, bad_bazel_commit=bad_bazel_commit, needs_clean=needs_clean, repeat_times=repeat_times, ) elif args.subparsers_name == "runner": git_repo_location = clone_git_repository(args.project_name, args.platform_name) bazelci.print_collapsed_group("Check good bazel commit " + args.good_bazel_commit) if not test_with_bazel_at_commit( project_name=args.project_name, platform_name=args.platform_name, git_repo_location=git_repo_location, bazel_commit=args.good_bazel_commit, needs_clean=args.needs_clean, repeat_times=args.repeat_times, ): raise Exception( "Given good commit (%s) is not actually good, abort bisecting." % args.good_bazel_commit ) start_bisecting( project_name=args.project_name, platform_name=args.platform_name, git_repo_location=git_repo_location, commits_list=get_bazel_commits_between(args.good_bazel_commit, args.bad_bazel_commit), needs_clean=args.needs_clean, repeat_times=args.repeat_times, ) else: parser.print_help() return 2 return 0
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script") subparsers = parser.add_subparsers(dest="subparsers_name") subparsers.add_parser("culprit_finder") runner = subparsers.add_parser("runner") runner.add_argument("--project_name", type=str) runner.add_argument("--task_name", type=str) runner.add_argument("--good_bazel_commit", type=str) runner.add_argument("--bad_bazel_commit", type=str) runner.add_argument("--needs_clean", type=bool, nargs="?", const=True) runner.add_argument("--repeat_times", type=int, default=1) args = parser.parse_args(argv) if args.subparsers_name == "culprit_finder": try: project_name = os.environ["PROJECT_NAME"] # For old config file, we can still set PLATFORM_NAME as task name. task = os.environ.get("PLATFORM_NAME") or os.environ.get( "TASK_NAME") if task: tasks = [task] else: tasks = get_tasks(project_name) good_bazel_commit = os.environ.get("GOOD_BAZEL_COMMIT") if not good_bazel_commit: # If GOOD_BAZEL_COMMIT is not set, use recorded last bazel green commit for downstream project last_green_commit_url = bazelci.bazelci_last_green_downstream_commit_url( ) good_bazel_commit = bazelci.get_last_green_commit( last_green_commit_url) bad_bazel_commit = os.environ.get("BAD_BAZEL_COMMIT") if not bad_bazel_commit: # If BAD_BAZEL_COMMIT is not set, use HEAD commit. bad_bazel_commit = (subprocess.check_output( ["git", "rev-parse", "HEAD"]).decode("utf-8").strip()) except KeyError as e: raise Exception("Environment variable %s must be set" % str(e)) needs_clean = False if "NEEDS_CLEAN" in os.environ: needs_clean = True repeat_times = 1 if "REPEAT_TIMES" in os.environ: repeat_times = int(os.environ["REPEAT_TIMES"]) if project_name not in bazelci.DOWNSTREAM_PROJECTS: raise Exception( "Project name '%s' not recognized, available projects are %s" % (project_name, str((bazelci.DOWNSTREAM_PROJECTS.keys())))) print_culprit_finder_pipeline( project_name=project_name, tasks=tasks, good_bazel_commit=good_bazel_commit, bad_bazel_commit=bad_bazel_commit, needs_clean=needs_clean, repeat_times=repeat_times, ) elif args.subparsers_name == "runner": git_repo_location = clone_git_repository(args.project_name, args.task_name) bazelci.print_collapsed_group("Check good bazel commit " + args.good_bazel_commit) if not test_with_bazel_at_commit( project_name=args.project_name, task_name=args.task_name, git_repo_location=git_repo_location, bazel_commit=args.good_bazel_commit, needs_clean=args.needs_clean, repeat_times=args.repeat_times, ): raise Exception( "Given good commit (%s) is not actually good, abort bisecting." % args.good_bazel_commit) start_bisecting( project_name=args.project_name, task_name=args.task_name, git_repo_location=git_repo_location, commits_list=get_bazel_commits_between(args.good_bazel_commit, args.bad_bazel_commit), needs_clean=args.needs_clean, repeat_times=args.repeat_times, ) else: parser.print_help() return 2 return 0
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script") subparsers = parser.add_subparsers(dest="subparsers_name") culprit_finder = subparsers.add_parser("culprit_finder") runner = subparsers.add_parser("runner") runner.add_argument("--project_name", type=str) runner.add_argument("--platform_name", type=str) runner.add_argument("--good_bazel_commit", type=str) runner.add_argument("--bad_bazel_commit", type=str) args = parser.parse_args(argv) try: if args.subparsers_name == "culprit_finder": try: project_name = os.environ["PROJECT_NAME"] platform_name = os.environ["PLATFORM_NAME"] good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"] bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"] except KeyError as e: raise BuildkiteException("Environment variable %s must be set" % str(e)) if project_name not in DOWNSTREAM_PROJECTS: raise BuildkiteException( "Project name '%s' not recognized, available projects are %s" % (project_name, str((DOWNSTREAM_PROJECTS.keys()))) ) if platform_name not in PLATFORMS: raise BuildkiteException( "Platform name '%s' not recognized, available platforms are %s" % (platform_name, str((PLATFORMS.keys()))) ) print_culprit_finder_pipeline( project_name=project_name, platform_name=platform_name, good_bazel_commit=good_bazel_commit, bad_bazel_commit=bad_bazel_commit, ) elif args.subparsers_name == "runner": git_repo_location = clone_git_repository(args.project_name, args.platform_name) bazelci.print_collapsed_group("Check good bazel commit " + args.good_bazel_commit) if not test_with_bazel_at_commit( project_name=args.project_name, platform_name=args.platform_name, git_repo_location=git_repo_location, bazel_commit=args.good_bazel_commit, ): raise BuildkiteException( "Given good commit (%s) is not actually good, abort bisecting." % args.good_bazel_commit ) start_bisecting( project_name=args.project_name, platform_name=args.platform_name, git_repo_location=git_repo_location, commits_list=get_bazel_commits_between( args.good_bazel_commit, args.bad_bazel_commit ), ) else: parser.print_help() return 2 except BuildkiteException as e: bazelci.eprint(str(e)) return 1 return 0