def start_bisecting(project_name, platform_name, git_repo_location, commits_list, needs_clean, repeat_times):
left = 0
right = len(commits_list)
while left < right:
mid = (left + right) // 2
mid_commit = commits_list[mid]
bazelci.print_expanded_group(":bazel: Test with Bazel built at " + mid_commit)
bazelci.eprint("Remaining suspected commits are:\n")
for i in range(left, right):
bazelci.eprint(commits_list[i] + "\n")
if test_with_bazel_at_commit(
project_name, platform_name, git_repo_location, mid_commit, needs_clean, repeat_times
):
bazelci.print_collapsed_group(":bazel: Succeeded at " + mid_commit)
left = mid + 1
else:
bazelci.print_collapsed_group(":bazel: Failed at " + mid_commit)
right = mid
bazelci.print_expanded_group(":bazel: Bisect Result")
if right == len(commits_list):
bazelci.eprint("first bad commit not found, every commit succeeded.")
else:
first_bad_commit = commits_list[right]
bazelci.eprint("first bad commit is " + first_bad_commit)
os.chdir(BAZEL_REPO_DIR)
bazelci.execute_command(["git", "--no-pager", "log", "-n", "1", first_bad_commit])
def main(argv=None):
org = os.getenv("BUILDKITE_ORGANIZATION_SLUG")
repo = os.getenv("BUILDKITE_REPO")
settings = DOCGEN_SETTINGS.get(org, {}).get(repo)
if not settings:
bazelci.eprint("docgen is not enabled for '%s' org and repository %s",
org, repo)
return 1
bazelci.print_expanded_group(
":bazel: Building documentation from {}".format(repo))
try:
bazelci.execute_command(["bazel", "build"] + DEFAULT_FLAGS +
settings.build_flags + [settings.target])
except subprocess.CalledProcessError as e:
bazelci.eprint("Bazel failed with exit code {}".format(e.returncode))
return e.returncode
src_root = os.path.join(os.getcwd(), settings.output_dir)
if settings.rewrite:
bazelci.print_expanded_group(
":bazel: Rewriting links in documentation files")
dest_root = os.path.join(tempfile.mkdtemp(), "site")
rewrite_and_copy(src_root, dest_root, settings.rewrite)
src_root = dest_root
bucket = "gs://{}".format(settings.gcs_bucket)
dest = get_destination(bucket, settings.gcs_subdir)
bazelci.print_expanded_group(
":bazel: Uploading documentation to {}".format(dest))
try:
bazelci.execute_command(
["gsutil", "-m", "rsync", "-r", "-c", "-d", src_root, dest])
bazelci.execute_command([
"gsutil", "web", "set", "-m", "index.html", "-e", "404.html",
bucket
])
# TODO: does not work with 404 pages in sub directories
except subprocess.CalledProcessError as e:
bazelci.eprint("Upload to GCS failed with exit code {}".format(
e.returncode))
return e.returncode
bazelci.print_collapsed_group(":bazel: Publishing documentation URL")
message = "You can find the documentation at {}".format(get_url(settings))
bazelci.execute_command([
"buildkite-agent", "annotate", "--style=info", message, "--context",
"doc_url"
])
bazelci.execute_command(
["buildkite-agent", "meta-data", "set", "message", message])
return 0
def test_with_bazel_at_commit(project_name, task_name, git_repo_location,
bazel_commit, needs_clean, repeat_times):
http_config = bazelci.DOWNSTREAM_PROJECTS[project_name]["http_config"]
for i in range(1, repeat_times + 1):
if repeat_times > 1:
bazelci.print_collapsed_group(":bazel: Try %s time" % i)
try:
return_code = bazelci.main([
"runner",
"--task=" + task_name,
"--http_config=" + http_config,
"--git_repo_location=" + git_repo_location,
"--use_bazel_at_commit=" + bazel_commit,
] + (["--needs_clean"] if needs_clean else []))
except subprocess.CalledProcessError as e:
bazelci.eprint(str(e))
return False
if return_code != 0:
return False
return True
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script")
subparsers = parser.add_subparsers(dest="subparsers_name")
subparsers.add_parser("culprit_finder")
runner = subparsers.add_parser("runner")
runner.add_argument("--project_name", type=str)
runner.add_argument("--platform_name", type=str)
runner.add_argument("--good_bazel_commit", type=str)
runner.add_argument("--bad_bazel_commit", type=str)
runner.add_argument("--needs_clean", type=bool, nargs="?", const=True)
runner.add_argument("--repeat_times", type=int, default=1)
args = parser.parse_args(argv)
if args.subparsers_name == "culprit_finder":
try:
project_name = os.environ["PROJECT_NAME"]
platform_name = os.environ["PLATFORM_NAME"]
good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"]
bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"]
except KeyError as e:
raise Exception("Environment variable %s must be set" % str(e))
needs_clean = False
if "NEEDS_CLEAN" in os.environ:
needs_clean = True
repeat_times = 1
if "REPEAT_TIMES" in os.environ:
repeat_times = int(os.environ["REPEAT_TIMES"])
if project_name not in DOWNSTREAM_PROJECTS:
raise Exception(
"Project name '%s' not recognized, available projects are %s"
% (project_name, str((DOWNSTREAM_PROJECTS.keys())))
)
if platform_name not in PLATFORMS:
raise Exception(
"Platform name '%s' not recognized, available platforms are %s"
% (platform_name, str((PLATFORMS.keys())))
)
print_culprit_finder_pipeline(
project_name=project_name,
platform_name=platform_name,
good_bazel_commit=good_bazel_commit,
bad_bazel_commit=bad_bazel_commit,
needs_clean=needs_clean,
repeat_times=repeat_times,
)
elif args.subparsers_name == "runner":
git_repo_location = clone_git_repository(args.project_name, args.platform_name)
bazelci.print_collapsed_group("Check good bazel commit " + args.good_bazel_commit)
if not test_with_bazel_at_commit(
project_name=args.project_name,
platform_name=args.platform_name,
git_repo_location=git_repo_location,
bazel_commit=args.good_bazel_commit,
needs_clean=args.needs_clean,
repeat_times=args.repeat_times,
):
raise Exception(
"Given good commit (%s) is not actually good, abort bisecting."
% args.good_bazel_commit
)
start_bisecting(
project_name=args.project_name,
platform_name=args.platform_name,
git_repo_location=git_repo_location,
commits_list=get_bazel_commits_between(args.good_bazel_commit, args.bad_bazel_commit),
needs_clean=args.needs_clean,
repeat_times=args.repeat_times,
)
else:
parser.print_help()
return 2
return 0
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script")
subparsers = parser.add_subparsers(dest="subparsers_name")
subparsers.add_parser("culprit_finder")
runner = subparsers.add_parser("runner")
runner.add_argument("--project_name", type=str)
runner.add_argument("--task_name", type=str)
runner.add_argument("--good_bazel_commit", type=str)
runner.add_argument("--bad_bazel_commit", type=str)
runner.add_argument("--needs_clean", type=bool, nargs="?", const=True)
runner.add_argument("--repeat_times", type=int, default=1)
args = parser.parse_args(argv)
if args.subparsers_name == "culprit_finder":
try:
project_name = os.environ["PROJECT_NAME"]
# For old config file, we can still set PLATFORM_NAME as task name.
task = os.environ.get("PLATFORM_NAME") or os.environ.get(
"TASK_NAME")
if task:
tasks = [task]
else:
tasks = get_tasks(project_name)
good_bazel_commit = os.environ.get("GOOD_BAZEL_COMMIT")
if not good_bazel_commit:
# If GOOD_BAZEL_COMMIT is not set, use recorded last bazel green commit for downstream project
last_green_commit_url = bazelci.bazelci_last_green_downstream_commit_url(
)
good_bazel_commit = bazelci.get_last_green_commit(
last_green_commit_url)
bad_bazel_commit = os.environ.get("BAD_BAZEL_COMMIT")
if not bad_bazel_commit:
# If BAD_BAZEL_COMMIT is not set, use HEAD commit.
bad_bazel_commit = (subprocess.check_output(
["git", "rev-parse", "HEAD"]).decode("utf-8").strip())
except KeyError as e:
raise Exception("Environment variable %s must be set" % str(e))
needs_clean = False
if "NEEDS_CLEAN" in os.environ:
needs_clean = True
repeat_times = 1
if "REPEAT_TIMES" in os.environ:
repeat_times = int(os.environ["REPEAT_TIMES"])
if project_name not in bazelci.DOWNSTREAM_PROJECTS:
raise Exception(
"Project name '%s' not recognized, available projects are %s" %
(project_name, str((bazelci.DOWNSTREAM_PROJECTS.keys()))))
print_culprit_finder_pipeline(
project_name=project_name,
tasks=tasks,
good_bazel_commit=good_bazel_commit,
bad_bazel_commit=bad_bazel_commit,
needs_clean=needs_clean,
repeat_times=repeat_times,
)
elif args.subparsers_name == "runner":
git_repo_location = clone_git_repository(args.project_name,
args.task_name)
bazelci.print_collapsed_group("Check good bazel commit " +
args.good_bazel_commit)
if not test_with_bazel_at_commit(
project_name=args.project_name,
task_name=args.task_name,
git_repo_location=git_repo_location,
bazel_commit=args.good_bazel_commit,
needs_clean=args.needs_clean,
repeat_times=args.repeat_times,
):
raise Exception(
"Given good commit (%s) is not actually good, abort bisecting."
% args.good_bazel_commit)
start_bisecting(
project_name=args.project_name,
task_name=args.task_name,
git_repo_location=git_repo_location,
commits_list=get_bazel_commits_between(args.good_bazel_commit,
args.bad_bazel_commit),
needs_clean=args.needs_clean,
repeat_times=args.repeat_times,
)
else:
parser.print_help()
return 2
return 0
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description="Bazel Culprit Finder Script")
subparsers = parser.add_subparsers(dest="subparsers_name")
culprit_finder = subparsers.add_parser("culprit_finder")
runner = subparsers.add_parser("runner")
runner.add_argument("--project_name", type=str)
runner.add_argument("--platform_name", type=str)
runner.add_argument("--good_bazel_commit", type=str)
runner.add_argument("--bad_bazel_commit", type=str)
args = parser.parse_args(argv)
try:
if args.subparsers_name == "culprit_finder":
try:
project_name = os.environ["PROJECT_NAME"]
platform_name = os.environ["PLATFORM_NAME"]
good_bazel_commit = os.environ["GOOD_BAZEL_COMMIT"]
bad_bazel_commit = os.environ["BAD_BAZEL_COMMIT"]
except KeyError as e:
raise BuildkiteException("Environment variable %s must be set" % str(e))
if project_name not in DOWNSTREAM_PROJECTS:
raise BuildkiteException(
"Project name '%s' not recognized, available projects are %s"
% (project_name, str((DOWNSTREAM_PROJECTS.keys())))
)
if platform_name not in PLATFORMS:
raise BuildkiteException(
"Platform name '%s' not recognized, available platforms are %s"
% (platform_name, str((PLATFORMS.keys())))
)
print_culprit_finder_pipeline(
project_name=project_name,
platform_name=platform_name,
good_bazel_commit=good_bazel_commit,
bad_bazel_commit=bad_bazel_commit,
)
elif args.subparsers_name == "runner":
git_repo_location = clone_git_repository(args.project_name, args.platform_name)
bazelci.print_collapsed_group("Check good bazel commit " + args.good_bazel_commit)
if not test_with_bazel_at_commit(
project_name=args.project_name,
platform_name=args.platform_name,
git_repo_location=git_repo_location,
bazel_commit=args.good_bazel_commit,
):
raise BuildkiteException(
"Given good commit (%s) is not actually good, abort bisecting."
% args.good_bazel_commit
)
start_bisecting(
project_name=args.project_name,
platform_name=args.platform_name,
git_repo_location=git_repo_location,
commits_list=get_bazel_commits_between(
args.good_bazel_commit, args.bad_bazel_commit
),
)
else:
parser.print_help()
return 2
except BuildkiteException as e:
bazelci.eprint(str(e))
return 1
return 0
