def latest_generation_and_build_number():
output = None
attempt = 0
while attempt < 5:
output = subprocess.check_output(
[gsutil_command(), "stat",
bazelci_builds_metadata_url()],
env=os.environ)
match = re.search("Generation:[ ]*([0-9]+)", output.decode("utf-8"))
if not match:
raise Exception(
"Couldn't parse generation. gsutil output format changed?")
generation = match.group(1)
match = re.search(r"Hash \(md5\):[ ]*([^\s]+)", output.decode("utf-8"))
if not match:
raise Exception(
"Couldn't parse md5 hash. gsutil output format changed?")
expected_md5hash = base64.b64decode(match.group(1))
output = subprocess.check_output(
[gsutil_command(), "cat",
bazelci_builds_metadata_url()],
env=os.environ)
hasher = hashlib.md5()
hasher.update(output)
actual_md5hash = hasher.digest()
if expected_md5hash == actual_md5hash:
break
attempt += 1
info = json.loads(output.decode("utf-8"))
return (generation, info["build_number"])
def try_publish_binaries(build_number, expected_generation):
now = datetime.datetime.now()
git_commit = os.environ["BUILDKITE_COMMIT"]
info = {
"build_number": build_number,
"build_time": now.strftime("%d-%m-%Y %H:%M"),
"git_commit": git_commit,
"platforms": {},
}
for platform in (name for name in PLATFORMS
if PLATFORMS[name]["publish_binary"]):
tmpdir = tempfile.mkdtemp()
try:
bazel_binary_path = download_bazel_binary(tmpdir, platform)
execute_command([
gsutil_command(),
"cp",
"-a",
"public-read",
bazel_binary_path,
bazelci_builds_gs_url(platform, git_commit),
])
info["platforms"][platform] = {
"url": bazelci_builds_download_url(platform, git_commit),
"sha256": sha256_hexdigest(bazel_binary_path),
}
finally:
shutil.rmtree(tmpdir)
tmpdir = tempfile.mkdtemp()
try:
info_file = os.path.join(tmpdir, "info.json")
with open(info_file, mode="w", encoding="utf-8") as fp:
json.dump(info, fp, indent=2, sort_keys=True)
try:
execute_command([
gsutil_command(),
"-h",
"x-goog-if-generation-match:" + expected_generation,
"-h",
"Content-Type:application/json",
"cp",
"-a",
"public-read",
info_file,
bazelci_builds_metadata_url(),
])
except subprocess.CalledProcessError:
raise BinaryUploadRaceException()
finally:
shutil.rmtree(tmpdir)
def main():
pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
git_repository = os.getenv("BUILDKITE_REPO")
last_green_commit = get_last_green_commit(git_repository, pipeline_slug)
current_commit = subprocess.check_output(["git", "rev-parse",
"HEAD"]).decode("utf-8").strip()
if last_green_commit:
execute_command(["git", "fetch", "-v", "origin", last_green_commit])
result = (subprocess.check_output([
"git", "rev-list",
"%s..%s" % (last_green_commit, current_commit)
]).decode("utf-8").strip())
# If current_commit is newer that last_green_commit, `git rev-list A..B` will output a bunch of
# commits, otherwise the output should be empty.
if not last_green_commit or result:
execute_command(
[
"echo %s | %s cp - %s" % (
current_commit,
gsutil_command(),
bazelci_last_green_commit_url(git_repository,
pipeline_slug),
)
],
shell=True,
)
else:
eprint(
"Updating abandoned: last green commit (%s) is not older than current commit (%s)."
% (last_green_commit, current_commit))
def get_last_green_commit(git_repository, pipeline_slug):
last_green_commit_url = bazelci_last_green_commit_url(
git_repository, pipeline_slug)
try:
return (subprocess.check_output(
[gsutil_command(), "cat", last_green_commit_url],
env=os.environ).decode("utf-8").strip())
except subprocess.CalledProcessError:
return None
def download_bazel_binary_at_commit(dest_dir, platform, bazel_git_commit):
# We only build bazel binary on ubuntu14.04 for every bazel commit.
# It should be OK to use it on other ubuntu platforms.
if "ubuntu" in PLATFORMS[platform].get("host-platform", platform):
platform = "ubuntu1404"
bazel_binary_path = os.path.join(
dest_dir, "bazel.exe" if platform == "windows" else "bazel")
try:
execute_command([
gsutil_command(),
"cp",
bazelci_builds_gs_url(platform, bazel_git_commit),
bazel_binary_path,
])
except subprocess.CalledProcessError as e:
raise Exception(
"Failed to download Bazel binary at %s, error message:\n%s" %
(bazel_git_commit, str(e)))
st = os.stat(bazel_binary_path)
os.chmod(bazel_binary_path, st.st_mode | stat.S_IEXEC)
return bazel_binary_path
def main(
config,
platform,
git_repository,
git_commit,
git_repo_location,
use_bazel_at_commit,
use_but,
save_but,
needs_clean,
build_only,
test_only,
monitor_flaky_tests,
incompatible_flags,
):
build_only = build_only or "test_targets" not in config
test_only = test_only or "build_targets" not in config
if build_only and test_only:
raise Exception(
"build_only and test_only cannot be true at the same time")
if use_bazel_at_commit and use_but:
raise Exception(
"use_bazel_at_commit cannot be set when use_but is true")
tmpdir = tempfile.mkdtemp()
sc_process = None
try:
if git_repo_location:
os.chdir(git_repo_location)
elif git_repository:
clone_git_repository(git_repository, platform, git_commit)
else:
git_repository = os.getenv("BUILDKITE_REPO")
if use_bazel_at_commit:
print_collapsed_group(":gcloud: Downloading Bazel built at " +
use_bazel_at_commit)
bazel_binary = download_bazel_binary_at_commit(
tmpdir, platform, use_bazel_at_commit)
elif use_but:
print_collapsed_group(":gcloud: Downloading Bazel Under Test")
bazel_binary = download_bazel_binary(tmpdir, platform)
else:
bazel_binary = "bazel"
print_bazel_version_info(bazel_binary, platform)
print_environment_variables_info()
if incompatible_flags:
print_expanded_group(
"Build and test with the following incompatible flags:")
for flag in incompatible_flags:
eprint(flag + "\n")
if platform == "windows":
execute_batch_commands(config.get("batch_commands", None))
else:
execute_shell_commands(config.get("shell_commands", None))
execute_bazel_run(bazel_binary, platform,
config.get("run_targets", None), incompatible_flags)
if config.get("sauce", None):
print_collapsed_group(":saucelabs: Starting Sauce Connect Proxy")
os.environ["SAUCE_USERNAME"] = "******"
os.environ["SAUCE_ACCESS_KEY"] = saucelabs_token()
os.environ["TUNNEL_IDENTIFIER"] = str(uuid.uuid4())
os.environ["BUILD_TAG"] = str(uuid.uuid4())
readyfile = os.path.join(tmpdir, "sc_is_ready")
if platform == "windows":
cmd = [
"sauce-connect.exe", "/i", os.environ["TUNNEL_IDENTIFIER"],
"/f", readyfile
]
else:
cmd = [
"sc", "-i", os.environ["TUNNEL_IDENTIFIER"], "-f",
readyfile
]
sc_process = execute_command_background(cmd)
wait_start = time.time()
while not os.path.exists(readyfile):
if time.time() - wait_start > 30:
raise Exception(
"Sauce Connect Proxy is still not ready after 30 seconds, aborting!"
)
time.sleep(1)
print("Sauce Connect Proxy is ready, continuing...")
if needs_clean:
execute_bazel_clean(bazel_binary, platform)
if not test_only:
execute_bazel_build(
bazel_binary,
platform,
config.get("build_flags", []),
config.get("build_targets", None),
None,
incompatible_flags,
)
if save_but:
upload_bazel_binary(platform)
if not build_only:
test_bep_file = os.path.join(tmpdir, "test_bep.json")
try:
execute_bazel_test(
bazel_binary,
platform,
config.get("test_flags", []),
config.get("test_targets", None),
test_bep_file,
monitor_flaky_tests,
incompatible_flags,
)
if has_flaky_tests(test_bep_file):
if monitor_flaky_tests:
# Upload the BEP logs from Bazel builds for later analysis on flaky tests
build_id = os.getenv("BUILDKITE_BUILD_ID")
pipeline_slug = os.getenv("BUILDKITE_PIPELINE_SLUG")
execute_command([
gsutil_command(),
"cp",
test_bep_file,
"gs://bazel-buildkite-stats/flaky-tests-bep/" +
pipeline_slug + "/" + build_id + ".json",
])
finally:
upload_test_logs(test_bep_file, tmpdir)
finally:
if sc_process:
sc_process.terminate()
try:
sc_process.wait(timeout=10)
except subprocess.TimeoutExpired:
sc_process.kill()
if tmpdir:
shutil.rmtree(tmpdir)