def main(argv=None):
if argv is None:
argv = sys.argv[1:]
utils.PRINT_COMMANDS = True
parser = argparse.ArgumentParser(description="Bazel Federation CI Patch Repositories Script")
parser.add_argument(
"--repositories_file",
type=str,
default="repositories.bzl",
help="Path of the file that contains the repository functions.",
)
args = parser.parse_args(argv)
utils.print_group("Executing patch_repositories.py...")
patching_required = utils.get_meta_data(
build_project_distro.REPO_PATCHING_REQUIRED_META_DATA_KEY, ""
)
if not patching_required:
utils.eprint("Running as part of a regular presubmit -> no repositories patching required")
return 0
project_name = utils.get_meta_data(build_project_distro.REPO_META_DATA_KEY)
archive_path = download_distro(project_name)
project_root = extract_distro(project_name, archive_path)
path = os.path.join(utils.REPO_ROOT, args.repositories_file)
rewrite_repositories_file(path, project_name, project_root)
upload_repositories_file(path)
return 0
def build_distro(repo_dir, target):
utils.print_group("Building distro {} in {}".format(target, repo_dir))
os.chdir(repo_dir)
process = utils.execute_command("bazel", "build", target)
if process.returncode:
raise Exception("Failed to build {}: {}".format(
target, process.stderr))
print(process.stdout)
process = utils.execute_command("find", "-L",
os.path.join(repo_dir, "bazel-bin"),
"-name", "*.tar.gz")
if process.returncode:
raise Exception("Unable to find tar.gz output file: {}".format(
process.stderr))
lines = process.stdout.split("\n")
files = [l for l in lines if l.strip()]
if len(files) != 1:
raise Exception("Expected exactly one tar.gz file, not {}: {}".format(
len(files), ", ".join(files)))
return files[0]
def rewrite_repositories_file(repositories_file, project_name, project_root):
utils.print_group("Rewriting {} to point {} at {}".format(
repositories_file, project_name, project_root))
transformations = [
lambda content: fix_function(repositories_file, content, project_name,
project_root)
]
rewrite_file(repositories_file, transformations)
def extract_archive(archive_path):
utils.print_group("Extracting {}".format(archive_path))
dest = tempfile.mkdtemp()
process = utils.execute_command("unzip", archive_path, "-d", dest)
if process.returncode:
raise Exception("Failed to extract {} to {}: {}".format(
archive_path, dest, process.stderr))
return dest
def fetch_source(org, repo, commit):
url = "https://github.com/{}/{}/archive/{}.zip".format(org, repo, commit)
utils.print_group("Fetching source from {}".format(url))
dest_path = os.path.join(tempfile.mkdtemp(), "{}.zip".format(repo))
process = utils.execute_command("curl", "-sSL", url, "-o", dest_path)
if process.returncode:
raise Exception("Unable to download from {}: {}".format(
url, process.stderr))
return dest_path
def download_distro(project_name):
src_path = utils.get_meta_data(build_project_distro.ARCHIVE_META_DATA_KEY)
dest_dir = tempfile.mkdtemp()
# Buildkite wants a trailing slash
if not dest_dir.endswith("/"):
dest_dir += "/"
utils.print_group("Downloading {} distro from Buildkite".format(project_name))
process = utils.execute_command("buildkite-agent", "artifact", "download", src_path, dest_dir)
if process.returncode:
raise Exception("Failed to download distro from {}: {}".format(src_path, process.stderr))
return os.path.join(dest_dir, src_path)
def extract_distro(project_name, archive_path):
utils.print_group("Extracting {} distro from {}".format(project_name, archive_path))
dest_dir = tempfile.mkdtemp()
process = utils.execute_command("tar", "-xf", archive_path, "-C", dest_dir)
if process.returncode:
raise Exception(
"Failed to extract {} distro to {}: {}".format(project_name, dest_dir, process.stderr)
)
# Unlike other repository rules, local_repository requires the presence of a WORKSPACE file
# (even though its content is being ignored).
# However, most distributions don't contain a WORKSPACE file, hence we create one in that case.
process = utils.execute_command("touch", os.path.join(dest_dir, "WORKSPACE"))
if process.returncode:
raise Exception("Failed to ensure that a WORKSPACE file exists: {}".format(process.stderr))
return dest_dir
def upload_repositories_file(repositories_file):
utils.print_group("Uploading {} file".format(repositories_file))
utils.upload_file(repositories_file)
def request_repo_patching():
utils.print_group(
"Requesting repositories patching in subsequent steps {}")
utils.set_meta_data(REPO_PATCHING_REQUIRED_META_DATA_KEY, "True")
def save_distro(distro_path):
utils.print_group("Uploading distro from {}".format(distro_path))
basename = utils.upload_file(distro_path)
utils.set_meta_data(ARCHIVE_META_DATA_KEY, basename)