def build_pex(fetch: bool) -> None:
if fetch:
extra_pex_args = [
f"--platform={plat}-{abi}"
for plat in ("linux_x86_64", "macosx_10.15_x86_64")
for abi in ("cp-37-m", "cp-38-cp38", "cp-39-cp39")
]
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.pex"
banner(f"Building {pex_name} by fetching wheels.")
else:
extra_pex_args = [f"--python={sys.executable}"]
plat = os.uname()[0].lower()
py = f"cp{''.join(map(str, sys.version_info[:2]))}"
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.{plat}-{py}.pex"
banner(f"Building {pex_name} by building wheels.")
if CONSTANTS.deploy_dir.exists():
shutil.rmtree(CONSTANTS.deploy_dir)
CONSTANTS.deploy_dir.mkdir(parents=True)
if fetch:
fetch_prebuilt_wheels(CONSTANTS.deploy_dir)
check_prebuilt_wheels_present(CONSTANTS.deploy_dir)
else:
build_pants_wheels()
build_3rdparty_wheels()
dest = Path("dist") / pex_name
with download_pex_bin() as pex_bin:
subprocess.run(
[
sys.executable,
str(pex_bin),
"-o",
str(dest),
"--no-build",
"--no-pypi",
"--disable-cache",
"-f",
str(CONSTANTS.deploy_pants_wheel_dir / CONSTANTS.pants_unstable_version),
"-f",
str(CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version),
"--no-strip-pex-env",
"--console-script=pants",
"--unzip",
*extra_pex_args,
f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}",
],
check=True,
)
if os.environ.get("PANTS_PEX_RELEASE", "") == "STABLE":
stable_dest = CONSTANTS.deploy_dir / "pex" / f"pants.{CONSTANTS.pants_stable_version}.pex"
stable_dest.parent.mkdir(parents=True, exist_ok=True)
dest.rename(stable_dest)
dest = stable_dest
green(f"Built {dest}")
subprocess.run([sys.executable, str(dest), "--version"], check=True)
green(f"Validated {dest}")
def check_prebuilt_wheels(check_dir: str) -> None:
banner(f"Checking prebuilt wheels for {CONSTANTS.pants_unstable_version}")
missing_packages = []
for package in sorted(all_packages()):
local_files = package.find_locally(
version=CONSTANTS.pants_unstable_version, search_dir=check_dir
)
if not local_files:
missing_packages.append(package.name)
continue
# If the package is cross platform, confirm that we have whls for two platforms.
is_cross_platform = not all(
local_file.name.endswith("-none-any.whl") for local_file in local_files
)
if is_cross_platform and len(local_files) != 2:
formatted_local_files = ", ".join(f.name for f in local_files)
missing_packages.append(
f"{package.name} (expected a macOS wheel and a linux wheel, but found "
f"{formatted_local_files})"
)
if missing_packages:
formatted_missing = "\n ".join(missing_packages)
die(f"Failed to find prebuilt wheels:\n {formatted_missing}")
green(f"All {len(all_packages())} pantsbuild.pants packages were fetched and are valid.")
def run_python_tests_v2() -> None:
known_v2_failures_file = "build-support/unit_test_v2_blacklist.txt"
with open(known_v2_failures_file, "r") as f:
blacklisted_targets = {line.strip() for line in f.readlines()}
with travis_section("PythonTestsV1",
"Running Python unit tests with V2 test runner"):
check_pants_pex_exists()
try:
all_targets = subprocess.run([
"./pants.pex",
"--tag=-integration",
"--filter-type=python_tests",
"filter",
"src/python::",
"tests/python::",
],
stdout=subprocess.PIPE,
encoding="utf-8",
check=True).stdout.strip().split("\n")
v2_targets = set(all_targets) - blacklisted_targets
subprocess.run(["./pants.pex", "--no-v1", "--v2", "test.pytest"] +
sorted(v2_targets) + PYTEST_PASSTHRU_ARGS,
check=True)
except subprocess.CalledProcessError:
die("Python unit tests failure (V2 test runner)")
else:
green("V2 unit tests passed.")
def run_python_tests_v1() -> None:
check_pants_pex_exists()
blacklisted_v2_targets = get_blacklisted_targets(
"unit_test_v2_blacklist.txt")
blacklisted_chroot_targets = get_blacklisted_targets(
"unit_test_chroot_blacklist.txt")
chrooted_targets = blacklisted_v2_targets - blacklisted_chroot_targets
with travis_section("PythonTestsV1",
"Running Python unit tests with V1 test runner"):
try:
subprocess.run(
["./pants.pex", "--test-pytest-chroot", "test.pytest"] +
sorted(chrooted_targets) + PYTEST_PASSTHRU_ARGS,
check=True)
subprocess.run(["./pants.pex", "test.pytest"] +
sorted(blacklisted_chroot_targets) +
PYTEST_PASSTHRU_ARGS,
check=True)
except subprocess.CalledProcessError:
die("Python unit test failure (V1 test runner")
else:
green("V1 unit tests passed.")
def build_fs_util() -> None:
# See https://www.pantsbuild.org/docs/contributions-rust for a description of fs_util. We include
# it in our releases because it can be a useful standalone tool.
with travis_section("fs_util", "Building fs_util"):
subprocess.run(
[
"build-support/bin/native/cargo",
"build",
"--release",
"--manifest-path=src/rust/engine/Cargo.toml",
"-p",
"fs_util",
],
check=True,
env={
**os.environ, "RUST_BACKTRACE": "1"
},
)
current_os = (subprocess.run(["build-support/bin/get_os.sh"],
stdout=subprocess.PIPE,
check=True).stdout.decode().strip())
dest_dir = (Path(CONSTANTS.deploy_dir) / "bin" / "fs_util" /
current_os / CONSTANTS.pants_unstable_version)
dest_dir.mkdir(parents=True, exist_ok=True)
shutil.copy("src/rust/engine/target/release/fs_util", dest_dir)
green(f"Built fs_util at {dest_dir / 'fs_util'}.")
def build_pants_wheels() -> None:
banner(f"Building Pants wheels with Python {CONSTANTS.python_version}")
version = CONSTANTS.pants_unstable_version
dest = CONSTANTS.deploy_pants_wheel_dir / version
dest.mkdir(parents=True, exist_ok=True)
args = (
"./pants",
# TODO(#9924).
"--no-dynamic-ui",
# TODO(#7654): It's not safe to use Pantsd because we're already using Pants to run
# this script.
"--concurrent",
"package",
*(package.target for package in PACKAGES),
)
with set_pants_version(CONSTANTS.pants_unstable_version):
try:
subprocess.run(args, check=True)
except subprocess.CalledProcessError as e:
failed_packages = ",".join(package.name for package in PACKAGES)
failed_targets = " ".join(package.target for package in PACKAGES)
die(
softwrap(f"""
Failed to build packages {failed_packages} for {version} with targets
{failed_targets}.
{e!r}
"""))
# TODO(#10718): Allow for sdist releases. We can build an sdist for
# `pantsbuild.pants.testutil`, but need to wire it up to the rest of our release process.
for package in PACKAGES:
found_wheels = sorted(
Path("dist").glob(f"{package}-{version}-*.whl"))
# NB: For any platform-specific wheels, like pantsbuild.pants, we assume that the
# top-level `dist` will only have wheels built for the current platform. This
# should be safe because it is not possible to build native wheels for another
# platform.
if not is_cross_platform(found_wheels) and len(found_wheels) > 1:
die(
softwrap(f"""
Found multiple wheels for {package} in the `dist/` folder, but was
expecting only one wheel: {sorted(wheel.name for wheel in found_wheels)}.
"""))
for wheel in found_wheels:
if not (dest / wheel.name).exists():
# We use `copy2` to preserve metadata.
shutil.copy2(wheel, dest)
green(f"Wrote Pants wheels to {dest}.")
banner(f"Validating Pants wheels for {CONSTANTS.python_version}.")
create_twine_venv()
subprocess.run(
[CONSTANTS.twine_venv_dir / "bin/twine", "check", dest / "*.whl"],
check=True)
green(f"Validated Pants wheels for {CONSTANTS.python_version}.")
def run_python_tests_v1() -> None:
known_v2_failures_file = "build-support/unit_test_v2_blacklist.txt"
with travis_section("PythonTestsV1",
"Running Python unit tests with V1 test runner"):
check_pants_pex_exists()
try:
subprocess.run([
"./pants.pex",
f"--target-spec-file={known_v2_failures_file}",
"test.pytest",
"--chroot",
] + PYTEST_PASSTHRU_ARGS,
check=True)
except subprocess.CalledProcessError:
die("Python unit test failure (V1 test runner")
else:
green("V1 unit tests passed.")
try:
subprocess.run([
"./pants.pex",
"--tag=-integration",
"--exclude-target-regexp=./*testprojects/.*",
"test.pytest",
"contrib::",
] + PYTEST_PASSTHRU_ARGS,
check=True)
except subprocess.CalledProcessError:
die("Contrib Python test failure")
else:
green("Contrib unit tests passed.")
def install_and_test_packages(version: str, *, extra_pip_args: list[str] | None = None) -> None:
with create_tmp_venv() as venv_tmpdir:
for pkg in PACKAGES:
pip_req = f"{pkg.name}=={version}"
banner(f"Installing and testing {pip_req}")
pkg.validate(version, venv_tmpdir, extra_pip_args or [])
green(f"Tests succeeded for {pip_req}")
def build_fs_util() -> None:
# See https://www.pantsbuild.org/docs/contributions-rust for a description of fs_util. We
# include it in our releases because it can be a useful standalone tool.
with travis_section("fs_util", "Building fs_util"):
command = ["./cargo", "build", "-p", "fs_util"]
release_mode = os.environ.get("MODE", "") != "debug"
if release_mode:
command.append("--release")
subprocess.run(command, check=True, env={**os.environ, "RUST_BACKTRACE": "1"})
current_os = (
subprocess.run(["build-support/bin/get_os.sh"], stdout=subprocess.PIPE, check=True)
.stdout.decode()
.strip()
)
dest_dir = (
Path(CONSTANTS.deploy_dir)
/ "bin"
/ "fs_util"
/ current_os
/ CONSTANTS.pants_unstable_version
)
dest_dir.mkdir(parents=True, exist_ok=True)
shutil.copy(
f"src/rust/engine/target/{'release' if release_mode else 'debug'}/fs_util",
dest_dir,
)
green(f"Built fs_util at {dest_dir / 'fs_util'}.")
def run_unit_tests(*, remote_execution_enabled: bool) -> None:
check_pants_pex_exists()
all_targets = get_all_python_tests(tag="-integration")
blacklisted_chroot_targets = get_blacklisted_targets(
"unit_test_chroot_blacklist.txt")
blacklisted_v2_targets = get_blacklisted_targets(
"unit_test_v2_blacklist.txt")
blacklisted_remote_targets = get_blacklisted_targets(
"unit_test_remote_blacklist.txt")
v1_no_chroot_targets = blacklisted_chroot_targets
v1_chroot_targets = blacklisted_v2_targets
v2_local_targets = blacklisted_remote_targets
v2_remote_targets = all_targets - v2_local_targets - v1_chroot_targets - v1_no_chroot_targets
basic_command = ["./pants.pex", "test.pytest"]
v2_command = ["./pants.pex", "--no-v1", "--v2", "test.pytest"]
v1_no_chroot_command = basic_command + sorted(
v1_no_chroot_targets) + PYTEST_PASSTHRU_ARGS
v1_chroot_command = basic_command + [
"--test-pytest-chroot"
] + sorted(v1_chroot_targets) + PYTEST_PASSTHRU_ARGS
v2_local_command = v2_command + sorted(v2_local_targets)
if not remote_execution_enabled:
v2_local_targets = v2_local_targets | v2_remote_targets
v2_local_command = v2_command + sorted(v2_local_targets)
else:
with travis_section(
"UnitTestsRemote", "Running unit tests via remote execution"
), get_remote_execution_oauth_token_path() as oauth_token_path:
v2_remote_command = v2_command[:-1] + [
"--pants-config-files=pants.remote.ini",
# We turn off speculation to reduce the risk of flakiness, where a test passes locally but
# fails remoting and we have a race condition for which environment executes first.
"--process-execution-speculation-strategy=none",
f"--remote-oauth-bearer-token-path={oauth_token_path}",
"test.pytest",
] + sorted(v2_remote_targets)
try:
subprocess.run(v2_remote_command, check=True)
except subprocess.CalledProcessError:
die("Unit test failure (remote execution)")
else:
green("Unit tests passed (remote execution)")
with travis_section("UnitTestsLocal",
"Running unit tests via local execution"):
try:
subprocess.run(v2_local_command, check=True)
subprocess.run(v1_chroot_command, check=True)
subprocess.run(v1_no_chroot_command, check=True)
except subprocess.CalledProcessError:
die("Unit test failure (local execution)")
else:
green("Unit tests passed (local execution)")
def run_shellcheck() -> None:
targets = glob("./**/*.sh", recursive=True) + ["./pants"]
command = ["shellcheck", "--shell=bash"] + targets
try:
subprocess.run(command, check=True)
except subprocess.CalledProcessError:
die("Please fix the above errors and run again.")
else:
green("./pants passed the shellcheck!")
def build_pants_wheels() -> None:
banner("Building Pants wheels")
version = CONSTANTS.pants_unstable_version
destination = CONSTANTS.deploy_pants_wheel_dir / version
destination.mkdir(parents=True, exist_ok=True)
def build(packages: Iterable[Package],
bdist_wheel_flags: Iterable[str]) -> None:
args = (
"./pants",
# TODO(#9924).
"--no-dynamic-ui",
# TODO(#7654): It's not safe to use Pantsd because we're already using Pants to run
# this script.
"--concurrent",
"setup-py",
*(package.target for package in packages),
"--",
"bdist_wheel",
*bdist_wheel_flags,
)
try:
subprocess.run(args, check=True)
for package in packages:
found_wheels = sorted(
Path("dist").glob(f"{package}-{version}-*.whl"))
# NB: For any platform-specific wheels, like pantsbuild.pants, we assume that the
# top-level `dist` will only have wheels built for the current platform. This
# should be safe because it is not possible to build native wheels for another
# platform.
if not is_cross_platform(
found_wheels) and len(found_wheels) > 1:
raise ValueError(
f"Found multiple wheels for {package} in the `dist/` folder, but was "
f"expecting only one wheel: {sorted(wheel.name for wheel in found_wheels)}."
)
for wheel in found_wheels:
if not (destination / wheel.name).exists():
# We use `copy2` to preserve metadata.
shutil.copy2(wheel, destination)
except subprocess.CalledProcessError as e:
failed_packages = ",".join(package.name for package in packages)
failed_targets = " ".join(package.target for package in packages)
die(
f"Failed to build packages {failed_packages} for {version} with targets "
f"{failed_targets}.\n\n{e!r}", )
packages_by_flags = defaultdict(list)
for package in PACKAGES:
packages_by_flags[package.bdist_wheel_flags].append(package)
with set_pants_version(CONSTANTS.pants_unstable_version):
for flags, packages in packages_by_flags.items():
build(packages, flags)
green(f"Wrote Pants wheels to {destination}.")
def main() -> None:
banner("CI BEGINS")
args = create_parser().parse_args()
setup_environment(python_version=args.python_version)
with maybe_get_remote_execution_oauth_token_path(
remote_execution_enabled=args.remote_execution_enabled
) as remote_execution_oauth_token_path:
if args.bootstrap:
bootstrap(
clean=args.bootstrap_clean,
try_to_skip_rust_compilation=args.
bootstrap_try_to_skip_rust_compilation,
python_version=args.python_version,
)
set_run_from_pex()
if args.githooks:
run_githooks()
if args.sanity_checks:
run_sanity_checks()
if args.lint:
run_lint(oauth_token_path=remote_execution_oauth_token_path)
if args.doc_gen:
run_doc_gen_tests()
if args.clippy:
run_clippy()
if args.cargo_audit:
run_cargo_audit()
if args.unit_tests:
run_unit_tests(oauth_token_path=remote_execution_oauth_token_path)
if args.rust_tests:
run_rust_tests()
if args.jvm_tests:
run_jvm_tests()
if args.integration_tests_v1:
run_integration_tests_v1(shard=args.integration_shard)
if args.integration_tests_v2:
run_integration_tests_v2(
oauth_token_path=remote_execution_oauth_token_path)
if args.plugin_tests:
run_plugin_tests(
oauth_token_path=remote_execution_oauth_token_path)
if args.platform_specific_tests:
run_platform_specific_tests()
banner("CI ENDS")
print()
green("SUCCESS")
def build_3rdparty_wheels() -> None:
banner(f"Building 3rdparty wheels with Python {CONSTANTS.python_version}")
dest = CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version
pkg_tgts = [pkg.target for pkg in PACKAGES]
with create_tmp_venv() as bin_dir:
deps = (subprocess.run(
[
"./pants",
"--concurrent",
"dependencies",
"--transitive",
*pkg_tgts,
],
stdout=subprocess.PIPE,
check=True,
).stdout.decode().strip().splitlines())
python_requirements = (subprocess.run(
[
"./pants",
"--concurrent",
"filter",
"--target-type=python_requirement",
*deps,
],
stdout=subprocess.PIPE,
check=True,
).stdout.decode().strip().splitlines())
if not python_requirements:
die(
softwrap(f"""
No 3rd-party dependencies detected for {pkg_tgts}. Is `./pants dependencies`
broken?
"""))
reqs = itertools.chain.from_iterable(obj["requirements"]
for obj in json.loads(
subprocess.run(
[
"./pants",
"--concurrent",
"peek",
*python_requirements,
],
stdout=subprocess.PIPE,
check=True,
).stdout))
subprocess.run(
[bin_dir / "pip", "wheel", f"--wheel-dir={dest}", *reqs],
check=True,
)
green(f"Wrote 3rdparty wheels to {dest}")
def run_shellcheck() -> None:
targets = set(glob("./**/*.sh", recursive=True)) | {
"./pants",
"./pants2",
"./build-support/pants_venv",
"./build-support/virtualenv",
"./build-support/githooks/pre-commit",
"./build-support/githooks/prepare-commit-msg",
}
targets -= set(glob("./build-support/bin/native/src/**/*.sh", recursive=True))
targets -= set(glob("./build-support/virtualenv.dist/**/*.sh", recursive=True))
command = ["shellcheck", "--shell=bash", "--external-sources"] + sorted(targets)
try:
subprocess.run(command, check=True)
except subprocess.CalledProcessError:
die("Please fix the above errors and run again.")
else:
green("./pants passed the shellcheck!")
def main() -> None:
if not Path("pants.pex").is_file:
die("pants.pex not found! Ensure you are in the repository root, then run " \
"'./build-support/bin/ci.sh -b' to bootstrap pants.pex with Python 3 or " \
"'./build-support/bin/ci.sh -2b' to bootstrap pants.pex with Python 2.")
expected_abis = frozenset(create_parser().parse_args().abis)
with zipfile.ZipFile("pants.pex", "r") as pex:
with pex.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
parsed_abis = frozenset(
parse_abi_from_filename(filename)
for filename in json.loads(pex_info_content)["distributions"].keys()
if parse_abi_from_filename(filename) != "none"
)
if not parsed_abis.issubset(expected_abis):
die("pants.pex was built with the incorrect ABI. Expected wheels with: {}, found: {}."
.format(' or '.join(sorted(expected_abis)), ', '.join(sorted(parsed_abis))))
green("Success. The pants.pex was built with wheels carrying the expected ABIs: {}."
.format(', '.join(sorted(parsed_abis))))
def check_pants_wheels_present(check_dir: str | Path) -> None:
banner(f"Checking prebuilt wheels for {CONSTANTS.pants_unstable_version}")
missing_packages = []
for package in PACKAGES:
local_files = package.find_locally(
version=CONSTANTS.pants_unstable_version, search_dir=check_dir)
if not local_files:
missing_packages.append(package.name)
continue
if is_cross_platform(local_files) and len(local_files) != 6:
formatted_local_files = ", ".join(f.name for f in local_files)
missing_packages.append(
f"{package.name} (expected 6 wheels, {{macosx, linux}} x {{cp37m, cp38, cp39}}, "
f"but found {formatted_local_files})")
if missing_packages:
formatted_missing = "\n ".join(missing_packages)
die(f"Failed to find prebuilt wheels:\n {formatted_missing}")
green(
f"All {len(PACKAGES)} pantsbuild.pants packages were fetched and are valid."
)
def main() -> None:
if not Path("pants.pex").is_file:
die("pants.pex not found! Ensure you are in the repository root, then run "
"`./build-support/bin/ci.py --bootstrap` to bootstrap pants.pex with Python 3.6 or "
"`./build-support/bin/ci.py --bootstrap --python-version 2.7` to bootstrap pants.pex with "
"Python 2.7.")
expected_abis = frozenset(create_parser().parse_args().abis)
with zipfile.ZipFile("pants.pex", "r") as pex:
with pex.open("PEX-INFO", "r") as pex_info:
pex_info_content = pex_info.readline().decode()
parsed_abis = frozenset(
parse_abi_from_filename(filename)
for filename in json.loads(pex_info_content)["distributions"].keys()
if parse_abi_from_filename(filename) != "none"
)
if not parsed_abis.issubset(expected_abis):
die("pants.pex was built with the incorrect ABI. Expected wheels with: {}, found: {}."
.format(' or '.join(sorted(expected_abis)), ', '.join(sorted(parsed_abis))))
green("Success. The pants.pex was built with wheels carrying the expected ABIs: {}."
.format(', '.join(sorted(parsed_abis))))
def main() -> None:
banner("CI BEGINS")
args = create_parser().parse_args()
setup_environment(python_version=args.python_version)
if args.bootstrap:
bootstrap(clean=args.bootstrap_clean,
python_version=args.python_version)
set_run_from_pex()
if args.githooks:
run_githooks()
if args.sanity_checks:
run_sanity_checks()
if args.lint:
run_lint()
if args.doc_gen:
run_doc_gen_tests()
if args.clippy:
run_clippy()
if args.cargo_audit:
run_cargo_audit()
if args.python_tests_v1:
run_python_tests_v1()
if args.python_tests_v2:
run_python_tests_v2()
if args.rust_tests:
run_rust_tests()
if args.jvm_tests:
run_jvm_tests()
if args.integration_tests:
run_integration_tests(shard=args.integration_shard)
if args.plugin_tests:
run_plugin_tests()
if args.platform_specific_tests:
run_platform_specific_tests()
banner("CI ENDS")
print()
green("SUCCESS")
def run_shellcheck() -> None:
targets = set(glob("./**/*.sh", recursive=True)) | {
"./pants",
"./pants2",
"./build-support/pants_venv",
"./build-support/virtualenv",
"./build-support/githooks/pre-commit",
"./build-support/githooks/prepare-commit-msg",
}
targets -= set(
glob("./build-support/bin/native/src/**/*.sh", recursive=True))
targets -= set(
glob("./build-support/virtualenv.dist/**/*.sh", recursive=True))
command = ["shellcheck", "--shell=bash", "--external-sources"
] + sorted(targets)
try:
subprocess.run(command, check=True)
except subprocess.CalledProcessError:
die("Please fix the above errors and run again.")
else:
green("./pants passed the shellcheck!")
def main() -> None:
banner("CI BEGINS")
args = create_parser().parse_args()
setup_environment(python_version=args.python_version)
if args.bootstrap:
bootstrap(
clean=args.bootstrap_clean,
try_to_skip_rust_compilation=args.
bootstrap_try_to_skip_rust_compilation,
python_version=args.python_version,
)
set_run_from_pex()
if args.githooks:
run_githooks()
if args.smoke_tests:
run_smoke_tests()
if args.lint:
run_lint(remote_cache_enabled=args.remote_cache_enabled)
if args.clippy:
run_clippy()
if args.cargo_audit:
run_cargo_audit()
if args.unit_tests or args.integration_tests:
run_python_tests(
include_unit=args.unit_tests,
include_integration=args.integration_tests,
remote_cache_enabled=args.remote_cache_enabled,
)
if args.rust_tests:
run_rust_tests()
if args.platform_specific_tests:
run_platform_specific_tests()
banner("CI ENDS")
print()
green("SUCCESS")
def run_v2_tests(*,
targets: Set[str],
execution_strategy: str,
oauth_token_path: Optional[str] = None) -> None:
try:
command = (["./pants.pex", "--no-v1", "--v2", "test.pytest"] +
sorted(targets) + PYTEST_PASSTHRU_ARGS)
if oauth_token_path is not None:
command[3:3] = [
"--pants-config-files=pants.remote.ini",
# We turn off speculation to reduce the risk of flakiness, where a test passes locally but
# fails remoting and we have a race condition for which environment executes first.
"--process-execution-speculation-strategy=none",
f"--remote-oauth-bearer-token-path={oauth_token_path}"
]
subprocess.run(command, check=True)
except subprocess.CalledProcessError:
die(f"V2 unit tests failure ({execution_strategy} build execution)."
)
else:
green(
f"V2 unit tests passed ({execution_strategy} build execution)."
)
def build_pex(fetch: bool) -> None:
if fetch:
extra_pex_args = [
f"--platform={plat}-{abi}"
for plat in ("linux_x86_64", "macosx_10.15_x86_64")
for abi in ("cp-37-m", "cp-38-cp38", "cp-39-cp39")
]
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.pex"
banner(f"Building {pex_name} by fetching wheels.")
else:
extra_pex_args = [f"--python={sys.executable}"]
plat = os.uname()[0].lower()
py = f"cp{''.join(map(str, sys.version_info[:2]))}"
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.{plat}-{py}.pex"
banner(f"Building {pex_name} by building wheels.")
if CONSTANTS.deploy_dir.exists():
shutil.rmtree(CONSTANTS.deploy_dir)
CONSTANTS.deploy_dir.mkdir(parents=True)
if fetch:
fetch_prebuilt_wheels(CONSTANTS.deploy_dir, include_3rdparty=True)
check_pants_wheels_present(CONSTANTS.deploy_dir)
else:
build_pants_wheels()
build_3rdparty_wheels()
# We need to both run Pex and the Pants PEX we build with it with clean environments since we
# ourselves may be running via `./pants run ...` which injects confounding environment variables
# like PEX_EXTRA_SYS_PATH, PEX_PATH and PEX_ROOT that need not or should not apply to these
# sub-processes.
env = {k: v for k, v in os.environ.items() if not k.startswith("PEX_")}
dest = Path("dist") / pex_name
with download_pex_bin() as pex_bin:
subprocess.run(
[
sys.executable,
str(pex_bin),
"-o",
str(dest),
"--no-build",
"--no-pypi",
"--disable-cache",
"-f",
str(CONSTANTS.deploy_pants_wheel_dir / CONSTANTS.pants_unstable_version),
"-f",
str(CONSTANTS.deploy_3rdparty_wheel_dir / CONSTANTS.pants_unstable_version),
"--no-strip-pex-env",
"--console-script=pants",
*extra_pex_args,
f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}",
],
env=env,
check=True,
)
if os.environ.get("PANTS_PEX_RELEASE", "") == "STABLE":
stable_dest = CONSTANTS.deploy_dir / "pex" / f"pants.{CONSTANTS.pants_stable_version}.pex"
stable_dest.parent.mkdir(parents=True, exist_ok=True)
dest.rename(stable_dest)
dest = stable_dest
green(f"Built {dest}")
subprocess.run([sys.executable, str(dest), "--no-pantsd", "--version"], env=env, check=True)
green(f"Validated {dest}")
def build_pex(fetch: bool) -> None:
stable = os.environ.get("PANTS_PEX_RELEASE", "") == "STABLE"
if fetch:
# TODO: Support macOS on ARM64.
extra_pex_args = [
"--python-shebang",
"/usr/bin/env python",
"--interpreter-constraint",
"CPython>=3.7,<3.10",
*(f"--platform={plat}-{abi}"
for plat in ("linux_x86_64", "macosx_10.15_x86_64")
for abi in ("cp-37-m", "cp-38-cp38", "cp-39-cp39")),
]
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.pex"
banner(f"Building {pex_name} by fetching wheels.")
else:
# TODO: Support macOS on ARM64. Will require qualifying the pex name with the arch.
major, minor = sys.version_info[:2]
extra_pex_args = [
f"--interpreter-constraint=CPython=={major}.{minor}.*",
f"--python={sys.executable}",
]
plat = os.uname()[0].lower()
py = f"cp{major}{minor}"
pex_name = f"pants.{CONSTANTS.pants_unstable_version}.{plat}-{py}.pex"
banner(f"Building {pex_name} by building wheels.")
if CONSTANTS.deploy_dir.exists():
shutil.rmtree(CONSTANTS.deploy_dir)
CONSTANTS.deploy_dir.mkdir(parents=True)
if fetch:
fetch_prebuilt_wheels(CONSTANTS.deploy_dir, include_3rdparty=True)
check_pants_wheels_present(CONSTANTS.deploy_dir)
if stable:
reversion_prebuilt_wheels()
else:
build_pants_wheels()
build_3rdparty_wheels()
# We need to both run Pex and the Pants PEX we build with it with clean environments since we
# ourselves may be running via `./pants run ...` which injects confounding environment variables
# like PEX_EXTRA_SYS_PATH, PEX_PATH and PEX_ROOT that need not or should not apply to these
# sub-processes.
env = {k: v for k, v in os.environ.items() if not k.startswith("PEX_")}
dest = Path("dist") / pex_name
with download_pex_bin() as pex_bin:
subprocess.run(
[
sys.executable,
str(pex_bin),
"-o",
str(dest),
"--no-build",
"--no-pypi",
"--disable-cache",
"-f",
str(CONSTANTS.deploy_pants_wheel_dir /
CONSTANTS.pants_unstable_version),
"-f",
str(CONSTANTS.deploy_3rdparty_wheel_dir /
CONSTANTS.pants_unstable_version),
"--no-strip-pex-env",
"--console-script=pants",
*extra_pex_args,
f"pantsbuild.pants=={CONSTANTS.pants_unstable_version}",
"--venv",
],
env=env,
check=True,
)
if stable:
stable_dest = CONSTANTS.deploy_dir / "pex" / f"pants.{CONSTANTS.pants_stable_version}.pex"
stable_dest.parent.mkdir(parents=True, exist_ok=True)
dest.rename(stable_dest)
dest = stable_dest
green(f"Built {dest}")
with TemporaryDirectory() as tmpdir:
validated_pex_path = Path(tmpdir, "pants.pex")
shutil.copyfile(dest, validated_pex_path)
validated_pex_path.chmod(0o777)
Path(tmpdir, "BUILD_ROOT").touch()
# We also need to filter out Pants options like `PANTS_CONFIG_FILES` and disable certain internal backends.
env = {k: v for k, v in env.items() if not k.startswith("PANTS_")}
env.update(DISABLED_BACKENDS_CONFIG)
subprocess.run([validated_pex_path, "--version"],
env=env,
check=True,
cwd=dest.parent)
green(f"Validated {dest}")
