def build_gcc(client, image, host_platform):
"""Build GCC in the Docker image."""
gcc_archive = download_entry("gcc", DOWNLOADS_PATH)
gmp_archive = download_entry("gmp", DOWNLOADS_PATH)
isl_archive = download_entry("isl", DOWNLOADS_PATH)
mpc_archive = download_entry("mpc", DOWNLOADS_PATH)
mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
log("copying archives to container...")
for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive,
mpfr_archive):
build_env.copy_file(a)
build_env.copy_file(toolchain_archive_path("binutils", host_platform))
build_env.copy_file(SUPPORT / "build-gcc.sh")
env = {
"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"GMP_VERSION": DOWNLOADS["gmp"]["version"],
"ISL_VERSION": DOWNLOADS["isl"]["version"],
"MPC_VERSION": DOWNLOADS["mpc"]["version"],
"MPFR_VERSION": DOWNLOADS["mpfr"]["version"],
}
add_env_common(env)
build_env.run("build-gcc.sh", environment=env)
build_env.get_tools_archive(
toolchain_archive_path("gcc", host_platform), "host")
def add_target_env(env, platform, build_env):
add_env_common(env)
env["PYBUILD_PLATFORM"] = platform
env["TOOLS_PATH"] = build_env.tools_path
if platform == "linux64":
env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
env["TARGET_TRIPLE"] = "x86_64-unknown-linux-gnu"
if platform == "macos":
env["MACOSX_DEPLOYMENT_TARGET"] = MACOSX_DEPLOYMENT_TARGET
env["BUILD_TRIPLE"] = "x86_64-apple-darwin18.7.0"
env["TARGET_TRIPLE"] = "x86_64-apple-darwin18.7.0"
env["PATH"] = "/usr/bin:/bin"
env["EXTRA_TARGET_CFLAGS"] = " ".join([
# Suppress extremely verbose warnings we see with LLVM 10.
"-Wno-nullability-completeness",
"-Wno-expansion-to-defined",
# LLVM 11 contains commit https://reviews.llvm.org/D83250,
# which enables -Werror for undef-prefix=TARGET_OS_.
# However, the macOS SDK has headers that reference deprecated
# TARGET_OS defines, like TARGET_OS_EMBEDDED. So LLVM 11 refuses
# to work with the macOS SDKs out of the box. We work around
# this by undoing the -Werror=undef-prefix in that commit.
"-Wno-undef-prefix",
])
# This path exists on GitHub Actions workers and is the 10.15 SDK. Using this
# SDK works around issues with the 11.0 SDK not working with CPython.
macosx_sdk_path_10_15 = (
"/Applications/Xcode_12.1.1.app/Contents/Developer/Platforms/MacOSX.platform"
"/Developer/SDKs/MacOSX.sdk ")
if "MACOS_SDK_PATH" in os.environ:
sdk_path = os.environ["MACOS_SDK_PATH"]
elif os.path.exists(macosx_sdk_path_10_15):
sdk_path = macosx_sdk_path_10_15
else:
# macOS SDK has historically been in /usr courtesy of an
# installer provided by Xcode. But with Catalina, the files
# are now typically in
# /Applications/Xcode.app/Contents/Developer/Platforms/.
# The proper way to resolve this path is with xcrun, which
# will give us the headers that Xcode is configured to use.
res = subprocess.run(
["xcrun", "--show-sdk-path"],
check=True,
capture_output=True,
encoding="utf-8",
)
sdk_path = res.stdout.strip()
if not os.path.exists(sdk_path):
raise Exception("macOS SDK path %s does not exist" % sdk_path)
env["MACOS_SDK_PATH"] = sdk_path
env["CPATH"] = "%s/usr/include" % sdk_path
def build_clang(client, image, host_platform):
if "linux" in host_platform:
cmake_archive = download_entry("cmake-linux-bin", DOWNLOADS_PATH)
ninja_archive = download_entry("ninja-linux-bin", DOWNLOADS_PATH)
elif "macos" in host_platform:
cmake_archive = download_entry("cmake-macos-bin", DOWNLOADS_PATH)
ninja_archive = download_entry("ninja-macos-bin", DOWNLOADS_PATH)
clang_archive = download_entry("clang", DOWNLOADS_PATH)
clang_rt_archive = download_entry("clang-compiler-rt", DOWNLOADS_PATH)
lld_archive = download_entry("lld", DOWNLOADS_PATH)
llvm_archive = download_entry("llvm", DOWNLOADS_PATH)
libcxx_archive = download_entry("libc++", DOWNLOADS_PATH)
libcxxabi_archive = download_entry("libc++abi", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
log("copying archives to container...")
for a in (
cmake_archive,
ninja_archive,
clang_archive,
clang_rt_archive,
lld_archive,
llvm_archive,
libcxx_archive,
libcxxabi_archive,
):
build_env.copy_file(a)
tools_path = "clang-%s" % host_platform
build_sh = "build-clang-%s.sh" % host_platform
binutils = install_binutils(host_platform)
gcc = binutils
env = {
"CLANG_COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"],
"CLANG_VERSION": DOWNLOADS["clang"]["version"],
"CMAKE_VERSION": DOWNLOADS["cmake-linux-bin"]["version"],
"COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"LIBCXX_VERSION": DOWNLOADS["libc++"]["version"],
"LIBCXXABI_VERSION": DOWNLOADS["libc++abi"]["version"],
"LLD_VERSION": DOWNLOADS["lld"]["version"],
"LLVM_VERSION": DOWNLOADS["llvm"]["version"],
}
add_env_common(env)
build_env.install_toolchain(BUILD, host_platform, binutils=binutils, gcc=gcc)
build_env.copy_file(SUPPORT / build_sh)
build_env.run(build_sh, environment=env)
build_env.get_tools_archive(
toolchain_archive_path("clang", host_platform), tools_path
)
def build_binutils(client, image, host_platform):
"""Build binutils in the Docker image."""
archive = download_entry("binutils", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / "build-binutils.sh")
env = {"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]}
add_env_common(env)
build_env.run(
"build-binutils.sh",
environment=env,
)
build_env.get_tools_archive(
toolchain_archive_path("binutils", host_platform), "host")
def add_target_env(env, build_platform, target_triple, build_env):
add_env_common(env)
settings = get_target_settings(TARGETS_CONFIG, target_triple)
env["HOST_CC"] = settings["host_cc"]
env["CC"] = settings["target_cc"]
env["PYBUILD_PLATFORM"] = build_platform
env["TOOLS_PATH"] = build_env.tools_path
extra_target_cflags = list(settings.get("target_cflags", []))
extra_target_ldflags = list(settings.get("target_ldflags", []))
extra_host_cflags = []
extra_host_ldflags = []
if build_platform == "linux64":
env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
# TODO should the musl target be normalized?
if target_triple == "x86_64-unknown-linux-musl":
env["TARGET_TRIPLE"] = "x86_64-unknown-linux-gnu"
else:
env["TARGET_TRIPLE"] = target_triple
if build_platform == "macos":
machine = platform.machine()
if machine == "arm64":
env["BUILD_TRIPLE"] = "aarch64-apple-darwin"
elif machine == "x86_64":
env["BUILD_TRIPLE"] = "x86_64-apple-darwin"
else:
raise Exception("unhandled macOS machine value: %s" % machine)
# Sniff out the Apple SDK minimum deployment target from cflags and
# export in its own variable. This is used by CPython's configure, as
# it doesn't sniff the cflag.
for flag in extra_target_cflags:
m = re.search("-version-min=(.*)$", flag)
if m:
env["APPLE_MIN_DEPLOYMENT_TARGET"] = m.group(1)
break
else:
raise Exception("could not find minimum Apple SDK version in cflags")
sdk_platform = settings["apple_sdk_platform"]
env["APPLE_SDK_PLATFORM"] = sdk_platform
env["TARGET_TRIPLE"] = target_triple
# We don't have build isolation on macOS. We nerf PATH to prevent
# non-system (e.g. Homebrew) executables from being used.
env["PATH"] = "/usr/bin:/bin"
if "APPLE_SDK_PATH" in os.environ:
sdk_path = os.environ["APPLE_SDK_PATH"]
else:
# macOS SDK has historically been in /usr courtesy of an
# installer provided by Xcode. But with Catalina, the files
# are now typically in
# /Applications/Xcode.app/Contents/Developer/Platforms/.
# The proper way to resolve this path is with xcrun, which
# will give us the headers that Xcode is configured to use.
res = subprocess.run(
["xcrun", "--sdk", sdk_platform, "--show-sdk-path"],
check=True,
capture_output=True,
encoding="utf-8",
)
sdk_path = res.stdout.strip()
if not os.path.exists(sdk_path):
raise Exception("macOS SDK path %s does not exist" % sdk_path)
# Grab the version from the SDK so we can put it in PYTHON.json.
sdk_settings_path = pathlib.Path(sdk_path) / "SDKSettings.json"
with sdk_settings_path.open("rb") as fh:
sdk_settings = json.load(fh)
env["APPLE_SDK_VERSION"] = sdk_settings["Version"]
env["APPLE_SDK_CANONICAL_NAME"] = sdk_settings["CanonicalName"]
extra_target_cflags.extend(["-isysroot", sdk_path])
extra_target_ldflags.extend(["-isysroot", sdk_path])
# The host SDK may be for a different platform from the target SDK.
# Resolve that separately.
if "APPLE_HOST_SDK_PATH" in os.environ:
host_sdk_path = os.environ["APPLE_HOST_SDK_PATH"]
else:
host_sdk_path = subprocess.run(
["xcrun", "--show-sdk-path"],
check=True,
capture_output=True,
encoding="utf-8",
).stdout.strip()
if not os.path.exists(host_sdk_path):
raise Exception("macOS host SDK path %s does not exist" % host_sdk_path)
extra_host_cflags.extend(["-isysroot", host_sdk_path])
extra_host_ldflags.extend(["-isysroot", host_sdk_path])
env["EXTRA_HOST_CFLAGS"] = " ".join(extra_host_cflags)
env["EXTRA_HOST_LDFLAGS"] = " ".join(extra_host_ldflags)
env["EXTRA_TARGET_CFLAGS"] = " ".join(extra_target_cflags)
env["EXTRA_TARGET_LDFLAGS"] = " ".join(extra_target_ldflags)
