def build_gcc(client, image, host_platform): """Build GCC in the Docker image.""" gcc_archive = download_entry("gcc", DOWNLOADS_PATH) gmp_archive = download_entry("gmp", DOWNLOADS_PATH) isl_archive = download_entry("isl", DOWNLOADS_PATH) mpc_archive = download_entry("mpc", DOWNLOADS_PATH) mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH) with build_environment(client, image) as build_env: install_sccache(build_env) log("copying archives to container...") for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive, mpfr_archive): build_env.copy_file(a) build_env.copy_file(toolchain_archive_path("binutils", host_platform)) build_env.copy_file(SUPPORT / "build-gcc.sh") env = { "BINUTILS_VERSION": DOWNLOADS["binutils"]["version"], "GCC_VERSION": DOWNLOADS["gcc"]["version"], "GMP_VERSION": DOWNLOADS["gmp"]["version"], "ISL_VERSION": DOWNLOADS["isl"]["version"], "MPC_VERSION": DOWNLOADS["mpc"]["version"], "MPFR_VERSION": DOWNLOADS["mpfr"]["version"], } add_env_common(env) build_env.run("build-gcc.sh", environment=env) build_env.get_tools_archive( toolchain_archive_path("gcc", host_platform), "host")
def add_target_env(env, platform, build_env): add_env_common(env) env["PYBUILD_PLATFORM"] = platform env["TOOLS_PATH"] = build_env.tools_path if platform == "linux64": env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" env["TARGET_TRIPLE"] = "x86_64-unknown-linux-gnu" if platform == "macos": env["MACOSX_DEPLOYMENT_TARGET"] = MACOSX_DEPLOYMENT_TARGET env["BUILD_TRIPLE"] = "x86_64-apple-darwin18.7.0" env["TARGET_TRIPLE"] = "x86_64-apple-darwin18.7.0" env["PATH"] = "/usr/bin:/bin" env["EXTRA_TARGET_CFLAGS"] = " ".join([ # Suppress extremely verbose warnings we see with LLVM 10. "-Wno-nullability-completeness", "-Wno-expansion-to-defined", # LLVM 11 contains commit https://reviews.llvm.org/D83250, # which enables -Werror for undef-prefix=TARGET_OS_. # However, the macOS SDK has headers that reference deprecated # TARGET_OS defines, like TARGET_OS_EMBEDDED. So LLVM 11 refuses # to work with the macOS SDKs out of the box. We work around # this by undoing the -Werror=undef-prefix in that commit. "-Wno-undef-prefix", ]) # This path exists on GitHub Actions workers and is the 10.15 SDK. Using this # SDK works around issues with the 11.0 SDK not working with CPython. macosx_sdk_path_10_15 = ( "/Applications/Xcode_12.1.1.app/Contents/Developer/Platforms/MacOSX.platform" "/Developer/SDKs/MacOSX.sdk ") if "MACOS_SDK_PATH" in os.environ: sdk_path = os.environ["MACOS_SDK_PATH"] elif os.path.exists(macosx_sdk_path_10_15): sdk_path = macosx_sdk_path_10_15 else: # macOS SDK has historically been in /usr courtesy of an # installer provided by Xcode. But with Catalina, the files # are now typically in # /Applications/Xcode.app/Contents/Developer/Platforms/. # The proper way to resolve this path is with xcrun, which # will give us the headers that Xcode is configured to use. res = subprocess.run( ["xcrun", "--show-sdk-path"], check=True, capture_output=True, encoding="utf-8", ) sdk_path = res.stdout.strip() if not os.path.exists(sdk_path): raise Exception("macOS SDK path %s does not exist" % sdk_path) env["MACOS_SDK_PATH"] = sdk_path env["CPATH"] = "%s/usr/include" % sdk_path
def build_clang(client, image, host_platform): if "linux" in host_platform: cmake_archive = download_entry("cmake-linux-bin", DOWNLOADS_PATH) ninja_archive = download_entry("ninja-linux-bin", DOWNLOADS_PATH) elif "macos" in host_platform: cmake_archive = download_entry("cmake-macos-bin", DOWNLOADS_PATH) ninja_archive = download_entry("ninja-macos-bin", DOWNLOADS_PATH) clang_archive = download_entry("clang", DOWNLOADS_PATH) clang_rt_archive = download_entry("clang-compiler-rt", DOWNLOADS_PATH) lld_archive = download_entry("lld", DOWNLOADS_PATH) llvm_archive = download_entry("llvm", DOWNLOADS_PATH) libcxx_archive = download_entry("libc++", DOWNLOADS_PATH) libcxxabi_archive = download_entry("libc++abi", DOWNLOADS_PATH) with build_environment(client, image) as build_env: install_sccache(build_env) log("copying archives to container...") for a in ( cmake_archive, ninja_archive, clang_archive, clang_rt_archive, lld_archive, llvm_archive, libcxx_archive, libcxxabi_archive, ): build_env.copy_file(a) tools_path = "clang-%s" % host_platform build_sh = "build-clang-%s.sh" % host_platform binutils = install_binutils(host_platform) gcc = binutils env = { "CLANG_COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"], "CLANG_VERSION": DOWNLOADS["clang"]["version"], "CMAKE_VERSION": DOWNLOADS["cmake-linux-bin"]["version"], "COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"], "GCC_VERSION": DOWNLOADS["gcc"]["version"], "LIBCXX_VERSION": DOWNLOADS["libc++"]["version"], "LIBCXXABI_VERSION": DOWNLOADS["libc++abi"]["version"], "LLD_VERSION": DOWNLOADS["lld"]["version"], "LLVM_VERSION": DOWNLOADS["llvm"]["version"], } add_env_common(env) build_env.install_toolchain(BUILD, host_platform, binutils=binutils, gcc=gcc) build_env.copy_file(SUPPORT / build_sh) build_env.run(build_sh, environment=env) build_env.get_tools_archive( toolchain_archive_path("clang", host_platform), tools_path )
def build_binutils(client, image, host_platform): """Build binutils in the Docker image.""" archive = download_entry("binutils", DOWNLOADS_PATH) with build_environment(client, image) as build_env: install_sccache(build_env) build_env.copy_file(archive) build_env.copy_file(SUPPORT / "build-binutils.sh") env = {"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]} add_env_common(env) build_env.run( "build-binutils.sh", environment=env, ) build_env.get_tools_archive( toolchain_archive_path("binutils", host_platform), "host")
def add_target_env(env, build_platform, target_triple, build_env): add_env_common(env) settings = get_target_settings(TARGETS_CONFIG, target_triple) env["HOST_CC"] = settings["host_cc"] env["CC"] = settings["target_cc"] env["PYBUILD_PLATFORM"] = build_platform env["TOOLS_PATH"] = build_env.tools_path extra_target_cflags = list(settings.get("target_cflags", [])) extra_target_ldflags = list(settings.get("target_ldflags", [])) extra_host_cflags = [] extra_host_ldflags = [] if build_platform == "linux64": env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" # TODO should the musl target be normalized? if target_triple == "x86_64-unknown-linux-musl": env["TARGET_TRIPLE"] = "x86_64-unknown-linux-gnu" else: env["TARGET_TRIPLE"] = target_triple if build_platform == "macos": machine = platform.machine() if machine == "arm64": env["BUILD_TRIPLE"] = "aarch64-apple-darwin" elif machine == "x86_64": env["BUILD_TRIPLE"] = "x86_64-apple-darwin" else: raise Exception("unhandled macOS machine value: %s" % machine) # Sniff out the Apple SDK minimum deployment target from cflags and # export in its own variable. This is used by CPython's configure, as # it doesn't sniff the cflag. for flag in extra_target_cflags: m = re.search("-version-min=(.*)$", flag) if m: env["APPLE_MIN_DEPLOYMENT_TARGET"] = m.group(1) break else: raise Exception("could not find minimum Apple SDK version in cflags") sdk_platform = settings["apple_sdk_platform"] env["APPLE_SDK_PLATFORM"] = sdk_platform env["TARGET_TRIPLE"] = target_triple # We don't have build isolation on macOS. We nerf PATH to prevent # non-system (e.g. Homebrew) executables from being used. env["PATH"] = "/usr/bin:/bin" if "APPLE_SDK_PATH" in os.environ: sdk_path = os.environ["APPLE_SDK_PATH"] else: # macOS SDK has historically been in /usr courtesy of an # installer provided by Xcode. But with Catalina, the files # are now typically in # /Applications/Xcode.app/Contents/Developer/Platforms/. # The proper way to resolve this path is with xcrun, which # will give us the headers that Xcode is configured to use. res = subprocess.run( ["xcrun", "--sdk", sdk_platform, "--show-sdk-path"], check=True, capture_output=True, encoding="utf-8", ) sdk_path = res.stdout.strip() if not os.path.exists(sdk_path): raise Exception("macOS SDK path %s does not exist" % sdk_path) # Grab the version from the SDK so we can put it in PYTHON.json. sdk_settings_path = pathlib.Path(sdk_path) / "SDKSettings.json" with sdk_settings_path.open("rb") as fh: sdk_settings = json.load(fh) env["APPLE_SDK_VERSION"] = sdk_settings["Version"] env["APPLE_SDK_CANONICAL_NAME"] = sdk_settings["CanonicalName"] extra_target_cflags.extend(["-isysroot", sdk_path]) extra_target_ldflags.extend(["-isysroot", sdk_path]) # The host SDK may be for a different platform from the target SDK. # Resolve that separately. if "APPLE_HOST_SDK_PATH" in os.environ: host_sdk_path = os.environ["APPLE_HOST_SDK_PATH"] else: host_sdk_path = subprocess.run( ["xcrun", "--show-sdk-path"], check=True, capture_output=True, encoding="utf-8", ).stdout.strip() if not os.path.exists(host_sdk_path): raise Exception("macOS host SDK path %s does not exist" % host_sdk_path) extra_host_cflags.extend(["-isysroot", host_sdk_path]) extra_host_ldflags.extend(["-isysroot", host_sdk_path]) env["EXTRA_HOST_CFLAGS"] = " ".join(extra_host_cflags) env["EXTRA_HOST_LDFLAGS"] = " ".join(extra_host_ldflags) env["EXTRA_TARGET_CFLAGS"] = " ".join(extra_target_cflags) env["EXTRA_TARGET_LDFLAGS"] = " ".join(extra_target_ldflags)