def build_gcc(client, image): """Build GCC in the Docker image.""" gcc_archive = download_entry("gcc", DOWNLOADS_PATH) gmp_archive = download_entry("gmp", DOWNLOADS_PATH) isl_archive = download_entry("isl", DOWNLOADS_PATH) mpc_archive = download_entry("mpc", DOWNLOADS_PATH) mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH) with build_environment(client, image) as build_env: log("copying archives to container...") for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive, mpfr_archive): build_env.copy_file(a) build_env.copy_file(archive_path("binutils", "linux64")) build_env.copy_file(SUPPORT / "build-gcc.sh") build_env.run( "build-gcc.sh", environment={ "BINUTILS_VERSION": DOWNLOADS["binutils"]["version"], "GCC_VERSION": DOWNLOADS["gcc"]["version"], "GMP_VERSION": DOWNLOADS["gmp"]["version"], "ISL_VERSION": DOWNLOADS["isl"]["version"], "MPC_VERSION": DOWNLOADS["mpc"]["version"], "MPFR_VERSION": DOWNLOADS["mpfr"]["version"], }, ) build_env.get_tools_archive(archive_path("gcc", "linux64"), "host")
def simple_build(client, image, entry, platform, musl=False, extra_archives=None): archive = download_entry(entry, DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain( BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl ) for a in extra_archives or []: build_env.install_artifact_archive(BUILD, a, platform, musl=musl) build_env.copy_file(archive) build_env.copy_file(SUPPORT / ("build-%s.sh" % entry)) env = { "CC": "clang", "TOOLCHAIN": "clang-%s" % platform, "%s_VERSION" % entry.upper().replace("-", "_"): DOWNLOADS[entry]["version"], } if musl: env["CC"] = "musl-clang" add_target_env(env, platform, build_env) build_env.run("build-%s.sh" % entry, environment=env) build_env.get_tools_archive(archive_path(entry, platform, musl=musl), "deps")
def build_gcc(client, image, host_platform): """Build GCC in the Docker image.""" gcc_archive = download_entry("gcc", DOWNLOADS_PATH) gmp_archive = download_entry("gmp", DOWNLOADS_PATH) isl_archive = download_entry("isl", DOWNLOADS_PATH) mpc_archive = download_entry("mpc", DOWNLOADS_PATH) mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH) with build_environment(client, image) as build_env: install_sccache(build_env) log("copying archives to container...") for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive, mpfr_archive): build_env.copy_file(a) build_env.copy_file(toolchain_archive_path("binutils", host_platform)) build_env.copy_file(SUPPORT / "build-gcc.sh") env = { "BINUTILS_VERSION": DOWNLOADS["binutils"]["version"], "GCC_VERSION": DOWNLOADS["gcc"]["version"], "GMP_VERSION": DOWNLOADS["gmp"]["version"], "ISL_VERSION": DOWNLOADS["isl"]["version"], "MPC_VERSION": DOWNLOADS["mpc"]["version"], "MPFR_VERSION": DOWNLOADS["mpfr"]["version"], } add_env_common(env) build_env.run("build-gcc.sh", environment=env) build_env.get_tools_archive( toolchain_archive_path("gcc", host_platform), "host")
def build_libedit( settings, client, image, host_platform, target_triple, optimizations, dest_archive ): libedit_archive = download_entry("libedit", DOWNLOADS_PATH) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): build_env.install_toolchain( BUILD, host_platform, binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, ) build_env.install_artifact_archive( BUILD, "ncurses", target_triple, optimizations ) build_env.copy_file(libedit_archive) build_env.copy_file(SUPPORT / "build-libedit.sh") env = { "TOOLCHAIN": "clang-%s" % host_platform, "LIBEDIT_VERSION": DOWNLOADS["libedit"]["version"], } add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-libedit.sh", environment=env) build_env.get_tools_archive(dest_archive, "deps")
def build_tix(client, image, platform, musl=False): tcl_archive = download_entry("tcl", DOWNLOADS_PATH) tk_archive = download_entry("tk", DOWNLOADS_PATH) tix_archive = download_entry("tix", DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain( BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl ) depends = {"tcl", "tk"} if platform != "macos": depends |= {"libX11", "xorgproto"} for p in sorted(depends): build_env.install_artifact_archive(BUILD, p, platform, musl=musl) for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"): build_env.copy_file(p) env = { "CC": "clang", "TOOLCHAIN": "clang-%s" % platform, "TCL_VERSION": DOWNLOADS["tcl"]["version"], "TIX_VERSION": DOWNLOADS["tix"]["version"], "TK_VERSION": DOWNLOADS["tk"]["version"], } if musl: env["CC"] = "musl-clang" add_target_env(env, platform, build_env) build_env.run("build-tix.sh", environment=env) build_env.get_tools_archive(archive_path("tix", platform, musl=musl), "deps")
def build_readline(client, image, host_platform, target_triple, optimizations, dest_archive): readline_archive = download_entry("readline", DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain( BUILD, host_platform, binutils=True, clang=True, musl="musl" in target_triple, ) build_env.install_artifact_archive(BUILD, "ncurses", target_triple, optimizations) build_env.copy_file(readline_archive) build_env.copy_file(SUPPORT / "build-readline.sh") env = { "CC": "clang", "TOOLCHAIN": "clang-%s" % host_platform, "READLINE_VERSION": DOWNLOADS["readline"]["version"], } if "musl" in target_triple: env["CC"] = "musl-clang" add_target_env(env, host_platform, build_env) build_env.run("build-readline.sh", environment=env) build_env.get_tools_archive(dest_archive, "deps")
def build_readline(client, image, platform, musl=False): readline_archive = download_entry("readline", DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain(BUILD, platform, binutils=True, clang=True, musl=musl) dep_platform = platform if musl: dep_platform += "-musl" build_env.install_artifact_archive(BUILD, "ncurses", platform, musl=musl) build_env.copy_file(readline_archive) build_env.copy_file(SUPPORT / "build-readline.sh") env = { "CC": "clang", "TOOLCHAIN": "clang-%s" % platform, "READLINE_VERSION": DOWNLOADS["readline"]["version"], } if musl: env["CC"] = "musl-clang" add_target_env(env, platform, build_env) build_env.run("build-readline.sh", environment=env) build_env.get_tools_archive( archive_path("readline", platform, musl=musl), "deps")
def build_clang(client, image, host_platform): if "linux" in host_platform: cmake_archive = download_entry("cmake-linux-bin", DOWNLOADS_PATH) ninja_archive = download_entry("ninja-linux-bin", DOWNLOADS_PATH) elif "macos" in host_platform: cmake_archive = download_entry("cmake-macos-bin", DOWNLOADS_PATH) ninja_archive = download_entry("ninja-macos-bin", DOWNLOADS_PATH) clang_archive = download_entry("clang", DOWNLOADS_PATH) clang_rt_archive = download_entry("clang-compiler-rt", DOWNLOADS_PATH) lld_archive = download_entry("lld", DOWNLOADS_PATH) llvm_archive = download_entry("llvm", DOWNLOADS_PATH) libcxx_archive = download_entry("libc++", DOWNLOADS_PATH) libcxxabi_archive = download_entry("libc++abi", DOWNLOADS_PATH) with build_environment(client, image) as build_env: install_sccache(build_env) log("copying archives to container...") for a in ( cmake_archive, ninja_archive, clang_archive, clang_rt_archive, lld_archive, llvm_archive, libcxx_archive, libcxxabi_archive, ): build_env.copy_file(a) tools_path = "clang-%s" % host_platform build_sh = "build-clang-%s.sh" % host_platform binutils = install_binutils(host_platform) gcc = binutils env = { "CLANG_COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"], "CLANG_VERSION": DOWNLOADS["clang"]["version"], "CMAKE_VERSION": DOWNLOADS["cmake-linux-bin"]["version"], "COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"], "GCC_VERSION": DOWNLOADS["gcc"]["version"], "LIBCXX_VERSION": DOWNLOADS["libc++"]["version"], "LIBCXXABI_VERSION": DOWNLOADS["libc++abi"]["version"], "LLD_VERSION": DOWNLOADS["lld"]["version"], "LLVM_VERSION": DOWNLOADS["llvm"]["version"], } add_env_common(env) build_env.install_toolchain(BUILD, host_platform, binutils=binutils, gcc=gcc) build_env.copy_file(SUPPORT / build_sh) build_env.run(build_sh, environment=env) build_env.get_tools_archive( toolchain_archive_path("clang", host_platform), tools_path )
def build_binutils(client, image): """Build binutils in the Docker image.""" archive = download_entry("binutils", DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.copy_file(archive) build_env.copy_file(SUPPORT / "build-binutils.sh") build_env.run( "build-binutils.sh", environment={"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]}, ) build_env.get_tools_archive(archive_path("binutils", "linux64"), "host")
def build_musl(client, image): musl_archive = download_entry("musl", DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain(BUILD, "linux64", binutils=True, clang=True) build_env.copy_file(musl_archive) build_env.copy_file(SUPPORT / "build-musl.sh") env = { "MUSL_VERSION": DOWNLOADS["musl"]["version"], "TOOLCHAIN": "clang-linux64", } build_env.run("build-musl.sh", environment=env) build_env.get_tools_archive(archive_path("musl", "linux64"), "host")
def simple_build( settings, client, image, entry, host_platform, target_triple, optimizations, dest_archive, extra_archives=None, tools_path="deps", ): archive = download_entry(entry, DOWNLOADS_PATH) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): build_env.install_toolchain( BUILD, host_platform, binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, ) for a in extra_archives or []: build_env.install_artifact_archive(BUILD, a, target_triple, optimizations) build_env.copy_file(archive) build_env.copy_file(SUPPORT / ("build-%s.sh" % entry)) env = { "%s_VERSION" % entry.upper().replace("-", "_"): DOWNLOADS[entry]["version"], } add_target_env(env, host_platform, target_triple, build_env) if entry == "openssl": settings = get_targets(TARGETS_CONFIG)[target_triple] env["OPENSSL_TARGET"] = settings["openssl_target"] build_env.run("build-%s.sh" % entry, environment=env) build_env.get_tools_archive(dest_archive, tools_path)
def simple_build( client, image, entry, host_platform, target_triple, optimizations, dest_archive, extra_archives=None, ): archive = download_entry(entry, DOWNLOADS_PATH) with build_environment(client, image) as build_env: build_env.install_toolchain( BUILD, host_platform, binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, ) for a in extra_archives or []: build_env.install_artifact_archive(BUILD, a, target_triple, optimizations) build_env.copy_file(archive) build_env.copy_file(SUPPORT / ("build-%s.sh" % entry)) env = { "CC": "clang", "TOOLCHAIN": "clang-%s" % host_platform, "%s_VERSION" % entry.upper().replace("-", "_"): DOWNLOADS[entry]["version"], } if "musl" in target_triple: env["CC"] = "musl-clang" add_target_env(env, host_platform, build_env) build_env.run("build-%s.sh" % entry, environment=env) build_env.get_tools_archive(dest_archive, "deps")
def build_binutils(client, image, host_platform): """Build binutils in the Docker image.""" archive = download_entry("binutils", DOWNLOADS_PATH) with build_environment(client, image) as build_env: install_sccache(build_env) build_env.copy_file(archive) build_env.copy_file(SUPPORT / "build-binutils.sh") env = {"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]} add_env_common(env) build_env.run( "build-binutils.sh", environment=env, ) build_env.get_tools_archive( toolchain_archive_path("binutils", host_platform), "host")
def build_tix(settings, client, image, host_platform, target_triple, optimizations, dest_archive): tcl_archive = download_entry("tcl", DOWNLOADS_PATH) tk_archive = download_entry("tk", DOWNLOADS_PATH) tix_archive = download_entry("tix", DOWNLOADS_PATH) with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): build_env.install_toolchain( BUILD, host_platform, binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, ) depends = {"tcl", "tk"} if host_platform != "macos": depends |= {"libX11", "xorgproto"} for p in sorted(depends): build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"): build_env.copy_file(p) env = { "TOOLCHAIN": "clang-%s" % host_platform, "TCL_VERSION": DOWNLOADS["tcl"]["version"], "TIX_VERSION": DOWNLOADS["tix"]["version"], "TK_VERSION": DOWNLOADS["tk"]["version"], } add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-tix.sh", environment=env) build_env.get_tools_archive(dest_archive, "deps")
def build_cpython( client, image, host_platform, target_triple, optimizations, dest_archive, libressl=False, version=None, ): """Build CPython in a Docker image'""" entry_name = "cpython-%s" % version entry = DOWNLOADS[entry_name] python_archive = download_entry(entry_name, DOWNLOADS_PATH) setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) pip_archive = download_entry("pip", DOWNLOADS_PATH) with (SUPPORT / ("static-modules.%s.%s" % (version, host_platform))).open("rb") as fh: static_modules_lines = [ l.rstrip() for l in fh if not l.startswith(b"#") ] with (SUPPORT / ("disabled-static-modules.%s.%s" % (version, host_platform))).open("rb") as fh: disabled_static_modules = { l.strip() for l in fh if l.strip() and not l.strip().startswith(b"#") } setup = derive_setup_local( static_modules_lines, python_archive, python_version=entry["version"], musl="musl" in target_triple, debug=optimizations == "debug", disabled=disabled_static_modules, ) config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8")) setup_dist_content = setup["setup_dist"] setup_local_content = setup["setup_local"] extra_make_content = setup["make_data"] with build_environment(client, image) as build_env: build_env.install_toolchain( BUILD, host_platform, binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, ) # TODO support bdb/gdbm toggle packages = { "bdb", "bzip2", "libedit", "libffi", "sqlite", "tcl", "tix", "tk", "uuid", "xz", "zlib", } if libressl: packages.add("libressl") else: packages.add("openssl") # We use the system ncurses on macOS for now. ncurses = host_platform != "macos" if ncurses: packages.add("ncurses") readline = host_platform != "macos" if readline: packages.add("readline") if host_platform == "linux64": packages |= {"libX11", "libXau", "libxcb", "xorgproto"} for p in sorted(packages): build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) for p in ( python_archive, setuptools_archive, pip_archive, SUPPORT / "build-cpython.sh", ): build_env.copy_file(p) for f in sorted(os.listdir(ROOT)): if f.startswith("LICENSE.") and f.endswith(".txt"): build_env.copy_file(ROOT / f) with tempfile.NamedTemporaryFile("wb") as fh: # In case default file masks cause wonkiness. os.chmod(fh.name, 0o644) fh.write(setup_local_content) fh.flush() build_env.copy_file(fh.name, dest_name="Setup.local") with tempfile.NamedTemporaryFile("wb") as fh: os.chmod(fh.name, 0o644) fh.write(extra_make_content) fh.flush() build_env.copy_file(fh.name, dest_name="Makefile.extra") env = { "CC": "clang", "PIP_VERSION": DOWNLOADS["pip"]["version"], "PYTHON_VERSION": entry["version"], "PYTHON_MAJMIN_VERSION": entry["version"][:3], "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"], "TOOLCHAIN": "clang-%s" % host_platform, } if "musl" in target_triple: env["CC"] = "musl-clang" if optimizations == "debug": env["CPYTHON_DEBUG"] = "1" if optimizations in ("pgo", "pgo+lto"): env["CPYTHON_OPTIMIZED"] = "1" if optimizations in ("lto", "pgo+lto"): env["CPYTHON_LTO"] = "1" add_target_env(env, host_platform, build_env) build_env.run("build-cpython.sh", environment=env) extension_module_loading = ["builtin"] crt_features = [] if host_platform == "linux64": if "musl" in target_triple: crt_features.append("static") else: extension_module_loading.append("shared-library") crt_features.append("glibc-dynamic") glibc_max_version = build_env.get_file( "glibc_version.txt").strip() if not glibc_max_version: raise Exception( "failed to retrieve glibc max symbol version") crt_features.append("glibc-max-symbol-version:%s" % glibc_max_version.decode("ascii")) python_symbol_visibility = "global-default" elif host_platform == "macos": python_symbol_visibility = "global-default" extension_module_loading.append("shared-library") crt_features.append("libSystem") else: raise ValueError("unhandled platform: %s" % host_platform) # Create PYTHON.json file describing this distribution. python_info = { "version": "6", "target_triple": target_triple, "optimizations": optimizations, "python_tag": entry["python_tag"], "python_version": entry["version"], "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, "libpython_link_mode": "static" if "musl" in target_triple else "shared", "crt_features": crt_features, "run_tests": "build/run_tests.py", "build_info": python_build_info( build_env, version, host_platform, "musl" in target_triple, optimizations, config_c_in, setup_dist_content, setup_local_content, libressl=libressl, ), "licenses": entry["licenses"], "license_path": "licenses/LICENSE.cpython.txt", } python_info["tcl_library_path"] = "install/lib" python_info["tcl_library_paths"] = [ "tcl8", "tcl8.6", "thread2.8.5", "Tix8.4.3", "tk8.6", ] # Add metadata derived from built distribution. extra_metadata = build_env.get_file("metadata.json") python_info.update(json.loads(extra_metadata)) validate_python_json(python_info) with tempfile.NamedTemporaryFile("w") as fh: json.dump(python_info, fh, sort_keys=True, indent=4) fh.flush() if image: dest_path = "/build/out/python" else: dest_path = "out/python" build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json") with open(dest_archive, "wb") as fh: fh.write(build_env.get_output_archive("python"))
def build_cpython( settings, client, image, host_platform, target_triple, optimizations, dest_archive, version=None, ): """Build CPython in a Docker image'""" entry_name = "cpython-%s" % version entry = DOWNLOADS[entry_name] python_version = entry["version"] python_archive = download_entry(entry_name, DOWNLOADS_PATH) setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) pip_archive = download_entry("pip", DOWNLOADS_PATH) with get_target_support_file(SUPPORT, "static-modules", version, host_platform, target_triple).open("rb") as fh: static_modules_lines = [ l.rstrip() for l in fh if not l.startswith(b"#") ] with get_target_support_file(SUPPORT, "disabled-static-modules", version, host_platform, target_triple).open("rb") as fh: disabled_static_modules = { l.strip() for l in fh if l.strip() and not l.strip().startswith(b"#") } setup = derive_setup_local( static_modules_lines, python_archive, python_version=python_version, musl="musl" in target_triple, debug=optimizations == "debug", disabled=disabled_static_modules, ) config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8")) setup_dist_content = setup["setup_dist"] setup_local_content = setup["setup_local"] extra_make_content = setup["make_data"] with build_environment(client, image) as build_env: if settings.get("needs_toolchain"): build_env.install_toolchain( BUILD, host_platform, binutils=install_binutils(host_platform), clang=True, musl="musl" in target_triple, ) packages = target_needs(TARGETS_CONFIG, target_triple, python_version) # Toolchain packages are handled specially. packages.discard("binutils") packages.discard("musl") for p in sorted(packages): build_env.install_artifact_archive(BUILD, p, target_triple, optimizations) for p in ( python_archive, setuptools_archive, pip_archive, SUPPORT / "build-cpython.sh", ): build_env.copy_file(p) for f in sorted(os.listdir(ROOT)): if f.startswith("LICENSE.") and f.endswith(".txt"): build_env.copy_file(ROOT / f) with tempfile.NamedTemporaryFile("wb") as fh: # In case default file masks cause wonkiness. os.chmod(fh.name, 0o644) fh.write(setup_local_content) fh.flush() build_env.copy_file(fh.name, dest_name="Setup.local") with tempfile.NamedTemporaryFile("wb") as fh: os.chmod(fh.name, 0o644) fh.write(extra_make_content) fh.flush() build_env.copy_file(fh.name, dest_name="Makefile.extra") env = { "PIP_VERSION": DOWNLOADS["pip"]["version"], "PYTHON_VERSION": entry["version"], "PYTHON_MAJMIN_VERSION": ".".join(entry["version"].split(".")[0:2]), "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"], "TOOLCHAIN": "clang-%s" % host_platform, } if optimizations == "debug": env["CPYTHON_DEBUG"] = "1" if optimizations in ("pgo", "pgo+lto"): env["CPYTHON_OPTIMIZED"] = "1" if optimizations in ("lto", "pgo+lto"): env["CPYTHON_LTO"] = "1" add_target_env(env, host_platform, target_triple, build_env) build_env.run("build-cpython.sh", environment=env) extension_module_loading = ["builtin"] crt_features = [] if host_platform == "linux64": if "musl" in target_triple: crt_features.append("static") else: extension_module_loading.append("shared-library") crt_features.append("glibc-dynamic") glibc_max_version = build_env.get_file( "glibc_version.txt").strip() if not glibc_max_version: raise Exception( "failed to retrieve glibc max symbol version") crt_features.append("glibc-max-symbol-version:%s" % glibc_max_version.decode("ascii")) python_symbol_visibility = "global-default" elif host_platform == "macos": python_symbol_visibility = "global-default" extension_module_loading.append("shared-library") crt_features.append("libSystem") else: raise ValueError("unhandled platform: %s" % host_platform) extra_metadata = json.loads(build_env.get_file("metadata.json")) # Create PYTHON.json file describing this distribution. python_info = { "version": "7", "target_triple": target_triple, "optimizations": optimizations, "python_tag": entry["python_tag"], "python_version": entry["version"], "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, "libpython_link_mode": "static" if "musl" in target_triple else "shared", "crt_features": crt_features, "run_tests": "build/run_tests.py", "build_info": python_build_info( build_env, version, host_platform, target_triple, "musl" in target_triple, optimizations, config_c_in, setup_dist_content, setup_local_content, extra_metadata, ), "licenses": entry["licenses"], "license_path": "licenses/LICENSE.cpython.txt", } python_info["tcl_library_path"] = "install/lib" python_info["tcl_library_paths"] = [ "itcl4.2.2", "tcl8", "tcl8.6", "thread2.8.7", "tk8.6", ] if "-apple" not in target_triple: python_info["tcl_library_paths"].append("Tix8.4.3") if "-apple" in target_triple: python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"] python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"] python_info["apple_sdk_canonical_name"] = env[ "APPLE_SDK_CANONICAL_NAME"] python_info["apple_sdk_deployment_target"] = env[ "APPLE_MIN_DEPLOYMENT_TARGET"] # Add metadata derived from built distribution. python_info.update(extra_metadata) validate_python_json(python_info) with tempfile.NamedTemporaryFile("w") as fh: json.dump(python_info, fh, sort_keys=True, indent=4) fh.flush() if image: dest_path = "/build/out/python" else: dest_path = "out/python" build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json") with open(dest_archive, "wb") as fh: fh.write(build_env.get_output_archive("python"))
def build_cpython( client, image, platform, debug=False, optimized=False, musl=False, libressl=False, version=None, ): """Build CPython in a Docker image'""" entry_name = "cpython-%s" % version entry = DOWNLOADS[entry_name] python_archive = download_entry(entry_name, DOWNLOADS_PATH) setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) pip_archive = download_entry("pip", DOWNLOADS_PATH) with (SUPPORT / ("static-modules.%s" % platform)).open("rb") as fh: static_modules_lines = [ l.rstrip() for l in fh if not l.startswith(b"#") ] with (SUPPORT / ("disabled-static-modules.%s" % platform)).open("rb") as fh: disabled_static_modules = { l.strip() for l in fh if l.strip() and not l.strip().startswith(b"#") } setup = derive_setup_local( static_modules_lines, python_archive, python_version=entry["version"], musl=musl, debug=debug, disabled=disabled_static_modules, ) config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8")) setup_dist_content = setup["setup_dist"] setup_local_content = setup["setup_local"] extra_make_content = setup["make_data"] with build_environment(client, image) as build_env: build_env.install_toolchain(BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl) dep_platform = platform if musl: dep_platform += "-musl" # TODO support bdb/gdbm toggle packages = { "bdb", "bzip2", "libedit", "libffi", "sqlite", "uuid", "xz", "zlib", } if libressl: packages.add("libressl") else: packages.add("openssl") # We use the system ncurses on macOS for now. ncurses = platform != "macos" if ncurses: packages.add("ncurses") readline = platform != "macos" if readline: packages.add("readline") if platform == "linux64": packages |= {"libX11", "libXau", "libxcb", "xorgproto"} tix = platform != "macos" if tix: packages |= {"tcl", "tix", "tk"} for p in sorted(packages): build_env.install_artifact_archive(BUILD, p, platform, musl=musl) for p in ( python_archive, setuptools_archive, pip_archive, SUPPORT / "build-cpython.sh", ): build_env.copy_file(p) for f in sorted(os.listdir(ROOT)): if f.startswith("LICENSE.") and f.endswith(".txt"): build_env.copy_file(ROOT / f) with tempfile.NamedTemporaryFile("wb") as fh: # In case default file masks cause wonkiness. os.chmod(fh.name, 0o644) fh.write(setup_local_content) fh.flush() build_env.copy_file(fh.name, dest_name="Setup.local") with tempfile.NamedTemporaryFile("wb") as fh: os.chmod(fh.name, 0o644) fh.write(extra_make_content) fh.flush() build_env.copy_file(fh.name, dest_name="Makefile.extra") env = { "CC": "clang", "PIP_VERSION": DOWNLOADS["pip"]["version"], "PYTHON_VERSION": entry["version"], "PYTHON_MAJMIN_VERSION": entry["version"][:3], "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"], "TOOLCHAIN": "clang-%s" % platform, } if musl: env["CC"] = "musl-clang" if debug: env["CPYTHON_DEBUG"] = "1" if optimized: env["CPYTHON_OPTIMIZED"] = "1" add_target_env(env, platform, build_env) build_env.run("build-cpython.sh", environment=env) fully_qualified_name = "python%s%sm" % ( entry["version"][0:3], "d" if debug else "", ) if platform == "linux64": os_name = "linux" elif platform == "macos": os_name = "macos" else: raise ValueError("unhandled platform: %s" % platform) # Create PYTHON.json file describing this distribution. python_info = { "version": "4", "os": os_name, "arch": "x86_64", "python_flavor": "cpython", "python_version": entry["version"], "python_exe": "install/bin/%s" % fully_qualified_name, "python_include": "install/include/%s" % fully_qualified_name, "python_stdlib": "install/lib/python%s" % entry["version"][0:3], "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "link_mode": "static", "build_info": python_build_info( build_env, platform, config_c_in, setup_dist_content, setup_local_content, libressl=libressl, ), "licenses": entry["licenses"], "license_path": "licenses/LICENSE.cpython.txt", "tcl_library_path": "install/lib/tcl", } with tempfile.NamedTemporaryFile("w") as fh: json.dump(python_info, fh, sort_keys=True, indent=4) fh.flush() if image: dest_path = "/build/out/python" else: dest_path = "out/python" build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json") basename = "cpython-%s-%s" % (entry["version"], platform) if musl: basename += "-musl" if debug: basename += "-debug" if optimized: basename += "-pgo" basename += ".tar" dest_path = BUILD / basename with dest_path.open("wb") as fh: fh.write(build_env.get_output_archive("python"))