def build_gcc(client, image):
"""Build GCC in the Docker image."""
gcc_archive = download_entry("gcc", DOWNLOADS_PATH)
gmp_archive = download_entry("gmp", DOWNLOADS_PATH)
isl_archive = download_entry("isl", DOWNLOADS_PATH)
mpc_archive = download_entry("mpc", DOWNLOADS_PATH)
mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
log("copying archives to container...")
for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive,
mpfr_archive):
build_env.copy_file(a)
build_env.copy_file(archive_path("binutils", "linux64"))
build_env.copy_file(SUPPORT / "build-gcc.sh")
build_env.run(
"build-gcc.sh",
environment={
"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"GMP_VERSION": DOWNLOADS["gmp"]["version"],
"ISL_VERSION": DOWNLOADS["isl"]["version"],
"MPC_VERSION": DOWNLOADS["mpc"]["version"],
"MPFR_VERSION": DOWNLOADS["mpfr"]["version"],
},
)
build_env.get_tools_archive(archive_path("gcc", "linux64"), "host")
def simple_build(client, image, entry, platform, musl=False, extra_archives=None):
archive = download_entry(entry, DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl
)
for a in extra_archives or []:
build_env.install_artifact_archive(BUILD, a, platform, musl=musl)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % platform,
"%s_VERSION" % entry.upper().replace("-", "_"): DOWNLOADS[entry]["version"],
}
if musl:
env["CC"] = "musl-clang"
add_target_env(env, platform, build_env)
build_env.run("build-%s.sh" % entry, environment=env)
build_env.get_tools_archive(archive_path(entry, platform, musl=musl), "deps")
def build_gcc(client, image, host_platform):
"""Build GCC in the Docker image."""
gcc_archive = download_entry("gcc", DOWNLOADS_PATH)
gmp_archive = download_entry("gmp", DOWNLOADS_PATH)
isl_archive = download_entry("isl", DOWNLOADS_PATH)
mpc_archive = download_entry("mpc", DOWNLOADS_PATH)
mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
log("copying archives to container...")
for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive,
mpfr_archive):
build_env.copy_file(a)
build_env.copy_file(toolchain_archive_path("binutils", host_platform))
build_env.copy_file(SUPPORT / "build-gcc.sh")
env = {
"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"GMP_VERSION": DOWNLOADS["gmp"]["version"],
"ISL_VERSION": DOWNLOADS["isl"]["version"],
"MPC_VERSION": DOWNLOADS["mpc"]["version"],
"MPFR_VERSION": DOWNLOADS["mpfr"]["version"],
}
add_env_common(env)
build_env.run("build-gcc.sh", environment=env)
build_env.get_tools_archive(
toolchain_archive_path("gcc", host_platform), "host")
def build_libedit(
settings, client, image, host_platform, target_triple, optimizations, dest_archive
):
libedit_archive = download_entry("libedit", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
build_env.install_artifact_archive(
BUILD, "ncurses", target_triple, optimizations
)
build_env.copy_file(libedit_archive)
build_env.copy_file(SUPPORT / "build-libedit.sh")
env = {
"TOOLCHAIN": "clang-%s" % host_platform,
"LIBEDIT_VERSION": DOWNLOADS["libedit"]["version"],
}
add_target_env(env, host_platform, target_triple, build_env)
build_env.run("build-libedit.sh", environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def build_tix(client, image, platform, musl=False):
tcl_archive = download_entry("tcl", DOWNLOADS_PATH)
tk_archive = download_entry("tk", DOWNLOADS_PATH)
tix_archive = download_entry("tix", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl
)
depends = {"tcl", "tk"}
if platform != "macos":
depends |= {"libX11", "xorgproto"}
for p in sorted(depends):
build_env.install_artifact_archive(BUILD, p, platform, musl=musl)
for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"):
build_env.copy_file(p)
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % platform,
"TCL_VERSION": DOWNLOADS["tcl"]["version"],
"TIX_VERSION": DOWNLOADS["tix"]["version"],
"TK_VERSION": DOWNLOADS["tk"]["version"],
}
if musl:
env["CC"] = "musl-clang"
add_target_env(env, platform, build_env)
build_env.run("build-tix.sh", environment=env)
build_env.get_tools_archive(archive_path("tix", platform, musl=musl), "deps")
def build_readline(client, image, host_platform, target_triple, optimizations,
dest_archive):
readline_archive = download_entry("readline", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=True,
clang=True,
musl="musl" in target_triple,
)
build_env.install_artifact_archive(BUILD, "ncurses", target_triple,
optimizations)
build_env.copy_file(readline_archive)
build_env.copy_file(SUPPORT / "build-readline.sh")
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % host_platform,
"READLINE_VERSION": DOWNLOADS["readline"]["version"],
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
add_target_env(env, host_platform, build_env)
build_env.run("build-readline.sh", environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def build_readline(client, image, platform, musl=False):
readline_archive = download_entry("readline", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(BUILD,
platform,
binutils=True,
clang=True,
musl=musl)
dep_platform = platform
if musl:
dep_platform += "-musl"
build_env.install_artifact_archive(BUILD,
"ncurses",
platform,
musl=musl)
build_env.copy_file(readline_archive)
build_env.copy_file(SUPPORT / "build-readline.sh")
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % platform,
"READLINE_VERSION": DOWNLOADS["readline"]["version"],
}
if musl:
env["CC"] = "musl-clang"
add_target_env(env, platform, build_env)
build_env.run("build-readline.sh", environment=env)
build_env.get_tools_archive(
archive_path("readline", platform, musl=musl), "deps")
def build_clang(client, image, host_platform):
if "linux" in host_platform:
cmake_archive = download_entry("cmake-linux-bin", DOWNLOADS_PATH)
ninja_archive = download_entry("ninja-linux-bin", DOWNLOADS_PATH)
elif "macos" in host_platform:
cmake_archive = download_entry("cmake-macos-bin", DOWNLOADS_PATH)
ninja_archive = download_entry("ninja-macos-bin", DOWNLOADS_PATH)
clang_archive = download_entry("clang", DOWNLOADS_PATH)
clang_rt_archive = download_entry("clang-compiler-rt", DOWNLOADS_PATH)
lld_archive = download_entry("lld", DOWNLOADS_PATH)
llvm_archive = download_entry("llvm", DOWNLOADS_PATH)
libcxx_archive = download_entry("libc++", DOWNLOADS_PATH)
libcxxabi_archive = download_entry("libc++abi", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
log("copying archives to container...")
for a in (
cmake_archive,
ninja_archive,
clang_archive,
clang_rt_archive,
lld_archive,
llvm_archive,
libcxx_archive,
libcxxabi_archive,
):
build_env.copy_file(a)
tools_path = "clang-%s" % host_platform
build_sh = "build-clang-%s.sh" % host_platform
binutils = install_binutils(host_platform)
gcc = binutils
env = {
"CLANG_COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"],
"CLANG_VERSION": DOWNLOADS["clang"]["version"],
"CMAKE_VERSION": DOWNLOADS["cmake-linux-bin"]["version"],
"COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"LIBCXX_VERSION": DOWNLOADS["libc++"]["version"],
"LIBCXXABI_VERSION": DOWNLOADS["libc++abi"]["version"],
"LLD_VERSION": DOWNLOADS["lld"]["version"],
"LLVM_VERSION": DOWNLOADS["llvm"]["version"],
}
add_env_common(env)
build_env.install_toolchain(BUILD, host_platform, binutils=binutils, gcc=gcc)
build_env.copy_file(SUPPORT / build_sh)
build_env.run(build_sh, environment=env)
build_env.get_tools_archive(
toolchain_archive_path("clang", host_platform), tools_path
)
def build_binutils(client, image):
"""Build binutils in the Docker image."""
archive = download_entry("binutils", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / "build-binutils.sh")
build_env.run(
"build-binutils.sh",
environment={"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]},
)
build_env.get_tools_archive(archive_path("binutils", "linux64"), "host")
def build_musl(client, image):
musl_archive = download_entry("musl", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(BUILD, "linux64", binutils=True, clang=True)
build_env.copy_file(musl_archive)
build_env.copy_file(SUPPORT / "build-musl.sh")
env = {
"MUSL_VERSION": DOWNLOADS["musl"]["version"],
"TOOLCHAIN": "clang-linux64",
}
build_env.run("build-musl.sh", environment=env)
build_env.get_tools_archive(archive_path("musl", "linux64"), "host")
def simple_build(
settings,
client,
image,
entry,
host_platform,
target_triple,
optimizations,
dest_archive,
extra_archives=None,
tools_path="deps",
):
archive = download_entry(entry, DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
for a in extra_archives or []:
build_env.install_artifact_archive(BUILD, a, target_triple,
optimizations)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
env = {
"%s_VERSION" % entry.upper().replace("-", "_"):
DOWNLOADS[entry]["version"],
}
add_target_env(env, host_platform, target_triple, build_env)
if entry == "openssl":
settings = get_targets(TARGETS_CONFIG)[target_triple]
env["OPENSSL_TARGET"] = settings["openssl_target"]
build_env.run("build-%s.sh" % entry, environment=env)
build_env.get_tools_archive(dest_archive, tools_path)
def simple_build(
client,
image,
entry,
host_platform,
target_triple,
optimizations,
dest_archive,
extra_archives=None,
):
archive = download_entry(entry, DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
for a in extra_archives or []:
build_env.install_artifact_archive(BUILD, a, target_triple,
optimizations)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
env = {
"CC":
"clang",
"TOOLCHAIN":
"clang-%s" % host_platform,
"%s_VERSION" % entry.upper().replace("-", "_"):
DOWNLOADS[entry]["version"],
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
add_target_env(env, host_platform, build_env)
build_env.run("build-%s.sh" % entry, environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def build_binutils(client, image, host_platform):
"""Build binutils in the Docker image."""
archive = download_entry("binutils", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / "build-binutils.sh")
env = {"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]}
add_env_common(env)
build_env.run(
"build-binutils.sh",
environment=env,
)
build_env.get_tools_archive(
toolchain_archive_path("binutils", host_platform), "host")
def build_tix(settings, client, image, host_platform, target_triple,
optimizations, dest_archive):
tcl_archive = download_entry("tcl", DOWNLOADS_PATH)
tk_archive = download_entry("tk", DOWNLOADS_PATH)
tix_archive = download_entry("tix", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
depends = {"tcl", "tk"}
if host_platform != "macos":
depends |= {"libX11", "xorgproto"}
for p in sorted(depends):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (tcl_archive, tk_archive, tix_archive,
SUPPORT / "build-tix.sh"):
build_env.copy_file(p)
env = {
"TOOLCHAIN": "clang-%s" % host_platform,
"TCL_VERSION": DOWNLOADS["tcl"]["version"],
"TIX_VERSION": DOWNLOADS["tix"]["version"],
"TK_VERSION": DOWNLOADS["tk"]["version"],
}
add_target_env(env, host_platform, target_triple, build_env)
build_env.run("build-tix.sh", environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def build_cpython(
client,
image,
host_platform,
target_triple,
optimizations,
dest_archive,
libressl=False,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with (SUPPORT / ("static-modules.%s.%s" %
(version, host_platform))).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with (SUPPORT / ("disabled-static-modules.%s.%s" %
(version, host_platform))).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=entry["version"],
musl="musl" in target_triple,
debug=optimizations == "debug",
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
# TODO support bdb/gdbm toggle
packages = {
"bdb",
"bzip2",
"libedit",
"libffi",
"sqlite",
"tcl",
"tix",
"tk",
"uuid",
"xz",
"zlib",
}
if libressl:
packages.add("libressl")
else:
packages.add("openssl")
# We use the system ncurses on macOS for now.
ncurses = host_platform != "macos"
if ncurses:
packages.add("ncurses")
readline = host_platform != "macos"
if readline:
packages.add("readline")
if host_platform == "linux64":
packages |= {"libX11", "libXau", "libxcb", "xorgproto"}
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"CC": "clang",
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION": entry["version"][:3],
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % host_platform,
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
if optimizations == "debug":
env["CPYTHON_DEBUG"] = "1"
if optimizations in ("pgo", "pgo+lto"):
env["CPYTHON_OPTIMIZED"] = "1"
if optimizations in ("lto", "pgo+lto"):
env["CPYTHON_LTO"] = "1"
add_target_env(env, host_platform, build_env)
build_env.run("build-cpython.sh", environment=env)
extension_module_loading = ["builtin"]
crt_features = []
if host_platform == "linux64":
if "musl" in target_triple:
crt_features.append("static")
else:
extension_module_loading.append("shared-library")
crt_features.append("glibc-dynamic")
glibc_max_version = build_env.get_file(
"glibc_version.txt").strip()
if not glibc_max_version:
raise Exception(
"failed to retrieve glibc max symbol version")
crt_features.append("glibc-max-symbol-version:%s" %
glibc_max_version.decode("ascii"))
python_symbol_visibility = "global-default"
elif host_platform == "macos":
python_symbol_visibility = "global-default"
extension_module_loading.append("shared-library")
crt_features.append("libSystem")
else:
raise ValueError("unhandled platform: %s" % host_platform)
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"6",
"target_triple":
target_triple,
"optimizations":
optimizations,
"python_tag":
entry["python_tag"],
"python_version":
entry["version"],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"python_symbol_visibility":
python_symbol_visibility,
"python_extension_module_loading":
extension_module_loading,
"libpython_link_mode":
"static" if "musl" in target_triple else "shared",
"crt_features":
crt_features,
"run_tests":
"build/run_tests.py",
"build_info":
python_build_info(
build_env,
version,
host_platform,
"musl" in target_triple,
optimizations,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
}
python_info["tcl_library_path"] = "install/lib"
python_info["tcl_library_paths"] = [
"tcl8",
"tcl8.6",
"thread2.8.5",
"Tix8.4.3",
"tk8.6",
]
# Add metadata derived from built distribution.
extra_metadata = build_env.get_file("metadata.json")
python_info.update(json.loads(extra_metadata))
validate_python_json(python_info)
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
with open(dest_archive, "wb") as fh:
fh.write(build_env.get_output_archive("python"))
def build_cpython(
settings,
client,
image,
host_platform,
target_triple,
optimizations,
dest_archive,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_version = entry["version"]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with get_target_support_file(SUPPORT, "static-modules", version,
host_platform,
target_triple).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with get_target_support_file(SUPPORT, "disabled-static-modules", version,
host_platform,
target_triple).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=python_version,
musl="musl" in target_triple,
debug=optimizations == "debug",
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
packages = target_needs(TARGETS_CONFIG, target_triple, python_version)
# Toolchain packages are handled specially.
packages.discard("binutils")
packages.discard("musl")
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION":
".".join(entry["version"].split(".")[0:2]),
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % host_platform,
}
if optimizations == "debug":
env["CPYTHON_DEBUG"] = "1"
if optimizations in ("pgo", "pgo+lto"):
env["CPYTHON_OPTIMIZED"] = "1"
if optimizations in ("lto", "pgo+lto"):
env["CPYTHON_LTO"] = "1"
add_target_env(env, host_platform, target_triple, build_env)
build_env.run("build-cpython.sh", environment=env)
extension_module_loading = ["builtin"]
crt_features = []
if host_platform == "linux64":
if "musl" in target_triple:
crt_features.append("static")
else:
extension_module_loading.append("shared-library")
crt_features.append("glibc-dynamic")
glibc_max_version = build_env.get_file(
"glibc_version.txt").strip()
if not glibc_max_version:
raise Exception(
"failed to retrieve glibc max symbol version")
crt_features.append("glibc-max-symbol-version:%s" %
glibc_max_version.decode("ascii"))
python_symbol_visibility = "global-default"
elif host_platform == "macos":
python_symbol_visibility = "global-default"
extension_module_loading.append("shared-library")
crt_features.append("libSystem")
else:
raise ValueError("unhandled platform: %s" % host_platform)
extra_metadata = json.loads(build_env.get_file("metadata.json"))
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"7",
"target_triple":
target_triple,
"optimizations":
optimizations,
"python_tag":
entry["python_tag"],
"python_version":
entry["version"],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"python_symbol_visibility":
python_symbol_visibility,
"python_extension_module_loading":
extension_module_loading,
"libpython_link_mode":
"static" if "musl" in target_triple else "shared",
"crt_features":
crt_features,
"run_tests":
"build/run_tests.py",
"build_info":
python_build_info(
build_env,
version,
host_platform,
target_triple,
"musl" in target_triple,
optimizations,
config_c_in,
setup_dist_content,
setup_local_content,
extra_metadata,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
}
python_info["tcl_library_path"] = "install/lib"
python_info["tcl_library_paths"] = [
"itcl4.2.2",
"tcl8",
"tcl8.6",
"thread2.8.7",
"tk8.6",
]
if "-apple" not in target_triple:
python_info["tcl_library_paths"].append("Tix8.4.3")
if "-apple" in target_triple:
python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"]
python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"]
python_info["apple_sdk_canonical_name"] = env[
"APPLE_SDK_CANONICAL_NAME"]
python_info["apple_sdk_deployment_target"] = env[
"APPLE_MIN_DEPLOYMENT_TARGET"]
# Add metadata derived from built distribution.
python_info.update(extra_metadata)
validate_python_json(python_info)
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
with open(dest_archive, "wb") as fh:
fh.write(build_env.get_output_archive("python"))
def build_cpython(
client,
image,
platform,
debug=False,
optimized=False,
musl=False,
libressl=False,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with (SUPPORT / ("static-modules.%s" % platform)).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with (SUPPORT /
("disabled-static-modules.%s" % platform)).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=entry["version"],
musl=musl,
debug=debug,
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
build_env.install_toolchain(BUILD,
platform,
binutils=install_binutils(platform),
clang=True,
musl=musl)
dep_platform = platform
if musl:
dep_platform += "-musl"
# TODO support bdb/gdbm toggle
packages = {
"bdb",
"bzip2",
"libedit",
"libffi",
"sqlite",
"uuid",
"xz",
"zlib",
}
if libressl:
packages.add("libressl")
else:
packages.add("openssl")
# We use the system ncurses on macOS for now.
ncurses = platform != "macos"
if ncurses:
packages.add("ncurses")
readline = platform != "macos"
if readline:
packages.add("readline")
if platform == "linux64":
packages |= {"libX11", "libXau", "libxcb", "xorgproto"}
tix = platform != "macos"
if tix:
packages |= {"tcl", "tix", "tk"}
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, platform, musl=musl)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"CC": "clang",
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION": entry["version"][:3],
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % platform,
}
if musl:
env["CC"] = "musl-clang"
if debug:
env["CPYTHON_DEBUG"] = "1"
if optimized:
env["CPYTHON_OPTIMIZED"] = "1"
add_target_env(env, platform, build_env)
build_env.run("build-cpython.sh", environment=env)
fully_qualified_name = "python%s%sm" % (
entry["version"][0:3],
"d" if debug else "",
)
if platform == "linux64":
os_name = "linux"
elif platform == "macos":
os_name = "macos"
else:
raise ValueError("unhandled platform: %s" % platform)
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"4",
"os":
os_name,
"arch":
"x86_64",
"python_flavor":
"cpython",
"python_version":
entry["version"],
"python_exe":
"install/bin/%s" % fully_qualified_name,
"python_include":
"install/include/%s" % fully_qualified_name,
"python_stdlib":
"install/lib/python%s" % entry["version"][0:3],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"link_mode":
"static",
"build_info":
python_build_info(
build_env,
platform,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
"tcl_library_path":
"install/lib/tcl",
}
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
basename = "cpython-%s-%s" % (entry["version"], platform)
if musl:
basename += "-musl"
if debug:
basename += "-debug"
if optimized:
basename += "-pgo"
basename += ".tar"
dest_path = BUILD / basename
with dest_path.open("wb") as fh:
fh.write(build_env.get_output_archive("python"))
