def build_tix(client, image, platform, musl=False):
tcl_archive = download_entry("tcl", DOWNLOADS_PATH)
tk_archive = download_entry("tk", DOWNLOADS_PATH)
tix_archive = download_entry("tix", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl
)
depends = {"tcl", "tk"}
if platform != "macos":
depends |= {"libX11", "xorgproto"}
for p in sorted(depends):
build_env.install_artifact_archive(BUILD, p, platform, musl=musl)
for p in (tcl_archive, tk_archive, tix_archive, SUPPORT / "build-tix.sh"):
build_env.copy_file(p)
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % platform,
"TCL_VERSION": DOWNLOADS["tcl"]["version"],
"TIX_VERSION": DOWNLOADS["tix"]["version"],
"TK_VERSION": DOWNLOADS["tk"]["version"],
}
if musl:
env["CC"] = "musl-clang"
add_target_env(env, platform, build_env)
build_env.run("build-tix.sh", environment=env)
build_env.get_tools_archive(archive_path("tix", platform, musl=musl), "deps")
def build_gcc(client, image, host_platform):
"""Build GCC in the Docker image."""
gcc_archive = download_entry("gcc", DOWNLOADS_PATH)
gmp_archive = download_entry("gmp", DOWNLOADS_PATH)
isl_archive = download_entry("isl", DOWNLOADS_PATH)
mpc_archive = download_entry("mpc", DOWNLOADS_PATH)
mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
log("copying archives to container...")
for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive,
mpfr_archive):
build_env.copy_file(a)
build_env.copy_file(toolchain_archive_path("binutils", host_platform))
build_env.copy_file(SUPPORT / "build-gcc.sh")
env = {
"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"GMP_VERSION": DOWNLOADS["gmp"]["version"],
"ISL_VERSION": DOWNLOADS["isl"]["version"],
"MPC_VERSION": DOWNLOADS["mpc"]["version"],
"MPFR_VERSION": DOWNLOADS["mpfr"]["version"],
}
add_env_common(env)
build_env.run("build-gcc.sh", environment=env)
build_env.get_tools_archive(
toolchain_archive_path("gcc", host_platform), "host")
def build_gcc(client, image):
"""Build GCC in the Docker image."""
gcc_archive = download_entry("gcc", DOWNLOADS_PATH)
gmp_archive = download_entry("gmp", DOWNLOADS_PATH)
isl_archive = download_entry("isl", DOWNLOADS_PATH)
mpc_archive = download_entry("mpc", DOWNLOADS_PATH)
mpfr_archive = download_entry("mpfr", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
log("copying archives to container...")
for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive,
mpfr_archive):
build_env.copy_file(a)
build_env.copy_file(archive_path("binutils", "linux64"))
build_env.copy_file(SUPPORT / "build-gcc.sh")
build_env.run(
"build-gcc.sh",
environment={
"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"GMP_VERSION": DOWNLOADS["gmp"]["version"],
"ISL_VERSION": DOWNLOADS["isl"]["version"],
"MPC_VERSION": DOWNLOADS["mpc"]["version"],
"MPFR_VERSION": DOWNLOADS["mpfr"]["version"],
},
)
build_env.get_tools_archive(archive_path("gcc", "linux64"), "host")
def build_gcc(client, image):
"""Build GCC in the Docker image."""
gcc_archive = download_entry('gcc', BUILD)
gmp_archive = download_entry('gmp', BUILD)
isl_archive = download_entry('isl', BUILD)
mpc_archive = download_entry('mpc', BUILD)
mpfr_archive = download_entry('mpfr', BUILD)
with run_container(client, image) as container:
log('copying archives to container...')
for a in (gcc_archive, gmp_archive, isl_archive, mpc_archive,
mpfr_archive):
copy_file_to_container(a, container, '/build')
copy_file_to_container(archive_path('binutils', 'linux64'), container,
'/build')
copy_file_to_container(SUPPORT / 'build-gcc.sh', container, '/build')
container_exec(container,
'/build/build-gcc.sh',
environment={
'BINUTILS_VERSION':
DOWNLOADS['binutils']['version'],
'GCC_VERSION': DOWNLOADS['gcc']['version'],
'GMP_VERSION': DOWNLOADS['gmp']['version'],
'ISL_VERSION': DOWNLOADS['isl']['version'],
'MPC_VERSION': DOWNLOADS['mpc']['version'],
'MPFR_VERSION': DOWNLOADS['mpfr']['version'],
})
download_tools_archive(container, archive_path('gcc', 'linux64'),
'host')
def build_clang(client, image, host_platform):
if "linux" in host_platform:
cmake_archive = download_entry("cmake-linux-bin", DOWNLOADS_PATH)
ninja_archive = download_entry("ninja-linux-bin", DOWNLOADS_PATH)
elif "macos" in host_platform:
cmake_archive = download_entry("cmake-macos-bin", DOWNLOADS_PATH)
ninja_archive = download_entry("ninja-macos-bin", DOWNLOADS_PATH)
clang_archive = download_entry("clang", DOWNLOADS_PATH)
clang_rt_archive = download_entry("clang-compiler-rt", DOWNLOADS_PATH)
lld_archive = download_entry("lld", DOWNLOADS_PATH)
llvm_archive = download_entry("llvm", DOWNLOADS_PATH)
libcxx_archive = download_entry("libc++", DOWNLOADS_PATH)
libcxxabi_archive = download_entry("libc++abi", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
log("copying archives to container...")
for a in (
cmake_archive,
ninja_archive,
clang_archive,
clang_rt_archive,
lld_archive,
llvm_archive,
libcxx_archive,
libcxxabi_archive,
):
build_env.copy_file(a)
tools_path = "clang-%s" % host_platform
build_sh = "build-clang-%s.sh" % host_platform
binutils = install_binutils(host_platform)
gcc = binutils
env = {
"CLANG_COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"],
"CLANG_VERSION": DOWNLOADS["clang"]["version"],
"CMAKE_VERSION": DOWNLOADS["cmake-linux-bin"]["version"],
"COMPILER_RT_VERSION": DOWNLOADS["clang-compiler-rt"]["version"],
"GCC_VERSION": DOWNLOADS["gcc"]["version"],
"LIBCXX_VERSION": DOWNLOADS["libc++"]["version"],
"LIBCXXABI_VERSION": DOWNLOADS["libc++abi"]["version"],
"LLD_VERSION": DOWNLOADS["lld"]["version"],
"LLVM_VERSION": DOWNLOADS["llvm"]["version"],
}
add_env_common(env)
build_env.install_toolchain(BUILD, host_platform, binutils=binutils, gcc=gcc)
build_env.copy_file(SUPPORT / build_sh)
build_env.run(build_sh, environment=env)
build_env.get_tools_archive(
toolchain_archive_path("clang", host_platform), tools_path
)
def simple_build(client, image, entry, platform, musl=False, extra_archives=None):
archive = download_entry(entry, DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD, platform, binutils=install_binutils(platform), clang=True, musl=musl
)
for a in extra_archives or []:
build_env.install_artifact_archive(BUILD, a, platform, musl=musl)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % platform,
"%s_VERSION" % entry.upper().replace("-", "_"): DOWNLOADS[entry]["version"],
}
if musl:
env["CC"] = "musl-clang"
add_target_env(env, platform, build_env)
build_env.run("build-%s.sh" % entry, environment=env)
build_env.get_tools_archive(archive_path(entry, platform, musl=musl), "deps")
def build_readline(client, image, platform, musl=False):
readline_archive = download_entry("readline", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(BUILD,
platform,
binutils=True,
clang=True,
musl=musl)
dep_platform = platform
if musl:
dep_platform += "-musl"
build_env.install_artifact_archive(BUILD,
"ncurses",
platform,
musl=musl)
build_env.copy_file(readline_archive)
build_env.copy_file(SUPPORT / "build-readline.sh")
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % platform,
"READLINE_VERSION": DOWNLOADS["readline"]["version"],
}
if musl:
env["CC"] = "musl-clang"
add_target_env(env, platform, build_env)
build_env.run("build-readline.sh", environment=env)
build_env.get_tools_archive(
archive_path("readline", platform, musl=musl), "deps")
def build_readline(client, image, platform, musl=False):
readline_archive = download_entry('readline', BUILD)
with run_container(client, image) as container:
copy_toolchain(container, musl=musl)
dep_platform = platform
if musl:
dep_platform += '-musl'
install_tools_archive(container,
archive_path('ncurses', platform, musl=musl))
copy_file_to_container(readline_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-readline.sh', container,
'/build')
env = {
'CC': 'clang',
'TOOLCHAIN': 'clang-linux64',
'READLINE_VERSION': DOWNLOADS['readline']['version'],
}
if musl:
env['CC'] = 'musl-clang'
add_target_env(env, platform)
container_exec(container, '/build/build-readline.sh', environment=env)
download_tools_archive(container,
archive_path('readline', platform, musl=musl),
'deps')
def build_readline(client, image, host_platform, target_triple, optimizations,
dest_archive):
readline_archive = download_entry("readline", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=True,
clang=True,
musl="musl" in target_triple,
)
build_env.install_artifact_archive(BUILD, "ncurses", target_triple,
optimizations)
build_env.copy_file(readline_archive)
build_env.copy_file(SUPPORT / "build-readline.sh")
env = {
"CC": "clang",
"TOOLCHAIN": "clang-%s" % host_platform,
"READLINE_VERSION": DOWNLOADS["readline"]["version"],
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
add_target_env(env, host_platform, build_env)
build_env.run("build-readline.sh", environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def build_libedit(
settings, client, image, host_platform, target_triple, optimizations, dest_archive
):
libedit_archive = download_entry("libedit", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
build_env.install_artifact_archive(
BUILD, "ncurses", target_triple, optimizations
)
build_env.copy_file(libedit_archive)
build_env.copy_file(SUPPORT / "build-libedit.sh")
env = {
"TOOLCHAIN": "clang-%s" % host_platform,
"LIBEDIT_VERSION": DOWNLOADS["libedit"]["version"],
}
add_target_env(env, host_platform, target_triple, build_env)
build_env.run("build-libedit.sh", environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def simple_build(entry):
archive = download_entry(entry, BUILD)
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
shutil.copyfile(archive, td / archive.name)
extract_tar_to_directory(BUILD / 'clang-macos.tar', td)
toolchain_path = td / 'clang-macos' / 'bin'
env = dict(os.environ)
env['%s_VERSION' % entry.upper()] = DOWNLOADS[entry]['version']
# We force a PATH only containing system files: we don't want
# pollution from homebrew, macports, etc.
env['PATH'] = '%s:/usr/bin:/bin' % toolchain_path
env['MACOSX_DEPLOYMENT_TARGET'] = MACOSX_DEPLOYMENT_TARGET
env['NUM_CPUS'] = '%s' % multiprocessing.cpu_count()
exec_and_log([SUPPORT / ('build-%s.sh' % entry)], td, env)
dest_path = BUILD / ('%s-macos.tar' % entry)
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def copy_rust(container):
rust = download_entry('rust', BUILD)
copy_file_to_container(rust, container, '/build')
container.exec_run(['/bin/mkdir', 'p', '/tools/rust'])
container.exec_run([
'/bin/tar', '-C', '/tools/rust', '--strip-components', '1', '-xf',
'/build/%s' % rust.name
])
def build_tcltk(client, image, platform):
tcl_archive = download_entry('tcl', BUILD)
tk_archive = download_entry('tk', BUILD)
x11_archive = download_entry('libx11', BUILD)
with run_container(client, image) as container:
copy_toolchain(container, platform=platform)
copy_file_to_container(tcl_archive, container, '/build')
copy_file_to_container(tk_archive, container, '/build')
copy_file_to_container(x11_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-tcltk.sh', container, '/build')
env = {
'TOOLCHAIN': 'clang-%s' % platform,
}
container_exec(container, '/build/build-tcltk.sh', environment=env)
dest_path = 'tcltk-%s.tar' % platform
download_tools_archive(container, BUILD / dest_path)
def build_tix(settings, client, image, host_platform, target_triple,
optimizations, dest_archive):
tcl_archive = download_entry("tcl", DOWNLOADS_PATH)
tk_archive = download_entry("tk", DOWNLOADS_PATH)
tix_archive = download_entry("tix", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
depends = {"tcl", "tk"}
if host_platform != "macos":
depends |= {"libX11", "xorgproto"}
for p in sorted(depends):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (tcl_archive, tk_archive, tix_archive,
SUPPORT / "build-tix.sh"):
build_env.copy_file(p)
env = {
"TOOLCHAIN": "clang-%s" % host_platform,
"TCL_VERSION": DOWNLOADS["tcl"]["version"],
"TIX_VERSION": DOWNLOADS["tix"]["version"],
"TK_VERSION": DOWNLOADS["tk"]["version"],
}
add_target_env(env, host_platform, target_triple, build_env)
build_env.run("build-tix.sh", environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def materialize_clang(host_platform):
entry = clang_toolchain(host_platform)
tar_zst = download_entry(entry, DOWNLOADS_PATH)
local_filename = "%s-%s-%s.tar" % (
entry,
DOWNLOADS[entry]["version"],
host_platform,
)
dctx = zstandard.ZstdDecompressor()
with open(tar_zst, "rb") as ifh:
with open(BUILD / local_filename, "wb") as ofh:
dctx.copy_stream(ifh, ofh)
def build_binutils(client, image):
"""Build binutils in the Docker image."""
archive = download_entry("binutils", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / "build-binutils.sh")
build_env.run(
"build-binutils.sh",
environment={"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]},
)
build_env.get_tools_archive(archive_path("binutils", "linux64"), "host")
def build_clang():
cmake_archive = download_entry('cmake-macos-bin', BUILD)
ninja_archive = download_entry('ninja-macos-bin', BUILD)
clang_archive = download_entry('clang-6', BUILD)
clang_rt_archive = download_entry('clang-compiler-rt-6', BUILD)
lld_archive = download_entry('lld-6', BUILD)
llvm_archive = download_entry('llvm-6', BUILD)
libcxx_archive = download_entry('libc++-6', BUILD)
libcxxabi_archive = download_entry('libc++abi-6', BUILD)
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
for a in (cmake_archive, ninja_archive, clang_archive,
clang_rt_archive, lld_archive, llvm_archive, libcxx_archive,
libcxxabi_archive):
shutil.copyfile(a, td / a.name)
env = {
'CMAKE_VERSION':
DOWNLOADS['cmake-macos-bin']['version'],
'NINJA_VERSION':
DOWNLOADS['ninja-macos-bin']['version'],
'CLANG_COMPILER_RT_VERSION':
DOWNLOADS['clang-compiler-rt-6']['version'],
'CLANG_VERSION':
DOWNLOADS['clang-6']['version'],
'COMPILER_RT_VERSION':
DOWNLOADS['clang-compiler-rt-6']['version'],
'LIBCXX_VERSION':
DOWNLOADS['libc++-6']['version'],
'LIBCXXABI_VERSION':
DOWNLOADS['libc++abi-6']['version'],
'LLD_VERSION':
DOWNLOADS['lld-6']['version'],
'LLVM_VERSION':
DOWNLOADS['llvm-6']['version'],
'PATH':
'/usr/bin:/bin',
}
exec_and_log([SUPPORT / 'build-clang.sh'], td, env)
dest_path = BUILD / 'clang-macos.tar'
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def build_openssl(perl_path: pathlib.Path):
"""Build OpenSSL from sources using the Perl executable specified."""
# First ensure the dependencies are in place.
openssl_archive = download_entry('openssl', BUILD)
nasm_archive = download_entry('nasm-windows-bin', BUILD)
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
root_32 = td / 'x86'
root_64 = td / 'x64'
root_32.mkdir()
root_64.mkdir()
# Then build the 32 and 64 bit OpenSSL installs in parallel
# (because nmake doesn't do parallel builds).
# TODO we need to adjust the environment to pull in a x86 toolchain
# in order for this to work.
fs = []
with concurrent.futures.ThreadPoolExecutor(2) as e:
#fs.append(e.submit(build_openssl_for_arch, perl_path, 'x86',
# openssl_archive, nasm_archive, root_32))
fs.append(e.submit(build_openssl_for_arch, perl_path, 'amd64',
openssl_archive, nasm_archive, root_64))
for f in fs:
f.result()
install = td / 'out'
#shutil.copytree(root_32 / 'install' / '32', install / 'openssl' / 'win32')
shutil.copytree(root_64 / 'install' / '64', install / 'openssl' / 'amd64')
dest_archive = BUILD / 'openssl-windows.tar'
with dest_archive.open('wb') as fh:
create_tar_from_directory(fh, install)
def build_musl(client, image):
musl_archive = download_entry("musl", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(BUILD, "linux64", binutils=True, clang=True)
build_env.copy_file(musl_archive)
build_env.copy_file(SUPPORT / "build-musl.sh")
env = {
"MUSL_VERSION": DOWNLOADS["musl"]["version"],
"TOOLCHAIN": "clang-linux64",
}
build_env.run("build-musl.sh", environment=env)
build_env.get_tools_archive(archive_path("musl", "linux64"), "host")
def build_clang(client, image):
cmake_archive = download_entry('cmake-linux-bin', BUILD)
ninja_archive = download_entry('ninja-linux-bin', BUILD)
clang_archive = download_entry('clang', BUILD)
clang_rt_archive = download_entry('clang-compiler-rt', BUILD)
lld_archive = download_entry('lld', BUILD)
llvm_archive = download_entry('llvm', BUILD)
libcxx_archive = download_entry('libc++', BUILD)
libcxxabi_archive = download_entry('libc++abi', BUILD)
with run_container(client, image) as container:
log('copying archives to container...')
for a in (cmake_archive, ninja_archive, clang_archive,
clang_rt_archive, lld_archive, llvm_archive, libcxx_archive,
libcxxabi_archive):
copy_file_to_container(a, container, '/build')
toolchain_platform = None
tools_path = 'clang-linux64'
suffix = 'linux64'
build_sh = 'build-clang.sh'
gcc = True
env = {
'CLANG_COMPILER_RT_VERSION':
DOWNLOADS['clang-compiler-rt']['version'],
'CLANG_VERSION': DOWNLOADS['clang']['version'],
'CMAKE_VERSION': DOWNLOADS['cmake-linux-bin']['version'],
'COMPILER_RT_VERSION': DOWNLOADS['clang-compiler-rt']['version'],
'GCC_VERSION': DOWNLOADS['gcc']['version'],
'LIBCXX_VERSION': DOWNLOADS['libc++']['version'],
'LIBCXXABI_VERSION': DOWNLOADS['libc++abi']['version'],
'LLD_VERSION': DOWNLOADS['lld']['version'],
'LLVM_VERSION': DOWNLOADS['llvm']['version'],
}
copy_toolchain(container, toolchain_platform, gcc=gcc)
copy_file_to_container(SUPPORT / build_sh, container, '/build')
container_exec(container, '/build/%s' % build_sh, environment=env)
download_tools_archive(container, BUILD / ('clang-%s.tar' % suffix),
tools_path)
def build_musl(client, image):
musl_archive = download_entry('musl', BUILD)
with run_container(client, image) as container:
copy_toolchain(container)
copy_file_to_container(musl_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-musl.sh', container, '/build')
env = {
'MUSL_VERSION': DOWNLOADS['musl']['version'],
'TOOLCHAIN': 'clang-linux64',
}
container_exec(container, '/build/build-musl.sh', environment=env)
download_tools_archive(container, archive_path('musl', 'linux64'),
'host')
def simple_build(
settings,
client,
image,
entry,
host_platform,
target_triple,
optimizations,
dest_archive,
extra_archives=None,
tools_path="deps",
):
archive = download_entry(entry, DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
for a in extra_archives or []:
build_env.install_artifact_archive(BUILD, a, target_triple,
optimizations)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
env = {
"%s_VERSION" % entry.upper().replace("-", "_"):
DOWNLOADS[entry]["version"],
}
add_target_env(env, host_platform, target_triple, build_env)
if entry == "openssl":
settings = get_targets(TARGETS_CONFIG)[target_triple]
env["OPENSSL_TARGET"] = settings["openssl_target"]
build_env.run("build-%s.sh" % entry, environment=env)
build_env.get_tools_archive(dest_archive, tools_path)
def build_binutils(client, image):
"""Build binutils in the Docker image."""
archive = download_entry('binutils', BUILD)
with run_container(client, image) as container:
copy_file_to_container(archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-binutils.sh', container,
'/build')
container_exec(container,
'/build/build-binutils.sh',
environment={
'BINUTILS_VERSION':
DOWNLOADS['binutils']['version'],
})
download_tools_archive(container, archive_path('binutils', 'linux64'),
'host')
def simple_build(
client,
image,
entry,
host_platform,
target_triple,
optimizations,
dest_archive,
extra_archives=None,
):
archive = download_entry(entry, DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
for a in extra_archives or []:
build_env.install_artifact_archive(BUILD, a, target_triple,
optimizations)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / ("build-%s.sh" % entry))
env = {
"CC":
"clang",
"TOOLCHAIN":
"clang-%s" % host_platform,
"%s_VERSION" % entry.upper().replace("-", "_"):
DOWNLOADS[entry]["version"],
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
add_target_env(env, host_platform, build_env)
build_env.run("build-%s.sh" % entry, environment=env)
build_env.get_tools_archive(dest_archive, "deps")
def build_readline(client, image, platform):
readline_archive = download_entry('readline', BUILD)
with run_container(client, image) as container:
copy_toolchain(container, platform=platform)
install_tools_archive(container, BUILD / ('ncurses-%s.tar' % platform))
copy_file_to_container(readline_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-readline.sh', container,
'/build')
env = {
'TOOLCHAIN': 'clang-linux64',
'READLINE_VERSION': DOWNLOADS['readline']['version'],
}
add_target_env(env, platform)
container_exec(container, '/build/build-readline.sh', environment=env)
dest_path = 'readline-%s.tar' % platform
download_tools_archive(container, BUILD / dest_path, 'deps')
def simple_build(client, image, entry, platform):
archive = download_entry(entry, BUILD)
with run_container(client, image) as container:
copy_toolchain(container, platform=platform)
copy_file_to_container(archive, container, '/build')
copy_file_to_container(SUPPORT / ('build-%s.sh' % entry), container,
'/build')
env = {
'TOOLCHAIN': 'clang-linux64',
'%s_VERSION' % entry.upper(): DOWNLOADS[entry]['version'],
}
add_target_env(env, platform)
container_exec(container,
'/build/build-%s.sh' % entry,
environment=env)
dest_path = '%s-%s.tar' % (entry, platform)
download_tools_archive(container, BUILD / dest_path, 'deps')
def build_binutils(client, image, host_platform):
"""Build binutils in the Docker image."""
archive = download_entry("binutils", DOWNLOADS_PATH)
with build_environment(client, image) as build_env:
install_sccache(build_env)
build_env.copy_file(archive)
build_env.copy_file(SUPPORT / "build-binutils.sh")
env = {"BINUTILS_VERSION": DOWNLOADS["binutils"]["version"]}
add_env_common(env)
build_env.run(
"build-binutils.sh",
environment=env,
)
build_env.get_tools_archive(
toolchain_archive_path("binutils", host_platform), "host")
def simple_build(client,
image,
entry,
platform,
musl=False,
extra_archives=None):
archive = download_entry(entry, BUILD)
with run_container(client, image) as container:
copy_toolchain(container, musl=musl)
for a in extra_archives or []:
install_tools_archive(container,
archive_path(a, platform, musl=musl))
copy_file_to_container(archive, container, '/build')
copy_file_to_container(SUPPORT / ('build-%s.sh' % entry), container,
'/build')
env = {
'CC':
'clang',
'TOOLCHAIN':
'clang-linux64',
'%s_VERSION' % entry.upper().replace('-', '_'):
DOWNLOADS[entry]['version'],
}
if musl:
env['CC'] = 'musl-clang'
add_target_env(env, platform)
container_exec(container,
'/build/build-%s.sh' % entry,
environment=env)
download_tools_archive(container,
archive_path(entry, platform, musl=musl),
'deps')
def build_cpython(
client,
image,
host_platform,
target_triple,
optimizations,
dest_archive,
libressl=False,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with (SUPPORT / ("static-modules.%s.%s" %
(version, host_platform))).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with (SUPPORT / ("disabled-static-modules.%s.%s" %
(version, host_platform))).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=entry["version"],
musl="musl" in target_triple,
debug=optimizations == "debug",
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
# TODO support bdb/gdbm toggle
packages = {
"bdb",
"bzip2",
"libedit",
"libffi",
"sqlite",
"tcl",
"tix",
"tk",
"uuid",
"xz",
"zlib",
}
if libressl:
packages.add("libressl")
else:
packages.add("openssl")
# We use the system ncurses on macOS for now.
ncurses = host_platform != "macos"
if ncurses:
packages.add("ncurses")
readline = host_platform != "macos"
if readline:
packages.add("readline")
if host_platform == "linux64":
packages |= {"libX11", "libXau", "libxcb", "xorgproto"}
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"CC": "clang",
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION": entry["version"][:3],
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % host_platform,
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
if optimizations == "debug":
env["CPYTHON_DEBUG"] = "1"
if optimizations in ("pgo", "pgo+lto"):
env["CPYTHON_OPTIMIZED"] = "1"
if optimizations in ("lto", "pgo+lto"):
env["CPYTHON_LTO"] = "1"
add_target_env(env, host_platform, build_env)
build_env.run("build-cpython.sh", environment=env)
extension_module_loading = ["builtin"]
crt_features = []
if host_platform == "linux64":
if "musl" in target_triple:
crt_features.append("static")
else:
extension_module_loading.append("shared-library")
crt_features.append("glibc-dynamic")
glibc_max_version = build_env.get_file(
"glibc_version.txt").strip()
if not glibc_max_version:
raise Exception(
"failed to retrieve glibc max symbol version")
crt_features.append("glibc-max-symbol-version:%s" %
glibc_max_version.decode("ascii"))
python_symbol_visibility = "global-default"
elif host_platform == "macos":
python_symbol_visibility = "global-default"
extension_module_loading.append("shared-library")
crt_features.append("libSystem")
else:
raise ValueError("unhandled platform: %s" % host_platform)
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"6",
"target_triple":
target_triple,
"optimizations":
optimizations,
"python_tag":
entry["python_tag"],
"python_version":
entry["version"],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"python_symbol_visibility":
python_symbol_visibility,
"python_extension_module_loading":
extension_module_loading,
"libpython_link_mode":
"static" if "musl" in target_triple else "shared",
"crt_features":
crt_features,
"run_tests":
"build/run_tests.py",
"build_info":
python_build_info(
build_env,
version,
host_platform,
"musl" in target_triple,
optimizations,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
}
python_info["tcl_library_path"] = "install/lib"
python_info["tcl_library_paths"] = [
"tcl8",
"tcl8.6",
"thread2.8.5",
"Tix8.4.3",
"tk8.6",
]
# Add metadata derived from built distribution.
extra_metadata = build_env.get_file("metadata.json")
python_info.update(json.loads(extra_metadata))
validate_python_json(python_info)
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
with open(dest_archive, "wb") as fh:
fh.write(build_env.get_output_archive("python"))
def build_cpython(client,
image,
platform,
debug=False,
optimized=False,
musl=False,
libressl=False,
version=None):
"""Build CPython in a Docker image'"""
entry_name = 'cpython-%s' % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, BUILD)
setuptools_archive = download_entry('setuptools', BUILD)
pip_archive = download_entry('pip', BUILD)
with (SUPPORT / 'static-modules').open('rb') as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b'#')
]
setup = derive_setup_local(static_modules_lines,
python_archive,
python_version=entry['version'],
musl=musl,
debug=debug)
config_c_in = parse_config_c(setup['config_c_in'].decode('utf-8'))
setup_dist_content = setup['setup_dist']
setup_local_content = setup['setup_local']
extra_make_content = setup['make_data']
with run_container(client, image) as container:
copy_toolchain(container, musl=musl)
dep_platform = platform
if musl:
dep_platform += '-musl'
# TODO support bdb/gdbm toggle
install_tools_archive(container,
archive_path('bdb', platform, musl=musl))
install_tools_archive(container,
archive_path('bzip2', platform, musl=musl))
install_tools_archive(container,
archive_path('libedit', platform, musl=musl))
install_tools_archive(container,
archive_path('libffi', platform, musl=musl))
install_tools_archive(container,
archive_path('libX11', platform, musl=musl))
install_tools_archive(container,
archive_path('libXau', platform, musl=musl))
install_tools_archive(container,
archive_path('libxcb', platform, musl=musl))
install_tools_archive(container,
archive_path('ncurses', platform, musl=musl))
if libressl:
install_tools_archive(
container, archive_path('libressl', platform, musl=musl))
else:
install_tools_archive(container,
archive_path('openssl', platform, musl=musl))
install_tools_archive(container,
archive_path('readline', platform, musl=musl))
install_tools_archive(container,
archive_path('sqlite', platform, musl=musl))
install_tools_archive(container,
archive_path('tcl', platform, musl=musl))
install_tools_archive(container, archive_path('tk',
platform,
musl=musl))
install_tools_archive(container,
archive_path('uuid', platform, musl=musl))
install_tools_archive(container,
archive_path('xorgproto', platform, musl=musl))
install_tools_archive(container, archive_path('xz',
platform,
musl=musl))
install_tools_archive(container,
archive_path('zlib', platform, musl=musl))
#copy_rust(container)
copy_file_to_container(python_archive, container, '/build')
copy_file_to_container(setuptools_archive, container, '/build')
copy_file_to_container(pip_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-cpython.sh', container,
'/build')
for f in sorted(os.listdir(ROOT)):
if f.startswith('LICENSE.') and f.endswith('.txt'):
copy_file_to_container(ROOT / f, container, '/build')
# TODO copy latest pip/setuptools.
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(setup_local_content)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build',
archive_path='Setup.local')
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(extra_make_content)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build',
archive_path='Makefile.extra')
env = {
'CC': 'clang',
'PIP_VERSION': DOWNLOADS['pip']['version'],
'PYTHON_VERSION': entry['version'],
'PYTHON_MAJMIN_VERSION': entry['version'][:3],
'SETUPTOOLS_VERSION': DOWNLOADS['setuptools']['version'],
}
if musl:
env['CC'] = 'musl-clang'
if debug:
env['CPYTHON_DEBUG'] = '1'
if optimized:
env['CPYTHON_OPTIMIZED'] = '1'
container_exec(container, '/build/build-cpython.sh', environment=env)
fully_qualified_name = 'python%s%sm' % (entry['version'][0:3],
'd' if debug else '')
# Create PYTHON.json file describing this distribution.
python_info = {
'version':
'2',
'os':
'linux',
'arch':
'x86_64',
'python_flavor':
'cpython',
'python_version':
entry['version'],
'python_exe':
'install/bin/%s' % fully_qualified_name,
'python_include':
'install/include/%s' % fully_qualified_name,
'python_stdlib':
'install/lib/python%s' % entry['version'][0:3],
'build_info':
python_build_info(container,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl),
'licenses':
entry['licenses'],
'license_path':
'licenses/LICENSE.cpython.txt',
}
with tempfile.NamedTemporaryFile('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build/out/python',
archive_path='PYTHON.json')
basename = 'cpython-%s-%s' % (entry['version'], platform)
if musl:
basename += '-musl'
if debug:
basename += '-debug'
if optimized:
basename += '-pgo'
basename += '.tar'
dest_path = BUILD / basename
data = container_get_archive(container, '/build/out/python')
with dest_path.open('wb') as fh:
fh.write(data)
