def build_cpython(client,
image,
platform,
debug=False,
optimized=False,
musl=False,
libressl=False,
version=None):
"""Build CPython in a Docker image'"""
entry_name = 'cpython-%s' % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, BUILD)
setuptools_archive = download_entry('setuptools', BUILD)
pip_archive = download_entry('pip', BUILD)
with (SUPPORT / 'static-modules').open('rb') as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b'#')
]
setup = derive_setup_local(static_modules_lines,
python_archive,
python_version=entry['version'],
musl=musl,
debug=debug)
config_c_in = parse_config_c(setup['config_c_in'].decode('utf-8'))
setup_dist_content = setup['setup_dist']
setup_local_content = setup['setup_local']
extra_make_content = setup['make_data']
with run_container(client, image) as container:
copy_toolchain(container, musl=musl)
dep_platform = platform
if musl:
dep_platform += '-musl'
# TODO support bdb/gdbm toggle
install_tools_archive(container,
archive_path('bdb', platform, musl=musl))
install_tools_archive(container,
archive_path('bzip2', platform, musl=musl))
install_tools_archive(container,
archive_path('libedit', platform, musl=musl))
install_tools_archive(container,
archive_path('libffi', platform, musl=musl))
install_tools_archive(container,
archive_path('libX11', platform, musl=musl))
install_tools_archive(container,
archive_path('libXau', platform, musl=musl))
install_tools_archive(container,
archive_path('libxcb', platform, musl=musl))
install_tools_archive(container,
archive_path('ncurses', platform, musl=musl))
if libressl:
install_tools_archive(
container, archive_path('libressl', platform, musl=musl))
else:
install_tools_archive(container,
archive_path('openssl', platform, musl=musl))
install_tools_archive(container,
archive_path('readline', platform, musl=musl))
install_tools_archive(container,
archive_path('sqlite', platform, musl=musl))
install_tools_archive(container,
archive_path('tcl', platform, musl=musl))
install_tools_archive(container, archive_path('tk',
platform,
musl=musl))
install_tools_archive(container,
archive_path('uuid', platform, musl=musl))
install_tools_archive(container,
archive_path('xorgproto', platform, musl=musl))
install_tools_archive(container, archive_path('xz',
platform,
musl=musl))
install_tools_archive(container,
archive_path('zlib', platform, musl=musl))
#copy_rust(container)
copy_file_to_container(python_archive, container, '/build')
copy_file_to_container(setuptools_archive, container, '/build')
copy_file_to_container(pip_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-cpython.sh', container,
'/build')
for f in sorted(os.listdir(ROOT)):
if f.startswith('LICENSE.') and f.endswith('.txt'):
copy_file_to_container(ROOT / f, container, '/build')
# TODO copy latest pip/setuptools.
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(setup_local_content)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build',
archive_path='Setup.local')
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(extra_make_content)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build',
archive_path='Makefile.extra')
env = {
'CC': 'clang',
'PIP_VERSION': DOWNLOADS['pip']['version'],
'PYTHON_VERSION': entry['version'],
'PYTHON_MAJMIN_VERSION': entry['version'][:3],
'SETUPTOOLS_VERSION': DOWNLOADS['setuptools']['version'],
}
if musl:
env['CC'] = 'musl-clang'
if debug:
env['CPYTHON_DEBUG'] = '1'
if optimized:
env['CPYTHON_OPTIMIZED'] = '1'
container_exec(container, '/build/build-cpython.sh', environment=env)
fully_qualified_name = 'python%s%sm' % (entry['version'][0:3],
'd' if debug else '')
# Create PYTHON.json file describing this distribution.
python_info = {
'version':
'2',
'os':
'linux',
'arch':
'x86_64',
'python_flavor':
'cpython',
'python_version':
entry['version'],
'python_exe':
'install/bin/%s' % fully_qualified_name,
'python_include':
'install/include/%s' % fully_qualified_name,
'python_stdlib':
'install/lib/python%s' % entry['version'][0:3],
'build_info':
python_build_info(container,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl),
'licenses':
entry['licenses'],
'license_path':
'licenses/LICENSE.cpython.txt',
}
with tempfile.NamedTemporaryFile('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build/out/python',
archive_path='PYTHON.json')
basename = 'cpython-%s-%s' % (entry['version'], platform)
if musl:
basename += '-musl'
if debug:
basename += '-debug'
if optimized:
basename += '-pgo'
basename += '.tar'
dest_path = BUILD / basename
data = container_get_archive(container, '/build/out/python')
with dest_path.open('wb') as fh:
fh.write(data)
def build_cpython(
client,
image,
host_platform,
target_triple,
optimizations,
dest_archive,
libressl=False,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with (SUPPORT / ("static-modules.%s.%s" %
(version, host_platform))).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with (SUPPORT / ("disabled-static-modules.%s.%s" %
(version, host_platform))).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=entry["version"],
musl="musl" in target_triple,
debug=optimizations == "debug",
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
# TODO support bdb/gdbm toggle
packages = {
"bdb",
"bzip2",
"libedit",
"libffi",
"sqlite",
"tcl",
"tix",
"tk",
"uuid",
"xz",
"zlib",
}
if libressl:
packages.add("libressl")
else:
packages.add("openssl")
# We use the system ncurses on macOS for now.
ncurses = host_platform != "macos"
if ncurses:
packages.add("ncurses")
readline = host_platform != "macos"
if readline:
packages.add("readline")
if host_platform == "linux64":
packages |= {"libX11", "libXau", "libxcb", "xorgproto"}
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"CC": "clang",
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION": entry["version"][:3],
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % host_platform,
}
if "musl" in target_triple:
env["CC"] = "musl-clang"
if optimizations == "debug":
env["CPYTHON_DEBUG"] = "1"
if optimizations in ("pgo", "pgo+lto"):
env["CPYTHON_OPTIMIZED"] = "1"
if optimizations in ("lto", "pgo+lto"):
env["CPYTHON_LTO"] = "1"
add_target_env(env, host_platform, build_env)
build_env.run("build-cpython.sh", environment=env)
extension_module_loading = ["builtin"]
crt_features = []
if host_platform == "linux64":
if "musl" in target_triple:
crt_features.append("static")
else:
extension_module_loading.append("shared-library")
crt_features.append("glibc-dynamic")
glibc_max_version = build_env.get_file(
"glibc_version.txt").strip()
if not glibc_max_version:
raise Exception(
"failed to retrieve glibc max symbol version")
crt_features.append("glibc-max-symbol-version:%s" %
glibc_max_version.decode("ascii"))
python_symbol_visibility = "global-default"
elif host_platform == "macos":
python_symbol_visibility = "global-default"
extension_module_loading.append("shared-library")
crt_features.append("libSystem")
else:
raise ValueError("unhandled platform: %s" % host_platform)
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"6",
"target_triple":
target_triple,
"optimizations":
optimizations,
"python_tag":
entry["python_tag"],
"python_version":
entry["version"],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"python_symbol_visibility":
python_symbol_visibility,
"python_extension_module_loading":
extension_module_loading,
"libpython_link_mode":
"static" if "musl" in target_triple else "shared",
"crt_features":
crt_features,
"run_tests":
"build/run_tests.py",
"build_info":
python_build_info(
build_env,
version,
host_platform,
"musl" in target_triple,
optimizations,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
}
python_info["tcl_library_path"] = "install/lib"
python_info["tcl_library_paths"] = [
"tcl8",
"tcl8.6",
"thread2.8.5",
"Tix8.4.3",
"tk8.6",
]
# Add metadata derived from built distribution.
extra_metadata = build_env.get_file("metadata.json")
python_info.update(json.loads(extra_metadata))
validate_python_json(python_info)
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
with open(dest_archive, "wb") as fh:
fh.write(build_env.get_output_archive("python"))
def build_cpython(
client,
image,
platform,
debug=False,
optimized=False,
musl=False,
libressl=False,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with (SUPPORT / ("static-modules.%s" % platform)).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with (SUPPORT /
("disabled-static-modules.%s" % platform)).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=entry["version"],
musl=musl,
debug=debug,
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
build_env.install_toolchain(BUILD,
platform,
binutils=install_binutils(platform),
clang=True,
musl=musl)
dep_platform = platform
if musl:
dep_platform += "-musl"
# TODO support bdb/gdbm toggle
packages = {
"bdb",
"bzip2",
"libedit",
"libffi",
"sqlite",
"uuid",
"xz",
"zlib",
}
if libressl:
packages.add("libressl")
else:
packages.add("openssl")
# We use the system ncurses on macOS for now.
ncurses = platform != "macos"
if ncurses:
packages.add("ncurses")
readline = platform != "macos"
if readline:
packages.add("readline")
if platform == "linux64":
packages |= {"libX11", "libXau", "libxcb", "xorgproto"}
tix = platform != "macos"
if tix:
packages |= {"tcl", "tix", "tk"}
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, platform, musl=musl)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"CC": "clang",
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION": entry["version"][:3],
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % platform,
}
if musl:
env["CC"] = "musl-clang"
if debug:
env["CPYTHON_DEBUG"] = "1"
if optimized:
env["CPYTHON_OPTIMIZED"] = "1"
add_target_env(env, platform, build_env)
build_env.run("build-cpython.sh", environment=env)
fully_qualified_name = "python%s%sm" % (
entry["version"][0:3],
"d" if debug else "",
)
if platform == "linux64":
os_name = "linux"
elif platform == "macos":
os_name = "macos"
else:
raise ValueError("unhandled platform: %s" % platform)
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"4",
"os":
os_name,
"arch":
"x86_64",
"python_flavor":
"cpython",
"python_version":
entry["version"],
"python_exe":
"install/bin/%s" % fully_qualified_name,
"python_include":
"install/include/%s" % fully_qualified_name,
"python_stdlib":
"install/lib/python%s" % entry["version"][0:3],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"link_mode":
"static",
"build_info":
python_build_info(
build_env,
platform,
config_c_in,
setup_dist_content,
setup_local_content,
libressl=libressl,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
"tcl_library_path":
"install/lib/tcl",
}
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
basename = "cpython-%s-%s" % (entry["version"], platform)
if musl:
basename += "-musl"
if debug:
basename += "-debug"
if optimized:
basename += "-pgo"
basename += ".tar"
dest_path = BUILD / basename
with dest_path.open("wb") as fh:
fh.write(build_env.get_output_archive("python"))
def build_cpython(pgo=False):
msbuild = find_msbuild()
log('found MSBuild at %s' % msbuild)
# The python.props file keys off MSBUILD, so it needs to be set.
os.environ['MSBUILD'] = str(msbuild)
activeperl_installer = download_entry('activeperl', BUILD)
bzip2_archive = download_entry('bzip2', BUILD)
sqlite_archive = download_entry('sqlite', BUILD)
tk_bin_archive = download_entry('tk-windows-bin', BUILD, local_name='tk-windows-bin.tar.gz')
xz_archive = download_entry('xz', BUILD)
zlib_archive = download_entry('zlib', BUILD)
python_archive = download_entry('cpython-3.7', BUILD)
python_version = DOWNLOADS['cpython-3.7']['version']
openssl_bin_archive = BUILD / 'openssl-windows.tar'
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
with concurrent.futures.ThreadPoolExecutor(7) as e:
for a in (python_archive, bzip2_archive, openssl_bin_archive,
sqlite_archive, tk_bin_archive, xz_archive, zlib_archive):
log('extracting %s to %s' % (a, td))
e.submit(extract_tar_to_directory, a, td)
cpython_source_path = td / ('Python-%s' % python_version)
pcbuild_path = cpython_source_path / 'PCBuild'
out_dir = td / 'out'
build_dir = out_dir / 'python' / 'build'
build_dir.mkdir(parents=True)
# Parse config.c before we hack it up: we want a pristine copy.
config_c_path = cpython_source_path / 'PC' / 'config.c'
with config_c_path.open('r') as fh:
config_c = fh.read()
builtin_extensions = parse_config_c(config_c)
hack_project_files(td, cpython_source_path)
hack_source_files(cpython_source_path)
if pgo:
run_msbuild(msbuild, pcbuild_path, configuration='PGInstrument')
exec_and_log([
str(cpython_source_path / 'python.bat'), '-m', 'test', '--pgo'],
str(pcbuild_path),
os.environ,
exit_on_error=False)
exec_and_log(
[
str(msbuild), str(pcbuild_path / 'pythoncore.vcxproj'),
'/target:KillPython',
'/verbosity:normal',
'/property:Configuration=PGInstrument',
'/property:Platform=x64',
'/property:KillPython=true',
],
pcbuild_path,
os.environ)
run_msbuild(msbuild, pcbuild_path, configuration='PGUpdate')
artifact_config = 'PGUpdate'
else:
run_msbuild(msbuild, pcbuild_path, configuration='Release')
artifact_config = 'Release'
install_dir = out_dir / 'python' / 'install'
# The PC/layout directory contains a script for copying files into
# a release-like directory. Use that for assembling the standalone
# build.
# It doesn't clean up the temp directory it creates. So pass one to it
# under our tempdir.
layout_tmp = td / 'layouttmp'
layout_tmp.mkdir()
exec_and_log(
[
str(cpython_source_path / 'python.bat'),
str(cpython_source_path / 'PC' / 'layout'),
'-vv',
'--source', str(cpython_source_path),
'--build', str(pcbuild_path / 'amd64'),
'--copy', str(install_dir),
'--temp', str(layout_tmp),
'--flat-dlls',
'--include-dev',
'--include-distutils',
],
pcbuild_path,
os.environ)
# Now copy the build artifacts into the output directory.
build_info = collect_python_build_artifacts(
pcbuild_path, out_dir / 'python', 'amd64', artifact_config)
for ext, init_fn in sorted(builtin_extensions.items()):
if ext in build_info['extensions']:
log('built-in extension should not have a build entry: %s' % ext)
sys.exit(1)
build_info['extensions'][ext] = {
'in_core': True,
'objs': [],
'init_fn': init_fn,
'links': [],
'static_lib': None,
}
# Copy OpenSSL libraries as a one-off.
for lib in ('crypto', 'ssl'):
name = 'lib%s_static.lib' % lib
source = td / 'openssl' / 'amd64' / 'lib' / name
dest = out_dir / 'python' / 'build' / 'lib' / name
log('copying %s to %s' % (source, dest))
shutil.copyfile(source, dest)
# Create PYTHON.json file describing this distribution.
python_info = {
# TODO bump version number once format is somewhat stable.
'version': '0',
'os': 'windows',
'arch': 'x86_64',
'python_flavor': 'cpython',
'python_version': python_version,
'python_exe': 'install/python.exe',
'python_include': 'install/include',
'python_stdlib': 'install/Lib',
'build_info': build_info,
}
with (out_dir / 'python' / 'PYTHON.json').open('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
# Copy software licenses file.
shutil.copyfile(ROOT / 'python-licenses.rst', out_dir / 'python' / 'LICENSE.rst')
dest_path = BUILD / 'cpython-windows.tar'
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def build_cpython():
python_archive = download_entry('cpython-3.7', BUILD)
python_version = DOWNLOADS['cpython-3.7']['version']
with (SUPPORT / 'static-modules').open('rb') as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b'#')
]
setup = derive_setup_local(static_modules_lines,
python_archive,
disabled=DISABLED_STATIC_MODULES)
config_c_in = parse_config_c(setup['config_c_in'].decode('utf-8'))
setup_dist_content = setup['setup_dist']
setup_local_content = setup['setup_local']
extra_make_content = setup['make_data']
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
extract_tar_to_directory(BUILD / 'clang-macos.tar', td)
toolchain_path = td / 'clang-macos' / 'bin'
deps_dir = td / 'deps'
deps_dir.mkdir()
extract_tar_to_directory(BUILD / 'bdb-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'bzip2-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'libedit-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'libffi-macos.tar', deps_dir)
# We use the system ncurses and statically link (for now).
#extract_tar_to_directory(BUILD / 'ncurses-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'openssl-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'sqlite-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'uuid-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'xz-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'zlib-macos.tar', deps_dir)
extract_tar_to_directory(python_archive, td)
setup_local_path = td / ('Python-%s' %
python_version) / 'Modules' / 'Setup.local'
with setup_local_path.open('wb') as fh:
fh.write(setup_local_content)
makefile_extra_path = td / 'Makefile.extra'
with makefile_extra_path.open('wb') as fh:
fh.write(extra_make_content)
shutil.copyfile(ROOT / 'python-licenses.rst',
td / 'python-licenses.rst')
env = dict(os.environ)
env['PYTHON_VERSION'] = python_version
# We force a PATH only containing system files: we don't want
# pollution from homebrew, macports, etc.
env['PATH'] = '%s:/usr/bin:/bin' % toolchain_path
env['MACOSX_DEPLOYMENT_TARGET'] = MACOSX_DEPLOYMENT_TARGET
env['NUM_CPUS'] = '%s' % multiprocessing.cpu_count()
env['CPYTHON_OPTIMIZED'] = '1'
exec_and_log([SUPPORT / 'build-cpython.sh'], td, env)
# Create PYTHON.json file describing this distribution.
python_info = {
# TODO bump version number once format is somewhat stable.
'version':
'0',
'os':
'macos',
'arch':
'x86_64',
'python_flavor':
'cpython',
'python_version':
python_version,
'python_exe':
'install/bin/python3',
'python_include':
'install/include/python3.7m',
'python_stdlib':
'install/lib/python3.7',
'build_info':
python_build_info(td / 'out' / 'python', config_c_in,
setup_dist_content, setup_local_content),
}
with (td / 'out' / 'python' / 'PYTHON.json').open('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
dest_path = BUILD / 'cpython-macos.tar'
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def build_cpython(client, image, platform, optimized=False):
"""Build CPythin in a Docker image'"""
python_archive = download_entry('cpython-3.7', BUILD)
with (SUPPORT / 'static-modules').open('rb') as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b'#')
]
setup = derive_setup_local(static_modules_lines, python_archive)
config_c_in = parse_config_c(setup['config_c_in'].decode('utf-8'))
setup_dist_content = setup['setup_dist']
setup_local_content = setup['setup_local']
extra_make_content = setup['make_data']
with run_container(client, image) as container:
copy_toolchain(container, platform=platform)
# TODO support bdb/gdbm toggle
install_tools_archive(container, BUILD / ('bdb-%s.tar' % platform))
install_tools_archive(container, BUILD / ('bzip2-%s.tar' % platform))
install_tools_archive(container, BUILD / ('libedit-%s.tar' % platform))
install_tools_archive(container, BUILD / ('libffi-%s.tar' % platform))
install_tools_archive(container, BUILD / ('ncurses-%s.tar' % platform))
install_tools_archive(container, BUILD / ('openssl-%s.tar' % platform))
install_tools_archive(container,
BUILD / ('readline-%s.tar' % platform))
install_tools_archive(container, BUILD / ('sqlite-%s.tar' % platform))
# tk requires a bunch of X11 stuff.
#install_tools_archive(container, BUILD / ('tcltk-%s.tar' % platform))
install_tools_archive(container, BUILD / ('uuid-%s.tar' % platform))
install_tools_archive(container, BUILD / ('xz-%s.tar' % platform))
install_tools_archive(container, BUILD / ('zlib-%s.tar' % platform))
#copy_rust(container)
copy_file_to_container(python_archive, container, '/build')
copy_file_to_container(SUPPORT / 'build-cpython.sh', container,
'/build')
copy_file_to_container(ROOT / 'python-licenses.rst', container,
'/build')
# TODO copy latest pip/setuptools.
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(setup_local_content)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build',
archive_path='Setup.local')
with tempfile.NamedTemporaryFile('wb') as fh:
fh.write(extra_make_content)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build',
archive_path='Makefile.extra')
env = {
'PYTHON_VERSION': DOWNLOADS['cpython-3.7']['version'],
}
if optimized:
env['CPYTHON_OPTIMIZED'] = '1'
container_exec(container, '/build/build-cpython.sh', environment=env)
# Create PYTHON.json file describing this distribution.
python_info = {
# TODO bump version number once format is somewhat stable.
'version':
'0',
'os':
'linux',
'arch':
'x86_64',
'python_flavor':
'cpython',
'python_version':
DOWNLOADS['cpython-3.7']['version'],
'python_exe':
'install/bin/python',
'python_include':
'install/include/python3.7m',
'python_stdlib':
'install/lib/python3.7',
'build_info':
python_build_info(container, config_c_in, setup_dist_content,
setup_local_content),
}
with tempfile.NamedTemporaryFile('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
copy_file_to_container(fh.name,
container,
'/build/out/python',
archive_path='PYTHON.json')
basename = 'cpython-%s' % platform
if optimized:
basename += '-pgo'
basename += '.tar'
dest_path = BUILD / basename
data, stat = container.get_archive('/build/out/python')
with dest_path.open('wb') as fh:
for chunk in data:
fh.write(chunk)
def build_cpython(
settings,
client,
image,
host_platform,
target_triple,
optimizations,
dest_archive,
version=None,
):
"""Build CPython in a Docker image'"""
entry_name = "cpython-%s" % version
entry = DOWNLOADS[entry_name]
python_version = entry["version"]
python_archive = download_entry(entry_name, DOWNLOADS_PATH)
setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
pip_archive = download_entry("pip", DOWNLOADS_PATH)
with get_target_support_file(SUPPORT, "static-modules", version,
host_platform,
target_triple).open("rb") as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b"#")
]
with get_target_support_file(SUPPORT, "disabled-static-modules", version,
host_platform,
target_triple).open("rb") as fh:
disabled_static_modules = {
l.strip()
for l in fh if l.strip() and not l.strip().startswith(b"#")
}
setup = derive_setup_local(
static_modules_lines,
python_archive,
python_version=python_version,
musl="musl" in target_triple,
debug=optimizations == "debug",
disabled=disabled_static_modules,
)
config_c_in = parse_config_c(setup["config_c_in"].decode("utf-8"))
setup_dist_content = setup["setup_dist"]
setup_local_content = setup["setup_local"]
extra_make_content = setup["make_data"]
with build_environment(client, image) as build_env:
if settings.get("needs_toolchain"):
build_env.install_toolchain(
BUILD,
host_platform,
binutils=install_binutils(host_platform),
clang=True,
musl="musl" in target_triple,
)
packages = target_needs(TARGETS_CONFIG, target_triple, python_version)
# Toolchain packages are handled specially.
packages.discard("binutils")
packages.discard("musl")
for p in sorted(packages):
build_env.install_artifact_archive(BUILD, p, target_triple,
optimizations)
for p in (
python_archive,
setuptools_archive,
pip_archive,
SUPPORT / "build-cpython.sh",
):
build_env.copy_file(p)
for f in sorted(os.listdir(ROOT)):
if f.startswith("LICENSE.") and f.endswith(".txt"):
build_env.copy_file(ROOT / f)
with tempfile.NamedTemporaryFile("wb") as fh:
# In case default file masks cause wonkiness.
os.chmod(fh.name, 0o644)
fh.write(setup_local_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Setup.local")
with tempfile.NamedTemporaryFile("wb") as fh:
os.chmod(fh.name, 0o644)
fh.write(extra_make_content)
fh.flush()
build_env.copy_file(fh.name, dest_name="Makefile.extra")
env = {
"PIP_VERSION": DOWNLOADS["pip"]["version"],
"PYTHON_VERSION": entry["version"],
"PYTHON_MAJMIN_VERSION":
".".join(entry["version"].split(".")[0:2]),
"SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
"TOOLCHAIN": "clang-%s" % host_platform,
}
if optimizations == "debug":
env["CPYTHON_DEBUG"] = "1"
if optimizations in ("pgo", "pgo+lto"):
env["CPYTHON_OPTIMIZED"] = "1"
if optimizations in ("lto", "pgo+lto"):
env["CPYTHON_LTO"] = "1"
add_target_env(env, host_platform, target_triple, build_env)
build_env.run("build-cpython.sh", environment=env)
extension_module_loading = ["builtin"]
crt_features = []
if host_platform == "linux64":
if "musl" in target_triple:
crt_features.append("static")
else:
extension_module_loading.append("shared-library")
crt_features.append("glibc-dynamic")
glibc_max_version = build_env.get_file(
"glibc_version.txt").strip()
if not glibc_max_version:
raise Exception(
"failed to retrieve glibc max symbol version")
crt_features.append("glibc-max-symbol-version:%s" %
glibc_max_version.decode("ascii"))
python_symbol_visibility = "global-default"
elif host_platform == "macos":
python_symbol_visibility = "global-default"
extension_module_loading.append("shared-library")
crt_features.append("libSystem")
else:
raise ValueError("unhandled platform: %s" % host_platform)
extra_metadata = json.loads(build_env.get_file("metadata.json"))
# Create PYTHON.json file describing this distribution.
python_info = {
"version":
"7",
"target_triple":
target_triple,
"optimizations":
optimizations,
"python_tag":
entry["python_tag"],
"python_version":
entry["version"],
"python_stdlib_test_packages":
sorted(STDLIB_TEST_PACKAGES),
"python_symbol_visibility":
python_symbol_visibility,
"python_extension_module_loading":
extension_module_loading,
"libpython_link_mode":
"static" if "musl" in target_triple else "shared",
"crt_features":
crt_features,
"run_tests":
"build/run_tests.py",
"build_info":
python_build_info(
build_env,
version,
host_platform,
target_triple,
"musl" in target_triple,
optimizations,
config_c_in,
setup_dist_content,
setup_local_content,
extra_metadata,
),
"licenses":
entry["licenses"],
"license_path":
"licenses/LICENSE.cpython.txt",
}
python_info["tcl_library_path"] = "install/lib"
python_info["tcl_library_paths"] = [
"itcl4.2.2",
"tcl8",
"tcl8.6",
"thread2.8.7",
"tk8.6",
]
if "-apple" not in target_triple:
python_info["tcl_library_paths"].append("Tix8.4.3")
if "-apple" in target_triple:
python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"]
python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"]
python_info["apple_sdk_canonical_name"] = env[
"APPLE_SDK_CANONICAL_NAME"]
python_info["apple_sdk_deployment_target"] = env[
"APPLE_MIN_DEPLOYMENT_TARGET"]
# Add metadata derived from built distribution.
python_info.update(extra_metadata)
validate_python_json(python_info)
with tempfile.NamedTemporaryFile("w") as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
fh.flush()
if image:
dest_path = "/build/out/python"
else:
dest_path = "out/python"
build_env.copy_file(fh.name, dest_path, dest_name="PYTHON.json")
with open(dest_archive, "wb") as fh:
fh.write(build_env.get_output_archive("python"))