def simple_build(entry):
archive = download_entry(entry, BUILD)
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
shutil.copyfile(archive, td / archive.name)
extract_tar_to_directory(BUILD / 'clang-macos.tar', td)
toolchain_path = td / 'clang-macos' / 'bin'
env = dict(os.environ)
env['%s_VERSION' % entry.upper()] = DOWNLOADS[entry]['version']
# We force a PATH only containing system files: we don't want
# pollution from homebrew, macports, etc.
env['PATH'] = '%s:/usr/bin:/bin' % toolchain_path
env['MACOSX_DEPLOYMENT_TARGET'] = MACOSX_DEPLOYMENT_TARGET
env['NUM_CPUS'] = '%s' % multiprocessing.cpu_count()
exec_and_log([SUPPORT / ('build-%s.sh' % entry)], td, env)
dest_path = BUILD / ('%s-macos.tar' % entry)
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def build_clang():
cmake_archive = download_entry('cmake-macos-bin', BUILD)
ninja_archive = download_entry('ninja-macos-bin', BUILD)
clang_archive = download_entry('clang-6', BUILD)
clang_rt_archive = download_entry('clang-compiler-rt-6', BUILD)
lld_archive = download_entry('lld-6', BUILD)
llvm_archive = download_entry('llvm-6', BUILD)
libcxx_archive = download_entry('libc++-6', BUILD)
libcxxabi_archive = download_entry('libc++abi-6', BUILD)
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
for a in (cmake_archive, ninja_archive, clang_archive,
clang_rt_archive, lld_archive, llvm_archive, libcxx_archive,
libcxxabi_archive):
shutil.copyfile(a, td / a.name)
env = {
'CMAKE_VERSION':
DOWNLOADS['cmake-macos-bin']['version'],
'NINJA_VERSION':
DOWNLOADS['ninja-macos-bin']['version'],
'CLANG_COMPILER_RT_VERSION':
DOWNLOADS['clang-compiler-rt-6']['version'],
'CLANG_VERSION':
DOWNLOADS['clang-6']['version'],
'COMPILER_RT_VERSION':
DOWNLOADS['clang-compiler-rt-6']['version'],
'LIBCXX_VERSION':
DOWNLOADS['libc++-6']['version'],
'LIBCXXABI_VERSION':
DOWNLOADS['libc++abi-6']['version'],
'LLD_VERSION':
DOWNLOADS['lld-6']['version'],
'LLVM_VERSION':
DOWNLOADS['llvm-6']['version'],
'PATH':
'/usr/bin:/bin',
}
exec_and_log([SUPPORT / 'build-clang.sh'], td, env)
dest_path = BUILD / 'clang-macos.tar'
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def build_openssl(perl_path: pathlib.Path):
"""Build OpenSSL from sources using the Perl executable specified."""
# First ensure the dependencies are in place.
openssl_archive = download_entry('openssl', BUILD)
nasm_archive = download_entry('nasm-windows-bin', BUILD)
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
root_32 = td / 'x86'
root_64 = td / 'x64'
root_32.mkdir()
root_64.mkdir()
# Then build the 32 and 64 bit OpenSSL installs in parallel
# (because nmake doesn't do parallel builds).
# TODO we need to adjust the environment to pull in a x86 toolchain
# in order for this to work.
fs = []
with concurrent.futures.ThreadPoolExecutor(2) as e:
#fs.append(e.submit(build_openssl_for_arch, perl_path, 'x86',
# openssl_archive, nasm_archive, root_32))
fs.append(e.submit(build_openssl_for_arch, perl_path, 'amd64',
openssl_archive, nasm_archive, root_64))
for f in fs:
f.result()
install = td / 'out'
#shutil.copytree(root_32 / 'install' / '32', install / 'openssl' / 'win32')
shutil.copytree(root_64 / 'install' / '64', install / 'openssl' / 'amd64')
dest_archive = BUILD / 'openssl-windows.tar'
with dest_archive.open('wb') as fh:
create_tar_from_directory(fh, install)
def build_cpython(pgo=False):
msbuild = find_msbuild()
log('found MSBuild at %s' % msbuild)
# The python.props file keys off MSBUILD, so it needs to be set.
os.environ['MSBUILD'] = str(msbuild)
activeperl_installer = download_entry('activeperl', BUILD)
bzip2_archive = download_entry('bzip2', BUILD)
sqlite_archive = download_entry('sqlite', BUILD)
tk_bin_archive = download_entry('tk-windows-bin', BUILD, local_name='tk-windows-bin.tar.gz')
xz_archive = download_entry('xz', BUILD)
zlib_archive = download_entry('zlib', BUILD)
python_archive = download_entry('cpython-3.7', BUILD)
python_version = DOWNLOADS['cpython-3.7']['version']
openssl_bin_archive = BUILD / 'openssl-windows.tar'
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
with concurrent.futures.ThreadPoolExecutor(7) as e:
for a in (python_archive, bzip2_archive, openssl_bin_archive,
sqlite_archive, tk_bin_archive, xz_archive, zlib_archive):
log('extracting %s to %s' % (a, td))
e.submit(extract_tar_to_directory, a, td)
cpython_source_path = td / ('Python-%s' % python_version)
pcbuild_path = cpython_source_path / 'PCBuild'
out_dir = td / 'out'
build_dir = out_dir / 'python' / 'build'
build_dir.mkdir(parents=True)
# Parse config.c before we hack it up: we want a pristine copy.
config_c_path = cpython_source_path / 'PC' / 'config.c'
with config_c_path.open('r') as fh:
config_c = fh.read()
builtin_extensions = parse_config_c(config_c)
hack_project_files(td, cpython_source_path)
hack_source_files(cpython_source_path)
if pgo:
run_msbuild(msbuild, pcbuild_path, configuration='PGInstrument')
exec_and_log([
str(cpython_source_path / 'python.bat'), '-m', 'test', '--pgo'],
str(pcbuild_path),
os.environ,
exit_on_error=False)
exec_and_log(
[
str(msbuild), str(pcbuild_path / 'pythoncore.vcxproj'),
'/target:KillPython',
'/verbosity:normal',
'/property:Configuration=PGInstrument',
'/property:Platform=x64',
'/property:KillPython=true',
],
pcbuild_path,
os.environ)
run_msbuild(msbuild, pcbuild_path, configuration='PGUpdate')
artifact_config = 'PGUpdate'
else:
run_msbuild(msbuild, pcbuild_path, configuration='Release')
artifact_config = 'Release'
install_dir = out_dir / 'python' / 'install'
# The PC/layout directory contains a script for copying files into
# a release-like directory. Use that for assembling the standalone
# build.
# It doesn't clean up the temp directory it creates. So pass one to it
# under our tempdir.
layout_tmp = td / 'layouttmp'
layout_tmp.mkdir()
exec_and_log(
[
str(cpython_source_path / 'python.bat'),
str(cpython_source_path / 'PC' / 'layout'),
'-vv',
'--source', str(cpython_source_path),
'--build', str(pcbuild_path / 'amd64'),
'--copy', str(install_dir),
'--temp', str(layout_tmp),
'--flat-dlls',
'--include-dev',
'--include-distutils',
],
pcbuild_path,
os.environ)
# Now copy the build artifacts into the output directory.
build_info = collect_python_build_artifacts(
pcbuild_path, out_dir / 'python', 'amd64', artifact_config)
for ext, init_fn in sorted(builtin_extensions.items()):
if ext in build_info['extensions']:
log('built-in extension should not have a build entry: %s' % ext)
sys.exit(1)
build_info['extensions'][ext] = {
'in_core': True,
'objs': [],
'init_fn': init_fn,
'links': [],
'static_lib': None,
}
# Copy OpenSSL libraries as a one-off.
for lib in ('crypto', 'ssl'):
name = 'lib%s_static.lib' % lib
source = td / 'openssl' / 'amd64' / 'lib' / name
dest = out_dir / 'python' / 'build' / 'lib' / name
log('copying %s to %s' % (source, dest))
shutil.copyfile(source, dest)
# Create PYTHON.json file describing this distribution.
python_info = {
# TODO bump version number once format is somewhat stable.
'version': '0',
'os': 'windows',
'arch': 'x86_64',
'python_flavor': 'cpython',
'python_version': python_version,
'python_exe': 'install/python.exe',
'python_include': 'install/include',
'python_stdlib': 'install/Lib',
'build_info': build_info,
}
with (out_dir / 'python' / 'PYTHON.json').open('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
# Copy software licenses file.
shutil.copyfile(ROOT / 'python-licenses.rst', out_dir / 'python' / 'LICENSE.rst')
dest_path = BUILD / 'cpython-windows.tar'
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')
def build_cpython():
python_archive = download_entry('cpython-3.7', BUILD)
python_version = DOWNLOADS['cpython-3.7']['version']
with (SUPPORT / 'static-modules').open('rb') as fh:
static_modules_lines = [
l.rstrip() for l in fh if not l.startswith(b'#')
]
setup = derive_setup_local(static_modules_lines,
python_archive,
disabled=DISABLED_STATIC_MODULES)
config_c_in = parse_config_c(setup['config_c_in'].decode('utf-8'))
setup_dist_content = setup['setup_dist']
setup_local_content = setup['setup_local']
extra_make_content = setup['make_data']
with tempfile.TemporaryDirectory() as td:
td = pathlib.Path(td)
extract_tar_to_directory(BUILD / 'clang-macos.tar', td)
toolchain_path = td / 'clang-macos' / 'bin'
deps_dir = td / 'deps'
deps_dir.mkdir()
extract_tar_to_directory(BUILD / 'bdb-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'bzip2-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'libedit-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'libffi-macos.tar', deps_dir)
# We use the system ncurses and statically link (for now).
#extract_tar_to_directory(BUILD / 'ncurses-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'openssl-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'sqlite-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'uuid-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'xz-macos.tar', deps_dir)
extract_tar_to_directory(BUILD / 'zlib-macos.tar', deps_dir)
extract_tar_to_directory(python_archive, td)
setup_local_path = td / ('Python-%s' %
python_version) / 'Modules' / 'Setup.local'
with setup_local_path.open('wb') as fh:
fh.write(setup_local_content)
makefile_extra_path = td / 'Makefile.extra'
with makefile_extra_path.open('wb') as fh:
fh.write(extra_make_content)
shutil.copyfile(ROOT / 'python-licenses.rst',
td / 'python-licenses.rst')
env = dict(os.environ)
env['PYTHON_VERSION'] = python_version
# We force a PATH only containing system files: we don't want
# pollution from homebrew, macports, etc.
env['PATH'] = '%s:/usr/bin:/bin' % toolchain_path
env['MACOSX_DEPLOYMENT_TARGET'] = MACOSX_DEPLOYMENT_TARGET
env['NUM_CPUS'] = '%s' % multiprocessing.cpu_count()
env['CPYTHON_OPTIMIZED'] = '1'
exec_and_log([SUPPORT / 'build-cpython.sh'], td, env)
# Create PYTHON.json file describing this distribution.
python_info = {
# TODO bump version number once format is somewhat stable.
'version':
'0',
'os':
'macos',
'arch':
'x86_64',
'python_flavor':
'cpython',
'python_version':
python_version,
'python_exe':
'install/bin/python3',
'python_include':
'install/include/python3.7m',
'python_stdlib':
'install/lib/python3.7',
'build_info':
python_build_info(td / 'out' / 'python', config_c_in,
setup_dist_content, setup_local_content),
}
with (td / 'out' / 'python' / 'PYTHON.json').open('w') as fh:
json.dump(python_info, fh, sort_keys=True, indent=4)
dest_path = BUILD / 'cpython-macos.tar'
with dest_path.open('wb') as fh:
create_tar_from_directory(fh, td / 'out')