def android_patch_ssl() -> None:
"""Run necessary patches on an android ssl before building."""
# We bundle our own SSL root certificates on various platforms and use
# the OpenSSL 'SSL_CERT_FILE' env var override to get them to be used
# by default. However, OpenSSL is picky about allowing env-vars to be
# used and something about the Android environment makes it disallow
# them. So we need to force the issue. Alternately we could explicitly
# pass 'cafile' args to SSLContexts whenever we do network-y stuff
# but it seems cleaner to just have things work by default.
fname = 'crypto/getenv.c'
txt = readfile(fname)
txt = replace_one(
txt,
('char *ossl_safe_getenv(const char *name)\n'
'{\n'),
('char *ossl_safe_getenv(const char *name)\n'
'{\n'
' // ERICF TWEAK: ALWAYS ALLOW GETENV.\n'
' return getenv(name);\n'),
)
writefile(fname, txt)
def gather() -> None:
"""Gather per-platform python headers, libs, and modules together.
This assumes all embeddable py builds have been run successfully,
and that PROJROOT is the cwd.
"""
# pylint: disable=too-many-locals
do_android = True
# First off, clear out any existing output.
existing_dirs = [
os.path.join('src/external', d) for d in os.listdir('src/external')
if d.startswith('python-') and d != 'python-notes.txt'
]
existing_dirs += [
os.path.join('assets/src', d) for d in os.listdir('assets/src')
if d.startswith('pylib-')
]
if not do_android:
existing_dirs = [d for d in existing_dirs if 'android' not in d]
for existing_dir in existing_dirs:
subprocess.run(['rm', '-rf', existing_dir], check=True)
apost2 = f'src/Python-{PY_VER_EXACT_ANDROID}/Android/sysroot'
for buildtype in ['debug', 'release']:
debug = buildtype == 'debug'
bsuffix = '_debug' if buildtype == 'debug' else ''
bsuffix2 = '-debug' if buildtype == 'debug' else ''
libname = 'python' + PYVER + ('d' if debug else '')
bases = {
'mac': f'build/python_apple_mac{bsuffix}/build/macOS',
'ios': f'build/python_apple_ios{bsuffix}/build/iOS',
'tvos': f'build/python_apple_tvos{bsuffix}/build/tvOS',
'android_arm': f'build/python_android_arm{bsuffix}/build',
'android_arm64': f'build/python_android_arm64{bsuffix}/build',
'android_x86': f'build/python_android_x86{bsuffix}/build',
'android_x86_64': f'build/python_android_x86_64{bsuffix}/build'
}
bases2 = {
'android_arm': f'build/python_android_arm{bsuffix}/{apost2}',
'android_arm64': f'build/python_android_arm64{bsuffix}/{apost2}',
'android_x86': f'build/python_android_x86{bsuffix}/{apost2}',
'android_x86_64': f'build/python_android_x86_64{bsuffix}/{apost2}'
}
# Note: only need pylib for the first in each group.
builds: list[dict[str, Any]] = [{
'name':
'macos',
'group':
'apple',
'headers':
bases['mac'] + '/Support/Python/Headers',
'libs': [
bases['mac'] + '/Support/Python/libPython.a',
bases['mac'] + '/Support/OpenSSL/libOpenSSL.a',
bases['mac'] + '/Support/XZ/libxz.a',
bases['mac'] + '/Support/BZip2/libbzip2.a',
],
'pylib':
(bases['mac'] + f'/Python-{PY_VER_EXACT_APPLE}-macOS/lib'),
}, {
'name':
'ios',
'group':
'apple',
'headers':
bases['ios'] + '/Support/Python/Headers',
'libs': [
bases['ios'] + '/Support/Python/libPython.a',
bases['ios'] + '/Support/OpenSSL/libOpenSSL.a',
bases['ios'] + '/Support/XZ/libxz.a',
bases['ios'] + '/Support/BZip2/libbzip2.a',
],
}, {
'name':
'tvos',
'group':
'apple',
'headers':
bases['tvos'] + '/Support/Python/Headers',
'libs': [
bases['tvos'] + '/Support/Python/libPython.a',
bases['tvos'] + '/Support/OpenSSL/libOpenSSL.a',
bases['tvos'] + '/Support/XZ/libxz.a',
bases['tvos'] + '/Support/BZip2/libbzip2.a',
],
}, {
'name': 'android_arm',
'group': 'android',
'headers': bases['android_arm'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm'] + f'/usr/lib/lib{libname}.a',
bases2['android_arm'] + '/usr/lib/libssl.a',
bases2['android_arm'] + '/usr/lib/libcrypto.a',
bases2['android_arm'] + '/usr/lib/liblzma.a',
bases2['android_arm'] + '/usr/lib/libsqlite3.a',
bases2['android_arm'] + '/usr/lib/libbz2.a',
bases2['android_arm'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_armeabi-v7a',
'pylib': (bases['android_arm'] + '/usr/lib/python' + PYVER),
}, {
'name': 'android_arm64',
'group': 'android',
'headers': bases['android_arm64'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm64'] + f'/usr/lib/lib{libname}.a',
bases2['android_arm64'] + '/usr/lib/libssl.a',
bases2['android_arm64'] + '/usr/lib/libcrypto.a',
bases2['android_arm64'] + '/usr/lib/liblzma.a',
bases2['android_arm64'] + '/usr/lib/libsqlite3.a',
bases2['android_arm64'] + '/usr/lib/libbz2.a',
bases2['android_arm64'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_arm64-v8a',
}, {
'name': 'android_x86',
'group': 'android',
'headers': bases['android_x86'] + f'/usr/include/{libname}',
'libs': [
bases['android_x86'] + f'/usr/lib/lib{libname}.a',
bases2['android_x86'] + '/usr/lib/libssl.a',
bases2['android_x86'] + '/usr/lib/libcrypto.a',
bases2['android_x86'] + '/usr/lib/liblzma.a',
bases2['android_x86'] + '/usr/lib/libsqlite3.a',
bases2['android_x86'] + '/usr/lib/libbz2.a',
bases2['android_x86'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_x86',
}, {
'name': 'android_x86_64',
'group': 'android',
'headers': bases['android_x86_64'] + f'/usr/include/{libname}',
'libs': [
bases['android_x86_64'] + f'/usr/lib/lib{libname}.a',
bases2['android_x86_64'] + '/usr/lib/libssl.a',
bases2['android_x86_64'] + '/usr/lib/libcrypto.a',
bases2['android_x86_64'] + '/usr/lib/liblzma.a',
bases2['android_x86_64'] + '/usr/lib/libsqlite3.a',
bases2['android_x86_64'] + '/usr/lib/libbz2.a',
bases2['android_x86_64'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_x86_64',
}]
for build in builds:
grp = build['group']
if not do_android and grp == 'android':
continue
builddir = f'src/external/python-{grp}{bsuffix2}'
header_dst = os.path.join(builddir, 'include')
lib_dst = os.path.join(builddir, 'lib')
assets_src_dst = f'assets/src/pylib-{grp}'
# Do some setup only once per group.
if not os.path.exists(builddir):
subprocess.run(['mkdir', '-p', builddir], check=True)
subprocess.run(['mkdir', '-p', lib_dst], check=True)
# Only pull modules into game assets on release pass.
if not debug:
# Copy system modules into the src assets
# dir for this group.
subprocess.run(['mkdir', '-p', assets_src_dst], check=True)
subprocess.run(
[
'rsync', '--recursive', '--include', '*.py',
'--exclude', '__pycache__', '--include', '*/',
'--exclude', '*', build['pylib'] + '/',
assets_src_dst
],
check=True,
)
# Prune a bunch of modules we don't need to cut
# down on size.
subprocess.run('cd "' + assets_src_dst + '" && rm -rf ' +
' '.join(PRUNE_LIB_NAMES),
shell=True,
check=True)
# Some minor filtering to system scripts:
# on iOS/tvOS, addusersitepackages() leads to a crash
# due to _sysconfigdata_dm_ios_darwin module not existing,
# so let's skip that.
fname = f'{assets_src_dst}/site.py'
txt = readfile(fname)
txt = replace_one(
txt,
' known_paths = addusersitepackages(known_paths)',
' # efro tweak: this craps out on ios/tvos.\n'
' # (and we don\'t use it anyway)\n'
' # known_paths = addusersitepackages(known_paths)')
writefile(fname, txt)
# Copy in a base set of headers (everything in a group should
# be using the same headers)
subprocess.run(f'cp -r "{build["headers"]}" "{header_dst}"',
shell=True,
check=True)
# Clear whatever pyconfigs came across; we'll build our own
# universal one below.
subprocess.run('rm ' + header_dst + '/pyconfig*',
shell=True,
check=True)
# Write a master pyconfig header that reroutes to each
# platform's actual header.
with open(header_dst + '/pyconfig.h', 'w',
encoding='utf-8') as hfile:
hfile.write(
'#if BA_OSTYPE_MACOS\n'
'#include "pyconfig-macos.h"\n\n'
'#elif BA_OSTYPE_IOS\n'
'#include "pyconfig-ios.h"\n\n'
'#elif BA_OSTYPE_TVOS\n'
'#include "pyconfig-tvos.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__arm__)\n'
'#include "pyconfig-android_arm.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__aarch64__)\n'
'#include "pyconfig-android_arm64.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__i386__)\n'
'#include "pyconfig-android_x86.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__x86_64__)\n'
'#include "pyconfig-android_x86_64.h"\n\n'
'#else\n'
'#error unknown platform\n\n'
'#endif\n')
# Now copy each build's config headers in with unique names.
cfgs = [
f for f in os.listdir(build['headers'])
if f.startswith('pyconfig')
]
# Copy config headers to their filtered names.
for cfg in cfgs:
out = cfg.replace('pyconfig', 'pyconfig-' + build['name'])
if cfg == 'pyconfig.h':
# For platform's root pyconfig.h we need to filter
# contents too (those headers can themselves include
# others; ios for instance points to a arm64 and a
# x86_64 variant).
contents = readfile(build['headers'] + '/' + cfg)
contents = contents.replace('pyconfig',
'pyconfig-' + build['name'])
writefile(header_dst + '/' + out, contents)
else:
# other configs we just rename
subprocess.run('cp "' + build['headers'] + '/' + cfg +
'" "' + header_dst + '/' + out + '"',
shell=True,
check=True)
# Copy in libs. If the lib gave a specific install name,
# use that; otherwise use name.
targetdir = lib_dst + '/' + build.get('libinst', build['name'])
subprocess.run(['rm', '-rf', targetdir], check=True)
subprocess.run(['mkdir', '-p', targetdir], check=True)
for lib in build['libs']:
subprocess.run(['cp', lib, targetdir], check=True)
print('Great success!')
def build_apple(arch: str, debug: bool = False) -> None:
"""Run a build for the provided apple arch (mac, ios, or tvos)."""
import platform
from efro.error import CleanError
# IMPORTANT; seems we currently wind up building against /usr/local gettext
# stuff. Hopefully the maintainer fixes this, but for now I need to
# remind myself to blow it away while building.
# (via brew remove gettext --ignore-dependencies)
# NOTE: Should check to see if this is still necessary on Apple silicon
# since homebrew stuff is no longer in /usr/local there.
if ('MacBook-Fro' in platform.node()
and os.environ.get('SKIP_GETTEXT_WARNING') != '1'):
if (subprocess.run('which gettext', shell=True,
check=False).returncode == 0):
raise CleanError(
'NEED TO TEMP-KILL GETTEXT (or set SKIP_GETTEXT_WARNING=1)')
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
subprocess.run(['rm', '-rf', builddir], check=True)
subprocess.run(['mkdir', '-p', 'build'], check=True)
subprocess.run(
[
'git', 'clone',
'https://github.com/beeware/Python-Apple-support.git', builddir
],
check=True,
)
os.chdir(builddir)
# TEMP: Check out a particular commit while the branch head is broken.
# We can actually fix this to use the current one, but something
# broke in the underlying build even on old commits so keeping it
# locked for now...
# run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
subprocess.run(['git', 'checkout', PYVER], check=True)
txt = readfile('Makefile')
# Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
txt = replace_one(
txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .',
'&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .')
# Turn doc strings on; looks like it only adds a few hundred k.
txt = txt.replace('--without-doc-strings', '--with-doc-strings')
# Set mac/ios version reqs
# (see issue with utimensat and futimens).
txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.15')
# And equivalent iOS (11+).
txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=13.0')
# Ditto for tvOS.
txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
'CFLAGS-tvOS=-mtvos-version-min=13.0')
if debug:
# Add debug build flag
# (Currently expect to find 2 instances of this).
dline = '--with-doc-strings --enable-ipv6 --without-ensurepip'
splitlen = len(txt.split(dline))
if splitlen != 3:
raise Exception('unexpected configure lines')
txt = txt.replace(dline, '--with-pydebug ' + dline)
# Debug has a different name.
# (Currently expect to replace 12 instances of this).
dline = ('python$(PYTHON_VER)'
if NEWER_PY_TEST else 'python$(PYTHON_VER)m')
splitlen = len(txt.split(dline))
if splitlen != 13:
raise RuntimeError(f'Unexpected configure line count {splitlen}.')
txt = txt.replace(
dline, 'python$(PYTHON_VER)d'
if NEWER_PY_TEST else 'python$(PYTHON_VER)dm')
# Inject our custom modifications to fire before building.
txt = txt.replace(
' # Configure target Python\n',
' cd $$(PYTHON_DIR-$1) && '
f'../../../../../tools/pcommand python_apple_patch {arch}\n'
' # Configure target Python\n',
)
# Use python3 instead of python for libffi setup script
txt = replace_one(
txt,
'cd $$(LIBFFI_DIR-$1) && python generate-darwin-source-and-headers.py'
" --only-$(shell echo $1 | tr '[:upper:]' '[:lower:]')",
'cd $$(LIBFFI_DIR-$1) && python3 generate-darwin-source-and-headers.py'
" --only-$(shell echo $1 | tr '[:upper:]' '[:lower:]')",
)
writefile('Makefile', txt)
# Ok; let 'er rip.
# (we run these in parallel so limit to 1 job a piece;
# otherwise they inherit the -j12 or whatever from the top level)
# (also this build seems to fail with multiple threads)
subprocess.run(
[
'make', '-j1', {
'mac': 'Python-macOS',
'ios': 'Python-iOS',
'tvos': 'Python-tvOS'
}[arch]
],
check=True,
)
print('python build complete! (apple/' + arch + ')')
def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
"""Run a build for android with the given architecture.
(can be arm, arm64, x86, or x86_64)
"""
builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
subprocess.run(['rm', '-rf', builddir], check=True)
subprocess.run(['mkdir', '-p', 'build'], check=True)
subprocess.run(
[
'git', 'clone', 'https://github.com/yan12125/python3-android.git',
builddir
],
check=True,
)
os.chdir(builddir)
# These builds require ANDROID_NDK to be set; make sure that's the case.
os.environ['ANDROID_NDK'] = subprocess.check_output(
[f'{rootdir}/tools/pcommand', 'android_sdk_utils',
'get-ndk-path']).decode().strip()
# Disable builds for dependencies we don't use.
ftxt = readfile('Android/build_deps.py')
# ftxt = replace_one(ftxt, ' NCurses,\n',
# '# NCurses,\n',)
ftxt = replace_one(
ftxt,
' '
'BZip2, GDBM, LibFFI, LibUUID, OpenSSL, Readline, SQLite, XZ, ZLib,\n',
' '
'BZip2, LibUUID, OpenSSL, SQLite, XZ, ZLib,\n',
)
# Older ssl seems to choke on newer ndk layouts.
ftxt = replace_one(
ftxt,
"source = 'https://www.openssl.org/source/openssl-1.1.1h.tar.gz'",
"source = 'https://www.openssl.org/source/openssl-1.1.1l.tar.gz'")
# Give ourselves a handle to patch the OpenSSL build.
ftxt = replace_one(
ftxt,
' # OpenSSL handles NDK internal paths by itself',
' # Ericf addition: do some patching:\n'
' self.run(["../../../../../../../tools/pcommand",'
' "python_android_patch_ssl"])\n'
' # OpenSSL handles NDK internal paths by itself',
)
writefile('Android/build_deps.py', ftxt)
# Tweak some things in the base build script; grab the right version
# of Python and also inject some code to modify bits of python
# after it is extracted.
ftxt = readfile('build.sh')
ftxt = replace_one(ftxt, 'PYVER=3.9.0', f'PYVER={PY_VER_EXACT_ANDROID}')
ftxt = replace_one(
ftxt, ' popd\n', f' ../../../tools/pcommand'
f' python_android_patch Python-{PY_VER_EXACT_ANDROID}\n popd\n')
writefile('build.sh', ftxt)
# Ok, let 'er rip
exargs = ' --with-pydebug' if debug else ''
subprocess.run(f'ARCH={arch} ANDROID_API=21 ./build.sh{exargs}',
shell=True,
check=True)
print('python build complete! (android/' + arch + ')')
def _patch_setup_file(platform: str, arch: str) -> None:
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
fname = 'Modules/Setup'
ftxt = readfile(fname)
if platform == 'android':
prefix = '$(srcdir)/Android/sysroot/usr'
uuid_ex = f' -L{prefix}/lib -luuid'
zlib_ex = f' -I{prefix}/include -L{prefix}/lib -lz'
bz2_ex = f' -I{prefix}/include -L{prefix}/lib -lbz2'
ssl_ex = f' -DUSE_SSL -I{prefix}/include -L{prefix}/lib -lssl -lcrypto'
sqlite_ex = f' -I{prefix}/include -L{prefix}/lib'
hash_ex = ' -DUSE_SSL -lssl -lcrypto'
lzma_ex = ' -llzma'
elif platform == 'apple':
prefix = '$(srcdir)/Android/sysroot/usr'
uuid_ex = ''
zlib_ex = ' -I$(prefix)/include -lz'
bz2_ex = (' -I$(srcdir)/../Support/BZip2/Headers'
' -L$(srcdir)/../Support/BZip2 -lbzip2')
ssl_ex = (' -I$(srcdir)/../Support/OpenSSL/Headers'
' -L$(srcdir)/../Support/OpenSSL -lOpenSSL -DUSE_SSL')
sqlite_ex = ' -I$(srcdir)/Modules/_sqlite'
hash_ex = (' -I$(srcdir)/../Support/OpenSSL/Headers'
' -L$(srcdir)/../Support/OpenSSL -lOpenSSL -DUSE_SSL')
lzma_ex = (' -I$(srcdir)/../Support/XZ/Headers'
' -L$(srcdir)/../Support/XZ/ -lxz')
else:
raise RuntimeError(f'Unknown platform {platform}')
# This list should contain all possible compiled modules to start.
# If any .so files are coming out of builds, their names should be
# added here to stop that.
cmodules = [
'_asyncio', '_bisect', '_blake2', '_codecs_cn', '_codecs_hk',
'_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw',
'_contextvars', '_crypt', '_csv', '_ctypes_test', '_ctypes',
'_curses_panel', '_curses', '_datetime', '_decimal', '_elementtree',
'_heapq', '_json', '_lsprof', '_lzma', '_md5', '_multibytecodec',
'_multiprocessing', '_opcode', '_pickle', '_posixsubprocess', '_queue',
'_random', '_sha1', '_sha3', '_sha256', '_sha512', '_socket',
'_statistics', '_struct', '_testbuffer', '_testcapi',
'_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_uuid',
'_xxsubinterpreters', '_xxtestfuzz', '_zoneinfo', 'array', 'audioop',
'binascii', 'cmath', 'fcntl', 'grp', 'math', 'mmap', 'ossaudiodev',
'parser', 'pyexpat', 'resource', 'select', 'syslog', 'termios',
'unicodedata', 'xxlimited', 'zlib'
]
# Selectively uncomment some existing modules for static compilation.
enables = [
'_asyncio', 'array', 'cmath', 'math', '_contextvars', '_struct',
'_random', '_elementtree', '_pickle', '_datetime', '_zoneinfo',
'_bisect', '_heapq', '_json', '_statistics', 'unicodedata', 'fcntl',
'select', 'mmap', '_csv', '_socket', '_sha3', '_blake2', 'binascii',
'_posixsubprocess'
]
# Note that the _md5 and _sha modules are normally only built if the
# system does not have the OpenSSL libs containing an optimized
# version.
if bool(False):
enables += ['_md5']
for enable in enables:
ftxt = replace_one(ftxt, f'#{enable} ', f'{enable} ')
cmodules.remove(enable)
# Disable ones that were enabled:
disables = ['xxsubtype']
for disable in disables:
ftxt = replace_one(ftxt, f'\n{disable} ', f'\n#{disable} ')
# Additions:
ftxt += '\n# Additions by efrotools:\n'
if bool(True):
ftxt += f'_uuid _uuidmodule.c{uuid_ex}\n'
cmodules.remove('_uuid')
ftxt += f'zlib zlibmodule.c{zlib_ex}\n'
cmodules.remove('zlib')
# Why isn't this getting built as a shared lib by default?
# Do we need it for sure?
ftxt += f'_hashlib _hashopenssl.c{hash_ex}\n'
ftxt += f'_lzma _lzmamodule.c{lzma_ex}\n'
cmodules.remove('_lzma')
ftxt += f'_bz2 _bz2module.c{bz2_ex}\n'
ftxt += f'_ssl _ssl.c{ssl_ex}\n'
ftxt += (f'_sqlite3'
f' _sqlite/cache.c'
f' _sqlite/connection.c'
f' _sqlite/cursor.c'
f' _sqlite/microprotocols.c'
f' _sqlite/module.c'
f' _sqlite/prepare_protocol.c'
f' _sqlite/row.c'
f' _sqlite/statement.c'
f' _sqlite/util.c'
f'{sqlite_ex}'
f' -DMODULE_NAME=\'\\"sqlite3\\"\''
f' -DSQLITE_OMIT_LOAD_EXTENSION'
f' -lsqlite3\n')
# Mac needs this:
if arch == 'mac':
ftxt += ('\n'
'# efrotools: mac urllib needs this:\n'
'_scproxy _scproxy.c '
'-framework SystemConfiguration '
'-framework CoreFoundation\n')
# Explicitly mark the remaining ones as disabled
# (so Python won't try to build them as dynamic libs).
remaining_disabled = ' '.join(cmodules)
ftxt += ('\n# Disabled by efrotools build:\n'
'*disabled*\n'
f'{remaining_disabled}\n')
writefile(fname, ftxt)
# Ok, this is weird.
# When applying the module Setup, python looks for any line containing *=*
# and interprets the whole thing a a global define?...
# This breaks things for our static sqlite compile above.
# The check used to look for [A-Z]*=* which didn't break, so let' just
# change it back to that for now.
# UPDATE: Currently this seems to only be necessary on Android;
# perhaps this broke between 3.9.6 and 3.9.7 or perhaps the apple
# bundle already patches it ¯\_(ツ)_/¯
fname = 'Modules/makesetup'
txt = readfile(fname)
if platform == 'android':
txt = replace_one(txt, ' *=*)'
' DEFS="$line$NL$DEFS"; continue;;',
' [A-Z]*=*) DEFS="$line$NL$DEFS";'
' continue;;')
assert txt.count('[A-Z]*=*') == 1
writefile(fname, txt)
def build_apple(arch: str, debug: bool = False) -> None:
"""Run a build for the provided apple arch (mac, ios, or tvos)."""
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
efrotools.run('rm -rf "' + builddir + '"')
efrotools.run('mkdir -p build')
efrotools.run('git clone '
'[email protected]:pybee/Python-Apple-support.git "' +
builddir + '"')
os.chdir(builddir)
# TEMP: Check out a particular commit while the branch head is broken.
# efrotools.run('git checkout 1a9c71dca298c03517e8236b81cf1d9c8c521cbf')
efrotools.run(f'git checkout {PYTHON_VERSION_MAJOR}')
# On mac we currently have to add the _scproxy module or urllib will
# fail.
txt = efrotools.readfile('patch/Python/Setup.embedded')
if arch == 'mac':
txt += ('\n'
'# ericf added - mac urllib needs this\n'
'_scproxy _scproxy.c '
'-framework SystemConfiguration '
'-framework CoreFoundation')
# Turn off sqlite module. (scratch that; leaving it in.)
# txt = efrotools.replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(')
# txt = txt.replace(' _sqlite/', '# _sqlite/')
# Turn off xz compression module. (scratch that; leaving it in.)
# txt = efrotools.replace_one(txt, '_lzma _', '#_lzma _')
# Turn off bzip2 module.
txt = efrotools.replace_one(txt, '_bz2 _b', '#_bz2 _b')
# Turn off openssl module (only if not doing openssl).
if not ENABLE_OPENSSL:
txt = efrotools.replace_one(txt, '_hashlib _hashopenssl.c',
'#_hashlib _hashopenssl.c')
# Turn off various other stuff we don't use.
for line in [
'_codecs _codecsmodule.c',
'_codecs_cn cjkcodecs/_codecs_cn.c',
'_codecs_hk cjkcodecs/_codecs_hk.c',
'_codecs_iso2022 cjkcodecs/',
'_codecs_jp cjkcodecs/_codecs_jp.c',
'_codecs_jp cjkcodecs/_codecs_jp.c',
'_codecs_kr cjkcodecs/_codecs_kr.c',
'_codecs_tw cjkcodecs/_codecs_tw.c',
'_lsprof _lsprof.o rotatingtree.c',
'_multibytecodec cjkcodecs/multibytecodec.c',
'_multiprocessing _multiprocessing/multiprocessing.c',
'_opcode _opcode.c',
'audioop audioop.c',
'grp grpmodule.c',
'mmap mmapmodule.c',
'parser parsermodule.c',
'pyexpat expat/xmlparse.c',
' expat/xmlrole.c ',
' expat/xmltok.c ',
' pyexpat.c ',
' -I$(srcdir)/Modules/expat ',
' -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI'
' -DXML_DEV_URANDOM',
'resource resource.c',
'syslog syslogmodule.c',
'termios termios.c',
'_ctypes_test _ctypes/_ctypes_test.c',
'_testbuffer _testbuffer.c',
'_testimportmultiple _testimportmultiple.c',
'_crypt _cryptmodule.c', # not on android so disabling here too
]:
txt = efrotools.replace_one(txt, line, '#' + line)
if ENABLE_OPENSSL:
# _md5 and _sha modules are normally only built if the
# system does not have the OpenSSL libs containing an optimized
# version.
# Note: seems we still need sha3 or we get errors
for line in [
'_md5 md5module.c',
'_sha1 sha1module.c',
# '_sha3 _sha3/sha3module.c',
'_sha256 sha256module.c',
'_sha512 sha512module.c',
]:
txt = efrotools.replace_one(txt, line, '#' + line)
else:
txt = efrotools.replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c')
efrotools.writefile('patch/Python/Setup.embedded', txt)
txt = efrotools.readfile('Makefile')
# Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
txt = efrotools.replace_one(
txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .',
'&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .')
# Remove makefile dependencies so we don't build the
# libs we're not using.
srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: '
txt = efrotools.replace_one(
txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' +
('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') +
'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt)
srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.'
'b$(BUILD_NUMBER).tar.gz: ')
txt = efrotools.replace_one(
txt, srctxt,
'dist/Python-$(PYTHON_VER)-$1-support.b$(BUILD_NUMBER).tar.gz:'
' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt)
# Turn doc strings on; looks like it only adds a few hundred k.
txt = txt.replace('--without-doc-strings', '--with-doc-strings')
# We're currently aiming at 10.13+ on mac
# (see issue with utimensat and futimens).
txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.13')
# And equivalent iOS (11+).
txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=11.0')
# Ditto for tvOS.
txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
'CFLAGS-tvOS=-mtvos-version-min=11.0')
if debug:
# Add debug build flag
# (Currently expect to find 2 instances of this).
dline = '--with-doc-strings --enable-ipv6 --without-ensurepip'
splitlen = len(txt.split(dline))
if splitlen != 3:
raise Exception("unexpected configure lines")
txt = txt.replace(dline, '--with-pydebug ' + dline)
# Debug has a different name.
# (Currently expect to replace 13 instances of this).
dline = 'python$(PYTHON_VER)m'
splitlen = len(txt.split(dline))
if splitlen != 14:
raise Exception("unexpected configure lines")
txt = txt.replace(dline, 'python$(PYTHON_VER)dm')
efrotools.writefile('Makefile', txt)
# Ok; let 'er rip.
# (we run these in parallel so limit to 1 job a piece;
# otherwise they inherit the -j12 or whatever from the top level)
# (also this build seems to fail with multiple threads)
efrotools.run('make -j1 ' + {
'mac': 'Python-macOS',
'ios': 'Python-iOS',
'tvos': 'Python-tvOS'
}[arch])
print('python build complete! (apple/' + arch + ')')
def gather() -> None:
"""Gather per-platform python headers, libs, and modules together.
This assumes all embeddable py builds have been run successfully,
and that PROJROOT is the cwd.
"""
# pylint: disable=too-many-locals
# First off, clear out any existing output.
existing_dirs = [
os.path.join('src/external', d) for d in os.listdir('src/external')
if d.startswith('python-') and d != 'python-notes.txt'
]
existing_dirs += [
os.path.join('assets/src', d) for d in os.listdir('assets/src')
if d.startswith('pylib-')
]
for existing_dir in existing_dirs:
efrotools.run('rm -rf "' + existing_dir + '"')
# Build our set of site-packages that we'll bundle in addition
# to the base system.
# FIXME: Should we perhaps make this part more explicit?..
# we might get unexpected changes sneaking if we're just
# pulling from installed python. But then again, anytime we're doing
# a new python build/gather we should expect *some* changes even if
# only at the build-system level since we pull some of that directly
# from latest git stuff.
efrotools.run('mkdir -p "assets/src/pylib-site-packages"')
efrotools.run('cp "/usr/local/lib/python' + PYTHON_VERSION_MAJOR +
'/site-packages/typing_extensions.py"'
' "assets/src/pylib-site-packages/"')
for buildtype in ['debug', 'release']:
debug = buildtype == 'debug'
bsuffix = '_debug' if buildtype == 'debug' else ''
bsuffix2 = '-debug' if buildtype == 'debug' else ''
libname = 'python' + PYTHON_VERSION_MAJOR + ('dm' if debug else 'm')
bases = {
'mac': f'build/python_apple_mac{bsuffix}/build/macOS',
'ios': f'build/python_apple_ios{bsuffix}/build/iOS',
'tvos': f'build/python_apple_tvos{bsuffix}/build/tvOS',
'android_arm': f'build/python_android_arm{bsuffix}/build/sysroot',
'android_arm64':
f'build/python_android_arm64{bsuffix}/build/sysroot',
'android_x86': f'build/python_android_x86{bsuffix}/build/sysroot',
'android_x86_64':
f'build/python_android_x86_64{bsuffix}/build/sysroot'
}
# Note: only need pylib for the first in each group.
builds: List[Dict[str, Any]] = [{
'name':
'macos',
'group':
'apple',
'headers':
bases['mac'] + '/Support/Python/Headers',
'libs': [
bases['mac'] + '/Support/Python/libPython.a',
bases['mac'] + '/Support/OpenSSL/libOpenSSL.a',
bases['mac'] + '/Support/XZ/libxz.a'
],
'pylib':
(bases['mac'] + '/python/lib/python' + PYTHON_VERSION_MAJOR),
}, {
'name':
'ios',
'group':
'apple',
'headers':
bases['ios'] + '/Support/Python/Headers',
'libs': [
bases['ios'] + '/Support/Python/libPython.a',
bases['ios'] + '/Support/OpenSSL/libOpenSSL.a',
bases['ios'] + '/Support/XZ/libxz.a'
],
}, {
'name':
'tvos',
'group':
'apple',
'headers':
bases['tvos'] + '/Support/Python/Headers',
'libs': [
bases['tvos'] + '/Support/Python/libPython.a',
bases['tvos'] + '/Support/OpenSSL/libOpenSSL.a',
bases['tvos'] + '/Support/XZ/libxz.a'
],
}, {
'name':
'android_arm',
'group':
'android',
'headers':
bases['android_arm'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm'] + f'/usr/lib/lib{libname}.a',
bases['android_arm'] + '/usr/lib/libssl.a',
bases['android_arm'] + '/usr/lib/libcrypto.a',
bases['android_arm'] + '/usr/lib/liblzma.a',
bases['android_arm'] + '/usr/lib/libsqlite3.a'
],
'libinst':
'android_armeabi-v7a',
'pylib':
(bases['android_arm'] + '/usr/lib/python' + PYTHON_VERSION_MAJOR),
}, {
'name':
'android_arm64',
'group':
'android',
'headers':
bases['android_arm64'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm64'] + f'/usr/lib/lib{libname}.a',
bases['android_arm64'] + '/usr/lib/libssl.a',
bases['android_arm64'] + '/usr/lib/libcrypto.a',
bases['android_arm64'] + '/usr/lib/liblzma.a',
bases['android_arm64'] + '/usr/lib/libsqlite3.a'
],
'libinst':
'android_arm64-v8a',
}, {
'name':
'android_x86',
'group':
'android',
'headers':
bases['android_x86'] + f'/usr/include/{libname}',
'libs': [
bases['android_x86'] + f'/usr/lib/lib{libname}.a',
bases['android_x86'] + '/usr/lib/libssl.a',
bases['android_x86'] + '/usr/lib/libcrypto.a',
bases['android_x86'] + '/usr/lib/liblzma.a',
bases['android_x86'] + '/usr/lib/libsqlite3.a'
],
'libinst':
'android_x86',
}, {
'name':
'android_x86_64',
'group':
'android',
'headers':
bases['android_x86_64'] + f'/usr/include/{libname}',
'libs': [
bases['android_x86_64'] + f'/usr/lib/lib{libname}.a',
bases['android_x86_64'] + '/usr/lib/libssl.a',
bases['android_x86_64'] + '/usr/lib/libcrypto.a',
bases['android_x86_64'] + '/usr/lib/liblzma.a',
bases['android_x86_64'] + '/usr/lib/libsqlite3.a'
],
'libinst':
'android_x86_64',
}]
for build in builds:
grp = build['group']
builddir = f'src/external/python-{grp}{bsuffix2}'
header_dst = os.path.join(builddir, 'include')
lib_dst = os.path.join(builddir, 'lib')
assets_src_dst = f'assets/src/pylib-{grp}'
# Do some setup only once per group.
if not os.path.exists(builddir):
efrotools.run('mkdir -p "' + builddir + '"')
efrotools.run('mkdir -p "' + lib_dst + '"')
# Only pull modules into game assets on release pass.
if not debug:
# Copy system modules into the src assets
# dir for this group.
efrotools.run('mkdir -p "' + assets_src_dst + '"')
efrotools.run(
'rsync --recursive --include "*.py"'
' --exclude __pycache__ --include "*/" --exclude "*" "'
+ build['pylib'] + '/" "' + assets_src_dst + '"')
# Prune a bunch of modules we don't need to cut
# down on size.
prune = [
'config-*', 'idlelib', 'lib-dynload', 'lib2to3',
'multiprocessing', 'pydoc_data', 'site-packages',
'ensurepip', 'tkinter', 'wsgiref', 'distutils',
'turtle.py', 'turtledemo', 'test', 'sqlite3/test',
'unittest', 'dbm', 'venv', 'ctypes/test', 'imaplib.py',
'_sysconfigdata_*'
]
efrotools.run('cd "' + assets_src_dst + '" && rm -rf ' +
' '.join(prune))
# Some minor filtering to system scripts:
# on iOS/tvOS, addusersitepackages() leads to a crash
# due to _sysconfigdata_dm_ios_darwin module not existing,
# so let's skip that.
fname = f'{assets_src_dst}/site.py'
txt = efrotools.readfile(fname)
txt = efrotools.replace_one(
txt,
' known_paths = addusersitepackages(known_paths)',
' # efro tweak: this craps out on ios/tvos.\n'
' # (and we don\'t use it anyway)\n'
' # known_paths = addusersitepackages(known_paths)')
efrotools.writefile(fname, txt)
# Copy in a base set of headers (everything in a group should
# be using the same headers)
efrotools.run(f'cp -r "{build["headers"]}" "{header_dst}"')
# Clear whatever pyconfigs came across; we'll build our own
# universal one below.
efrotools.run('rm ' + header_dst + '/pyconfig*')
# Write a master pyconfig header that reroutes to each
# platform's actual header.
with open(header_dst + '/pyconfig.h', 'w') as hfile:
hfile.write(
'#if BA_OSTYPE_MACOS\n'
'#include "pyconfig-macos.h"\n\n'
'#elif BA_OSTYPE_IOS\n'
'#include "pyconfig-ios.h"\n\n'
'#elif BA_OSTYPE_TVOS\n'
'#include "pyconfig-tvos.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__arm__)\n'
'#include "pyconfig-android_arm.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__aarch64__)\n'
'#include "pyconfig-android_arm64.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__i386__)\n'
'#include "pyconfig-android_x86.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__x86_64__)\n'
'#include "pyconfig-android_x86_64.h"\n\n'
'#else\n'
'#error unknown platform\n\n'
'#endif\n')
# Now copy each build's config headers in with unique names.
cfgs = [
f for f in os.listdir(build['headers'])
if f.startswith('pyconfig')
]
# Copy config headers to their filtered names.
for cfg in cfgs:
out = cfg.replace('pyconfig', 'pyconfig-' + build['name'])
if cfg == 'pyconfig.h':
# For platform's root pyconfig.h we need to filter
# contents too (those headers can themselves include
# others; ios for instance points to a arm64 and a
# x86_64 variant).
contents = efrotools.readfile(build['headers'] + '/' + cfg)
contents = contents.replace('pyconfig',
'pyconfig-' + build['name'])
efrotools.writefile(header_dst + '/' + out, contents)
else:
# other configs we just rename
efrotools.run('cp "' + build['headers'] + '/' + cfg +
'" "' + header_dst + '/' + out + '"')
# Copy in libs. If the lib gave a specific install name,
# use that; otherwise use name.
targetdir = lib_dst + '/' + build.get('libinst', build['name'])
efrotools.run('rm -rf "' + targetdir + '"')
efrotools.run('mkdir -p "' + targetdir + '"')
for lib in build['libs']:
efrotools.run('cp "' + lib + '" "' + targetdir + '"')
print('Great success!')
def android_patch() -> None:
"""Run necessary patches on an android archive before building."""
fname = 'src/cpython/Modules/Setup.dist'
txt = efrotools.readfile(fname)
# Need to switch some flags on this one.
txt = efrotools.replace_one(txt, '#zlib zlibmodule.c',
'zlib zlibmodule.c -lz\n#zlib zlibmodule.c')
# Just turn all these on.
for enable in [
'#array arraymodule.c', '#cmath cmathmodule.c _math.c',
'#math mathmodule.c', '#_contextvars _contextvarsmodule.c',
'#_struct _struct.c', '#_weakref _weakref.c',
'#_testcapi _testcapimodule.c', '#_random _randommodule.c',
'#_elementtree -I', '#_pickle _pickle.c',
'#_datetime _datetimemodule.c', '#_bisect _bisectmodule.c',
'#_heapq _heapqmodule.c', '#_asyncio _asynciomodule.c',
'#unicodedata unicodedata.c', '#fcntl fcntlmodule.c',
'#select selectmodule.c', '#_csv _csv.c',
'#_socket socketmodule.c', '#_blake2 _blake2/blake2module.c',
'#binascii binascii.c', '#_posixsubprocess _posixsubprocess.c',
'#_sha3 _sha3/sha3module.c'
]:
txt = efrotools.replace_one(txt, enable, enable[1:])
if ENABLE_OPENSSL:
txt = efrotools.replace_one(txt, '#_ssl _ssl.c \\',
'_ssl _ssl.c -DUSE_SSL -lssl -lcrypto')
else:
# Note that the _md5 and _sha modules are normally only built if the
# system does not have the OpenSSL libs containing an optimized
# version.
for enable in [
'#_md5 md5module.c', '#_sha1 sha1module.c',
'#_sha256 sha256module.c', '#_sha512 sha512module.c'
]:
txt = efrotools.replace_one(txt, enable, enable[1:])
# Turn this off (its just an example module).
txt = efrotools.replace_one(txt, 'xxsubtype xxsubtype.c',
'#xxsubtype xxsubtype.c')
# For whatever reason this stuff isn't in there at all; add it.
txt += '\n_json _json.c\n'
txt += '\n_lzma _lzmamodule.c -llzma\n'
txt += ('\n_sqlite3 -I$(srcdir)/Modules/_sqlite'
' -DMODULE_NAME=\'\\"sqlite3\\"\' -DSQLITE_OMIT_LOAD_EXTENSION'
' -lsqlite3 \\\n'
' _sqlite/cache.c \\\n'
' _sqlite/connection.c \\\n'
' _sqlite/cursor.c \\\n'
' _sqlite/microprotocols.c \\\n'
' _sqlite/module.c \\\n'
' _sqlite/prepare_protocol.c \\\n'
' _sqlite/row.c \\\n'
' _sqlite/statement.c \\\n'
' _sqlite/util.c\n')
if ENABLE_OPENSSL:
txt += '\n\n_hashlib _hashopenssl.c -DUSE_SSL -lssl -lcrypto\n'
txt += '\n\n*disabled*\n_ctypes _crypt grp'
efrotools.writefile(fname, txt)
# Ok, this is weird.
# When applying the module Setup, python looks for any line containing *=*
# and interprets the whole thing a a global define?...
# This breaks things for our static sqlite compile above.
# The check used to look for [A-Z]*=* which didn't break, so let' just
# change it back to that for now.
fname = 'src/cpython/Modules/makesetup'
txt = efrotools.readfile(fname)
txt = efrotools.replace_one(
txt, ' *=*) DEFS="$line$NL$DEFS"; continue;;',
' [A-Z]*=*) DEFS="$line$NL$DEFS"; continue;;')
efrotools.writefile(fname, txt)
print("APPLIED EFROTOOLS ANDROID BUILD PATCHES.")
def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
"""Run a build for android with the given architecture.
(can be arm, arm64, x86, or x86_64)
"""
import subprocess
builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
efrotools.run('rm -rf "' + builddir + '"')
efrotools.run('mkdir -p build')
efrotools.run('git clone '
'[email protected]:yan12125/python3-android.git "' + builddir +
'"')
os.chdir(builddir)
# It seems we now need 'autopoint' as part of this build, but on mac it
# is not available on the normal path, but only as part of the keg-only
# gettext homebrew formula.
if (subprocess.run('which autopoint', shell=True, check=False).returncode
!= 0):
print("Updating path for mac autopoint...")
appath = subprocess.run('brew ls gettext | grep bin/autopoint',
shell=True,
check=True,
capture_output=True)
appathout = os.path.dirname(appath.stdout.decode().strip())
os.environ['PATH'] += (':' + appathout)
print(f'ADDED "{appathout}" TO SYS PATH...')
# Commit from Jan 8, 2020. Right after this, the build system was switched
# a a completely new minimal one which will take some work to update here.
# Punting on that for now...
if True: # pylint: disable=using-constant-test
efrotools.run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae')
ftxt = efrotools.readfile('pybuild/env.py')
# Set the packages we build.
ftxt = efrotools.replace_one(
ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," +
(" 'openssl'" if ENABLE_OPENSSL else "") + ")\n# packages = (")
# Don't wanna bother with gpg signing stuff.
ftxt = efrotools.replace_one(ftxt, 'verify_source = True',
'verify_source = False')
# Sub in the min api level we're targeting.
ftxt = efrotools.replace_one(ftxt, 'android_api_level = 21',
'android_api_level = 21')
ftxt = efrotools.replace_one(ftxt, "target_arch = 'arm'",
"target_arch = '" + arch + "'")
efrotools.writefile('pybuild/env.py', ftxt)
ftxt = efrotools.readfile('Makefile')
# This needs to be python3 for us.
ftxt = efrotools.replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n')
efrotools.writefile('Makefile', ftxt)
ftxt = efrotools.readfile('pybuild/packages/python.py')
# We currently build as a static lib.
ftxt = efrotools.replace_one(ftxt, " '--enable-shared',\n", "")
ftxt = efrotools.replace_one(
ftxt, "super().__init__('https://github.com/python/cpython/')",
"super().__init__('https://github.com/python/cpython/', branch='3.7')")
# Turn ipv6 on (curious why its turned off here?...)
# Also, turn on low level debugging features for our debug builds.
ftxt = efrotools.replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',")
if debug:
ftxt = efrotools.replace_one(ftxt, "'./configure',",
"'./configure', '--with-pydebug',")
# We don't use this stuff so lets strip it out to simplify.
ftxt = efrotools.replace_one(ftxt, "'--without-ensurepip',", "")
# This builds all modules as dynamic libs, but we want to be consistent
# with our other embedded builds and just static-build the ones we
# need... so to change that we'll need to add a hook for ourself after
# python is downloaded but before it is built so we can muck with it.
ftxt = efrotools.replace_one(
ftxt, ' def build(self):',
' def build(self):\n import os\n'
' if os.system(\'"' + rootdir +
'/tools/snippets" python_android_patch "' + os.getcwd() +
'"\') != 0: raise Exception("patch apply failed")')
efrotools.writefile('pybuild/packages/python.py', ftxt)
# Set this to a particular cpython commit to target exact releases from git
commit = '43364a7ae01fbe4288ef42622259a0038ce1edcc' # 3.7.6 release
if commit is not None:
ftxt = efrotools.readfile('pybuild/source.py')
# Check out a particular commit right after the clone.
ftxt = efrotools.replace_one(
ftxt, "'git', 'clone', '--single-branch', '-b',"
" self.branch, self.source_url, self.dest])",
"'git', 'clone', '-b',"
" self.branch, self.source_url, self.dest])\n"
" # efro: hack to get the python we want.\n"
" print('DOING URL', self.source_url)\n"
" if self.source_url == "
"'https://github.com/python/cpython/':\n"
" run_in_dir(['git', 'checkout', '" + commit +
"'], self.source_dir)")
efrotools.writefile('pybuild/source.py', ftxt)
ftxt = efrotools.readfile('pybuild/util.py')
# Still don't wanna bother with gpg signing stuff.
ftxt = efrotools.replace_one(
ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n',
'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n'
' print("gpg-verify disabled by ericf")\n'
' return\n')
efrotools.writefile('pybuild/util.py', ftxt)
# These builds require ANDROID_NDK to be set, so make sure that's
# the case.
os.environ['ANDROID_NDK'] = subprocess.check_output(
[rootdir + '/tools/android_sdk_utils',
'get-ndk-path']).decode().strip()
# Ok, let 'er rip
# (we often run these builds in parallel so limit to 1 job a piece;
# otherwise they each inherit the -j12 or whatever from the top level).
efrotools.run('make -j1')
print('python build complete! (android/' + arch + ')')
def build_apple(arch: str, debug: bool = False) -> None:
"""Run a build for the provided apple arch (mac, ios, or tvos)."""
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
import platform
import subprocess
from efro.error import CleanError
# IMPORTANT; seems we currently wind up building against /usr/local gettext
# stuff. Hopefully the maintainer fixes this, but for now I need to
# remind myself to blow it away while building.
# (via brew remove gettext --ignore-dependencies)
if 'MacBook-Fro' in platform.node():
if (subprocess.run('which gettext', shell=True,
check=False).returncode == 0):
raise CleanError('NEED TO TEMP-KILL GETTEXT')
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
run('rm -rf "' + builddir + '"')
run('mkdir -p build')
run('git clone '
'[email protected]:pybee/Python-Apple-support.git "' + builddir + '"')
os.chdir(builddir)
# TEMP: Check out a particular commit while the branch head is broken.
# We can actually fix this to use the current one, but something
# broke in the underlying build even on old commits so keeping it
# locked for now...
# run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
run(f'git checkout {PYVER}')
# On mac we currently have to add the _scproxy module or urllib will
# fail.
txt = readfile('patch/Python/Setup.embedded')
if arch == 'mac':
txt += ('\n'
'# ericf added - mac urllib needs this\n'
'_scproxy _scproxy.c '
'-framework SystemConfiguration '
'-framework CoreFoundation')
# Turn off sqlite module. (scratch that; leaving it in.)
# txt = replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(')
# txt = txt.replace(' _sqlite/', '# _sqlite/')
# Turn off xz compression module. (scratch that; leaving it in.)
# txt = replace_one(txt, '_lzma _', '#_lzma _')
# Turn off bzip2 module.
txt = replace_one(txt, '_bz2 _b', '#_bz2 _b')
# Turn off openssl module (only if not doing openssl).
if not ENABLE_OPENSSL:
txt = replace_one(txt, '_hashlib _hashopenssl.c',
'#_hashlib _hashopenssl.c')
# Turn off various other stuff we don't use.
for line in [
'_codecs _codecsmodule.c',
'_codecs_cn cjkcodecs/_codecs_cn.c',
'_codecs_hk cjkcodecs/_codecs_hk.c',
'_codecs_iso2022 cjkcodecs/',
'_codecs_jp cjkcodecs/_codecs_jp.c',
'_codecs_jp cjkcodecs/_codecs_jp.c',
'_codecs_kr cjkcodecs/_codecs_kr.c',
'_codecs_tw cjkcodecs/_codecs_tw.c',
'_lsprof _lsprof.o rotatingtree.c',
'_multibytecodec cjkcodecs/multibytecodec.c',
'_multiprocessing _multiprocessing/multiprocessing.c',
'_opcode _opcode.c',
'audioop audioop.c',
'grp grpmodule.c',
'mmap mmapmodule.c',
'parser parsermodule.c',
'pyexpat expat/xmlparse.c',
' expat/xmlrole.c ',
' expat/xmltok.c ',
' pyexpat.c ',
' -I$(srcdir)/Modules/expat ',
' -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI'
' -DXML_DEV_URANDOM',
'resource resource.c',
'syslog syslogmodule.c',
'termios termios.c',
'_ctypes_test _ctypes/_ctypes_test.c',
'_testbuffer _testbuffer.c',
'_testimportmultiple _testimportmultiple.c',
'_crypt _cryptmodule.c', # not on android so disabling here too
]:
txt = replace_one(txt, line, '#' + line)
if ENABLE_OPENSSL:
# _md5 and _sha modules are normally only built if the
# system does not have the OpenSSL libs containing an optimized
# version.
# Note: seems we still need sha3 or we get errors
for line in [
'_md5 md5module.c',
'_sha1 sha1module.c',
# '_sha3 _sha3/sha3module.c',
'_sha256 sha256module.c',
'_sha512 sha512module.c',
]:
txt = replace_one(txt, line, '#' + line)
else:
txt = replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c')
writefile('patch/Python/Setup.embedded', txt)
txt = readfile('Makefile')
# Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
txt = replace_one(
txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .',
'&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .')
# Remove makefile dependencies so we don't build the
# libs we're not using.
srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: '
if PY38:
txt = replace_one(
txt, srctxt,
'$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: ' +
('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') +
'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt)
else:
txt = replace_one(
txt, srctxt,
'$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' +
('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') +
'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt)
srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.'
'$(BUILD_NUMBER).tar.gz: ')
txt = replace_one(
txt, srctxt,
'dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz:'
' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt)
# Turn doc strings on; looks like it only adds a few hundred k.
txt = txt.replace('--without-doc-strings', '--with-doc-strings')
# Set mac/ios version reqs
# (see issue with utimensat and futimens).
txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8',
'MACOSX_DEPLOYMENT_TARGET=10.15')
# And equivalent iOS (11+).
txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0',
'CFLAGS-iOS=-mios-version-min=13.0')
# Ditto for tvOS.
txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0',
'CFLAGS-tvOS=-mtvos-version-min=13.0')
if debug:
# Add debug build flag
# (Currently expect to find 2 instances of this).
dline = '--with-doc-strings --enable-ipv6 --without-ensurepip'
splitlen = len(txt.split(dline))
if splitlen != 3:
raise Exception('unexpected configure lines')
txt = txt.replace(dline, '--with-pydebug ' + dline)
# Debug has a different name.
# (Currently expect to replace 12 instances of this).
dline = 'python$(PYTHON_VER)' if PY38 else 'python$(PYTHON_VER)m'
splitlen = len(txt.split(dline))
if splitlen != 13:
raise RuntimeError(f'Unexpected configure line count {splitlen}.')
txt = txt.replace(
dline, 'python$(PYTHON_VER)d' if PY38 else 'python$(PYTHON_VER)dm')
writefile('Makefile', txt)
# Ok; let 'er rip.
# (we run these in parallel so limit to 1 job a piece;
# otherwise they inherit the -j12 or whatever from the top level)
# (also this build seems to fail with multiple threads)
run('make -j1 ' + {
'mac': 'Python-macOS',
'ios': 'Python-iOS',
'tvos': 'Python-tvOS'
}[arch])
print('python build complete! (apple/' + arch + ')')
def build_apple(arch: str, debug: bool = False) -> None:
"""Run a build for the provided apple arch (mac, ios, or tvos)."""
import platform
from efro.error import CleanError
# IMPORTANT; seems we currently wind up building against /usr/local gettext
# stuff. Hopefully the maintainer fixes this, but for now I need to
# remind myself to blow it away while building.
# (via brew remove gettext --ignore-dependencies)
# NOTE: Should check to see if this is still necessary on Apple silicon
# since homebrew stuff is no longer in /usr/local there.
if ('MacBook-Fro' in platform.node()
and os.environ.get('SKIP_GETTEXT_WARNING') != '1'):
if (subprocess.run('which gettext', shell=True,
check=False).returncode == 0):
raise CleanError(
'NEED TO TEMP-KILL GETTEXT (or set SKIP_GETTEXT_WARNING=1)')
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
subprocess.run(['rm', '-rf', builddir], check=True)
subprocess.run(['mkdir', '-p', 'build'], check=True)
subprocess.run(
[
'git', 'clone',
'https://github.com/beeware/Python-Apple-support.git', builddir
],
check=True,
)
os.chdir(builddir)
# TEMP: Check out a particular commit while the branch head is broken.
# We can actually fix this to use the current one, but something
# broke in the underlying build even on old commits so keeping it
# locked for now...
# run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
subprocess.run(['git', 'checkout', PY_VER], check=True)
txt = readfile('Makefile')
# Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
txt = replace_exact(
txt,
'\t\tPATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/_install/bin:$(PATH)',
'\t\tPATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/_install/bin:$(PATH)"')
# Turn doc strings on; looks like it only adds a few hundred k.
txt = replace_exact(txt,
'--without-doc-strings',
'--with-doc-strings',
count=2)
# Customize our minimum version requirements
txt = replace_exact(
txt,
'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
)
txt = replace_exact(
txt,
'CFLAGS-iOS=-mios-version-min=12.0 ',
'CFLAGS-iOS=-mios-version-min=12.0 ',
)
txt = replace_exact(
txt,
'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
)
assert '--with-pydebug' not in txt
if debug:
# Add debug build flag
txt = replace_exact(
txt,
'--enable-ipv6 --without-ensurepip ',
'--enable-ipv6 --with-pydebug --without-ensurepip ',
count=2,
)
# Debug lib has a different name.
txt = replace_exact(txt,
'python$(PYTHON_VER).a',
'python$(PYTHON_VER)d.a',
count=2)
txt = replace_exact(txt,
'/include/python$(PYTHON_VER)',
'/include/python$(PYTHON_VER)d',
count=4)
# Inject our custom modifications to fire right after their normal
# Setup.local filtering and right before building (and pass the same
# 'slice' value they use so we can use it too).
txt = replace_exact(
txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n',
'\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n'
'\tcd $$(PYTHON_DIR-$(target)) && '
f'../../../../../tools/pcommand python_apple_patch {arch} '
'"$$(SLICE-$$(SDK-$(target)))"\n')
txt = replace_exact(
txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n',
'\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n'
'\tcd $$(PYTHON_DIR-$(os)) && '
f'../../../../../tools/pcommand python_apple_patch {arch} '
'"$$(SLICE-macosx)"\n')
writefile('Makefile', txt)
# Ok; let 'er rip.
# (we run these in parallel so limit to 1 job a piece;
# otherwise they inherit the -j12 or whatever from the top level)
# (also this build seems to fail with multiple threads)
subprocess.run(
[
'make', '-j1', {
'mac': 'Python-macOS',
'ios': 'Python-iOS',
'tvos': 'Python-tvOS'
}[arch]
],
check=True,
)
print('python build complete! (apple/' + arch + ')')
def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
"""Run a build for android with the given architecture.
(can be arm, arm64, x86, or x86_64)
"""
# pylint: disable=too-many-statements
import subprocess
import platform
builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
run('rm -rf "' + builddir + '"')
run('mkdir -p build')
run('git clone '
'https://github.com/yan12125/python3-android.git "' + builddir + '"')
os.chdir(builddir)
# It seems we now need 'autopoint' as part of this build, but on mac it
# is not available on the normal path, but only as part of the keg-only
# gettext homebrew formula.
if platform.system() == 'Darwin':
if (subprocess.run('which autopoint', shell=True,
check=False).returncode != 0):
print('Updating path for mac autopoint...')
appath = subprocess.run('brew ls gettext | grep bin/autopoint',
shell=True,
check=True,
capture_output=True)
appathout = os.path.dirname(appath.stdout.decode().strip())
os.environ['PATH'] += (':' + appathout)
print(f'ADDED "{appathout}" TO SYS PATH...')
# Commit from Jan 8, 2020. Right after this, the build system was switched
# a a completely new minimal one which will take some work to update here.
# Punting on that for now... (tentative plan is to try and adopt the new
# one when we update for Python 3.10 in a year or two).
if True: # pylint: disable=using-constant-test
run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae')
ftxt = readfile('pybuild/env.py')
# Set the packages we build.
ftxt = replace_one(
ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," +
(" 'openssl'" if ENABLE_OPENSSL else '') + ')\n# packages = (')
# Don't wanna bother with gpg signing stuff.
ftxt = replace_one(ftxt, 'verify_source = True', 'verify_source = False')
# Sub in the min api level we're targeting.
ftxt = replace_one(ftxt, 'android_api_level = 21',
'android_api_level = 21')
ftxt = replace_one(ftxt, "target_arch = 'arm'",
"target_arch = '" + arch + "'")
writefile('pybuild/env.py', ftxt)
ftxt = readfile('Makefile')
# This needs to be python3 for us.
ftxt = replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n')
writefile('Makefile', ftxt)
ftxt = readfile('pybuild/packages/python.py')
# We currently build as a static lib.
ftxt = replace_one(ftxt, " '--enable-shared',\n", '')
ftxt = replace_one(
ftxt, "super().__init__('https://github.com/python/cpython/')",
"super().__init__('https://github.com/python/cpython/'"
f", branch='{PYVER}')")
# Turn ipv6 on (curious why its turned off here?...)
# Also, turn on low level debugging features for our debug builds.
ftxt = replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',")
if debug:
ftxt = replace_one(ftxt, "'./configure',",
"'./configure', '--with-pydebug',")
# We don't use this stuff so lets strip it out to simplify.
ftxt = replace_one(ftxt, "'--without-ensurepip',", '')
# This builds all modules as dynamic libs, but we want to be consistent
# with our other embedded builds and just static-build the ones we
# need... so to change that we'll need to add a hook for ourself after
# python is downloaded but before it is built so we can muck with it.
ftxt = replace_one(
ftxt, ' def build(self):',
' def build(self):\n import os\n'
' if os.system(\'"' + rootdir +
'/tools/pcommand" python_android_patch "' + os.getcwd() +
'"\') != 0: raise Exception("patch apply failed")')
writefile('pybuild/packages/python.py', ftxt)
# Set these to particular releases to use those.
# py_commit = '580fbb018fd0844806119614d752b41fc69660f9' # 3.8.5
py_commit = '6503f05dd59e26a9986bdea097b3da9b3546f45b' # 3.8.7
# cpython-source-deps stuff started failing for OpenSSL on Jan 8 2021.
# Pinning it to an older one for now.
py_ssl_commit = '7f34c3085feb4692bbbb6c8b19d053ebc5049dad' # From 6/12/20
commit_lines = ''
if py_commit is not None:
commit_lines += (' if self.source_url == '
"'https://github.com/python/cpython/':\n"
" run_in_dir(['git', 'checkout', '" +
py_commit + "'], self.source_dir)\n")
if py_ssl_commit is not None:
commit_lines += (' if self.source_url == '
"'https://github.com/python/cpython-source-deps'"
" and self.branch == 'openssl-1.1.1':\n"
" run_in_dir(['git', 'checkout', '" +
py_ssl_commit + "'], self.source_dir)\n")
ftxt = readfile('pybuild/source.py')
# Check out a particular commit right after the clone.
ftxt = replace_one(
ftxt, "'git', 'clone', '--single-branch', '-b',"
' self.branch, self.source_url, self.dest])', "'git', 'clone', '-b',"
' self.branch, self.source_url, self.dest])\n'
' # efro: hack to get exact commits we want.\n'
" print('DOING URL', self.source_url)\n" + commit_lines)
writefile('pybuild/source.py', ftxt)
ftxt = readfile('pybuild/util.py')
# Still don't wanna bother with gpg signing stuff.
ftxt = replace_one(
ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n',
'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n'
' print("gpg-verify disabled by ericf")\n'
' return\n')
writefile('pybuild/util.py', ftxt)
# These builds require ANDROID_NDK to be set, so make sure that's
# the case.
os.environ['ANDROID_NDK'] = subprocess.check_output(
[rootdir + '/tools/android_sdk_utils',
'get-ndk-path']).decode().strip()
# Ok, let 'er rip
# (we often run these builds in parallel so limit to 1 job a piece;
# otherwise they each inherit the -j12 or whatever from the top level).
run('make -j1')
print('python build complete! (android/' + arch + ')')
