def stage_server_file(projroot: str, mode: str, infilename: str,
outfilename: str) -> None:
"""Stage files for the server environment with some filtering."""
import batools.build
from efrotools import replace_exact
if mode not in ('debug', 'release'):
raise RuntimeError(f"Invalid server-file-staging mode '{mode}';"
f" expected 'debug' or 'release'.")
print(f'Building server file: {os.path.basename(outfilename)}')
os.makedirs(os.path.dirname(outfilename), exist_ok=True)
basename = os.path.basename(infilename)
if basename == 'config_template.yaml':
# Inject all available config values into the config file.
_write_if_changed(
outfilename,
batools.build.filter_server_config(str(projroot), infilename))
elif basename == 'ballisticacore_server.py':
# Run Python in opt mode for release builds.
with open(infilename, encoding='utf-8') as infile:
lines = infile.read().splitlines()
if mode == 'release':
lines[0] = replace_exact(
lines[0], f'#!/usr/bin/env python{PYVER}',
f'#!/usr/bin/env -S python{PYVER} -O')
_write_if_changed(outfilename,
'\n'.join(lines) + '\n',
make_executable=True)
elif basename == 'README.txt':
with open(infilename, encoding='utf-8') as infile:
readme = infile.read()
_write_if_changed(outfilename, readme)
elif basename == 'launch_ballisticacore_server.bat':
# Run Python in opt mode for release builds.
with open(infilename, encoding='utf-8') as infile:
lines = infile.read().splitlines()
if mode == 'release':
lines[1] = replace_exact(
lines[1], ':: Python interpreter.', ':: Python interpreter.'
' (in opt mode so we use bundled .opt-1.pyc files)')
lines[2] = replace_exact(
lines[2], 'dist\\\\python.exe ballisticacore_server.py',
'dist\\\\python.exe -O ballisticacore_server.py')
else:
# In debug mode we use the bundled debug interpreter.
lines[2] = replace_exact(
lines[2], 'dist\\\\python.exe ballisticacore_server.py',
'dist\\\\python_d.exe ballisticacore_server.py')
_write_if_changed(outfilename, '\n'.join(lines) + '\n')
else:
raise RuntimeError(f"Unknown server file for staging: '{basename}'.")
def android_patch_ssl() -> None:
"""Run necessary patches on an android ssl before building."""
# We bundle our own SSL root certificates on various platforms and use
# the OpenSSL 'SSL_CERT_FILE' env var override to get them to be used
# by default. However, OpenSSL is picky about allowing env-vars to be
# used and something about the Android environment makes it disallow
# them. So we need to force the issue. Alternately we could explicitly
# pass 'cafile' args to SSLContexts whenever we do network-y stuff
# but it seems cleaner to just have things work by default.
fname = 'crypto/getenv.c'
txt = readfile(fname)
txt = replace_exact(
txt,
('char *ossl_safe_getenv(const char *name)\n'
'{\n'),
('char *ossl_safe_getenv(const char *name)\n'
'{\n'
' // ERICF TWEAK: ALWAYS ALLOW GETENV.\n'
' return getenv(name);\n'),
)
writefile(fname, txt)
def gather() -> None:
"""Gather per-platform python headers, libs, and modules together.
This assumes all embeddable py builds have been run successfully,
and that PROJROOT is the cwd.
"""
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
do_android = True
# First off, clear out any existing output.
existing_dirs = [
os.path.join('src/external', d) for d in os.listdir('src/external')
if d.startswith('python-') and d != 'python-notes.txt'
]
existing_dirs += [
os.path.join('assets/src', d) for d in os.listdir('assets/src')
if d.startswith('pylib-')
]
if not do_android:
existing_dirs = [d for d in existing_dirs if 'android' not in d]
for existing_dir in existing_dirs:
subprocess.run(['rm', '-rf', existing_dir], check=True)
apost2 = f'src/Python-{PY_VER_EXACT_ANDROID}/Android/sysroot'
for buildtype in ['debug', 'release']:
debug = buildtype == 'debug'
bsuffix = '_debug' if buildtype == 'debug' else ''
bsuffix2 = '-debug' if buildtype == 'debug' else ''
libname = 'python' + PY_VER + ('d' if debug else '')
bases = {
'mac': f'build/python_apple_mac{bsuffix}/build/macOS',
'ios': f'build/python_apple_ios{bsuffix}/build/iOS',
'tvos': f'build/python_apple_tvos{bsuffix}/build/tvOS',
'android_arm': f'build/python_android_arm{bsuffix}/build',
'android_arm64': f'build/python_android_arm64{bsuffix}/build',
'android_x86': f'build/python_android_x86{bsuffix}/build',
'android_x86_64': f'build/python_android_x86_64{bsuffix}/build'
}
bases2 = {
'android_arm': f'build/python_android_arm{bsuffix}/{apost2}',
'android_arm64': f'build/python_android_arm64{bsuffix}/{apost2}',
'android_x86': f'build/python_android_x86{bsuffix}/{apost2}',
'android_x86_64': f'build/python_android_x86_64{bsuffix}/{apost2}'
}
# Note: only need pylib for the first in each group.
mac_arch_dir = 'macos-arm64_x86_64'
ios_arch_dir = 'ios-arm64'
tvos_arch_dir = 'tvos-arm64'
builds: list[dict[str, Any]] = [{
'name':
'macos',
'group':
'apple',
'headers':
bases['mac'] +
f'/Support/Python.xcframework/{mac_arch_dir}/Headers',
'libs': [
bases['mac'] +
f'/Support/Python.xcframework/{mac_arch_dir}/libPython.a',
bases['mac'] +
f'/Support/OpenSSL.xcframework/{mac_arch_dir}/libOpenSSL.a',
bases['mac'] +
f'/Support/XZ.xcframework/{mac_arch_dir}/libxz.a',
bases['mac'] +
f'/Support/BZip2.xcframework/{mac_arch_dir}/libbzip2.a',
],
'pylib':
(bases['mac'] + f'/Python-{PY_VER_EXACT_APPLE}-macOS/lib'),
}, {
'name':
'ios',
'group':
'apple',
'headers':
bases['ios'] +
f'/Support/Python.xcframework/{ios_arch_dir}/Headers',
'libs': [
bases['ios'] +
f'/Support/Python.xcframework/{ios_arch_dir}/libPython.a',
bases['ios'] +
f'/Support/OpenSSL.xcframework/{ios_arch_dir}/libOpenSSL.a',
bases['ios'] +
f'/Support/XZ.xcframework/{ios_arch_dir}/libxz.a',
bases['ios'] +
f'/Support/BZip2.xcframework/{ios_arch_dir}/libbzip2.a',
],
}, {
'name':
'tvos',
'group':
'apple',
'headers':
bases['tvos'] +
f'/Support/Python.xcframework/{tvos_arch_dir}/Headers',
'libs': [
bases['tvos'] +
f'/Support/Python.xcframework/{tvos_arch_dir}/libPython.a',
bases['tvos'] +
f'/Support/OpenSSL.xcframework/{tvos_arch_dir}/libOpenSSL.a',
bases['tvos'] +
f'/Support/XZ.xcframework/{tvos_arch_dir}/libxz.a',
bases['tvos'] +
f'/Support/BZip2.xcframework/{tvos_arch_dir}/libbzip2.a',
],
}, {
'name': 'android_arm',
'group': 'android',
'headers': bases['android_arm'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm'] + f'/usr/lib/lib{libname}.a',
bases2['android_arm'] + '/usr/lib/libssl.a',
bases2['android_arm'] + '/usr/lib/libcrypto.a',
bases2['android_arm'] + '/usr/lib/liblzma.a',
bases2['android_arm'] + '/usr/lib/libsqlite3.a',
bases2['android_arm'] + '/usr/lib/libbz2.a',
bases2['android_arm'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_armeabi-v7a',
'pylib': (bases['android_arm'] + '/usr/lib/python' + PY_VER),
}, {
'name': 'android_arm64',
'group': 'android',
'headers': bases['android_arm64'] + f'/usr/include/{libname}',
'libs': [
bases['android_arm64'] + f'/usr/lib/lib{libname}.a',
bases2['android_arm64'] + '/usr/lib/libssl.a',
bases2['android_arm64'] + '/usr/lib/libcrypto.a',
bases2['android_arm64'] + '/usr/lib/liblzma.a',
bases2['android_arm64'] + '/usr/lib/libsqlite3.a',
bases2['android_arm64'] + '/usr/lib/libbz2.a',
bases2['android_arm64'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_arm64-v8a',
}, {
'name': 'android_x86',
'group': 'android',
'headers': bases['android_x86'] + f'/usr/include/{libname}',
'libs': [
bases['android_x86'] + f'/usr/lib/lib{libname}.a',
bases2['android_x86'] + '/usr/lib/libssl.a',
bases2['android_x86'] + '/usr/lib/libcrypto.a',
bases2['android_x86'] + '/usr/lib/liblzma.a',
bases2['android_x86'] + '/usr/lib/libsqlite3.a',
bases2['android_x86'] + '/usr/lib/libbz2.a',
bases2['android_x86'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_x86',
}, {
'name': 'android_x86_64',
'group': 'android',
'headers': bases['android_x86_64'] + f'/usr/include/{libname}',
'libs': [
bases['android_x86_64'] + f'/usr/lib/lib{libname}.a',
bases2['android_x86_64'] + '/usr/lib/libssl.a',
bases2['android_x86_64'] + '/usr/lib/libcrypto.a',
bases2['android_x86_64'] + '/usr/lib/liblzma.a',
bases2['android_x86_64'] + '/usr/lib/libsqlite3.a',
bases2['android_x86_64'] + '/usr/lib/libbz2.a',
bases2['android_x86_64'] + '/usr/lib/libuuid.a',
],
'libinst': 'android_x86_64',
}]
for build in builds:
grp = build['group']
if not do_android and grp == 'android':
continue
builddir = f'src/external/python-{grp}{bsuffix2}'
header_dst = os.path.join(builddir, 'include')
lib_dst = os.path.join(builddir, 'lib')
assets_src_dst = f'assets/src/pylib-{grp}'
# Do some setup only once per group.
if not os.path.exists(builddir):
subprocess.run(['mkdir', '-p', builddir], check=True)
subprocess.run(['mkdir', '-p', lib_dst], check=True)
# Only pull modules into game assets on release pass.
if not debug:
# Copy system modules into the src assets
# dir for this group.
subprocess.run(['mkdir', '-p', assets_src_dst], check=True)
subprocess.run(
[
'rsync', '--recursive', '--include', '*.py',
'--exclude', '__pycache__', '--include', '*/',
'--exclude', '*', build['pylib'] + '/',
assets_src_dst
],
check=True,
)
# Prune a bunch of modules we don't need to cut
# down on size.
subprocess.run('cd "' + assets_src_dst + '" && rm -rf ' +
' '.join(PRUNE_LIB_NAMES),
shell=True,
check=True)
# Some minor filtering to system scripts:
# on iOS/tvOS, addusersitepackages() leads to a crash
# due to _sysconfigdata_dm_ios_darwin module not existing,
# so let's skip that.
fname = f'{assets_src_dst}/site.py'
txt = readfile(fname)
txt = replace_exact(
txt,
' known_paths = addusersitepackages(known_paths)',
' # efro tweak: this craps out on ios/tvos.\n'
' # (and we don\'t use it anyway)\n'
' # known_paths = addusersitepackages(known_paths)')
writefile(fname, txt)
# Copy in a base set of headers (everything in a group should
# be using the same headers)
subprocess.run(f'cp -r "{build["headers"]}" "{header_dst}"',
shell=True,
check=True)
# Clear whatever pyconfigs came across; we'll build our own
# universal one below.
subprocess.run('rm ' + header_dst + '/pyconfig*',
shell=True,
check=True)
# Write a master pyconfig header that reroutes to each
# platform's actual header.
with open(header_dst + '/pyconfig.h', 'w',
encoding='utf-8') as hfile:
hfile.write(
'#if BA_OSTYPE_MACOS\n'
'#include "pyconfig-macos.h"\n\n'
'#elif BA_OSTYPE_IOS\n'
'#include "pyconfig-ios.h"\n\n'
'#elif BA_OSTYPE_TVOS\n'
'#include "pyconfig-tvos.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__arm__)\n'
'#include "pyconfig-android_arm.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__aarch64__)\n'
'#include "pyconfig-android_arm64.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__i386__)\n'
'#include "pyconfig-android_x86.h"\n\n'
'#elif BA_OSTYPE_ANDROID and defined(__x86_64__)\n'
'#include "pyconfig-android_x86_64.h"\n\n'
'#else\n'
'#error unknown platform\n\n'
'#endif\n')
# Now copy each build's config headers in with unique names.
cfgs = [
f for f in os.listdir(build['headers'])
if f.startswith('pyconfig')
]
# Copy config headers to their filtered names.
for cfg in cfgs:
out = cfg.replace('pyconfig', 'pyconfig-' + build['name'])
if cfg == 'pyconfig.h':
# For platform's root pyconfig.h we need to filter
# contents too (those headers can themselves include
# others; ios for instance points to a arm64 and a
# x86_64 variant).
contents = readfile(build['headers'] + '/' + cfg)
contents = contents.replace('pyconfig',
'pyconfig-' + build['name'])
writefile(header_dst + '/' + out, contents)
else:
# other configs we just rename
subprocess.run('cp "' + build['headers'] + '/' + cfg +
'" "' + header_dst + '/' + out + '"',
shell=True,
check=True)
# Copy in libs. If the lib gave a specific install name,
# use that; otherwise use name.
targetdir = lib_dst + '/' + build.get('libinst', build['name'])
subprocess.run(['rm', '-rf', targetdir], check=True)
subprocess.run(['mkdir', '-p', targetdir], check=True)
for lib in build['libs']:
subprocess.run(['cp', lib, targetdir], check=True)
print('Great success!')
def build_apple(arch: str, debug: bool = False) -> None:
"""Run a build for the provided apple arch (mac, ios, or tvos)."""
import platform
from efro.error import CleanError
# IMPORTANT; seems we currently wind up building against /usr/local gettext
# stuff. Hopefully the maintainer fixes this, but for now I need to
# remind myself to blow it away while building.
# (via brew remove gettext --ignore-dependencies)
# NOTE: Should check to see if this is still necessary on Apple silicon
# since homebrew stuff is no longer in /usr/local there.
if ('MacBook-Fro' in platform.node()
and os.environ.get('SKIP_GETTEXT_WARNING') != '1'):
if (subprocess.run('which gettext', shell=True,
check=False).returncode == 0):
raise CleanError(
'NEED TO TEMP-KILL GETTEXT (or set SKIP_GETTEXT_WARNING=1)')
builddir = 'build/python_apple_' + arch + ('_debug' if debug else '')
subprocess.run(['rm', '-rf', builddir], check=True)
subprocess.run(['mkdir', '-p', 'build'], check=True)
subprocess.run(
[
'git', 'clone',
'https://github.com/beeware/Python-Apple-support.git', builddir
],
check=True,
)
os.chdir(builddir)
# TEMP: Check out a particular commit while the branch head is broken.
# We can actually fix this to use the current one, but something
# broke in the underlying build even on old commits so keeping it
# locked for now...
# run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6')
subprocess.run(['git', 'checkout', PY_VER], check=True)
txt = readfile('Makefile')
# Fix a bug where spaces in PATH cause errors (darn you vmware fusion!)
txt = replace_exact(
txt,
'\t\tPATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/_install/bin:$(PATH)',
'\t\tPATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/_install/bin:$(PATH)"')
# Turn doc strings on; looks like it only adds a few hundred k.
txt = replace_exact(txt,
'--without-doc-strings',
'--with-doc-strings',
count=2)
# Customize our minimum version requirements
txt = replace_exact(
txt,
'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
'CFLAGS-macOS=-mmacosx-version-min=10.15\n',
)
txt = replace_exact(
txt,
'CFLAGS-iOS=-mios-version-min=12.0 ',
'CFLAGS-iOS=-mios-version-min=12.0 ',
)
txt = replace_exact(
txt,
'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
'CFLAGS-tvOS=-mtvos-version-min=9.0 ',
)
assert '--with-pydebug' not in txt
if debug:
# Add debug build flag
txt = replace_exact(
txt,
'--enable-ipv6 --without-ensurepip ',
'--enable-ipv6 --with-pydebug --without-ensurepip ',
count=2,
)
# Debug lib has a different name.
txt = replace_exact(txt,
'python$(PYTHON_VER).a',
'python$(PYTHON_VER)d.a',
count=2)
txt = replace_exact(txt,
'/include/python$(PYTHON_VER)',
'/include/python$(PYTHON_VER)d',
count=4)
# Inject our custom modifications to fire right after their normal
# Setup.local filtering and right before building (and pass the same
# 'slice' value they use so we can use it too).
txt = replace_exact(
txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n',
'\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n'
'\tcd $$(PYTHON_DIR-$(target)) && '
f'../../../../../tools/pcommand python_apple_patch {arch} '
'"$$(SLICE-$$(SDK-$(target)))"\n')
txt = replace_exact(
txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n',
'\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n'
'\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n'
'\tcd $$(PYTHON_DIR-$(os)) && '
f'../../../../../tools/pcommand python_apple_patch {arch} '
'"$$(SLICE-macosx)"\n')
writefile('Makefile', txt)
# Ok; let 'er rip.
# (we run these in parallel so limit to 1 job a piece;
# otherwise they inherit the -j12 or whatever from the top level)
# (also this build seems to fail with multiple threads)
subprocess.run(
[
'make', '-j1', {
'mac': 'Python-macOS',
'ios': 'Python-iOS',
'tvos': 'Python-tvOS'
}[arch]
],
check=True,
)
print('python build complete! (apple/' + arch + ')')
def _patch_setup_file(platform: str, arch: str, slc: str) -> None:
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
fname = 'Modules/Setup'
ftxt = readfile(fname)
if platform == 'android':
prefix = '$(srcdir)/Android/sysroot/usr'
uuid_ex = f' -L{prefix}/lib -luuid'
zlib_ex = f' -I{prefix}/include -L{prefix}/lib -lz'
bz2_ex = f' -I{prefix}/include -L{prefix}/lib -lbz2'
ssl_ex = f' -DUSE_SSL -I{prefix}/include -L{prefix}/lib -lssl -lcrypto'
sqlite_ex = f' -I{prefix}/include -L{prefix}/lib'
hash_ex = ' -DUSE_SSL -lssl -lcrypto'
lzma_ex = ' -llzma'
elif platform == 'apple':
# These should basically match what the Python-Apple-support dist
# does in its patch/Python/Setup.embedded.
# (We do our own thing and ignore the Setup.local it generates
# for the sake of cross-platform consistency, but still need to
# do what we do the same way they do what they do)
def _slrp(val: str) -> str:
# In the distro, the Makefile does this filtering to Setup.local
return val.replace('{{slice}}', slc)
uuid_ex = ''
zlib_ex = ' -I$(prefix)/include -lz'
bz2_ex = _slrp(
' -I$(srcdir)/../Support/BZip2.xcframework/{{slice}}/Headers'
' -L$(srcdir)/../Support/BZip2.xcframework/{{slice}} -lbzip2')
ssl_ex = _slrp(
' -I$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}/Headers'
' -L$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}'
' -lOpenSSL -DUSE_SSL')
sqlite_ex = ' -I$(srcdir)/Modules/_sqlite'
hash_ex = _slrp(
' -I$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}/Headers'
' -L$(srcdir)/../Support/OpenSSL.xcframework/{{slice}}'
' -lOpenSSL -DUSE_SSL')
lzma_ex = _slrp(
' -I$(srcdir)/../Support/XZ.xcframework/{{slice}}/Headers'
' -L$(srcdir)/../Support/XZ.xcframework/{{slice}} -lxz')
else:
raise RuntimeError(f'Unknown platform {platform}')
# This list should contain all possible compiled modules to start.
# If any .so files are coming out of builds, their names should be
# added here to stop that.
cmodules = [
'_asyncio', '_bisect', '_blake2', '_codecs_cn', '_codecs_hk',
'_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw',
'_contextvars', '_crypt', '_csv', '_ctypes_test', '_ctypes',
'_curses_panel', '_curses', '_datetime', '_decimal', '_elementtree',
'_heapq', '_json', '_lsprof', '_lzma', '_md5', '_multibytecodec',
'_multiprocessing', '_opcode', '_pickle', '_posixsubprocess', '_queue',
'_random', '_sha1', '_sha3', '_sha256', '_sha512', '_socket',
'_statistics', '_struct', '_testbuffer', '_testcapi',
'_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_uuid',
'_xxsubinterpreters', '_xxtestfuzz', '_zoneinfo', 'array', 'audioop',
'binascii', 'cmath', 'fcntl', 'grp', 'math', 'mmap', 'ossaudiodev',
'parser', 'pyexpat', 'resource', 'select', 'syslog', 'termios',
'unicodedata', 'xxlimited', 'zlib'
]
# Selectively uncomment some existing modules for static compilation.
enables = [
'_asyncio', 'array', 'cmath', 'math', '_contextvars', '_struct',
'_random', '_elementtree', '_pickle', '_datetime', '_zoneinfo',
'_bisect', '_heapq', '_json', '_statistics', 'unicodedata', 'fcntl',
'select', 'mmap', '_csv', '_socket', '_sha3', '_blake2', 'binascii',
'_posixsubprocess'
]
# Note that the _md5 and _sha modules are normally only built if the
# system does not have the OpenSSL libs containing an optimized
# version.
if bool(False):
enables += ['_md5']
for enable in enables:
ftxt = replace_exact(ftxt, f'#{enable} ', f'{enable} ')
cmodules.remove(enable)
# Disable ones that were enabled:
disables = ['xxsubtype']
for disable in disables:
ftxt = replace_exact(ftxt, f'\n{disable} ', f'\n#{disable} ')
# Additions:
ftxt += '\n# Additions by efrotools:\n'
if bool(True):
ftxt += f'_uuid _uuidmodule.c{uuid_ex}\n'
cmodules.remove('_uuid')
ftxt += f'zlib zlibmodule.c{zlib_ex}\n'
cmodules.remove('zlib')
# Why isn't this getting built as a shared lib by default?
# Do we need it for sure?
ftxt += f'_hashlib _hashopenssl.c{hash_ex}\n'
ftxt += f'_lzma _lzmamodule.c{lzma_ex}\n'
cmodules.remove('_lzma')
ftxt += f'_bz2 _bz2module.c{bz2_ex}\n'
ftxt += f'_ssl _ssl.c{ssl_ex}\n'
ftxt += (f'_sqlite3'
f' _sqlite/cache.c'
f' _sqlite/connection.c'
f' _sqlite/cursor.c'
f' _sqlite/microprotocols.c'
f' _sqlite/module.c'
f' _sqlite/prepare_protocol.c'
f' _sqlite/row.c'
f' _sqlite/statement.c'
f' _sqlite/util.c'
f'{sqlite_ex}'
f' -DMODULE_NAME=\'\\"sqlite3\\"\''
f' -DSQLITE_OMIT_LOAD_EXTENSION'
f' -lsqlite3\n')
# Mac needs this:
if arch == 'mac':
ftxt += ('\n'
'# efrotools: mac urllib needs this:\n'
'_scproxy _scproxy.c '
'-framework SystemConfiguration '
'-framework CoreFoundation\n')
# Explicitly mark the remaining ones as disabled
# (so Python won't try to build them as dynamic libs).
remaining_disabled = ' '.join(cmodules)
ftxt += ('\n# Disabled by efrotools build:\n'
'*disabled*\n'
f'{remaining_disabled}\n')
writefile(fname, ftxt)
# Ok, this is weird.
# When applying the module Setup, python looks for any line containing *=*
# and interprets the whole thing a a global define?...
# This breaks things for our static sqlite compile above.
# The check used to look for [A-Z]*=* which didn't break, so let' just
# change it back to that for now.
# UPDATE: Currently this seems to only be necessary on Android;
# perhaps this broke between 3.9.6 and 3.9.7 or perhaps the apple
# bundle already patches it ¯\_(ツ)_/¯
fname = 'Modules/makesetup'
txt = readfile(fname)
if platform == 'android':
txt = replace_exact(txt, ' *=*)'
' DEFS="$line$NL$DEFS"; continue;;',
' [A-Z]*=*) DEFS="$line$NL$DEFS";'
' continue;;')
assert txt.count('[A-Z]*=*') == 1
writefile(fname, txt)
def build_android(rootdir: str, arch: str, debug: bool = False) -> None:
"""Run a build for android with the given architecture.
(can be arm, arm64, x86, or x86_64)
"""
builddir = 'build/python_android_' + arch + ('_debug' if debug else '')
subprocess.run(['rm', '-rf', builddir], check=True)
subprocess.run(['mkdir', '-p', 'build'], check=True)
subprocess.run(
['git', 'clone', ANDROID_PYTHON_REPO, builddir],
check=True,
)
os.chdir(builddir)
# TEMP - use 3.9.6 branch
# subprocess.run(['git', 'checkout', PY_VER_EXACT_ANDROID], check=True)
# These builds require ANDROID_NDK to be set; make sure that's the case.
os.environ['ANDROID_NDK'] = subprocess.check_output(
[f'{rootdir}/tools/pcommand', 'android_sdk_utils',
'get-ndk-path']).decode().strip()
# Disable builds for dependencies we don't use.
ftxt = readfile('Android/build_deps.py')
# ftxt = replace_exact(ftxt, ' NCurses,\n',
# '# NCurses,\n',)
ftxt = replace_exact(
ftxt,
' '
'BZip2, GDBM, LibFFI, LibUUID, OpenSSL, Readline, SQLite, XZ, ZLib,\n',
' '
'BZip2, LibUUID, OpenSSL, SQLite, XZ, ZLib,\n',
)
# Older ssl seems to choke on newer ndk layouts.
if bool(False):
ftxt = replace_exact(
ftxt,
"source = 'https://www.openssl.org/source/openssl-1.1.1h.tar.gz'",
"source = 'https://www.openssl.org/source/openssl-1.1.1l.tar.gz'")
# Give ourselves a handle to patch the OpenSSL build.
ftxt = replace_exact(
ftxt,
' # OpenSSL handles NDK internal paths by itself',
' # Ericf addition: do some patching:\n'
' self.run(["../../../../../../../tools/pcommand",'
' "python_android_patch_ssl"])\n'
' # OpenSSL handles NDK internal paths by itself',
)
writefile('Android/build_deps.py', ftxt)
# Tweak some things in the base build script; grab the right version
# of Python and also inject some code to modify bits of python
# after it is extracted.
ftxt = readfile('build.sh')
ftxt = replace_exact(ftxt, 'PYVER=3.10.4', f'PYVER={PY_VER_EXACT_ANDROID}')
ftxt = replace_exact(
ftxt, ' popd\n', f' ../../../tools/pcommand'
f' python_android_patch Python-{PY_VER_EXACT_ANDROID}\n popd\n')
writefile('build.sh', ftxt)
# Ok, let 'er rip
exargs = ' --with-pydebug' if debug else ''
subprocess.run(f'ARCH={arch} ANDROID_API=21 ./build.sh{exargs}',
shell=True,
check=True)
print('python build complete! (android/' + arch + ')')