def android_patch_ssl() -> None: """Run necessary patches on an android ssl before building.""" # We bundle our own SSL root certificates on various platforms and use # the OpenSSL 'SSL_CERT_FILE' env var override to get them to be used # by default. However, OpenSSL is picky about allowing env-vars to be # used and something about the Android environment makes it disallow # them. So we need to force the issue. Alternately we could explicitly # pass 'cafile' args to SSLContexts whenever we do network-y stuff # but it seems cleaner to just have things work by default. fname = 'crypto/getenv.c' txt = readfile(fname) txt = replace_one( txt, ('char *ossl_safe_getenv(const char *name)\n' '{\n'), ('char *ossl_safe_getenv(const char *name)\n' '{\n' ' // ERICF TWEAK: ALWAYS ALLOW GETENV.\n' ' return getenv(name);\n'), ) writefile(fname, txt)
def gather() -> None: """Gather per-platform python headers, libs, and modules together. This assumes all embeddable py builds have been run successfully, and that PROJROOT is the cwd. """ # pylint: disable=too-many-locals do_android = True # First off, clear out any existing output. existing_dirs = [ os.path.join('src/external', d) for d in os.listdir('src/external') if d.startswith('python-') and d != 'python-notes.txt' ] existing_dirs += [ os.path.join('assets/src', d) for d in os.listdir('assets/src') if d.startswith('pylib-') ] if not do_android: existing_dirs = [d for d in existing_dirs if 'android' not in d] for existing_dir in existing_dirs: subprocess.run(['rm', '-rf', existing_dir], check=True) apost2 = f'src/Python-{PY_VER_EXACT_ANDROID}/Android/sysroot' for buildtype in ['debug', 'release']: debug = buildtype == 'debug' bsuffix = '_debug' if buildtype == 'debug' else '' bsuffix2 = '-debug' if buildtype == 'debug' else '' libname = 'python' + PYVER + ('d' if debug else '') bases = { 'mac': f'build/python_apple_mac{bsuffix}/build/macOS', 'ios': f'build/python_apple_ios{bsuffix}/build/iOS', 'tvos': f'build/python_apple_tvos{bsuffix}/build/tvOS', 'android_arm': f'build/python_android_arm{bsuffix}/build', 'android_arm64': f'build/python_android_arm64{bsuffix}/build', 'android_x86': f'build/python_android_x86{bsuffix}/build', 'android_x86_64': f'build/python_android_x86_64{bsuffix}/build' } bases2 = { 'android_arm': f'build/python_android_arm{bsuffix}/{apost2}', 'android_arm64': f'build/python_android_arm64{bsuffix}/{apost2}', 'android_x86': f'build/python_android_x86{bsuffix}/{apost2}', 'android_x86_64': f'build/python_android_x86_64{bsuffix}/{apost2}' } # Note: only need pylib for the first in each group. builds: list[dict[str, Any]] = [{ 'name': 'macos', 'group': 'apple', 'headers': bases['mac'] + '/Support/Python/Headers', 'libs': [ bases['mac'] + '/Support/Python/libPython.a', bases['mac'] + '/Support/OpenSSL/libOpenSSL.a', bases['mac'] + '/Support/XZ/libxz.a', bases['mac'] + '/Support/BZip2/libbzip2.a', ], 'pylib': (bases['mac'] + f'/Python-{PY_VER_EXACT_APPLE}-macOS/lib'), }, { 'name': 'ios', 'group': 'apple', 'headers': bases['ios'] + '/Support/Python/Headers', 'libs': [ bases['ios'] + '/Support/Python/libPython.a', bases['ios'] + '/Support/OpenSSL/libOpenSSL.a', bases['ios'] + '/Support/XZ/libxz.a', bases['ios'] + '/Support/BZip2/libbzip2.a', ], }, { 'name': 'tvos', 'group': 'apple', 'headers': bases['tvos'] + '/Support/Python/Headers', 'libs': [ bases['tvos'] + '/Support/Python/libPython.a', bases['tvos'] + '/Support/OpenSSL/libOpenSSL.a', bases['tvos'] + '/Support/XZ/libxz.a', bases['tvos'] + '/Support/BZip2/libbzip2.a', ], }, { 'name': 'android_arm', 'group': 'android', 'headers': bases['android_arm'] + f'/usr/include/{libname}', 'libs': [ bases['android_arm'] + f'/usr/lib/lib{libname}.a', bases2['android_arm'] + '/usr/lib/libssl.a', bases2['android_arm'] + '/usr/lib/libcrypto.a', bases2['android_arm'] + '/usr/lib/liblzma.a', bases2['android_arm'] + '/usr/lib/libsqlite3.a', bases2['android_arm'] + '/usr/lib/libbz2.a', bases2['android_arm'] + '/usr/lib/libuuid.a', ], 'libinst': 'android_armeabi-v7a', 'pylib': (bases['android_arm'] + '/usr/lib/python' + PYVER), }, { 'name': 'android_arm64', 'group': 'android', 'headers': bases['android_arm64'] + f'/usr/include/{libname}', 'libs': [ bases['android_arm64'] + f'/usr/lib/lib{libname}.a', bases2['android_arm64'] + '/usr/lib/libssl.a', bases2['android_arm64'] + '/usr/lib/libcrypto.a', bases2['android_arm64'] + '/usr/lib/liblzma.a', bases2['android_arm64'] + '/usr/lib/libsqlite3.a', bases2['android_arm64'] + '/usr/lib/libbz2.a', bases2['android_arm64'] + '/usr/lib/libuuid.a', ], 'libinst': 'android_arm64-v8a', }, { 'name': 'android_x86', 'group': 'android', 'headers': bases['android_x86'] + f'/usr/include/{libname}', 'libs': [ bases['android_x86'] + f'/usr/lib/lib{libname}.a', bases2['android_x86'] + '/usr/lib/libssl.a', bases2['android_x86'] + '/usr/lib/libcrypto.a', bases2['android_x86'] + '/usr/lib/liblzma.a', bases2['android_x86'] + '/usr/lib/libsqlite3.a', bases2['android_x86'] + '/usr/lib/libbz2.a', bases2['android_x86'] + '/usr/lib/libuuid.a', ], 'libinst': 'android_x86', }, { 'name': 'android_x86_64', 'group': 'android', 'headers': bases['android_x86_64'] + f'/usr/include/{libname}', 'libs': [ bases['android_x86_64'] + f'/usr/lib/lib{libname}.a', bases2['android_x86_64'] + '/usr/lib/libssl.a', bases2['android_x86_64'] + '/usr/lib/libcrypto.a', bases2['android_x86_64'] + '/usr/lib/liblzma.a', bases2['android_x86_64'] + '/usr/lib/libsqlite3.a', bases2['android_x86_64'] + '/usr/lib/libbz2.a', bases2['android_x86_64'] + '/usr/lib/libuuid.a', ], 'libinst': 'android_x86_64', }] for build in builds: grp = build['group'] if not do_android and grp == 'android': continue builddir = f'src/external/python-{grp}{bsuffix2}' header_dst = os.path.join(builddir, 'include') lib_dst = os.path.join(builddir, 'lib') assets_src_dst = f'assets/src/pylib-{grp}' # Do some setup only once per group. if not os.path.exists(builddir): subprocess.run(['mkdir', '-p', builddir], check=True) subprocess.run(['mkdir', '-p', lib_dst], check=True) # Only pull modules into game assets on release pass. if not debug: # Copy system modules into the src assets # dir for this group. subprocess.run(['mkdir', '-p', assets_src_dst], check=True) subprocess.run( [ 'rsync', '--recursive', '--include', '*.py', '--exclude', '__pycache__', '--include', '*/', '--exclude', '*', build['pylib'] + '/', assets_src_dst ], check=True, ) # Prune a bunch of modules we don't need to cut # down on size. subprocess.run('cd "' + assets_src_dst + '" && rm -rf ' + ' '.join(PRUNE_LIB_NAMES), shell=True, check=True) # Some minor filtering to system scripts: # on iOS/tvOS, addusersitepackages() leads to a crash # due to _sysconfigdata_dm_ios_darwin module not existing, # so let's skip that. fname = f'{assets_src_dst}/site.py' txt = readfile(fname) txt = replace_one( txt, ' known_paths = addusersitepackages(known_paths)', ' # efro tweak: this craps out on ios/tvos.\n' ' # (and we don\'t use it anyway)\n' ' # known_paths = addusersitepackages(known_paths)') writefile(fname, txt) # Copy in a base set of headers (everything in a group should # be using the same headers) subprocess.run(f'cp -r "{build["headers"]}" "{header_dst}"', shell=True, check=True) # Clear whatever pyconfigs came across; we'll build our own # universal one below. subprocess.run('rm ' + header_dst + '/pyconfig*', shell=True, check=True) # Write a master pyconfig header that reroutes to each # platform's actual header. with open(header_dst + '/pyconfig.h', 'w', encoding='utf-8') as hfile: hfile.write( '#if BA_OSTYPE_MACOS\n' '#include "pyconfig-macos.h"\n\n' '#elif BA_OSTYPE_IOS\n' '#include "pyconfig-ios.h"\n\n' '#elif BA_OSTYPE_TVOS\n' '#include "pyconfig-tvos.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__arm__)\n' '#include "pyconfig-android_arm.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__aarch64__)\n' '#include "pyconfig-android_arm64.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__i386__)\n' '#include "pyconfig-android_x86.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__x86_64__)\n' '#include "pyconfig-android_x86_64.h"\n\n' '#else\n' '#error unknown platform\n\n' '#endif\n') # Now copy each build's config headers in with unique names. cfgs = [ f for f in os.listdir(build['headers']) if f.startswith('pyconfig') ] # Copy config headers to their filtered names. for cfg in cfgs: out = cfg.replace('pyconfig', 'pyconfig-' + build['name']) if cfg == 'pyconfig.h': # For platform's root pyconfig.h we need to filter # contents too (those headers can themselves include # others; ios for instance points to a arm64 and a # x86_64 variant). contents = readfile(build['headers'] + '/' + cfg) contents = contents.replace('pyconfig', 'pyconfig-' + build['name']) writefile(header_dst + '/' + out, contents) else: # other configs we just rename subprocess.run('cp "' + build['headers'] + '/' + cfg + '" "' + header_dst + '/' + out + '"', shell=True, check=True) # Copy in libs. If the lib gave a specific install name, # use that; otherwise use name. targetdir = lib_dst + '/' + build.get('libinst', build['name']) subprocess.run(['rm', '-rf', targetdir], check=True) subprocess.run(['mkdir', '-p', targetdir], check=True) for lib in build['libs']: subprocess.run(['cp', lib, targetdir], check=True) print('Great success!')
def build_apple(arch: str, debug: bool = False) -> None: """Run a build for the provided apple arch (mac, ios, or tvos).""" import platform from efro.error import CleanError # IMPORTANT; seems we currently wind up building against /usr/local gettext # stuff. Hopefully the maintainer fixes this, but for now I need to # remind myself to blow it away while building. # (via brew remove gettext --ignore-dependencies) # NOTE: Should check to see if this is still necessary on Apple silicon # since homebrew stuff is no longer in /usr/local there. if ('MacBook-Fro' in platform.node() and os.environ.get('SKIP_GETTEXT_WARNING') != '1'): if (subprocess.run('which gettext', shell=True, check=False).returncode == 0): raise CleanError( 'NEED TO TEMP-KILL GETTEXT (or set SKIP_GETTEXT_WARNING=1)') builddir = 'build/python_apple_' + arch + ('_debug' if debug else '') subprocess.run(['rm', '-rf', builddir], check=True) subprocess.run(['mkdir', '-p', 'build'], check=True) subprocess.run( [ 'git', 'clone', 'https://github.com/beeware/Python-Apple-support.git', builddir ], check=True, ) os.chdir(builddir) # TEMP: Check out a particular commit while the branch head is broken. # We can actually fix this to use the current one, but something # broke in the underlying build even on old commits so keeping it # locked for now... # run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6') subprocess.run(['git', 'checkout', PYVER], check=True) txt = readfile('Makefile') # Fix a bug where spaces in PATH cause errors (darn you vmware fusion!) txt = replace_one( txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .', '&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .') # Turn doc strings on; looks like it only adds a few hundred k. txt = txt.replace('--without-doc-strings', '--with-doc-strings') # Set mac/ios version reqs # (see issue with utimensat and futimens). txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8', 'MACOSX_DEPLOYMENT_TARGET=10.15') # And equivalent iOS (11+). txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0', 'CFLAGS-iOS=-mios-version-min=13.0') # Ditto for tvOS. txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0', 'CFLAGS-tvOS=-mtvos-version-min=13.0') if debug: # Add debug build flag # (Currently expect to find 2 instances of this). dline = '--with-doc-strings --enable-ipv6 --without-ensurepip' splitlen = len(txt.split(dline)) if splitlen != 3: raise Exception('unexpected configure lines') txt = txt.replace(dline, '--with-pydebug ' + dline) # Debug has a different name. # (Currently expect to replace 12 instances of this). dline = ('python$(PYTHON_VER)' if NEWER_PY_TEST else 'python$(PYTHON_VER)m') splitlen = len(txt.split(dline)) if splitlen != 13: raise RuntimeError(f'Unexpected configure line count {splitlen}.') txt = txt.replace( dline, 'python$(PYTHON_VER)d' if NEWER_PY_TEST else 'python$(PYTHON_VER)dm') # Inject our custom modifications to fire before building. txt = txt.replace( ' # Configure target Python\n', ' cd $$(PYTHON_DIR-$1) && ' f'../../../../../tools/pcommand python_apple_patch {arch}\n' ' # Configure target Python\n', ) # Use python3 instead of python for libffi setup script txt = replace_one( txt, 'cd $$(LIBFFI_DIR-$1) && python generate-darwin-source-and-headers.py' " --only-$(shell echo $1 | tr '[:upper:]' '[:lower:]')", 'cd $$(LIBFFI_DIR-$1) && python3 generate-darwin-source-and-headers.py' " --only-$(shell echo $1 | tr '[:upper:]' '[:lower:]')", ) writefile('Makefile', txt) # Ok; let 'er rip. # (we run these in parallel so limit to 1 job a piece; # otherwise they inherit the -j12 or whatever from the top level) # (also this build seems to fail with multiple threads) subprocess.run( [ 'make', '-j1', { 'mac': 'Python-macOS', 'ios': 'Python-iOS', 'tvos': 'Python-tvOS' }[arch] ], check=True, ) print('python build complete! (apple/' + arch + ')')
def build_android(rootdir: str, arch: str, debug: bool = False) -> None: """Run a build for android with the given architecture. (can be arm, arm64, x86, or x86_64) """ builddir = 'build/python_android_' + arch + ('_debug' if debug else '') subprocess.run(['rm', '-rf', builddir], check=True) subprocess.run(['mkdir', '-p', 'build'], check=True) subprocess.run( [ 'git', 'clone', 'https://github.com/yan12125/python3-android.git', builddir ], check=True, ) os.chdir(builddir) # These builds require ANDROID_NDK to be set; make sure that's the case. os.environ['ANDROID_NDK'] = subprocess.check_output( [f'{rootdir}/tools/pcommand', 'android_sdk_utils', 'get-ndk-path']).decode().strip() # Disable builds for dependencies we don't use. ftxt = readfile('Android/build_deps.py') # ftxt = replace_one(ftxt, ' NCurses,\n', # '# NCurses,\n',) ftxt = replace_one( ftxt, ' ' 'BZip2, GDBM, LibFFI, LibUUID, OpenSSL, Readline, SQLite, XZ, ZLib,\n', ' ' 'BZip2, LibUUID, OpenSSL, SQLite, XZ, ZLib,\n', ) # Older ssl seems to choke on newer ndk layouts. ftxt = replace_one( ftxt, "source = 'https://www.openssl.org/source/openssl-1.1.1h.tar.gz'", "source = 'https://www.openssl.org/source/openssl-1.1.1l.tar.gz'") # Give ourselves a handle to patch the OpenSSL build. ftxt = replace_one( ftxt, ' # OpenSSL handles NDK internal paths by itself', ' # Ericf addition: do some patching:\n' ' self.run(["../../../../../../../tools/pcommand",' ' "python_android_patch_ssl"])\n' ' # OpenSSL handles NDK internal paths by itself', ) writefile('Android/build_deps.py', ftxt) # Tweak some things in the base build script; grab the right version # of Python and also inject some code to modify bits of python # after it is extracted. ftxt = readfile('build.sh') ftxt = replace_one(ftxt, 'PYVER=3.9.0', f'PYVER={PY_VER_EXACT_ANDROID}') ftxt = replace_one( ftxt, ' popd\n', f' ../../../tools/pcommand' f' python_android_patch Python-{PY_VER_EXACT_ANDROID}\n popd\n') writefile('build.sh', ftxt) # Ok, let 'er rip exargs = ' --with-pydebug' if debug else '' subprocess.run(f'ARCH={arch} ANDROID_API=21 ./build.sh{exargs}', shell=True, check=True) print('python build complete! (android/' + arch + ')')
def _patch_setup_file(platform: str, arch: str) -> None: # pylint: disable=too-many-locals # pylint: disable=too-many-statements fname = 'Modules/Setup' ftxt = readfile(fname) if platform == 'android': prefix = '$(srcdir)/Android/sysroot/usr' uuid_ex = f' -L{prefix}/lib -luuid' zlib_ex = f' -I{prefix}/include -L{prefix}/lib -lz' bz2_ex = f' -I{prefix}/include -L{prefix}/lib -lbz2' ssl_ex = f' -DUSE_SSL -I{prefix}/include -L{prefix}/lib -lssl -lcrypto' sqlite_ex = f' -I{prefix}/include -L{prefix}/lib' hash_ex = ' -DUSE_SSL -lssl -lcrypto' lzma_ex = ' -llzma' elif platform == 'apple': prefix = '$(srcdir)/Android/sysroot/usr' uuid_ex = '' zlib_ex = ' -I$(prefix)/include -lz' bz2_ex = (' -I$(srcdir)/../Support/BZip2/Headers' ' -L$(srcdir)/../Support/BZip2 -lbzip2') ssl_ex = (' -I$(srcdir)/../Support/OpenSSL/Headers' ' -L$(srcdir)/../Support/OpenSSL -lOpenSSL -DUSE_SSL') sqlite_ex = ' -I$(srcdir)/Modules/_sqlite' hash_ex = (' -I$(srcdir)/../Support/OpenSSL/Headers' ' -L$(srcdir)/../Support/OpenSSL -lOpenSSL -DUSE_SSL') lzma_ex = (' -I$(srcdir)/../Support/XZ/Headers' ' -L$(srcdir)/../Support/XZ/ -lxz') else: raise RuntimeError(f'Unknown platform {platform}') # This list should contain all possible compiled modules to start. # If any .so files are coming out of builds, their names should be # added here to stop that. cmodules = [ '_asyncio', '_bisect', '_blake2', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_contextvars', '_crypt', '_csv', '_ctypes_test', '_ctypes', '_curses_panel', '_curses', '_datetime', '_decimal', '_elementtree', '_heapq', '_json', '_lsprof', '_lzma', '_md5', '_multibytecodec', '_multiprocessing', '_opcode', '_pickle', '_posixsubprocess', '_queue', '_random', '_sha1', '_sha3', '_sha256', '_sha512', '_socket', '_statistics', '_struct', '_testbuffer', '_testcapi', '_testimportmultiple', '_testinternalcapi', '_testmultiphase', '_uuid', '_xxsubinterpreters', '_xxtestfuzz', '_zoneinfo', 'array', 'audioop', 'binascii', 'cmath', 'fcntl', 'grp', 'math', 'mmap', 'ossaudiodev', 'parser', 'pyexpat', 'resource', 'select', 'syslog', 'termios', 'unicodedata', 'xxlimited', 'zlib' ] # Selectively uncomment some existing modules for static compilation. enables = [ '_asyncio', 'array', 'cmath', 'math', '_contextvars', '_struct', '_random', '_elementtree', '_pickle', '_datetime', '_zoneinfo', '_bisect', '_heapq', '_json', '_statistics', 'unicodedata', 'fcntl', 'select', 'mmap', '_csv', '_socket', '_sha3', '_blake2', 'binascii', '_posixsubprocess' ] # Note that the _md5 and _sha modules are normally only built if the # system does not have the OpenSSL libs containing an optimized # version. if bool(False): enables += ['_md5'] for enable in enables: ftxt = replace_one(ftxt, f'#{enable} ', f'{enable} ') cmodules.remove(enable) # Disable ones that were enabled: disables = ['xxsubtype'] for disable in disables: ftxt = replace_one(ftxt, f'\n{disable} ', f'\n#{disable} ') # Additions: ftxt += '\n# Additions by efrotools:\n' if bool(True): ftxt += f'_uuid _uuidmodule.c{uuid_ex}\n' cmodules.remove('_uuid') ftxt += f'zlib zlibmodule.c{zlib_ex}\n' cmodules.remove('zlib') # Why isn't this getting built as a shared lib by default? # Do we need it for sure? ftxt += f'_hashlib _hashopenssl.c{hash_ex}\n' ftxt += f'_lzma _lzmamodule.c{lzma_ex}\n' cmodules.remove('_lzma') ftxt += f'_bz2 _bz2module.c{bz2_ex}\n' ftxt += f'_ssl _ssl.c{ssl_ex}\n' ftxt += (f'_sqlite3' f' _sqlite/cache.c' f' _sqlite/connection.c' f' _sqlite/cursor.c' f' _sqlite/microprotocols.c' f' _sqlite/module.c' f' _sqlite/prepare_protocol.c' f' _sqlite/row.c' f' _sqlite/statement.c' f' _sqlite/util.c' f'{sqlite_ex}' f' -DMODULE_NAME=\'\\"sqlite3\\"\'' f' -DSQLITE_OMIT_LOAD_EXTENSION' f' -lsqlite3\n') # Mac needs this: if arch == 'mac': ftxt += ('\n' '# efrotools: mac urllib needs this:\n' '_scproxy _scproxy.c ' '-framework SystemConfiguration ' '-framework CoreFoundation\n') # Explicitly mark the remaining ones as disabled # (so Python won't try to build them as dynamic libs). remaining_disabled = ' '.join(cmodules) ftxt += ('\n# Disabled by efrotools build:\n' '*disabled*\n' f'{remaining_disabled}\n') writefile(fname, ftxt) # Ok, this is weird. # When applying the module Setup, python looks for any line containing *=* # and interprets the whole thing a a global define?... # This breaks things for our static sqlite compile above. # The check used to look for [A-Z]*=* which didn't break, so let' just # change it back to that for now. # UPDATE: Currently this seems to only be necessary on Android; # perhaps this broke between 3.9.6 and 3.9.7 or perhaps the apple # bundle already patches it ¯\_(ツ)_/¯ fname = 'Modules/makesetup' txt = readfile(fname) if platform == 'android': txt = replace_one(txt, ' *=*)' ' DEFS="$line$NL$DEFS"; continue;;', ' [A-Z]*=*) DEFS="$line$NL$DEFS";' ' continue;;') assert txt.count('[A-Z]*=*') == 1 writefile(fname, txt)
def build_apple(arch: str, debug: bool = False) -> None: """Run a build for the provided apple arch (mac, ios, or tvos).""" builddir = 'build/python_apple_' + arch + ('_debug' if debug else '') efrotools.run('rm -rf "' + builddir + '"') efrotools.run('mkdir -p build') efrotools.run('git clone ' '[email protected]:pybee/Python-Apple-support.git "' + builddir + '"') os.chdir(builddir) # TEMP: Check out a particular commit while the branch head is broken. # efrotools.run('git checkout 1a9c71dca298c03517e8236b81cf1d9c8c521cbf') efrotools.run(f'git checkout {PYTHON_VERSION_MAJOR}') # On mac we currently have to add the _scproxy module or urllib will # fail. txt = efrotools.readfile('patch/Python/Setup.embedded') if arch == 'mac': txt += ('\n' '# ericf added - mac urllib needs this\n' '_scproxy _scproxy.c ' '-framework SystemConfiguration ' '-framework CoreFoundation') # Turn off sqlite module. (scratch that; leaving it in.) # txt = efrotools.replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(') # txt = txt.replace(' _sqlite/', '# _sqlite/') # Turn off xz compression module. (scratch that; leaving it in.) # txt = efrotools.replace_one(txt, '_lzma _', '#_lzma _') # Turn off bzip2 module. txt = efrotools.replace_one(txt, '_bz2 _b', '#_bz2 _b') # Turn off openssl module (only if not doing openssl). if not ENABLE_OPENSSL: txt = efrotools.replace_one(txt, '_hashlib _hashopenssl.c', '#_hashlib _hashopenssl.c') # Turn off various other stuff we don't use. for line in [ '_codecs _codecsmodule.c', '_codecs_cn cjkcodecs/_codecs_cn.c', '_codecs_hk cjkcodecs/_codecs_hk.c', '_codecs_iso2022 cjkcodecs/', '_codecs_jp cjkcodecs/_codecs_jp.c', '_codecs_jp cjkcodecs/_codecs_jp.c', '_codecs_kr cjkcodecs/_codecs_kr.c', '_codecs_tw cjkcodecs/_codecs_tw.c', '_lsprof _lsprof.o rotatingtree.c', '_multibytecodec cjkcodecs/multibytecodec.c', '_multiprocessing _multiprocessing/multiprocessing.c', '_opcode _opcode.c', 'audioop audioop.c', 'grp grpmodule.c', 'mmap mmapmodule.c', 'parser parsermodule.c', 'pyexpat expat/xmlparse.c', ' expat/xmlrole.c ', ' expat/xmltok.c ', ' pyexpat.c ', ' -I$(srcdir)/Modules/expat ', ' -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI' ' -DXML_DEV_URANDOM', 'resource resource.c', 'syslog syslogmodule.c', 'termios termios.c', '_ctypes_test _ctypes/_ctypes_test.c', '_testbuffer _testbuffer.c', '_testimportmultiple _testimportmultiple.c', '_crypt _cryptmodule.c', # not on android so disabling here too ]: txt = efrotools.replace_one(txt, line, '#' + line) if ENABLE_OPENSSL: # _md5 and _sha modules are normally only built if the # system does not have the OpenSSL libs containing an optimized # version. # Note: seems we still need sha3 or we get errors for line in [ '_md5 md5module.c', '_sha1 sha1module.c', # '_sha3 _sha3/sha3module.c', '_sha256 sha256module.c', '_sha512 sha512module.c', ]: txt = efrotools.replace_one(txt, line, '#' + line) else: txt = efrotools.replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c') efrotools.writefile('patch/Python/Setup.embedded', txt) txt = efrotools.readfile('Makefile') # Fix a bug where spaces in PATH cause errors (darn you vmware fusion!) txt = efrotools.replace_one( txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .', '&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .') # Remove makefile dependencies so we don't build the # libs we're not using. srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' txt = efrotools.replace_one( txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' + ('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') + 'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt) srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.' 'b$(BUILD_NUMBER).tar.gz: ') txt = efrotools.replace_one( txt, srctxt, 'dist/Python-$(PYTHON_VER)-$1-support.b$(BUILD_NUMBER).tar.gz:' ' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt) # Turn doc strings on; looks like it only adds a few hundred k. txt = txt.replace('--without-doc-strings', '--with-doc-strings') # We're currently aiming at 10.13+ on mac # (see issue with utimensat and futimens). txt = efrotools.replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8', 'MACOSX_DEPLOYMENT_TARGET=10.13') # And equivalent iOS (11+). txt = efrotools.replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0', 'CFLAGS-iOS=-mios-version-min=11.0') # Ditto for tvOS. txt = efrotools.replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0', 'CFLAGS-tvOS=-mtvos-version-min=11.0') if debug: # Add debug build flag # (Currently expect to find 2 instances of this). dline = '--with-doc-strings --enable-ipv6 --without-ensurepip' splitlen = len(txt.split(dline)) if splitlen != 3: raise Exception("unexpected configure lines") txt = txt.replace(dline, '--with-pydebug ' + dline) # Debug has a different name. # (Currently expect to replace 13 instances of this). dline = 'python$(PYTHON_VER)m' splitlen = len(txt.split(dline)) if splitlen != 14: raise Exception("unexpected configure lines") txt = txt.replace(dline, 'python$(PYTHON_VER)dm') efrotools.writefile('Makefile', txt) # Ok; let 'er rip. # (we run these in parallel so limit to 1 job a piece; # otherwise they inherit the -j12 or whatever from the top level) # (also this build seems to fail with multiple threads) efrotools.run('make -j1 ' + { 'mac': 'Python-macOS', 'ios': 'Python-iOS', 'tvos': 'Python-tvOS' }[arch]) print('python build complete! (apple/' + arch + ')')
def gather() -> None: """Gather per-platform python headers, libs, and modules together. This assumes all embeddable py builds have been run successfully, and that PROJROOT is the cwd. """ # pylint: disable=too-many-locals # First off, clear out any existing output. existing_dirs = [ os.path.join('src/external', d) for d in os.listdir('src/external') if d.startswith('python-') and d != 'python-notes.txt' ] existing_dirs += [ os.path.join('assets/src', d) for d in os.listdir('assets/src') if d.startswith('pylib-') ] for existing_dir in existing_dirs: efrotools.run('rm -rf "' + existing_dir + '"') # Build our set of site-packages that we'll bundle in addition # to the base system. # FIXME: Should we perhaps make this part more explicit?.. # we might get unexpected changes sneaking if we're just # pulling from installed python. But then again, anytime we're doing # a new python build/gather we should expect *some* changes even if # only at the build-system level since we pull some of that directly # from latest git stuff. efrotools.run('mkdir -p "assets/src/pylib-site-packages"') efrotools.run('cp "/usr/local/lib/python' + PYTHON_VERSION_MAJOR + '/site-packages/typing_extensions.py"' ' "assets/src/pylib-site-packages/"') for buildtype in ['debug', 'release']: debug = buildtype == 'debug' bsuffix = '_debug' if buildtype == 'debug' else '' bsuffix2 = '-debug' if buildtype == 'debug' else '' libname = 'python' + PYTHON_VERSION_MAJOR + ('dm' if debug else 'm') bases = { 'mac': f'build/python_apple_mac{bsuffix}/build/macOS', 'ios': f'build/python_apple_ios{bsuffix}/build/iOS', 'tvos': f'build/python_apple_tvos{bsuffix}/build/tvOS', 'android_arm': f'build/python_android_arm{bsuffix}/build/sysroot', 'android_arm64': f'build/python_android_arm64{bsuffix}/build/sysroot', 'android_x86': f'build/python_android_x86{bsuffix}/build/sysroot', 'android_x86_64': f'build/python_android_x86_64{bsuffix}/build/sysroot' } # Note: only need pylib for the first in each group. builds: List[Dict[str, Any]] = [{ 'name': 'macos', 'group': 'apple', 'headers': bases['mac'] + '/Support/Python/Headers', 'libs': [ bases['mac'] + '/Support/Python/libPython.a', bases['mac'] + '/Support/OpenSSL/libOpenSSL.a', bases['mac'] + '/Support/XZ/libxz.a' ], 'pylib': (bases['mac'] + '/python/lib/python' + PYTHON_VERSION_MAJOR), }, { 'name': 'ios', 'group': 'apple', 'headers': bases['ios'] + '/Support/Python/Headers', 'libs': [ bases['ios'] + '/Support/Python/libPython.a', bases['ios'] + '/Support/OpenSSL/libOpenSSL.a', bases['ios'] + '/Support/XZ/libxz.a' ], }, { 'name': 'tvos', 'group': 'apple', 'headers': bases['tvos'] + '/Support/Python/Headers', 'libs': [ bases['tvos'] + '/Support/Python/libPython.a', bases['tvos'] + '/Support/OpenSSL/libOpenSSL.a', bases['tvos'] + '/Support/XZ/libxz.a' ], }, { 'name': 'android_arm', 'group': 'android', 'headers': bases['android_arm'] + f'/usr/include/{libname}', 'libs': [ bases['android_arm'] + f'/usr/lib/lib{libname}.a', bases['android_arm'] + '/usr/lib/libssl.a', bases['android_arm'] + '/usr/lib/libcrypto.a', bases['android_arm'] + '/usr/lib/liblzma.a', bases['android_arm'] + '/usr/lib/libsqlite3.a' ], 'libinst': 'android_armeabi-v7a', 'pylib': (bases['android_arm'] + '/usr/lib/python' + PYTHON_VERSION_MAJOR), }, { 'name': 'android_arm64', 'group': 'android', 'headers': bases['android_arm64'] + f'/usr/include/{libname}', 'libs': [ bases['android_arm64'] + f'/usr/lib/lib{libname}.a', bases['android_arm64'] + '/usr/lib/libssl.a', bases['android_arm64'] + '/usr/lib/libcrypto.a', bases['android_arm64'] + '/usr/lib/liblzma.a', bases['android_arm64'] + '/usr/lib/libsqlite3.a' ], 'libinst': 'android_arm64-v8a', }, { 'name': 'android_x86', 'group': 'android', 'headers': bases['android_x86'] + f'/usr/include/{libname}', 'libs': [ bases['android_x86'] + f'/usr/lib/lib{libname}.a', bases['android_x86'] + '/usr/lib/libssl.a', bases['android_x86'] + '/usr/lib/libcrypto.a', bases['android_x86'] + '/usr/lib/liblzma.a', bases['android_x86'] + '/usr/lib/libsqlite3.a' ], 'libinst': 'android_x86', }, { 'name': 'android_x86_64', 'group': 'android', 'headers': bases['android_x86_64'] + f'/usr/include/{libname}', 'libs': [ bases['android_x86_64'] + f'/usr/lib/lib{libname}.a', bases['android_x86_64'] + '/usr/lib/libssl.a', bases['android_x86_64'] + '/usr/lib/libcrypto.a', bases['android_x86_64'] + '/usr/lib/liblzma.a', bases['android_x86_64'] + '/usr/lib/libsqlite3.a' ], 'libinst': 'android_x86_64', }] for build in builds: grp = build['group'] builddir = f'src/external/python-{grp}{bsuffix2}' header_dst = os.path.join(builddir, 'include') lib_dst = os.path.join(builddir, 'lib') assets_src_dst = f'assets/src/pylib-{grp}' # Do some setup only once per group. if not os.path.exists(builddir): efrotools.run('mkdir -p "' + builddir + '"') efrotools.run('mkdir -p "' + lib_dst + '"') # Only pull modules into game assets on release pass. if not debug: # Copy system modules into the src assets # dir for this group. efrotools.run('mkdir -p "' + assets_src_dst + '"') efrotools.run( 'rsync --recursive --include "*.py"' ' --exclude __pycache__ --include "*/" --exclude "*" "' + build['pylib'] + '/" "' + assets_src_dst + '"') # Prune a bunch of modules we don't need to cut # down on size. prune = [ 'config-*', 'idlelib', 'lib-dynload', 'lib2to3', 'multiprocessing', 'pydoc_data', 'site-packages', 'ensurepip', 'tkinter', 'wsgiref', 'distutils', 'turtle.py', 'turtledemo', 'test', 'sqlite3/test', 'unittest', 'dbm', 'venv', 'ctypes/test', 'imaplib.py', '_sysconfigdata_*' ] efrotools.run('cd "' + assets_src_dst + '" && rm -rf ' + ' '.join(prune)) # Some minor filtering to system scripts: # on iOS/tvOS, addusersitepackages() leads to a crash # due to _sysconfigdata_dm_ios_darwin module not existing, # so let's skip that. fname = f'{assets_src_dst}/site.py' txt = efrotools.readfile(fname) txt = efrotools.replace_one( txt, ' known_paths = addusersitepackages(known_paths)', ' # efro tweak: this craps out on ios/tvos.\n' ' # (and we don\'t use it anyway)\n' ' # known_paths = addusersitepackages(known_paths)') efrotools.writefile(fname, txt) # Copy in a base set of headers (everything in a group should # be using the same headers) efrotools.run(f'cp -r "{build["headers"]}" "{header_dst}"') # Clear whatever pyconfigs came across; we'll build our own # universal one below. efrotools.run('rm ' + header_dst + '/pyconfig*') # Write a master pyconfig header that reroutes to each # platform's actual header. with open(header_dst + '/pyconfig.h', 'w') as hfile: hfile.write( '#if BA_OSTYPE_MACOS\n' '#include "pyconfig-macos.h"\n\n' '#elif BA_OSTYPE_IOS\n' '#include "pyconfig-ios.h"\n\n' '#elif BA_OSTYPE_TVOS\n' '#include "pyconfig-tvos.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__arm__)\n' '#include "pyconfig-android_arm.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__aarch64__)\n' '#include "pyconfig-android_arm64.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__i386__)\n' '#include "pyconfig-android_x86.h"\n\n' '#elif BA_OSTYPE_ANDROID and defined(__x86_64__)\n' '#include "pyconfig-android_x86_64.h"\n\n' '#else\n' '#error unknown platform\n\n' '#endif\n') # Now copy each build's config headers in with unique names. cfgs = [ f for f in os.listdir(build['headers']) if f.startswith('pyconfig') ] # Copy config headers to their filtered names. for cfg in cfgs: out = cfg.replace('pyconfig', 'pyconfig-' + build['name']) if cfg == 'pyconfig.h': # For platform's root pyconfig.h we need to filter # contents too (those headers can themselves include # others; ios for instance points to a arm64 and a # x86_64 variant). contents = efrotools.readfile(build['headers'] + '/' + cfg) contents = contents.replace('pyconfig', 'pyconfig-' + build['name']) efrotools.writefile(header_dst + '/' + out, contents) else: # other configs we just rename efrotools.run('cp "' + build['headers'] + '/' + cfg + '" "' + header_dst + '/' + out + '"') # Copy in libs. If the lib gave a specific install name, # use that; otherwise use name. targetdir = lib_dst + '/' + build.get('libinst', build['name']) efrotools.run('rm -rf "' + targetdir + '"') efrotools.run('mkdir -p "' + targetdir + '"') for lib in build['libs']: efrotools.run('cp "' + lib + '" "' + targetdir + '"') print('Great success!')
def android_patch() -> None: """Run necessary patches on an android archive before building.""" fname = 'src/cpython/Modules/Setup.dist' txt = efrotools.readfile(fname) # Need to switch some flags on this one. txt = efrotools.replace_one(txt, '#zlib zlibmodule.c', 'zlib zlibmodule.c -lz\n#zlib zlibmodule.c') # Just turn all these on. for enable in [ '#array arraymodule.c', '#cmath cmathmodule.c _math.c', '#math mathmodule.c', '#_contextvars _contextvarsmodule.c', '#_struct _struct.c', '#_weakref _weakref.c', '#_testcapi _testcapimodule.c', '#_random _randommodule.c', '#_elementtree -I', '#_pickle _pickle.c', '#_datetime _datetimemodule.c', '#_bisect _bisectmodule.c', '#_heapq _heapqmodule.c', '#_asyncio _asynciomodule.c', '#unicodedata unicodedata.c', '#fcntl fcntlmodule.c', '#select selectmodule.c', '#_csv _csv.c', '#_socket socketmodule.c', '#_blake2 _blake2/blake2module.c', '#binascii binascii.c', '#_posixsubprocess _posixsubprocess.c', '#_sha3 _sha3/sha3module.c' ]: txt = efrotools.replace_one(txt, enable, enable[1:]) if ENABLE_OPENSSL: txt = efrotools.replace_one(txt, '#_ssl _ssl.c \\', '_ssl _ssl.c -DUSE_SSL -lssl -lcrypto') else: # Note that the _md5 and _sha modules are normally only built if the # system does not have the OpenSSL libs containing an optimized # version. for enable in [ '#_md5 md5module.c', '#_sha1 sha1module.c', '#_sha256 sha256module.c', '#_sha512 sha512module.c' ]: txt = efrotools.replace_one(txt, enable, enable[1:]) # Turn this off (its just an example module). txt = efrotools.replace_one(txt, 'xxsubtype xxsubtype.c', '#xxsubtype xxsubtype.c') # For whatever reason this stuff isn't in there at all; add it. txt += '\n_json _json.c\n' txt += '\n_lzma _lzmamodule.c -llzma\n' txt += ('\n_sqlite3 -I$(srcdir)/Modules/_sqlite' ' -DMODULE_NAME=\'\\"sqlite3\\"\' -DSQLITE_OMIT_LOAD_EXTENSION' ' -lsqlite3 \\\n' ' _sqlite/cache.c \\\n' ' _sqlite/connection.c \\\n' ' _sqlite/cursor.c \\\n' ' _sqlite/microprotocols.c \\\n' ' _sqlite/module.c \\\n' ' _sqlite/prepare_protocol.c \\\n' ' _sqlite/row.c \\\n' ' _sqlite/statement.c \\\n' ' _sqlite/util.c\n') if ENABLE_OPENSSL: txt += '\n\n_hashlib _hashopenssl.c -DUSE_SSL -lssl -lcrypto\n' txt += '\n\n*disabled*\n_ctypes _crypt grp' efrotools.writefile(fname, txt) # Ok, this is weird. # When applying the module Setup, python looks for any line containing *=* # and interprets the whole thing a a global define?... # This breaks things for our static sqlite compile above. # The check used to look for [A-Z]*=* which didn't break, so let' just # change it back to that for now. fname = 'src/cpython/Modules/makesetup' txt = efrotools.readfile(fname) txt = efrotools.replace_one( txt, ' *=*) DEFS="$line$NL$DEFS"; continue;;', ' [A-Z]*=*) DEFS="$line$NL$DEFS"; continue;;') efrotools.writefile(fname, txt) print("APPLIED EFROTOOLS ANDROID BUILD PATCHES.")
def build_android(rootdir: str, arch: str, debug: bool = False) -> None: """Run a build for android with the given architecture. (can be arm, arm64, x86, or x86_64) """ import subprocess builddir = 'build/python_android_' + arch + ('_debug' if debug else '') efrotools.run('rm -rf "' + builddir + '"') efrotools.run('mkdir -p build') efrotools.run('git clone ' '[email protected]:yan12125/python3-android.git "' + builddir + '"') os.chdir(builddir) # It seems we now need 'autopoint' as part of this build, but on mac it # is not available on the normal path, but only as part of the keg-only # gettext homebrew formula. if (subprocess.run('which autopoint', shell=True, check=False).returncode != 0): print("Updating path for mac autopoint...") appath = subprocess.run('brew ls gettext | grep bin/autopoint', shell=True, check=True, capture_output=True) appathout = os.path.dirname(appath.stdout.decode().strip()) os.environ['PATH'] += (':' + appathout) print(f'ADDED "{appathout}" TO SYS PATH...') # Commit from Jan 8, 2020. Right after this, the build system was switched # a a completely new minimal one which will take some work to update here. # Punting on that for now... if True: # pylint: disable=using-constant-test efrotools.run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae') ftxt = efrotools.readfile('pybuild/env.py') # Set the packages we build. ftxt = efrotools.replace_one( ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," + (" 'openssl'" if ENABLE_OPENSSL else "") + ")\n# packages = (") # Don't wanna bother with gpg signing stuff. ftxt = efrotools.replace_one(ftxt, 'verify_source = True', 'verify_source = False') # Sub in the min api level we're targeting. ftxt = efrotools.replace_one(ftxt, 'android_api_level = 21', 'android_api_level = 21') ftxt = efrotools.replace_one(ftxt, "target_arch = 'arm'", "target_arch = '" + arch + "'") efrotools.writefile('pybuild/env.py', ftxt) ftxt = efrotools.readfile('Makefile') # This needs to be python3 for us. ftxt = efrotools.replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n') efrotools.writefile('Makefile', ftxt) ftxt = efrotools.readfile('pybuild/packages/python.py') # We currently build as a static lib. ftxt = efrotools.replace_one(ftxt, " '--enable-shared',\n", "") ftxt = efrotools.replace_one( ftxt, "super().__init__('https://github.com/python/cpython/')", "super().__init__('https://github.com/python/cpython/', branch='3.7')") # Turn ipv6 on (curious why its turned off here?...) # Also, turn on low level debugging features for our debug builds. ftxt = efrotools.replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',") if debug: ftxt = efrotools.replace_one(ftxt, "'./configure',", "'./configure', '--with-pydebug',") # We don't use this stuff so lets strip it out to simplify. ftxt = efrotools.replace_one(ftxt, "'--without-ensurepip',", "") # This builds all modules as dynamic libs, but we want to be consistent # with our other embedded builds and just static-build the ones we # need... so to change that we'll need to add a hook for ourself after # python is downloaded but before it is built so we can muck with it. ftxt = efrotools.replace_one( ftxt, ' def build(self):', ' def build(self):\n import os\n' ' if os.system(\'"' + rootdir + '/tools/snippets" python_android_patch "' + os.getcwd() + '"\') != 0: raise Exception("patch apply failed")') efrotools.writefile('pybuild/packages/python.py', ftxt) # Set this to a particular cpython commit to target exact releases from git commit = '43364a7ae01fbe4288ef42622259a0038ce1edcc' # 3.7.6 release if commit is not None: ftxt = efrotools.readfile('pybuild/source.py') # Check out a particular commit right after the clone. ftxt = efrotools.replace_one( ftxt, "'git', 'clone', '--single-branch', '-b'," " self.branch, self.source_url, self.dest])", "'git', 'clone', '-b'," " self.branch, self.source_url, self.dest])\n" " # efro: hack to get the python we want.\n" " print('DOING URL', self.source_url)\n" " if self.source_url == " "'https://github.com/python/cpython/':\n" " run_in_dir(['git', 'checkout', '" + commit + "'], self.source_dir)") efrotools.writefile('pybuild/source.py', ftxt) ftxt = efrotools.readfile('pybuild/util.py') # Still don't wanna bother with gpg signing stuff. ftxt = efrotools.replace_one( ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n', 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n' ' print("gpg-verify disabled by ericf")\n' ' return\n') efrotools.writefile('pybuild/util.py', ftxt) # These builds require ANDROID_NDK to be set, so make sure that's # the case. os.environ['ANDROID_NDK'] = subprocess.check_output( [rootdir + '/tools/android_sdk_utils', 'get-ndk-path']).decode().strip() # Ok, let 'er rip # (we often run these builds in parallel so limit to 1 job a piece; # otherwise they each inherit the -j12 or whatever from the top level). efrotools.run('make -j1') print('python build complete! (android/' + arch + ')')
def build_apple(arch: str, debug: bool = False) -> None: """Run a build for the provided apple arch (mac, ios, or tvos).""" # pylint: disable=too-many-branches # pylint: disable=too-many-statements import platform import subprocess from efro.error import CleanError # IMPORTANT; seems we currently wind up building against /usr/local gettext # stuff. Hopefully the maintainer fixes this, but for now I need to # remind myself to blow it away while building. # (via brew remove gettext --ignore-dependencies) if 'MacBook-Fro' in platform.node(): if (subprocess.run('which gettext', shell=True, check=False).returncode == 0): raise CleanError('NEED TO TEMP-KILL GETTEXT') builddir = 'build/python_apple_' + arch + ('_debug' if debug else '') run('rm -rf "' + builddir + '"') run('mkdir -p build') run('git clone ' '[email protected]:pybee/Python-Apple-support.git "' + builddir + '"') os.chdir(builddir) # TEMP: Check out a particular commit while the branch head is broken. # We can actually fix this to use the current one, but something # broke in the underlying build even on old commits so keeping it # locked for now... # run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6') run(f'git checkout {PYVER}') # On mac we currently have to add the _scproxy module or urllib will # fail. txt = readfile('patch/Python/Setup.embedded') if arch == 'mac': txt += ('\n' '# ericf added - mac urllib needs this\n' '_scproxy _scproxy.c ' '-framework SystemConfiguration ' '-framework CoreFoundation') # Turn off sqlite module. (scratch that; leaving it in.) # txt = replace_one(txt, '_sqlite3 -I$(', '#_sqlite3 -I$(') # txt = txt.replace(' _sqlite/', '# _sqlite/') # Turn off xz compression module. (scratch that; leaving it in.) # txt = replace_one(txt, '_lzma _', '#_lzma _') # Turn off bzip2 module. txt = replace_one(txt, '_bz2 _b', '#_bz2 _b') # Turn off openssl module (only if not doing openssl). if not ENABLE_OPENSSL: txt = replace_one(txt, '_hashlib _hashopenssl.c', '#_hashlib _hashopenssl.c') # Turn off various other stuff we don't use. for line in [ '_codecs _codecsmodule.c', '_codecs_cn cjkcodecs/_codecs_cn.c', '_codecs_hk cjkcodecs/_codecs_hk.c', '_codecs_iso2022 cjkcodecs/', '_codecs_jp cjkcodecs/_codecs_jp.c', '_codecs_jp cjkcodecs/_codecs_jp.c', '_codecs_kr cjkcodecs/_codecs_kr.c', '_codecs_tw cjkcodecs/_codecs_tw.c', '_lsprof _lsprof.o rotatingtree.c', '_multibytecodec cjkcodecs/multibytecodec.c', '_multiprocessing _multiprocessing/multiprocessing.c', '_opcode _opcode.c', 'audioop audioop.c', 'grp grpmodule.c', 'mmap mmapmodule.c', 'parser parsermodule.c', 'pyexpat expat/xmlparse.c', ' expat/xmlrole.c ', ' expat/xmltok.c ', ' pyexpat.c ', ' -I$(srcdir)/Modules/expat ', ' -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI' ' -DXML_DEV_URANDOM', 'resource resource.c', 'syslog syslogmodule.c', 'termios termios.c', '_ctypes_test _ctypes/_ctypes_test.c', '_testbuffer _testbuffer.c', '_testimportmultiple _testimportmultiple.c', '_crypt _cryptmodule.c', # not on android so disabling here too ]: txt = replace_one(txt, line, '#' + line) if ENABLE_OPENSSL: # _md5 and _sha modules are normally only built if the # system does not have the OpenSSL libs containing an optimized # version. # Note: seems we still need sha3 or we get errors for line in [ '_md5 md5module.c', '_sha1 sha1module.c', # '_sha3 _sha3/sha3module.c', '_sha256 sha256module.c', '_sha512 sha512module.c', ]: txt = replace_one(txt, line, '#' + line) else: txt = replace_one(txt, '_ssl _ssl.c', '#_ssl _ssl.c') writefile('patch/Python/Setup.embedded', txt) txt = readfile('Makefile') # Fix a bug where spaces in PATH cause errors (darn you vmware fusion!) txt = replace_one( txt, '&& PATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH) .', '&& PATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/dist/bin:$(PATH)" .') # Remove makefile dependencies so we don't build the # libs we're not using. srctxt = '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: ' if PY38: txt = replace_one( txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER).a: ' + ('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') + 'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt) else: txt = replace_one( txt, srctxt, '$$(PYTHON_DIR-$1)/dist/lib/libpython$(PYTHON_VER)m.a: ' + ('build/$2/Support/OpenSSL ' if ENABLE_OPENSSL else '') + 'build/$2/Support/XZ $$(PYTHON_DIR-$1)/Makefile\n#' + srctxt) srctxt = ('dist/Python-$(PYTHON_VER)-$1-support.' '$(BUILD_NUMBER).tar.gz: ') txt = replace_one( txt, srctxt, 'dist/Python-$(PYTHON_VER)-$1-support.$(BUILD_NUMBER).tar.gz:' ' $$(PYTHON_FRAMEWORK-$1)\n#' + srctxt) # Turn doc strings on; looks like it only adds a few hundred k. txt = txt.replace('--without-doc-strings', '--with-doc-strings') # Set mac/ios version reqs # (see issue with utimensat and futimens). txt = replace_one(txt, 'MACOSX_DEPLOYMENT_TARGET=10.8', 'MACOSX_DEPLOYMENT_TARGET=10.15') # And equivalent iOS (11+). txt = replace_one(txt, 'CFLAGS-iOS=-mios-version-min=8.0', 'CFLAGS-iOS=-mios-version-min=13.0') # Ditto for tvOS. txt = replace_one(txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0', 'CFLAGS-tvOS=-mtvos-version-min=13.0') if debug: # Add debug build flag # (Currently expect to find 2 instances of this). dline = '--with-doc-strings --enable-ipv6 --without-ensurepip' splitlen = len(txt.split(dline)) if splitlen != 3: raise Exception('unexpected configure lines') txt = txt.replace(dline, '--with-pydebug ' + dline) # Debug has a different name. # (Currently expect to replace 12 instances of this). dline = 'python$(PYTHON_VER)' if PY38 else 'python$(PYTHON_VER)m' splitlen = len(txt.split(dline)) if splitlen != 13: raise RuntimeError(f'Unexpected configure line count {splitlen}.') txt = txt.replace( dline, 'python$(PYTHON_VER)d' if PY38 else 'python$(PYTHON_VER)dm') writefile('Makefile', txt) # Ok; let 'er rip. # (we run these in parallel so limit to 1 job a piece; # otherwise they inherit the -j12 or whatever from the top level) # (also this build seems to fail with multiple threads) run('make -j1 ' + { 'mac': 'Python-macOS', 'ios': 'Python-iOS', 'tvos': 'Python-tvOS' }[arch]) print('python build complete! (apple/' + arch + ')')
def build_apple(arch: str, debug: bool = False) -> None: """Run a build for the provided apple arch (mac, ios, or tvos).""" import platform from efro.error import CleanError # IMPORTANT; seems we currently wind up building against /usr/local gettext # stuff. Hopefully the maintainer fixes this, but for now I need to # remind myself to blow it away while building. # (via brew remove gettext --ignore-dependencies) # NOTE: Should check to see if this is still necessary on Apple silicon # since homebrew stuff is no longer in /usr/local there. if ('MacBook-Fro' in platform.node() and os.environ.get('SKIP_GETTEXT_WARNING') != '1'): if (subprocess.run('which gettext', shell=True, check=False).returncode == 0): raise CleanError( 'NEED TO TEMP-KILL GETTEXT (or set SKIP_GETTEXT_WARNING=1)') builddir = 'build/python_apple_' + arch + ('_debug' if debug else '') subprocess.run(['rm', '-rf', builddir], check=True) subprocess.run(['mkdir', '-p', 'build'], check=True) subprocess.run( [ 'git', 'clone', 'https://github.com/beeware/Python-Apple-support.git', builddir ], check=True, ) os.chdir(builddir) # TEMP: Check out a particular commit while the branch head is broken. # We can actually fix this to use the current one, but something # broke in the underlying build even on old commits so keeping it # locked for now... # run('git checkout bf1ed73d0d5ff46862ba69dd5eb2ffaeff6f19b6') subprocess.run(['git', 'checkout', PY_VER], check=True) txt = readfile('Makefile') # Fix a bug where spaces in PATH cause errors (darn you vmware fusion!) txt = replace_exact( txt, '\t\tPATH=$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/_install/bin:$(PATH)', '\t\tPATH="$(PROJECT_DIR)/$(PYTHON_DIR-macOS)/_install/bin:$(PATH)"') # Turn doc strings on; looks like it only adds a few hundred k. txt = replace_exact(txt, '--without-doc-strings', '--with-doc-strings', count=2) # Customize our minimum version requirements txt = replace_exact( txt, 'CFLAGS-macOS=-mmacosx-version-min=10.15\n', 'CFLAGS-macOS=-mmacosx-version-min=10.15\n', ) txt = replace_exact( txt, 'CFLAGS-iOS=-mios-version-min=12.0 ', 'CFLAGS-iOS=-mios-version-min=12.0 ', ) txt = replace_exact( txt, 'CFLAGS-tvOS=-mtvos-version-min=9.0 ', 'CFLAGS-tvOS=-mtvos-version-min=9.0 ', ) assert '--with-pydebug' not in txt if debug: # Add debug build flag txt = replace_exact( txt, '--enable-ipv6 --without-ensurepip ', '--enable-ipv6 --with-pydebug --without-ensurepip ', count=2, ) # Debug lib has a different name. txt = replace_exact(txt, 'python$(PYTHON_VER).a', 'python$(PYTHON_VER)d.a', count=2) txt = replace_exact(txt, '/include/python$(PYTHON_VER)', '/include/python$(PYTHON_VER)d', count=4) # Inject our custom modifications to fire right after their normal # Setup.local filtering and right before building (and pass the same # 'slice' value they use so we can use it too). txt = replace_exact( txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n' '\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n', '\t\t\tsed -e "s/{{slice}}/$$(SLICE-$$(SDK-$(target)))/g" \\\n' '\t\t\t> $$(PYTHON_DIR-$(target))/Modules/Setup.local\n' '\tcd $$(PYTHON_DIR-$(target)) && ' f'../../../../../tools/pcommand python_apple_patch {arch} ' '"$$(SLICE-$$(SDK-$(target)))"\n') txt = replace_exact( txt, '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n' '\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n', '\t\t\tsed -e "s/{{slice}}/$$(SLICE-macosx)/g" \\\n' '\t\t\t> $$(PYTHON_DIR-$(os))/Modules/Setup.local\n' '\tcd $$(PYTHON_DIR-$(os)) && ' f'../../../../../tools/pcommand python_apple_patch {arch} ' '"$$(SLICE-macosx)"\n') writefile('Makefile', txt) # Ok; let 'er rip. # (we run these in parallel so limit to 1 job a piece; # otherwise they inherit the -j12 or whatever from the top level) # (also this build seems to fail with multiple threads) subprocess.run( [ 'make', '-j1', { 'mac': 'Python-macOS', 'ios': 'Python-iOS', 'tvos': 'Python-tvOS' }[arch] ], check=True, ) print('python build complete! (apple/' + arch + ')')
def build_android(rootdir: str, arch: str, debug: bool = False) -> None: """Run a build for android with the given architecture. (can be arm, arm64, x86, or x86_64) """ # pylint: disable=too-many-statements import subprocess import platform builddir = 'build/python_android_' + arch + ('_debug' if debug else '') run('rm -rf "' + builddir + '"') run('mkdir -p build') run('git clone ' 'https://github.com/yan12125/python3-android.git "' + builddir + '"') os.chdir(builddir) # It seems we now need 'autopoint' as part of this build, but on mac it # is not available on the normal path, but only as part of the keg-only # gettext homebrew formula. if platform.system() == 'Darwin': if (subprocess.run('which autopoint', shell=True, check=False).returncode != 0): print('Updating path for mac autopoint...') appath = subprocess.run('brew ls gettext | grep bin/autopoint', shell=True, check=True, capture_output=True) appathout = os.path.dirname(appath.stdout.decode().strip()) os.environ['PATH'] += (':' + appathout) print(f'ADDED "{appathout}" TO SYS PATH...') # Commit from Jan 8, 2020. Right after this, the build system was switched # a a completely new minimal one which will take some work to update here. # Punting on that for now... (tentative plan is to try and adopt the new # one when we update for Python 3.10 in a year or two). if True: # pylint: disable=using-constant-test run('git checkout 9adbcfaca37f40b7a86381f83f0f6af4187233ae') ftxt = readfile('pybuild/env.py') # Set the packages we build. ftxt = replace_one( ftxt, 'packages = (', "packages = ('zlib', 'sqlite', 'xz'," + (" 'openssl'" if ENABLE_OPENSSL else '') + ')\n# packages = (') # Don't wanna bother with gpg signing stuff. ftxt = replace_one(ftxt, 'verify_source = True', 'verify_source = False') # Sub in the min api level we're targeting. ftxt = replace_one(ftxt, 'android_api_level = 21', 'android_api_level = 21') ftxt = replace_one(ftxt, "target_arch = 'arm'", "target_arch = '" + arch + "'") writefile('pybuild/env.py', ftxt) ftxt = readfile('Makefile') # This needs to be python3 for us. ftxt = replace_one(ftxt, 'PYTHON?=python\n', 'PYTHON?=python3\n') writefile('Makefile', ftxt) ftxt = readfile('pybuild/packages/python.py') # We currently build as a static lib. ftxt = replace_one(ftxt, " '--enable-shared',\n", '') ftxt = replace_one( ftxt, "super().__init__('https://github.com/python/cpython/')", "super().__init__('https://github.com/python/cpython/'" f", branch='{PYVER}')") # Turn ipv6 on (curious why its turned off here?...) # Also, turn on low level debugging features for our debug builds. ftxt = replace_one(ftxt, "'--disable-ipv6',", "'--enable-ipv6',") if debug: ftxt = replace_one(ftxt, "'./configure',", "'./configure', '--with-pydebug',") # We don't use this stuff so lets strip it out to simplify. ftxt = replace_one(ftxt, "'--without-ensurepip',", '') # This builds all modules as dynamic libs, but we want to be consistent # with our other embedded builds and just static-build the ones we # need... so to change that we'll need to add a hook for ourself after # python is downloaded but before it is built so we can muck with it. ftxt = replace_one( ftxt, ' def build(self):', ' def build(self):\n import os\n' ' if os.system(\'"' + rootdir + '/tools/pcommand" python_android_patch "' + os.getcwd() + '"\') != 0: raise Exception("patch apply failed")') writefile('pybuild/packages/python.py', ftxt) # Set these to particular releases to use those. # py_commit = '580fbb018fd0844806119614d752b41fc69660f9' # 3.8.5 py_commit = '6503f05dd59e26a9986bdea097b3da9b3546f45b' # 3.8.7 # cpython-source-deps stuff started failing for OpenSSL on Jan 8 2021. # Pinning it to an older one for now. py_ssl_commit = '7f34c3085feb4692bbbb6c8b19d053ebc5049dad' # From 6/12/20 commit_lines = '' if py_commit is not None: commit_lines += (' if self.source_url == ' "'https://github.com/python/cpython/':\n" " run_in_dir(['git', 'checkout', '" + py_commit + "'], self.source_dir)\n") if py_ssl_commit is not None: commit_lines += (' if self.source_url == ' "'https://github.com/python/cpython-source-deps'" " and self.branch == 'openssl-1.1.1':\n" " run_in_dir(['git', 'checkout', '" + py_ssl_commit + "'], self.source_dir)\n") ftxt = readfile('pybuild/source.py') # Check out a particular commit right after the clone. ftxt = replace_one( ftxt, "'git', 'clone', '--single-branch', '-b'," ' self.branch, self.source_url, self.dest])', "'git', 'clone', '-b'," ' self.branch, self.source_url, self.dest])\n' ' # efro: hack to get exact commits we want.\n' " print('DOING URL', self.source_url)\n" + commit_lines) writefile('pybuild/source.py', ftxt) ftxt = readfile('pybuild/util.py') # Still don't wanna bother with gpg signing stuff. ftxt = replace_one( ftxt, 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n', 'def gpg_verify_file(sig_filename, filename, validpgpkeys):\n' ' print("gpg-verify disabled by ericf")\n' ' return\n') writefile('pybuild/util.py', ftxt) # These builds require ANDROID_NDK to be set, so make sure that's # the case. os.environ['ANDROID_NDK'] = subprocess.check_output( [rootdir + '/tools/android_sdk_utils', 'get-ndk-path']).decode().strip() # Ok, let 'er rip # (we often run these builds in parallel so limit to 1 job a piece; # otherwise they each inherit the -j12 or whatever from the top level). run('make -j1') print('python build complete! (android/' + arch + ')')