def __init__(self): super(Context, self).__init__() self.include_dirs = [] self._build_env_prepared = False self._sdk_dir = None self._ndk_dir = None self._android_api = None self._ndk_ver = None self.toolchain_prefix = None self.toolchain_version = None self.local_recipes = None # root of the toolchain self.setup_dirs() # this list should contain all Archs, it is pruned later self.archs = ( ArchARM(self), ArchARMv7_a(self), Archx86(self) ) ensure_dir(join(self.build_dir, 'bootstrap_builds')) ensure_dir(join(self.build_dir, 'other_builds')) # other_builds: where everything else is built # remove the most obvious flags that can break the compilation self.env.pop("LDFLAGS", None) self.env.pop("ARCHFLAGS", None) self.env.pop("CFLAGS", None)
def make_build_dest(dest): build_dest = join(build_root, dest) if not isdir(build_dest): ensure_dir(build_dest) return build_dest, False return build_dest, True
def get_recipe_env(self, arch, with_flags_in_cc=True): env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format( self.ctx.get_libs_dir(arch.arch) + ' -L{} '.format(self.ctx.libs_dir) + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))) if self.ctx.python_recipe.from_crystax: env['LDFLAGS'] = (env['LDFLAGS'] + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))) # ' -L/home/asandy/.local/share/python-for-android/build/bootstrap_builds/sdl2/libs/armeabi ' if self.ctx.python_recipe.from_crystax: env['LDSHARED'] = env['CC'] + ' -shared' else: env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh') # shprint(sh.whereis, env['LDSHARED'], _env=env) env['LIBLINK'] = 'NOTNONE' env['NDKPLATFORM'] = self.ctx.ndk_platform if self.ctx.copy_libs: env['COPYLIBS'] = '1' # Every recipe uses its own liblink path, object files are # collected and biglinked later liblink_path = join(self.get_build_container_dir(arch.arch), 'objects_{}'.format(self.name)) env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) if self.ctx.python_recipe.from_crystax: env['CFLAGS'] = '-I{} '.format( join(self.ctx.ndk_dir, 'sources', 'python', self.ctx.python_recipe.version, 'include', 'python')) + env['CFLAGS'] return env
def build_recipes(build_order, python_modules, ctx): # Put recipes in correct build order bs = ctx.bootstrap info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) # 4) biglink everything # AND: Should make this optional info_main('# Biglinking object files') if not ctx.python_recipe or not ctx.python_recipe.from_crystax: biglink(ctx, arch) else: info('NDK is crystax, skipping biglink (will this work?)') # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules) return
def build_arch(self, arch): super(LibZMQRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) # # libsodium_recipe = Recipe.get_recipe('libsodium', self.ctx) # libsodium_dir = libsodium_recipe.get_build_dir(arch.arch) # env['sodium_CFLAGS'] = '-I{}'.format(join( # libsodium_dir, 'src')) # env['sodium_LDLAGS'] = '-L{}'.format(join( # libsodium_dir, 'src', 'libsodium', '.libs')) curdir = self.get_build_dir(arch.arch) prefix = join(curdir, "install") with current_directory(curdir): bash = sh.Command('sh') shprint( bash, './configure', '--host=arm-linux-androideabi', '--without-documentation', '--prefix={}'.format(prefix), '--with-libsodium=no', _env=env) shprint(sh.make, _env=env) shprint(sh.make, 'install', _env=env) shutil.copyfile('.libs/libzmq.so', join( self.ctx.get_libs_dir(arch.arch), 'libzmq.so')) bootstrap_obj_dir = join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch) ensure_dir(bootstrap_obj_dir) shutil.copyfile( '{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}/libgnustl_shared.so'.format( self.ctx.ndk_dir, self.ctx.toolchain_version, arch), join(bootstrap_obj_dir, 'libgnustl_shared.so'))
def get_recipe_env(self, arch, with_flags_in_cc=True): env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc) env["LDFLAGS"] = env["LDFLAGS"] + " -L{} ".format( self.ctx.get_libs_dir(arch.arch) + " -L{} ".format(self.ctx.libs_dir) + " -L{}".format(join(self.ctx.bootstrap.build_dir, "obj", "local", arch.arch)) ) if self.ctx.python_recipe.from_crystax: env["LDFLAGS"] = env["LDFLAGS"] + " -L{}".format(join(self.ctx.bootstrap.build_dir, "libs", arch.arch)) # ' -L/home/asandy/.local/share/python-for-android/build/bootstrap_builds/sdl2/libs/armeabi ' if self.ctx.python_recipe.from_crystax: env["LDSHARED"] = env["CC"] + " -shared" else: env["LDSHARED"] = join(self.ctx.root_dir, "tools", "liblink.sh") # shprint(sh.whereis, env['LDSHARED'], _env=env) env["LIBLINK"] = "NOTNONE" env["NDKPLATFORM"] = self.ctx.ndk_platform if self.ctx.copy_libs: env["COPYLIBS"] = "1" # Every recipe uses its own liblink path, object files are # collected and biglinked later liblink_path = join(self.get_build_container_dir(arch.arch), "objects_{}".format(self.name)) env["LIBLINK_PATH"] = liblink_path ensure_dir(liblink_path) if self.ctx.python_recipe.from_crystax: env["CFLAGS"] = ( "-I{} ".format( join(self.ctx.ndk_dir, "sources", "python", self.ctx.python_recipe.version, "include", "python") ) + env["CFLAGS"] ) return env
def get_recipe_env(self, arch, with_flags_in_cc=True): env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format( self.ctx.get_libs_dir(arch.arch) + ' -L{} '.format(self.ctx.libs_dir) + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))) if self.ctx.python_recipe.from_crystax: env['LDFLAGS'] = (env['LDFLAGS'] + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))) if self.ctx.python_recipe.from_crystax or self.ctx.python_recipe.name == 'python3': env['LDSHARED'] = env['CC'] + ' -shared' else: env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh') # shprint(sh.whereis, env['LDSHARED'], _env=env) env['LIBLINK'] = 'NOTNONE' env['NDKPLATFORM'] = self.ctx.ndk_platform if self.ctx.copy_libs: env['COPYLIBS'] = '1' # Every recipe uses its own liblink path, object files are # collected and biglinked later liblink_path = join(self.get_build_container_dir(arch.arch), 'objects_{}'.format(self.name)) env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) return env
def prebuild_arch(self, arch): if not self.is_patched(arch): super(ReportLabRecipe, self).prebuild_arch(arch) self.apply_patch('patches/fix-setup.patch', arch.arch) recipe_dir = self.get_build_dir(arch.arch) shprint(sh.touch, os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include')) tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp")) info('reportlab recipe: recipe_dir={}'.format(recipe_dir)) info('reportlab recipe: tmp_dir={}'.format(tmp_dir)) info('reportlab recipe: ft_dir={}'.format(ft_dir)) info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir)) info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir)) with current_directory(recipe_dir): sh.ls('-lathr') ensure_dir(tmp_dir) pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip") if not os.path.isfile(pfbfile): sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile) sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile) if os.path.isfile("setup.py"): with open('setup.py', 'rb') as f: text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir) with open('setup.py', 'wb') as f: f.write(text)
def build_arch(self, arch): info("Extracting CrystaX python3 from NDK package") dirn = self.ctx.get_python_install_dir() ensure_dir(dirn) self.ctx.hostpython = "python{}".format(self.version)
def run_distribute(self): info_main('# Creating Android project from build and {} bootstrap'.format( self.name)) info('This currently just copies the build stuff straight from the build dir.') shprint(sh.rm, '-rf', self.dist_dir) shprint(sh.cp, '-r', self.build_dir, self.dist_dir) with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) arch = self.ctx.archs[0] if len(self.ctx.archs) > 1: raise ValueError('built for more than one arch, but bootstrap cannot handle that yet') info('Bootstrap running with arch {}'.format(arch)) with current_directory(self.dist_dir): info('Copying python distribution') self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_aars(arch) self.distribute_javaclasses(self.ctx.javaclass_dir) python_bundle_dir = join('_python_bundle', '_python_bundle') ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(ServiceOnlyBootstrap, self).run_distribute()
def build_arch(self, arch): recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, self.build_subdir) ensure_dir(build_dir) if not exists(join(build_dir, 'python')): with current_directory(recipe_build_dir): # Configure the build with current_directory(build_dir): if not exists('config.status'): shprint( sh.Command(join(recipe_build_dir, 'configure'))) # Create the Setup file. This copying from Setup.dist # seems to be the normal and expected procedure. shprint(sh.cp, join('Modules', 'Setup.dist'), join(build_dir, 'Modules', 'Setup')) result = shprint(sh.make, '-C', build_dir) else: info('Skipping {name} ({version}) build, as it has already ' 'been completed'.format(name=self.name, version=self.version)) self.ctx.hostpython = join(build_dir, 'python')
def _unpack_aar(self, aar, arch): '''Unpack content of .aar bundle and copy to current dist dir.''' with temp_directory() as temp_dir: name = splitext(basename(aar))[0] jar_name = name + '.jar' info("unpack {} aar".format(name)) debug(" from {}".format(aar)) debug(" to {}".format(temp_dir)) shprint(sh.unzip, '-o', aar, '-d', temp_dir) jar_src = join(temp_dir, 'classes.jar') jar_tgt = join('libs', jar_name) debug("copy {} jar".format(name)) debug(" from {}".format(jar_src)) debug(" to {}".format(jar_tgt)) ensure_dir('libs') shprint(sh.cp, '-a', jar_src, jar_tgt) so_src_dir = join(temp_dir, 'jni', arch.arch) so_tgt_dir = join('libs', arch.arch) debug("copy {} .so".format(name)) debug(" from {}".format(so_src_dir)) debug(" to {}".format(so_tgt_dir)) ensure_dir(so_tgt_dir) so_files = glob.glob(join(so_src_dir, '*.so')) for f in so_files: shprint(sh.cp, '-a', f, so_tgt_dir)
def run_distribute(self): info_main('# Creating Android project from build and {} bootstrap'.format( self.name)) # src_path = join(self.ctx.root_dir, 'bootstrap_templates', # self.name) src_path = join(self.bootstrap_dir, 'build') arch = self.ctx.archs[0] if len(self.ctx.archs) > 1: raise ValueError('built for more than one arch, but bootstrap cannot handle that yet') info('Bootstrap running with arch {}'.format(arch)) with current_directory(self.dist_dir): info('Creating initial layout') for dirname in ('assets', 'bin', 'private', 'res', 'templates'): if not exists(dirname): shprint(sh.mkdir, dirname) info('Copying default files') shprint(sh.cp, '-a', join(self.build_dir, 'project.properties'), '.') shprint(sh.cp, '-a', join(src_path, 'build.py'), '.') shprint(sh.cp, '-a', join(src_path, 'buildlib'), '.') shprint(sh.cp, '-a', join(src_path, 'src'), '.') shprint(sh.cp, '-a', join(src_path, 'templates'), '.') shprint(sh.cp, '-a', join(src_path, 'res'), '.') shprint(sh.cp, '-a', join(src_path, 'blacklist.txt'), '.') shprint(sh.cp, '-a', join(src_path, 'whitelist.txt'), '.') with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) info('Copying python distribution') python_bundle_dir = join('_python_bundle', '_python_bundle') if 'python2legacy' in self.ctx.recipe_build_order: # a special case with its own packaging location python_bundle_dir = 'private' # And also must had an install directory, make sure of that self.ctx.python_recipe.create_python_install(self.dist_dir) self.distribute_libs( arch, [join(self.build_dir, 'libs', arch.arch), self.ctx.get_libs_dir(arch.arch)]) self.distribute_aars(arch) self.distribute_javaclasses(self.ctx.javaclass_dir) ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(PygameBootstrap, self).run_distribute()
def distribute_libs(self, arch, src_dirs, wildcard='*', dest_dir="libs"): '''Copy existing arch libs from build dirs to current dist dir.''' info('Copying libs') tgt_dir = join(dest_dir, arch.arch) ensure_dir(tgt_dir) for src_dir in src_dirs: for lib in glob.glob(join(src_dir, wildcard)): shprint(sh.cp, '-a', lib, tgt_dir)
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('autoreconf'), '-vif', _env=env) shprint(sh.Command('./configure'), '--host=' + arch.command_prefix, '--prefix=' + self.get_build_dir(arch.arch), '--disable-builddir', '--enable-shared', _env=env) # '--with-sysroot={}'.format(self.ctx.ndk_platform), # '--target={}'.format(arch.toolchain_prefix), # ndk 15 introduces unified headers required --sysroot and # -isysroot for libraries and headers. libtool's head explodes # trying to weave them into it's own magic. The result is a link # failure trying to link libc. We call make to compile the bits # and manually link... try: shprint(sh.make, '-j5', 'libffi.la', _env=env) except sh.ErrorReturnCode_2: info("make libffi.la failed as expected") cc = sh.Command(env['CC'].split()[0]) cflags = env['CC'].split()[1:] host_build = self.get_build_dir(arch.arch) arch_flags = '' if '-march=' in env['CFLAGS']: arch_flags = '-march={}'.format(env['CFLAGS'].split('-march=')[1]) src_arch = arch.command_prefix.split('-')[0] if src_arch == 'x86_64': # libffi has not specific arch files for x86_64...so...using # the ones from x86 which seems to build fine... src_arch = 'x86' cflags.extend(arch_flags.split()) cflags.extend(['-shared', '-fPIC', '-DPIC']) cflags.extend(glob(join(host_build, 'src/.libs/*.o'))) cflags.extend(glob(join(host_build, 'src', src_arch, '.libs/*.o'))) ldflags = env['LDFLAGS'].split() cflags.extend(ldflags) cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o', '.libs/libffi.so']) with current_directory(host_build): shprint(cc, *cflags, _env=env) ensure_dir(self.ctx.get_libs_dir(arch.arch)) shprint(sh.cp, join(host_build, '.libs', 'libffi.so'), self.ctx.get_libs_dir(arch.arch))
def run_distribute(self): info_main("# Creating Android project ({})".format(self.name)) arch = self.ctx.archs[0] python_install_dir = self.ctx.get_python_install_dir() from_crystax = self.ctx.python_recipe.from_crystax if len(self.ctx.archs) > 1: raise ValueError("SDL2/gradle support only one arch") info("Copying SDL2/gradle build for {}".format(arch)) shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environment variable (ANDROID_HOME) # or the local.properties if exists with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) with current_directory(self.dist_dir): info("Copying Python distribution") hostpython = sh.Command(self.ctx.hostpython) if self.ctx.python_recipe.name == 'python2': try: shprint(hostpython, '-OO', '-m', 'compileall', python_install_dir, _tail=10, _filterout="^Listing") except sh.ErrorReturnCode: pass if 'python2' in self.ctx.recipe_build_order and not exists('python-install'): shprint( sh.cp, '-a', python_install_dir, './python-install') self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) python_bundle_dir = join('_python_bundle', '_python_bundle') if 'python2' in self.ctx.recipe_build_order: # Python 2 is a special case with its own packaging location python_bundle_dir = 'private' ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(SDL2GradleBootstrap, self).run_distribute()
def get_recipe_env(self, arch=None): env = super(PygameRecipe, self).get_recipe_env(arch) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( self.ctx.get_libs_dir(arch.arch)) env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink') env['LIBLINK'] = 'NOTNONE' env['NDKPLATFORM'] = self.ctx.ndk_platform # Every recipe uses its own liblink path, object files are collected and biglinked later liblink_path = join(self.get_build_container_dir(arch.arch), 'objects_{}'.format(self.name)) env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) return env
def get_recipe_env(self, arch, with_flags_in_cc=True): env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format( self.ctx.get_libs_dir(arch.arch) + ' -L{} '.format(self.ctx.libs_dir) + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))) if self.ctx.python_recipe.from_crystax: env['LDFLAGS'] = (env['LDFLAGS'] + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))) if self.ctx.python_recipe.name == 'python2legacy': env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh') else: env['LDSHARED'] = env['CC'] + ' -shared' # shprint(sh.whereis, env['LDSHARED'], _env=env) env['LIBLINK'] = 'NOTNONE' env['NDKPLATFORM'] = self.ctx.ndk_platform if self.ctx.copy_libs: env['COPYLIBS'] = '1' # Every recipe uses its own liblink path, object files are # collected and biglinked later liblink_path = join(self.get_build_container_dir(arch.arch), 'objects_{}'.format(self.name)) env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) # Add crystax-specific site packages: if self.ctx.python_recipe.from_crystax: command = sh.Command('python{}'.format(self.ctx.python_recipe.version)) site_packages_dirs = command( '-c', 'import site; print("\\n".join(site.getsitepackages()))') site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n') if 'PYTHONPATH' in env: env['PYTHONPATH'] = env['PYTHONPATH'] +\ ':{}'.format(':'.join(site_packages_dirs)) else: env['PYTHONPATH'] = ':'.join(site_packages_dirs) while env['PYTHONPATH'].find("::") > 0: env['PYTHONPATH'] = env['PYTHONPATH'].replace("::", ":") if env['PYTHONPATH'].endswith(":"): env['PYTHONPATH'] = env['PYTHONPATH'][:-1] if env['PYTHONPATH'].startswith(":"): env['PYTHONPATH'] = env['PYTHONPATH'][1:] return env
def unpack(self, arch): info_main('Unpacking {} for {}'.format(self.name, arch)) build_dir = self.get_build_container_dir(arch) user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower())) if user_dir is not None: info("Installing KivyMD development versoion (from source)") self.clean_build() shprint(sh.rm, '-rf', build_dir) shprint(sh.mkdir, '-p', build_dir) shprint(sh.rmdir, build_dir) ensure_dir(build_dir) ensure_dir(build_dir + "/kivymd") shprint(sh.cp, user_dir + '/setup.py', self.get_build_dir(arch) + "/setup.py") shprint(sh.cp, '-a', user_dir + "/kivymd", self.get_build_dir(arch) + "/kivymd") return
def build_arch(self, arch): # We don't have to actually build anything as CrystaX comes # with the necessary modules. They are included by modifying # the Android.mk in the jni folder. # If the Python version to be used is not prebuilt with the CrystaX # NDK, we do have to download it. crystax_python_dir = join(self.ctx.ndk_dir, 'sources', 'python') if not exists(join(crystax_python_dir, self.version)): info(('The NDK does not have a prebuilt Python {}, trying ' 'to obtain one.').format(self.version)) if self.version not in prebuilt_download_locations: error(('No prebuilt version for Python {} could be found, ' 'the built cannot continue.')) exit(1) with temp_directory() as td: self.download_file(prebuilt_download_locations[self.version], join(td, 'downloaded_python')) shprint(sh.tar, 'xf', join(td, 'downloaded_python'), '--directory', crystax_python_dir) if not exists(join(crystax_python_dir, self.version)): error(('Something went wrong, the directory at {} should ' 'have been created but does not exist.').format( join(crystax_python_dir, self.version))) if not exists(join( crystax_python_dir, self.version, 'libs', arch.arch)): error(('The prebuilt Python for version {} does not contain ' 'binaries for your chosen architecture "{}".').format( self.version, arch.arch)) exit(1) # TODO: We should have an option to build a new Python. This # would also allow linking to openssl and sqlite from CrystaX. dirn = self.ctx.get_python_install_dir() ensure_dir(dirn) # Instead of using a locally built hostpython, we use the # user's Python for now. They must have the right version # available. Using e.g. pyenv makes this easy. self.ctx.hostpython = 'python{}'.format(self.version)
def run_distribute(self): info_main("# Creating Android project ({})".format(self.name)) arch = self.ctx.archs[0] python_install_dir = self.ctx.get_python_install_dir() from_crystax = self.ctx.python_recipe.from_crystax if len(self.ctx.archs) > 1: raise ValueError("SDL2/gradle support only one arch") info("Copying SDL2/gradle build for {}".format(arch)) shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environment variable (ANDROID_HOME) # or the local.properties if exists with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) with current_directory(self.dist_dir): info("Copying Python distribution") python_bundle_dir = join('_python_bundle', '_python_bundle') if 'python2legacy' in self.ctx.recipe_build_order: # a special case with its own packaging location python_bundle_dir = 'private' # And also must had an install directory, make sure of that self.ctx.python_recipe.create_python_install(self.dist_dir) self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(SDL2GradleBootstrap, self).run_distribute()
def build_arch(self, arch): if self.ctx.ndk_api < self.MIN_NDK_API: raise BuildInterruptingException( 'Target ndk-api is {}, but the python3 recipe supports only' ' {}+'.format(self.ctx.ndk_api, self.MIN_NDK_API)) recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, 'android-build') ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does sys_prefix = '/usr/local' sys_exec_prefix = '/usr/local' with current_directory(build_dir): env = self.get_recipe_env(arch) env = self.set_libs_flags(env, arch) android_build = sh.Command( join(recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8') if not exists('config.status'): shprint( sh.Command(join(recipe_build_dir, 'configure')), *(' '.join(self.configure_args).format( android_host=env['HOSTARCH'], android_build=android_build, prefix=sys_prefix, exec_prefix=sys_exec_prefix)).split(' '), _env=env) if not exists('python'): py_version = self.major_minor_version_string if self.major_minor_version_string[0] == '3': py_version += 'm' shprint(sh.make, 'all', 'INSTSONAME=libpython{version}.so'.format( version=py_version), _env=env) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
def setUp(self): """ Setups recipe and context. """ self.context = Context() self.arch = ArchARMv7_a(self.context) self.recipe = Recipe.get_recipe('reportlab', self.context) self.recipe.ctx = self.context self.bootstrap = None recipe_build_order, python_modules, bootstrap = \ get_recipe_order_and_bootstrap( self.context, [self.recipe.name], self.bootstrap) self.context.recipe_build_order = recipe_build_order self.context.python_modules = python_modules self.context.setup_dirs(tempfile.gettempdir()) self.bootstrap = bootstrap self.recipe_dir = self.recipe.get_build_dir(self.arch.arch) ensure_dir(self.recipe_dir)
def get_recipe_env(self, arch, with_flags_in_cc=True): env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format( self.ctx.get_libs_dir(arch.arch) + ' -L{} '.format(self.ctx.libs_dir) + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))) if self.ctx.python_recipe.from_crystax: env['LDFLAGS'] = (env['LDFLAGS'] + ' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch))) # ' -L/home/asandy/.local/share/python-for-android/build/bootstrap_builds/sdl2/libs/armeabi ' if self.ctx.python_recipe.from_crystax: env['LDSHARED'] = env['CC'] + ' -shared' else: env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh') # shprint(sh.whereis, env['LDSHARED'], _env=env) env['LIBLINK'] = 'NOTNONE' env['NDKPLATFORM'] = self.ctx.ndk_platform if self.ctx.copy_libs: env['COPYLIBS'] = '1' # Every recipe uses its own liblink path, object files are # collected and biglinked later liblink_path = join(self.get_build_container_dir(arch.arch), 'objects_{}'.format(self.name)) env['LIBLINK_PATH'] = liblink_path ensure_dir(liblink_path) if self.ctx.python_recipe.from_crystax: env['CFLAGS'] = '-I{} '.format( join(self.ctx.ndk_dir, 'sources', 'python', self.ctx.python_recipe.version, 'include', 'python')) + env['CFLAGS'] # Temporarily hardcode the -lpython3.x as this does not # get applied automatically in some environments. This # will need generalising, along with the other hardcoded # py3.5 references, to support other python3 or crystax # python versions. python3_version = self.ctx.python_recipe.version python3_version = '.'.join(python3_version.split('.')[:2]) env['LDFLAGS'] = env['LDFLAGS'] + ' -lpython{}m'.format(python3_version) return env
def biglink(ctx, arch): # First, collate object files from each recipe info('Collating object files from each recipe') obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects') ensure_dir(obj_dir) recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order] for recipe in recipes: recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch), 'objects_{}'.format(recipe.name)) if not exists(recipe_obj_dir): info('{} recipe has no biglinkable files dir, skipping' .format(recipe.name)) continue files = glob.glob(join(recipe_obj_dir, '*')) if not len(files): info('{} recipe has no biglinkable files, skipping' .format(recipe.name)) continue info('{} recipe has object files, copying'.format(recipe.name)) files.append(obj_dir) shprint(sh.cp, '-r', *files) env = arch.get_env() env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)) if not len(glob.glob(join(obj_dir, '*'))): info('There seem to be no libraries to biglink, skipping.') return info('Biglinking') info('target {}'.format(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'))) do_biglink = copylibs_function if ctx.copy_libs else biglink_function # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? with current_directory(join(ctx.ndk_platform, 'usr', 'lib')): do_biglink( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), extra_link_dirs=[join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch), os.path.abspath('.')], env=env)
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('autoreconf'), '-vif', _env=env) shprint(sh.Command('./configure'), '--host=' + arch.command_prefix, '--prefix=' + self.get_build_dir(arch.arch), '--disable-builddir', '--enable-shared', _env=env) shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env) host_build = self.get_build_dir(arch.arch) ensure_dir(self.ctx.get_libs_dir(arch.arch)) shprint(sh.cp, join(host_build, '.libs', 'libffi.so'), self.ctx.get_libs_dir(arch.arch))
def copy_files(self, arch): ndk = self.ctx.ndk_dir env = self.get_recipe_env(arch) lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" lib = lib.format(ndk=self.ctx.ndk_dir, version=env["TOOLCHAIN_VERSION"], arch=arch.arch) stl_lib = join(lib, "libgnustl_shared.so") dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload") shprint(sh.cp, stl_lib, dst_dir) src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib") dst_lib = self.get_lib_dir(arch) src_suffix = "." + self.version dst_suffix = "." + self.version.split(".")[0] # main version for lib in self.generated_libraries: shprint(sh.cp, join(src_lib, lib+src_suffix), join(dst_lib, lib+dst_suffix)) src_include = join( self.get_build_dir(arch.arch), "icu_build", "include") dst_include = join( self.ctx.get_python_install_dir(), "include", "icu") ensure_dir(dst_include) shprint(sh.cp, "-r", join(src_include, "layout"), dst_include) shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include) # copy stl library lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}" lib = lib.format(ndk=self.ctx.ndk_dir, version=env["TOOLCHAIN_VERSION"], arch=arch.arch) stl_lib = join(lib, "libgnustl_shared.so") dst_dir = join(self.ctx.get_python_install_dir(), "lib") ensure_dir(dst_dir) shprint(sh.cp, stl_lib, dst_dir)
def prebuild_arch(self, arch): if not self.is_patched(arch): super(ReportLabRecipe, self).prebuild_arch(arch) recipe_dir = self.get_build_dir(arch.arch) # Some versions of reportlab ship with a GPL-licensed font. # Remove it, since this is problematic in .apks unless the # entire app is GPL: font_dir = os.path.join(recipe_dir, "src", "reportlab", "fonts") if os.path.exists(font_dir): for l in os.listdir(font_dir): if l.lower().startswith('darkgarden'): os.remove(os.path.join(font_dir, l)) # Apply patches: self.apply_patch('patches/fix-setup.patch', arch.arch) shprint(sh.touch, os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include')) tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp")) info('reportlab recipe: recipe_dir={}'.format(recipe_dir)) info('reportlab recipe: tmp_dir={}'.format(tmp_dir)) info('reportlab recipe: ft_dir={}'.format(ft_dir)) info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir)) info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir)) with current_directory(recipe_dir): ensure_dir(tmp_dir) pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip") if not os.path.isfile(pfbfile): sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile) sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile) if os.path.isfile("setup.py"): with open('setup.py', 'r') as f: text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir) with open('setup.py', 'w') as f: f.write(text)
def biglink(ctx, arch): # First, collate object files from each recipe info('Collating object files from each recipe') obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects') ensure_dir(obj_dir) recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order] for recipe in recipes: recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch), 'objects_{}'.format(recipe.name)) if not exists(recipe_obj_dir): info('{} recipe has no biglinkable files dir, skipping' .format(recipe.name)) continue files = glob.glob(join(recipe_obj_dir, '*')) if not len(files): info('{} recipe has no biglinkable files, skipping' .format(recipe.name)) info('{} recipe has object files, copying'.format(recipe.name)) files.append(obj_dir) shprint(sh.cp, '-r', *files) env = arch.get_env() env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)) if not len(glob.glob(join(obj_dir, '*'))): info('There seem to be no libraries to biglink, skipping.') return info('Biglinking') info('target {}'.format(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'))) biglink_function( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), extra_link_dirs=[join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)], env=env)
def setup_dirs(self): '''Calculates all the storage and build dirs, and makes sure the directories exist where necessary.''' self.root_dir = realpath(dirname(__file__)) # AND: TODO: Allow the user to set the build_dir self.storage_dir = user_data_dir('python-for-android') self.build_dir = join(self.storage_dir, 'build') self.dist_dir = join(self.storage_dir, 'dists') ensure_dir(self.storage_dir) ensure_dir(self.build_dir) ensure_dir(self.dist_dir)
def build_recipes(build_order, python_modules, ctx, project_dir, ignore_project_setup_py=False): # Put recipes in correct build order info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) recipe.install_libraries(arch) else: info('{} said it is already built, skipping'.format( recipe.name)) # 4) biglink everything info_main('# Biglinking object files') if not ctx.python_recipe: biglink(ctx, arch) else: warning("Context's python recipe found, " "skipping biglink (will this work?)") # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules, project_dir, ignore_setup_py=ignore_project_setup_py) return
def run_distribute(self): info_main("# Creating Android project ({})".format(self.name)) arch = self.ctx.archs[0] python_install_dir = self.ctx.get_python_install_dir() from_crystax = self.ctx.python_recipe.from_crystax if len(self.ctx.archs) > 1: raise ValueError("SDL2/gradle support only one arch") info("Copying SDL2/gradle build for {}".format(arch)) shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environment variable (ANDROID_HOME) # or the local.properties if exists with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) with current_directory(self.dist_dir): info("Copying Python distribution") hostpython = sh.Command(self.ctx.hostpython) if self.ctx.python_recipe.name == 'python2': try: shprint(hostpython, '-OO', '-m', 'compileall', '-q', python_install_dir, _tail=10, _filterout="^Listing") except sh.ErrorReturnCode: pass if 'python2' in self.ctx.recipe_build_order and not exists( 'python-install'): shprint(sh.cp, '-a', python_install_dir, './python-install') self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) python_bundle_dir = join('_python_bundle', '_python_bundle') if 'python2' in self.ctx.recipe_build_order: # Python 2 is a special case with its own packaging location python_bundle_dir = 'private' ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(SDL2GradleBootstrap, self).run_distribute()
def get_lib_dir(self, arch): lib_dir = join(self.ctx.get_python_install_dir(), "lib") ensure_dir(lib_dir) return lib_dir
def prepare_dist_dir(self, name): ensure_dir(self.dist_dir)
def distribute_javaclasses(self, javaclass_dir, dest_dir="src"): '''Copy existing javaclasses from build dir to current dist dir.''' info('Copying java files') ensure_dir(dest_dir) filenames = glob.glob(javaclass_dir) shprint(sh.cp, '-a', *filenames, dest_dir)
def javaclass_dir(self): # Was previously hardcoded as self.build_dir/java dir = join(self.build_dir, 'javaclasses', self.bootstrap.distribution.name) ensure_dir(dir) return dir
def aars_dir(self): dir = join(self.build_dir, 'aars', self.bootstrap.distribution.name) ensure_dir(dir) return dir
def libs_dir(self): # Was previously hardcoded as self.build_dir/libs dir = join(self.build_dir, 'libs_collections', self.bootstrap.distribution.name) ensure_dir(dir) return dir
def get_libs_dir(self, arch): '''The libs dir for a given arch.''' ensure_dir(join(self.libs_dir, arch)) return join(self.libs_dir, arch)
def run_distribute(self): info_main("# Creating Android project ({})".format(self.name)) arch = self.ctx.archs[0] python_install_dir = self.ctx.get_python_install_dir() from_crystax = self.ctx.python_recipe.from_crystax crystax_python_dir = join("crystax_python", "crystax_python") if len(self.ctx.archs) > 1: raise ValueError("SDL2/gradle support only one arch") info("Copying SDL2/gradle build for {}".format(arch)) shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) # either the build use environemnt variable (ANDROID_HOME) # or the local.properties if exists with current_directory(self.dist_dir): with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) with current_directory(self.dist_dir): info("Copying Python distribution") if not exists("private") and not from_crystax: ensure_dir("private") if not exists("crystax_python") and from_crystax: ensure_dir(crystax_python_dir) hostpython = sh.Command(self.ctx.hostpython) if not from_crystax: try: shprint(hostpython, '-OO', '-m', 'compileall', python_install_dir, _tail=10, _filterout="^Listing") except sh.ErrorReturnCode: pass if not exists('python-install'): shprint(sh.cp, '-a', python_install_dir, './python-install') self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) if not from_crystax: info("Filling private directory") if not exists(join("private", "lib")): info("private/lib does not exist, making") shprint(sh.cp, "-a", join("python-install", "lib"), "private") shprint(sh.mkdir, "-p", join("private", "include", "python2.7")) libpymodules_fn = join("libs", arch.arch, "libpymodules.so") if exists(libpymodules_fn): shprint(sh.mv, libpymodules_fn, 'private/') shprint( sh.cp, join('python-install', 'include', 'python2.7', 'pyconfig.h'), join('private', 'include', 'python2.7/')) info('Removing some unwanted files') shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so')) shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig')) libdir = join(self.dist_dir, 'private', 'lib', 'python2.7') site_packages_dir = join(libdir, 'site-packages') with current_directory(libdir): removes = [] for dirname, root, filenames in walk("."): for filename in filenames: for suffix in EXCLUDE_EXTS: if filename.endswith(suffix): removes.append(filename) shprint(sh.rm, '-f', *removes) info('Deleting some other stuff not used on android') # To quote the original distribute.sh, 'well...' shprint(sh.rm, '-rf', 'idlelib') for filename in glob.glob('config/libpython*.a'): shprint(sh.rm, '-f', filename) shprint(sh.rm, '-rf', 'config/python.o') else: # Python *is* loaded from crystax ndk_dir = self.ctx.ndk_dir py_recipe = self.ctx.python_recipe python_dir = join(ndk_dir, 'sources', 'python', py_recipe.version, 'libs', arch.arch) shprint(sh.cp, '-r', join(python_dir, 'stdlib.zip'), crystax_python_dir) shprint(sh.cp, '-r', join(python_dir, 'modules'), crystax_python_dir) shprint(sh.cp, '-r', self.ctx.get_python_install_dir(), join(crystax_python_dir, 'site-packages')) info('Renaming .so files to reflect cross-compile') site_packages_dir = join(crystax_python_dir, "site-packages") find_ret = shprint(sh.find, site_packages_dir, '-iname', '*.so') filenames = find_ret.stdout.decode('utf-8').split('\n')[:-1] for filename in filenames: parts = filename.split('.') if len(parts) <= 2: continue shprint(sh.mv, filename, filename.split('.')[0] + '.so') site_packages_dir = join(abspath(curdir), site_packages_dir) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(SDL2GradleBootstrap, self).run_distribute()
def ensure_dirs(self): ensure_dir(self.storage_dir) ensure_dir(self.build_dir) ensure_dir(self.dist_dir) ensure_dir(join(self.build_dir, 'bootstrap_builds')) ensure_dir(join(self.build_dir, 'other_builds'))
def create_python_bundle(self, dirn, arch): """ Create a packaged python bundle in the target directory, by copying all the modules and standard library to the right place. """ # Todo: find a better way to find the build libs folder modules_build_dir = join( self.get_build_dir(arch.arch), "android-build", "build", "lib.linux{}-{}-{}".format( "2" if self.version[0] == "2" else "", arch.command_prefix.split("-")[0], self.major_minor_version_string, ), ) # Compile to *.pyc/*.pyo the python modules self.compile_python_files(modules_build_dir) # Compile to *.pyc/*.pyo the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), "Lib")) # Compile to *.pyc/*.pyo the other python packages (site-packages) self.compile_python_files(self.ctx.get_python_install_dir()) # Bundle compiled python modules to a folder modules_dir = join(dirn, "modules") c_ext = self.compiled_extension ensure_dir(modules_dir) module_filens = glob.glob(join(modules_build_dir, "*.so")) + glob.glob( join(modules_build_dir, "*" + c_ext)) info("Copy {} files into the bundle".format(len(module_filens))) for filen in module_filens: info(" - copy {}".format(filen)) copy2(filen, modules_dir) # zip up the standard library stdlib_zip = join(dirn, "stdlib.zip") with current_directory(join(self.get_build_dir(arch.arch), "Lib")): stdlib_filens = list( walk_valid_filens(".", self.stdlib_dir_blacklist, self.stdlib_filen_blacklist)) info("Zip {} files into the bundle".format(len(stdlib_filens))) shprint(sh.zip, stdlib_zip, *stdlib_filens) # copy the site-packages into place ensure_dir(join(dirn, "site-packages")) ensure_dir(self.ctx.get_python_install_dir()) # TODO: Improve the API around walking and copying the files with current_directory(self.ctx.get_python_install_dir()): filens = list( walk_valid_filens( ".", self.site_packages_dir_blacklist, self.site_packages_filen_blacklist, )) info("Copy {} files into the site-packages".format(len(filens))) for filen in filens: info(" - copy {}".format(filen)) ensure_dir(join(dirn, "site-packages", dirname(filen))) copy2(filen, join(dirn, "site-packages", filen)) # copy the python .so files into place python_build_dir = join(self.get_build_dir(arch.arch), "android-build") python_lib_name = "libpython" + self.major_minor_version_string if self.major_minor_version_string[0] == "3": python_lib_name += "m" shprint( sh.cp, join(python_build_dir, python_lib_name + ".so"), join(self.ctx.bootstrap.dist_dir, "libs", arch.arch), ) info("Renaming .so files to reflect cross-compile") self.reduce_object_file_names(join(dirn, "site-packages")) return join(dirn, "site-packages")
def create_python_bundle(self, dirn, arch): """ Create a packaged python bundle in the target directory, by copying all the modules and standard library to the right place. """ # Todo: find a better way to find the build libs folder modules_build_dir = join( self.get_build_dir(arch.arch), 'android-build', 'build', 'lib.linux{}-{}-{}'.format('2' if self.version[0] == '2' else '', arch.command_prefix.split('-')[0], self.major_minor_version_string)) # Compile to *.pyc/*.pyo the python modules self.compile_python_files(modules_build_dir) # Compile to *.pyc/*.pyo the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib')) # Compile to *.pyc/*.pyo the other python packages (site-packages) self.compile_python_files(self.ctx.get_python_install_dir()) # Bundle compiled python modules to a folder modules_dir = join(dirn, 'modules') c_ext = self.compiled_extension ensure_dir(modules_dir) module_filens = (glob.glob(join(modules_build_dir, '*.so')) + glob.glob(join(modules_build_dir, '*' + c_ext))) info("Copy {} files into the bundle".format(len(module_filens))) for filen in module_filens: info(" - copy {}".format(filen)) copy2(filen, modules_dir) # zip up the standard library stdlib_zip = join(dirn, 'stdlib.zip') with current_directory(join(self.get_build_dir(arch.arch), 'Lib')): stdlib_filens = list( walk_valid_filens('.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist)) info("Zip {} files into the bundle".format(len(stdlib_filens))) shprint(sh.zip, stdlib_zip, *stdlib_filens) # copy the site-packages into place ensure_dir(join(dirn, 'site-packages')) ensure_dir(self.ctx.get_python_install_dir()) # TODO: Improve the API around walking and copying the files with current_directory(self.ctx.get_python_install_dir()): filens = list( walk_valid_filens('.', self.site_packages_dir_blacklist, self.site_packages_filen_blacklist)) info("Copy {} files into the site-packages".format(len(filens))) for filen in filens: info(" - copy {}".format(filen)) ensure_dir(join(dirn, 'site-packages', dirname(filen))) copy2(filen, join(dirn, 'site-packages', filen)) # copy the python .so files into place python_build_dir = join(self.get_build_dir(arch.arch), 'android-build') python_lib_name = 'libpython' + self.major_minor_version_string if self.major_minor_version_string[0] == '3': python_lib_name += 'm' shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'), join(self.ctx.dist_dir, self.ctx.dist_name, 'libs', arch.arch)) info('Renaming .so files to reflect cross-compile') self.reduce_object_file_names(join(dirn, 'site-packages')) return join(dirn, 'site-packages')
def build_arch(self, arch): if self.ctx.ndk_api < self.MIN_NDK_API: raise BuildInterruptingException( 'Target ndk-api is {}, but the python3 recipe supports only {}+' .format(self.ctx.ndk_api, self.MIN_NDK_API)) recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, 'android-build') ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does sys_prefix = '/usr/local' sys_exec_prefix = '/usr/local' # Skipping "Ensure that nl_langinfo is broken" from the original bpo-30386 platform_name = 'android-{}'.format(self.ctx.ndk_api) with current_directory(build_dir): env = environ.copy() # TODO: Get this information from p4a's arch system android_host = arch.command_prefix android_build = sh.Command( join(recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8') platform_dir = join(self.ctx.ndk_dir, 'platforms', platform_name, arch.platform_dir) toolchain = '{android_host}-4.9'.format( android_host=arch.toolchain_prefix) toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', 'linux-x86_64') target_data = arch.command_prefix.split('-') if target_data[0] == 'arm': target_data[0] = 'armv7a' target = '-'.join( [target_data[0], 'none', target_data[1], target_data[2]]) CC = '{clang} -target {target} -gcc-toolchain {toolchain}'.format( clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt', 'linux-x86_64', 'bin', 'clang'), target=target, toolchain=toolchain) AR = join(toolchain, 'bin', android_host) + '-ar' LD = join(toolchain, 'bin', android_host) + '-ld' RANLIB = join(toolchain, 'bin', android_host) + '-ranlib' READELF = join(toolchain, 'bin', android_host) + '-readelf' STRIP = join( toolchain, 'bin', android_host) + '-strip --strip-debug --strip-unneeded' env['CC'] = CC env['AR'] = AR env['LD'] = LD env['RANLIB'] = RANLIB env['READELF'] = READELF env['STRIP'] = STRIP env['PATH'] = '{hostpython_dir}:{old_path}'.format( hostpython_dir=self.get_recipe('hostpython3', self.ctx).get_path_to_python(), old_path=env['PATH']) ndk_flags = ( '-fPIC --sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} ' '-isystem {ndk_android_host}').format( ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'), android_api=self.ctx.ndk_api, ndk_android_host=join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include', android_host)) sysroot = join(self.ctx.ndk_dir, 'platforms', platform_name, arch.platform_dir) env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format( sysroot, join(sysroot, 'usr', 'lib')) # Manually add the libs directory, and copy some object # files to the current directory otherwise they aren't # picked up. This seems necessary because the --sysroot # setting in LDFLAGS is overridden by the other flags. # TODO: Work out why this doesn't happen in the original # bpo-30386 Makefile system. logger.warning('Doing some hacky stuff to link properly') lib_dir = join(sysroot, 'usr', 'lib') if arch.arch == 'x86_64': lib_dir = join(sysroot, 'usr', 'lib64') env['LDFLAGS'] += ' -L{}'.format(lib_dir) shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './') shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './') env['SYSROOT'] = sysroot env = self.set_libs_flags(env, arch) if not exists('config.status'): shprint(sh.Command(join(recipe_build_dir, 'configure')), *(' '.join( ('--host={android_host}', '--build={android_build}', '--enable-shared', '--disable-ipv6', 'ac_cv_file__dev_ptmx=yes', 'ac_cv_file__dev_ptc=no', '--without-ensurepip', 'ac_cv_little_endian_double=yes', '--prefix={prefix}', '--exec-prefix={exec_prefix}')).format( android_host=android_host, android_build=android_build, prefix=sys_prefix, exec_prefix=sys_exec_prefix)).split(' '), _env=env) if not exists('python'): shprint(sh.make, 'all', _env=env) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
def unpack(self, arch): info_main('Unpacking {} for {}'.format(self.name, arch)) build_dir = self.get_build_container_dir(arch) user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower())) if user_dir is not None: info('P4A_{}_DIR exists, symlinking instead'.format( self.name.lower())) if exists(self.get_build_dir(arch)): return shprint(sh.rm, '-rf', build_dir) shprint(sh.mkdir, '-p', build_dir) shprint(sh.rmdir, build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return if self.url is None: info('Skipping {} unpack as no URL is set'.format(self.name)) return filename = shprint(sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename) if ma: # fragmented URL? filename = ma.group(1) with current_directory(build_dir): directory_name = self.get_build_dir(arch) if not exists(directory_name) or not isdir(directory_name): extraction_filename = join(self.ctx.packages_path, self.name, filename) if isfile(extraction_filename): info('File type for {}'.format(extraction_filename)) os.system('file ' + extraction_filename) if extraction_filename.endswith('.zip'): try: sh.unzip(extraction_filename) except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2): # return code 1 means unzipping had # warnings but did complete, # apparently happens sometimes with # github zips pass import zipfile fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split( '/')[0] if root_directory != basename(directory_name): shprint(sh.mv, root_directory, directory_name) elif extraction_filename.endswith( ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')): sh.tar('xvf', extraction_filename) root_directory = sh.tar( 'tf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != basename(directory_name): shprint(sh.mv, root_directory, directory_name) else: raise Exception( 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2 or .tar.xz'.format( extraction_filename)) elif isdir(extraction_filename): mkdir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git', ): shprint(sh.cp, '-Rv', join(extraction_filename, entry), directory_name) else: raise Exception( 'Given path is neither a file nor a directory: {}'. format(extraction_filename)) else: info('{} is already unpacked, skipping'.format(self.name))
def prepare_dist_dir(self): ensure_dir(self.dist_dir)
def run_distribute(self): info_main( '# Creating Android project from build and {} bootstrap'.format( self.name)) # src_path = join(self.ctx.root_dir, 'bootstrap_templates', # self.name) src_path = join(self.bootstrap_dir, 'build') arch = self.ctx.archs[0] if len(self.ctx.archs) > 1: raise ValueError( 'built for more than one arch, but bootstrap cannot handle that yet' ) info('Bootstrap running with arch {}'.format(arch)) with current_directory(self.dist_dir): info('Creating initial layout') for dirname in ('assets', 'bin', 'private', 'res', 'templates'): if not exists(dirname): shprint(sh.mkdir, dirname) info('Copying default files') shprint(sh.cp, '-a', join(self.build_dir, 'project.properties'), '.') shprint(sh.cp, '-a', join(src_path, 'build.py'), '.') shprint(sh.cp, '-a', join(src_path, 'buildlib'), '.') shprint(sh.cp, '-a', join(src_path, 'src'), '.') shprint(sh.cp, '-a', join(src_path, 'templates'), '.') shprint(sh.cp, '-a', join(src_path, 'res'), '.') shprint(sh.cp, '-a', join(src_path, 'blacklist.txt'), '.') shprint(sh.cp, '-a', join(src_path, 'whitelist.txt'), '.') with open('local.properties', 'w') as fileh: fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir)) info('Copying python distribution') python_bundle_dir = join('_python_bundle', '_python_bundle') if 'python2legacy' in self.ctx.recipe_build_order: # a special case with its own packaging location python_bundle_dir = 'private' # And also must had an install directory, make sure of that self.ctx.python_recipe.create_python_install(self.dist_dir) self.distribute_libs(arch, [ join(self.build_dir, 'libs', arch.arch), self.ctx.get_libs_dir(arch.arch) ]) self.distribute_aars(arch) self.distribute_javaclasses(self.ctx.javaclass_dir) ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') self.strip_libraries(arch) self.fry_eggs(site_packages_dir) super(PygameBootstrap, self).run_distribute()
def unpack(self, arch): info_main('Unpacking {} for {}'.format(self.name, arch)) build_dir = self.get_build_container_dir(arch) user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower())) if user_dir is not None: info('P4A_{}_DIR exists, symlinking instead'.format( self.name.lower())) # AND: Currently there's something wrong if I use ln, fix this warning('Using cp -a instead of symlink...fix this!') if exists(self.get_build_dir(arch)): return shprint(sh.rm, '-rf', build_dir) shprint(sh.mkdir, '-p', build_dir) shprint(sh.rmdir, build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return if self.url is None: info('Skipping {} unpack as no URL is set'.format(self.name)) return filename = shprint(sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') with current_directory(build_dir): directory_name = self.get_build_dir(arch) # AND: Could use tito's get_archive_rootdir here if not exists(directory_name) or not isdir(directory_name): extraction_filename = join(self.ctx.packages_path, self.name, filename) if isfile(extraction_filename): if extraction_filename.endswith('.tar.gz') or \ extraction_filename.endswith('.tgz'): sh.tar('xzf', extraction_filename) root_directory = shprint( sh.tar, 'tzf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != directory_name: shprint(sh.mv, root_directory, directory_name) elif (extraction_filename.endswith('.tar.bz2') or extraction_filename.endswith('.tbz2')): info('Extracting {} at {}'.format( extraction_filename, filename)) sh.tar('xjf', extraction_filename) root_directory = sh.tar( 'tjf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != directory_name: shprint(sh.mv, root_directory, directory_name) elif extraction_filename.endswith('.zip'): sh.unzip(extraction_filename) import zipfile fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split( '/')[0] if root_directory != directory_name: shprint(sh.mv, root_directory, directory_name) else: raise Exception( 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2') elif isdir(extraction_filename): mkdir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git', ): shprint(sh.cp, '-Rv', join(extraction_filename, entry), directory_name) else: raise Exception( 'Given path is neither a file nor a directory: {}'. format(extraction_filename)) else: info('{} is already unpacked, skipping'.format(self.name))
def python_installs_dir(self): dir = join(self.build_dir, 'python-installs') ensure_dir(dir) return dir
def prepare_dist_dir(self, name): # self.dist_dir = self.get_dist_dir(name) ensure_dir(self.dist_dir)
def build_recipes(build_order, python_modules, ctx): # Put recipes in correct build order bs = ctx.bootstrap info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: if ctx.P4A_force_build and not recipe.force_build: info_main( 'recipe {0} not marked for force build, skip download'.format( recipe.name)) continue recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: if ctx.P4A_force_build and not recipe.force_build: info_main('recipe {0} not marked for force build, skip unpack'. format(recipe.name)) continue ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.force_build or recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping'.format( recipe.name)) # 4) biglink everything # AND: Should make this optional info_main('# Biglinking object files') if not ctx.python_recipe or not ctx.python_recipe.from_crystax: biglink(ctx, arch) else: info('NDK is crystax, skipping biglink (will this work?)') # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules) return