def build_arch(self, arch): recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, self.build_subdir) ensure_dir(build_dir) if not exists(join(build_dir, 'python')): with current_directory(recipe_build_dir): # Configure the build with current_directory(build_dir): if not exists('config.status'): shprint( sh.Command(join(recipe_build_dir, 'configure'))) # Create the Setup file. This copying from Setup.dist # seems to be the normal and expected procedure. shprint(sh.cp, join('Modules', 'Setup.dist'), join(build_dir, 'Modules', 'Setup')) result = shprint(sh.make, '-C', build_dir) else: info('Skipping {name} ({version}) build, as it has already ' 'been completed'.format(name=self.name, version=self.version)) self.ctx.hostpython = join(build_dir, 'python')
def build_arch(self, arch): info("Extracting CrystaX python3 from NDK package") dirn = self.ctx.get_python_install_dir() ensure_dir(dirn) self.ctx.hostpython = "python{}".format(self.version)
def clean_bootstrap_builds(self, args): '''Delete all the bootstrap builds.''' for bs in Bootstrap.list_bootstraps(): bs = Bootstrap.get_bootstrap(bs, self.ctx) if bs.build_dir and exists(bs.build_dir): info('Cleaning build for {} bootstrap.'.format(bs.name)) shutil.rmtree(bs.build_dir)
def clean_build(self, arch=None): """Deletes all the build information of the recipe. If arch is not None, only this arch dir is deleted. Otherwise (the default) all builds for all archs are deleted. By default, this just deletes the main build dir. If the recipe has e.g. object files biglinked, or .so files stored elsewhere, you should override this method. This method is intended for testing purposes, it may have strange results. Rebuild everything if this seems to happen. """ if arch is None: base_dir = join(self.ctx.build_dir, "other_builds", self.name) else: base_dir = self.get_build_container_dir(arch) dirs = glob.glob(base_dir + "-*") if exists(base_dir): dirs.append(base_dir) if not dirs: warning(("Attempted to clean build for {} but found no existing " "build dirs").format(self.name)) for directory in dirs: if exists(directory): info("Deleting {}".format(directory)) shutil.rmtree(directory)
def download_if_necessary(self): info_main("Downloading {}".format(self.name)) user_dir = environ.get("P4A_{}_DIR".format(self.name.lower())) if user_dir is not None: info("P4A_{}_DIR is set, skipping download for {}".format(self.name, self.name)) return self.download()
def build_recipes(build_order, python_modules, ctx): # Put recipes in correct build order bs = ctx.bootstrap info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) # 4) biglink everything # AND: Should make this optional info_main('# Biglinking object files') if not ctx.python_recipe or not ctx.python_recipe.from_crystax: biglink(ctx, arch) else: info('NDK is crystax, skipping biglink (will this work?)') # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules) return
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) try: shprint(hostpython, 'setup.py', 'build_ext', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) info('Running cython where appropriate') shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx', '-exec', self.ctx.cython, '{}', ';', _env=env) info('ran cython') shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) print('stripping') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) print('stripped!?')
def extract_source(self, source, cwd): """ (internal) Extract the `source` into the directory `cwd`. """ if not source: return if isfile(source): info("Extract {} into {}".format(source, cwd)) if source.endswith(".tgz") or source.endswith(".tar.gz"): shprint(sh.tar, "-C", cwd, "-xvzf", source) elif source.endswith(".tbz2") or source.endswith(".tar.bz2"): shprint(sh.tar, "-C", cwd, "-xvjf", source) elif source.endswith(".zip"): zf = zipfile.ZipFile(source) zf.extractall(path=cwd) zf.close() else: warning("Error: cannot extract, unrecognized extension for {}".format(source)) raise Exception() elif isdir(source): info("Copying {} into {}".format(source, cwd)) shprint(sh.cp, "-a", source, cwd) else: warning("Error: cannot extract or copy, unrecognized path {}".format(source)) raise Exception()
def clean_build(self, arch=None): '''Deletes all the build information of the recipe. If arch is not None, only this arch dir is deleted. Otherwise (the default) all builds for all archs are deleted. By default, this just deletes the main build dir. If the recipe has e.g. object files biglinked, or .so files stored elsewhere, you should override this method. This method is intended for testing purposes, it may have strange results. Rebuild everything if this seems to happen. ''' if arch is None: base_dir = join(self.ctx.build_dir, 'other_builds', self.name) else: base_dir = self.get_build_container_dir(arch) dirs = glob.glob(base_dir + '-*') if exists(base_dir): dirs.append(base_dir) if not dirs: warning(('Attempted to clean build for {} but found no existing ' 'build dirs').format(self.name)) for directory in dirs: if exists(directory): info('Deleting {}'.format(directory)) shutil.rmtree(directory) # Delete any Python distributions to ensure the recipe build # doesn't persist in site-packages shutil.rmtree(self.ctx.python_installs_dir)
def rebuild_compiled_components(self, arch, env): info('Rebuilding compiled components in {}'.format(self.name)) hostpython = sh.Command(self.real_hostpython_location) shprint(hostpython, 'setup.py', 'clean', '--all', _env=env) shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env, *self.setup_extra_args)
def should_build(self, arch): name = self.folder_name if self.ctx.has_package(name): info('Python package already exists in site-packages') return False info('{} apparently isn\'t already in site-packages'.format(name)) return True
def build_dist_from_args(ctx, dist, args): '''Parses out any bootstrap related arguments, and uses them to build a dist.''' bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) build_order, python_modules, bs \ = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs) ctx.recipe_build_order = build_order info('The selected bootstrap is {}'.format(bs.name)) info_main('# Creating dist with {} bootstrap'.format(bs.name)) bs.distribution = dist info_notify('Dist will have name {} and recipes ({})'.format( dist.name, ', '.join(dist.recipes))) ctx.dist_name = bs.distribution.name ctx.prepare_bootstrap(bs) ctx.prepare_dist(ctx.dist_name) build_recipes(build_order, python_modules, ctx) ctx.bootstrap.run_distribute() info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' .format(join(ctx.dist_dir, ctx.dist_name)))
def prebuild_arch(self, arch): if not self.is_patched(arch): super(ReportLabRecipe, self).prebuild_arch(arch) self.apply_patch('patches/fix-setup.patch', arch.arch) recipe_dir = self.get_build_dir(arch.arch) shprint(sh.touch, os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include')) tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp")) info('reportlab recipe: recipe_dir={}'.format(recipe_dir)) info('reportlab recipe: tmp_dir={}'.format(tmp_dir)) info('reportlab recipe: ft_dir={}'.format(ft_dir)) info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir)) info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir)) with current_directory(recipe_dir): sh.ls('-lathr') ensure_dir(tmp_dir) pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip") if not os.path.isfile(pfbfile): sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile) sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile) if os.path.isfile("setup.py"): with open('setup.py', 'rb') as f: text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir) with open('setup.py', 'wb') as f: f.write(text)
def build_arch(self, arch): """simple shared compile""" env = self.get_recipe_env(arch, with_flags_in_cc=False) for path in ( self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): if not exists(path): info("creating {}".format(path)) shprint(sh.mkdir, '-p', path) cli = env['CC'].split()[0] # makes sure first CC command is the compiler rather than ccache, refs: # https://github.com/kivy/python-for-android/issues/1399 if 'ccache' in cli: cli = env['CC'].split()[1] cc = sh.Command(cli) with current_directory(self.get_build_dir(arch.arch)): cflags = env['CFLAGS'].split() cflags.extend(['-I.', '-c', '-l.', 'glob.c', '-I.']) shprint(cc, *cflags, _env=env) cflags = env['CFLAGS'].split() cflags.extend(['-shared', '-I.', 'glob.o', '-o', 'libglob.so']) cflags.extend(env['LDFLAGS'].split()) shprint(cc, *cflags, _env=env) shprint(sh.cp, 'libglob.so', join(self.ctx.libs_dir, arch.arch))
def strip_libraries(self, arch): info('Stripping libraries') if self.ctx.python_recipe.from_crystax: info('Python was loaded from CrystaX, skipping strip') return env = arch.get_env() tokens = shlex.split(env['STRIP']) strip = sh.Command(tokens[0]) if len(tokens) > 1: strip = strip.bake(tokens[1:]) libs_dir = join(self.dist_dir, '_python_bundle', '_python_bundle', 'modules') if self.ctx.python_recipe.name == 'python2legacy': libs_dir = join(self.dist_dir, 'private') filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'), '-iname', '*.so', _env=env).stdout.decode('utf-8') logger.info('Stripping libraries in private dir') for filen in filens.split('\n'): if not filen: continue # skip the last '' try: strip(filen, _env=env) except sh.ErrorReturnCode_1: logger.debug('Failed to strip ' + filen)
def set_libs_flags(self, env, arch): '''Takes care to properly link libraries with python depending on our requirements and the attribute :attr:`opt_depends`. ''' if 'libffi' in self.ctx.recipe_build_order: info('Activating flags for libffi') recipe = Recipe.get_recipe('libffi', self.ctx) include = ' -I' + ' -I'.join(recipe.get_include_dirs(arch)) ldflag = ' -L' + join(recipe.get_build_dir(arch.arch), recipe.get_host(arch), '.libs') + ' -lffi' env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include env['LDFLAGS'] = env.get('LDFLAGS', '') + ldflag if 'openssl' in self.ctx.recipe_build_order: recipe = Recipe.get_recipe('openssl', self.ctx) openssl_build_dir = recipe.get_build_dir(arch.arch) setuplocal = join('Modules', 'Setup.local') shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal) shprint(sh.sed, '-i.backup', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal) env['OPENSSL_VERSION'] = recipe.version if 'sqlite3' in self.ctx.recipe_build_order: # Include sqlite3 in python2 build recipe = Recipe.get_recipe('sqlite3', self.ctx) include = ' -I' + recipe.get_build_dir(arch.arch) lib = ' -L' + recipe.get_lib_dir(arch) + ' -lsqlite3' # Insert or append to env flag = 'CPPFLAGS' env[flag] = env[flag] + include if flag in env else include flag = 'LDFLAGS' env[flag] = env[flag] + lib if flag in env else lib return env
def apk(self, args): '''Create an APK using the given distribution.''' # AND: Need to add a parser here for any extra options # parser = argparse.ArgumentParser( # description='Build an APK') # args = parser.parse_args(args) ctx = self.ctx dist = self._dist # Manually fixing these arguments at the string stage is # unsatisfactory and should probably be changed somehow, but # we can't leave it until later as the build.py scripts assume # they are in the current directory. for i, arg in enumerate(args[:-1]): if arg in ('--dir', '--private'): args[i+1] = realpath(expanduser(args[i+1])) build = imp.load_source('build', join(dist.dist_dir, 'build.py')) with current_directory(dist.dist_dir): build.parse_args(args) shprint(sh.ant, 'debug', _tail=20, _critical=True) # AND: This is very crude, needs improving. Also only works # for debug for now. info_main('# Copying APK to current directory') apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-debug.apk')) if len(apks) == 0: raise ValueError('Couldn\'t find the built APK') if len(apks) > 1: info('More than one built APK found...guessing you ' 'just built {}'.format(apks[-1])) shprint(sh.cp, apks[-1], './')
def build_arch(self, arch): env = self.get_recipe_env(arch) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format( jni_path=join(self.ctx.bootstrap.build_dir, 'jni')) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl/include -I{jni_path}/sdl_mixer'.format( jni_path=join(self.ctx.bootstrap.build_dir, 'jni')) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl_ttf -I{jni_path}/sdl_image'.format( jni_path=join(self.ctx.bootstrap.build_dir, 'jni')) debug('pygame cflags', env['CFLAGS']) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{libs_path} -L{src_path}/obj/local/{arch} -lm -lz'.format( libs_path=self.ctx.libs_dir, src_path=self.ctx.bootstrap.build_dir, arch=env['ARCH']) env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink') with current_directory(self.get_build_dir(arch.arch)): info('hostpython is ' + self.ctx.hostpython) hostpython = sh.Command(self.ctx.hostpython) shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, _tail=10, _critical=True) info('strip is ' + env['STRIP']) build_lib = glob.glob('./build/lib*') assert len(build_lib) == 1 print('stripping pygame') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';') python_install_path = join(self.ctx.build_dir, 'python-install') warning('Should remove pygame tests etc. here, but skipping for now')
def delete_dist(self, _args): dist = self._dist if not dist.folder_exists(): info('No dist exists that matches your specifications, ' 'exiting without deleting.') return dist.delete()
def build_cython_components(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): info('hostpython is ' + self.ctx.hostpython) hostpython = sh.Command(self.ctx.hostpython) app_mk = join(self.get_build_dir(arch.arch), 'Application.mk') if not exists(app_mk): shprint(sh.cp, join(self.get_recipe_dir(), 'Application.mk'), app_mk) app_setup = join(self.get_build_dir(arch.arch), 'setup.py') if not exists(app_setup): shprint(sh.cp, join(self.get_recipe_dir(), 'setup.py'), app_setup) # This first attempt *will* fail, because cython isn't # installed in the hostpython try: shprint(hostpython, 'setup.py', 'build_ext', _env=env) except sh.ErrorReturnCode_1: pass # ...so we manually run cython from the user's system shprint(sh.find, self.get_build_dir('armeabi'), '-iname', '*.pyx', '-exec', self.ctx.cython, '{}', ';', _env=env) # now cython has already been run so the build works shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env) # stripping debug symbols lowers the file size a lot build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env)
def prebuild_arch(self, arch): super(MobileInsightRecipe, self).prebuild_arch(arch) build_dir = self.get_build_dir(arch.arch) tmp_dir = join(build_dir, 'mi_tmp') info("Cleaning old MobileInsight-core sources at {}".format(build_dir)) try: shprint(sh.rm, '-r', build_dir, _tail = 20, _critical = True) except: pass if LOCAL_DEBUG is False: info("Cloning MobileInsight-core sources from {}".format(self.mi_git)) shprint(sh.git, 'clone', '-b', self.mi_branch, '--depth=1', self.mi_git, tmp_dir, _tail = 20, _critical = True) else: warning("Debugging using local sources of MobileInsight at {}".format(self.local_src)) shprint(sh.mkdir, build_dir, _tail = 20, _critical = True) shprint(sh.mkdir, tmp_dir, _tail = 20, _critical = True) shprint(sh.cp, '-fr', self.local_src, tmp_dir, _tail = 20, _critical = True) tmp_dir = join(tmp_dir, 'MobileInsight-core') shprint(sh.mv, join(tmp_dir, 'mobile_insight'), build_dir, _tail = 20, _critical = True) shprint(sh.mv, join(tmp_dir, 'dm_collector_c'), build_dir, _tail = 20, _critical = True) # remove unnecessary codes shprint(sh.rm, '-r', tmp_dir, _tail = 20, _critical = True) self.get_newest_toolchain(arch)
def build_dist_from_args(ctx, dist, args): '''Parses out any bootstrap related arguments, and uses them to build a dist.''' bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) build_order, python_modules, bs \ = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs) ctx.recipe_build_order = build_order ctx.python_modules = python_modules if python_modules and hasattr(sys, 'real_prefix'): error('virtualenv is needed to install pure-Python modules, but') error('virtualenv does not support nesting, and you are running') error('python-for-android in one. Please run p4a outside of a') error('virtualenv instead.') exit(1) info('The selected bootstrap is {}'.format(bs.name)) info_main('# Creating dist with {} bootstrap'.format(bs.name)) bs.distribution = dist info_notify('Dist will have name {} and recipes ({})'.format( dist.name, ', '.join(dist.recipes))) ctx.dist_name = bs.distribution.name ctx.prepare_bootstrap(bs) ctx.prepare_dist(ctx.dist_name) build_recipes(build_order, python_modules, ctx) ctx.bootstrap.run_distribute() info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' .format(join(ctx.dist_dir, ctx.dist_name)))
def _unpack_aar(self, aar, arch): '''Unpack content of .aar bundle and copy to current dist dir.''' with temp_directory() as temp_dir: name = splitext(basename(aar))[0] jar_name = name + '.jar' info("unpack {} aar".format(name)) debug(" from {}".format(aar)) debug(" to {}".format(temp_dir)) shprint(sh.unzip, '-o', aar, '-d', temp_dir) jar_src = join(temp_dir, 'classes.jar') jar_tgt = join('libs', jar_name) debug("copy {} jar".format(name)) debug(" from {}".format(jar_src)) debug(" to {}".format(jar_tgt)) ensure_dir('libs') shprint(sh.cp, '-a', jar_src, jar_tgt) so_src_dir = join(temp_dir, 'jni', arch.arch) so_tgt_dir = join('libs', arch.arch) debug("copy {} .so".format(name)) debug(" from {}".format(so_src_dir)) debug(" to {}".format(so_tgt_dir)) ensure_dir(so_tgt_dir) so_files = glob.glob(join(so_src_dir, '*.so')) for f in so_files: shprint(sh.cp, '-a', f, so_tgt_dir)
def build_arch(self, arch): """simple shared compile""" env = self.get_recipe_env(arch, with_flags_in_cc=False) for path in ( self.get_build_dir(arch.arch), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'), join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')): if not exists(path): info("creating {}".format(path)) shprint(sh.mkdir, '-p', path) cli = env['CC'].split() cc = sh.Command(cli[0]) with current_directory(self.get_build_dir(arch.arch)): cflags = env['CFLAGS'].split() cflags.extend(['-I.', '-c', '-l.', 'ifaddrs.c', '-I.']) shprint(cc, *cflags, _env=env) cflags = env['CFLAGS'].split() cflags.extend(['-shared', '-I.', 'ifaddrs.o', '-o', 'libifaddrs.so']) cflags.extend(env['LDFLAGS'].split()) shprint(cc, *cflags, _env=env) shprint(sh.cp, 'libifaddrs.so', self.ctx.get_libs_dir(arch.arch)) shprint(sh.cp, "libifaddrs.so", join(self.ctx.get_python_install_dir(), 'lib')) # drop header in to the Python include directory python_version = self.ctx.python_recipe.version[0:3] shprint(sh.cp, "ifaddrs.h", join( self.ctx.get_python_install_dir(), 'include/python{}'.format(python_version)) ) include_path = join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include') shprint(sh.cp, "ifaddrs.h", include_path)
def strip_libraries(self, arch): info('Stripping libraries') if self.ctx.python_recipe.from_crystax: info('Python was loaded from CrystaX, skipping strip') return env = arch.get_env() strip = which('arm-linux-androideabi-strip', env['PATH']) if strip is None: warning('Can\'t find strip in PATH...') return strip = sh.Command(strip) libs_dir = join(self.dist_dir, '_python_bundle', '_python_bundle', 'modules') if self.ctx.python_recipe.name == 'python2legacy': libs_dir = join(self.dist_dir, 'private') filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'), '-iname', '*.so', _env=env).stdout.decode('utf-8') logger.info('Stripping libraries in private dir') for filen in filens.split('\n'): try: strip(filen, _env=env) except sh.ErrorReturnCode_1: logger.debug('Failed to strip ' + filen)
def cythonize_build(self, env, build_dir="."): if not self.cythonize: info('Running cython cancelled per recipe setting') return info('Running cython where appropriate') for root, dirnames, filenames in walk("."): for filename in fnmatch.filter(filenames, "*.pyx"): self.cythonize_file(env, build_dir, join(root, filename))
def distribute_libs(self, arch, src_dirs, wildcard='*', dest_dir="libs"): '''Copy existing arch libs from build dirs to current dist dir.''' info('Copying libs') tgt_dir = join(dest_dir, arch.arch) ensure_dir(tgt_dir) for src_dir in src_dirs: for lib in glob.glob(join(src_dir, wildcard)): shprint(sh.cp, '-a', lib, tgt_dir)
def append_file(self, filename, dest): info("Append {} to {}".format(filename, dest)) filename = join(self.recipe_dir, filename) dest = join(self.build_dir, dest) with open(filename, "rb") as fd: data = fd.read() with open(dest, "ab") as fd: fd.write(data)
def apply_patch(self, filename, arch): """ Apply a patch from the current recipe directory into the current build directory. """ info("Applying patch {}".format(filename)) filename = join(self.recipe_dir, filename) shprint(sh.patch, "-t", "-d", self.get_build_dir(arch), "-p1", "-i", filename, _tail=10)
def install_python_package(self, arch, name=None, env=None, is_dir=True): '''Automate the installation of a Python package (or a cython package where the cython components are pre-built).''' # arch = self.filtered_archs[0] # old kivy-ios way if name is None: name = self.name if env is None: env = self.get_recipe_env(arch) info('Installing {} into site-packages'.format(self.name)) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) # hostpython = sh.Command('python3.5') if self.ctx.python_recipe.from_crystax: # hppath = join(dirname(self.hostpython_location), 'Lib', # 'site-packages') hpenv = env.copy() # if 'PYTHONPATH' in hpenv: # hpenv['PYTHONPATH'] = ':'.join([hppath] + # hpenv['PYTHONPATH'].split(':')) # else: # hpenv['PYTHONPATH'] = hppath # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir() # shprint(hostpython, 'setup.py', 'build', # _env=hpenv, *self.setup_extra_args) shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions) _env=hpenv, *self.setup_extra_args) # site_packages_dir = self.ctx.get_site_packages_dir() # built_files = glob.glob(join('build', 'lib*', '*')) # for filen in built_files: # shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1])) elif self.call_hostpython_via_targetpython: shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, *self.setup_extra_args) else: hppath = join(dirname(self.hostpython_location), 'Lib', 'site-packages') hpenv = env.copy() if 'PYTHONPATH' in hpenv: hpenv['PYTHONPATH'] = ':'.join([hppath] + hpenv['PYTHONPATH'].split(':')) else: hpenv['PYTHONPATH'] = hppath shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=lib/python2.7/site-packages', _env=hpenv, *self.setup_extra_args) # AND: Hardcoded python2.7 needs fixing # If asked, also install in the hostpython build dir if self.install_in_hostpython: self.install_hostpython_package(arch)
def delete_dist(self, args): dist = self._dist if dist.needs_build: info('No dist exists that matches your specifications, ' 'exiting without deleting.') shutil.rmtree(dist.dist_dir)
Out_Fore, Err_Style, Err_Fore, info_notify, info_main, shprint, Null_Fore, Null_Style) from pythonforandroid.util import current_directory, ensure_dir from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution, pretty_log_dists from pythonforandroid.graph import get_recipe_order_and_bootstrap from pythonforandroid.build import Context, build_recipes user_dir = dirname(realpath(os.path.curdir)) toolchain_dir = dirname(__file__) sys.path.insert(0, join(toolchain_dir, "tools", "external")) info(''.join([ Err_Style.BRIGHT, Err_Fore.RED, 'This python-for-android revamp is an experimental alpha release!', Err_Style.RESET_ALL ])) info(''.join([ Err_Fore.RED, ('It should work (mostly), but you may experience ' 'missing features or bugs.'), Err_Style.RESET_ALL ])) def add_boolean_option(parser, names, no_names=None, default=True, dest=None, description=None):
def distribute_javaclasses(self, javaclass_dir, dest_dir="src"): '''Copy existing javaclasses from build dir to current dist dir.''' info('Copying java files') ensure_dir(dest_dir) filenames = glob.glob(javaclass_dir) shprint(sh.cp, '-a', *filenames, dest_dir)
def apk(self, args): '''Create an APK using the given distribution.''' ctx = self.ctx dist = self._dist # Manually fixing these arguments at the string stage is # unsatisfactory and should probably be changed somehow, but # we can't leave it until later as the build.py scripts assume # they are in the current directory. fix_args = ('--dir', '--private', '--add-jar', '--add-source', '--whitelist', '--blacklist', '--presplash', '--icon') unknown_args = args.unknown_args for i, arg in enumerate(unknown_args[:-1]): argx = arg.split('=') if argx[0] in fix_args: if len(argx) > 1: unknown_args[i] = '='.join((argx[0], realpath(expanduser(argx[1])))) else: unknown_args[i+1] = realpath(expanduser(unknown_args[i+1])) env = os.environ.copy() if args.build_mode == 'release': if args.keystore: env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore)) if args.signkey: env['P4A_RELEASE_KEYALIAS'] = args.signkey if args.keystorepw: env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw if args.signkeypw: env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env: env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw build = imp.load_source('build', join(dist.dist_dir, 'build.py')) with current_directory(dist.dist_dir): build_args = build.parse_args(args.unknown_args) output = shprint(sh.ant, args.build_mode, _tail=20, _critical=True, _env=env) info_main('# Copying APK to current directory') apk_re = re.compile(r'.*Package: (.*\.apk)$') apk_file = None for line in reversed(output.splitlines()): m = apk_re.match(line) if m: apk_file = m.groups()[0] break if not apk_file: info_main('# APK filename not found in build output, trying to guess') suffix = args.build_mode if suffix == 'release': suffix = suffix + '-unsigned' apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-{}.apk'.format(suffix))) if len(apks) == 0: raise ValueError('Couldn\'t find the built APK') if len(apks) > 1: info('More than one built APK found...guessing you ' 'just built {}'.format(apks[-1])) apk_file = apks[-1] info_main('# Found APK file: {}'.format(apk_file)) shprint(sh.cp, apk_file, './')
def unpack(self, arch): info_main('Unpacking {} for {}'.format(self.name, arch)) build_dir = self.get_build_container_dir(arch) user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower())) if user_dir is not None: info('P4A_{}_DIR exists, symlinking instead'.format( self.name.lower())) # AND: Currently there's something wrong if I use ln, fix this warning('Using cp -a instead of symlink...fix this!') if exists(self.get_build_dir(arch)): return shprint(sh.rm, '-rf', build_dir) shprint(sh.mkdir, '-p', build_dir) shprint(sh.rmdir, build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return if self.url is None: info('Skipping {} unpack as no URL is set'.format(self.name)) return filename = shprint(sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') with current_directory(build_dir): directory_name = self.get_build_dir(arch) # AND: Could use tito's get_archive_rootdir here if not exists(directory_name) or not isdir(directory_name): extraction_filename = join(self.ctx.packages_path, self.name, filename) if isfile(extraction_filename): if extraction_filename.endswith('.zip'): sh.unzip(extraction_filename) import zipfile fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split( '/')[0] if root_directory != directory_name: shprint(sh.mv, root_directory, directory_name) elif (extraction_filename.endswith('.tar.gz') or extraction_filename.endswith('.tgz') or extraction_filename.endswith('.tar.bz2') or extraction_filename.endswith('.tbz2') or extraction_filename.endswith('.tar.xz') or extraction_filename.endswith('.txz')): sh.tar('xf', extraction_filename) root_directory = shprint( sh.tar, 'tf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != directory_name: shprint(sh.mv, root_directory, directory_name) else: raise Exception( 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2 or .tar.xz') elif isdir(extraction_filename): mkdir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git', ): shprint(sh.cp, '-Rv', join(extraction_filename, entry), directory_name) else: raise Exception( 'Given path is neither a file nor a directory: {}'. format(extraction_filename)) else: info('{} is already unpacked, skipping'.format(self.name))
def run_pymodules_install(ctx, modules, project_dir=None, ignore_setup_py=False): """ This function will take care of all non-recipe things, by: 1. Processing them from --requirements (the modules argument) and installing them 2. Installing the user project/app itself via setup.py if ignore_setup_py=True """ info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***') modules = list(filter(ctx.not_has_package, modules)) # Bail out if no python deps and no setup.py to process: if not modules and (ignore_setup_py or project_dir is None or not project_has_setup_py(project_dir)): info('No Python modules and no setup.py to process, skipping') return # Output messages about what we're going to do: if modules: info('The requirements ({}) don\'t have recipes, attempting to ' 'install them with pip'.format(', '.join(modules))) info('If this fails, it may mean that the module has compiled ' 'components and needs a recipe.') if project_dir is not None and \ project_has_setup_py(project_dir) and not ignore_setup_py: info('Will process project install, if it fails then the ' 'project may not be compatible for Android install.') venv = sh.Command(ctx.virtualenv) with current_directory(join(ctx.build_dir)): shprint( venv, '--python=python{}'.format( ctx.python_recipe.major_minor_version_string.partition(".") [0]), 'venv') # Prepare base environment and upgrade pip: base_env = copy.copy(os.environ) base_env["PYTHONPATH"] = ctx.get_site_packages_dir() info('Upgrade pip to latest version') shprint(sh.bash, '-c', ("source venv/bin/activate && pip install -U pip"), _env=copy.copy(base_env)) # Install Cython in case modules need it to build: info('Install Cython in case one of the modules needs it to build') shprint(sh.bash, '-c', ("venv/bin/pip install Cython"), _env=copy.copy(base_env)) # Get environment variables for build (with CC/compiler set): standard_recipe = CythonRecipe() standard_recipe.ctx = ctx # (note: following line enables explicit -lpython... linker options) standard_recipe.call_hostpython_via_targetpython = False recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) env = copy.copy(base_env) env.update(recipe_env) # Make sure our build package dir is available, and the virtualenv # site packages come FIRST (so the proper pip version is used): env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir() env["PYTHONPATH"] = os.path.abspath( join(ctx.build_dir, "venv", "lib", "python" + ctx.python_recipe.major_minor_version_string, "site-packages")) + ":" + env["PYTHONPATH"] # Install the manually specified requirements first: if not modules: info('There are no Python modules to install, skipping') else: info('Creating a requirements.txt file for the Python modules') with open('requirements.txt', 'w') as fileh: for module in modules: key = 'VERSION_' + module if key in environ: line = '{}=={}\n'.format(module, environ[key]) else: line = '{}\n'.format(module) fileh.write(line) info('Installing Python modules with pip') info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. ' 'A reason for this is often modules compiling ' 'native code that is unaware of Android cross-compilation ' 'and does not work without additional ' 'changes / workarounds.') shprint(sh.bash, '-c', ("venv/bin/pip " + "install -v --target '{0}' --no-deps -r requirements.txt" ).format(ctx.get_site_packages_dir().replace( "'", "'\"'\"'")), _env=copy.copy(env)) # Afterwards, run setup.py if present: if project_dir is not None and (project_has_setup_py(project_dir) and not ignore_setup_py): with current_directory(project_dir): info('got setup.py or similar, running project install. ' + '(disable this behavior with --ignore-setup-py)') # Compute & output the constraints we will use: info('Contents that will be used for constraints.txt:') constraints = subprocess.check_output( [join(ctx.build_dir, "venv", "bin", "pip"), "freeze"], env=copy.copy(env)) try: constraints = constraints.decode("utf-8", "replace") except AttributeError: pass info(constraints) # Make sure all packages found are fixed in version # by writing a constraint file, to avoid recipes being # upgraded & reinstalled: with open('constraints.txt', 'wb') as fileh: fileh.write(constraints.encode("utf-8", "replace")) info('Populating venv\'s site-packages with ' 'ctx.get_site_packages_dir()...') # Copy dist contents into site-packages for discovery. # Why this is needed: # --target is somewhat evil and messes with discovery of # packages in PYTHONPATH if that also includes the target # folder. So we need to use the regular virtualenv # site-packages folder instead. # Reference: # https://github.com/pypa/pip/issues/6223 ctx_site_packages_dir = os.path.normpath( os.path.abspath(ctx.get_site_packages_dir())) venv_site_packages_dir = os.path.normpath( os.path.join(ctx.build_dir, "venv", "lib", [ f for f in os.listdir( os.path.join(ctx.build_dir, "venv", "lib")) if f.startswith("python") ][0], "site-packages")) copied_over_contents = [] for f in os.listdir(ctx_site_packages_dir): full_path = os.path.join(ctx_site_packages_dir, f) if not os.path.exists( os.path.join(venv_site_packages_dir, f)): if os.path.isdir(full_path): shutil.copytree( full_path, os.path.join(venv_site_packages_dir, f)) else: shutil.copy2( full_path, os.path.join(venv_site_packages_dir, f)) copied_over_contents.append(f) # Get listing of virtualenv's site-packages, to see the # newly added things afterwards & copy them back into # the distribution folder / build context site-packages: previous_venv_contents = os.listdir(venv_site_packages_dir) # Actually run setup.py: info('Launching package install...') shprint(sh.bash, '-c', ("'" + join(ctx.build_dir, "venv", "bin", "pip").replace("'", "'\"'\"'") + "' " + "install -c constraints.txt -v .").format( ctx.get_site_packages_dir().replace( "'", "'\"'\"'")), _env=copy.copy(env)) # Go over all new additions and copy them back: info('Copying additions resulting from setup.py back ' + 'into ctx.get_site_packages_dir()...') new_venv_additions = [] for f in (set(os.listdir(venv_site_packages_dir)) - set(previous_venv_contents)): new_venv_additions.append(f) full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): shutil.copytree(full_path, os.path.join(ctx_site_packages_dir, f)) else: shutil.copy2(full_path, os.path.join(ctx_site_packages_dir, f)) # Undo all the changes we did to the venv-site packages: info('Reverting additions to virtualenv\'s site-packages...') for f in set(copied_over_contents + new_venv_additions): full_path = os.path.join(venv_site_packages_dir, f) if os.path.isdir(full_path): shutil.rmtree(full_path) else: os.remove(full_path) elif not ignore_setup_py: info("No setup.py found in project directory: " + str(project_dir)) # Strip object files after potential Cython or native code builds: standard_recipe.strip_object_files(ctx.archs[0], env, build_dir=ctx.build_dir)
def get_recipe_order_and_bootstrap(ctx, names, bs=None): recipes_to_load = set(names) if bs is not None and bs.recipe_depends: recipes_to_load = recipes_to_load.union(set(bs.recipe_depends)) possible_orders = [] # get all possible order graphs, as names may include tuples/lists # of alternative dependencies names = [([name] if not isinstance(name, (list, tuple)) else name) for name in names] for name_set in product(*names): new_possible_orders = [RecipeOrder(ctx)] for name in name_set: new_possible_orders = recursively_collect_orders( name, ctx, orders=new_possible_orders) possible_orders.extend(new_possible_orders) # turn each order graph into a linear list if possible orders = [] for possible_order in possible_orders: try: order = find_order(possible_order) except ValueError: # a circular dependency was found info('Circular dependency found in graph {}, skipping it.'.format( possible_order)) continue except: warning('Failed to import recipe named {}; the recipe exists ' 'but appears broken.'.format(name)) warning('Exception was:') raise orders.append(list(order)) # prefer python2 and SDL2 if available orders = sorted(orders, key=lambda order: -('python2' in order) - ('sdl2' in order)) if not orders: error('Didn\'t find any valid dependency graphs.') error('This means that some of your requirements pull in ' 'conflicting dependencies.') error('Exiting.') exit(1) # It would be better to check against possible orders other # than the first one, but in practice clashes will be rare, # and can be resolved by specifying more parameters chosen_order = orders[0] if len(orders) > 1: info('Found multiple valid dependency orders:') for order in orders: info(' {}'.format(order)) info('Using the first of these: {}'.format(chosen_order)) else: info('Found a single valid recipe set: {}'.format(chosen_order)) if bs is None: bs = Bootstrap.get_bootstrap_from_recipes(chosen_order, ctx) recipes, python_modules, bs = get_recipe_order_and_bootstrap( ctx, chosen_order, bs=bs) else: # check if each requirement has a recipe recipes = [] python_modules = [] for name in chosen_order: try: Recipe.get_recipe(name, ctx) except IOError: python_modules.append(name) else: recipes.append(name) return recipes, python_modules, bs
def run_pymodules_install(ctx, modules): modules = list(filter(ctx.not_has_package, modules)) if not modules: info('There are no Python modules to install, skipping') return info('The requirements ({}) don\'t have recipes, attempting to install ' 'them with pip'.format(', '.join(modules))) info('If this fails, it may mean that the module has compiled ' 'components and needs a recipe.') venv = sh.Command(ctx.virtualenv) with current_directory(join(ctx.build_dir)): shprint(venv, '--python=python{}'.format(ctx.python_recipe.major_minor_version_string), 'venv') info('Creating a requirements.txt file for the Python modules') with open('requirements.txt', 'w') as fileh: for module in modules: key = 'VERSION_' + module if key in environ: line = '{}=={}\n'.format(module, environ[key]) else: line = '{}\n'.format(module) fileh.write(line) info('Installing Python modules with pip') info('If this fails with a message about /bin/false, this ' 'probably means the package cannot be installed with ' 'pip as it needs a compilation recipe.') # This bash method is what old-p4a used # It works but should be replaced with something better shprint(sh.bash, '-c', ( "env CC=/bin/false CXX=/bin/false " "PYTHONPATH={0} venv/bin/pip install --target '{0}' --no-deps -r requirements.txt" ).format(ctx.get_site_packages_dir()))
def prepare_build_environment(self, user_sdk_dir, user_ndk_dir, user_android_api, user_ndk_ver, user_ndk_api): '''Checks that build dependencies exist and sets internal variables for the Android SDK etc. ..warning:: This *must* be called before trying any build stuff ''' self.ensure_dirs() if self._build_env_prepared: return ok = True # Work out where the Android SDK is sdk_dir = None if user_sdk_dir: sdk_dir = user_sdk_dir # This is the old P4A-specific var if sdk_dir is None: sdk_dir = environ.get('ANDROIDSDK', None) # This seems used more conventionally if sdk_dir is None: sdk_dir = environ.get('ANDROID_HOME', None) # Checks in the buildozer SDK dir, useful for debug tests of p4a if sdk_dir is None: possible_dirs = glob.glob(expanduser(join( '~', '.buildozer', 'android', 'platform', 'android-sdk-*'))) possible_dirs = [d for d in possible_dirs if not (d.endswith('.bz2') or d.endswith('.gz'))] if possible_dirs: info('Found possible SDK dirs in buildozer dir: {}'.format( ', '.join([d.split(os.sep)[-1] for d in possible_dirs]))) info('Will attempt to use SDK at {}'.format(possible_dirs[0])) warning('This SDK lookup is intended for debug only, if you ' 'use python-for-android much you should probably ' 'maintain your own SDK download.') sdk_dir = possible_dirs[0] if sdk_dir is None: raise BuildInterruptingException('Android SDK dir was not specified, exiting.') self.sdk_dir = realpath(sdk_dir) # Check what Android API we're using android_api = None if user_android_api: android_api = user_android_api info('Getting Android API version from user argument: {}'.format(android_api)) elif 'ANDROIDAPI' in environ: android_api = environ['ANDROIDAPI'] info('Found Android API target in $ANDROIDAPI: {}'.format(android_api)) else: info('Android API target was not set manually, using ' 'the default of {}'.format(DEFAULT_ANDROID_API)) android_api = DEFAULT_ANDROID_API android_api = int(android_api) self.android_api = android_api if self.android_api >= 21 and self.archs[0].arch == 'armeabi': raise BuildInterruptingException( 'Asked to build for armeabi architecture with API ' '{}, but API 21 or greater does not support armeabi'.format( self.android_api), instructions='You probably want to build with --arch=armeabi-v7a instead') info('Checking SDK: {}'.format(sdk_dir)) if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): info('valid adv manager:') avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n') elif exists(join(sdk_dir, 'tools', 'android')): info('valid android:') android = sh.Command(join(sdk_dir, 'tools', 'android')) targets = android('list').stdout.decode('utf-8').split('\n') else: info('Could not find `android` or `sdkmanager`') raise BuildInterruptingException( 'Could not find `android` or `sdkmanager` binaries in Android SDK', instructions='Make sure the path to the Android SDK is correct') apis = [s for s in targets if re.match(r'^ *API level: ', s)] apis = [re.findall(r'[0-9]+', s) for s in apis] apis = [int(s[0]) for s in apis if s] info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) if android_api in apis: info(('Requested API target {} is available, ' 'continuing.').format(android_api)) else: raise BuildInterruptingException( ('Requested API target {} is not available, install ' 'it with the SDK android tool.').format(android_api)) # Find the Android NDK # Could also use ANDROID_NDK, but doesn't look like many tools use this ndk_dir = None if user_ndk_dir: ndk_dir = user_ndk_dir info('Getting NDK dir from from user argument') if ndk_dir is None: # The old P4A-specific dir ndk_dir = environ.get('ANDROIDNDK', None) if ndk_dir is not None: info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir)) if ndk_dir is None: # Apparently the most common convention ndk_dir = environ.get('NDK_HOME', None) if ndk_dir is not None: info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir)) if ndk_dir is None: # Another convention (with maven?) ndk_dir = environ.get('ANDROID_NDK_HOME', None) if ndk_dir is not None: info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir)) if ndk_dir is None: # Checks in the buildozer NDK dir, useful # # for debug tests of p4a possible_dirs = glob.glob(expanduser(join( '~', '.buildozer', 'android', 'platform', 'android-ndk-r*'))) if possible_dirs: info('Found possible NDK dirs in buildozer dir: {}'.format( ', '.join([d.split(os.sep)[-1] for d in possible_dirs]))) info('Will attempt to use NDK at {}'.format(possible_dirs[0])) warning('This NDK lookup is intended for debug only, if you ' 'use python-for-android much you should probably ' 'maintain your own NDK download.') ndk_dir = possible_dirs[0] if ndk_dir is None: raise BuildInterruptingException('Android NDK dir was not specified') self.ndk_dir = realpath(ndk_dir) # Find the NDK version, and check it against what the NDK dir # seems to report ndk_ver = None if user_ndk_ver: ndk_ver = user_ndk_ver if ndk_dir is not None: info('Got NDK version from from user argument: {}'.format(ndk_ver)) if ndk_ver is None: ndk_ver = environ.get('ANDROIDNDKVER', None) if ndk_ver is not None: info('Got NDK version from $ANDROIDNDKVER: {}'.format(ndk_ver)) self.ndk = 'google' try: with open(join(ndk_dir, 'RELEASE.TXT')) as fileh: reported_ndk_ver = fileh.read().split(' ')[0].strip() except IOError: pass else: if reported_ndk_ver.startswith('crystax-ndk-'): reported_ndk_ver = reported_ndk_ver[12:] self.ndk = 'crystax' if ndk_ver is None: ndk_ver = reported_ndk_ver info(('Got Android NDK version from the NDK dir: {}').format(ndk_ver)) else: if ndk_ver != reported_ndk_ver: warning('NDK version was set as {}, but checking ' 'the NDK dir claims it is {}.'.format( ndk_ver, reported_ndk_ver)) warning('The build will try to continue, but it may ' 'fail and you should check ' 'that your setting is correct.') warning('If the NDK dir result is correct, you don\'t ' 'need to manually set the NDK ver.') if ndk_ver is None: warning('Android NDK version could not be found. This probably' 'won\'t cause any problems, but if necessary you can' 'set it with `--ndk-version=...`.') self.ndk_ver = ndk_ver ndk_api = None if user_ndk_api: ndk_api = user_ndk_api info('Getting NDK API version (i.e. minimum supported API) from user argument') elif 'NDKAPI' in environ: ndk_api = environ.get('NDKAPI', None) info('Found Android API target in $NDKAPI') else: ndk_api = min(self.android_api, DEFAULT_NDK_API) warning('NDK API target was not set manually, using ' 'the default of {} = min(android-api={}, default ndk-api={})'.format( ndk_api, self.android_api, DEFAULT_NDK_API)) ndk_api = int(ndk_api) self.ndk_api = ndk_api if self.ndk_api > self.android_api: raise BuildInterruptingException( 'Target NDK API is {}, higher than the target Android API {}.'.format( self.ndk_api, self.android_api), instructions=('The NDK API is a minimum supported API number and must be lower ' 'than the target Android API')) info('Using {} NDK {}'.format(self.ndk.capitalize(), self.ndk_ver)) virtualenv = None if virtualenv is None: virtualenv = sh.which('virtualenv2') if virtualenv is None: virtualenv = sh.which('virtualenv-2.7') if virtualenv is None: virtualenv = sh.which('virtualenv') if virtualenv is None: raise IOError('Couldn\'t find a virtualenv executable, ' 'you must install this to use p4a.') self.virtualenv = virtualenv info('Found virtualenv at {}'.format(virtualenv)) # path to some tools self.ccache = sh.which("ccache") if not self.ccache: info('ccache is missing, the build will not be optimized in the ' 'future.') for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"): cython = sh.which(cython_fn) if cython: self.cython = cython break else: raise BuildInterruptingException('No cython binary found.') if not self.cython: ok = False warning("Missing requirement: cython is not installed") # This would need to be changed if supporting multiarch APKs arch = self.archs[0] platform_dir = arch.platform_dir toolchain_prefix = arch.toolchain_prefix toolchain_version = None self.ndk_platform = join( self.ndk_dir, 'platforms', 'android-{}'.format(self.ndk_api), platform_dir) if not exists(self.ndk_platform): warning('ndk_platform doesn\'t exist: {}'.format( self.ndk_platform)) ok = False py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' toolchain_versions = [] toolchain_path = join(self.ndk_dir, 'toolchains') if isdir(toolchain_path): toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path, toolchain_prefix)) toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:] for path in toolchain_contents] else: warning('Could not find toolchain subdirectory!') ok = False toolchain_versions.sort() toolchain_versions_gcc = [] for toolchain_version in toolchain_versions: if toolchain_version[0].isdigit(): # GCC toolchains begin with a number toolchain_versions_gcc.append(toolchain_version) if toolchain_versions: info('Found the following toolchain versions: {}'.format( toolchain_versions)) info('Picking the latest gcc toolchain, here {}'.format( toolchain_versions_gcc[-1])) toolchain_version = toolchain_versions_gcc[-1] else: warning('Could not find any toolchain for {}!'.format( toolchain_prefix)) ok = False self.toolchain_prefix = toolchain_prefix self.toolchain_version = toolchain_version # Modify the path so that sh finds modules appropriately environ['PATH'] = ( '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' '{toolchain_prefix}-{toolchain_version}/prebuilt/' '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' 'tools:{path}').format( sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, toolchain_prefix=toolchain_prefix, toolchain_version=toolchain_version, py_platform=py_platform, path=environ.get('PATH')) for executable in ("pkg-config", "autoconf", "automake", "libtoolize", "tar", "bzip2", "unzip", "make", "gcc", "g++"): if not sh.which(executable): warning("Missing executable: {} is not installed".format( executable)) if not ok: raise BuildInterruptingException( 'python-for-android cannot continue due to the missing executables above')
def get_distribution( cls, ctx, *, arch_name, # required keyword argument: there is no sensible default name=None, recipes=[], ndk_api=None, force_build=False, extra_dist_dirs=[], require_perfect_match=False, allow_replace_dist=True): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. ndk_api : int The NDK API to compile against, included in the dist because it cannot be changed later during APK packaging. arch_name : str The target architecture name to compile against, included in the dist because it cannot be changed later during APK packaging. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. allow_replace_dist : bool If True, will allow an existing dist with the specified name but incompatible requirements to be overwritten by a new one with the current requirements. ''' possible_dists = Distribution.get_distributions(ctx) # Will hold dists that would be built in the same folder as an existing dist folder_match_dist = None # 0) Check if a dist with that name and architecture already exists if name is not None and name: possible_dists = [ d for d in possible_dists if (d.name == name) and (arch_name in d.archs) ] if possible_dists: # There should only be one folder with a given dist name *and* arch. # We could check that here, but for compatibility let's let it slide # and just record the details of one of them. We only use this data to # possibly fail the build later, so it doesn't really matter if there # was more than one clash. folder_match_dist = possible_dists[0] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: if (ndk_api is not None and dist.ndk_api != ndk_api) or dist.ndk_api is None: continue for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes, arch and NDK API, return it for dist in possible_dists: if force_build: continue if ndk_api is not None and dist.ndk_api != ndk_api: continue if arch_name is not None and arch_name not in dist.archs: continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): info_notify('{} has compatible recipes, using this one'.format( dist.name)) return dist # If there was a name match but we didn't already choose it, # then the existing dist is incompatible with the requested # configuration and the build cannot continue if folder_match_dist is not None and not allow_replace_dist: raise BuildInterruptingException( 'Asked for dist with name {name} with recipes ({req_recipes}) and ' 'NDK API {req_ndk_api}, but a dist ' 'with this name already exists and has either incompatible recipes ' '({dist_recipes}) or NDK API {dist_ndk_api}'.format( name=name, req_ndk_api=ndk_api, dist_ndk_api=folder_match_dist.ndk_api, req_recipes=', '.join(recipes), dist_recipes=', '.join(folder_match_dist.recipes))) assert len(possible_dists) < 2 # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join( ctx.dist_dir, generate_dist_folder_name( name, [arch_name] if arch_name is not None else None, )) dist.recipes = recipes dist.ndk_api = ctx.ndk_api dist.archs = [arch_name] return dist
def install_python_package(self, arch, name=None, env=None, is_dir=True): for package in packages: #srcdir = os.path.join(self.get_recipe_dir(), "..", "..", "..", "..", "tribler", "src", package) srcdir = os.path.join("/work/tribler/src", package) info('Installing {} into site-packages'.format(package)) copytree(srcdir, self.ctx.get_python_install_dir())
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) if self.ctx.python_recipe.from_crystax: command = sh.Command('python{}'.format( self.ctx.python_recipe.version)) site_packages_dirs = command( '-c', 'import site; print("\\n".join(site.getsitepackages()))') site_packages_dirs = site_packages_dirs.stdout.decode( 'utf-8').split('\n') # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5' if 'PYTHONPATH' in env: env['PYTHONPATH'] = env + ':{}'.format( ':'.join(site_packages_dirs)) else: env['PYTHONPATH'] = ':'.join(site_packages_dirs) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) # hostpython = sh.Command('python3.5') shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env) print('cwd is', realpath(curdir)) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) manually_cythonise = False try: shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) manually_cythonise = True if manually_cythonise: self.cythonize_build(env=env) shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) else: info( 'First build appeared to complete correctly, skipping manual' 'cythonising.') print('stripping') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) print('stripped!?')
def create_python_bundle(self, dirn, arch): """ Create a packaged python bundle in the target directory, by copying all the modules and standard library to the right place. """ # Todo: find a better way to find the build libs folder modules_build_dir = join( self.get_build_dir(arch.arch), 'android-build', 'build', 'lib.linux{}-{}-{}'.format('2' if self.version[0] == '2' else '', arch.command_prefix.split('-')[0], self.major_minor_version_string)) # Compile to *.pyc/*.pyo the python modules self.compile_python_files(modules_build_dir) # Compile to *.pyc/*.pyo the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib')) # Compile to *.pyc/*.pyo the other python packages (site-packages) self.compile_python_files(self.ctx.get_python_install_dir()) # Bundle compiled python modules to a folder modules_dir = join(dirn, 'modules') c_ext = self.compiled_extension ensure_dir(modules_dir) module_filens = (glob.glob(join(modules_build_dir, '*.so')) + glob.glob(join(modules_build_dir, '*' + c_ext))) info("Copy {} files into the bundle".format(len(module_filens))) for filen in module_filens: info(" - copy {}".format(filen)) copy2(filen, modules_dir) # zip up the standard library stdlib_zip = join(dirn, 'stdlib.zip') with current_directory(join(self.get_build_dir(arch.arch), 'Lib')): stdlib_filens = list( walk_valid_filens('.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist)) info("Zip {} files into the bundle".format(len(stdlib_filens))) shprint(sh.zip, stdlib_zip, *stdlib_filens) # copy the site-packages into place ensure_dir(join(dirn, 'site-packages')) ensure_dir(self.ctx.get_python_install_dir()) # TODO: Improve the API around walking and copying the files with current_directory(self.ctx.get_python_install_dir()): filens = list( walk_valid_filens('.', self.site_packages_dir_blacklist, self.site_packages_filen_blacklist)) info("Copy {} files into the site-packages".format(len(filens))) for filen in filens: info(" - copy {}".format(filen)) ensure_dir(join(dirn, 'site-packages', dirname(filen))) copy2(filen, join(dirn, 'site-packages', filen)) # copy the python .so files into place python_build_dir = join(self.get_build_dir(arch.arch), 'android-build') python_lib_name = 'libpython' + self.major_minor_version_string if self.major_minor_version_string[0] == '3': python_lib_name += 'm' shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'), join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch)) info('Renaming .so files to reflect cross-compile') self.reduce_object_file_names(join(dirn, 'site-packages')) return join(dirn, 'site-packages')
def biglink(ctx, arch): # First, collate object files from each recipe info('Collating object files from each recipe') obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects') ensure_dir(obj_dir) recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order] for recipe in recipes: recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch), 'objects_{}'.format(recipe.name)) if not exists(recipe_obj_dir): info('{} recipe has no biglinkable files dir, skipping' .format(recipe.name)) continue files = glob.glob(join(recipe_obj_dir, '*')) if not len(files): info('{} recipe has no biglinkable files, skipping' .format(recipe.name)) continue info('{} recipe has object files, copying'.format(recipe.name)) files.append(obj_dir) shprint(sh.cp, '-r', *files) env = arch.get_env() env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)) if not len(glob.glob(join(obj_dir, '*'))): info('There seem to be no libraries to biglink, skipping.') return info('Biglinking') info('target {}'.format(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'))) do_biglink = copylibs_function if ctx.copy_libs else biglink_function # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? with current_directory(join(ctx.ndk_platform, 'usr', 'lib')): do_biglink( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), extra_link_dirs=[join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch), os.path.abspath('.')], env=env)
def set_libs_flags(self, env, arch): '''Takes care to properly link libraries with python depending on our requirements and the attribute :attr:`opt_depends`. ''' def add_flags(include_flags, link_dirs, link_libs): env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs env['LIBS'] = env.get('LIBS', '') + link_libs if 'sqlite3' in self.ctx.recipe_build_order: info('Activating flags for sqlite3') recipe = Recipe.get_recipe('sqlite3', self.ctx) add_flags(' -I' + recipe.get_build_dir(arch.arch), ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') if 'libffi' in self.ctx.recipe_build_order: info('Activating flags for libffi') recipe = Recipe.get_recipe('libffi', self.ctx) # In order to force the correct linkage for our libffi library, we # set the following variable to point where is our libffi.pc file, # because the python build system uses pkg-config to configure it. env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch) add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), ' -lffi') if 'openssl' in self.ctx.recipe_build_order: info('Activating flags for openssl') recipe = Recipe.get_recipe('openssl', self.ctx) self.configure_args += \ ('--with-openssl=' + recipe.get_build_dir(arch.arch),) add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch), recipe.link_libs_flags()) for library_name in {'libbz2', 'liblzma'}: if library_name in self.ctx.recipe_build_order: info(f'Activating flags for {library_name}') recipe = Recipe.get_recipe(library_name, self.ctx) add_flags(recipe.get_library_includes(arch), recipe.get_library_ldflags(arch), recipe.get_library_libs_flag()) # python build system contains hardcoded zlib version which prevents # the build of zlib module, here we search for android's zlib version # and sets the right flags, so python can be build with android's zlib info("Activating flags for android's zlib") zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') zlib_h = join(zlib_includes, 'zlib.h') try: with open(zlib_h) as fileh: zlib_data = fileh.read() except IOError: raise BuildInterruptingException( "Could not determine android's zlib version, no zlib.h ({}) in" " the NDK dir includes".format(zlib_h)) for line in zlib_data.split('\n'): if line.startswith('#define ZLIB_VERSION '): break else: raise BuildInterruptingException( 'Could not parse zlib.h...so we cannot find zlib version,' 'required by python build,') env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '') add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz') return env
def build_recipes(build_order, python_modules, ctx, project_dir, ignore_project_setup_py=False): # Put recipes in correct build order info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping'.format( recipe.name)) # 4) biglink everything info_main('# Biglinking object files') if not ctx.python_recipe or not ctx.python_recipe.from_crystax: biglink(ctx, arch) else: info('NDK is crystax, skipping biglink (will this work?)') # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules, project_dir, ignore_setup_py=ignore_project_setup_py) return
def apk(self, args): """Create an APK using the given distribution.""" ctx = self.ctx dist = self._dist # Manually fixing these arguments at the string stage is # unsatisfactory and should probably be changed somehow, but # we can't leave it until later as the build.py scripts assume # they are in the current directory. fix_args = ('--dir', '--private', '--add-jar', '--add-source', '--whitelist', '--blacklist', '--presplash', '--icon') unknown_args = args.unknown_args for i, arg in enumerate(unknown_args): argx = arg.split('=') if argx[0] in fix_args: if len(argx) > 1: unknown_args[i] = '='.join( (argx[0], realpath(expanduser(argx[1])))) else: unknown_args[i+1] = realpath(expanduser(unknown_args[i+1])) env = os.environ.copy() if args.build_mode == 'release': if args.keystore: env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore)) if args.signkey: env['P4A_RELEASE_KEYALIAS'] = args.signkey if args.keystorepw: env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw if args.signkeypw: env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env: env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw build = imp.load_source('build', join(dist.dist_dir, 'build.py')) with current_directory(dist.dist_dir): self.hook("before_apk_build") os.environ["ANDROID_API"] = str(self.ctx.android_api) build_args = build.parse_args(args.unknown_args) self.hook("after_apk_build") self.hook("before_apk_assemble") build_type = ctx.java_build_tool if build_type == 'auto': info('Selecting java build tool:') build_tools_versions = os.listdir(join(ctx.sdk_dir, 'build-tools')) build_tools_versions = sorted(build_tools_versions, key=LooseVersion) build_tools_version = build_tools_versions[-1] info(('Detected highest available build tools ' 'version to be {}').format(build_tools_version)) if build_tools_version >= '25.0' and exists('gradlew'): build_type = 'gradle' info(' Building with gradle, as gradle executable is ' 'present') else: build_type = 'ant' if build_tools_version < '25.0': info((' Building with ant, as the highest ' 'build-tools-version is only {}').format( build_tools_version)) else: info(' Building with ant, as no gradle executable ' 'detected') if build_type == 'gradle': # gradle-based build env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir env["ANDROID_HOME"] = self.ctx.sdk_dir gradlew = sh.Command('./gradlew') if exists('/usr/bin/dos2unix'): # .../dists/bdisttest_python3/gradlew # .../build/bootstrap_builds/sdl2-python3crystax/gradlew # if docker on windows, gradle contains CRLF output = shprint( sh.Command('dos2unix'), gradlew._path.decode('utf8'), _tail=20, _critical=True, _env=env ) if args.build_mode == "debug": gradle_task = "assembleDebug" elif args.build_mode == "release": gradle_task = "assembleRelease" else: error("Unknown build mode {} for apk()".format( args.build_mode)) exit(1) output = shprint(gradlew, gradle_task, _tail=20, _critical=True, _env=env) # gradle output apks somewhere else # and don't have version in file apk_dir = join(dist.dist_dir, "build", "outputs", "apk", args.build_mode) apk_glob = "*-{}.apk" apk_add_version = True else: # ant-based build try: ant = sh.Command('ant') except sh.CommandNotFound: error('Could not find ant binary, please install it ' 'and make sure it is in your $PATH.') exit(1) output = shprint(ant, args.build_mode, _tail=20, _critical=True, _env=env) apk_dir = join(dist.dist_dir, "bin") apk_glob = "*-*-{}.apk" apk_add_version = False self.hook("after_apk_assemble") info_main('# Copying APK to current directory') apk_re = re.compile(r'.*Package: (.*\.apk)$') apk_file = None for line in reversed(output.splitlines()): m = apk_re.match(line) if m: apk_file = m.groups()[0] break if not apk_file: info_main('# APK filename not found in build output. Guessing...') if args.build_mode == "release": suffixes = ("release", "release-unsigned") else: suffixes = ("debug", ) for suffix in suffixes: apks = glob.glob(join(apk_dir, apk_glob.format(suffix))) if apks: if len(apks) > 1: info('More than one built APK found... guessing you ' 'just built {}'.format(apks[-1])) apk_file = apks[-1] break else: raise ValueError('Couldn\'t find the built APK') info_main('# Found APK file: {}'.format(apk_file)) if apk_add_version: info('# Add version number to APK') apk_name, apk_suffix = basename(apk_file).split("-", 1) apk_file_dest = "{}-{}-{}".format( apk_name, build_args.version, apk_suffix) info('# APK renamed to {}'.format(apk_file_dest)) shprint(sh.cp, apk_file, apk_file_dest) else: shprint(sh.cp, apk_file, './')
def check_ndk_version(ndk_dir): """ Check the NDK version against what is currently recommended and raise an exception of :class:`~pythonforandroid.util.BuildInterruptingException` in case that the user tries to use an NDK lower than minimum supported, specified via attribute `MIN_NDK_VERSION`. .. versionchanged:: 2019.06.06.1.dev0 Added the ability to get android's NDK `letter version` and also rewrote to raise an exception in case that an NDK version lower than the minimum supported is detected. """ version = read_ndk_version(ndk_dir) if version is None: warning(READ_ERROR_NDK_MESSAGE.format(ndk_dir=ndk_dir)) warning( ENSURE_RIGHT_NDK_MESSAGE.format( min_supported=MIN_NDK_VERSION, rec_version=RECOMMENDED_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL, ) ) return # create a dictionary which will describe the relationship of the android's # NDK minor version with the `human readable` letter version, egs: # Pkg.Revision = 17.1.4828580 => ndk-17b # Pkg.Revision = 17.2.4988734 => ndk-17c # Pkg.Revision = 19.0.5232133 => ndk-19 (No letter) minor_to_letter = {0: ''} minor_to_letter.update( {n + 1: chr(i) for n, i in enumerate(range(ord('b'), ord('b') + 25))} ) major_version = version.version[0] letter_version = minor_to_letter[version.version[1]] string_version = '{major_version}{letter_version}'.format( major_version=major_version, letter_version=letter_version ) info(CURRENT_NDK_VERSION_MESSAGE.format(ndk_version=string_version)) if major_version < MIN_NDK_VERSION: raise BuildInterruptingException( NDK_LOWER_THAN_SUPPORTED_MESSAGE.format( min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL ), instructions=( 'Please, go to the android NDK page ({ndk_url}) and download a' ' supported version.\n*** The currently recommended NDK' ' version is {rec_version} ***'.format( ndk_url=NDK_DOWNLOAD_URL, rec_version=RECOMMENDED_NDK_VERSION, ) ), ) elif major_version > MAX_NDK_VERSION: warning( RECOMMENDED_NDK_VERSION_MESSAGE.format( recommended_ndk_version=RECOMMENDED_NDK_VERSION ) ) warning(NEW_NDK_MESSAGE)
def build_arch(self, arch): # If openssl is needed we may have to recompile cPython to get the # ssl.py module working properly if self.from_crystax and 'openssl' in self.ctx.recipe_build_order: info('Openssl and crystax-python combination may require ' 'recompilation of python...') ssl_recipe = self.get_recipe('openssl', self.ctx) stage, msg = self.check_for_sslso(ssl_recipe, arch) stage = 0 if stage < 5 else stage info(msg) openssl_build_dir = ssl_recipe.get_build_dir(arch.arch) openssl_ndk_dir = join(self.ctx.ndk_dir, 'sources', 'openssl', ssl_recipe.version) if stage < 2: info('Copying openssl headers and Android.mk to ndk') ensure_dir(openssl_ndk_dir) if stage < 1.2: # copy include folder and Android.mk to ndk mk_path = self.find_Android_mk() if mk_path is None: raise IOError('Android.mk file could not be found in ' 'any versions in ndk->sources->openssl') shprint(sh.cp, mk_path, openssl_ndk_dir) include_dir = join(openssl_build_dir, 'include') if stage < 1.3: ndk_include_dir = join(openssl_ndk_dir, 'include', 'openssl') self.copy_include_dir(join(include_dir, 'openssl'), ndk_include_dir) target_conf = join(openssl_ndk_dir, 'include', 'openssl', 'opensslconf.h') shprint(sh.rm, '-f', target_conf) # overwrite opensslconf.h with open(target_conf, 'w') as fp: fp.write(OPENSSLCONF) if stage < 1.4: # move current conf to arch specific conf in ndk under_scored_arch = arch.arch.replace('-', '_') shprint( sh.ln, '-sf', realpath(join(include_dir, 'openssl', 'opensslconf.h')), join(openssl_ndk_dir, 'include', 'openssl', 'opensslconf_{}.h'.format(under_scored_arch))) if stage < 3: info('Copying openssl libs to ndk') arch_ndk_lib = join(openssl_ndk_dir, 'libs', arch.arch) ensure_dir(arch_ndk_lib) shprint( sh.ln, '-sf', realpath( join(openssl_build_dir, 'libcrypto{}.so'.format(ssl_recipe.version))), join(openssl_build_dir, 'libcrypto.so')) shprint( sh.ln, '-sf', realpath( join(openssl_build_dir, 'libssl{}.so'.format(ssl_recipe.version))), join(openssl_build_dir, 'libssl.so')) libs = ['libcrypto.a', 'libcrypto.so', 'libssl.a', 'libssl.so'] cmd = [join(openssl_build_dir, lib) for lib in libs] + [arch_ndk_lib] shprint(sh.cp, '-f', *cmd) if stage < 10: info('Recompiling python-crystax') self.patch_dev_defaults(ssl_recipe) build_script = join(self.ctx.ndk_dir, 'build', 'tools', 'build-target-python.sh') shprint(Command(build_script), '--ndk-dir={}'.format(self.ctx.ndk_dir), '--abis={}'.format(arch.arch), '-j5', '--verbose', self.get_build_dir(arch.arch)) info('Extracting CrystaX python3 from NDK package') dirn = self.ctx.get_python_install_dir() ensure_dir(dirn) self.ctx.hostpython = 'python{}'.format(self.version)
def copy_file(self, filename, dest): info("Copy {} to {}".format(filename, dest)) filename = join(self.recipe_dir, filename) dest = join(self.build_dir, dest) shutil.copy(filename, dest)
def __init__(self): argv = sys.argv # Buildozer used to pass these arguments in a now-invalid order # If that happens, apply this fix # This fix will be removed once a fixed buildozer is released if (len(argv) > 2 and argv[1].startswith('--color') and argv[2].startswith('--storage-dir')): argv.append(argv.pop(1)) # the --color arg argv.append(argv.pop(1)) # the --storage-dir arg parser = NoAbbrevParser( description=('A packaging tool for turning Python scripts and apps ' 'into Android APKs')) generic_parser = argparse.ArgumentParser( add_help=False, description=('Generic arguments applied to all commands')) dist_parser = argparse.ArgumentParser( add_help=False, description=('Arguments for dist building')) generic_parser.add_argument( '--debug', dest='debug', action='store_true', default=False, help='Display debug output and all build info') generic_parser.add_argument( '--color', dest='color', choices=['always', 'never', 'auto'], help='Enable or disable color output (default enabled on tty)') generic_parser.add_argument( '--sdk-dir', '--sdk_dir', dest='sdk_dir', default='', help='The filepath where the Android SDK is installed') generic_parser.add_argument( '--ndk-dir', '--ndk_dir', dest='ndk_dir', default='', help='The filepath where the Android NDK is installed') generic_parser.add_argument( '--android-api', '--android_api', dest='android_api', default=0, type=int, help='The Android API level to build against.') generic_parser.add_argument( '--ndk-version', '--ndk_version', dest='ndk_version', default='', help=('The version of the Android NDK. This is optional, ' 'we try to work it out automatically from the ndk_dir.')) generic_parser.add_argument( '--symlink-java-src', '--symlink_java_src', action='store_true', dest='symlink_java_src', default=False, help=('If True, symlinks the java src folder during build and dist ' 'creation. This is useful for development only, it could also ' 'cause weird problems.')) default_storage_dir = user_data_dir('python-for-android') if ' ' in default_storage_dir: default_storage_dir = '~/.python-for-android' generic_parser.add_argument( '--storage-dir', dest='storage_dir', default=default_storage_dir, help=('Primary storage directory for downloads and builds ' '(default: {})'.format(default_storage_dir))) # AND: This option doesn't really fit in the other categories, the # arg structure needs a rethink generic_parser.add_argument( '--arch', help='The archs to build for, separated by commas.', default='armeabi') # Options for specifying the Distribution generic_parser.add_argument( '--dist-name', '--dist_name', help='The name of the distribution to use or create', default='') generic_parser.add_argument( '--requirements', help=('Dependencies of your app, should be recipe names or ' 'Python modules'), default='') generic_parser.add_argument( '--bootstrap', help='The bootstrap to build with. Leave unset to choose automatically.', default=None) add_boolean_option( generic_parser, ["force-build"], default=False, description='Whether to force compilation of a new distribution:') generic_parser.add_argument( '--extra-dist-dirs', '--extra_dist_dirs', dest='extra_dist_dirs', default='', help='Directories in which to look for distributions') add_boolean_option( generic_parser, ["require-perfect-match"], default=False, description=('Whether the dist recipes must perfectly match ' 'those requested')) generic_parser.add_argument( '--local-recipes', '--local_recipes', dest='local_recipes', default='./p4a-recipes', help='Directory to look for local recipes') add_boolean_option( generic_parser, ['copy-libs'], default=False, description='Copy libraries instead of using biglink (Android 4.3+)') self._read_configuration() subparsers = parser.add_subparsers(dest='subparser_name', help='The command to run') def add_parser(subparsers, *args, **kwargs): ''' argparse in python2 doesn't support the aliases option, so we just don't provide the aliases there. ''' if 'aliases' in kwargs and sys.version_info.major < 3: kwargs.pop('aliases') return subparsers.add_parser(*args, **kwargs) parser_recipes = add_parser(subparsers, 'recipes', parents=[generic_parser], help='List the available recipes') parser_recipes.add_argument( "--compact", action="store_true", default=False, help="Produce a compact list suitable for scripting") parser_bootstraps = add_parser(subparsers, 'bootstraps', help='List the available bootstraps', parents=[generic_parser]) parser_clean_all = add_parser(subparsers, 'clean_all', aliases=['clean-all'], help='Delete all builds, dists and caches', parents=[generic_parser]) parser_clean_dists = add_parser(subparsers, 'clean_dists', aliases=['clean-dists'], help='Delete all dists', parents=[generic_parser]) parser_clean_bootstrap_builds = add_parser(subparsers, 'clean_bootstrap_builds', aliases=['clean-bootstrap-builds'], help='Delete all bootstrap builds', parents=[generic_parser]) parser_clean_builds = add_parser(subparsers, 'clean_builds', aliases=['clean-builds'], help='Delete all builds', parents=[generic_parser]) parser_clean_recipe_build = add_parser(subparsers, 'clean_recipe_build', aliases=['clean-recipe-build'], help=('Delete the build components of the given recipe. ' 'By default this will also delete built dists'), parents=[generic_parser]) parser_clean_recipe_build.add_argument('recipe', help='The recipe name') parser_clean_recipe_build.add_argument('--no-clean-dists', default=False, dest='no_clean_dists', action='store_true', help='If passed, do not delete existing dists') parser_clean_download_cache= add_parser(subparsers, 'clean_download_cache', aliases=['clean-download-cache'], help='Delete cached downloads for requirement builds', parents=[generic_parser]) parser_clean_download_cache.add_argument( 'recipes', nargs='*', help=('The recipes to clean (space-separated). If no recipe name is ' 'provided, the entire cache is cleared.')) parser_export_dist = add_parser(subparsers, 'export_dist', aliases=['export-dist'], help='Copy the named dist to the given path', parents=[generic_parser]) parser_export_dist.add_argument('output_dir', help=('The output dir to copy to')) parser_export_dist.add_argument('--symlink', action='store_true', help=('Symlink the dist instead of copying')) parser_apk = add_parser(subparsers, 'apk', help='Build an APK', parents=[generic_parser]) parser_apk.add_argument('--release', dest='build_mode', action='store_const', const='release', default='debug', help='Build the PARSER_APK. in Release mode') parser_apk.add_argument('--keystore', dest='keystore', action='store', default=None, help=('Keystore for JAR signing key, will use jarsigner ' 'default if not specified (release build only)')) parser_apk.add_argument('--signkey', dest='signkey', action='store', default=None, help='Key alias to sign PARSER_APK. with (release build only)') parser_apk.add_argument('--keystorepw', dest='keystorepw', action='store', default=None, help='Password for keystore') parser_apk.add_argument('--signkeypw', dest='signkeypw', action='store', default=None, help='Password for key alias') parser_create = add_parser(subparsers, 'create', help='Compile a set of requirements into a dist', parents=[generic_parser]) parser_archs = add_parser(subparsers, 'archs', help='List the available target architectures', parents=[generic_parser]) parser_distributions = add_parser(subparsers, 'distributions', aliases=['dists'], help='List the currently available (compiled) dists', parents=[generic_parser]) parser_delete_dist = add_parser(subparsers, 'delete_dist', aliases=['delete-dist'], help='Delete a compiled dist', parents=[generic_parser]) parser_sdk_tools = add_parser(subparsers, 'sdk_tools', aliases=['sdk-tools'], help='Run the given binary from the SDK tools dis', parents=[generic_parser]) parser_sdk_tools.add_argument( 'tool', help=('The tool binary name to run')) parser_adb = add_parser(subparsers, 'adb', help='Run adb from the given SDK', parents=[generic_parser]) parser_logcat = add_parser(subparsers, 'logcat', help='Run logcat from the given SDK', parents=[generic_parser]) parser_build_status = add_parser(subparsers, 'build_status', aliases=['build-status'], help='Print some debug information about current built components', parents=[generic_parser]) args, unknown = parser.parse_known_args(sys.argv[1:]) args.unknown_args = unknown self.args = args setup_color(args.color) if args.debug: logger.setLevel(logging.DEBUG) # strip version from requirements, and put them in environ if hasattr(args, 'requirements'): requirements = [] for requirement in split_argument_list(args.requirements): if "==" in requirement: requirement, version = requirement.split(u"==", 1) os.environ["VERSION_{}".format(requirement)] = version info('Recipe {}: version "{}" requested'.format( requirement, version)) requirements.append(requirement) args.requirements = u",".join(requirements) self.ctx = Context() self.storage_dir = args.storage_dir self.ctx.setup_dirs(self.storage_dir) self.sdk_dir = args.sdk_dir self.ndk_dir = args.ndk_dir self.android_api = args.android_api self.ndk_version = args.ndk_version self.ctx.symlink_java_src = args.symlink_java_src self._archs = split_argument_list(args.arch) # AND: Fail nicely if the args aren't handled yet if args.extra_dist_dirs: warning('Received --extra_dist_dirs but this arg currently is not ' 'handled, exiting.') exit(1) self.ctx.local_recipes = args.local_recipes self.ctx.copy_libs = args.copy_libs # Each subparser corresponds to a method getattr(self, args.subparser_name.replace('-', '_'))(args)
def unpack(self, arch): info_main('Unpacking {} for {}'.format(self.name, arch)) build_dir = self.get_build_container_dir(arch) user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower())) if user_dir is not None: info('P4A_{}_DIR exists, symlinking instead'.format( self.name.lower())) if exists(self.get_build_dir(arch)): return shprint(sh.rm, '-rf', build_dir) shprint(sh.mkdir, '-p', build_dir) shprint(sh.rmdir, build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return if self.url is None: info('Skipping {} unpack as no URL is set'.format(self.name)) return filename = shprint(sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename) if ma: # fragmented URL? filename = ma.group(1) with current_directory(build_dir): directory_name = self.get_build_dir(arch) if not exists(directory_name) or not isdir(directory_name): extraction_filename = join(self.ctx.packages_path, self.name, filename) if isfile(extraction_filename): if extraction_filename.endswith('.zip'): try: sh.unzip(extraction_filename) except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2): # return code 1 means unzipping had # warnings but did complete, # apparently happens sometimes with # github zips pass import zipfile fileh = zipfile.ZipFile(extraction_filename, 'r') root_directory = fileh.filelist[0].filename.split( '/')[0] if root_directory != basename(directory_name): shprint(sh.mv, root_directory, directory_name) elif extraction_filename.endswith( ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz')): sh.tar('xf', extraction_filename) root_directory = sh.tar( 'tf', extraction_filename).stdout.decode( 'utf-8').split('\n')[0].split('/')[0] if root_directory != basename(directory_name): shprint(sh.mv, root_directory, directory_name) else: raise Exception( 'Could not extract {} download, it must be .zip, ' '.tar.gz or .tar.bz2 or .tar.xz'.format( extraction_filename)) elif isdir(extraction_filename): mkdir(directory_name) for entry in listdir(extraction_filename): if entry not in ('.git', ): shprint(sh.cp, '-Rv', join(extraction_filename, entry), directory_name) else: raise Exception( 'Given path is neither a file nor a directory: {}'. format(extraction_filename)) else: info('{} is already unpacked, skipping'.format(self.name))
def get_distribution(cls, ctx, name=None, recipes=[], ndk_api=None, force_build=False, extra_dist_dirs=[], require_perfect_match=False, allow_replace_dist=True): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. allow_replace_dist : bool If True, will allow an existing dist with the specified name but incompatible requirements to be overwritten by a new one with the current requirements. ''' existing_dists = Distribution.get_distributions(ctx) needs_build = True # whether the dist needs building, will be returned possible_dists = existing_dists name_match_dist = None # 0) Check if a dist with that name already exists if name is not None and name: possible_dists = [d for d in possible_dists if d.name == name] if possible_dists: name_match_dist = possible_dists[0] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: if ( ndk_api is not None and dist.ndk_api != ndk_api ) or dist.ndk_api is None: continue for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes and ndk API, return it for dist in possible_dists: if force_build: continue if ndk_api is not None and dist.ndk_api != ndk_api: continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): info_notify('{} has compatible recipes, using this one' .format(dist.name)) return dist assert len(possible_dists) < 2 # If there was a name match but we didn't already choose it, # then the existing dist is incompatible with the requested # configuration and the build cannot continue if name_match_dist is not None and not allow_replace_dist: raise BuildInterruptingException( 'Asked for dist with name {name} with recipes ({req_recipes}) and ' 'NDK API {req_ndk_api}, but a dist ' 'with this name already exists and has either incompatible recipes ' '({dist_recipes}) or NDK API {dist_ndk_api}'.format( name=name, req_ndk_api=ndk_api, dist_ndk_api=name_match_dist.ndk_api, req_recipes=', '.join(recipes), dist_recipes=', '.join(name_match_dist.recipes))) # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join(ctx.dist_dir, dist.name) dist.recipes = recipes dist.ndk_api = ctx.ndk_api return dist
def get_distribution(cls, ctx, name=None, recipes=[], force_build=False, extra_dist_dirs=[], require_perfect_match=False): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. ''' # AND: This whole function is a bit hacky, it needs checking # properly to make sure it follows logically correct # possibilities existing_dists = Distribution.get_distributions(ctx) needs_build = True # whether the dist needs building, will be returned possible_dists = existing_dists # 0) Check if a dist with that name already exists if name is not None and name: possible_dists = [d for d in possible_dists if d.name == name] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes, return it P4A_force_build = False for dist in possible_dists: if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): if force_build: # distribution exists, rebuild forced P4A_force_build = True continue else: # existing distribution returned, no build required info_notify( '{} has compatible recipes, using this one'.format( dist.name)) return dist assert len(possible_dists) < 2 if not name and possible_dists: info('Asked for dist with name {} with recipes ({}), but a dist ' 'with this name already exists and has incompatible recipes ' '({})'.format(name, ', '.join(recipes), ', '.join(possible_dists[0].recipes))) info('No compatible dist found, so exiting.') exit(1) # # 2) Check if any downloadable dists meet the requirements # online_dists = [('testsdl2', ['hostpython2', 'sdl2_image', # 'sdl2_mixer', 'sdl2_ttf', # 'python2', 'sdl2', # 'pyjniussdl2', 'kivysdl2'], # 'https://github.com/inclement/sdl2-example-dist/archive/master.zip'), # ] # _possible_dists = [] # for dist_name, dist_recipes, dist_url in online_dists: # for recipe in recipes: # if recipe not in dist_recipes: # break # else: # dist = Distribution(ctx) # dist.name = dist_name # dist.url = dist_url # _possible_dists.append(dist) # # if _possible_dists # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True """Locally modified recipes will be forced to build, others are reused environmental variable P4A_{recipe_name}_DIR points to local recipe P4A_force_build affects all local recipes""" dist.P4A_force_build = P4A_force_build and os.environ.get( 'P4A_force_build') if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join(ctx.dist_dir, dist.name) dist.recipes = recipes return dist
def prebuild_arch(self, arch): """Make the build and target directories""" path = self.get_build_dir(arch.arch) if not exists(path): info("creating {}".format(path)) shprint(sh.mkdir, '-p', path)
def run_pymodules_install(ctx, modules): modules = list(filter(ctx.not_has_package, modules)) if not modules: info('There are no Python modules to install, skipping') return info('The requirements ({}) don\'t have recipes, attempting to install ' 'them with pip'.format(', '.join(modules))) info('If this fails, it may mean that the module has compiled ' 'components and needs a recipe.') venv = sh.Command(ctx.virtualenv) with current_directory(join(ctx.build_dir)): shprint( venv, '--python=python{}.{}'.format( ctx.python_recipe.major_minor_version_string.partition(".")[0], ctx.python_recipe.major_minor_version_string.partition(".") [2]), 'venv') info('Creating a requirements.txt file for the Python modules') with open('requirements.txt', 'w') as fileh: for module in modules: key = 'VERSION_' + module if key in environ: line = '{}=={}\n'.format(module, environ[key]) else: line = '{}\n'.format(module) fileh.write(line) # Prepare base environment and upgrade pip: base_env = copy.copy(os.environ) base_env["PYTHONPATH"] = ctx.get_site_packages_dir() info('Upgrade pip to latest version') shprint(sh.bash, '-c', ("source venv/bin/activate && pip install -U pip"), _env=copy.copy(base_env)) # Install Cython in case modules need it to build: info('Install Cython in case one of the modules needs it to build') shprint(sh.bash, '-c', ("venv/bin/pip install Cython"), _env=copy.copy(base_env)) # Get environment variables for build (with CC/compiler set): standard_recipe = CythonRecipe() standard_recipe.ctx = ctx # (note: following line enables explicit -lpython... linker options) standard_recipe.call_hostpython_via_targetpython = False recipe_env = standard_recipe.get_recipe_env(ctx.archs[0]) env = copy.copy(base_env) env.update(recipe_env) info('Installing Python modules with pip') info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. ' 'A reason for this is often modules compiling ' 'native code that is unaware of Android cross-compilation ' 'and does not work without additional ' 'changes / workarounds.') # Make sure our build package dir is available, and the virtualenv # site packages come FIRST (so the proper pip version is used): env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir() env["PYTHONPATH"] = os.path.abspath( join(ctx.build_dir, "venv", "lib", "python" + ctx.python_recipe.major_minor_version_string, "site-packages")) + ":" + env["PYTHONPATH"] ''' # Do actual install: shprint(sh.bash, '-c', ( "venv/bin/pip " + "install -v --target '{0}' --no-deps -r requirements.txt" ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")), _env=copy.copy(env)) ''' # use old install script shprint(sh.bash, '-c', ( "source venv/bin/activate && env CC=/bin/false CXX=/bin/false " "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt" ).format(ctx.get_site_packages_dir())) # Strip object files after potential Cython or native code builds: standard_recipe.strip_object_files(ctx.archs[0], env, build_dir=ctx.build_dir)
def distribute_javaclasses(self, javaclass_dir): '''Copy existing javaclasses from build dir to current dist dir.''' info('Copying java files') for filename in glob.glob(javaclass_dir): shprint(sh.cp, '-a', filename, 'src')
def distribute_aars(self, arch): '''Process existing .aar bundles and copy to current dist dir.''' info('Unpacking aars') for aar in glob.glob(join(self.ctx.aars_dir, '*.aar')): self._unpack_aar(aar, arch)
def install_python_package(self, arch, name=None, env=None, is_dir=True): '''Automate the installation of a Python package (or a cython package where the cython components are pre-built).''' # arch = self.filtered_archs[0] # old kivy-ios way if name is None: name = self.name if env is None: env = self.get_recipe_env(arch) info('Installing {} into site-packages'.format(self.name)) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) # hostpython = sh.Command('python3.5') if self.ctx.python_recipe.from_crystax: # hppath = join(dirname(self.hostpython_location), 'Lib', # 'site-packages') hpenv = env.copy() # if 'PYTHONPATH' in hpenv: # hpenv['PYTHONPATH'] = ':'.join([hppath] + # hpenv['PYTHONPATH'].split(':')) # else: # hpenv['PYTHONPATH'] = hppath # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir() # shprint(hostpython, 'setup.py', 'build', # _env=hpenv, *self.setup_extra_args) shprint( hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions) _env=hpenv, *self.setup_extra_args) # site_packages_dir = self.ctx.get_site_packages_dir() # built_files = glob.glob(join('build', 'lib*', '*')) # for filen in built_files: # shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1])) elif self.call_hostpython_via_targetpython: shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, *self.setup_extra_args) else: hppath = join(dirname(self.hostpython_location), 'Lib', 'site-packages') hpenv = env.copy() if 'PYTHONPATH' in hpenv: hpenv['PYTHONPATH'] = ':'.join( [hppath] + hpenv['PYTHONPATH'].split(':')) else: hpenv['PYTHONPATH'] = hppath shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=lib/python2.7/site-packages', _env=hpenv, *self.setup_extra_args) # AND: Hardcoded python2.7 needs fixing # If asked, also install in the hostpython build dir if self.install_in_hostpython: self.install_hostpython_package(arch)
def apk(self, args): '''Create an APK using the given distribution.''' ap = argparse.ArgumentParser(description='Build an APK') ap.add_argument('--release', dest='build_mode', action='store_const', const='release', default='debug', help='Build the APK in Release mode') ap.add_argument( '--keystore', dest='keystore', action='store', default=None, help=('Keystore for JAR signing key, will use jarsigner ' 'default if not specified (release build only)')) ap.add_argument('--signkey', dest='signkey', action='store', default=None, help='Key alias to sign APK with (release build only)') ap.add_argument('--keystorepw', dest='keystorepw', action='store', default=None, help='Password for keystore') ap.add_argument('--signkeypw', dest='signkeypw', action='store', default=None, help='Password for key alias') apk_args, args = ap.parse_known_args(args) ctx = self.ctx dist = self._dist # Manually fixing these arguments at the string stage is # unsatisfactory and should probably be changed somehow, but # we can't leave it until later as the build.py scripts assume # they are in the current directory. fix_args = ('--dir', '--private', '--add-jar', '--add-source', '--whitelist', '--blacklist', '--presplash', '--icon') for i, arg in enumerate(args[:-1]): argx = arg.split('=') if argx[0] in fix_args: if len(argx) > 1: args[i] = '='.join( (argx[0], realpath(expanduser(argx[1])))) else: args[i + 1] = realpath(expanduser(args[i + 1])) env = os.environ.copy() if apk_args.build_mode == 'release': if apk_args.keystore: env['P4A_RELEASE_KEYSTORE'] = realpath( expanduser(apk_args.keystore)) if apk_args.signkey: env['P4A_RELEASE_KEYALIAS'] = apk_args.signkey if apk_args.keystorepw: env['P4A_RELEASE_KEYSTORE_PASSWD'] = apk_args.keystorepw if apk_args.signkeypw: env['P4A_RELEASE_KEYALIAS_PASSWD'] = apk_args.signkeypw elif apk_args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env: env['P4A_RELEASE_KEYALIAS_PASSWD'] = apk_args.keystorepw build = imp.load_source('build', join(dist.dist_dir, 'build.py')) with current_directory(dist.dist_dir): build_args = build.parse_args(args) output = shprint(sh.ant, apk_args.build_mode, _tail=20, _critical=True, _env=env) info_main('# Copying APK to current directory') apk_re = re.compile(r'.*Please sign (/.*\.apk) manually$') apk_file = None for line in reversed(output.splitlines()): m = apk_re.match(line) if m: apk_file = m.groups()[0] break if not apk_file: info_main( '# APK filename not found in build output, trying to guess') apks = glob.glob( join(dist.dist_dir, 'bin', '*-*-{}.apk'.format(apk_args.build_mode))) if len(apks) == 0: raise ValueError('Couldn\'t find the built APK') if len(apks) > 1: info('More than one built APK found...guessing you ' 'just built {}'.format(apks[-1])) apk_file = apks[-1] info_main('# Found APK file: {}'.format(apk_file)) shprint(sh.cp, apk_file, './')