def build_arch(self, arch):
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, self.build_subdir)
ensure_dir(build_dir)
if not exists(join(build_dir, 'python')):
with current_directory(recipe_build_dir):
# Configure the build
with current_directory(build_dir):
if not exists('config.status'):
shprint(
sh.Command(join(recipe_build_dir, 'configure')))
# Create the Setup file. This copying from Setup.dist
# seems to be the normal and expected procedure.
shprint(sh.cp, join('Modules', 'Setup.dist'),
join(build_dir, 'Modules', 'Setup'))
result = shprint(sh.make, '-C', build_dir)
else:
info('Skipping {name} ({version}) build, as it has already '
'been completed'.format(name=self.name, version=self.version))
self.ctx.hostpython = join(build_dir, 'python')
def build_arch(self, arch):
info("Extracting CrystaX python3 from NDK package")
dirn = self.ctx.get_python_install_dir()
ensure_dir(dirn)
self.ctx.hostpython = "python{}".format(self.version)
def clean_bootstrap_builds(self, args):
'''Delete all the bootstrap builds.'''
for bs in Bootstrap.list_bootstraps():
bs = Bootstrap.get_bootstrap(bs, self.ctx)
if bs.build_dir and exists(bs.build_dir):
info('Cleaning build for {} bootstrap.'.format(bs.name))
shutil.rmtree(bs.build_dir)
def clean_build(self, arch=None):
"""Deletes all the build information of the recipe.
If arch is not None, only this arch dir is deleted. Otherwise
(the default) all builds for all archs are deleted.
By default, this just deletes the main build dir. If the
recipe has e.g. object files biglinked, or .so files stored
elsewhere, you should override this method.
This method is intended for testing purposes, it may have
strange results. Rebuild everything if this seems to happen.
"""
if arch is None:
base_dir = join(self.ctx.build_dir, "other_builds", self.name)
else:
base_dir = self.get_build_container_dir(arch)
dirs = glob.glob(base_dir + "-*")
if exists(base_dir):
dirs.append(base_dir)
if not dirs:
warning(("Attempted to clean build for {} but found no existing " "build dirs").format(self.name))
for directory in dirs:
if exists(directory):
info("Deleting {}".format(directory))
shutil.rmtree(directory)
def download_if_necessary(self):
info_main("Downloading {}".format(self.name))
user_dir = environ.get("P4A_{}_DIR".format(self.name.lower()))
if user_dir is not None:
info("P4A_{}_DIR is set, skipping download for {}".format(self.name, self.name))
return
self.download()
def build_recipes(build_order, python_modules, ctx):
# Put recipes in correct build order
bs = ctx.bootstrap
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'
.format(recipe.name))
# 4) biglink everything
# AND: Should make this optional
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx, python_modules)
return
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
try:
shprint(hostpython, 'setup.py', 'build_ext', _env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
info('Running cython where appropriate')
shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx',
'-exec', self.ctx.cython, '{}', ';', _env=env)
info('ran cython')
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
_tail=20, _critical=True, *self.setup_extra_args)
print('stripping')
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
print('stripped!?')
def extract_source(self, source, cwd):
"""
(internal) Extract the `source` into the directory `cwd`.
"""
if not source:
return
if isfile(source):
info("Extract {} into {}".format(source, cwd))
if source.endswith(".tgz") or source.endswith(".tar.gz"):
shprint(sh.tar, "-C", cwd, "-xvzf", source)
elif source.endswith(".tbz2") or source.endswith(".tar.bz2"):
shprint(sh.tar, "-C", cwd, "-xvjf", source)
elif source.endswith(".zip"):
zf = zipfile.ZipFile(source)
zf.extractall(path=cwd)
zf.close()
else:
warning("Error: cannot extract, unrecognized extension for {}".format(source))
raise Exception()
elif isdir(source):
info("Copying {} into {}".format(source, cwd))
shprint(sh.cp, "-a", source, cwd)
else:
warning("Error: cannot extract or copy, unrecognized path {}".format(source))
raise Exception()
def clean_build(self, arch=None):
'''Deletes all the build information of the recipe.
If arch is not None, only this arch dir is deleted. Otherwise
(the default) all builds for all archs are deleted.
By default, this just deletes the main build dir. If the
recipe has e.g. object files biglinked, or .so files stored
elsewhere, you should override this method.
This method is intended for testing purposes, it may have
strange results. Rebuild everything if this seems to happen.
'''
if arch is None:
base_dir = join(self.ctx.build_dir, 'other_builds', self.name)
else:
base_dir = self.get_build_container_dir(arch)
dirs = glob.glob(base_dir + '-*')
if exists(base_dir):
dirs.append(base_dir)
if not dirs:
warning(('Attempted to clean build for {} but found no existing '
'build dirs').format(self.name))
for directory in dirs:
if exists(directory):
info('Deleting {}'.format(directory))
shutil.rmtree(directory)
# Delete any Python distributions to ensure the recipe build
# doesn't persist in site-packages
shutil.rmtree(self.ctx.python_installs_dir)
def rebuild_compiled_components(self, arch, env):
info('Rebuilding compiled components in {}'.format(self.name))
hostpython = sh.Command(self.real_hostpython_location)
shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
*self.setup_extra_args)
def should_build(self, arch):
name = self.folder_name
if self.ctx.has_package(name):
info('Python package already exists in site-packages')
return False
info('{} apparently isn\'t already in site-packages'.format(name))
return True
def build_dist_from_args(ctx, dist, args):
'''Parses out any bootstrap related arguments, and uses them to build
a dist.'''
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
build_order, python_modules, bs \
= get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
ctx.recipe_build_order = build_order
info('The selected bootstrap is {}'.format(bs.name))
info_main('# Creating dist with {} bootstrap'.format(bs.name))
bs.distribution = dist
info_notify('Dist will have name {} and recipes ({})'.format(
dist.name, ', '.join(dist.recipes)))
ctx.dist_name = bs.distribution.name
ctx.prepare_bootstrap(bs)
ctx.prepare_dist(ctx.dist_name)
build_recipes(build_order, python_modules, ctx)
ctx.bootstrap.run_distribute()
info_main('# Your distribution was created successfully, exiting.')
info('Dist can be found at (for now) {}'
.format(join(ctx.dist_dir, ctx.dist_name)))
def prebuild_arch(self, arch):
if not self.is_patched(arch):
super(ReportLabRecipe, self).prebuild_arch(arch)
self.apply_patch('patches/fix-setup.patch', arch.arch)
recipe_dir = self.get_build_dir(arch.arch)
shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
ft = self.get_recipe('freetype', self.ctx)
ft_dir = ft.get_build_dir(arch.arch)
ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
info('reportlab recipe: ft_dir={}'.format(ft_dir))
info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
with current_directory(recipe_dir):
sh.ls('-lathr')
ensure_dir(tmp_dir)
pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
if not os.path.isfile(pfbfile):
sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
if os.path.isfile("setup.py"):
with open('setup.py', 'rb') as f:
text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
with open('setup.py', 'wb') as f:
f.write(text)
def build_arch(self, arch):
"""simple shared compile"""
env = self.get_recipe_env(arch, with_flags_in_cc=False)
for path in (
self.get_build_dir(arch.arch),
join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'),
join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')):
if not exists(path):
info("creating {}".format(path))
shprint(sh.mkdir, '-p', path)
cli = env['CC'].split()[0]
# makes sure first CC command is the compiler rather than ccache, refs:
# https://github.com/kivy/python-for-android/issues/1399
if 'ccache' in cli:
cli = env['CC'].split()[1]
cc = sh.Command(cli)
with current_directory(self.get_build_dir(arch.arch)):
cflags = env['CFLAGS'].split()
cflags.extend(['-I.', '-c', '-l.', 'glob.c', '-I.'])
shprint(cc, *cflags, _env=env)
cflags = env['CFLAGS'].split()
cflags.extend(['-shared', '-I.', 'glob.o', '-o', 'libglob.so'])
cflags.extend(env['LDFLAGS'].split())
shprint(cc, *cflags, _env=env)
shprint(sh.cp, 'libglob.so', join(self.ctx.libs_dir, arch.arch))
def strip_libraries(self, arch):
info('Stripping libraries')
if self.ctx.python_recipe.from_crystax:
info('Python was loaded from CrystaX, skipping strip')
return
env = arch.get_env()
tokens = shlex.split(env['STRIP'])
strip = sh.Command(tokens[0])
if len(tokens) > 1:
strip = strip.bake(tokens[1:])
libs_dir = join(self.dist_dir, '_python_bundle',
'_python_bundle', 'modules')
if self.ctx.python_recipe.name == 'python2legacy':
libs_dir = join(self.dist_dir, 'private')
filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'),
'-iname', '*.so', _env=env).stdout.decode('utf-8')
logger.info('Stripping libraries in private dir')
for filen in filens.split('\n'):
if not filen:
continue # skip the last ''
try:
strip(filen, _env=env)
except sh.ErrorReturnCode_1:
logger.debug('Failed to strip ' + filen)
def set_libs_flags(self, env, arch):
'''Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
'''
if 'libffi' in self.ctx.recipe_build_order:
info('Activating flags for libffi')
recipe = Recipe.get_recipe('libffi', self.ctx)
include = ' -I' + ' -I'.join(recipe.get_include_dirs(arch))
ldflag = ' -L' + join(recipe.get_build_dir(arch.arch),
recipe.get_host(arch), '.libs') + ' -lffi'
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include
env['LDFLAGS'] = env.get('LDFLAGS', '') + ldflag
if 'openssl' in self.ctx.recipe_build_order:
recipe = Recipe.get_recipe('openssl', self.ctx)
openssl_build_dir = recipe.get_build_dir(arch.arch)
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal)
shprint(sh.sed, '-i.backup', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal)
env['OPENSSL_VERSION'] = recipe.version
if 'sqlite3' in self.ctx.recipe_build_order:
# Include sqlite3 in python2 build
recipe = Recipe.get_recipe('sqlite3', self.ctx)
include = ' -I' + recipe.get_build_dir(arch.arch)
lib = ' -L' + recipe.get_lib_dir(arch) + ' -lsqlite3'
# Insert or append to env
flag = 'CPPFLAGS'
env[flag] = env[flag] + include if flag in env else include
flag = 'LDFLAGS'
env[flag] = env[flag] + lib if flag in env else lib
return env
def apk(self, args):
'''Create an APK using the given distribution.'''
# AND: Need to add a parser here for any extra options
# parser = argparse.ArgumentParser(
# description='Build an APK')
# args = parser.parse_args(args)
ctx = self.ctx
dist = self._dist
# Manually fixing these arguments at the string stage is
# unsatisfactory and should probably be changed somehow, but
# we can't leave it until later as the build.py scripts assume
# they are in the current directory.
for i, arg in enumerate(args[:-1]):
if arg in ('--dir', '--private'):
args[i+1] = realpath(expanduser(args[i+1]))
build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
with current_directory(dist.dist_dir):
build.parse_args(args)
shprint(sh.ant, 'debug', _tail=20, _critical=True)
# AND: This is very crude, needs improving. Also only works
# for debug for now.
info_main('# Copying APK to current directory')
apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-debug.apk'))
if len(apks) == 0:
raise ValueError('Couldn\'t find the built APK')
if len(apks) > 1:
info('More than one built APK found...guessing you '
'just built {}'.format(apks[-1]))
shprint(sh.cp, apks[-1], './')
def build_arch(self, arch):
env = self.get_recipe_env(arch)
env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format(
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl/include -I{jni_path}/sdl_mixer'.format(
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl_ttf -I{jni_path}/sdl_image'.format(
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
debug('pygame cflags', env['CFLAGS'])
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{libs_path} -L{src_path}/obj/local/{arch} -lm -lz'.format(
libs_path=self.ctx.libs_dir, src_path=self.ctx.bootstrap.build_dir, arch=env['ARCH'])
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
with current_directory(self.get_build_dir(arch.arch)):
info('hostpython is ' + self.ctx.hostpython)
hostpython = sh.Command(self.ctx.hostpython)
shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
_tail=10, _critical=True)
info('strip is ' + env['STRIP'])
build_lib = glob.glob('./build/lib*')
assert len(build_lib) == 1
print('stripping pygame')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';')
python_install_path = join(self.ctx.build_dir, 'python-install')
warning('Should remove pygame tests etc. here, but skipping for now')
def delete_dist(self, _args):
dist = self._dist
if not dist.folder_exists():
info('No dist exists that matches your specifications, '
'exiting without deleting.')
return
dist.delete()
def build_cython_components(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
info('hostpython is ' + self.ctx.hostpython)
hostpython = sh.Command(self.ctx.hostpython)
app_mk = join(self.get_build_dir(arch.arch), 'Application.mk')
if not exists(app_mk):
shprint(sh.cp, join(self.get_recipe_dir(), 'Application.mk'), app_mk)
app_setup = join(self.get_build_dir(arch.arch), 'setup.py')
if not exists(app_setup):
shprint(sh.cp, join(self.get_recipe_dir(), 'setup.py'), app_setup)
# This first attempt *will* fail, because cython isn't
# installed in the hostpython
try:
shprint(hostpython, 'setup.py', 'build_ext', _env=env)
except sh.ErrorReturnCode_1:
pass
# ...so we manually run cython from the user's system
shprint(sh.find, self.get_build_dir('armeabi'), '-iname', '*.pyx', '-exec',
self.ctx.cython, '{}', ';', _env=env)
# now cython has already been run so the build works
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env)
# stripping debug symbols lowers the file size a lot
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
def prebuild_arch(self, arch):
super(MobileInsightRecipe, self).prebuild_arch(arch)
build_dir = self.get_build_dir(arch.arch)
tmp_dir = join(build_dir, 'mi_tmp')
info("Cleaning old MobileInsight-core sources at {}".format(build_dir))
try:
shprint(sh.rm, '-r',
build_dir,
_tail = 20,
_critical = True)
except:
pass
if LOCAL_DEBUG is False:
info("Cloning MobileInsight-core sources from {}".format(self.mi_git))
shprint(sh.git,
'clone', '-b',
self.mi_branch,
'--depth=1',
self.mi_git,
tmp_dir,
_tail = 20,
_critical = True)
else:
warning("Debugging using local sources of MobileInsight at {}".format(self.local_src))
shprint(sh.mkdir,
build_dir,
_tail = 20,
_critical = True)
shprint(sh.mkdir,
tmp_dir,
_tail = 20,
_critical = True)
shprint(sh.cp,
'-fr',
self.local_src,
tmp_dir,
_tail = 20,
_critical = True)
tmp_dir = join(tmp_dir, 'MobileInsight-core')
shprint(sh.mv,
join(tmp_dir, 'mobile_insight'),
build_dir,
_tail = 20,
_critical = True)
shprint(sh.mv,
join(tmp_dir, 'dm_collector_c'),
build_dir,
_tail = 20,
_critical = True)
# remove unnecessary codes
shprint(sh.rm, '-r', tmp_dir,
_tail = 20,
_critical = True)
self.get_newest_toolchain(arch)
def build_dist_from_args(ctx, dist, args):
'''Parses out any bootstrap related arguments, and uses them to build
a dist.'''
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
build_order, python_modules, bs \
= get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
ctx.recipe_build_order = build_order
ctx.python_modules = python_modules
if python_modules and hasattr(sys, 'real_prefix'):
error('virtualenv is needed to install pure-Python modules, but')
error('virtualenv does not support nesting, and you are running')
error('python-for-android in one. Please run p4a outside of a')
error('virtualenv instead.')
exit(1)
info('The selected bootstrap is {}'.format(bs.name))
info_main('# Creating dist with {} bootstrap'.format(bs.name))
bs.distribution = dist
info_notify('Dist will have name {} and recipes ({})'.format(
dist.name, ', '.join(dist.recipes)))
ctx.dist_name = bs.distribution.name
ctx.prepare_bootstrap(bs)
ctx.prepare_dist(ctx.dist_name)
build_recipes(build_order, python_modules, ctx)
ctx.bootstrap.run_distribute()
info_main('# Your distribution was created successfully, exiting.')
info('Dist can be found at (for now) {}'
.format(join(ctx.dist_dir, ctx.dist_name)))
def _unpack_aar(self, aar, arch):
'''Unpack content of .aar bundle and copy to current dist dir.'''
with temp_directory() as temp_dir:
name = splitext(basename(aar))[0]
jar_name = name + '.jar'
info("unpack {} aar".format(name))
debug(" from {}".format(aar))
debug(" to {}".format(temp_dir))
shprint(sh.unzip, '-o', aar, '-d', temp_dir)
jar_src = join(temp_dir, 'classes.jar')
jar_tgt = join('libs', jar_name)
debug("copy {} jar".format(name))
debug(" from {}".format(jar_src))
debug(" to {}".format(jar_tgt))
ensure_dir('libs')
shprint(sh.cp, '-a', jar_src, jar_tgt)
so_src_dir = join(temp_dir, 'jni', arch.arch)
so_tgt_dir = join('libs', arch.arch)
debug("copy {} .so".format(name))
debug(" from {}".format(so_src_dir))
debug(" to {}".format(so_tgt_dir))
ensure_dir(so_tgt_dir)
so_files = glob.glob(join(so_src_dir, '*.so'))
for f in so_files:
shprint(sh.cp, '-a', f, so_tgt_dir)
def build_arch(self, arch):
"""simple shared compile"""
env = self.get_recipe_env(arch, with_flags_in_cc=False)
for path in (
self.get_build_dir(arch.arch),
join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'),
join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')):
if not exists(path):
info("creating {}".format(path))
shprint(sh.mkdir, '-p', path)
cli = env['CC'].split()
cc = sh.Command(cli[0])
with current_directory(self.get_build_dir(arch.arch)):
cflags = env['CFLAGS'].split()
cflags.extend(['-I.', '-c', '-l.', 'ifaddrs.c', '-I.'])
shprint(cc, *cflags, _env=env)
cflags = env['CFLAGS'].split()
cflags.extend(['-shared', '-I.', 'ifaddrs.o', '-o', 'libifaddrs.so'])
cflags.extend(env['LDFLAGS'].split())
shprint(cc, *cflags, _env=env)
shprint(sh.cp, 'libifaddrs.so', self.ctx.get_libs_dir(arch.arch))
shprint(sh.cp, "libifaddrs.so", join(self.ctx.get_python_install_dir(), 'lib'))
# drop header in to the Python include directory
python_version = self.ctx.python_recipe.version[0:3]
shprint(sh.cp, "ifaddrs.h",
join(
self.ctx.get_python_install_dir(),
'include/python{}'.format(python_version))
)
include_path = join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')
shprint(sh.cp, "ifaddrs.h", include_path)
def strip_libraries(self, arch):
info('Stripping libraries')
if self.ctx.python_recipe.from_crystax:
info('Python was loaded from CrystaX, skipping strip')
return
env = arch.get_env()
strip = which('arm-linux-androideabi-strip', env['PATH'])
if strip is None:
warning('Can\'t find strip in PATH...')
return
strip = sh.Command(strip)
libs_dir = join(self.dist_dir, '_python_bundle',
'_python_bundle', 'modules')
if self.ctx.python_recipe.name == 'python2legacy':
libs_dir = join(self.dist_dir, 'private')
filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'),
'-iname', '*.so', _env=env).stdout.decode('utf-8')
logger.info('Stripping libraries in private dir')
for filen in filens.split('\n'):
try:
strip(filen, _env=env)
except sh.ErrorReturnCode_1:
logger.debug('Failed to strip ' + filen)
def cythonize_build(self, env, build_dir="."):
if not self.cythonize:
info('Running cython cancelled per recipe setting')
return
info('Running cython where appropriate')
for root, dirnames, filenames in walk("."):
for filename in fnmatch.filter(filenames, "*.pyx"):
self.cythonize_file(env, build_dir, join(root, filename))
def distribute_libs(self, arch, src_dirs, wildcard='*', dest_dir="libs"):
'''Copy existing arch libs from build dirs to current dist dir.'''
info('Copying libs')
tgt_dir = join(dest_dir, arch.arch)
ensure_dir(tgt_dir)
for src_dir in src_dirs:
for lib in glob.glob(join(src_dir, wildcard)):
shprint(sh.cp, '-a', lib, tgt_dir)
def append_file(self, filename, dest):
info("Append {} to {}".format(filename, dest))
filename = join(self.recipe_dir, filename)
dest = join(self.build_dir, dest)
with open(filename, "rb") as fd:
data = fd.read()
with open(dest, "ab") as fd:
fd.write(data)
def apply_patch(self, filename, arch):
"""
Apply a patch from the current recipe directory into the current
build directory.
"""
info("Applying patch {}".format(filename))
filename = join(self.recipe_dir, filename)
shprint(sh.patch, "-t", "-d", self.get_build_dir(arch), "-p1", "-i", filename, _tail=10)
def install_python_package(self, arch, name=None, env=None, is_dir=True):
'''Automate the installation of a Python package (or a cython
package where the cython components are pre-built).'''
# arch = self.filtered_archs[0] # old kivy-ios way
if name is None:
name = self.name
if env is None:
env = self.get_recipe_env(arch)
info('Installing {} into site-packages'.format(self.name))
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.hostpython_location)
# hostpython = sh.Command('python3.5')
if self.ctx.python_recipe.from_crystax:
# hppath = join(dirname(self.hostpython_location), 'Lib',
# 'site-packages')
hpenv = env.copy()
# if 'PYTHONPATH' in hpenv:
# hpenv['PYTHONPATH'] = ':'.join([hppath] +
# hpenv['PYTHONPATH'].split(':'))
# else:
# hpenv['PYTHONPATH'] = hppath
# hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
# shprint(hostpython, 'setup.py', 'build',
# _env=hpenv, *self.setup_extra_args)
shprint(hostpython, 'setup.py', 'install', '-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=.',
# AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
_env=hpenv, *self.setup_extra_args)
# site_packages_dir = self.ctx.get_site_packages_dir()
# built_files = glob.glob(join('build', 'lib*', '*'))
# for filen in built_files:
# shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
elif self.call_hostpython_via_targetpython:
shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
*self.setup_extra_args)
else:
hppath = join(dirname(self.hostpython_location), 'Lib',
'site-packages')
hpenv = env.copy()
if 'PYTHONPATH' in hpenv:
hpenv['PYTHONPATH'] = ':'.join([hppath] +
hpenv['PYTHONPATH'].split(':'))
else:
hpenv['PYTHONPATH'] = hppath
shprint(hostpython, 'setup.py', 'install', '-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=lib/python2.7/site-packages',
_env=hpenv, *self.setup_extra_args)
# AND: Hardcoded python2.7 needs fixing
# If asked, also install in the hostpython build dir
if self.install_in_hostpython:
self.install_hostpython_package(arch)
def delete_dist(self, args):
dist = self._dist
if dist.needs_build:
info('No dist exists that matches your specifications, '
'exiting without deleting.')
shutil.rmtree(dist.dist_dir)
Out_Fore, Err_Style, Err_Fore,
info_notify, info_main, shprint,
Null_Fore, Null_Style)
from pythonforandroid.util import current_directory, ensure_dir
from pythonforandroid.bootstrap import Bootstrap
from pythonforandroid.distribution import Distribution, pretty_log_dists
from pythonforandroid.graph import get_recipe_order_and_bootstrap
from pythonforandroid.build import Context, build_recipes
user_dir = dirname(realpath(os.path.curdir))
toolchain_dir = dirname(__file__)
sys.path.insert(0, join(toolchain_dir, "tools", "external"))
info(''.join([
Err_Style.BRIGHT, Err_Fore.RED,
'This python-for-android revamp is an experimental alpha release!',
Err_Style.RESET_ALL
]))
info(''.join([
Err_Fore.RED,
('It should work (mostly), but you may experience '
'missing features or bugs.'), Err_Style.RESET_ALL
]))
def add_boolean_option(parser,
names,
no_names=None,
default=True,
dest=None,
description=None):
def distribute_javaclasses(self, javaclass_dir, dest_dir="src"):
'''Copy existing javaclasses from build dir to current dist dir.'''
info('Copying java files')
ensure_dir(dest_dir)
filenames = glob.glob(javaclass_dir)
shprint(sh.cp, '-a', *filenames, dest_dir)
def apk(self, args):
'''Create an APK using the given distribution.'''
ctx = self.ctx
dist = self._dist
# Manually fixing these arguments at the string stage is
# unsatisfactory and should probably be changed somehow, but
# we can't leave it until later as the build.py scripts assume
# they are in the current directory.
fix_args = ('--dir', '--private', '--add-jar', '--add-source',
'--whitelist', '--blacklist', '--presplash', '--icon')
unknown_args = args.unknown_args
for i, arg in enumerate(unknown_args[:-1]):
argx = arg.split('=')
if argx[0] in fix_args:
if len(argx) > 1:
unknown_args[i] = '='.join((argx[0],
realpath(expanduser(argx[1]))))
else:
unknown_args[i+1] = realpath(expanduser(unknown_args[i+1]))
env = os.environ.copy()
if args.build_mode == 'release':
if args.keystore:
env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore))
if args.signkey:
env['P4A_RELEASE_KEYALIAS'] = args.signkey
if args.keystorepw:
env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
if args.signkeypw:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw
build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
with current_directory(dist.dist_dir):
build_args = build.parse_args(args.unknown_args)
output = shprint(sh.ant, args.build_mode, _tail=20, _critical=True, _env=env)
info_main('# Copying APK to current directory')
apk_re = re.compile(r'.*Package: (.*\.apk)$')
apk_file = None
for line in reversed(output.splitlines()):
m = apk_re.match(line)
if m:
apk_file = m.groups()[0]
break
if not apk_file:
info_main('# APK filename not found in build output, trying to guess')
suffix = args.build_mode
if suffix == 'release':
suffix = suffix + '-unsigned'
apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-{}.apk'.format(suffix)))
if len(apks) == 0:
raise ValueError('Couldn\'t find the built APK')
if len(apks) > 1:
info('More than one built APK found...guessing you '
'just built {}'.format(apks[-1]))
apk_file = apks[-1]
info_main('# Found APK file: {}'.format(apk_file))
shprint(sh.cp, apk_file, './')
def unpack(self, arch):
info_main('Unpacking {} for {}'.format(self.name, arch))
build_dir = self.get_build_container_dir(arch)
user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
if user_dir is not None:
info('P4A_{}_DIR exists, symlinking instead'.format(
self.name.lower()))
# AND: Currently there's something wrong if I use ln, fix this
warning('Using cp -a instead of symlink...fix this!')
if exists(self.get_build_dir(arch)):
return
shprint(sh.rm, '-rf', build_dir)
shprint(sh.mkdir, '-p', build_dir)
shprint(sh.rmdir, build_dir)
ensure_dir(build_dir)
shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
return
if self.url is None:
info('Skipping {} unpack as no URL is set'.format(self.name))
return
filename = shprint(sh.basename,
self.versioned_url).stdout[:-1].decode('utf-8')
with current_directory(build_dir):
directory_name = self.get_build_dir(arch)
# AND: Could use tito's get_archive_rootdir here
if not exists(directory_name) or not isdir(directory_name):
extraction_filename = join(self.ctx.packages_path, self.name,
filename)
if isfile(extraction_filename):
if extraction_filename.endswith('.zip'):
sh.unzip(extraction_filename)
import zipfile
fileh = zipfile.ZipFile(extraction_filename, 'r')
root_directory = fileh.filelist[0].filename.split(
'/')[0]
if root_directory != directory_name:
shprint(sh.mv, root_directory, directory_name)
elif (extraction_filename.endswith('.tar.gz')
or extraction_filename.endswith('.tgz')
or extraction_filename.endswith('.tar.bz2')
or extraction_filename.endswith('.tbz2')
or extraction_filename.endswith('.tar.xz')
or extraction_filename.endswith('.txz')):
sh.tar('xf', extraction_filename)
root_directory = shprint(
sh.tar, 'tf', extraction_filename).stdout.decode(
'utf-8').split('\n')[0].split('/')[0]
if root_directory != directory_name:
shprint(sh.mv, root_directory, directory_name)
else:
raise Exception(
'Could not extract {} download, it must be .zip, '
'.tar.gz or .tar.bz2 or .tar.xz')
elif isdir(extraction_filename):
mkdir(directory_name)
for entry in listdir(extraction_filename):
if entry not in ('.git', ):
shprint(sh.cp, '-Rv',
join(extraction_filename, entry),
directory_name)
else:
raise Exception(
'Given path is neither a file nor a directory: {}'.
format(extraction_filename))
else:
info('{} is already unpacked, skipping'.format(self.name))
def run_pymodules_install(ctx,
modules,
project_dir=None,
ignore_setup_py=False):
""" This function will take care of all non-recipe things, by:
1. Processing them from --requirements (the modules argument)
and installing them
2. Installing the user project/app itself via setup.py if
ignore_setup_py=True
"""
info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
modules = list(filter(ctx.not_has_package, modules))
# Bail out if no python deps and no setup.py to process:
if not modules and (ignore_setup_py or project_dir is None
or not project_has_setup_py(project_dir)):
info('No Python modules and no setup.py to process, skipping')
return
# Output messages about what we're going to do:
if modules:
info('The requirements ({}) don\'t have recipes, attempting to '
'install them with pip'.format(', '.join(modules)))
info('If this fails, it may mean that the module has compiled '
'components and needs a recipe.')
if project_dir is not None and \
project_has_setup_py(project_dir) and not ignore_setup_py:
info('Will process project install, if it fails then the '
'project may not be compatible for Android install.')
venv = sh.Command(ctx.virtualenv)
with current_directory(join(ctx.build_dir)):
shprint(
venv, '--python=python{}'.format(
ctx.python_recipe.major_minor_version_string.partition(".")
[0]), 'venv')
# Prepare base environment and upgrade pip:
base_env = copy.copy(os.environ)
base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
info('Upgrade pip to latest version')
shprint(sh.bash,
'-c', ("source venv/bin/activate && pip install -U pip"),
_env=copy.copy(base_env))
# Install Cython in case modules need it to build:
info('Install Cython in case one of the modules needs it to build')
shprint(sh.bash,
'-c', ("venv/bin/pip install Cython"),
_env=copy.copy(base_env))
# Get environment variables for build (with CC/compiler set):
standard_recipe = CythonRecipe()
standard_recipe.ctx = ctx
# (note: following line enables explicit -lpython... linker options)
standard_recipe.call_hostpython_via_targetpython = False
recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
env = copy.copy(base_env)
env.update(recipe_env)
# Make sure our build package dir is available, and the virtualenv
# site packages come FIRST (so the proper pip version is used):
env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
env["PYTHONPATH"] = os.path.abspath(
join(ctx.build_dir, "venv", "lib",
"python" + ctx.python_recipe.major_minor_version_string,
"site-packages")) + ":" + env["PYTHONPATH"]
# Install the manually specified requirements first:
if not modules:
info('There are no Python modules to install, skipping')
else:
info('Creating a requirements.txt file for the Python modules')
with open('requirements.txt', 'w') as fileh:
for module in modules:
key = 'VERSION_' + module
if key in environ:
line = '{}=={}\n'.format(module, environ[key])
else:
line = '{}\n'.format(module)
fileh.write(line)
info('Installing Python modules with pip')
info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
'A reason for this is often modules compiling '
'native code that is unaware of Android cross-compilation '
'and does not work without additional '
'changes / workarounds.')
shprint(sh.bash,
'-c',
("venv/bin/pip " +
"install -v --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir().replace(
"'", "'\"'\"'")),
_env=copy.copy(env))
# Afterwards, run setup.py if present:
if project_dir is not None and (project_has_setup_py(project_dir)
and not ignore_setup_py):
with current_directory(project_dir):
info('got setup.py or similar, running project install. ' +
'(disable this behavior with --ignore-setup-py)')
# Compute & output the constraints we will use:
info('Contents that will be used for constraints.txt:')
constraints = subprocess.check_output(
[join(ctx.build_dir, "venv", "bin", "pip"), "freeze"],
env=copy.copy(env))
try:
constraints = constraints.decode("utf-8", "replace")
except AttributeError:
pass
info(constraints)
# Make sure all packages found are fixed in version
# by writing a constraint file, to avoid recipes being
# upgraded & reinstalled:
with open('constraints.txt', 'wb') as fileh:
fileh.write(constraints.encode("utf-8", "replace"))
info('Populating venv\'s site-packages with '
'ctx.get_site_packages_dir()...')
# Copy dist contents into site-packages for discovery.
# Why this is needed:
# --target is somewhat evil and messes with discovery of
# packages in PYTHONPATH if that also includes the target
# folder. So we need to use the regular virtualenv
# site-packages folder instead.
# Reference:
# https://github.com/pypa/pip/issues/6223
ctx_site_packages_dir = os.path.normpath(
os.path.abspath(ctx.get_site_packages_dir()))
venv_site_packages_dir = os.path.normpath(
os.path.join(ctx.build_dir, "venv", "lib", [
f for f in os.listdir(
os.path.join(ctx.build_dir, "venv", "lib"))
if f.startswith("python")
][0], "site-packages"))
copied_over_contents = []
for f in os.listdir(ctx_site_packages_dir):
full_path = os.path.join(ctx_site_packages_dir, f)
if not os.path.exists(
os.path.join(venv_site_packages_dir, f)):
if os.path.isdir(full_path):
shutil.copytree(
full_path,
os.path.join(venv_site_packages_dir, f))
else:
shutil.copy2(
full_path,
os.path.join(venv_site_packages_dir, f))
copied_over_contents.append(f)
# Get listing of virtualenv's site-packages, to see the
# newly added things afterwards & copy them back into
# the distribution folder / build context site-packages:
previous_venv_contents = os.listdir(venv_site_packages_dir)
# Actually run setup.py:
info('Launching package install...')
shprint(sh.bash,
'-c',
("'" + join(ctx.build_dir, "venv", "bin",
"pip").replace("'", "'\"'\"'") + "' " +
"install -c constraints.txt -v .").format(
ctx.get_site_packages_dir().replace(
"'", "'\"'\"'")),
_env=copy.copy(env))
# Go over all new additions and copy them back:
info('Copying additions resulting from setup.py back ' +
'into ctx.get_site_packages_dir()...')
new_venv_additions = []
for f in (set(os.listdir(venv_site_packages_dir)) -
set(previous_venv_contents)):
new_venv_additions.append(f)
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.copytree(full_path,
os.path.join(ctx_site_packages_dir, f))
else:
shutil.copy2(full_path,
os.path.join(ctx_site_packages_dir, f))
# Undo all the changes we did to the venv-site packages:
info('Reverting additions to virtualenv\'s site-packages...')
for f in set(copied_over_contents + new_venv_additions):
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.rmtree(full_path)
else:
os.remove(full_path)
elif not ignore_setup_py:
info("No setup.py found in project directory: " + str(project_dir))
# Strip object files after potential Cython or native code builds:
standard_recipe.strip_object_files(ctx.archs[0],
env,
build_dir=ctx.build_dir)
def get_recipe_order_and_bootstrap(ctx, names, bs=None):
recipes_to_load = set(names)
if bs is not None and bs.recipe_depends:
recipes_to_load = recipes_to_load.union(set(bs.recipe_depends))
possible_orders = []
# get all possible order graphs, as names may include tuples/lists
# of alternative dependencies
names = [([name] if not isinstance(name, (list, tuple)) else name)
for name in names]
for name_set in product(*names):
new_possible_orders = [RecipeOrder(ctx)]
for name in name_set:
new_possible_orders = recursively_collect_orders(
name, ctx, orders=new_possible_orders)
possible_orders.extend(new_possible_orders)
# turn each order graph into a linear list if possible
orders = []
for possible_order in possible_orders:
try:
order = find_order(possible_order)
except ValueError: # a circular dependency was found
info('Circular dependency found in graph {}, skipping it.'.format(
possible_order))
continue
except:
warning('Failed to import recipe named {}; the recipe exists '
'but appears broken.'.format(name))
warning('Exception was:')
raise
orders.append(list(order))
# prefer python2 and SDL2 if available
orders = sorted(orders,
key=lambda order: -('python2' in order) -
('sdl2' in order))
if not orders:
error('Didn\'t find any valid dependency graphs.')
error('This means that some of your requirements pull in '
'conflicting dependencies.')
error('Exiting.')
exit(1)
# It would be better to check against possible orders other
# than the first one, but in practice clashes will be rare,
# and can be resolved by specifying more parameters
chosen_order = orders[0]
if len(orders) > 1:
info('Found multiple valid dependency orders:')
for order in orders:
info(' {}'.format(order))
info('Using the first of these: {}'.format(chosen_order))
else:
info('Found a single valid recipe set: {}'.format(chosen_order))
if bs is None:
bs = Bootstrap.get_bootstrap_from_recipes(chosen_order, ctx)
recipes, python_modules, bs = get_recipe_order_and_bootstrap(
ctx, chosen_order, bs=bs)
else:
# check if each requirement has a recipe
recipes = []
python_modules = []
for name in chosen_order:
try:
Recipe.get_recipe(name, ctx)
except IOError:
python_modules.append(name)
else:
recipes.append(name)
return recipes, python_modules, bs
def run_pymodules_install(ctx, modules):
modules = list(filter(ctx.not_has_package, modules))
if not modules:
info('There are no Python modules to install, skipping')
return
info('The requirements ({}) don\'t have recipes, attempting to install '
'them with pip'.format(', '.join(modules)))
info('If this fails, it may mean that the module has compiled '
'components and needs a recipe.')
venv = sh.Command(ctx.virtualenv)
with current_directory(join(ctx.build_dir)):
shprint(venv,
'--python=python{}'.format(ctx.python_recipe.major_minor_version_string),
'venv')
info('Creating a requirements.txt file for the Python modules')
with open('requirements.txt', 'w') as fileh:
for module in modules:
key = 'VERSION_' + module
if key in environ:
line = '{}=={}\n'.format(module, environ[key])
else:
line = '{}\n'.format(module)
fileh.write(line)
info('Installing Python modules with pip')
info('If this fails with a message about /bin/false, this '
'probably means the package cannot be installed with '
'pip as it needs a compilation recipe.')
# This bash method is what old-p4a used
# It works but should be replaced with something better
shprint(sh.bash, '-c', (
"env CC=/bin/false CXX=/bin/false "
"PYTHONPATH={0} venv/bin/pip install --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir()))
def prepare_build_environment(self,
user_sdk_dir,
user_ndk_dir,
user_android_api,
user_ndk_ver,
user_ndk_api):
'''Checks that build dependencies exist and sets internal variables
for the Android SDK etc.
..warning:: This *must* be called before trying any build stuff
'''
self.ensure_dirs()
if self._build_env_prepared:
return
ok = True
# Work out where the Android SDK is
sdk_dir = None
if user_sdk_dir:
sdk_dir = user_sdk_dir
# This is the old P4A-specific var
if sdk_dir is None:
sdk_dir = environ.get('ANDROIDSDK', None)
# This seems used more conventionally
if sdk_dir is None:
sdk_dir = environ.get('ANDROID_HOME', None)
# Checks in the buildozer SDK dir, useful for debug tests of p4a
if sdk_dir is None:
possible_dirs = glob.glob(expanduser(join(
'~', '.buildozer', 'android', 'platform', 'android-sdk-*')))
possible_dirs = [d for d in possible_dirs if not
(d.endswith('.bz2') or d.endswith('.gz'))]
if possible_dirs:
info('Found possible SDK dirs in buildozer dir: {}'.format(
', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
info('Will attempt to use SDK at {}'.format(possible_dirs[0]))
warning('This SDK lookup is intended for debug only, if you '
'use python-for-android much you should probably '
'maintain your own SDK download.')
sdk_dir = possible_dirs[0]
if sdk_dir is None:
raise BuildInterruptingException('Android SDK dir was not specified, exiting.')
self.sdk_dir = realpath(sdk_dir)
# Check what Android API we're using
android_api = None
if user_android_api:
android_api = user_android_api
info('Getting Android API version from user argument: {}'.format(android_api))
elif 'ANDROIDAPI' in environ:
android_api = environ['ANDROIDAPI']
info('Found Android API target in $ANDROIDAPI: {}'.format(android_api))
else:
info('Android API target was not set manually, using '
'the default of {}'.format(DEFAULT_ANDROID_API))
android_api = DEFAULT_ANDROID_API
android_api = int(android_api)
self.android_api = android_api
if self.android_api >= 21 and self.archs[0].arch == 'armeabi':
raise BuildInterruptingException(
'Asked to build for armeabi architecture with API '
'{}, but API 21 or greater does not support armeabi'.format(
self.android_api),
instructions='You probably want to build with --arch=armeabi-v7a instead')
info('Checking SDK: {}'.format(sdk_dir))
if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):
info('valid adv manager:')
avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager'))
targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n')
elif exists(join(sdk_dir, 'tools', 'android')):
info('valid android:')
android = sh.Command(join(sdk_dir, 'tools', 'android'))
targets = android('list').stdout.decode('utf-8').split('\n')
else:
info('Could not find `android` or `sdkmanager`')
raise BuildInterruptingException(
'Could not find `android` or `sdkmanager` binaries in Android SDK',
instructions='Make sure the path to the Android SDK is correct')
apis = [s for s in targets if re.match(r'^ *API level: ', s)]
apis = [re.findall(r'[0-9]+', s) for s in apis]
apis = [int(s[0]) for s in apis if s]
info('Available Android APIs are ({})'.format(
', '.join(map(str, apis))))
if android_api in apis:
info(('Requested API target {} is available, '
'continuing.').format(android_api))
else:
raise BuildInterruptingException(
('Requested API target {} is not available, install '
'it with the SDK android tool.').format(android_api))
# Find the Android NDK
# Could also use ANDROID_NDK, but doesn't look like many tools use this
ndk_dir = None
if user_ndk_dir:
ndk_dir = user_ndk_dir
info('Getting NDK dir from from user argument')
if ndk_dir is None: # The old P4A-specific dir
ndk_dir = environ.get('ANDROIDNDK', None)
if ndk_dir is not None:
info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir))
if ndk_dir is None: # Apparently the most common convention
ndk_dir = environ.get('NDK_HOME', None)
if ndk_dir is not None:
info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir))
if ndk_dir is None: # Another convention (with maven?)
ndk_dir = environ.get('ANDROID_NDK_HOME', None)
if ndk_dir is not None:
info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir))
if ndk_dir is None: # Checks in the buildozer NDK dir, useful
# # for debug tests of p4a
possible_dirs = glob.glob(expanduser(join(
'~', '.buildozer', 'android', 'platform', 'android-ndk-r*')))
if possible_dirs:
info('Found possible NDK dirs in buildozer dir: {}'.format(
', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
info('Will attempt to use NDK at {}'.format(possible_dirs[0]))
warning('This NDK lookup is intended for debug only, if you '
'use python-for-android much you should probably '
'maintain your own NDK download.')
ndk_dir = possible_dirs[0]
if ndk_dir is None:
raise BuildInterruptingException('Android NDK dir was not specified')
self.ndk_dir = realpath(ndk_dir)
# Find the NDK version, and check it against what the NDK dir
# seems to report
ndk_ver = None
if user_ndk_ver:
ndk_ver = user_ndk_ver
if ndk_dir is not None:
info('Got NDK version from from user argument: {}'.format(ndk_ver))
if ndk_ver is None:
ndk_ver = environ.get('ANDROIDNDKVER', None)
if ndk_ver is not None:
info('Got NDK version from $ANDROIDNDKVER: {}'.format(ndk_ver))
self.ndk = 'google'
try:
with open(join(ndk_dir, 'RELEASE.TXT')) as fileh:
reported_ndk_ver = fileh.read().split(' ')[0].strip()
except IOError:
pass
else:
if reported_ndk_ver.startswith('crystax-ndk-'):
reported_ndk_ver = reported_ndk_ver[12:]
self.ndk = 'crystax'
if ndk_ver is None:
ndk_ver = reported_ndk_ver
info(('Got Android NDK version from the NDK dir: {}').format(ndk_ver))
else:
if ndk_ver != reported_ndk_ver:
warning('NDK version was set as {}, but checking '
'the NDK dir claims it is {}.'.format(
ndk_ver, reported_ndk_ver))
warning('The build will try to continue, but it may '
'fail and you should check '
'that your setting is correct.')
warning('If the NDK dir result is correct, you don\'t '
'need to manually set the NDK ver.')
if ndk_ver is None:
warning('Android NDK version could not be found. This probably'
'won\'t cause any problems, but if necessary you can'
'set it with `--ndk-version=...`.')
self.ndk_ver = ndk_ver
ndk_api = None
if user_ndk_api:
ndk_api = user_ndk_api
info('Getting NDK API version (i.e. minimum supported API) from user argument')
elif 'NDKAPI' in environ:
ndk_api = environ.get('NDKAPI', None)
info('Found Android API target in $NDKAPI')
else:
ndk_api = min(self.android_api, DEFAULT_NDK_API)
warning('NDK API target was not set manually, using '
'the default of {} = min(android-api={}, default ndk-api={})'.format(
ndk_api, self.android_api, DEFAULT_NDK_API))
ndk_api = int(ndk_api)
self.ndk_api = ndk_api
if self.ndk_api > self.android_api:
raise BuildInterruptingException(
'Target NDK API is {}, higher than the target Android API {}.'.format(
self.ndk_api, self.android_api),
instructions=('The NDK API is a minimum supported API number and must be lower '
'than the target Android API'))
info('Using {} NDK {}'.format(self.ndk.capitalize(), self.ndk_ver))
virtualenv = None
if virtualenv is None:
virtualenv = sh.which('virtualenv2')
if virtualenv is None:
virtualenv = sh.which('virtualenv-2.7')
if virtualenv is None:
virtualenv = sh.which('virtualenv')
if virtualenv is None:
raise IOError('Couldn\'t find a virtualenv executable, '
'you must install this to use p4a.')
self.virtualenv = virtualenv
info('Found virtualenv at {}'.format(virtualenv))
# path to some tools
self.ccache = sh.which("ccache")
if not self.ccache:
info('ccache is missing, the build will not be optimized in the '
'future.')
for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"):
cython = sh.which(cython_fn)
if cython:
self.cython = cython
break
else:
raise BuildInterruptingException('No cython binary found.')
if not self.cython:
ok = False
warning("Missing requirement: cython is not installed")
# This would need to be changed if supporting multiarch APKs
arch = self.archs[0]
platform_dir = arch.platform_dir
toolchain_prefix = arch.toolchain_prefix
toolchain_version = None
self.ndk_platform = join(
self.ndk_dir,
'platforms',
'android-{}'.format(self.ndk_api),
platform_dir)
if not exists(self.ndk_platform):
warning('ndk_platform doesn\'t exist: {}'.format(
self.ndk_platform))
ok = False
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_versions = []
toolchain_path = join(self.ndk_dir, 'toolchains')
if isdir(toolchain_path):
toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path,
toolchain_prefix))
toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:]
for path in toolchain_contents]
else:
warning('Could not find toolchain subdirectory!')
ok = False
toolchain_versions.sort()
toolchain_versions_gcc = []
for toolchain_version in toolchain_versions:
if toolchain_version[0].isdigit():
# GCC toolchains begin with a number
toolchain_versions_gcc.append(toolchain_version)
if toolchain_versions:
info('Found the following toolchain versions: {}'.format(
toolchain_versions))
info('Picking the latest gcc toolchain, here {}'.format(
toolchain_versions_gcc[-1]))
toolchain_version = toolchain_versions_gcc[-1]
else:
warning('Could not find any toolchain for {}!'.format(
toolchain_prefix))
ok = False
self.toolchain_prefix = toolchain_prefix
self.toolchain_version = toolchain_version
# Modify the path so that sh finds modules appropriately
environ['PATH'] = (
'{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/'
'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/'
'{toolchain_prefix}-{toolchain_version}/prebuilt/'
'{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/'
'tools:{path}').format(
sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir,
toolchain_prefix=toolchain_prefix,
toolchain_version=toolchain_version,
py_platform=py_platform, path=environ.get('PATH'))
for executable in ("pkg-config", "autoconf", "automake", "libtoolize",
"tar", "bzip2", "unzip", "make", "gcc", "g++"):
if not sh.which(executable):
warning("Missing executable: {} is not installed".format(
executable))
if not ok:
raise BuildInterruptingException(
'python-for-android cannot continue due to the missing executables above')
def get_distribution(
cls,
ctx,
*,
arch_name, # required keyword argument: there is no sensible default
name=None,
recipes=[],
ndk_api=None,
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False,
allow_replace_dist=True):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
ndk_api : int
The NDK API to compile against, included in the dist because it cannot
be changed later during APK packaging.
arch_name : str
The target architecture name to compile against, included in the dist because
it cannot be changed later during APK packaging.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
allow_replace_dist : bool
If True, will allow an existing dist with the specified
name but incompatible requirements to be overwritten by
a new one with the current requirements.
'''
possible_dists = Distribution.get_distributions(ctx)
# Will hold dists that would be built in the same folder as an existing dist
folder_match_dist = None
# 0) Check if a dist with that name and architecture already exists
if name is not None and name:
possible_dists = [
d for d in possible_dists
if (d.name == name) and (arch_name in d.archs)
]
if possible_dists:
# There should only be one folder with a given dist name *and* arch.
# We could check that here, but for compatibility let's let it slide
# and just record the details of one of them. We only use this data to
# possibly fail the build later, so it doesn't really matter if there
# was more than one clash.
folder_match_dist = possible_dists[0]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
if (ndk_api is not None
and dist.ndk_api != ndk_api) or dist.ndk_api is None:
continue
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes, arch and NDK API, return it
for dist in possible_dists:
if force_build:
continue
if ndk_api is not None and dist.ndk_api != ndk_api:
continue
if arch_name is not None and arch_name not in dist.archs:
continue
if (set(dist.recipes) == set(recipes)
or (set(recipes).issubset(set(dist.recipes))
and not require_perfect_match)):
info_notify('{} has compatible recipes, using this one'.format(
dist.name))
return dist
# If there was a name match but we didn't already choose it,
# then the existing dist is incompatible with the requested
# configuration and the build cannot continue
if folder_match_dist is not None and not allow_replace_dist:
raise BuildInterruptingException(
'Asked for dist with name {name} with recipes ({req_recipes}) and '
'NDK API {req_ndk_api}, but a dist '
'with this name already exists and has either incompatible recipes '
'({dist_recipes}) or NDK API {dist_ndk_api}'.format(
name=name,
req_ndk_api=ndk_api,
dist_ndk_api=folder_match_dist.ndk_api,
req_recipes=', '.join(recipes),
dist_recipes=', '.join(folder_match_dist.recipes)))
assert len(possible_dists) < 2
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(
ctx.dist_dir,
generate_dist_folder_name(
name,
[arch_name] if arch_name is not None else None,
))
dist.recipes = recipes
dist.ndk_api = ctx.ndk_api
dist.archs = [arch_name]
return dist
def install_python_package(self, arch, name=None, env=None, is_dir=True):
for package in packages:
#srcdir = os.path.join(self.get_recipe_dir(), "..", "..", "..", "..", "tribler", "src", package)
srcdir = os.path.join("/work/tribler/src", package)
info('Installing {} into site-packages'.format(package))
copytree(srcdir, self.ctx.get_python_install_dir())
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
if self.ctx.python_recipe.from_crystax:
command = sh.Command('python{}'.format(
self.ctx.python_recipe.version))
site_packages_dirs = command(
'-c', 'import site; print("\\n".join(site.getsitepackages()))')
site_packages_dirs = site_packages_dirs.stdout.decode(
'utf-8').split('\n')
# env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
if 'PYTHONPATH' in env:
env['PYTHONPATH'] = env + ':{}'.format(
':'.join(site_packages_dirs))
else:
env['PYTHONPATH'] = ':'.join(site_packages_dirs)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
# hostpython = sh.Command('python3.5')
shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
print('cwd is', realpath(curdir))
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
manually_cythonise = False
try:
shprint(hostpython,
'setup.py',
'build_ext',
'-v',
_env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
manually_cythonise = True
if manually_cythonise:
self.cythonize_build(env=env)
shprint(hostpython,
'setup.py',
'build_ext',
'-v',
_env=env,
_tail=20,
_critical=True,
*self.setup_extra_args)
else:
info(
'First build appeared to complete correctly, skipping manual'
'cythonising.')
print('stripping')
build_lib = glob.glob('./build/lib*')
shprint(sh.find,
build_lib[0],
'-name',
'*.o',
'-exec',
env['STRIP'],
'{}',
';',
_env=env)
print('stripped!?')
def create_python_bundle(self, dirn, arch):
"""
Create a packaged python bundle in the target directory, by
copying all the modules and standard library to the right
place.
"""
# Todo: find a better way to find the build libs folder
modules_build_dir = join(
self.get_build_dir(arch.arch), 'android-build', 'build',
'lib.linux{}-{}-{}'.format('2' if self.version[0] == '2' else '',
arch.command_prefix.split('-')[0],
self.major_minor_version_string))
# Compile to *.pyc/*.pyo the python modules
self.compile_python_files(modules_build_dir)
# Compile to *.pyc/*.pyo the standard python library
self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
# Compile to *.pyc/*.pyo the other python packages (site-packages)
self.compile_python_files(self.ctx.get_python_install_dir())
# Bundle compiled python modules to a folder
modules_dir = join(dirn, 'modules')
c_ext = self.compiled_extension
ensure_dir(modules_dir)
module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
glob.glob(join(modules_build_dir, '*' + c_ext)))
info("Copy {} files into the bundle".format(len(module_filens)))
for filen in module_filens:
info(" - copy {}".format(filen))
copy2(filen, modules_dir)
# zip up the standard library
stdlib_zip = join(dirn, 'stdlib.zip')
with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
stdlib_filens = list(
walk_valid_filens('.', self.stdlib_dir_blacklist,
self.stdlib_filen_blacklist))
info("Zip {} files into the bundle".format(len(stdlib_filens)))
shprint(sh.zip, stdlib_zip, *stdlib_filens)
# copy the site-packages into place
ensure_dir(join(dirn, 'site-packages'))
ensure_dir(self.ctx.get_python_install_dir())
# TODO: Improve the API around walking and copying the files
with current_directory(self.ctx.get_python_install_dir()):
filens = list(
walk_valid_filens('.', self.site_packages_dir_blacklist,
self.site_packages_filen_blacklist))
info("Copy {} files into the site-packages".format(len(filens)))
for filen in filens:
info(" - copy {}".format(filen))
ensure_dir(join(dirn, 'site-packages', dirname(filen)))
copy2(filen, join(dirn, 'site-packages', filen))
# copy the python .so files into place
python_build_dir = join(self.get_build_dir(arch.arch), 'android-build')
python_lib_name = 'libpython' + self.major_minor_version_string
if self.major_minor_version_string[0] == '3':
python_lib_name += 'm'
shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'),
join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch))
info('Renaming .so files to reflect cross-compile')
self.reduce_object_file_names(join(dirn, 'site-packages'))
return join(dirn, 'site-packages')
def biglink(ctx, arch):
# First, collate object files from each recipe
info('Collating object files from each recipe')
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
'objects_{}'.format(recipe.name))
if not exists(recipe_obj_dir):
info('{} recipe has no biglinkable files dir, skipping'
.format(recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, '*'))
if not len(files):
info('{} recipe has no biglinkable files, skipping'
.format(recipe.name))
continue
info('{} recipe has object files, copying'.format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, '-r', *files)
env = arch.get_env()
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
if not len(glob.glob(join(obj_dir, '*'))):
info('There seem to be no libraries to biglink, skipping.')
return
info('Biglinking')
info('target {}'.format(join(ctx.get_libs_dir(arch.arch),
'libpymodules.so')))
do_biglink = copylibs_function if ctx.copy_libs else biglink_function
# Move to the directory containing crtstart_so.o and crtend_so.o
# This is necessary with newer NDKs? A gcc bug?
with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):
do_biglink(
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
obj_dir.split(' '),
extra_link_dirs=[join(ctx.bootstrap.build_dir,
'obj', 'local', arch.arch),
os.path.abspath('.')],
env=env)
def set_libs_flags(self, env, arch):
'''Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
'''
def add_flags(include_flags, link_dirs, link_libs):
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
env['LIBS'] = env.get('LIBS', '') + link_libs
if 'sqlite3' in self.ctx.recipe_build_order:
info('Activating flags for sqlite3')
recipe = Recipe.get_recipe('sqlite3', self.ctx)
add_flags(' -I' + recipe.get_build_dir(arch.arch),
' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
if 'libffi' in self.ctx.recipe_build_order:
info('Activating flags for libffi')
recipe = Recipe.get_recipe('libffi', self.ctx)
# In order to force the correct linkage for our libffi library, we
# set the following variable to point where is our libffi.pc file,
# because the python build system uses pkg-config to configure it.
env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)
add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
' -lffi')
if 'openssl' in self.ctx.recipe_build_order:
info('Activating flags for openssl')
recipe = Recipe.get_recipe('openssl', self.ctx)
self.configure_args += \
('--with-openssl=' + recipe.get_build_dir(arch.arch),)
add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch),
recipe.link_libs_flags())
for library_name in {'libbz2', 'liblzma'}:
if library_name in self.ctx.recipe_build_order:
info(f'Activating flags for {library_name}')
recipe = Recipe.get_recipe(library_name, self.ctx)
add_flags(recipe.get_library_includes(arch),
recipe.get_library_ldflags(arch),
recipe.get_library_libs_flag())
# python build system contains hardcoded zlib version which prevents
# the build of zlib module, here we search for android's zlib version
# and sets the right flags, so python can be build with android's zlib
info("Activating flags for android's zlib")
zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib')
zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
zlib_h = join(zlib_includes, 'zlib.h')
try:
with open(zlib_h) as fileh:
zlib_data = fileh.read()
except IOError:
raise BuildInterruptingException(
"Could not determine android's zlib version, no zlib.h ({}) in"
" the NDK dir includes".format(zlib_h))
for line in zlib_data.split('\n'):
if line.startswith('#define ZLIB_VERSION '):
break
else:
raise BuildInterruptingException(
'Could not parse zlib.h...so we cannot find zlib version,'
'required by python build,')
env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')
return env
def build_recipes(build_order,
python_modules,
ctx,
project_dir,
ignore_project_setup_py=False):
# Put recipes in correct build order
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'.format(
recipe.name))
# 4) biglink everything
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx,
python_modules,
project_dir,
ignore_setup_py=ignore_project_setup_py)
return
def apk(self, args):
"""Create an APK using the given distribution."""
ctx = self.ctx
dist = self._dist
# Manually fixing these arguments at the string stage is
# unsatisfactory and should probably be changed somehow, but
# we can't leave it until later as the build.py scripts assume
# they are in the current directory.
fix_args = ('--dir', '--private', '--add-jar', '--add-source',
'--whitelist', '--blacklist', '--presplash', '--icon')
unknown_args = args.unknown_args
for i, arg in enumerate(unknown_args):
argx = arg.split('=')
if argx[0] in fix_args:
if len(argx) > 1:
unknown_args[i] = '='.join(
(argx[0], realpath(expanduser(argx[1]))))
else:
unknown_args[i+1] = realpath(expanduser(unknown_args[i+1]))
env = os.environ.copy()
if args.build_mode == 'release':
if args.keystore:
env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore))
if args.signkey:
env['P4A_RELEASE_KEYALIAS'] = args.signkey
if args.keystorepw:
env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
if args.signkeypw:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw
build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
with current_directory(dist.dist_dir):
self.hook("before_apk_build")
os.environ["ANDROID_API"] = str(self.ctx.android_api)
build_args = build.parse_args(args.unknown_args)
self.hook("after_apk_build")
self.hook("before_apk_assemble")
build_type = ctx.java_build_tool
if build_type == 'auto':
info('Selecting java build tool:')
build_tools_versions = os.listdir(join(ctx.sdk_dir,
'build-tools'))
build_tools_versions = sorted(build_tools_versions,
key=LooseVersion)
build_tools_version = build_tools_versions[-1]
info(('Detected highest available build tools '
'version to be {}').format(build_tools_version))
if build_tools_version >= '25.0' and exists('gradlew'):
build_type = 'gradle'
info(' Building with gradle, as gradle executable is '
'present')
else:
build_type = 'ant'
if build_tools_version < '25.0':
info((' Building with ant, as the highest '
'build-tools-version is only {}').format(
build_tools_version))
else:
info(' Building with ant, as no gradle executable '
'detected')
if build_type == 'gradle':
# gradle-based build
env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir
env["ANDROID_HOME"] = self.ctx.sdk_dir
gradlew = sh.Command('./gradlew')
if exists('/usr/bin/dos2unix'):
# .../dists/bdisttest_python3/gradlew
# .../build/bootstrap_builds/sdl2-python3crystax/gradlew
# if docker on windows, gradle contains CRLF
output = shprint(
sh.Command('dos2unix'), gradlew._path.decode('utf8'),
_tail=20, _critical=True, _env=env
)
if args.build_mode == "debug":
gradle_task = "assembleDebug"
elif args.build_mode == "release":
gradle_task = "assembleRelease"
else:
error("Unknown build mode {} for apk()".format(
args.build_mode))
exit(1)
output = shprint(gradlew, gradle_task, _tail=20,
_critical=True, _env=env)
# gradle output apks somewhere else
# and don't have version in file
apk_dir = join(dist.dist_dir,
"build", "outputs", "apk",
args.build_mode)
apk_glob = "*-{}.apk"
apk_add_version = True
else:
# ant-based build
try:
ant = sh.Command('ant')
except sh.CommandNotFound:
error('Could not find ant binary, please install it '
'and make sure it is in your $PATH.')
exit(1)
output = shprint(ant, args.build_mode, _tail=20,
_critical=True, _env=env)
apk_dir = join(dist.dist_dir, "bin")
apk_glob = "*-*-{}.apk"
apk_add_version = False
self.hook("after_apk_assemble")
info_main('# Copying APK to current directory')
apk_re = re.compile(r'.*Package: (.*\.apk)$')
apk_file = None
for line in reversed(output.splitlines()):
m = apk_re.match(line)
if m:
apk_file = m.groups()[0]
break
if not apk_file:
info_main('# APK filename not found in build output. Guessing...')
if args.build_mode == "release":
suffixes = ("release", "release-unsigned")
else:
suffixes = ("debug", )
for suffix in suffixes:
apks = glob.glob(join(apk_dir, apk_glob.format(suffix)))
if apks:
if len(apks) > 1:
info('More than one built APK found... guessing you '
'just built {}'.format(apks[-1]))
apk_file = apks[-1]
break
else:
raise ValueError('Couldn\'t find the built APK')
info_main('# Found APK file: {}'.format(apk_file))
if apk_add_version:
info('# Add version number to APK')
apk_name, apk_suffix = basename(apk_file).split("-", 1)
apk_file_dest = "{}-{}-{}".format(
apk_name, build_args.version, apk_suffix)
info('# APK renamed to {}'.format(apk_file_dest))
shprint(sh.cp, apk_file, apk_file_dest)
else:
shprint(sh.cp, apk_file, './')
def check_ndk_version(ndk_dir):
"""
Check the NDK version against what is currently recommended and raise an
exception of :class:`~pythonforandroid.util.BuildInterruptingException` in
case that the user tries to use an NDK lower than minimum supported,
specified via attribute `MIN_NDK_VERSION`.
.. versionchanged:: 2019.06.06.1.dev0
Added the ability to get android's NDK `letter version` and also
rewrote to raise an exception in case that an NDK version lower than
the minimum supported is detected.
"""
version = read_ndk_version(ndk_dir)
if version is None:
warning(READ_ERROR_NDK_MESSAGE.format(ndk_dir=ndk_dir))
warning(
ENSURE_RIGHT_NDK_MESSAGE.format(
min_supported=MIN_NDK_VERSION,
rec_version=RECOMMENDED_NDK_VERSION,
ndk_url=NDK_DOWNLOAD_URL,
)
)
return
# create a dictionary which will describe the relationship of the android's
# NDK minor version with the `human readable` letter version, egs:
# Pkg.Revision = 17.1.4828580 => ndk-17b
# Pkg.Revision = 17.2.4988734 => ndk-17c
# Pkg.Revision = 19.0.5232133 => ndk-19 (No letter)
minor_to_letter = {0: ''}
minor_to_letter.update(
{n + 1: chr(i) for n, i in enumerate(range(ord('b'), ord('b') + 25))}
)
major_version = version.version[0]
letter_version = minor_to_letter[version.version[1]]
string_version = '{major_version}{letter_version}'.format(
major_version=major_version, letter_version=letter_version
)
info(CURRENT_NDK_VERSION_MESSAGE.format(ndk_version=string_version))
if major_version < MIN_NDK_VERSION:
raise BuildInterruptingException(
NDK_LOWER_THAN_SUPPORTED_MESSAGE.format(
min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL
),
instructions=(
'Please, go to the android NDK page ({ndk_url}) and download a'
' supported version.\n*** The currently recommended NDK'
' version is {rec_version} ***'.format(
ndk_url=NDK_DOWNLOAD_URL,
rec_version=RECOMMENDED_NDK_VERSION,
)
),
)
elif major_version > MAX_NDK_VERSION:
warning(
RECOMMENDED_NDK_VERSION_MESSAGE.format(
recommended_ndk_version=RECOMMENDED_NDK_VERSION
)
)
warning(NEW_NDK_MESSAGE)
def build_arch(self, arch):
# If openssl is needed we may have to recompile cPython to get the
# ssl.py module working properly
if self.from_crystax and 'openssl' in self.ctx.recipe_build_order:
info('Openssl and crystax-python combination may require '
'recompilation of python...')
ssl_recipe = self.get_recipe('openssl', self.ctx)
stage, msg = self.check_for_sslso(ssl_recipe, arch)
stage = 0 if stage < 5 else stage
info(msg)
openssl_build_dir = ssl_recipe.get_build_dir(arch.arch)
openssl_ndk_dir = join(self.ctx.ndk_dir, 'sources', 'openssl',
ssl_recipe.version)
if stage < 2:
info('Copying openssl headers and Android.mk to ndk')
ensure_dir(openssl_ndk_dir)
if stage < 1.2:
# copy include folder and Android.mk to ndk
mk_path = self.find_Android_mk()
if mk_path is None:
raise IOError('Android.mk file could not be found in '
'any versions in ndk->sources->openssl')
shprint(sh.cp, mk_path, openssl_ndk_dir)
include_dir = join(openssl_build_dir, 'include')
if stage < 1.3:
ndk_include_dir = join(openssl_ndk_dir, 'include',
'openssl')
self.copy_include_dir(join(include_dir, 'openssl'),
ndk_include_dir)
target_conf = join(openssl_ndk_dir, 'include', 'openssl',
'opensslconf.h')
shprint(sh.rm, '-f', target_conf)
# overwrite opensslconf.h
with open(target_conf, 'w') as fp:
fp.write(OPENSSLCONF)
if stage < 1.4:
# move current conf to arch specific conf in ndk
under_scored_arch = arch.arch.replace('-', '_')
shprint(
sh.ln, '-sf',
realpath(join(include_dir, 'openssl',
'opensslconf.h')),
join(openssl_ndk_dir, 'include', 'openssl',
'opensslconf_{}.h'.format(under_scored_arch)))
if stage < 3:
info('Copying openssl libs to ndk')
arch_ndk_lib = join(openssl_ndk_dir, 'libs', arch.arch)
ensure_dir(arch_ndk_lib)
shprint(
sh.ln, '-sf',
realpath(
join(openssl_build_dir,
'libcrypto{}.so'.format(ssl_recipe.version))),
join(openssl_build_dir, 'libcrypto.so'))
shprint(
sh.ln, '-sf',
realpath(
join(openssl_build_dir,
'libssl{}.so'.format(ssl_recipe.version))),
join(openssl_build_dir, 'libssl.so'))
libs = ['libcrypto.a', 'libcrypto.so', 'libssl.a', 'libssl.so']
cmd = [join(openssl_build_dir, lib)
for lib in libs] + [arch_ndk_lib]
shprint(sh.cp, '-f', *cmd)
if stage < 10:
info('Recompiling python-crystax')
self.patch_dev_defaults(ssl_recipe)
build_script = join(self.ctx.ndk_dir, 'build', 'tools',
'build-target-python.sh')
shprint(Command(build_script),
'--ndk-dir={}'.format(self.ctx.ndk_dir),
'--abis={}'.format(arch.arch), '-j5', '--verbose',
self.get_build_dir(arch.arch))
info('Extracting CrystaX python3 from NDK package')
dirn = self.ctx.get_python_install_dir()
ensure_dir(dirn)
self.ctx.hostpython = 'python{}'.format(self.version)
def copy_file(self, filename, dest):
info("Copy {} to {}".format(filename, dest))
filename = join(self.recipe_dir, filename)
dest = join(self.build_dir, dest)
shutil.copy(filename, dest)
def __init__(self):
argv = sys.argv
# Buildozer used to pass these arguments in a now-invalid order
# If that happens, apply this fix
# This fix will be removed once a fixed buildozer is released
if (len(argv) > 2 and
argv[1].startswith('--color') and
argv[2].startswith('--storage-dir')):
argv.append(argv.pop(1)) # the --color arg
argv.append(argv.pop(1)) # the --storage-dir arg
parser = NoAbbrevParser(
description=('A packaging tool for turning Python scripts and apps '
'into Android APKs'))
generic_parser = argparse.ArgumentParser(
add_help=False,
description=('Generic arguments applied to all commands'))
dist_parser = argparse.ArgumentParser(
add_help=False,
description=('Arguments for dist building'))
generic_parser.add_argument(
'--debug', dest='debug', action='store_true',
default=False,
help='Display debug output and all build info')
generic_parser.add_argument(
'--color', dest='color', choices=['always', 'never', 'auto'],
help='Enable or disable color output (default enabled on tty)')
generic_parser.add_argument(
'--sdk-dir', '--sdk_dir', dest='sdk_dir', default='',
help='The filepath where the Android SDK is installed')
generic_parser.add_argument(
'--ndk-dir', '--ndk_dir', dest='ndk_dir', default='',
help='The filepath where the Android NDK is installed')
generic_parser.add_argument(
'--android-api', '--android_api', dest='android_api', default=0, type=int,
help='The Android API level to build against.')
generic_parser.add_argument(
'--ndk-version', '--ndk_version', dest='ndk_version', default='',
help=('The version of the Android NDK. This is optional, '
'we try to work it out automatically from the ndk_dir.'))
generic_parser.add_argument(
'--symlink-java-src', '--symlink_java_src',
action='store_true',
dest='symlink_java_src',
default=False,
help=('If True, symlinks the java src folder during build and dist '
'creation. This is useful for development only, it could also '
'cause weird problems.'))
default_storage_dir = user_data_dir('python-for-android')
if ' ' in default_storage_dir:
default_storage_dir = '~/.python-for-android'
generic_parser.add_argument(
'--storage-dir', dest='storage_dir',
default=default_storage_dir,
help=('Primary storage directory for downloads and builds '
'(default: {})'.format(default_storage_dir)))
# AND: This option doesn't really fit in the other categories, the
# arg structure needs a rethink
generic_parser.add_argument(
'--arch',
help='The archs to build for, separated by commas.',
default='armeabi')
# Options for specifying the Distribution
generic_parser.add_argument(
'--dist-name', '--dist_name',
help='The name of the distribution to use or create',
default='')
generic_parser.add_argument(
'--requirements',
help=('Dependencies of your app, should be recipe names or '
'Python modules'),
default='')
generic_parser.add_argument(
'--bootstrap',
help='The bootstrap to build with. Leave unset to choose automatically.',
default=None)
add_boolean_option(
generic_parser, ["force-build"],
default=False,
description='Whether to force compilation of a new distribution:')
generic_parser.add_argument(
'--extra-dist-dirs', '--extra_dist_dirs',
dest='extra_dist_dirs', default='',
help='Directories in which to look for distributions')
add_boolean_option(
generic_parser, ["require-perfect-match"],
default=False,
description=('Whether the dist recipes must perfectly match '
'those requested'))
generic_parser.add_argument(
'--local-recipes', '--local_recipes',
dest='local_recipes', default='./p4a-recipes',
help='Directory to look for local recipes')
add_boolean_option(
generic_parser, ['copy-libs'],
default=False,
description='Copy libraries instead of using biglink (Android 4.3+)')
self._read_configuration()
subparsers = parser.add_subparsers(dest='subparser_name',
help='The command to run')
def add_parser(subparsers, *args, **kwargs):
'''
argparse in python2 doesn't support the aliases option,
so we just don't provide the aliases there.
'''
if 'aliases' in kwargs and sys.version_info.major < 3:
kwargs.pop('aliases')
return subparsers.add_parser(*args, **kwargs)
parser_recipes = add_parser(subparsers,
'recipes',
parents=[generic_parser],
help='List the available recipes')
parser_recipes.add_argument(
"--compact", action="store_true", default=False,
help="Produce a compact list suitable for scripting")
parser_bootstraps = add_parser(subparsers,
'bootstraps', help='List the available bootstraps',
parents=[generic_parser])
parser_clean_all = add_parser(subparsers,
'clean_all', aliases=['clean-all'],
help='Delete all builds, dists and caches',
parents=[generic_parser])
parser_clean_dists = add_parser(subparsers,
'clean_dists', aliases=['clean-dists'],
help='Delete all dists',
parents=[generic_parser])
parser_clean_bootstrap_builds = add_parser(subparsers,
'clean_bootstrap_builds', aliases=['clean-bootstrap-builds'],
help='Delete all bootstrap builds',
parents=[generic_parser])
parser_clean_builds = add_parser(subparsers,
'clean_builds', aliases=['clean-builds'],
help='Delete all builds',
parents=[generic_parser])
parser_clean_recipe_build = add_parser(subparsers,
'clean_recipe_build', aliases=['clean-recipe-build'],
help=('Delete the build components of the given recipe. '
'By default this will also delete built dists'),
parents=[generic_parser])
parser_clean_recipe_build.add_argument('recipe', help='The recipe name')
parser_clean_recipe_build.add_argument('--no-clean-dists', default=False,
dest='no_clean_dists',
action='store_true',
help='If passed, do not delete existing dists')
parser_clean_download_cache= add_parser(subparsers,
'clean_download_cache', aliases=['clean-download-cache'],
help='Delete cached downloads for requirement builds',
parents=[generic_parser])
parser_clean_download_cache.add_argument(
'recipes', nargs='*',
help=('The recipes to clean (space-separated). If no recipe name is '
'provided, the entire cache is cleared.'))
parser_export_dist = add_parser(subparsers,
'export_dist', aliases=['export-dist'],
help='Copy the named dist to the given path',
parents=[generic_parser])
parser_export_dist.add_argument('output_dir', help=('The output dir to copy to'))
parser_export_dist.add_argument('--symlink', action='store_true',
help=('Symlink the dist instead of copying'))
parser_apk = add_parser(subparsers,
'apk', help='Build an APK',
parents=[generic_parser])
parser_apk.add_argument('--release', dest='build_mode', action='store_const',
const='release', default='debug',
help='Build the PARSER_APK. in Release mode')
parser_apk.add_argument('--keystore', dest='keystore', action='store', default=None,
help=('Keystore for JAR signing key, will use jarsigner '
'default if not specified (release build only)'))
parser_apk.add_argument('--signkey', dest='signkey', action='store', default=None,
help='Key alias to sign PARSER_APK. with (release build only)')
parser_apk.add_argument('--keystorepw', dest='keystorepw', action='store', default=None,
help='Password for keystore')
parser_apk.add_argument('--signkeypw', dest='signkeypw', action='store', default=None,
help='Password for key alias')
parser_create = add_parser(subparsers,
'create', help='Compile a set of requirements into a dist',
parents=[generic_parser])
parser_archs = add_parser(subparsers,
'archs', help='List the available target architectures',
parents=[generic_parser])
parser_distributions = add_parser(subparsers,
'distributions', aliases=['dists'],
help='List the currently available (compiled) dists',
parents=[generic_parser])
parser_delete_dist = add_parser(subparsers,
'delete_dist', aliases=['delete-dist'], help='Delete a compiled dist',
parents=[generic_parser])
parser_sdk_tools = add_parser(subparsers,
'sdk_tools', aliases=['sdk-tools'],
help='Run the given binary from the SDK tools dis',
parents=[generic_parser])
parser_sdk_tools.add_argument(
'tool', help=('The tool binary name to run'))
parser_adb = add_parser(subparsers,
'adb', help='Run adb from the given SDK',
parents=[generic_parser])
parser_logcat = add_parser(subparsers,
'logcat', help='Run logcat from the given SDK',
parents=[generic_parser])
parser_build_status = add_parser(subparsers,
'build_status', aliases=['build-status'],
help='Print some debug information about current built components',
parents=[generic_parser])
args, unknown = parser.parse_known_args(sys.argv[1:])
args.unknown_args = unknown
self.args = args
setup_color(args.color)
if args.debug:
logger.setLevel(logging.DEBUG)
# strip version from requirements, and put them in environ
if hasattr(args, 'requirements'):
requirements = []
for requirement in split_argument_list(args.requirements):
if "==" in requirement:
requirement, version = requirement.split(u"==", 1)
os.environ["VERSION_{}".format(requirement)] = version
info('Recipe {}: version "{}" requested'.format(
requirement, version))
requirements.append(requirement)
args.requirements = u",".join(requirements)
self.ctx = Context()
self.storage_dir = args.storage_dir
self.ctx.setup_dirs(self.storage_dir)
self.sdk_dir = args.sdk_dir
self.ndk_dir = args.ndk_dir
self.android_api = args.android_api
self.ndk_version = args.ndk_version
self.ctx.symlink_java_src = args.symlink_java_src
self._archs = split_argument_list(args.arch)
# AND: Fail nicely if the args aren't handled yet
if args.extra_dist_dirs:
warning('Received --extra_dist_dirs but this arg currently is not '
'handled, exiting.')
exit(1)
self.ctx.local_recipes = args.local_recipes
self.ctx.copy_libs = args.copy_libs
# Each subparser corresponds to a method
getattr(self, args.subparser_name.replace('-', '_'))(args)
def unpack(self, arch):
info_main('Unpacking {} for {}'.format(self.name, arch))
build_dir = self.get_build_container_dir(arch)
user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
if user_dir is not None:
info('P4A_{}_DIR exists, symlinking instead'.format(
self.name.lower()))
if exists(self.get_build_dir(arch)):
return
shprint(sh.rm, '-rf', build_dir)
shprint(sh.mkdir, '-p', build_dir)
shprint(sh.rmdir, build_dir)
ensure_dir(build_dir)
shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
return
if self.url is None:
info('Skipping {} unpack as no URL is set'.format(self.name))
return
filename = shprint(sh.basename,
self.versioned_url).stdout[:-1].decode('utf-8')
ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename)
if ma: # fragmented URL?
filename = ma.group(1)
with current_directory(build_dir):
directory_name = self.get_build_dir(arch)
if not exists(directory_name) or not isdir(directory_name):
extraction_filename = join(self.ctx.packages_path, self.name,
filename)
if isfile(extraction_filename):
if extraction_filename.endswith('.zip'):
try:
sh.unzip(extraction_filename)
except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2):
# return code 1 means unzipping had
# warnings but did complete,
# apparently happens sometimes with
# github zips
pass
import zipfile
fileh = zipfile.ZipFile(extraction_filename, 'r')
root_directory = fileh.filelist[0].filename.split(
'/')[0]
if root_directory != basename(directory_name):
shprint(sh.mv, root_directory, directory_name)
elif extraction_filename.endswith(
('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz',
'.txz')):
sh.tar('xf', extraction_filename)
root_directory = sh.tar(
'tf', extraction_filename).stdout.decode(
'utf-8').split('\n')[0].split('/')[0]
if root_directory != basename(directory_name):
shprint(sh.mv, root_directory, directory_name)
else:
raise Exception(
'Could not extract {} download, it must be .zip, '
'.tar.gz or .tar.bz2 or .tar.xz'.format(
extraction_filename))
elif isdir(extraction_filename):
mkdir(directory_name)
for entry in listdir(extraction_filename):
if entry not in ('.git', ):
shprint(sh.cp, '-Rv',
join(extraction_filename, entry),
directory_name)
else:
raise Exception(
'Given path is neither a file nor a directory: {}'.
format(extraction_filename))
else:
info('{} is already unpacked, skipping'.format(self.name))
def get_distribution(cls, ctx, name=None, recipes=[],
ndk_api=None,
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False,
allow_replace_dist=True):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
allow_replace_dist : bool
If True, will allow an existing dist with the specified
name but incompatible requirements to be overwritten by
a new one with the current requirements.
'''
existing_dists = Distribution.get_distributions(ctx)
needs_build = True # whether the dist needs building, will be returned
possible_dists = existing_dists
name_match_dist = None
# 0) Check if a dist with that name already exists
if name is not None and name:
possible_dists = [d for d in possible_dists if d.name == name]
if possible_dists:
name_match_dist = possible_dists[0]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
if (
ndk_api is not None and dist.ndk_api != ndk_api
) or dist.ndk_api is None:
continue
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes and ndk API, return it
for dist in possible_dists:
if force_build:
continue
if ndk_api is not None and dist.ndk_api != ndk_api:
continue
if (set(dist.recipes) == set(recipes) or
(set(recipes).issubset(set(dist.recipes)) and
not require_perfect_match)):
info_notify('{} has compatible recipes, using this one'
.format(dist.name))
return dist
assert len(possible_dists) < 2
# If there was a name match but we didn't already choose it,
# then the existing dist is incompatible with the requested
# configuration and the build cannot continue
if name_match_dist is not None and not allow_replace_dist:
raise BuildInterruptingException(
'Asked for dist with name {name} with recipes ({req_recipes}) and '
'NDK API {req_ndk_api}, but a dist '
'with this name already exists and has either incompatible recipes '
'({dist_recipes}) or NDK API {dist_ndk_api}'.format(
name=name,
req_ndk_api=ndk_api,
dist_ndk_api=name_match_dist.ndk_api,
req_recipes=', '.join(recipes),
dist_recipes=', '.join(name_match_dist.recipes)))
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(ctx.dist_dir, dist.name)
dist.recipes = recipes
dist.ndk_api = ctx.ndk_api
return dist
def get_distribution(cls,
ctx,
name=None,
recipes=[],
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
'''
# AND: This whole function is a bit hacky, it needs checking
# properly to make sure it follows logically correct
# possibilities
existing_dists = Distribution.get_distributions(ctx)
needs_build = True # whether the dist needs building, will be returned
possible_dists = existing_dists
# 0) Check if a dist with that name already exists
if name is not None and name:
possible_dists = [d for d in possible_dists if d.name == name]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes, return it
P4A_force_build = False
for dist in possible_dists:
if (set(dist.recipes) == set(recipes)
or (set(recipes).issubset(set(dist.recipes))
and not require_perfect_match)):
if force_build:
# distribution exists, rebuild forced
P4A_force_build = True
continue
else:
# existing distribution returned, no build required
info_notify(
'{} has compatible recipes, using this one'.format(
dist.name))
return dist
assert len(possible_dists) < 2
if not name and possible_dists:
info('Asked for dist with name {} with recipes ({}), but a dist '
'with this name already exists and has incompatible recipes '
'({})'.format(name, ', '.join(recipes),
', '.join(possible_dists[0].recipes)))
info('No compatible dist found, so exiting.')
exit(1)
# # 2) Check if any downloadable dists meet the requirements
# online_dists = [('testsdl2', ['hostpython2', 'sdl2_image',
# 'sdl2_mixer', 'sdl2_ttf',
# 'python2', 'sdl2',
# 'pyjniussdl2', 'kivysdl2'],
# 'https://github.com/inclement/sdl2-example-dist/archive/master.zip'),
# ]
# _possible_dists = []
# for dist_name, dist_recipes, dist_url in online_dists:
# for recipe in recipes:
# if recipe not in dist_recipes:
# break
# else:
# dist = Distribution(ctx)
# dist.name = dist_name
# dist.url = dist_url
# _possible_dists.append(dist)
# # if _possible_dists
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
"""Locally modified recipes will be forced to build, others are reused
environmental variable P4A_{recipe_name}_DIR points to local recipe
P4A_force_build affects all local recipes"""
dist.P4A_force_build = P4A_force_build and os.environ.get(
'P4A_force_build')
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(ctx.dist_dir, dist.name)
dist.recipes = recipes
return dist
def prebuild_arch(self, arch):
"""Make the build and target directories"""
path = self.get_build_dir(arch.arch)
if not exists(path):
info("creating {}".format(path))
shprint(sh.mkdir, '-p', path)
def run_pymodules_install(ctx, modules):
modules = list(filter(ctx.not_has_package, modules))
if not modules:
info('There are no Python modules to install, skipping')
return
info('The requirements ({}) don\'t have recipes, attempting to install '
'them with pip'.format(', '.join(modules)))
info('If this fails, it may mean that the module has compiled '
'components and needs a recipe.')
venv = sh.Command(ctx.virtualenv)
with current_directory(join(ctx.build_dir)):
shprint(
venv, '--python=python{}.{}'.format(
ctx.python_recipe.major_minor_version_string.partition(".")[0],
ctx.python_recipe.major_minor_version_string.partition(".")
[2]), 'venv')
info('Creating a requirements.txt file for the Python modules')
with open('requirements.txt', 'w') as fileh:
for module in modules:
key = 'VERSION_' + module
if key in environ:
line = '{}=={}\n'.format(module, environ[key])
else:
line = '{}\n'.format(module)
fileh.write(line)
# Prepare base environment and upgrade pip:
base_env = copy.copy(os.environ)
base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
info('Upgrade pip to latest version')
shprint(sh.bash,
'-c', ("source venv/bin/activate && pip install -U pip"),
_env=copy.copy(base_env))
# Install Cython in case modules need it to build:
info('Install Cython in case one of the modules needs it to build')
shprint(sh.bash,
'-c', ("venv/bin/pip install Cython"),
_env=copy.copy(base_env))
# Get environment variables for build (with CC/compiler set):
standard_recipe = CythonRecipe()
standard_recipe.ctx = ctx
# (note: following line enables explicit -lpython... linker options)
standard_recipe.call_hostpython_via_targetpython = False
recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
env = copy.copy(base_env)
env.update(recipe_env)
info('Installing Python modules with pip')
info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
'A reason for this is often modules compiling '
'native code that is unaware of Android cross-compilation '
'and does not work without additional '
'changes / workarounds.')
# Make sure our build package dir is available, and the virtualenv
# site packages come FIRST (so the proper pip version is used):
env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
env["PYTHONPATH"] = os.path.abspath(
join(ctx.build_dir, "venv", "lib",
"python" + ctx.python_recipe.major_minor_version_string,
"site-packages")) + ":" + env["PYTHONPATH"]
'''
# Do actual install:
shprint(sh.bash, '-c', (
"venv/bin/pip " +
"install -v --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
_env=copy.copy(env))
'''
# use old install script
shprint(sh.bash, '-c', (
"source venv/bin/activate && env CC=/bin/false CXX=/bin/false "
"PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir()))
# Strip object files after potential Cython or native code builds:
standard_recipe.strip_object_files(ctx.archs[0],
env,
build_dir=ctx.build_dir)
def distribute_javaclasses(self, javaclass_dir):
'''Copy existing javaclasses from build dir to current dist dir.'''
info('Copying java files')
for filename in glob.glob(javaclass_dir):
shprint(sh.cp, '-a', filename, 'src')
def distribute_aars(self, arch):
'''Process existing .aar bundles and copy to current dist dir.'''
info('Unpacking aars')
for aar in glob.glob(join(self.ctx.aars_dir, '*.aar')):
self._unpack_aar(aar, arch)
def install_python_package(self, arch, name=None, env=None, is_dir=True):
'''Automate the installation of a Python package (or a cython
package where the cython components are pre-built).'''
# arch = self.filtered_archs[0] # old kivy-ios way
if name is None:
name = self.name
if env is None:
env = self.get_recipe_env(arch)
info('Installing {} into site-packages'.format(self.name))
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.hostpython_location)
# hostpython = sh.Command('python3.5')
if self.ctx.python_recipe.from_crystax:
# hppath = join(dirname(self.hostpython_location), 'Lib',
# 'site-packages')
hpenv = env.copy()
# if 'PYTHONPATH' in hpenv:
# hpenv['PYTHONPATH'] = ':'.join([hppath] +
# hpenv['PYTHONPATH'].split(':'))
# else:
# hpenv['PYTHONPATH'] = hppath
# hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
# shprint(hostpython, 'setup.py', 'build',
# _env=hpenv, *self.setup_extra_args)
shprint(
hostpython,
'setup.py',
'install',
'-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=.',
# AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
_env=hpenv,
*self.setup_extra_args)
# site_packages_dir = self.ctx.get_site_packages_dir()
# built_files = glob.glob(join('build', 'lib*', '*'))
# for filen in built_files:
# shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
elif self.call_hostpython_via_targetpython:
shprint(hostpython,
'setup.py',
'install',
'-O2',
_env=env,
*self.setup_extra_args)
else:
hppath = join(dirname(self.hostpython_location), 'Lib',
'site-packages')
hpenv = env.copy()
if 'PYTHONPATH' in hpenv:
hpenv['PYTHONPATH'] = ':'.join(
[hppath] + hpenv['PYTHONPATH'].split(':'))
else:
hpenv['PYTHONPATH'] = hppath
shprint(hostpython,
'setup.py',
'install',
'-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=lib/python2.7/site-packages',
_env=hpenv,
*self.setup_extra_args)
# AND: Hardcoded python2.7 needs fixing
# If asked, also install in the hostpython build dir
if self.install_in_hostpython:
self.install_hostpython_package(arch)
def apk(self, args):
'''Create an APK using the given distribution.'''
ap = argparse.ArgumentParser(description='Build an APK')
ap.add_argument('--release',
dest='build_mode',
action='store_const',
const='release',
default='debug',
help='Build the APK in Release mode')
ap.add_argument(
'--keystore',
dest='keystore',
action='store',
default=None,
help=('Keystore for JAR signing key, will use jarsigner '
'default if not specified (release build only)'))
ap.add_argument('--signkey',
dest='signkey',
action='store',
default=None,
help='Key alias to sign APK with (release build only)')
ap.add_argument('--keystorepw',
dest='keystorepw',
action='store',
default=None,
help='Password for keystore')
ap.add_argument('--signkeypw',
dest='signkeypw',
action='store',
default=None,
help='Password for key alias')
apk_args, args = ap.parse_known_args(args)
ctx = self.ctx
dist = self._dist
# Manually fixing these arguments at the string stage is
# unsatisfactory and should probably be changed somehow, but
# we can't leave it until later as the build.py scripts assume
# they are in the current directory.
fix_args = ('--dir', '--private', '--add-jar', '--add-source',
'--whitelist', '--blacklist', '--presplash', '--icon')
for i, arg in enumerate(args[:-1]):
argx = arg.split('=')
if argx[0] in fix_args:
if len(argx) > 1:
args[i] = '='.join(
(argx[0], realpath(expanduser(argx[1]))))
else:
args[i + 1] = realpath(expanduser(args[i + 1]))
env = os.environ.copy()
if apk_args.build_mode == 'release':
if apk_args.keystore:
env['P4A_RELEASE_KEYSTORE'] = realpath(
expanduser(apk_args.keystore))
if apk_args.signkey:
env['P4A_RELEASE_KEYALIAS'] = apk_args.signkey
if apk_args.keystorepw:
env['P4A_RELEASE_KEYSTORE_PASSWD'] = apk_args.keystorepw
if apk_args.signkeypw:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = apk_args.signkeypw
elif apk_args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = apk_args.keystorepw
build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
with current_directory(dist.dist_dir):
build_args = build.parse_args(args)
output = shprint(sh.ant,
apk_args.build_mode,
_tail=20,
_critical=True,
_env=env)
info_main('# Copying APK to current directory')
apk_re = re.compile(r'.*Please sign (/.*\.apk) manually$')
apk_file = None
for line in reversed(output.splitlines()):
m = apk_re.match(line)
if m:
apk_file = m.groups()[0]
break
if not apk_file:
info_main(
'# APK filename not found in build output, trying to guess')
apks = glob.glob(
join(dist.dist_dir, 'bin',
'*-*-{}.apk'.format(apk_args.build_mode)))
if len(apks) == 0:
raise ValueError('Couldn\'t find the built APK')
if len(apks) > 1:
info('More than one built APK found...guessing you '
'just built {}'.format(apks[-1]))
apk_file = apks[-1]
info_main('# Found APK file: {}'.format(apk_file))
shprint(sh.cp, apk_file, './')