def __init__(self):
super(Context, self).__init__()
self.include_dirs = []
self._build_env_prepared = False
self._sdk_dir = None
self._ndk_dir = None
self._android_api = None
self._ndk_ver = None
self.toolchain_prefix = None
self.toolchain_version = None
self.local_recipes = None
# root of the toolchain
self.setup_dirs()
# this list should contain all Archs, it is pruned later
self.archs = (
ArchARM(self),
ArchARMv7_a(self),
Archx86(self)
)
ensure_dir(join(self.build_dir, 'bootstrap_builds'))
ensure_dir(join(self.build_dir, 'other_builds'))
# other_builds: where everything else is built
# remove the most obvious flags that can break the compilation
self.env.pop("LDFLAGS", None)
self.env.pop("ARCHFLAGS", None)
self.env.pop("CFLAGS", None)
def make_build_dest(dest):
build_dest = join(build_root, dest)
if not isdir(build_dest):
ensure_dir(build_dest)
return build_dest, False
return build_dest, True
def get_recipe_env(self, arch, with_flags_in_cc=True):
env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc)
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format(
self.ctx.get_libs_dir(arch.arch) +
' -L{} '.format(self.ctx.libs_dir) +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local',
arch.arch)))
if self.ctx.python_recipe.from_crystax:
env['LDFLAGS'] = (env['LDFLAGS'] +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)))
# ' -L/home/asandy/.local/share/python-for-android/build/bootstrap_builds/sdl2/libs/armeabi '
if self.ctx.python_recipe.from_crystax:
env['LDSHARED'] = env['CC'] + ' -shared'
else:
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh')
# shprint(sh.whereis, env['LDSHARED'], _env=env)
env['LIBLINK'] = 'NOTNONE'
env['NDKPLATFORM'] = self.ctx.ndk_platform
if self.ctx.copy_libs:
env['COPYLIBS'] = '1'
# Every recipe uses its own liblink path, object files are
# collected and biglinked later
liblink_path = join(self.get_build_container_dir(arch.arch),
'objects_{}'.format(self.name))
env['LIBLINK_PATH'] = liblink_path
ensure_dir(liblink_path)
if self.ctx.python_recipe.from_crystax:
env['CFLAGS'] = '-I{} '.format(
join(self.ctx.ndk_dir, 'sources', 'python',
self.ctx.python_recipe.version, 'include',
'python')) + env['CFLAGS']
return env
def build_recipes(build_order, python_modules, ctx):
# Put recipes in correct build order
bs = ctx.bootstrap
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'
.format(recipe.name))
# 4) biglink everything
# AND: Should make this optional
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx, python_modules)
return
def build_arch(self, arch):
super(LibZMQRecipe, self).build_arch(arch)
env = self.get_recipe_env(arch)
#
# libsodium_recipe = Recipe.get_recipe('libsodium', self.ctx)
# libsodium_dir = libsodium_recipe.get_build_dir(arch.arch)
# env['sodium_CFLAGS'] = '-I{}'.format(join(
# libsodium_dir, 'src'))
# env['sodium_LDLAGS'] = '-L{}'.format(join(
# libsodium_dir, 'src', 'libsodium', '.libs'))
curdir = self.get_build_dir(arch.arch)
prefix = join(curdir, "install")
with current_directory(curdir):
bash = sh.Command('sh')
shprint(
bash, './configure',
'--host=arm-linux-androideabi',
'--without-documentation',
'--prefix={}'.format(prefix),
'--with-libsodium=no',
_env=env)
shprint(sh.make, _env=env)
shprint(sh.make, 'install', _env=env)
shutil.copyfile('.libs/libzmq.so', join(
self.ctx.get_libs_dir(arch.arch), 'libzmq.so'))
bootstrap_obj_dir = join(self.ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)
ensure_dir(bootstrap_obj_dir)
shutil.copyfile(
'{}/sources/cxx-stl/gnu-libstdc++/{}/libs/{}/libgnustl_shared.so'.format(
self.ctx.ndk_dir, self.ctx.toolchain_version, arch),
join(bootstrap_obj_dir, 'libgnustl_shared.so'))
def get_recipe_env(self, arch, with_flags_in_cc=True):
env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc)
env["LDFLAGS"] = env["LDFLAGS"] + " -L{} ".format(
self.ctx.get_libs_dir(arch.arch)
+ " -L{} ".format(self.ctx.libs_dir)
+ " -L{}".format(join(self.ctx.bootstrap.build_dir, "obj", "local", arch.arch))
)
if self.ctx.python_recipe.from_crystax:
env["LDFLAGS"] = env["LDFLAGS"] + " -L{}".format(join(self.ctx.bootstrap.build_dir, "libs", arch.arch))
# ' -L/home/asandy/.local/share/python-for-android/build/bootstrap_builds/sdl2/libs/armeabi '
if self.ctx.python_recipe.from_crystax:
env["LDSHARED"] = env["CC"] + " -shared"
else:
env["LDSHARED"] = join(self.ctx.root_dir, "tools", "liblink.sh")
# shprint(sh.whereis, env['LDSHARED'], _env=env)
env["LIBLINK"] = "NOTNONE"
env["NDKPLATFORM"] = self.ctx.ndk_platform
if self.ctx.copy_libs:
env["COPYLIBS"] = "1"
# Every recipe uses its own liblink path, object files are
# collected and biglinked later
liblink_path = join(self.get_build_container_dir(arch.arch), "objects_{}".format(self.name))
env["LIBLINK_PATH"] = liblink_path
ensure_dir(liblink_path)
if self.ctx.python_recipe.from_crystax:
env["CFLAGS"] = (
"-I{} ".format(
join(self.ctx.ndk_dir, "sources", "python", self.ctx.python_recipe.version, "include", "python")
)
+ env["CFLAGS"]
)
return env
def get_recipe_env(self, arch, with_flags_in_cc=True):
env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc)
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format(
self.ctx.get_libs_dir(arch.arch) +
' -L{} '.format(self.ctx.libs_dir) +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local',
arch.arch)))
if self.ctx.python_recipe.from_crystax:
env['LDFLAGS'] = (env['LDFLAGS'] +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)))
if self.ctx.python_recipe.from_crystax or self.ctx.python_recipe.name == 'python3':
env['LDSHARED'] = env['CC'] + ' -shared'
else:
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh')
# shprint(sh.whereis, env['LDSHARED'], _env=env)
env['LIBLINK'] = 'NOTNONE'
env['NDKPLATFORM'] = self.ctx.ndk_platform
if self.ctx.copy_libs:
env['COPYLIBS'] = '1'
# Every recipe uses its own liblink path, object files are
# collected and biglinked later
liblink_path = join(self.get_build_container_dir(arch.arch),
'objects_{}'.format(self.name))
env['LIBLINK_PATH'] = liblink_path
ensure_dir(liblink_path)
return env
def prebuild_arch(self, arch):
if not self.is_patched(arch):
super(ReportLabRecipe, self).prebuild_arch(arch)
self.apply_patch('patches/fix-setup.patch', arch.arch)
recipe_dir = self.get_build_dir(arch.arch)
shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
ft = self.get_recipe('freetype', self.ctx)
ft_dir = ft.get_build_dir(arch.arch)
ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
info('reportlab recipe: ft_dir={}'.format(ft_dir))
info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
with current_directory(recipe_dir):
sh.ls('-lathr')
ensure_dir(tmp_dir)
pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
if not os.path.isfile(pfbfile):
sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
if os.path.isfile("setup.py"):
with open('setup.py', 'rb') as f:
text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
with open('setup.py', 'wb') as f:
f.write(text)
def build_arch(self, arch):
info("Extracting CrystaX python3 from NDK package")
dirn = self.ctx.get_python_install_dir()
ensure_dir(dirn)
self.ctx.hostpython = "python{}".format(self.version)
def run_distribute(self):
info_main('# Creating Android project from build and {} bootstrap'.format(
self.name))
info('This currently just copies the build stuff straight from the build dir.')
shprint(sh.rm, '-rf', self.dist_dir)
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Copying python distribution')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
python_bundle_dir = join('_python_bundle', '_python_bundle')
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(ServiceOnlyBootstrap, self).run_distribute()
def build_arch(self, arch):
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, self.build_subdir)
ensure_dir(build_dir)
if not exists(join(build_dir, 'python')):
with current_directory(recipe_build_dir):
# Configure the build
with current_directory(build_dir):
if not exists('config.status'):
shprint(
sh.Command(join(recipe_build_dir, 'configure')))
# Create the Setup file. This copying from Setup.dist
# seems to be the normal and expected procedure.
shprint(sh.cp, join('Modules', 'Setup.dist'),
join(build_dir, 'Modules', 'Setup'))
result = shprint(sh.make, '-C', build_dir)
else:
info('Skipping {name} ({version}) build, as it has already '
'been completed'.format(name=self.name, version=self.version))
self.ctx.hostpython = join(build_dir, 'python')
def _unpack_aar(self, aar, arch):
'''Unpack content of .aar bundle and copy to current dist dir.'''
with temp_directory() as temp_dir:
name = splitext(basename(aar))[0]
jar_name = name + '.jar'
info("unpack {} aar".format(name))
debug(" from {}".format(aar))
debug(" to {}".format(temp_dir))
shprint(sh.unzip, '-o', aar, '-d', temp_dir)
jar_src = join(temp_dir, 'classes.jar')
jar_tgt = join('libs', jar_name)
debug("copy {} jar".format(name))
debug(" from {}".format(jar_src))
debug(" to {}".format(jar_tgt))
ensure_dir('libs')
shprint(sh.cp, '-a', jar_src, jar_tgt)
so_src_dir = join(temp_dir, 'jni', arch.arch)
so_tgt_dir = join('libs', arch.arch)
debug("copy {} .so".format(name))
debug(" from {}".format(so_src_dir))
debug(" to {}".format(so_tgt_dir))
ensure_dir(so_tgt_dir)
so_files = glob.glob(join(so_src_dir, '*.so'))
for f in so_files:
shprint(sh.cp, '-a', f, so_tgt_dir)
def run_distribute(self):
info_main('# Creating Android project from build and {} bootstrap'.format(
self.name))
# src_path = join(self.ctx.root_dir, 'bootstrap_templates',
# self.name)
src_path = join(self.bootstrap_dir, 'build')
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError('built for more than one arch, but bootstrap cannot handle that yet')
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Creating initial layout')
for dirname in ('assets', 'bin', 'private', 'res', 'templates'):
if not exists(dirname):
shprint(sh.mkdir, dirname)
info('Copying default files')
shprint(sh.cp, '-a', join(self.build_dir, 'project.properties'), '.')
shprint(sh.cp, '-a', join(src_path, 'build.py'), '.')
shprint(sh.cp, '-a', join(src_path, 'buildlib'), '.')
shprint(sh.cp, '-a', join(src_path, 'src'), '.')
shprint(sh.cp, '-a', join(src_path, 'templates'), '.')
shprint(sh.cp, '-a', join(src_path, 'res'), '.')
shprint(sh.cp, '-a', join(src_path, 'blacklist.txt'), '.')
shprint(sh.cp, '-a', join(src_path, 'whitelist.txt'), '.')
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
info('Copying python distribution')
python_bundle_dir = join('_python_bundle', '_python_bundle')
if 'python2legacy' in self.ctx.recipe_build_order:
# a special case with its own packaging location
python_bundle_dir = 'private'
# And also must had an install directory, make sure of that
self.ctx.python_recipe.create_python_install(self.dist_dir)
self.distribute_libs(
arch, [join(self.build_dir, 'libs', arch.arch),
self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(PygameBootstrap, self).run_distribute()
def distribute_libs(self, arch, src_dirs, wildcard='*', dest_dir="libs"):
'''Copy existing arch libs from build dirs to current dist dir.'''
info('Copying libs')
tgt_dir = join(dest_dir, arch.arch)
ensure_dir(tgt_dir)
for src_dir in src_dirs:
for lib in glob.glob(join(src_dir, wildcard)):
shprint(sh.cp, '-a', lib, tgt_dir)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('autoreconf'), '-vif', _env=env)
shprint(sh.Command('./configure'),
'--host=' + arch.command_prefix,
'--prefix=' + self.get_build_dir(arch.arch),
'--disable-builddir',
'--enable-shared', _env=env)
# '--with-sysroot={}'.format(self.ctx.ndk_platform),
# '--target={}'.format(arch.toolchain_prefix),
# ndk 15 introduces unified headers required --sysroot and
# -isysroot for libraries and headers. libtool's head explodes
# trying to weave them into it's own magic. The result is a link
# failure trying to link libc. We call make to compile the bits
# and manually link...
try:
shprint(sh.make, '-j5', 'libffi.la', _env=env)
except sh.ErrorReturnCode_2:
info("make libffi.la failed as expected")
cc = sh.Command(env['CC'].split()[0])
cflags = env['CC'].split()[1:]
host_build = self.get_build_dir(arch.arch)
arch_flags = ''
if '-march=' in env['CFLAGS']:
arch_flags = '-march={}'.format(env['CFLAGS'].split('-march=')[1])
src_arch = arch.command_prefix.split('-')[0]
if src_arch == 'x86_64':
# libffi has not specific arch files for x86_64...so...using
# the ones from x86 which seems to build fine...
src_arch = 'x86'
cflags.extend(arch_flags.split())
cflags.extend(['-shared', '-fPIC', '-DPIC'])
cflags.extend(glob(join(host_build, 'src/.libs/*.o')))
cflags.extend(glob(join(host_build, 'src', src_arch, '.libs/*.o')))
ldflags = env['LDFLAGS'].split()
cflags.extend(ldflags)
cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o',
'.libs/libffi.so'])
with current_directory(host_build):
shprint(cc, *cflags, _env=env)
ensure_dir(self.ctx.get_libs_dir(arch.arch))
shprint(sh.cp,
join(host_build, '.libs', 'libffi.so'),
self.ctx.get_libs_dir(arch.arch))
def run_distribute(self):
info_main("# Creating Android project ({})".format(self.name))
arch = self.ctx.archs[0]
python_install_dir = self.ctx.get_python_install_dir()
from_crystax = self.ctx.python_recipe.from_crystax
if len(self.ctx.archs) > 1:
raise ValueError("SDL2/gradle support only one arch")
info("Copying SDL2/gradle build for {}".format(arch))
shprint(sh.rm, "-rf", self.dist_dir)
shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
# either the build use environment variable (ANDROID_HOME)
# or the local.properties if exists
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
with current_directory(self.dist_dir):
info("Copying Python distribution")
hostpython = sh.Command(self.ctx.hostpython)
if self.ctx.python_recipe.name == 'python2':
try:
shprint(hostpython, '-OO', '-m', 'compileall',
python_install_dir,
_tail=10, _filterout="^Listing")
except sh.ErrorReturnCode:
pass
if 'python2' in self.ctx.recipe_build_order and not exists('python-install'):
shprint(
sh.cp, '-a', python_install_dir, './python-install')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_javaclasses(self.ctx.javaclass_dir,
dest_dir=join("src", "main", "java"))
python_bundle_dir = join('_python_bundle', '_python_bundle')
if 'python2' in self.ctx.recipe_build_order:
# Python 2 is a special case with its own packaging location
python_bundle_dir = 'private'
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(SDL2GradleBootstrap, self).run_distribute()
def get_recipe_env(self, arch=None):
env = super(PygameRecipe, self).get_recipe_env(arch)
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
self.ctx.get_libs_dir(arch.arch))
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
env['LIBLINK'] = 'NOTNONE'
env['NDKPLATFORM'] = self.ctx.ndk_platform
# Every recipe uses its own liblink path, object files are collected and biglinked later
liblink_path = join(self.get_build_container_dir(arch.arch), 'objects_{}'.format(self.name))
env['LIBLINK_PATH'] = liblink_path
ensure_dir(liblink_path)
return env
def get_recipe_env(self, arch, with_flags_in_cc=True):
env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc)
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format(
self.ctx.get_libs_dir(arch.arch) +
' -L{} '.format(self.ctx.libs_dir) +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local',
arch.arch)))
if self.ctx.python_recipe.from_crystax:
env['LDFLAGS'] = (env['LDFLAGS'] +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)))
if self.ctx.python_recipe.name == 'python2legacy':
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh')
else:
env['LDSHARED'] = env['CC'] + ' -shared'
# shprint(sh.whereis, env['LDSHARED'], _env=env)
env['LIBLINK'] = 'NOTNONE'
env['NDKPLATFORM'] = self.ctx.ndk_platform
if self.ctx.copy_libs:
env['COPYLIBS'] = '1'
# Every recipe uses its own liblink path, object files are
# collected and biglinked later
liblink_path = join(self.get_build_container_dir(arch.arch),
'objects_{}'.format(self.name))
env['LIBLINK_PATH'] = liblink_path
ensure_dir(liblink_path)
# Add crystax-specific site packages:
if self.ctx.python_recipe.from_crystax:
command = sh.Command('python{}'.format(self.ctx.python_recipe.version))
site_packages_dirs = command(
'-c', 'import site; print("\\n".join(site.getsitepackages()))')
site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n')
if 'PYTHONPATH' in env:
env['PYTHONPATH'] = env['PYTHONPATH'] +\
':{}'.format(':'.join(site_packages_dirs))
else:
env['PYTHONPATH'] = ':'.join(site_packages_dirs)
while env['PYTHONPATH'].find("::") > 0:
env['PYTHONPATH'] = env['PYTHONPATH'].replace("::", ":")
if env['PYTHONPATH'].endswith(":"):
env['PYTHONPATH'] = env['PYTHONPATH'][:-1]
if env['PYTHONPATH'].startswith(":"):
env['PYTHONPATH'] = env['PYTHONPATH'][1:]
return env
def unpack(self, arch):
info_main('Unpacking {} for {}'.format(self.name, arch))
build_dir = self.get_build_container_dir(arch)
user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
if user_dir is not None:
info("Installing KivyMD development versoion (from source)")
self.clean_build()
shprint(sh.rm, '-rf', build_dir)
shprint(sh.mkdir, '-p', build_dir)
shprint(sh.rmdir, build_dir)
ensure_dir(build_dir)
ensure_dir(build_dir + "/kivymd")
shprint(sh.cp, user_dir + '/setup.py', self.get_build_dir(arch) + "/setup.py")
shprint(sh.cp, '-a', user_dir + "/kivymd", self.get_build_dir(arch) + "/kivymd")
return
def build_arch(self, arch):
# We don't have to actually build anything as CrystaX comes
# with the necessary modules. They are included by modifying
# the Android.mk in the jni folder.
# If the Python version to be used is not prebuilt with the CrystaX
# NDK, we do have to download it.
crystax_python_dir = join(self.ctx.ndk_dir, 'sources', 'python')
if not exists(join(crystax_python_dir, self.version)):
info(('The NDK does not have a prebuilt Python {}, trying '
'to obtain one.').format(self.version))
if self.version not in prebuilt_download_locations:
error(('No prebuilt version for Python {} could be found, '
'the built cannot continue.'))
exit(1)
with temp_directory() as td:
self.download_file(prebuilt_download_locations[self.version],
join(td, 'downloaded_python'))
shprint(sh.tar, 'xf', join(td, 'downloaded_python'),
'--directory', crystax_python_dir)
if not exists(join(crystax_python_dir, self.version)):
error(('Something went wrong, the directory at {} should '
'have been created but does not exist.').format(
join(crystax_python_dir, self.version)))
if not exists(join(
crystax_python_dir, self.version, 'libs', arch.arch)):
error(('The prebuilt Python for version {} does not contain '
'binaries for your chosen architecture "{}".').format(
self.version, arch.arch))
exit(1)
# TODO: We should have an option to build a new Python. This
# would also allow linking to openssl and sqlite from CrystaX.
dirn = self.ctx.get_python_install_dir()
ensure_dir(dirn)
# Instead of using a locally built hostpython, we use the
# user's Python for now. They must have the right version
# available. Using e.g. pyenv makes this easy.
self.ctx.hostpython = 'python{}'.format(self.version)
def run_distribute(self):
info_main("# Creating Android project ({})".format(self.name))
arch = self.ctx.archs[0]
python_install_dir = self.ctx.get_python_install_dir()
from_crystax = self.ctx.python_recipe.from_crystax
if len(self.ctx.archs) > 1:
raise ValueError("SDL2/gradle support only one arch")
info("Copying SDL2/gradle build for {}".format(arch))
shprint(sh.rm, "-rf", self.dist_dir)
shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
# either the build use environment variable (ANDROID_HOME)
# or the local.properties if exists
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
with current_directory(self.dist_dir):
info("Copying Python distribution")
python_bundle_dir = join('_python_bundle', '_python_bundle')
if 'python2legacy' in self.ctx.recipe_build_order:
# a special case with its own packaging location
python_bundle_dir = 'private'
# And also must had an install directory, make sure of that
self.ctx.python_recipe.create_python_install(self.dist_dir)
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_javaclasses(self.ctx.javaclass_dir,
dest_dir=join("src", "main", "java"))
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(SDL2GradleBootstrap, self).run_distribute()
def build_arch(self, arch):
if self.ctx.ndk_api < self.MIN_NDK_API:
raise BuildInterruptingException(
'Target ndk-api is {}, but the python3 recipe supports only'
' {}+'.format(self.ctx.ndk_api, self.MIN_NDK_API))
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, 'android-build')
ensure_dir(build_dir)
# TODO: Get these dynamically, like bpo-30386 does
sys_prefix = '/usr/local'
sys_exec_prefix = '/usr/local'
with current_directory(build_dir):
env = self.get_recipe_env(arch)
env = self.set_libs_flags(env, arch)
android_build = sh.Command(
join(recipe_build_dir,
'config.guess'))().stdout.strip().decode('utf-8')
if not exists('config.status'):
shprint(
sh.Command(join(recipe_build_dir, 'configure')),
*(' '.join(self.configure_args).format(
android_host=env['HOSTARCH'],
android_build=android_build,
prefix=sys_prefix,
exec_prefix=sys_exec_prefix)).split(' '),
_env=env)
if not exists('python'):
py_version = self.major_minor_version_string
if self.major_minor_version_string[0] == '3':
py_version += 'm'
shprint(sh.make, 'all',
'INSTSONAME=libpython{version}.so'.format(
version=py_version), _env=env)
# TODO: Look into passing the path to pyconfig.h in a
# better way, although this is probably acceptable
sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
def setUp(self):
"""
Setups recipe and context.
"""
self.context = Context()
self.arch = ArchARMv7_a(self.context)
self.recipe = Recipe.get_recipe('reportlab', self.context)
self.recipe.ctx = self.context
self.bootstrap = None
recipe_build_order, python_modules, bootstrap = \
get_recipe_order_and_bootstrap(
self.context, [self.recipe.name], self.bootstrap)
self.context.recipe_build_order = recipe_build_order
self.context.python_modules = python_modules
self.context.setup_dirs(tempfile.gettempdir())
self.bootstrap = bootstrap
self.recipe_dir = self.recipe.get_build_dir(self.arch.arch)
ensure_dir(self.recipe_dir)
def get_recipe_env(self, arch, with_flags_in_cc=True):
env = super(CythonRecipe, self).get_recipe_env(arch, with_flags_in_cc)
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{} '.format(
self.ctx.get_libs_dir(arch.arch) +
' -L{} '.format(self.ctx.libs_dir) +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'obj', 'local',
arch.arch)))
if self.ctx.python_recipe.from_crystax:
env['LDFLAGS'] = (env['LDFLAGS'] +
' -L{}'.format(join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)))
# ' -L/home/asandy/.local/share/python-for-android/build/bootstrap_builds/sdl2/libs/armeabi '
if self.ctx.python_recipe.from_crystax:
env['LDSHARED'] = env['CC'] + ' -shared'
else:
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink.sh')
# shprint(sh.whereis, env['LDSHARED'], _env=env)
env['LIBLINK'] = 'NOTNONE'
env['NDKPLATFORM'] = self.ctx.ndk_platform
if self.ctx.copy_libs:
env['COPYLIBS'] = '1'
# Every recipe uses its own liblink path, object files are
# collected and biglinked later
liblink_path = join(self.get_build_container_dir(arch.arch),
'objects_{}'.format(self.name))
env['LIBLINK_PATH'] = liblink_path
ensure_dir(liblink_path)
if self.ctx.python_recipe.from_crystax:
env['CFLAGS'] = '-I{} '.format(
join(self.ctx.ndk_dir, 'sources', 'python',
self.ctx.python_recipe.version, 'include',
'python')) + env['CFLAGS']
# Temporarily hardcode the -lpython3.x as this does not
# get applied automatically in some environments. This
# will need generalising, along with the other hardcoded
# py3.5 references, to support other python3 or crystax
# python versions.
python3_version = self.ctx.python_recipe.version
python3_version = '.'.join(python3_version.split('.')[:2])
env['LDFLAGS'] = env['LDFLAGS'] + ' -lpython{}m'.format(python3_version)
return env
def biglink(ctx, arch):
# First, collate object files from each recipe
info('Collating object files from each recipe')
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
'objects_{}'.format(recipe.name))
if not exists(recipe_obj_dir):
info('{} recipe has no biglinkable files dir, skipping'
.format(recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, '*'))
if not len(files):
info('{} recipe has no biglinkable files, skipping'
.format(recipe.name))
continue
info('{} recipe has object files, copying'.format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, '-r', *files)
env = arch.get_env()
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
if not len(glob.glob(join(obj_dir, '*'))):
info('There seem to be no libraries to biglink, skipping.')
return
info('Biglinking')
info('target {}'.format(join(ctx.get_libs_dir(arch.arch),
'libpymodules.so')))
do_biglink = copylibs_function if ctx.copy_libs else biglink_function
# Move to the directory containing crtstart_so.o and crtend_so.o
# This is necessary with newer NDKs? A gcc bug?
with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):
do_biglink(
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
obj_dir.split(' '),
extra_link_dirs=[join(ctx.bootstrap.build_dir,
'obj', 'local', arch.arch),
os.path.abspath('.')],
env=env)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('autoreconf'), '-vif', _env=env)
shprint(sh.Command('./configure'),
'--host=' + arch.command_prefix,
'--prefix=' + self.get_build_dir(arch.arch),
'--disable-builddir',
'--enable-shared', _env=env)
shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env)
host_build = self.get_build_dir(arch.arch)
ensure_dir(self.ctx.get_libs_dir(arch.arch))
shprint(sh.cp,
join(host_build, '.libs', 'libffi.so'),
self.ctx.get_libs_dir(arch.arch))
def copy_files(self, arch):
ndk = self.ctx.ndk_dir
env = self.get_recipe_env(arch)
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
lib = lib.format(ndk=self.ctx.ndk_dir,
version=env["TOOLCHAIN_VERSION"],
arch=arch.arch)
stl_lib = join(lib, "libgnustl_shared.so")
dst_dir = join(self.ctx.get_site_packages_dir(), "..", "lib-dynload")
shprint(sh.cp, stl_lib, dst_dir)
src_lib = join(self.get_build_dir(arch.arch), "icu_build", "lib")
dst_lib = self.get_lib_dir(arch)
src_suffix = "." + self.version
dst_suffix = "." + self.version.split(".")[0] # main version
for lib in self.generated_libraries:
shprint(sh.cp, join(src_lib, lib+src_suffix),
join(dst_lib, lib+dst_suffix))
src_include = join(
self.get_build_dir(arch.arch), "icu_build", "include")
dst_include = join(
self.ctx.get_python_install_dir(), "include", "icu")
ensure_dir(dst_include)
shprint(sh.cp, "-r", join(src_include, "layout"), dst_include)
shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include)
# copy stl library
lib = "{ndk}/sources/cxx-stl/gnu-libstdc++/{version}/libs/{arch}"
lib = lib.format(ndk=self.ctx.ndk_dir,
version=env["TOOLCHAIN_VERSION"],
arch=arch.arch)
stl_lib = join(lib, "libgnustl_shared.so")
dst_dir = join(self.ctx.get_python_install_dir(), "lib")
ensure_dir(dst_dir)
shprint(sh.cp, stl_lib, dst_dir)
def prebuild_arch(self, arch):
if not self.is_patched(arch):
super(ReportLabRecipe, self).prebuild_arch(arch)
recipe_dir = self.get_build_dir(arch.arch)
# Some versions of reportlab ship with a GPL-licensed font.
# Remove it, since this is problematic in .apks unless the
# entire app is GPL:
font_dir = os.path.join(recipe_dir,
"src", "reportlab", "fonts")
if os.path.exists(font_dir):
for l in os.listdir(font_dir):
if l.lower().startswith('darkgarden'):
os.remove(os.path.join(font_dir, l))
# Apply patches:
self.apply_patch('patches/fix-setup.patch', arch.arch)
shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
ft = self.get_recipe('freetype', self.ctx)
ft_dir = ft.get_build_dir(arch.arch)
ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
info('reportlab recipe: ft_dir={}'.format(ft_dir))
info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
with current_directory(recipe_dir):
ensure_dir(tmp_dir)
pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
if not os.path.isfile(pfbfile):
sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
if os.path.isfile("setup.py"):
with open('setup.py', 'r') as f:
text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
with open('setup.py', 'w') as f:
f.write(text)
def biglink(ctx, arch):
# First, collate object files from each recipe
info('Collating object files from each recipe')
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
'objects_{}'.format(recipe.name))
if not exists(recipe_obj_dir):
info('{} recipe has no biglinkable files dir, skipping'
.format(recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, '*'))
if not len(files):
info('{} recipe has no biglinkable files, skipping'
.format(recipe.name))
info('{} recipe has object files, copying'.format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, '-r', *files)
env = arch.get_env()
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
if not len(glob.glob(join(obj_dir, '*'))):
info('There seem to be no libraries to biglink, skipping.')
return
info('Biglinking')
info('target {}'.format(join(ctx.get_libs_dir(arch.arch),
'libpymodules.so')))
biglink_function(
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
obj_dir.split(' '),
extra_link_dirs=[join(ctx.bootstrap.build_dir,
'obj', 'local', arch.arch)],
env=env)
def setup_dirs(self):
'''Calculates all the storage and build dirs, and makes sure
the directories exist where necessary.'''
self.root_dir = realpath(dirname(__file__))
# AND: TODO: Allow the user to set the build_dir
self.storage_dir = user_data_dir('python-for-android')
self.build_dir = join(self.storage_dir, 'build')
self.dist_dir = join(self.storage_dir, 'dists')
ensure_dir(self.storage_dir)
ensure_dir(self.build_dir)
ensure_dir(self.dist_dir)
def build_recipes(build_order,
python_modules,
ctx,
project_dir,
ignore_project_setup_py=False):
# Put recipes in correct build order
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
recipe.install_libraries(arch)
else:
info('{} said it is already built, skipping'.format(
recipe.name))
# 4) biglink everything
info_main('# Biglinking object files')
if not ctx.python_recipe:
biglink(ctx, arch)
else:
warning("Context's python recipe found, "
"skipping biglink (will this work?)")
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx,
python_modules,
project_dir,
ignore_setup_py=ignore_project_setup_py)
return
def run_distribute(self):
info_main("# Creating Android project ({})".format(self.name))
arch = self.ctx.archs[0]
python_install_dir = self.ctx.get_python_install_dir()
from_crystax = self.ctx.python_recipe.from_crystax
if len(self.ctx.archs) > 1:
raise ValueError("SDL2/gradle support only one arch")
info("Copying SDL2/gradle build for {}".format(arch))
shprint(sh.rm, "-rf", self.dist_dir)
shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
# either the build use environment variable (ANDROID_HOME)
# or the local.properties if exists
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
with current_directory(self.dist_dir):
info("Copying Python distribution")
hostpython = sh.Command(self.ctx.hostpython)
if self.ctx.python_recipe.name == 'python2':
try:
shprint(hostpython,
'-OO',
'-m',
'compileall',
'-q',
python_install_dir,
_tail=10,
_filterout="^Listing")
except sh.ErrorReturnCode:
pass
if 'python2' in self.ctx.recipe_build_order and not exists(
'python-install'):
shprint(sh.cp, '-a', python_install_dir,
'./python-install')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_javaclasses(self.ctx.javaclass_dir,
dest_dir=join("src", "main", "java"))
python_bundle_dir = join('_python_bundle', '_python_bundle')
if 'python2' in self.ctx.recipe_build_order:
# Python 2 is a special case with its own packaging location
python_bundle_dir = 'private'
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(SDL2GradleBootstrap, self).run_distribute()
def get_lib_dir(self, arch):
lib_dir = join(self.ctx.get_python_install_dir(), "lib")
ensure_dir(lib_dir)
return lib_dir
def prepare_dist_dir(self, name):
ensure_dir(self.dist_dir)
def distribute_javaclasses(self, javaclass_dir, dest_dir="src"):
'''Copy existing javaclasses from build dir to current dist dir.'''
info('Copying java files')
ensure_dir(dest_dir)
filenames = glob.glob(javaclass_dir)
shprint(sh.cp, '-a', *filenames, dest_dir)
def javaclass_dir(self):
# Was previously hardcoded as self.build_dir/java
dir = join(self.build_dir, 'javaclasses',
self.bootstrap.distribution.name)
ensure_dir(dir)
return dir
def aars_dir(self):
dir = join(self.build_dir, 'aars', self.bootstrap.distribution.name)
ensure_dir(dir)
return dir
def libs_dir(self):
# Was previously hardcoded as self.build_dir/libs
dir = join(self.build_dir, 'libs_collections',
self.bootstrap.distribution.name)
ensure_dir(dir)
return dir
def get_libs_dir(self, arch):
'''The libs dir for a given arch.'''
ensure_dir(join(self.libs_dir, arch))
return join(self.libs_dir, arch)
def run_distribute(self):
info_main("# Creating Android project ({})".format(self.name))
arch = self.ctx.archs[0]
python_install_dir = self.ctx.get_python_install_dir()
from_crystax = self.ctx.python_recipe.from_crystax
crystax_python_dir = join("crystax_python", "crystax_python")
if len(self.ctx.archs) > 1:
raise ValueError("SDL2/gradle support only one arch")
info("Copying SDL2/gradle build for {}".format(arch))
shprint(sh.rm, "-rf", self.dist_dir)
shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
# either the build use environemnt variable (ANDROID_HOME)
# or the local.properties if exists
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
with current_directory(self.dist_dir):
info("Copying Python distribution")
if not exists("private") and not from_crystax:
ensure_dir("private")
if not exists("crystax_python") and from_crystax:
ensure_dir(crystax_python_dir)
hostpython = sh.Command(self.ctx.hostpython)
if not from_crystax:
try:
shprint(hostpython,
'-OO',
'-m',
'compileall',
python_install_dir,
_tail=10,
_filterout="^Listing")
except sh.ErrorReturnCode:
pass
if not exists('python-install'):
shprint(sh.cp, '-a', python_install_dir,
'./python-install')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_javaclasses(self.ctx.javaclass_dir,
dest_dir=join("src", "main", "java"))
if not from_crystax:
info("Filling private directory")
if not exists(join("private", "lib")):
info("private/lib does not exist, making")
shprint(sh.cp, "-a", join("python-install", "lib"),
"private")
shprint(sh.mkdir, "-p", join("private", "include",
"python2.7"))
libpymodules_fn = join("libs", arch.arch, "libpymodules.so")
if exists(libpymodules_fn):
shprint(sh.mv, libpymodules_fn, 'private/')
shprint(
sh.cp,
join('python-install', 'include',
'python2.7', 'pyconfig.h'),
join('private', 'include', 'python2.7/'))
info('Removing some unwanted files')
shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so'))
shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig'))
libdir = join(self.dist_dir, 'private', 'lib', 'python2.7')
site_packages_dir = join(libdir, 'site-packages')
with current_directory(libdir):
removes = []
for dirname, root, filenames in walk("."):
for filename in filenames:
for suffix in EXCLUDE_EXTS:
if filename.endswith(suffix):
removes.append(filename)
shprint(sh.rm, '-f', *removes)
info('Deleting some other stuff not used on android')
# To quote the original distribute.sh, 'well...'
shprint(sh.rm, '-rf', 'idlelib')
for filename in glob.glob('config/libpython*.a'):
shprint(sh.rm, '-f', filename)
shprint(sh.rm, '-rf', 'config/python.o')
else: # Python *is* loaded from crystax
ndk_dir = self.ctx.ndk_dir
py_recipe = self.ctx.python_recipe
python_dir = join(ndk_dir, 'sources', 'python',
py_recipe.version, 'libs', arch.arch)
shprint(sh.cp, '-r', join(python_dir, 'stdlib.zip'),
crystax_python_dir)
shprint(sh.cp, '-r', join(python_dir, 'modules'),
crystax_python_dir)
shprint(sh.cp, '-r', self.ctx.get_python_install_dir(),
join(crystax_python_dir, 'site-packages'))
info('Renaming .so files to reflect cross-compile')
site_packages_dir = join(crystax_python_dir, "site-packages")
find_ret = shprint(sh.find, site_packages_dir, '-iname',
'*.so')
filenames = find_ret.stdout.decode('utf-8').split('\n')[:-1]
for filename in filenames:
parts = filename.split('.')
if len(parts) <= 2:
continue
shprint(sh.mv, filename, filename.split('.')[0] + '.so')
site_packages_dir = join(abspath(curdir), site_packages_dir)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(SDL2GradleBootstrap, self).run_distribute()
def ensure_dirs(self):
ensure_dir(self.storage_dir)
ensure_dir(self.build_dir)
ensure_dir(self.dist_dir)
ensure_dir(join(self.build_dir, 'bootstrap_builds'))
ensure_dir(join(self.build_dir, 'other_builds'))
def create_python_bundle(self, dirn, arch):
"""
Create a packaged python bundle in the target directory, by
copying all the modules and standard library to the right
place.
"""
# Todo: find a better way to find the build libs folder
modules_build_dir = join(
self.get_build_dir(arch.arch),
"android-build",
"build",
"lib.linux{}-{}-{}".format(
"2" if self.version[0] == "2" else "",
arch.command_prefix.split("-")[0],
self.major_minor_version_string,
),
)
# Compile to *.pyc/*.pyo the python modules
self.compile_python_files(modules_build_dir)
# Compile to *.pyc/*.pyo the standard python library
self.compile_python_files(join(self.get_build_dir(arch.arch), "Lib"))
# Compile to *.pyc/*.pyo the other python packages (site-packages)
self.compile_python_files(self.ctx.get_python_install_dir())
# Bundle compiled python modules to a folder
modules_dir = join(dirn, "modules")
c_ext = self.compiled_extension
ensure_dir(modules_dir)
module_filens = glob.glob(join(modules_build_dir, "*.so")) + glob.glob(
join(modules_build_dir, "*" + c_ext))
info("Copy {} files into the bundle".format(len(module_filens)))
for filen in module_filens:
info(" - copy {}".format(filen))
copy2(filen, modules_dir)
# zip up the standard library
stdlib_zip = join(dirn, "stdlib.zip")
with current_directory(join(self.get_build_dir(arch.arch), "Lib")):
stdlib_filens = list(
walk_valid_filens(".", self.stdlib_dir_blacklist,
self.stdlib_filen_blacklist))
info("Zip {} files into the bundle".format(len(stdlib_filens)))
shprint(sh.zip, stdlib_zip, *stdlib_filens)
# copy the site-packages into place
ensure_dir(join(dirn, "site-packages"))
ensure_dir(self.ctx.get_python_install_dir())
# TODO: Improve the API around walking and copying the files
with current_directory(self.ctx.get_python_install_dir()):
filens = list(
walk_valid_filens(
".",
self.site_packages_dir_blacklist,
self.site_packages_filen_blacklist,
))
info("Copy {} files into the site-packages".format(len(filens)))
for filen in filens:
info(" - copy {}".format(filen))
ensure_dir(join(dirn, "site-packages", dirname(filen)))
copy2(filen, join(dirn, "site-packages", filen))
# copy the python .so files into place
python_build_dir = join(self.get_build_dir(arch.arch), "android-build")
python_lib_name = "libpython" + self.major_minor_version_string
if self.major_minor_version_string[0] == "3":
python_lib_name += "m"
shprint(
sh.cp,
join(python_build_dir, python_lib_name + ".so"),
join(self.ctx.bootstrap.dist_dir, "libs", arch.arch),
)
info("Renaming .so files to reflect cross-compile")
self.reduce_object_file_names(join(dirn, "site-packages"))
return join(dirn, "site-packages")
def create_python_bundle(self, dirn, arch):
"""
Create a packaged python bundle in the target directory, by
copying all the modules and standard library to the right
place.
"""
# Todo: find a better way to find the build libs folder
modules_build_dir = join(
self.get_build_dir(arch.arch), 'android-build', 'build',
'lib.linux{}-{}-{}'.format('2' if self.version[0] == '2' else '',
arch.command_prefix.split('-')[0],
self.major_minor_version_string))
# Compile to *.pyc/*.pyo the python modules
self.compile_python_files(modules_build_dir)
# Compile to *.pyc/*.pyo the standard python library
self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
# Compile to *.pyc/*.pyo the other python packages (site-packages)
self.compile_python_files(self.ctx.get_python_install_dir())
# Bundle compiled python modules to a folder
modules_dir = join(dirn, 'modules')
c_ext = self.compiled_extension
ensure_dir(modules_dir)
module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
glob.glob(join(modules_build_dir, '*' + c_ext)))
info("Copy {} files into the bundle".format(len(module_filens)))
for filen in module_filens:
info(" - copy {}".format(filen))
copy2(filen, modules_dir)
# zip up the standard library
stdlib_zip = join(dirn, 'stdlib.zip')
with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
stdlib_filens = list(
walk_valid_filens('.', self.stdlib_dir_blacklist,
self.stdlib_filen_blacklist))
info("Zip {} files into the bundle".format(len(stdlib_filens)))
shprint(sh.zip, stdlib_zip, *stdlib_filens)
# copy the site-packages into place
ensure_dir(join(dirn, 'site-packages'))
ensure_dir(self.ctx.get_python_install_dir())
# TODO: Improve the API around walking and copying the files
with current_directory(self.ctx.get_python_install_dir()):
filens = list(
walk_valid_filens('.', self.site_packages_dir_blacklist,
self.site_packages_filen_blacklist))
info("Copy {} files into the site-packages".format(len(filens)))
for filen in filens:
info(" - copy {}".format(filen))
ensure_dir(join(dirn, 'site-packages', dirname(filen)))
copy2(filen, join(dirn, 'site-packages', filen))
# copy the python .so files into place
python_build_dir = join(self.get_build_dir(arch.arch), 'android-build')
python_lib_name = 'libpython' + self.major_minor_version_string
if self.major_minor_version_string[0] == '3':
python_lib_name += 'm'
shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'),
join(self.ctx.dist_dir, self.ctx.dist_name, 'libs', arch.arch))
info('Renaming .so files to reflect cross-compile')
self.reduce_object_file_names(join(dirn, 'site-packages'))
return join(dirn, 'site-packages')
def build_arch(self, arch):
if self.ctx.ndk_api < self.MIN_NDK_API:
raise BuildInterruptingException(
'Target ndk-api is {}, but the python3 recipe supports only {}+'
.format(self.ctx.ndk_api, self.MIN_NDK_API))
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, 'android-build')
ensure_dir(build_dir)
# TODO: Get these dynamically, like bpo-30386 does
sys_prefix = '/usr/local'
sys_exec_prefix = '/usr/local'
# Skipping "Ensure that nl_langinfo is broken" from the original bpo-30386
platform_name = 'android-{}'.format(self.ctx.ndk_api)
with current_directory(build_dir):
env = environ.copy()
# TODO: Get this information from p4a's arch system
android_host = arch.command_prefix
android_build = sh.Command(
join(recipe_build_dir,
'config.guess'))().stdout.strip().decode('utf-8')
platform_dir = join(self.ctx.ndk_dir, 'platforms', platform_name,
arch.platform_dir)
toolchain = '{android_host}-4.9'.format(
android_host=arch.toolchain_prefix)
toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain,
'prebuilt', 'linux-x86_64')
target_data = arch.command_prefix.split('-')
if target_data[0] == 'arm':
target_data[0] = 'armv7a'
target = '-'.join(
[target_data[0], 'none', target_data[1], target_data[2]])
CC = '{clang} -target {target} -gcc-toolchain {toolchain}'.format(
clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt',
'linux-x86_64', 'bin', 'clang'),
target=target,
toolchain=toolchain)
AR = join(toolchain, 'bin', android_host) + '-ar'
LD = join(toolchain, 'bin', android_host) + '-ld'
RANLIB = join(toolchain, 'bin', android_host) + '-ranlib'
READELF = join(toolchain, 'bin', android_host) + '-readelf'
STRIP = join(
toolchain, 'bin',
android_host) + '-strip --strip-debug --strip-unneeded'
env['CC'] = CC
env['AR'] = AR
env['LD'] = LD
env['RANLIB'] = RANLIB
env['READELF'] = READELF
env['STRIP'] = STRIP
env['PATH'] = '{hostpython_dir}:{old_path}'.format(
hostpython_dir=self.get_recipe('hostpython3',
self.ctx).get_path_to_python(),
old_path=env['PATH'])
ndk_flags = (
'-fPIC --sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} '
'-isystem {ndk_android_host}').format(
ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'),
android_api=self.ctx.ndk_api,
ndk_android_host=join(self.ctx.ndk_dir, 'sysroot', 'usr',
'include', android_host))
sysroot = join(self.ctx.ndk_dir, 'platforms', platform_name,
arch.platform_dir)
env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags
env['LDFLAGS'] = env.get('LDFLAGS',
'') + ' --sysroot={} -L{}'.format(
sysroot, join(sysroot, 'usr', 'lib'))
# Manually add the libs directory, and copy some object
# files to the current directory otherwise they aren't
# picked up. This seems necessary because the --sysroot
# setting in LDFLAGS is overridden by the other flags.
# TODO: Work out why this doesn't happen in the original
# bpo-30386 Makefile system.
logger.warning('Doing some hacky stuff to link properly')
lib_dir = join(sysroot, 'usr', 'lib')
if arch.arch == 'x86_64':
lib_dir = join(sysroot, 'usr', 'lib64')
env['LDFLAGS'] += ' -L{}'.format(lib_dir)
shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './')
shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './')
env['SYSROOT'] = sysroot
env = self.set_libs_flags(env, arch)
if not exists('config.status'):
shprint(sh.Command(join(recipe_build_dir, 'configure')),
*(' '.join(
('--host={android_host}',
'--build={android_build}', '--enable-shared',
'--disable-ipv6', 'ac_cv_file__dev_ptmx=yes',
'ac_cv_file__dev_ptc=no', '--without-ensurepip',
'ac_cv_little_endian_double=yes',
'--prefix={prefix}',
'--exec-prefix={exec_prefix}')).format(
android_host=android_host,
android_build=android_build,
prefix=sys_prefix,
exec_prefix=sys_exec_prefix)).split(' '),
_env=env)
if not exists('python'):
shprint(sh.make, 'all', _env=env)
# TODO: Look into passing the path to pyconfig.h in a
# better way, although this is probably acceptable
sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
def unpack(self, arch):
info_main('Unpacking {} for {}'.format(self.name, arch))
build_dir = self.get_build_container_dir(arch)
user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
if user_dir is not None:
info('P4A_{}_DIR exists, symlinking instead'.format(
self.name.lower()))
if exists(self.get_build_dir(arch)):
return
shprint(sh.rm, '-rf', build_dir)
shprint(sh.mkdir, '-p', build_dir)
shprint(sh.rmdir, build_dir)
ensure_dir(build_dir)
shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
return
if self.url is None:
info('Skipping {} unpack as no URL is set'.format(self.name))
return
filename = shprint(sh.basename,
self.versioned_url).stdout[:-1].decode('utf-8')
ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename)
if ma: # fragmented URL?
filename = ma.group(1)
with current_directory(build_dir):
directory_name = self.get_build_dir(arch)
if not exists(directory_name) or not isdir(directory_name):
extraction_filename = join(self.ctx.packages_path, self.name,
filename)
if isfile(extraction_filename):
info('File type for {}'.format(extraction_filename))
os.system('file ' + extraction_filename)
if extraction_filename.endswith('.zip'):
try:
sh.unzip(extraction_filename)
except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2):
# return code 1 means unzipping had
# warnings but did complete,
# apparently happens sometimes with
# github zips
pass
import zipfile
fileh = zipfile.ZipFile(extraction_filename, 'r')
root_directory = fileh.filelist[0].filename.split(
'/')[0]
if root_directory != basename(directory_name):
shprint(sh.mv, root_directory, directory_name)
elif extraction_filename.endswith(
('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz',
'.txz')):
sh.tar('xvf', extraction_filename)
root_directory = sh.tar(
'tf', extraction_filename).stdout.decode(
'utf-8').split('\n')[0].split('/')[0]
if root_directory != basename(directory_name):
shprint(sh.mv, root_directory, directory_name)
else:
raise Exception(
'Could not extract {} download, it must be .zip, '
'.tar.gz or .tar.bz2 or .tar.xz'.format(
extraction_filename))
elif isdir(extraction_filename):
mkdir(directory_name)
for entry in listdir(extraction_filename):
if entry not in ('.git', ):
shprint(sh.cp, '-Rv',
join(extraction_filename, entry),
directory_name)
else:
raise Exception(
'Given path is neither a file nor a directory: {}'.
format(extraction_filename))
else:
info('{} is already unpacked, skipping'.format(self.name))
def prepare_dist_dir(self):
ensure_dir(self.dist_dir)
def run_distribute(self):
info_main(
'# Creating Android project from build and {} bootstrap'.format(
self.name))
# src_path = join(self.ctx.root_dir, 'bootstrap_templates',
# self.name)
src_path = join(self.bootstrap_dir, 'build')
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError(
'built for more than one arch, but bootstrap cannot handle that yet'
)
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Creating initial layout')
for dirname in ('assets', 'bin', 'private', 'res', 'templates'):
if not exists(dirname):
shprint(sh.mkdir, dirname)
info('Copying default files')
shprint(sh.cp, '-a', join(self.build_dir, 'project.properties'),
'.')
shprint(sh.cp, '-a', join(src_path, 'build.py'), '.')
shprint(sh.cp, '-a', join(src_path, 'buildlib'), '.')
shprint(sh.cp, '-a', join(src_path, 'src'), '.')
shprint(sh.cp, '-a', join(src_path, 'templates'), '.')
shprint(sh.cp, '-a', join(src_path, 'res'), '.')
shprint(sh.cp, '-a', join(src_path, 'blacklist.txt'), '.')
shprint(sh.cp, '-a', join(src_path, 'whitelist.txt'), '.')
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
info('Copying python distribution')
python_bundle_dir = join('_python_bundle', '_python_bundle')
if 'python2legacy' in self.ctx.recipe_build_order:
# a special case with its own packaging location
python_bundle_dir = 'private'
# And also must had an install directory, make sure of that
self.ctx.python_recipe.create_python_install(self.dist_dir)
self.distribute_libs(arch, [
join(self.build_dir, 'libs', arch.arch),
self.ctx.get_libs_dir(arch.arch)
])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
ensure_dir(python_bundle_dir)
site_packages_dir = self.ctx.python_recipe.create_python_bundle(
join(self.dist_dir, python_bundle_dir), arch)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(PygameBootstrap, self).run_distribute()
def unpack(self, arch):
info_main('Unpacking {} for {}'.format(self.name, arch))
build_dir = self.get_build_container_dir(arch)
user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
if user_dir is not None:
info('P4A_{}_DIR exists, symlinking instead'.format(
self.name.lower()))
# AND: Currently there's something wrong if I use ln, fix this
warning('Using cp -a instead of symlink...fix this!')
if exists(self.get_build_dir(arch)):
return
shprint(sh.rm, '-rf', build_dir)
shprint(sh.mkdir, '-p', build_dir)
shprint(sh.rmdir, build_dir)
ensure_dir(build_dir)
shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
return
if self.url is None:
info('Skipping {} unpack as no URL is set'.format(self.name))
return
filename = shprint(sh.basename,
self.versioned_url).stdout[:-1].decode('utf-8')
with current_directory(build_dir):
directory_name = self.get_build_dir(arch)
# AND: Could use tito's get_archive_rootdir here
if not exists(directory_name) or not isdir(directory_name):
extraction_filename = join(self.ctx.packages_path, self.name,
filename)
if isfile(extraction_filename):
if extraction_filename.endswith('.tar.gz') or \
extraction_filename.endswith('.tgz'):
sh.tar('xzf', extraction_filename)
root_directory = shprint(
sh.tar, 'tzf', extraction_filename).stdout.decode(
'utf-8').split('\n')[0].split('/')[0]
if root_directory != directory_name:
shprint(sh.mv, root_directory, directory_name)
elif (extraction_filename.endswith('.tar.bz2')
or extraction_filename.endswith('.tbz2')):
info('Extracting {} at {}'.format(
extraction_filename, filename))
sh.tar('xjf', extraction_filename)
root_directory = sh.tar(
'tjf', extraction_filename).stdout.decode(
'utf-8').split('\n')[0].split('/')[0]
if root_directory != directory_name:
shprint(sh.mv, root_directory, directory_name)
elif extraction_filename.endswith('.zip'):
sh.unzip(extraction_filename)
import zipfile
fileh = zipfile.ZipFile(extraction_filename, 'r')
root_directory = fileh.filelist[0].filename.split(
'/')[0]
if root_directory != directory_name:
shprint(sh.mv, root_directory, directory_name)
else:
raise Exception(
'Could not extract {} download, it must be .zip, '
'.tar.gz or .tar.bz2')
elif isdir(extraction_filename):
mkdir(directory_name)
for entry in listdir(extraction_filename):
if entry not in ('.git', ):
shprint(sh.cp, '-Rv',
join(extraction_filename, entry),
directory_name)
else:
raise Exception(
'Given path is neither a file nor a directory: {}'.
format(extraction_filename))
else:
info('{} is already unpacked, skipping'.format(self.name))
def python_installs_dir(self):
dir = join(self.build_dir, 'python-installs')
ensure_dir(dir)
return dir
def prepare_dist_dir(self, name):
# self.dist_dir = self.get_dist_dir(name)
ensure_dir(self.dist_dir)
def build_recipes(build_order, python_modules, ctx):
# Put recipes in correct build order
bs = ctx.bootstrap
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
if ctx.P4A_force_build and not recipe.force_build:
info_main(
'recipe {0} not marked for force build, skip download'.format(
recipe.name))
continue
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
if ctx.P4A_force_build and not recipe.force_build:
info_main('recipe {0} not marked for force build, skip unpack'.
format(recipe.name))
continue
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.force_build or recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'.format(
recipe.name))
# 4) biglink everything
# AND: Should make this optional
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx, python_modules)
return
