def recipes(self, args):
ctx = self.ctx
if args.compact:
print(" ".join(set(Recipe.list_recipes(ctx))))
else:
for name in sorted(Recipe.list_recipes(ctx)):
try:
recipe = Recipe.get_recipe(name, ctx)
except IOError:
warning('Recipe "{}" could not be loaded'.format(name))
except SyntaxError:
import traceback
traceback.print_exc()
warning(('Recipe "{}" could not be loaded due to a '
'syntax error').format(name))
version = str(recipe.version)
print('{Fore.BLUE}{Style.BRIGHT}{recipe.name:<12} '
'{Style.RESET_ALL}{Fore.LIGHTBLUE_EX}'
'{version:<8}{Style.RESET_ALL}'.format(
recipe=recipe, Fore=Out_Fore, Style=Out_Style,
version=version))
print(' {Fore.GREEN}depends: {recipe.depends}'
'{Fore.RESET}'.format(recipe=recipe, Fore=Out_Fore))
if recipe.conflicts:
print(' {Fore.RED}conflicts: {recipe.conflicts}'
'{Fore.RESET}'
.format(recipe=recipe, Fore=Out_Fore))
if recipe.opt_depends:
print(' {Fore.YELLOW}optional depends: '
'{recipe.opt_depends}{Fore.RESET}'
.format(recipe=recipe, Fore=Out_Fore))
def set_libs_flags(self, env, arch):
'''Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
'''
if 'libffi' in self.ctx.recipe_build_order:
info('Activating flags for libffi')
recipe = Recipe.get_recipe('libffi', self.ctx)
include = ' -I' + ' -I'.join(recipe.get_include_dirs(arch))
ldflag = ' -L' + join(recipe.get_build_dir(arch.arch),
recipe.get_host(arch), '.libs') + ' -lffi'
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include
env['LDFLAGS'] = env.get('LDFLAGS', '') + ldflag
if 'openssl' in self.ctx.recipe_build_order:
recipe = Recipe.get_recipe('openssl', self.ctx)
openssl_build_dir = recipe.get_build_dir(arch.arch)
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal)
shprint(sh.sed, '-i.backup', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal)
env['OPENSSL_VERSION'] = recipe.version
if 'sqlite3' in self.ctx.recipe_build_order:
# Include sqlite3 in python2 build
recipe = Recipe.get_recipe('sqlite3', self.ctx)
include = ' -I' + recipe.get_build_dir(arch.arch)
lib = ' -L' + recipe.get_lib_dir(arch) + ' -lsqlite3'
# Insert or append to env
flag = 'CPPFLAGS'
env[flag] = env[flag] + include if flag in env else include
flag = 'LDFLAGS'
env[flag] = env[flag] + lib if flag in env else lib
return env
def get_bootstrap_from_recipes(cls, recipes, ctx):
'''Returns a bootstrap whose recipe requirements do not conflict with
the given recipes.'''
info('Trying to find a bootstrap that matches the given recipes.')
bootstraps = [cls.get_bootstrap(name, ctx)
for name in cls.list_bootstraps()]
acceptable_bootstraps = []
for bs in bootstraps:
if not bs.can_be_chosen_automatically:
continue
possible_dependency_lists = expand_dependencies(bs.recipe_depends)
for possible_dependencies in possible_dependency_lists:
ok = True
for recipe in possible_dependencies:
recipe = Recipe.get_recipe(recipe, ctx)
if any([conflict in recipes for conflict in recipe.conflicts]):
ok = False
break
for recipe in recipes:
recipe = Recipe.get_recipe(recipe, ctx)
if any([conflict in possible_dependencies
for conflict in recipe.conflicts]):
ok = False
break
if ok:
acceptable_bootstraps.append(bs)
info('Found {} acceptable bootstraps: {}'.format(
len(acceptable_bootstraps),
[bs.name for bs in acceptable_bootstraps]))
if acceptable_bootstraps:
info('Using the first of these: {}'
.format(acceptable_bootstraps[0].name))
return acceptable_bootstraps[0]
return None
def recipes(self, args):
parser = argparse.ArgumentParser(
description="List all the available recipes")
parser.add_argument(
"--compact", action="store_true", default=False,
help="Produce a compact list suitable for scripting")
args = parser.parse_args(args)
ctx = self.ctx
if args.compact:
print(" ".join(set(Recipe.list_recipes(ctx))))
else:
for name in sorted(Recipe.list_recipes(ctx)):
recipe = Recipe.get_recipe(name, ctx)
version = str(recipe.version)
print('{Fore.BLUE}{Style.BRIGHT}{recipe.name:<12} '
'{Style.RESET_ALL}{Fore.LIGHTBLUE_EX}'
'{version:<8}{Style.RESET_ALL}'.format(
recipe=recipe, Fore=Out_Fore, Style=Out_Style,
version=version))
print(' {Fore.GREEN}depends: {recipe.depends}'
'{Fore.RESET}'.format(recipe=recipe, Fore=Out_Fore))
if recipe.conflicts:
print(' {Fore.RED}conflicts: {recipe.conflicts}'
'{Fore.RESET}'
.format(recipe=recipe, Fore=Out_Fore))
if recipe.opt_depends:
print(' {Fore.YELLOW}optional depends: '
'{recipe.opt_depends}{Fore.RESET}'
.format(recipe=recipe, Fore=Out_Fore))
def set_libs_flags(self, env, arch):
'''Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
'''
def add_flags(include_flags, link_dirs, link_libs):
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
env['LIBS'] = env.get('LIBS', '') + link_libs
if 'sqlite3' in self.ctx.recipe_build_order:
info('Activating flags for sqlite3')
recipe = Recipe.get_recipe('sqlite3', self.ctx)
add_flags(' -I' + recipe.get_build_dir(arch.arch),
' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
if 'libffi' in self.ctx.recipe_build_order:
info('Activating flags for libffi')
recipe = Recipe.get_recipe('libffi', self.ctx)
# In order to force the correct linkage for our libffi library, we
# set the following variable to point where is our libffi.pc file,
# because the python build system uses pkg-config to configure it.
env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)
add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
' -lffi')
if 'openssl' in self.ctx.recipe_build_order:
info('Activating flags for openssl')
recipe = Recipe.get_recipe('openssl', self.ctx)
add_flags(recipe.include_flags(arch),
recipe.link_dirs_flags(arch), recipe.link_libs_flags())
return env
def test_get_recipe(self):
"""
Makes sure `get_recipe()` returns a `Recipe` object when possible.
"""
ctx = Context()
recipe_name = 'python3'
recipe = Recipe.get_recipe(recipe_name, ctx)
self.assertTrue(isinstance(recipe, Recipe))
self.assertEqual(recipe.name, recipe_name)
recipe_name = 'does_not_exist'
with self.assertRaises(ValueError) as e:
Recipe.get_recipe(recipe_name, ctx)
self.assertEqual(
e.exception.args[0], 'Recipe does not exist: {}'.format(recipe_name))
def has_package(self, name, arch=None):
# If this is a file path, it'll need special handling:
if (name.find("/") >= 0 or name.find("\\") >= 0) and \
name.find("://") < 0: # (:// would indicate an url)
if not os.path.exists(name):
# Non-existing dir, cannot look this up.
return False
try:
name = get_package_name(os.path.abspath(name))
except ValueError:
# Failed to look up any meaningful name.
return False
# Try to look up recipe by name:
try:
recipe = Recipe.get_recipe(name, self)
except ValueError:
pass
else:
name = getattr(recipe, 'site_packages_name', None) or name
name = name.replace('.', '/')
site_packages_dir = self.get_site_packages_dir(arch)
return (exists(join(site_packages_dir, name)) or
exists(join(site_packages_dir, name + '.py')) or
exists(join(site_packages_dir, name + '.pyc')) or
exists(join(site_packages_dir, name + '.pyo')) or
exists(join(site_packages_dir, name + '.so')) or
glob.glob(join(site_packages_dir, name + '-*.egg')))
def build_recipes(build_order, python_modules, ctx):
# Put recipes in correct build order
bs = ctx.bootstrap
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'
.format(recipe.name))
# 4) biglink everything
# AND: Should make this optional
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx, python_modules)
return
def set_libs_flags(self, env, arch):
env = super(Python3Recipe, self).set_libs_flags(env, arch)
if 'openssl' in self.ctx.recipe_build_order:
recipe = Recipe.get_recipe('openssl', self.ctx)
self.configure_args += \
('--with-openssl=' + recipe.get_build_dir(arch.arch),)
return env
def build_arch(self, arch):
super(LibxsltRecipe, self).build_arch(arch)
env = self.get_recipe_env(arch)
build_dir = self.get_build_dir(arch.arch)
with current_directory(build_dir):
# If the build is done with /bin/sh things blow up,
# try really hard to use bash
libxml2_recipe = Recipe.get_recipe('libxml2', self.ctx)
libxml2_build_dir = libxml2_recipe.get_build_dir(arch.arch)
build_arch = shprint(sh.gcc, '-dumpmachine').stdout.decode(
'utf-8').split('\n')[0]
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('autoreconf'), '-vif', _env=env)
shprint(sh.Command('./configure'),
'--build=' + build_arch,
'--host=' + arch.command_prefix,
'--target=' + arch.command_prefix,
'--without-plugins',
'--without-debug',
'--without-python',
'--without-crypto',
'--with-libxml-src=' + libxml2_build_dir,
'--disable-shared',
_env=env)
shprint(sh.make, "V=1", _env=env)
shutil.copyfile('libxslt/.libs/libxslt.a',
join(self.ctx.libs_dir, 'libxslt.a'))
shutil.copyfile('libexslt/.libs/libexslt.a',
join(self.ctx.libs_dir, 'libexslt.a'))
def get_env(self):
env = {}
env["CFLAGS"] = " ".join([
"-DANDROID", "-mandroid", "-fomit-frame-pointer",
"--sysroot", self.ctx.ndk_platform])
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] = " ".join(['-lm'])
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
print('path is', environ['PATH'])
cc = find_executable('{command_prefix}-gcc'.format(
command_prefix=command_prefix), path=environ['PATH'])
if cc is None:
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
env['CC'] = '{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
cflags=env['CFLAGS'])
env['CXX'] = '{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
cxxflags=env['CXXFLAGS'])
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# AND: This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
return env
def setUp(self):
"""
Setups recipe and context.
"""
self.context = Context()
self.context.ndk_api = 21
self.context.android_api = 27
self.arch = ArchARMv7_a(self.context)
self.recipe = Recipe.get_recipe('gevent', self.context)
def test_list_recipes(self):
"""
Trivial test verifying list_recipes returns a generator with some recipes.
"""
ctx = Context()
recipes = Recipe.list_recipes(ctx)
self.assertTrue(isinstance(recipes, types.GeneratorType))
recipes = list(recipes)
self.assertIn('python3', recipes)
def get_recipe_env(self, arch):
env = super(CryptographyRecipe, self).get_recipe_env(arch)
openssl_recipe = Recipe.get_recipe('openssl', self.ctx)
env['CFLAGS'] += openssl_recipe.include_flags(arch)
env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch)
env['LIBS'] = openssl_recipe.link_libs_flags()
return env
def test_recipe_dirs(self):
"""
Trivial `recipe_dirs()` test.
Makes sure the list is not empty and has the root directory.
"""
ctx = Context()
recipes_dir = Recipe.recipe_dirs(ctx)
# by default only the root dir `recipes` directory
self.assertEqual(len(recipes_dir), 1)
self.assertTrue(recipes_dir[0].startswith(ctx.root_dir))
def conflicts(self, name):
for name in self.keys():
try:
recipe = Recipe.get_recipe(name, self.ctx)
conflicts = recipe.conflicts
except IOError:
conflicts = []
if any([c in self for c in conflicts]):
return True
return False
def conflicts(self):
for name in self.keys():
try:
recipe = Recipe.get_recipe(name, self.ctx)
conflicts = [dep.lower() for dep in recipe.conflicts]
except ValueError:
conflicts = []
if any([c in self for c in conflicts]):
return True
return False
def get_recipe_env(self, arch=None):
env = super(PyCryptoRecipe, self).get_recipe_env(arch)
openssl_build_dir = Recipe.get_recipe('openssl', self.ctx).get_build_dir(arch.arch)
env['CC'] = '%s -I%s' % (env['CC'], join(openssl_build_dir, 'include'))
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
self.ctx.get_libs_dir(arch.arch) +
'-L{}'.format(self.ctx.libs_dir)) + ' -L{}'.format(
openssl_build_dir)
env['EXTRA_CFLAGS'] = '--host linux-armv'
env['ac_cv_func_malloc_0_nonnull'] = 'yes'
return env
def clean_recipe_build(self, args):
'''Deletes the build files of the given recipe.
This is intended for debug purposes, you may experience
strange behaviour or problems with some recipes (if their
build has done unexpected state changes). If this happens, run
clean_builds, or attempt to clean other recipes until things
work again.
'''
recipe = Recipe.get_recipe(args.recipe, self.ctx)
info('Cleaning build for {} recipe.'.format(recipe.name))
recipe.clean_build()
def recursively_collect_orders(name, ctx, all_inputs, orders=[]):
'''For each possible recipe ordering, try to add the new recipe name
to that order. Recursively do the same thing with all the
dependencies of each recipe.
'''
try:
recipe = Recipe.get_recipe(name, ctx)
if recipe.depends is None:
dependencies = []
else:
# make all dependencies into lists so that product will work
dependencies = [([dependency] if not isinstance(
dependency, (list, tuple))
else dependency) for dependency in recipe.depends]
# handle opt_depends: these impose requirements on the build
# order only if already present in the list of recipes to build
dependencies.extend([[d] for d in recipe.get_opt_depends_in_list(all_inputs)])
if recipe.conflicts is None:
conflicts = []
else:
conflicts = recipe.conflicts
except IOError:
# The recipe does not exist, so we assume it can be installed
# via pip with no extra dependencies
dependencies = []
conflicts = []
new_orders = []
# for each existing recipe order, see if we can add the new recipe name
for order in orders:
if name in order:
new_orders.append(deepcopy(order))
continue
if order.conflicts(name):
continue
if any([conflict in order for conflict in conflicts]):
continue
for dependency_set in product(*dependencies):
new_order = deepcopy(order)
new_order[name] = set(dependency_set)
dependency_new_orders = [new_order]
for dependency in dependency_set:
dependency_new_orders = recursively_collect_orders(
dependency, ctx, all_inputs, dependency_new_orders)
new_orders.extend(dependency_new_orders)
return new_orders
def set_libs_flags(self, env, arch):
env = super(Python2Recipe, self).set_libs_flags(env, arch)
if 'libffi' in self.ctx.recipe_build_order:
# For python2 we need to tell configure that we want to use our
# compiled libffi, this step is not necessary for python3.
self.configure_args += ('--with-system-ffi',)
if 'openssl' in self.ctx.recipe_build_order:
recipe = Recipe.get_recipe('openssl', self.ctx)
openssl_build = recipe.get_build_dir(arch.arch)
env['OPENSSL_BUILD'] = openssl_build
env['OPENSSL_VERSION'] = recipe.version
return env
def get_recipe_env(self, arch=None, clang=True):
env = super(PyCryptoRecipe, self).get_recipe_env(arch)
openssl_recipe = Recipe.get_recipe('openssl', self.ctx)
env['CC'] = env['CC'] + openssl_recipe.include_flags(arch)
env['LDFLAGS'] += ' -L{}'.format(self.ctx.get_libs_dir(arch.arch))
env['LDFLAGS'] += ' -L{}'.format(self.ctx.libs_dir)
env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch)
env['LIBS'] = env.get('LIBS', '') + openssl_recipe.link_libs_flags()
env['EXTRA_CFLAGS'] = '--host linux-armv'
env['ac_cv_func_malloc_0_nonnull'] = 'yes'
return env
def set_libs_flags(self, env, arch):
'''Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
'''
if 'libffi' in self.ctx.recipe_build_order:
info('Activating flags for libffi')
recipe = Recipe.get_recipe('libffi', self.ctx)
include = ' -I' + ' -I'.join(recipe.get_include_dirs(arch))
ldflag = ' -L' + join(recipe.get_build_dir(arch.arch),
recipe.get_host(arch), '.libs') + ' -lffi'
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include
env['LDFLAGS'] = env.get('LDFLAGS', '') + ldflag
return env
def has_package(self, name, arch=None):
try:
recipe = Recipe.get_recipe(name, self)
except IOError:
pass
else:
name = getattr(recipe, 'site_packages_name', None) or name
name = name.replace('.', '/')
site_packages_dir = self.get_site_packages_dir(arch)
return (exists(join(site_packages_dir, name)) or
exists(join(site_packages_dir, name + '.py')) or
exists(join(site_packages_dir, name + '.pyc')) or
exists(join(site_packages_dir, name + '.pyo')) or
exists(join(site_packages_dir, name + '.so')))
def remove_remaining_conflicts(self, ctx):
# It's unpleasant to have to pass ctx as an argument...
'''Checks all possible graphs for conflicts that have arisen during
the additon of alternative repice branches, as these are not checked
for conflicts at the time.'''
new_graphs = []
for i, graph in enumerate(self.graphs):
for name in graph.keys():
recipe = Recipe.get_recipe(name, ctx)
if any([c in graph for c in recipe.conflicts]):
break
else:
new_graphs.append(graph)
self.graphs = new_graphs
def recipes(self, args):
ctx = self.ctx
if args.compact:
print(" ".join(set(Recipe.list_recipes(ctx))))
else:
for name in sorted(Recipe.list_recipes(ctx)):
recipe = Recipe.get_recipe(name, ctx)
version = str(recipe.version)
print('{Fore.BLUE}{Style.BRIGHT}{recipe.name:<12} '
'{Style.RESET_ALL}{Fore.LIGHTBLUE_EX}'
'{version:<8}{Style.RESET_ALL}'.format(
recipe=recipe, Fore=Out_Fore, Style=Out_Style,
version=version))
print(' {Fore.GREEN}depends: {recipe.depends}'
'{Fore.RESET}'.format(recipe=recipe, Fore=Out_Fore))
if recipe.conflicts:
print(' {Fore.RED}conflicts: {recipe.conflicts}'
'{Fore.RESET}'
.format(recipe=recipe, Fore=Out_Fore))
if recipe.opt_depends:
print(' {Fore.YELLOW}optional depends: '
'{recipe.opt_depends}{Fore.RESET}'
.format(recipe=recipe, Fore=Out_Fore))
def clean_recipe_build(self, args):
'''Deletes the build files of the given recipe.
This is intended for debug purposes, you may experience
strange behaviour or problems with some recipes (if their
build has done unexpected state changes). If this happens, run
clean_builds, or attempt to clean other recipes until things
work again.
'''
parser = argparse.ArgumentParser(
description="Delete all build files for the given recipe name.")
parser.add_argument('recipe', help='The recipe name')
args = parser.parse_args(args)
recipe = Recipe.get_recipe(args.recipe, self.ctx)
info('Cleaning build for {} recipe.'.format(recipe.name))
recipe.clean_build()
def setUp(self):
"""
Setups recipe and context.
"""
self.context = Context()
self.arch = ArchARMv7_a(self.context)
self.recipe = Recipe.get_recipe('reportlab', self.context)
self.recipe.ctx = self.context
self.bootstrap = None
recipe_build_order, python_modules, bootstrap = \
get_recipe_order_and_bootstrap(
self.context, [self.recipe.name], self.bootstrap)
self.context.recipe_build_order = recipe_build_order
self.context.python_modules = python_modules
self.context.setup_dirs(tempfile.gettempdir())
self.bootstrap = bootstrap
self.recipe_dir = self.recipe.get_build_dir(self.arch.arch)
ensure_dir(self.recipe_dir)
def biglink(ctx, arch):
# First, collate object files from each recipe
info('Collating object files from each recipe')
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
'objects_{}'.format(recipe.name))
if not exists(recipe_obj_dir):
info('{} recipe has no biglinkable files dir, skipping'
.format(recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, '*'))
if not len(files):
info('{} recipe has no biglinkable files, skipping'
.format(recipe.name))
continue
info('{} recipe has object files, copying'.format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, '-r', *files)
env = arch.get_env()
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
if not len(glob.glob(join(obj_dir, '*'))):
info('There seem to be no libraries to biglink, skipping.')
return
info('Biglinking')
info('target {}'.format(join(ctx.get_libs_dir(arch.arch),
'libpymodules.so')))
do_biglink = copylibs_function if ctx.copy_libs else biglink_function
# Move to the directory containing crtstart_so.o and crtend_so.o
# This is necessary with newer NDKs? A gcc bug?
with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):
do_biglink(
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
obj_dir.split(' '),
extra_link_dirs=[join(ctx.bootstrap.build_dir,
'obj', 'local', arch.arch),
os.path.abspath('.')],
env=env)
def get_recipe_env(self, arch):
env = super(LibxsltRecipe, self).get_recipe_env(arch)
env['CONFIG_SHELL'] = '/bin/bash'
env['SHELL'] = '/bin/bash'
libxml2_recipe = Recipe.get_recipe('libxml2', self.ctx)
libxml2_build_dir = libxml2_recipe.get_build_dir(arch.arch)
libxml2_libs_dir = join(libxml2_build_dir, '.libs')
env['CFLAGS'] = ' '.join([
env['CFLAGS'],
'-I' + libxml2_build_dir,
'-I' + join(libxml2_build_dir, 'include', 'libxml'),
'-I' + self.get_build_dir(arch.arch),
])
env['LDFLAGS'] += ' -L' + libxml2_libs_dir
env['LIBS'] = '-lxml2 -lz -lm'
return env
def biglink(ctx, arch):
# First, collate object files from each recipe
info('Collating object files from each recipe')
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
'objects_{}'.format(recipe.name))
if not exists(recipe_obj_dir):
info('{} recipe has no biglinkable files dir, skipping'.format(
recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, '*'))
if not len(files):
info('{} recipe has no biglinkable files, skipping'.format(
recipe.name))
info('{} recipe has object files, copying'.format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, '-r', *files)
env = arch.get_env()
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
if not len(glob.glob(join(obj_dir, '*'))):
info('There seem to be no libraries to biglink, skipping.')
return
info('Biglinking')
info('target {}'.format(
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so')))
biglink_function(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
obj_dir.split(' '),
extra_link_dirs=[
join(ctx.bootstrap.build_dir, 'obj', 'local',
arch.arch)
],
env=env)
def has_package(self, name, arch=None):
# If this is a file path, it'll need special handling:
if (name.find("/") >= 0 or name.find("\\") >= 0) and \
name.find("://") < 0: # (:// would indicate an url)
if not os.path.exists(name):
# Non-existing dir, cannot look this up.
return False
if os.path.exists(os.path.join(name, "setup.py")):
# Get name from setup.py:
name = subprocess.check_output(
[sys.executable, "setup.py", "--name"], cwd=name)
try:
name = name.decode('utf-8', 'replace')
except AttributeError:
pass
name = name.strip()
if len(name) == 0:
# Failed to look up any meaningful name.
return False
else:
# A folder with whatever, cannot look this up.
return False
# Try to look up recipe by name:
try:
recipe = Recipe.get_recipe(name, self)
except IOError:
pass
else:
name = getattr(recipe, 'site_packages_name', None) or name
name = name.replace('.', '/')
site_packages_dir = self.get_site_packages_dir(arch)
return (exists(join(site_packages_dir, name))
or exists(join(site_packages_dir, name + '.py'))
or exists(join(site_packages_dir, name + '.pyc'))
or exists(join(site_packages_dir, name + '.pyo'))
or exists(join(site_packages_dir, name + '.so'))
or glob.glob(join(site_packages_dir, name + '-*.egg')))
def expand_dependencies(recipes, ctx):
""" This function expands to lists of all different available
alternative recipe combinations, with the dependencies added in
ONLY for all the not-with-alternative recipes.
(So this is like the deps graph very simplified and incomplete, but
hopefully good enough for most basic bootstrap compatibility checks)
"""
# Add in all the deps of recipes where there is no alternative:
recipes_with_deps = list(recipes)
for entry in recipes:
if not isinstance(entry, (tuple, list)) or len(entry) == 1:
if isinstance(entry, (tuple, list)):
entry = entry[0]
try:
recipe = Recipe.get_recipe(entry, ctx)
recipes_with_deps += recipe.depends
except ValueError:
# it's a pure python package without a recipe, so we
# don't know the dependencies...skipping for now
pass
# Split up lists by available alternatives:
recipe_lists = [[]]
for recipe in recipes_with_deps:
if isinstance(recipe, (tuple, list)):
new_recipe_lists = []
for alternative in recipe:
for old_list in recipe_lists:
new_list = [i for i in old_list]
new_list.append(alternative)
new_recipe_lists.append(new_list)
recipe_lists = new_recipe_lists
else:
for existing_list in recipe_lists:
existing_list.append(recipe)
return recipe_lists
def build_cython_components(self, arch):
libzmq_recipe = Recipe.get_recipe('libzmq', self.ctx)
libzmq_prefix = join(libzmq_recipe.get_build_dir(arch.arch), "install")
self.setup_extra_args = ["--zmq={}".format(libzmq_prefix)]
self.build_cmd = "configure"
env = self.get_recipe_env(arch)
setup_cfg = join(self.get_build_dir(arch.arch), "setup.cfg")
with open(setup_cfg, "wb") as fd:
fd.write("""
[global]
zmq_prefix = {}
skip_check_zmq = True
""".format(libzmq_prefix).encode())
return super().build_cython_components(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.hostpython_location)
shprint(hostpython, 'setup.py', 'configure', '-v', _env=env)
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env)
build_dir = glob.glob('build/lib.*')[0]
shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
env['STRIP'], '{}', ';', _env=env)
def test_get_recipe_env(
self,
mock_find_executable,
mock_glob,
mock_ensure_dir,
mock_check_recipe_choices,
):
"""
Test that method
:meth:`~pythonforandroid.recipes.pyicu.PyICURecipe.get_recipe_env`
returns the expected flags
"""
icu_recipe = Recipe.get_recipe("icu", self.ctx)
mock_find_executable.return_value = (
"/opt/android/android-ndk/toolchains/"
"llvm/prebuilt/linux-x86_64/bin/clang"
)
mock_glob.return_value = ["llvm"]
mock_check_recipe_choices.return_value = sorted(
self.ctx.recipe_build_order
)
expected_pyicu_libs = [
lib[3:-3] for lib in icu_recipe.built_libraries.keys()
]
env = self.recipe.get_recipe_env(self.arch)
self.assertEqual(":".join(expected_pyicu_libs), env["PYICU_LIBRARIES"])
self.assertIn("include/icu", env["CPPFLAGS"])
self.assertIn("icu4c/icu_build/lib", env["LDFLAGS"])
# make sure that the mocked methods are actually called
mock_glob.assert_called()
mock_ensure_dir.assert_called()
mock_find_executable.assert_called()
mock_check_recipe_choices.assert_called()
def recursively_collect_orders(name,
ctx,
all_inputs,
orders=None,
blacklist=None):
'''For each possible recipe ordering, try to add the new recipe name
to that order. Recursively do the same thing with all the
dependencies of each recipe.
'''
name = name.lower()
if orders is None:
orders = []
if blacklist is None:
blacklist = set()
try:
recipe = Recipe.get_recipe(name, ctx)
dependencies = get_dependency_tuple_list_for_recipe(
recipe, blacklist=blacklist)
# handle opt_depends: these impose requirements on the build
# order only if already present in the list of recipes to build
dependencies.extend(
fix_deplist([[d]
for d in recipe.get_opt_depends_in_list(all_inputs)
if d.lower() not in blacklist]))
if recipe.conflicts is None:
conflicts = []
else:
conflicts = [dep.lower() for dep in recipe.conflicts]
except ValueError:
# The recipe does not exist, so we assume it can be installed
# via pip with no extra dependencies
dependencies = []
conflicts = []
new_orders = []
# for each existing recipe order, see if we can add the new recipe name
for order in orders:
if name in order:
new_orders.append(deepcopy(order))
continue
if order.conflicts():
continue
if any([conflict in order for conflict in conflicts]):
continue
for dependency_set in product(*dependencies):
new_order = deepcopy(order)
new_order[name] = set(dependency_set)
dependency_new_orders = [new_order]
for dependency in dependency_set:
dependency_new_orders = recursively_collect_orders(
dependency,
ctx,
all_inputs,
dependency_new_orders,
blacklist=blacklist)
new_orders.extend(dependency_new_orders)
return new_orders
def set_libs_flags(self, env, arch):
"""Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
"""
def add_flags(include_flags, link_dirs, link_libs):
env["CPPFLAGS"] = env.get("CPPFLAGS", "") + include_flags
env["LDFLAGS"] = env.get("LDFLAGS", "") + link_dirs
env["LIBS"] = env.get("LIBS", "") + link_libs
if "sqlite3" in self.ctx.recipe_build_order:
info("Activating flags for sqlite3")
recipe = Recipe.get_recipe("sqlite3", self.ctx)
add_flags(
" -I" + recipe.get_build_dir(arch.arch),
" -L" + recipe.get_lib_dir(arch),
" -lsqlite3",
)
if "libffi" in self.ctx.recipe_build_order:
info("Activating flags for libffi")
recipe = Recipe.get_recipe("libffi", self.ctx)
# In order to force the correct linkage for our libffi library, we
# set the following variable to point where is our libffi.pc file,
# because the python build system uses pkg-config to configure it.
env["PKG_CONFIG_PATH"] = recipe.get_build_dir(arch.arch)
add_flags(
" -I" + " -I".join(recipe.get_include_dirs(arch)),
" -L" + join(recipe.get_build_dir(arch.arch), ".libs"),
" -lffi",
)
if "openssl" in self.ctx.recipe_build_order:
info("Activating flags for openssl")
recipe = Recipe.get_recipe("openssl", self.ctx)
self.configure_args += ("--with-openssl=" +
recipe.get_build_dir(arch.arch), )
add_flags(
recipe.include_flags(arch),
recipe.link_dirs_flags(arch),
recipe.link_libs_flags(),
)
for library_name in {"libbz2", "liblzma"}:
if library_name in self.ctx.recipe_build_order:
info(f"Activating flags for {library_name}")
recipe = Recipe.get_recipe(library_name, self.ctx)
add_flags(
recipe.get_library_includes(arch),
recipe.get_library_ldflags(arch),
recipe.get_library_libs_flag(),
)
# python build system contains hardcoded zlib version which prevents
# the build of zlib module, here we search for android's zlib version
# and sets the right flags, so python can be build with android's zlib
info("Activating flags for android's zlib")
zlib_lib_path = join(self.ctx.ndk_platform, "usr", "lib")
zlib_includes = join(self.ctx.ndk_dir, "sysroot", "usr", "include")
zlib_h = join(zlib_includes, "zlib.h")
try:
with open(zlib_h) as fileh:
zlib_data = fileh.read()
except IOError:
raise BuildInterruptingException(
"Could not determine android's zlib version, no zlib.h ({}) in"
" the NDK dir includes".format(zlib_h))
for line in zlib_data.split("\n"):
if line.startswith("#define ZLIB_VERSION "):
break
else:
raise BuildInterruptingException(
"Could not parse zlib.h...so we cannot find zlib version,"
"required by python build,")
env["ZLIB_VERSION"] = line.replace("#define ZLIB_VERSION ", "")
add_flags(" -I" + zlib_includes, " -L" + zlib_lib_path, " -lz")
return env
def get_env(self):
env = {}
env["CFLAGS"] = " ".join([
"-DANDROID", "-mandroid", "-fomit-frame-pointer", "--sysroot",
self.ctx.ndk_platform
])
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] = " ".join(['-lm'])
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
print('path is', environ['PATH'])
cc = find_executable(
'{command_prefix}-gcc'.format(command_prefix=command_prefix),
path=environ['PATH'])
if cc is None:
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix, ccache=ccache, cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# AND: This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(hostpython_recipe.get_build_dir(self.arch),
'build',
'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
return env
def obvious_conflict_checker(ctx, name_tuples, blacklist=None):
""" This is a pre-flight check function that will completely ignore
recipe order or choosing an actual value in any of the multiple
choice tuples/dependencies, and just do a very basic obvious
conflict check.
"""
deps_were_added_by = dict()
deps = set()
if blacklist is None:
blacklist = set()
# Add dependencies for all recipes:
to_be_added = [(name_tuple, None) for name_tuple in name_tuples]
while len(to_be_added) > 0:
current_to_be_added = list(to_be_added)
to_be_added = []
for (added_tuple, adding_recipe) in current_to_be_added:
assert (type(added_tuple) == tuple)
if len(added_tuple) > 1:
# No obvious commitment in what to add, don't check it itself
# but throw it into deps for later comparing against
# (Remember this function only catches obvious issues)
deps.add(added_tuple)
continue
name = added_tuple[0]
recipe_conflicts = set()
recipe_dependencies = []
try:
# Get recipe to add and who's ultimately adding it:
recipe = Recipe.get_recipe(name, ctx)
recipe_conflicts = {c.lower() for c in recipe.conflicts}
recipe_dependencies = get_dependency_tuple_list_for_recipe(
recipe, blacklist=blacklist)
except ValueError:
pass
adder_first_recipe_name = adding_recipe or name
# Collect the conflicts:
triggered_conflicts = []
for dep_tuple_list in deps:
# See if the new deps conflict with things added before:
if set(dep_tuple_list).intersection(recipe_conflicts) == set(
dep_tuple_list):
triggered_conflicts.append(dep_tuple_list)
continue
# See if what was added before conflicts with the new deps:
if len(dep_tuple_list) > 1:
# Not an obvious commitment to a specific recipe/dep
# to be added, so we won't check.
# (remember this function only catches obvious issues)
continue
try:
dep_recipe = Recipe.get_recipe(dep_tuple_list[0], ctx)
except ValueError:
continue
conflicts = [c.lower() for c in dep_recipe.conflicts]
if name in conflicts:
triggered_conflicts.append(dep_tuple_list)
# Throw error on conflict:
if triggered_conflicts:
# Get first conflict and see who added that one:
adder_second_recipe_name = "'||'".join(triggered_conflicts[0])
second_recipe_original_adder = deps_were_added_by.get(
(adder_second_recipe_name, ), None)
if second_recipe_original_adder:
adder_second_recipe_name = second_recipe_original_adder
# Prompt error:
raise BuildInterruptingException(
"Conflict detected: '{}'"
" inducing dependencies {}, and '{}'"
" inducing conflicting dependencies {}".format(
adder_first_recipe_name, (recipe.name, ),
adder_second_recipe_name, triggered_conflicts[0]))
# Actually add it to our list:
deps.add(added_tuple)
deps_were_added_by[added_tuple] = adding_recipe
# Schedule dependencies to be added
to_be_added += [(dep, adder_first_recipe_name or name)
for dep in recipe_dependencies if dep not in deps]
# If we came here, then there were no obvious conflicts.
return None
def get_recipe_order_and_bootstrap(ctx, names, bs=None):
recipes_to_load = set(names)
if bs is not None and bs.recipe_depends:
recipes_to_load = recipes_to_load.union(set(bs.recipe_depends))
possible_orders = []
# get all possible order graphs, as names may include tuples/lists
# of alternative dependencies
names = [([name] if not isinstance(name, (list, tuple)) else name)
for name in names]
for name_set in product(*names):
new_possible_orders = [RecipeOrder(ctx)]
for name in name_set:
new_possible_orders = recursively_collect_orders(
name, ctx, orders=new_possible_orders)
possible_orders.extend(new_possible_orders)
# turn each order graph into a linear list if possible
orders = []
for possible_order in possible_orders:
try:
order = find_order(possible_order)
except ValueError: # a circular dependency was found
info('Circular dependency found in graph {}, skipping it.'.format(
possible_order))
continue
except:
warning('Failed to import recipe named {}; the recipe exists '
'but appears broken.'.format(name))
warning('Exception was:')
raise
orders.append(list(order))
# prefer python2 and SDL2 if available
orders = sorted(orders,
key=lambda order: -('python2' in order) -
('sdl2' in order))
if not orders:
raise BuildInterruptingException(
'Didn\'t find any valid dependency graphs. This means that some of your '
'requirements pull in conflicting dependencies.')
# It would be better to check against possible orders other
# than the first one, but in practice clashes will be rare,
# and can be resolved by specifying more parameters
chosen_order = orders[0]
if len(orders) > 1:
info('Found multiple valid dependency orders:')
for order in orders:
info(' {}'.format(order))
info('Using the first of these: {}'.format(chosen_order))
else:
info('Found a single valid recipe set: {}'.format(chosen_order))
if bs is None:
bs = Bootstrap.get_bootstrap_from_recipes(chosen_order, ctx)
recipes, python_modules, bs = get_recipe_order_and_bootstrap(
ctx, chosen_order, bs=bs)
else:
# check if each requirement has a recipe
recipes = []
python_modules = []
for name in chosen_order:
try:
recipe = Recipe.get_recipe(name, ctx)
python_modules += recipe.python_depends
except IOError:
python_modules.append(name)
else:
recipes.append(name)
python_modules = list(set(python_modules))
return recipes, python_modules, bs
def do_python_build(self, arch):
shprint(sh.cp, self.ctx.hostpython, self.get_build_dir(arch.arch))
shprint(sh.cp, self.ctx.hostpgen,
join(self.get_build_dir(arch.arch), 'Parser'))
hostpython = join(self.get_build_dir(arch.arch), 'hostpython')
hostpgen = join(self.get_build_dir(arch.arch), 'hostpython')
with current_directory(self.get_build_dir(arch.arch)):
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, join(hostpython_recipe.get_recipe_dir(), 'Setup'),
'Modules')
env = arch.get_env()
# AND: Should probably move these to get_recipe_env for
# neatness, but the whole recipe needs tidying along these
# lines
env['HOSTARCH'] = 'arm-linux-androideabi'
env['BUILDARCH'] = shprint(
sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
env['CFLAGS'] = ' '.join([env['CFLAGS'], '-DNO_MALLINFO'])
env['LDFLAGS'] += ' -Wl,--allow-multiple-definition'
# TODO need to add a should_build that checks if optional
# dependencies have changed (possibly in a generic way)
if 'openssl' in self.ctx.recipe_build_order:
r = Recipe.get_recipe('openssl', self.ctx)
openssl_build_dir = r.get_build_dir(arch.arch)
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'),
setuplocal)
#shprint(sh.cat, join(self.get_recipe_dir(), 'Setup.local-ssl'), '>>', setuplocal)
shprint(sh.sed, '-i.backup',
's#^SSL=.*#SSL={}#'.format(openssl_build_dir),
setuplocal)
env['OPENSSL_VERSION'] = r.version
env['CFLAGS'] += ' -I%s' % join(openssl_build_dir, 'include')
env['LDFLAGS'] += ' -L%s' % openssl_build_dir
if 'sqlite3' in self.ctx.recipe_build_order:
# Include sqlite3 in python2 build
r = Recipe.get_recipe('sqlite3', self.ctx)
i = ' -I' + r.get_build_dir(arch.arch)
l = ' -L' + r.get_lib_dir(arch) + ' -lsqlite3'
# Insert or append to env
f = 'CPPFLAGS'
env[f] = env[f] + i if f in env else i
f = 'LDFLAGS'
env[f] = env[f] + l if f in env else l
with open('config.site', 'w') as fileh:
fileh.write('''
ac_cv_file__dev_ptmx=no
ac_cv_file__dev_ptc=no
ac_cv_have_long_long_format=yes
''')
configure = sh.Command('./configure')
# AND: OFLAG isn't actually set, should it be?
shprint(configure,
'CROSS_COMPILE_TARGET=yes',
'CONFIG_SITE=config.site',
'--host={}'.format(env['HOSTARCH']),
'--build={}'.format(env['BUILDARCH']),
'--prefix={}'.format(realpath('./python-install')),
'--enable-shared',
'--enable-ipv6',
'--disable-toolbox-glue',
'--disable-framework',
'--with-system-ffi',
_env=env)
# AND: tito left this comment in the original source. It's still true!
# FIXME, the first time, we got a error at:
# python$EXE ../../Tools/scripts/h2py.py -i '(u_long)' /usr/include/netinet/in.h
# /home/tito/code/python-for-android/build/python/Python-2.7.2/python: 1: Syntax error: word unexpected (expecting ")")
# because at this time, python is arm, not x86. even that, why /usr/include/netinet/in.h is used ?
# check if we can avoid this part.
# Hardcoded, remove -I/usr/include/x86_64-linux-gnu from CCSHARED and CFLAGS
# to prevent overriding android arm sysroot includes
make = sh.Command(env['MAKE'].split(' ')[0])
print('First install (expected to fail...')
try:
shprint(make,
'-j5',
'install',
'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
except sh.ErrorReturnCode_2:
print(
'First python2 make failed. This is expected, trying again.'
)
print('Make compile...')
shprint(make,
'-j5',
'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
print('Second install (expected to work)')
shprint(sh.touch, 'python.exe', 'python')
# -k added to keep going (need to figure out the reason for make: *** [libinstall] Error 1)
shprint(make,
'-j5',
'install',
'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
if is_darwin():
shprint(sh.cp,
join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'Lib'))
shprint(sh.cp,
join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'lib', 'python2.7'))
# reduce python
for dir_name in ('test', join('json', 'tests'), 'lib-tk',
join('sqlite3', 'test'), join('unittest, test'),
join('lib2to3', 'tests'), join('bsddb', 'tests'),
join('distutils',
'tests'), join('email', 'test'), 'curses'):
shprint(sh.rm, '-rf',
join('python-install', 'lib', 'python2.7', dir_name))
def get_recipe_order_and_bootstrap(ctx, names, bs=None, blacklist=None):
# Get set of recipe/dependency names, clean up and add bootstrap deps:
names = set(names)
if bs is not None and bs.recipe_depends:
names = names.union(set(bs.recipe_depends))
names = fix_deplist([([name] if not isinstance(name,
(list, tuple)) else name)
for name in names])
if blacklist is None:
blacklist = set()
blacklist = {bitem.lower() for bitem in blacklist}
# Remove all values that are in the blacklist:
names_before_blacklist = list(names)
names = []
for name in names_before_blacklist:
cleaned_up_tuple = tuple(
[item for item in name if item not in blacklist])
if cleaned_up_tuple:
names.append(cleaned_up_tuple)
# Do check for obvious conflicts (that would trigger in any order, and
# without comitting to any specific choice in a multi-choice tuple of
# dependencies):
obvious_conflict_checker(ctx, names, blacklist=blacklist)
# If we get here, no obvious conflicts!
# get all possible order graphs, as names may include tuples/lists
# of alternative dependencies
possible_orders = []
for name_set in product(*names):
new_possible_orders = [RecipeOrder(ctx)]
for name in name_set:
new_possible_orders = recursively_collect_orders(
name,
ctx,
name_set,
orders=new_possible_orders,
blacklist=blacklist)
possible_orders.extend(new_possible_orders)
# turn each order graph into a linear list if possible
orders = []
for possible_order in possible_orders:
try:
order = find_order(possible_order)
except ValueError: # a circular dependency was found
info('Circular dependency found in graph {}, skipping it.'.format(
possible_order))
continue
orders.append(list(order))
# prefer python3 and SDL2 if available
orders = sorted(orders,
key=lambda order: -('python3' in order) -
('sdl2' in order))
if not orders:
raise BuildInterruptingException(
'Didn\'t find any valid dependency graphs. '
'This means that some of your '
'requirements pull in conflicting dependencies.')
# It would be better to check against possible orders other
# than the first one, but in practice clashes will be rare,
# and can be resolved by specifying more parameters
chosen_order = orders[0]
if len(orders) > 1:
info('Found multiple valid dependency orders:')
for order in orders:
info(' {}'.format(order))
info('Using the first of these: {}'.format(chosen_order))
else:
info('Found a single valid recipe set: {}'.format(chosen_order))
if bs is None:
bs = Bootstrap.get_bootstrap_from_recipes(chosen_order, ctx)
if bs is None:
# Note: don't remove this without thought, causes infinite loop
raise BuildInterruptingException(
"Could not find any compatible bootstrap!")
recipes, python_modules, bs = get_recipe_order_and_bootstrap(
ctx, chosen_order, bs=bs, blacklist=blacklist)
else:
# check if each requirement has a recipe
recipes = []
python_modules = []
for name in chosen_order:
try:
recipe = Recipe.get_recipe(name, ctx)
python_modules += recipe.python_depends
except ValueError:
python_modules.append(name)
else:
recipes.append(name)
python_modules = list(set(python_modules))
return recipes, python_modules, bs
def __init__(self):
argv = sys.argv
self.warn_on_carriage_return_args(argv)
# Buildozer used to pass these arguments in a now-invalid order
# If that happens, apply this fix
# This fix will be removed once a fixed buildozer is released
if (len(argv) > 2
and argv[1].startswith('--color')
and argv[2].startswith('--storage-dir')):
argv.append(argv.pop(1)) # the --color arg
argv.append(argv.pop(1)) # the --storage-dir arg
parser = NoAbbrevParser(
description='A packaging tool for turning Python scripts and apps '
'into Android APKs')
generic_parser = argparse.ArgumentParser(
add_help=False,
description='Generic arguments applied to all commands')
argparse.ArgumentParser(
add_help=False, description='Arguments for dist building')
generic_parser.add_argument(
'--debug', dest='debug', action='store_true', default=False,
help='Display debug output and all build info')
generic_parser.add_argument(
'--color', dest='color', choices=['always', 'never', 'auto'],
help='Enable or disable color output (default enabled on tty)')
generic_parser.add_argument(
'--sdk-dir', '--sdk_dir', dest='sdk_dir', default='',
help='The filepath where the Android SDK is installed')
generic_parser.add_argument(
'--ndk-dir', '--ndk_dir', dest='ndk_dir', default='',
help='The filepath where the Android NDK is installed')
generic_parser.add_argument(
'--android-api',
'--android_api',
dest='android_api',
default=0,
type=int,
help=('The Android API level to build against defaults to {} if '
'not specified.').format(RECOMMENDED_TARGET_API))
generic_parser.add_argument(
'--ndk-version', '--ndk_version', dest='ndk_version', default=None,
help=('DEPRECATED: the NDK version is now found automatically or '
'not at all.'))
generic_parser.add_argument(
'--ndk-api', type=int, default=None,
help=('The Android API level to compile against. This should be your '
'*minimal supported* API, not normally the same as your --android-api. '
'Defaults to min(ANDROID_API, {}) if not specified.').format(RECOMMENDED_NDK_API))
generic_parser.add_argument(
'--symlink-java-src', '--symlink_java_src',
action='store_true',
dest='symlink_java_src',
default=False,
help=('If True, symlinks the java src folder during build and dist '
'creation. This is useful for development only, it could also'
' cause weird problems.'))
default_storage_dir = user_data_dir('python-for-android')
if ' ' in default_storage_dir:
default_storage_dir = '~/.python-for-android'
generic_parser.add_argument(
'--storage-dir', dest='storage_dir', default=default_storage_dir,
help=('Primary storage directory for downloads and builds '
'(default: {})'.format(default_storage_dir)))
generic_parser.add_argument(
'--arch', help='The arch to build for.',
default='armeabi-v7a')
# Options for specifying the Distribution
generic_parser.add_argument(
'--dist-name', '--dist_name',
help='The name of the distribution to use or create', default='')
generic_parser.add_argument(
'--requirements',
help=('Dependencies of your app, should be recipe names or '
'Python modules. NOT NECESSARY if you are using '
'Python 3 with --use-setup-py'),
default='')
generic_parser.add_argument(
'--recipe-blacklist',
help=('Blacklist an internal recipe from use. Allows '
'disabling Python 3 core modules to save size'),
dest="recipe_blacklist",
default='')
generic_parser.add_argument(
'--blacklist-requirements',
help=('Blacklist an internal recipe from use. Allows '
'disabling Python 3 core modules to save size'),
dest="blacklist_requirements",
default='')
generic_parser.add_argument(
'--bootstrap',
help='The bootstrap to build with. Leave unset to choose '
'automatically.',
default=None)
generic_parser.add_argument(
'--hook',
help='Filename to a module that contains python-for-android hooks',
default=None)
add_boolean_option(
generic_parser, ["force-build"],
default=False,
description='Whether to force compilation of a new distribution')
add_boolean_option(
generic_parser, ["require-perfect-match"],
default=False,
description=('Whether the dist recipes must perfectly match '
'those requested'))
add_boolean_option(
generic_parser, ["allow-replace-dist"],
default=True,
description='Whether existing dist names can be automatically replaced'
)
generic_parser.add_argument(
'--local-recipes', '--local_recipes',
dest='local_recipes', default='./p4a-recipes',
help='Directory to look for local recipes')
generic_parser.add_argument(
'--java-build-tool',
dest='java_build_tool', default='auto',
choices=['auto', 'ant', 'gradle'],
help=('The java build tool to use when packaging the APK, defaults '
'to automatically selecting an appropriate tool.'))
add_boolean_option(
generic_parser, ['copy-libs'],
default=False,
description='Copy libraries instead of using biglink (Android 4.3+)'
)
self._read_configuration()
subparsers = parser.add_subparsers(dest='subparser_name',
help='The command to run')
def add_parser(subparsers, *args, **kwargs):
"""
argparse in python2 doesn't support the aliases option,
so we just don't provide the aliases there.
"""
if 'aliases' in kwargs and sys.version_info.major < 3:
kwargs.pop('aliases')
return subparsers.add_parser(*args, **kwargs)
add_parser(
subparsers,
'recommendations',
parents=[generic_parser],
help='List recommended p4a dependencies')
parser_recipes = add_parser(
subparsers,
'recipes',
parents=[generic_parser],
help='List the available recipes')
parser_recipes.add_argument(
"--compact",
action="store_true", default=False,
help="Produce a compact list suitable for scripting")
add_parser(
subparsers, 'bootstraps',
help='List the available bootstraps',
parents=[generic_parser])
add_parser(
subparsers, 'clean_all',
aliases=['clean-all'],
help='Delete all builds, dists and caches',
parents=[generic_parser])
add_parser(
subparsers, 'clean_dists',
aliases=['clean-dists'],
help='Delete all dists',
parents=[generic_parser])
add_parser(
subparsers, 'clean_bootstrap_builds',
aliases=['clean-bootstrap-builds'],
help='Delete all bootstrap builds',
parents=[generic_parser])
add_parser(
subparsers, 'clean_builds',
aliases=['clean-builds'],
help='Delete all builds',
parents=[generic_parser])
parser_clean = add_parser(
subparsers, 'clean',
help='Delete build components.',
parents=[generic_parser])
parser_clean.add_argument(
'component', nargs='+',
help=('The build component(s) to delete. You can pass any '
'number of arguments from "all", "builds", "dists", '
'"distributions", "bootstrap_builds", "downloads".'))
parser_clean_recipe_build = add_parser(
subparsers,
'clean_recipe_build', aliases=['clean-recipe-build'],
help=('Delete the build components of the given recipe. '
'By default this will also delete built dists'),
parents=[generic_parser])
parser_clean_recipe_build.add_argument(
'recipe', help='The recipe name')
parser_clean_recipe_build.add_argument(
'--no-clean-dists', default=False,
dest='no_clean_dists',
action='store_true',
help='If passed, do not delete existing dists')
parser_clean_download_cache = add_parser(
subparsers,
'clean_download_cache', aliases=['clean-download-cache'],
help='Delete cached downloads for requirement builds',
parents=[generic_parser])
parser_clean_download_cache.add_argument(
'recipes',
nargs='*',
help='The recipes to clean (space-separated). If no recipe name is'
' provided, the entire cache is cleared.')
parser_export_dist = add_parser(
subparsers,
'export_dist', aliases=['export-dist'],
help='Copy the named dist to the given path',
parents=[generic_parser])
parser_export_dist.add_argument('output_dir',
help='The output dir to copy to')
parser_export_dist.add_argument(
'--symlink',
action='store_true',
help='Symlink the dist instead of copying')
parser_apk = add_parser(
subparsers,
'apk', help='Build an APK',
parents=[generic_parser])
# This is actually an internal argument of the build.py
# (see pythonforandroid/bootstraps/common/build/build.py).
# However, it is also needed before the distribution is finally
# assembled for locating the setup.py / other build systems, which
# is why we also add it here:
parser_apk.add_argument(
'--private', dest='private',
help='the directory with the app source code files' +
' (containing your main.py entrypoint)',
required=False, default=None)
parser_apk.add_argument(
'--release', dest='build_mode', action='store_const',
const='release', default='debug',
help='Build the PARSER_APK. in Release mode')
parser_apk.add_argument(
'--use-setup-py', dest="use_setup_py",
action='store_true', default=False,
help="Process the setup.py of a project if present. " +
"(Experimental!")
parser_apk.add_argument(
'--ignore-setup-py', dest="ignore_setup_py",
action='store_true', default=False,
help="Don't run the setup.py of a project if present. " +
"This may be required if the setup.py is not " +
"designed to work inside p4a (e.g. by installing " +
"dependencies that won't work or aren't desired " +
"on Android")
parser_apk.add_argument(
'--keystore', dest='keystore', action='store', default=None,
help=('Keystore for JAR signing key, will use jarsigner '
'default if not specified (release build only)'))
parser_apk.add_argument(
'--signkey', dest='signkey', action='store', default=None,
help='Key alias to sign PARSER_APK. with (release build only)')
parser_apk.add_argument(
'--keystorepw', dest='keystorepw', action='store', default=None,
help='Password for keystore')
parser_apk.add_argument(
'--signkeypw', dest='signkeypw', action='store', default=None,
help='Password for key alias')
add_parser(
subparsers,
'create', help='Compile a set of requirements into a dist',
parents=[generic_parser])
add_parser(
subparsers,
'archs', help='List the available target architectures',
parents=[generic_parser])
add_parser(
subparsers,
'distributions', aliases=['dists'],
help='List the currently available (compiled) dists',
parents=[generic_parser])
add_parser(
subparsers,
'delete_dist', aliases=['delete-dist'], help='Delete a compiled dist',
parents=[generic_parser])
parser_sdk_tools = add_parser(
subparsers,
'sdk_tools', aliases=['sdk-tools'],
help='Run the given binary from the SDK tools dis',
parents=[generic_parser])
parser_sdk_tools.add_argument(
'tool', help='The binary tool name to run')
add_parser(
subparsers,
'adb', help='Run adb from the given SDK',
parents=[generic_parser])
add_parser(
subparsers,
'logcat', help='Run logcat from the given SDK',
parents=[generic_parser])
add_parser(
subparsers,
'build_status', aliases=['build-status'],
help='Print some debug information about current built components',
parents=[generic_parser])
parser.add_argument('-v', '--version', action='version',
version=__version__)
args, unknown = parser.parse_known_args(sys.argv[1:])
args.unknown_args = unknown
if hasattr(args, "private") and args.private is not None:
# Pass this value on to the internal bootstrap build.py:
args.unknown_args += ["--private", args.private]
if hasattr(args, "ignore_setup_py") and args.ignore_setup_py:
args.use_setup_py = False
self.args = args
if args.subparser_name is None:
parser.print_help()
exit(1)
setup_color(args.color)
if args.debug:
logger.setLevel(logging.DEBUG)
self.ctx = Context()
self.ctx.use_setup_py = getattr(args, "use_setup_py", True)
have_setup_py_or_similar = False
if getattr(args, "private", None) is not None:
project_dir = getattr(args, "private")
if (os.path.exists(os.path.join(project_dir, "setup.py")) or
os.path.exists(os.path.join(project_dir,
"pyproject.toml"))):
have_setup_py_or_similar = True
# Process requirements and put version in environ
if hasattr(args, 'requirements'):
requirements = []
# Add dependencies from setup.py, but only if they are recipes
# (because otherwise, setup.py itself will install them later)
if (have_setup_py_or_similar and
getattr(args, "use_setup_py", False)):
try:
info("Analyzing package dependencies. MAY TAKE A WHILE.")
# Get all the dependencies corresponding to a recipe:
dependencies = [
dep.lower() for dep in
get_dep_names_of_package(
args.private,
keep_version_pins=True,
recursive=True,
verbose=True,
)
]
info("Dependencies obtained: " + str(dependencies))
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(self.ctx))
]
dependencies = set(dependencies).intersection(
set(all_recipes)
)
# Add dependencies to argument list:
if len(dependencies) > 0:
if len(args.requirements) > 0:
args.requirements += u","
args.requirements += u",".join(dependencies)
except ValueError:
# Not a python package, apparently.
warning(
"Processing failed, is this project a valid "
"package? Will continue WITHOUT setup.py deps."
)
# Parse --requirements argument list:
for requirement in split_argument_list(args.requirements):
if "==" in requirement:
requirement, version = requirement.split(u"==", 1)
os.environ["VERSION_{}".format(requirement)] = version
info('Recipe {}: version "{}" requested'.format(
requirement, version))
requirements.append(requirement)
args.requirements = u",".join(requirements)
self.warn_on_deprecated_args(args)
self.storage_dir = args.storage_dir
self.ctx.setup_dirs(self.storage_dir)
self.sdk_dir = args.sdk_dir
self.ndk_dir = args.ndk_dir
self.android_api = args.android_api
self.ndk_api = args.ndk_api
self.ctx.symlink_java_src = args.symlink_java_src
self.ctx.java_build_tool = args.java_build_tool
self._archs = split_argument_list(args.arch)
self.ctx.local_recipes = args.local_recipes
self.ctx.copy_libs = args.copy_libs
# Each subparser corresponds to a method
getattr(self, args.subparser_name.replace('-', '_'))(args)
def get_recipe_env(self, arch, with_flags_in_cc=True):
""" Add libgeos headers to path """
env = super(ShapelyRecipe, self).get_recipe_env(arch, with_flags_in_cc)
libgeos_dir = Recipe.get_recipe('libgeos', self.ctx).get_build_dir(arch.arch)
env['CFLAGS'] += " -I{}/dist/include".format(libgeos_dir)
return env
def get_recipe_order_and_bootstrap(ctx, names, bs=None):
'''Takes a list of recipe names and (optionally) a bootstrap. Then
works out the dependency graph (including bootstrap recipes if
necessary). Finally, if no bootstrap was initially selected,
chooses one that supports all the recipes.
'''
graph = Graph()
recipes_to_load = set(names)
if bs is not None and bs.recipe_depends:
info_notify('Bootstrap requires recipes {}'.format(bs.recipe_depends))
recipes_to_load = recipes_to_load.union(set(bs.recipe_depends))
recipes_to_load = list(recipes_to_load)
recipe_loaded = []
python_modules = []
while recipes_to_load:
name = recipes_to_load.pop(0)
if name in recipe_loaded or isinstance(name, (list, tuple)):
continue
try:
recipe = Recipe.get_recipe(name, ctx)
except IOError:
info('No recipe named {}; will attempt to install with pip'.format(
name))
python_modules.append(name)
continue
except (KeyboardInterrupt, SystemExit):
raise
except:
warning('Failed to import recipe named {}; the recipe exists '
'but appears broken.'.format(name))
warning('Exception was:')
raise
graph.add(name, name)
info('Loaded recipe {} (depends on {}{})'.format(
name, recipe.depends, ', conflicts {}'.format(recipe.conflicts)
if recipe.conflicts else ''))
for depend in recipe.depends:
graph.add(name, depend)
recipes_to_load += recipe.depends
for conflict in recipe.conflicts:
if graph.conflicts(conflict):
warning(('{} conflicts with {}, but both have been '
'included or pulled into the requirements.'.format(
recipe.name, conflict)))
warning(
'Due to this conflict the build cannot continue, exiting.')
exit(1)
python_modules += recipe.python_depends
recipe_loaded.append(name)
graph.remove_remaining_conflicts(ctx)
if len(graph.graphs) > 1:
info('Found multiple valid recipe sets:')
for g in graph.graphs:
info(' {}'.format(g.keys()))
info_notify('Using the first of these: {}'.format(
graph.graphs[0].keys()))
elif len(graph.graphs) == 0:
warning('Didn\'t find any valid dependency graphs, exiting.')
exit(1)
else:
info('Found a single valid recipe set (this is good)')
build_order = list(graph.find_order(0))
if bs is None: # It would be better to check against possible
# orders other than the first one, but in practice
# there will rarely be clashes, and the user can
# specify more parameters if necessary to resolve
# them.
bs = Bootstrap.get_bootstrap_from_recipes(build_order, ctx)
if bs is None:
info('Could not find a bootstrap compatible with the '
'required recipes.')
info('If you think such a combination should exist, try '
'specifying the bootstrap manually with --bootstrap.')
exit(1)
info('{} bootstrap appears compatible with the required recipes.'.
format(bs.name))
info('Checking this...')
recipes_to_load = bs.recipe_depends
# This code repeats the code from earlier! Should move to a function:
while recipes_to_load:
name = recipes_to_load.pop(0)
if name in recipe_loaded or isinstance(name, (list, tuple)):
continue
try:
recipe = Recipe.get_recipe(name, ctx)
except ImportError:
info('No recipe named {}; will attempt to install with pip'.
format(name))
python_modules.append(name)
continue
graph.add(name, name)
info('Loaded recipe {} (depends on {}{})'.format(
name, recipe.depends, ', conflicts {}'.format(recipe.conflicts)
if recipe.conflicts else ''))
for depend in recipe.depends:
graph.add(name, depend)
recipes_to_load += recipe.depends
for conflict in recipe.conflicts:
if graph.conflicts(conflict):
warning(
('{} conflicts with {}, but both have been '
'included or pulled into the requirements.'.format(
recipe.name, conflict)))
warning('Due to this conflict the build cannot continue, '
'exiting.')
exit(1)
recipe_loaded.append(name)
graph.remove_remaining_conflicts(ctx)
build_order = list(graph.find_order(0))
return build_order, python_modules, bs
def test_postarch_build(self, mock_install_stl_lib):
arch = ArchAarch_64(self.ctx)
recipe = Recipe.get_recipe('icu', self.ctx)
assert recipe.need_stl_shared, True
recipe.postbuild_arch(arch)
mock_install_stl_lib.assert_called_once_with(arch)
def do_python_build(self, arch):
if 'sqlite' in self.ctx.recipe_build_order:
print('sqlite support not yet enabled in python recipe')
exit(1)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, self.ctx.hostpython, self.get_build_dir(arch.arch))
shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir(arch.arch))
hostpython = join(self.get_build_dir(arch.arch), 'hostpython')
hostpgen = join(self.get_build_dir(arch.arch), 'hostpython')
with current_directory(self.get_build_dir(arch.arch)):
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, join(hostpython_recipe.get_recipe_dir(), 'Setup'), 'Modules')
env = arch.get_env()
# AND: Should probably move these to get_recipe_env for
# neatness, but the whole recipe needs tidying along these
# lines
env['HOSTARCH'] = 'arm-eabi'
env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
env['CFLAGS'] = ' '.join([env['CFLAGS'], '-DNO_MALLINFO'])
# TODO need to add a should_build that checks if optional
# dependencies have changed (possibly in a generic way)
if 'openssl' in self.ctx.recipe_build_order:
openssl_build_dir = Recipe.get_recipe('openssl', self.ctx).get_build_dir(arch.arch)
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal)
shprint(sh.sed, '-i', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal)
configure = sh.Command('./configure')
# AND: OFLAG isn't actually set, should it be?
shprint(configure,
'--host={}'.format(env['HOSTARCH']),
'--build={}'.format(env['BUILDARCH']),
# 'OPT={}'.format(env['OFLAG']),
'--prefix={}'.format(realpath('./python-install')),
'--enable-shared',
'--disable-toolbox-glue',
'--disable-framework',
_env=env)
# AND: tito left this comment in the original source. It's still true!
# FIXME, the first time, we got a error at:
# python$EXE ../../Tools/scripts/h2py.py -i '(u_long)' /usr/include/netinet/in.h
# /home/tito/code/python-for-android/build/python/Python-2.7.2/python: 1: Syntax error: word unexpected (expecting ")")
# because at this time, python is arm, not x86. even that, why /usr/include/netinet/in.h is used ?
# check if we can avoid this part.
make = sh.Command(env['MAKE'].split(' ')[0])
print('First install (expected to fail...')
try:
shprint(make, '-j5', 'install', 'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
except sh.ErrorReturnCode_2:
print('First python2 make failed. This is expected, trying again.')
print('Second install (expected to work)')
shprint(sh.touch, 'python.exe', 'python')
shprint(make, '-j5', 'install', 'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
if is_darwin():
shprint(sh.cp, join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'Lib'))
shprint(sh.cp, join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'lib', 'python2.7'))
# reduce python
for dir_name in ('test', join('json', 'tests'), 'lib-tk',
join('sqlite3', 'test'), join('unittest, test'),
join('lib2to3', 'tests'), join('bsddb', 'tests'),
join('distutils', 'tests'), join('email', 'test'),
'curses'):
shprint(sh.rm, '-rf', join('python-install',
'lib', 'python2.7', dir_name))
def set_libs_flags(self, env, arch):
'''Takes care to properly link libraries with python depending on our
requirements and the attribute :attr:`opt_depends`.
'''
def add_flags(include_flags, link_dirs, link_libs):
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
env['LIBS'] = env.get('LIBS', '') + link_libs
if 'sqlite3' in self.ctx.recipe_build_order:
info('Activating flags for sqlite3')
recipe = Recipe.get_recipe('sqlite3', self.ctx)
add_flags(' -I' + recipe.get_build_dir(arch.arch),
' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')
if 'libffi' in self.ctx.recipe_build_order:
info('Activating flags for libffi')
recipe = Recipe.get_recipe('libffi', self.ctx)
# In order to force the correct linkage for our libffi library, we
# set the following variable to point where is our libffi.pc file,
# because the python build system uses pkg-config to configure it.
env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)
add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
' -lffi')
if 'openssl' in self.ctx.recipe_build_order:
info('Activating flags for openssl')
recipe = Recipe.get_recipe('openssl', self.ctx)
add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch),
recipe.link_libs_flags())
for library_name in {'libbz2', 'liblzma'}:
if library_name in self.ctx.recipe_build_order:
info(f'Activating flags for {library_name}')
recipe = Recipe.get_recipe(library_name, self.ctx)
add_flags(recipe.get_library_includes(arch),
recipe.get_library_ldflags(arch),
recipe.get_library_libs_flag())
# python build system contains hardcoded zlib version which prevents
# the build of zlib module, here we search for android's zlib version
# and sets the right flags, so python can be build with android's zlib
info("Activating flags for android's zlib")
zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib')
zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
zlib_h = join(zlib_includes, 'zlib.h')
try:
with open(zlib_h) as fileh:
zlib_data = fileh.read()
except IOError:
raise BuildInterruptingException(
"Could not determine android's zlib version, no zlib.h ({}) in"
" the NDK dir includes".format(zlib_h))
for line in zlib_data.split('\n'):
if line.startswith('#define ZLIB_VERSION '):
break
else:
raise BuildInterruptingException(
'Could not parse zlib.h...so we cannot find zlib version,'
'required by python build,')
env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')
return env
def get_env(self, with_flags_in_cc=True, clang=False):
env = {}
cflags = [
'-DANDROID', '-fomit-frame-pointer',
'-D__ANDROID_API__={}'.format(self.ctx.ndk_api)
]
if not clang:
cflags.append('-mandroid')
else:
cflags.append('-target ' + self.target)
toolchain = '{android_host}-{toolchain_version}'.format(
android_host=self.ctx.toolchain_prefix,
toolchain_version=self.ctx.toolchain_version)
toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain,
'prebuilt', 'linux-x86_64')
cflags.append('-gcc-toolchain {}'.format(toolchain))
env['CFLAGS'] = ' '.join(cflags)
env['LDFLAGS'] = ' '
sysroot = join(self.ctx._ndk_dir, 'sysroot')
if exists(sysroot):
# post-15 NDK per
# https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md
env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.ctx.toolchain_prefix)
env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.command_prefix)
else:
sysroot = self.ctx.ndk_platform
env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform)
env['CFLAGS'] += ' -isysroot {} '.format(sysroot)
env['CFLAGS'] += '-I' + join(
self.ctx.get_python_install_dir(), 'include/python{}'.format(
self.ctx.python_recipe.version[0:3]))
env['LDFLAGS'] += '--sysroot {} '.format(self.ctx.ndk_platform)
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] += " ".join(
['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(
self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update(
{k: v
for k, v in environ.items() if k.startswith('CCACHE_')})
if clang:
llvm_dirname = split(
glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1]
clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname,
'prebuilt', 'linux-x86_64', 'bin')
environ['PATH'] = '{clang_path}:{path}'.format(
clang_path=clang_path, path=environ['PATH'])
exe = join(clang_path, 'clang')
execxx = join(clang_path, 'clang++')
else:
exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix)
execxx = '{command_prefix}-g++'.format(
command_prefix=command_prefix)
cc = find_executable(exe, path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
raise BuildInterruptingException(
'Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
if with_flags_in_cc:
env['CC'] = '{ccache}{exe} {cflags}'.format(exe=exe,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{execxx} {cxxflags}'.format(
execxx=execxx, ccache=ccache, cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{exe}'.format(exe=exe, ccache=ccache)
env['CXX'] = '{ccache}{execxx}'.format(execxx=execxx,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
env['LDSHARED'] = env["CC"] + " -pthread -shared " +\
"-Wl,-O1 -Wl,-Bsymbolic-functions "
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
# For crystax python, we can't use the host python headers:
env["CFLAGS"] += ' -I{}/sources/python/{}/include/python/'.\
format(self.ctx.ndk_dir, self.ctx.python_recipe.version[0:3])
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe(
'host' + self.ctx.python_recipe.name, self.ctx)
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch), 'build',
'lib.linux-{}-{}'.format(
uname()[-1],
self.ctx.python_recipe.major_minor_version_string))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
def get_env(self, with_flags_in_cc=True):
env = {}
# CFLAGS/CXXFLAGS: the processor flags
env['CFLAGS'] = ' '.join(self.common_cflags).format(target=self.target)
if self.arch_cflags:
# each architecture may have has his own CFLAGS
env['CFLAGS'] += ' ' + ' '.join(self.arch_cflags)
env['CXXFLAGS'] = env['CFLAGS']
# CPPFLAGS (for macros and includes)
env['CPPFLAGS'] = ' '.join(self.common_cppflags).format(
ctx=self.ctx,
command_prefix=self.command_prefix,
python_includes=join(
self.ctx.get_python_install_dir(),
'include/python{}'.format(self.ctx.python_recipe.version[0:3]),
),
)
# LDFLAGS: Link the extra global link paths first before anything else
# (such that overriding system libraries with them is possible)
env['LDFLAGS'] = (
' ' + " ".join([
"-L'" + link_path.replace("'", "'\"'\"'") +
"'" # no shlex.quote in py2
for link_path in self.extra_global_link_paths
]) + ' ' + ' '.join(self.common_ldflags).format(
ctx_libs_dir=self.ctx.get_libs_dir(self.arch)))
# LDLIBS: Library flags or names given to compilers when they are
# supposed to invoke the linker.
env['LDLIBS'] = ' '.join(self.common_ldlibs)
# CCACHE
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update(
{k: v
for k, v in environ.items() if k.startswith('CCACHE_')})
# Compiler: `CC` and `CXX` (and make sure that the compiler exists)
environ['PATH'] = '{clang_path}:{path}'.format(
clang_path=self.clang_path, path=environ['PATH'])
cc = find_executable(self.clang_exe, path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
raise BuildInterruptingException(
'Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.'.format(self.clang_exe))
if with_flags_in_cc:
env['CC'] = '{ccache}{exe} {cflags}'.format(exe=self.clang_exe,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{execxx} {cxxflags}'.format(
execxx=self.clang_exe_cxx,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{exe}'.format(exe=self.clang_exe,
ccache=ccache)
env['CXX'] = '{ccache}{execxx}'.format(execxx=self.clang_exe_cxx,
ccache=ccache)
# Android's binaries
command_prefix = self.command_prefix
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j{}'.format(str(cpu_count()))
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
# Android's arch/toolchain
env['ARCH'] = self.arch
env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
env['TOOLCHAIN_PREFIX'] = self.ctx.toolchain_prefix
env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version
# Custom linker options
env['LDSHARED'] = env['CC'] + ' ' + ' '.join(self.common_ldshared)
# Host python (used by some recipes)
hostpython_recipe = Recipe.get_recipe(
'host' + self.ctx.python_recipe.name, self.ctx)
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'native-build',
'build',
'lib.{}-{}'.format(
build_platform,
self.ctx.python_recipe.major_minor_version_string,
),
)
env['PATH'] = environ['PATH']
# for reproducible builds
if 'SOURCE_DATE_EPOCH' in environ:
for k in 'LC_ALL TZ SOURCE_DATE_EPOCH PYTHONHASHSEED BUILD_DATE BUILD_TIME'.split(
):
if k in environ:
env[k] = environ[k]
return env
def build_recipes(build_order,
python_modules,
ctx,
project_dir,
ignore_project_setup_py=False):
# Put recipes in correct build order
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
("The requirements ({}) were not found as recipes, they will be "
"installed with pip.").format(", ".join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main("# Downloading recipes ")
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main("# Building all recipes for arch {}".format(arch.arch))
info_main("# Unpacking recipes")
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main("# Prebuilding recipes")
# 2) prebuild packages
for recipe in recipes:
info_main("Prebuilding {} for {}".format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main("# Building recipes")
for recipe in recipes:
info_main("Building {} for {}".format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info("{} said it is already built, skipping".format(
recipe.name))
recipe.install_libraries(arch)
# 4) biglink everything
info_main("# Biglinking object files")
if not ctx.python_recipe:
biglink(ctx, arch)
else:
warning("Context's python recipe found, "
"skipping biglink (will this work?)")
# 5) postbuild packages
info_main("# Postbuilding recipes")
for recipe in recipes:
info_main("Postbuilding {} for {}".format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main("# Installing pure Python modules")
run_pymodules_install(ctx,
python_modules,
project_dir,
ignore_setup_py=ignore_project_setup_py)
def do_python_build(self, arch):
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, self.ctx.hostpython, self.get_build_dir(arch.arch))
shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir(arch.arch))
hostpython = join(self.get_build_dir(arch.arch), 'hostpython')
hostpgen = join(self.get_build_dir(arch.arch), 'hostpython')
with current_directory(self.get_build_dir(arch.arch)):
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, join(hostpython_recipe.get_recipe_dir(), 'Setup'),
'Modules')
env = arch.get_env()
env['HOSTARCH'] = 'arm-eabi'
env['BUILDARCH'] = shprint(
sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
env['CFLAGS'] = ' '.join([env['CFLAGS'], '-DNO_MALLINFO'])
# TODO need to add a should_build that checks if optional
# dependencies have changed (possibly in a generic way)
if 'openssl' in self.ctx.recipe_build_order:
recipe = Recipe.get_recipe('openssl', self.ctx)
openssl_build_dir = recipe.get_build_dir(arch.arch)
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'),
setuplocal)
shprint(sh.sed, '-i.backup',
's#^SSL=.*#SSL={}#'.format(openssl_build_dir),
setuplocal)
env['OPENSSL_VERSION'] = recipe.version
if 'sqlite3' in self.ctx.recipe_build_order:
# Include sqlite3 in python2 build
recipe = Recipe.get_recipe('sqlite3', self.ctx)
include = ' -I' + recipe.get_build_dir(arch.arch)
lib = ' -L' + recipe.get_lib_dir(arch) + ' -lsqlite3'
# Insert or append to env
flag = 'CPPFLAGS'
env[flag] = env[flag] + include if flag in env else include
flag = 'LDFLAGS'
env[flag] = env[flag] + lib if flag in env else lib
# NDK has langinfo.h but doesn't define nl_langinfo()
env['ac_cv_header_langinfo_h'] = 'no'
configure = sh.Command('./configure')
shprint(
configure,
'--host={}'.format(env['HOSTARCH']),
'--build={}'.format(env['BUILDARCH']),
# 'OPT={}'.format(env['OFLAG']),
'--prefix={}'.format(realpath('./python-install')),
'--enable-shared',
'--disable-toolbox-glue',
'--disable-framework',
_env=env)
# tito left this comment in the original source. It's still true!
# FIXME, the first time, we got a error at:
# python$EXE ../../Tools/scripts/h2py.py -i '(u_long)' /usr/include/netinet/in.h
# /home/tito/code/python-for-android/build/python/Python-2.7.2/python: 1: Syntax error: word unexpected (expecting ")")
# because at this time, python is arm, not x86. even that, why /usr/include/netinet/in.h is used ?
# check if we can avoid this part.
make = sh.Command(env['MAKE'].split(' ')[0])
print('First install (expected to fail...')
try:
shprint(make,
'-j5',
'install',
'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
except sh.ErrorReturnCode_2:
print(
'First python2 make failed. This is expected, trying again.'
)
print('Second install (expected to work)')
shprint(sh.touch, 'python.exe', 'python')
shprint(make,
'-j5',
'install',
'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
if is_darwin():
shprint(sh.cp,
join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'Lib'))
shprint(sh.cp,
join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'lib', 'python2.7'))
# reduce python
for dir_name in ('test', join('json', 'tests'), 'lib-tk',
join('sqlite3', 'test'), join('unittest, test'),
join('lib2to3', 'tests'), join('bsddb', 'tests'),
join('distutils',
'tests'), join('email', 'test'), 'curses'):
shprint(sh.rm, '-rf',
join('python-install', 'lib', 'python2.7', dir_name))
def get_env(self, with_flags_in_cc=True):
env = {}
env['CFLAGS'] = ' '.join([
'-DANDROID', '-mandroid', '-fomit-frame-pointer'
' -D__ANDROID_API__={}'.format(self.ctx._android_api),
])
env['LDFLAGS'] = ' '
sysroot = join(self.ctx._ndk_dir, 'sysroot')
if exists(sysroot):
# post-15 NDK per
# https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md
env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format(
self.ctx.ndk_dir, self.ctx.toolchain_prefix)
else:
sysroot = self.ctx.ndk_platform
env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform)
env['CFLAGS'] += ' -isysroot {} '.format(sysroot)
env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(),
'include/python{}'.format(
self.ctx.python_recipe.version[0:3])
)
env['LDFLAGS'] += '--sysroot {} '.format(self.ctx.ndk_platform)
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
cc = find_executable('{command_prefix}-gcc'.format(
command_prefix=command_prefix), path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the {} executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
if with_flags_in_cc:
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{command_prefix}-gcc'.format(
command_prefix=command_prefix,
ccache=ccache)
env['CXX'] = '{ccache}{command_prefix}-g++'.format(
command_prefix=command_prefix,
ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
# env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
# env['LDSHARED'] = env['LD']
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# AND: This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch),
'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
def build_recipes(build_order,
python_modules,
ctx,
project_dir,
ignore_project_setup_py=False):
# Put recipes in correct build order
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'.format(
recipe.name))
# 4) biglink everything
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx,
python_modules,
project_dir,
ignore_setup_py=ignore_project_setup_py)
return
def get_env(self, with_flags_in_cc=True):
env = {}
env["CFLAGS"] = " ".join([
"-DANDROID", "-mandroid", "-fomit-frame-pointer", "--sysroot",
self.ctx.ndk_platform
])
env["CXXFLAGS"] = env["CFLAGS"]
env["LDFLAGS"] = " ".join(
['-lm', '-L' + self.ctx.get_libs_dir(self.arch)])
if self.ctx.ndk == 'crystax':
env['LDFLAGS'] += ' -L{}/sources/crystax/libs/{} -lcrystax'.format(
self.ctx.ndk_dir, self.arch)
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_prefix = self.ctx.toolchain_prefix
toolchain_version = self.ctx.toolchain_version
command_prefix = self.command_prefix
env['TOOLCHAIN_PREFIX'] = toolchain_prefix
env['TOOLCHAIN_VERSION'] = toolchain_version
ccache = ''
if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))):
# print('ccache found, will optimize builds')
ccache = self.ctx.ccache + ' '
env['USE_CCACHE'] = '1'
env['NDK_CCACHE'] = self.ctx.ccache
env.update(
{k: v
for k, v in environ.items() if k.startswith('CCACHE_')})
cc = find_executable(
'{command_prefix}-gcc'.format(command_prefix=command_prefix),
path=environ['PATH'])
if cc is None:
print('Searching path are: {!r}'.format(environ['PATH']))
warning('Couldn\'t find executable for CC. This indicates a '
'problem locating the CC executable in the Android '
'NDK, not that you don\'t have a normal compiler '
'installed. Exiting.')
exit(1)
if with_flags_in_cc:
env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cflags=env['CFLAGS'])
env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
command_prefix=command_prefix,
ccache=ccache,
cxxflags=env['CXXFLAGS'])
else:
env['CC'] = '{ccache}{command_prefix}-gcc'.format(
command_prefix=command_prefix, ccache=ccache)
env['CXX'] = '{ccache}{command_prefix}-g++'.format(
command_prefix=command_prefix, ccache=ccache)
env['AR'] = '{}-ar'.format(command_prefix)
env['RANLIB'] = '{}-ranlib'.format(command_prefix)
env['LD'] = '{}-ld'.format(command_prefix)
# env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
# env['LDSHARED'] = env['LD']
env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
env['MAKE'] = 'make -j5'
env['READELF'] = '{}-readelf'.format(command_prefix)
env['NM'] = '{}-nm'.format(command_prefix)
#: frmdstryr: This seems to not be used anywhere, is it needed?
try:
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
# AND: This hardcodes python version 2.7, needs fixing
env['BUILDLIB_PATH'] = join(
hostpython_recipe.get_build_dir(self.arch), 'build',
'lib.linux-{}-2.7'.format(uname()[-1]))
except IOError:
pass
env['PATH'] = environ['PATH']
env['ARCH'] = self.arch
if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version
return env
def test_run_distribute(
self,
mock_sh_cp,
mock_sh_rm,
mock_listdir,
mock_chdir,
mock_ensure_dir,
mock_strip_libraries,
mock_create_python_bundle,
mock_open_dist_files,
mock_open_sdl2_files,
mock_open_webview_files,
mock_open_service_only_files,
):
"""
A test for any overwritten method of
`~pythonforandroid.bootstrap.Bootstrap.run_distribute`. Here we mock
any file/dir operation that it could slow down our tests, and there is
a lot to mock, because the `run_distribute` method it should take care
of prepare all compiled files to generate the final `apk`. The targets
of this test will be:
- :meth:`~pythonforandroid.bootstraps.sdl2.BootstrapSdl2
.run_distribute`
- :meth:`~pythonforandroid.bootstraps.service_only
.ServiceOnlyBootstrap.run_distribute`
- :meth:`~pythonforandroid.bootstraps.webview.WebViewBootstrap
.run_distribute`
- :meth:`~pythonforandroid.bootstraps.empty.EmptyBootstrap.
run_distribute`
Here we will tests all those methods that are specific for each class.
"""
# prepare bootstrap and distribution
bs = Bootstrap().get_bootstrap(self.bootstrap_name, self.ctx)
bs.build_dir = bs.get_build_dir()
self.setUp_distribution_with_bootstrap(bs)
self.ctx.hostpython = "/some/fake/hostpython3"
self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx)
self.ctx.python_modules = ["requests"]
self.ctx.archs = [ArchARMv7_a(self.ctx)]
bs.run_distribute()
mock_open_dist_files.assert_called_once_with("dist_info.json", "w")
mock_open_bootstraps = {
"sdl2": mock_open_sdl2_files,
"webview": mock_open_webview_files,
"service_only": mock_open_service_only_files,
}
expected_open_calls = {
"sdl2": [
mock.call("local.properties", "w"),
mock.call("blacklist.txt", "a"),
],
"webview": [mock.call("local.properties", "w")],
"service_only": [mock.call("local.properties", "w")],
}
mock_open_bs = mock_open_bootstraps[self.bootstrap_name]
# test that the expected calls has been called
for expected_call in expected_open_calls[self.bootstrap_name]:
self.assertIn(expected_call, mock_open_bs.call_args_list)
# test that the write function has been called with the expected args
self.assertIn(
mock.call().__enter__().write("sdk.dir=/opt/android/android-sdk"),
mock_open_bs.mock_calls,
)
if self.bootstrap_name == "sdl2":
self.assertIn(
mock.call().__enter__().write(
"\nsqlite3/*\nlib-dynload/_sqlite3.so\n"),
mock_open_bs.mock_calls,
)
# check that the other mocks we made are actually called
mock_sh_rm.assert_called()
mock_sh_cp.assert_called()
mock_chdir.assert_called()
mock_listdir.assert_called()
mock_strip_libraries.assert_called()
mock_create_python_bundle.assert_called()
def build_arch(self, arch):
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, 'android-build')
ensure_dir(build_dir)
# TODO: Get these dynamically, like bpo-30386 does
sys_prefix = '/usr/local'
sys_exec_prefix = '/usr/local'
# Skipping "Ensure that nl_langinfo is broken" from the original bpo-30386
platform_name = 'android-{}'.format(self.ctx.ndk_api)
with current_directory(build_dir):
env = environ.copy()
# TODO: Get this information from p4a's arch system
android_host = 'arm-linux-androideabi'
android_build = sh.Command(join(recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8')
platform_dir = join(self.ctx.ndk_dir, 'platforms', platform_name, 'arch-arm')
toolchain = '{android_host}-4.9'.format(android_host=android_host)
toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', 'linux-x86_64')
CC = '{clang} -target {target} -gcc-toolchain {toolchain}'.format(
clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt', 'linux-x86_64', 'bin', 'clang'),
target='armv7-none-linux-androideabi',
toolchain=toolchain)
AR = join(toolchain, 'bin', android_host) + '-ar'
LD = join(toolchain, 'bin', android_host) + '-ld'
RANLIB = join(toolchain, 'bin', android_host) + '-ranlib'
READELF = join(toolchain, 'bin', android_host) + '-readelf'
STRIP = join(toolchain, 'bin', android_host) + '-strip --strip-debug --strip-unneeded'
env['CC'] = CC
env['AR'] = AR
env['LD'] = LD
env['RANLIB'] = RANLIB
env['READELF'] = READELF
env['STRIP'] = STRIP
env['PATH'] = '{hostpython_dir}:{old_path}'.format(
hostpython_dir=self.get_recipe('hostpython3', self.ctx).get_path_to_python(),
old_path=env['PATH'])
ndk_flags = ('--sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} '
'-isystem {ndk_android_host}').format(
ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'),
android_api=self.ctx.ndk_api,
ndk_android_host=join(
self.ctx.ndk_dir, 'sysroot', 'usr', 'include', android_host))
sysroot = join(self.ctx.ndk_dir, 'platforms', platform_name, 'arch-arm')
env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags
env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags
env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format(sysroot, join(sysroot, 'usr', 'lib'))
if 'openssl' in self.ctx.recipe_build_order:
recipe = Recipe.get_recipe('openssl', self.ctx)
openssl_build_dir = recipe.get_build_dir(arch.arch)
ensure_dir('Modules')
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal)
shprint(sh.sed, '-i.backup', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal)
env['OPENSSL_VERSION'] = recipe.lib_version
if 'sqlite3' in self.ctx.recipe_build_order:
# Include sqlite3 in python2 build
recipe = Recipe.get_recipe('sqlite3', self.ctx)
include = ' -I' + recipe.get_build_dir(arch.arch)
lib = ' -L' + recipe.get_lib_dir(arch) + ' -lsqlite3'
# Insert or append to env
flag = 'CPPFLAGS'
env[flag] = env[flag] + include if flag in env else include
flag = 'LDFLAGS'
env[flag] = env[flag] + lib if flag in env else lib
# Manually add the libs directory, and copy some object
# files to the current directory otherwise they aren't
# picked up. This seems necessary because the --sysroot
# setting in LDFLAGS is overridden by the other flags.
# TODO: Work out why this doesn't happen in the original
# bpo-30386 Makefile system.
logger.warning('Doing some hacky stuff to link properly')
lib_dir = join(sysroot, 'usr', 'lib')
env['LDFLAGS'] += ' -L{}'.format(lib_dir)
shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './')
shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './')
env['SYSROOT'] = sysroot
if not exists('config.status'):
shprint(sh.Command(join(recipe_build_dir, 'configure')),
*(' '.join(('--host={android_host}',
'--build={android_build}',
'--enable-shared',
'--disable-ipv6',
'ac_cv_file__dev_ptmx=yes',
'ac_cv_file__dev_ptc=no',
'--without-ensurepip',
'ac_cv_little_endian_double=yes',
'--prefix={prefix}',
'--exec-prefix={exec_prefix}')).format(
android_host=android_host,
android_build=android_build,
prefix=sys_prefix,
exec_prefix=sys_exec_prefix)).split(' '), _env=env)
if not exists('python'):
shprint(sh.make, 'all', _env=env)
# TODO: Look into passing the path to pyconfig.h in a
# better way, although this is probably acceptable
sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))