def build_arch(self, arch):
env = self.get_recipe_env(arch)
# Build libproto.a
with current_directory(self.get_build_dir(arch.arch)):
env['HOSTARCH'] = 'arm-eabi'
env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('./configure'),
'--host={}'.format(env['HOSTARCH']),
'--enable-shared',
_env=env)
with current_directory(join(self.get_build_dir(arch.arch), 'src')):
shprint(sh.make, 'libprotobuf.la', '-j'+str(cpu_count()), _env=env)
shprint(sh.cp, '.libs/libprotobuf.a', join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a'))
# Copy stl library
shutil.copyfile(
self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so',
join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so'))
# Build python bindings and _message.so
with current_directory(join(self.get_build_dir(arch.arch), 'python')):
hostpython = sh.Command(self.hostpython_location)
shprint(hostpython,
'setup.py',
'build_ext',
'--cpp_implementation', _env=env)
# Install python bindings
self.install_python_package(arch)
def build_arch(self, arch):
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, self.build_subdir)
ensure_dir(build_dir)
if not exists(join(build_dir, 'python')):
with current_directory(recipe_build_dir):
# Configure the build
with current_directory(build_dir):
if not exists('config.status'):
shprint(
sh.Command(join(recipe_build_dir, 'configure')))
# Create the Setup file. This copying from Setup.dist
# seems to be the normal and expected procedure.
shprint(sh.cp, join('Modules', 'Setup.dist'),
join(build_dir, 'Modules', 'Setup'))
result = shprint(sh.make, '-C', build_dir)
else:
info('Skipping {name} ({version}) build, as it has already '
'been completed'.format(name=self.name, version=self.version))
self.ctx.hostpython = join(build_dir, 'python')
def download_file(self, url, target, cwd=None):
"""
(internal) Download an ``url`` to a ``target``.
"""
if not url:
return
info('Downloading {} from {}'.format(self.name, url))
if cwd:
target = join(cwd, target)
parsed_url = urlparse(url)
if parsed_url.scheme in ('http', 'https'):
def report_hook(index, blksize, size):
if size <= 0:
progression = '{0} bytes'.format(index * blksize)
else:
progression = '{0:.2f}%'.format(
index * blksize * 100. / float(size))
if "CI" not in environ:
stdout.write('- Download {}\r'.format(progression))
stdout.flush()
if exists(target):
unlink(target)
# Download item with multiple attempts (for bad connections):
attempts = 0
while True:
try:
urlretrieve(url, target, report_hook)
except OSError as e:
attempts += 1
if attempts >= 5:
raise e
stdout.write('Download failed retrying in a second...')
time.sleep(1)
continue
break
return target
elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http', 'git+https'):
if isdir(target):
with current_directory(target):
shprint(sh.git, 'fetch', '--tags')
if self.version:
shprint(sh.git, 'checkout', self.version)
shprint(sh.git, 'pull')
shprint(sh.git, 'pull', '--recurse-submodules')
shprint(sh.git, 'submodule', 'update', '--recursive')
else:
if url.startswith('git+'):
url = url[4:]
shprint(sh.git, 'clone', '--recursive', url, target)
if self.version:
with current_directory(target):
shprint(sh.git, 'checkout', self.version)
shprint(sh.git, 'submodule', 'update', '--recursive')
return target
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('autoreconf'), '-vif', _env=env)
shprint(sh.Command('./configure'),
'--host=' + arch.command_prefix,
'--prefix=' + self.get_build_dir(arch.arch),
'--disable-builddir',
'--enable-shared', _env=env)
# '--with-sysroot={}'.format(self.ctx.ndk_platform),
# '--target={}'.format(arch.toolchain_prefix),
# ndk 15 introduces unified headers required --sysroot and
# -isysroot for libraries and headers. libtool's head explodes
# trying to weave them into it's own magic. The result is a link
# failure trying to link libc. We call make to compile the bits
# and manually link...
try:
shprint(sh.make, '-j5', 'libffi.la', _env=env)
except sh.ErrorReturnCode_2:
info("make libffi.la failed as expected")
cc = sh.Command(env['CC'].split()[0])
cflags = env['CC'].split()[1:]
host_build = self.get_build_dir(arch.arch)
arch_flags = ''
if '-march=' in env['CFLAGS']:
arch_flags = '-march={}'.format(env['CFLAGS'].split('-march=')[1])
src_arch = arch.command_prefix.split('-')[0]
if src_arch == 'x86_64':
# libffi has not specific arch files for x86_64...so...using
# the ones from x86 which seems to build fine...
src_arch = 'x86'
cflags.extend(arch_flags.split())
cflags.extend(['-shared', '-fPIC', '-DPIC'])
cflags.extend(glob(join(host_build, 'src/.libs/*.o')))
cflags.extend(glob(join(host_build, 'src', src_arch, '.libs/*.o')))
ldflags = env['LDFLAGS'].split()
cflags.extend(ldflags)
cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o',
'.libs/libffi.so'])
with current_directory(host_build):
shprint(cc, *cflags, _env=env)
ensure_dir(self.ctx.get_libs_dir(arch.arch))
shprint(sh.cp,
join(host_build, '.libs', 'libffi.so'),
self.ctx.get_libs_dir(arch.arch))
def build_arch(self, arch):
env = self.get_recipe_env(arch)
env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format(
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl/include -I{jni_path}/sdl_mixer'.format(
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl_ttf -I{jni_path}/sdl_image'.format(
jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
debug('pygame cflags', env['CFLAGS'])
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{libs_path} -L{src_path}/obj/local/{arch} -lm -lz'.format(
libs_path=self.ctx.libs_dir, src_path=self.ctx.bootstrap.build_dir, arch=env['ARCH'])
env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')
with current_directory(self.get_build_dir(arch.arch)):
info('hostpython is ' + self.ctx.hostpython)
hostpython = sh.Command(self.ctx.hostpython)
shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
_tail=10, _critical=True)
info('strip is ' + env['STRIP'])
build_lib = glob.glob('./build/lib*')
assert len(build_lib) == 1
print('stripping pygame')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';')
python_install_path = join(self.ctx.build_dir, 'python-install')
warning('Should remove pygame tests etc. here, but skipping for now')
def prebuild_arch(self, arch):
if not self.is_patched(arch):
super(ReportLabRecipe, self).prebuild_arch(arch)
self.apply_patch('patches/fix-setup.patch', arch.arch)
recipe_dir = self.get_build_dir(arch.arch)
shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
ft = self.get_recipe('freetype', self.ctx)
ft_dir = ft.get_build_dir(arch.arch)
ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
info('reportlab recipe: ft_dir={}'.format(ft_dir))
info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
with current_directory(recipe_dir):
sh.ls('-lathr')
ensure_dir(tmp_dir)
pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
if not os.path.isfile(pfbfile):
sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
if os.path.isfile("setup.py"):
with open('setup.py', 'rb') as f:
text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
with open('setup.py', 'wb') as f:
f.write(text)
def apk(self, args):
'''Create an APK using the given distribution.'''
# AND: Need to add a parser here for any extra options
# parser = argparse.ArgumentParser(
# description='Build an APK')
# args = parser.parse_args(args)
ctx = self.ctx
dist = self._dist
# Manually fixing these arguments at the string stage is
# unsatisfactory and should probably be changed somehow, but
# we can't leave it until later as the build.py scripts assume
# they are in the current directory.
for i, arg in enumerate(args[:-1]):
if arg in ('--dir', '--private'):
args[i+1] = realpath(expanduser(args[i+1]))
build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
with current_directory(dist.dist_dir):
build.parse_args(args)
shprint(sh.ant, 'debug', _tail=20, _critical=True)
# AND: This is very crude, needs improving. Also only works
# for debug for now.
info_main('# Copying APK to current directory')
apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-debug.apk'))
if len(apks) == 0:
raise ValueError('Couldn\'t find the built APK')
if len(apks) > 1:
info('More than one built APK found...guessing you '
'just built {}'.format(apks[-1]))
shprint(sh.cp, apks[-1], './')
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
debug('cwd is {}'.format(realpath(curdir)))
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
manually_cythonise = False
try:
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
manually_cythonise = True
if manually_cythonise:
self.cythonize_build(env=env)
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
_tail=20, _critical=True, *self.setup_extra_args)
else:
info('First build appeared to complete correctly, skipping manual'
'cythonising.')
self.strip_object_files(arch, env)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('./configure'), '--host=' + arch.toolchain_prefix,
'--prefix=' + self.ctx.get_python_install_dir(),
'--enable-shared', _env=env)
shprint(sh.make, '-j5', 'libffi.la', _env=env)
# dlname = None
# with open(join(host, 'libffi.la')) as f:
# for line in f:
# if line.startswith('dlname='):
# dlname = line.strip()[8:-1]
# break
#
# if not dlname or not exists(join(host, '.libs', dlname)):
# raise RuntimeError('failed to locate shared object! ({})'
# .format(dlname))
# shprint(sh.sed, '-i', 's/^dlname=.*$/dlname=\'libffi.so\'/', join(host, 'libffi.la'))
shprint(sh.cp, '-t', self.ctx.get_libs_dir(arch.arch),
join(self.get_host(arch), '.libs', 'libffi.so')) #,
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
try:
shprint(hostpython, 'setup.py', 'build_ext', _env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
info('Running cython where appropriate')
shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx',
'-exec', self.ctx.cython, '{}', ';', _env=env)
info('ran cython')
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
_tail=20, _critical=True, *self.setup_extra_args)
print('stripping')
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
print('stripped!?')
def run_pymodules_install(ctx, modules):
modules = filter(ctx.not_has_package, modules)
if not modules:
info('There are no Python modules to install, skipping')
return
info('The requirements ({}) don\'t have recipes, attempting to install '
'them with pip'.format(', '.join(modules)))
info('If this fails, it may mean that the module has compiled '
'components and needs a recipe.')
venv = sh.Command(ctx.virtualenv)
with current_directory(join(ctx.build_dir)):
shprint(venv, '--python=python2.7', 'venv')
info('Creating a requirements.txt file for the Python modules')
with open('requirements.txt', 'w') as fileh:
for module in modules:
fileh.write('{}\n'.format(module))
info('Installing Python modules with pip')
info('If this fails with a message about /bin/false, this '
'probably means the package cannot be installed with '
'pip as it needs a compilation recipe.')
# This bash method is what old-p4a used
# It works but should be replaced with something better
shprint(sh.bash, '-c', (
"source venv/bin/activate && env CC=/bin/false CXX=/bin/false "
"PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir()))
def install_python_package(self, arch):
env = self.get_recipe_env(arch)
info('Installing {} into site-packages'.format(self.name))
with current_directory(join(self.get_build_dir(arch.arch), 'python')):
hostpython = sh.Command(self.hostpython_location)
if self.ctx.python_recipe.from_crystax:
hpenv = env.copy()
shprint(hostpython, 'setup.py', 'install', '-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=.',
'--cpp_implementation',
_env=hpenv, *self.setup_extra_args)
else:
hppath = join(dirname(self.hostpython_location), 'Lib',
'site-packages')
hpenv = env.copy()
if 'PYTHONPATH' in hpenv:
hpenv['PYTHONPATH'] = ':'.join([hppath] +
hpenv['PYTHONPATH'].split(':'))
else:
hpenv['PYTHONPATH'] = hppath
shprint(hostpython, 'setup.py', 'install', '-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=lib/python2.7/site-packages',
'--cpp_implementation',
_env=hpenv, *self.setup_extra_args)
def prebuild_arch(self, arch):
super(AndroidRecipe, self).prebuild_arch(arch)
ctx_bootstrap = self.ctx.bootstrap.name
# define macros for Cython, C, Python
tpxi = 'DEF {} = {}\n'
th = '#define {} {}\n'
tpy = '{} = {}\n'
# make sure bootstrap name is in unicode
if isinstance(ctx_bootstrap, bytes):
ctx_bootstrap = ctx_bootstrap.decode('utf-8')
bootstrap = bootstrap_name = ctx_bootstrap
is_sdl2 = bootstrap_name in ('sdl2', 'sdl2python3', 'sdl2_gradle')
is_webview = bootstrap_name in ('webview',)
if is_sdl2 or is_webview:
if is_sdl2:
bootstrap = 'sdl2'
java_ns = u'org.kivy.android'
jni_ns = u'org/kivy/android'
else:
logger.error((
'unsupported bootstrap for android recipe: {}'
''.format(bootstrap_name)
))
exit(1)
config = {
'BOOTSTRAP': bootstrap,
'IS_SDL2': int(is_sdl2),
'PY2': int(will_build('python2')(self)),
'JAVA_NAMESPACE': java_ns,
'JNI_NAMESPACE': jni_ns,
}
# create config files for Cython, C and Python
with (
current_directory(self.get_build_dir(arch.arch))), (
open(join('android', 'config.pxi'), 'w')) as fpxi, (
open(join('android', 'config.h'), 'w')) as fh, (
open(join('android', 'config.py'), 'w')) as fpy:
for key, value in config.items():
fpxi.write(tpxi.format(key, repr(value)))
fpy.write(tpy.format(key, repr(value)))
fh.write(th.format(
key,
value if isinstance(value, int) else '"{}"'.format(value)
))
self.config_env[key] = str(value)
if is_sdl2:
fh.write('JNIEnv *SDL_AndroidGetJNIEnv(void);\n')
fh.write(
'#define SDL_ANDROID_GetJNIEnv SDL_AndroidGetJNIEnv\n'
)
def install_python_package(self, arch, name=None, env=None, is_dir=True):
'''Automate the installation of a Python package (or a cython
package where the cython components are pre-built).'''
# arch = self.filtered_archs[0] # old kivy-ios way
if name is None:
name = self.name
if env is None:
env = self.get_recipe_env(arch)
info('Installing {} into site-packages'.format(self.name))
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.hostpython_location)
# hostpython = sh.Command('python3.5')
if self.ctx.python_recipe.from_crystax:
# hppath = join(dirname(self.hostpython_location), 'Lib',
# 'site-packages')
hpenv = env.copy()
# if 'PYTHONPATH' in hpenv:
# hpenv['PYTHONPATH'] = ':'.join([hppath] +
# hpenv['PYTHONPATH'].split(':'))
# else:
# hpenv['PYTHONPATH'] = hppath
# hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
# shprint(hostpython, 'setup.py', 'build',
# _env=hpenv, *self.setup_extra_args)
shprint(hostpython, 'setup.py', 'install', '-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=.',
# AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
_env=hpenv, *self.setup_extra_args)
# site_packages_dir = self.ctx.get_site_packages_dir()
# built_files = glob.glob(join('build', 'lib*', '*'))
# for filen in built_files:
# shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
elif self.call_hostpython_via_targetpython:
shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
*self.setup_extra_args)
else:
hppath = join(dirname(self.hostpython_location), 'Lib',
'site-packages')
hpenv = env.copy()
if 'PYTHONPATH' in hpenv:
hpenv['PYTHONPATH'] = ':'.join([hppath] +
hpenv['PYTHONPATH'].split(':'))
else:
hpenv['PYTHONPATH'] = hppath
shprint(hostpython, 'setup.py', 'install', '-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=lib/python2.7/site-packages',
_env=hpenv, *self.setup_extra_args)
# AND: Hardcoded python2.7 needs fixing
# If asked, also install in the hostpython build dir
if self.install_in_hostpython:
self.install_hostpython_package(arch)
def download(self):
if self.url is None:
info('Skipping {} download as no URL is set'.format(self.name))
return
url = self.versioned_url
ma = match(u'^(.+)#md5=([0-9a-f]{32})$', url)
if ma: # fragmented URL?
if self.md5sum:
raise ValueError(
('Received md5sum from both the {} recipe '
'and its url').format(self.name))
url = ma.group(1)
expected_md5 = ma.group(2)
else:
expected_md5 = self.md5sum
shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name))
with current_directory(join(self.ctx.packages_path, self.name)):
filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')
do_download = True
marker_filename = '.mark-{}'.format(filename)
if exists(filename) and isfile(filename):
if not exists(marker_filename):
shprint(sh.rm, filename)
elif expected_md5:
current_md5 = md5sum(filename)
if current_md5 != expected_md5:
debug('* Generated md5sum: {}'.format(current_md5))
debug('* Expected md5sum: {}'.format(expected_md5))
raise ValueError(
('Generated md5sum does not match expected md5sum '
'for {} recipe').format(self.name))
do_download = False
else:
do_download = False
# If we got this far, we will download
if do_download:
debug('Downloading {} from {}'.format(self.name, url))
shprint(sh.rm, '-f', marker_filename)
self.download_file(self.versioned_url, filename)
shprint(sh.touch, marker_filename)
if exists(filename) and isfile(filename) and expected_md5:
current_md5 = md5sum(filename)
if expected_md5 is not None:
if current_md5 != expected_md5:
debug('* Generated md5sum: {}'.format(current_md5))
debug('* Expected md5sum: {}'.format(expected_md5))
raise ValueError(
('Generated md5sum does not match expected md5sum '
'for {} recipe').format(self.name))
else:
info('{} download already cached, skipping'.format(self.name))
def download_file(self, url, target, cwd=None):
"""
(internal) Download an ``url`` to a ``target``.
"""
if not url:
return
info('Downloading {} from {}'.format(self.name, url))
if cwd:
target = join(cwd, target)
parsed_url = urlparse(url)
if parsed_url.scheme in ('http', 'https'):
def report_hook(index, blksize, size):
if size <= 0:
progression = '{0} bytes'.format(index * blksize)
else:
progression = '{0:.2f}%'.format(
index * blksize * 100. / float(size))
stdout.write('- Download {}\r'.format(progression))
stdout.flush()
if exists(target):
unlink(target)
urlretrieve(url, target, report_hook)
return target
elif parsed_url.scheme in ('git', 'git+ssh', 'git+http', 'git+https'):
if isdir(target):
with current_directory(target):
shprint(sh.git, 'fetch', '--tags')
if self.version:
shprint(sh.git, 'checkout', self.version)
shprint(sh.git, 'pull')
shprint(sh.git, 'pull', '--recurse-submodules')
shprint(sh.git, 'submodule', 'update', '--recursive')
else:
if url.startswith('git+'):
url = url[4:]
shprint(sh.git, 'clone', '--recursive', url, target)
if self.version:
with current_directory(target):
shprint(sh.git, 'checkout', self.version)
shprint(sh.git, 'submodule', 'update', '--recursive')
return target
def download_file(self, url, target, cwd=None):
"""
(internal) Download an ``url`` to a ``target``.
"""
if not url:
return
info("Downloading {} from {}".format(self.name, url))
if cwd:
target = join(cwd, target)
parsed_url = urlparse(url)
if parsed_url.scheme in ("http", "https"):
def report_hook(index, blksize, size):
if size <= 0:
progression = "{0} bytes".format(index * blksize)
else:
progression = "{0:.2f}%".format(index * blksize * 100.0 / float(size))
stdout.write("- Download {}\r".format(progression))
stdout.flush()
if exists(target):
unlink(target)
urlretrieve(url, target, report_hook)
return target
elif parsed_url.scheme in ("git", "git+ssh", "git+http", "git+https"):
if isdir(target):
with current_directory(target):
shprint(sh.git, "fetch", "--tags")
if self.version:
shprint(sh.git, "checkout", self.version)
shprint(sh.git, "pull")
shprint(sh.git, "pull", "--recurse-submodules")
shprint(sh.git, "submodule", "update", "--recursive")
else:
if url.startswith("git+"):
url = url[4:]
shprint(sh.git, "clone", "--recursive", url, target)
if self.version:
with current_directory(target):
shprint(sh.git, "checkout", self.version)
shprint(sh.git, "submodule", "update", "--recursive")
return target
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
if self.ctx.python_recipe.from_crystax:
command = sh.Command('python{}'.format(self.ctx.python_recipe.version))
site_packages_dirs = command(
'-c', 'import site; print("\\n".join(site.getsitepackages()))')
site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n')
# env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
if 'PYTHONPATH' in env:
env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs))
else:
env['PYTHONPATH'] = ':'.join(site_packages_dirs)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
# hostpython = sh.Command('python3.5')
shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
print('cwd is', realpath(curdir))
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
manually_cythonise = False
try:
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
manually_cythonise = True
if manually_cythonise:
info('Running cython where appropriate')
cyenv = env.copy()
if 'CYTHONPATH' in cyenv:
cyenv['PYTHONPATH'] = cyenv['CYTHONPATH']
elif 'PYTHONPATH' in cyenv:
del cyenv['PYTHONPATH']
cython = 'cython' if self.ctx.python_recipe.from_crystax else self.ctx.cython
cython_cmd = 'find "{}" -iname *.pyx | xargs "{}"'.format(
self.get_build_dir(arch.arch), cython)
shprint(sh.sh, '-c', cython_cmd, _env=cyenv)
info('ran cython')
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
_tail=20, _critical=True, *self.setup_extra_args)
else:
info('First build appeared to complete correctly, skipping manual'
'cythonising.')
print('stripping')
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
print('stripped!?')
def build_compiled_components(self,arch):
super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch)
# Copy libgnustl_shared.so
with current_directory(self.get_build_dir(arch.arch)):
sh.cp(
"{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx,arch=arch),
self.ctx.get_libs_dir(arch.arch)
)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('autoreconf'), '-vif', _env=env)
shprint(sh.Command('./configure'),
'--host=' + arch.toolchain_prefix,
'--prefix=' + self.ctx.get_python_install_dir(),
'--enable-shared', _env=env)
#'--with-sysroot={}'.format(self.ctx.ndk_platform),
#'--target={}'.format(arch.toolchain_prefix),
# ndk 15 introduces unified headers required --sysroot and
# -isysroot for libraries and headers. libtool's head explodes
# trying to weave them into it's own magic. The result is a link
# failure tryng to link libc. We call make to compile the bits
# and manually link...
try:
shprint(sh.make, '-j5', 'libffi.la', _env=env)
except sh.ErrorReturnCode_2:
info("make libffi.la failed as expected")
cc = sh.Command(env['CC'].split()[0])
cflags = env['CC'].split()[1:]
cflags.extend(['-march=armv7-a', '-mfloat-abi=softfp', '-mfpu=vfp',
'-mthumb', '-shared', '-fPIC', '-DPIC',
'src/.libs/prep_cif.o', 'src/.libs/types.o',
'src/.libs/raw_api.o', 'src/.libs/java_raw_api.o',
'src/.libs/closures.o', 'src/arm/.libs/sysv.o',
'src/arm/.libs/ffi.o', ]
)
ldflags = env['LDFLAGS'].split()
cflags.extend(ldflags)
cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o',
'.libs/libffi.so'])
with current_directory(self.get_host(arch)):
shprint(cc, *cflags, _env=env)
shprint(sh.cp, '-t', self.ctx.get_libs_dir(arch.arch),
join(self.get_host(arch), '.libs', 'libffi.so'))
def load_info(self):
'''Load information about the dist from the info file that p4a
automatically creates.'''
with current_directory(self.dist_dir):
filen = 'dist_info.json'
if not exists(filen):
return None
with open('dist_info.json', 'r') as fileh:
dist_info = json.load(fileh)
return dist_info
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
if self.ctx.python_recipe.from_crystax:
command = sh.Command('python{}'.format(self.ctx.python_recipe.version))
site_packages_dirs = command(
'-c', 'import site; print("\\n".join(site.getsitepackages()))')
site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n')
# env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
if 'PYTHONPATH' in env:
env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs))
else:
env['PYTHONPATH'] = ':'.join(site_packages_dirs)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
# hostpython = sh.Command('python3.5')
shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
print('cwd is', realpath(curdir))
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
manually_cythonise = False
try:
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
manually_cythonise = True
if manually_cythonise:
self.cythonize_build(env=env)
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
_tail=20, _critical=True, *self.setup_extra_args)
else:
info('First build appeared to complete correctly, skipping manual'
'cythonising.')
if 'python2' in self.ctx.recipe_build_order:
info('Stripping object files')
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
if 'python3crystax' in self.ctx.recipe_build_order:
info('Stripping object files')
shprint(sh.find, '.', '-iname', '*.so', '-exec',
'/usr/bin/echo', '{}', ';', _env=env)
shprint(sh.find, '.', '-iname', '*.so', '-exec',
env['STRIP'].split(' ')[0], '--strip-unneeded',
# '/usr/bin/strip', '--strip-unneeded',
'{}', ';', _env=env)
def build_compiled_components(self, arch):
info("Building compiled components in {}".format(self.name))
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.hostpython_location)
if self.install_in_hostpython:
shprint(hostpython, "setup.py", "clean", "--all", _env=env)
shprint(hostpython, "setup.py", self.build_cmd, "-v", _env=env, *self.setup_extra_args)
build_dir = glob.glob("build/lib.*")[0]
shprint(sh.find, build_dir, "-name", '"*.o"', "-exec", env["STRIP"], "{}", ";", _env=env)
def save_info(self):
'''
Save information about the distribution in its dist_dir.
'''
with current_directory(self.dist_dir):
info('Saving distribution info')
with open('dist_info.json', 'w') as fileh:
json.dump({'dist_name': self.name,
'archs': [arch.arch for arch in self.ctx.archs],
'recipes': self.ctx.recipe_build_order},
fileh)
def build_arch(self, arch, *extra_args):
super(NDKRecipe, self).build_arch(arch)
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
shprint(
sh.ndk_build,
'V=1',
'APP_PLATFORM=android-' + str(self.ctx.ndk_api),
'APP_ABI=' + arch.arch,
*extra_args, _env=env
)
def run_distribute(self):
# print('Default bootstrap being used doesn\'t know how '
# 'to distribute...failing.')
# exit(1)
with current_directory(self.dist_dir):
info('Saving distribution info')
with open('dist_info.json', 'w') as fileh:
json.dump({'dist_name': self.ctx.dist_name,
'bootstrap': self.ctx.bootstrap.name,
'archs': [arch.arch for arch in self.ctx.archs],
'recipes': self.ctx.recipe_build_order + self.ctx.python_modules},
fileh)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
# Build libproto.a
with current_directory(self.get_build_dir(arch.arch)):
env['HOSTARCH'] = 'arm-eabi'
env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('./configure'),
'--host={}'.format(env['HOSTARCH']),
'--enable-shared',
_env=env)
with current_directory(join(self.get_build_dir(arch.arch), 'src')):
shprint(sh.make, 'libprotobuf.la', '-j'+str(cpu_count()), _env=env)
shprint(sh.cp, '.libs/libprotobuf.a', join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a'))
# Copy stl library
shutil.copyfile(
self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so',
join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so'))
# Build python bindings and _message.so
with current_directory(join(self.get_build_dir(arch.arch), 'python')):
hostpython = sh.Command(self.hostpython_location)
shprint(hostpython,
'setup.py',
'build_ext',
'--cpp_implementation', _env=env)
# Install python bindings
self.install_python_package(arch)
# Create __init__.py which is missing (cf. https://github.com/protocolbuffers/protobuf/issues/1296
# and https://stackoverflow.com/questions/13862562/google-protocol-buffers-not-found-when-trying-to-freeze-python-app)
open(join(self.ctx.get_site_packages_dir(), 'google', '__init__.py'), 'a').close()
def prepare_build_dir(self):
'''Ensure that a build dir exists for the recipe. This same single
dir will be used for building all different archs.'''
self.build_dir = self.get_build_dir()
shprint(sh.cp, '-r',
join(self.bootstrap_dir, 'build'),
# join(self.ctx.root_dir,
# 'bootstrap_templates',
# self.name),
self.build_dir)
with current_directory(self.build_dir):
with open('project.properties', 'w') as fileh:
fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_compiled_components(self, arch):
info('Building compiled components in {}'.format(self.name))
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.hostpython_location)
if self.install_in_hostpython:
shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
shprint(hostpython, 'setup.py', self.build_cmd, '-v',
_env=env, *self.setup_extra_args)
build_dir = glob.glob('build/lib.*')[0]
shprint(sh.find, build_dir, '-name', '"*.o"', '-exec',
env['STRIP'], '{}', ';', _env=env)
def get_host(self, arch):
with current_directory(self.get_build_dir(arch.arch)):
host = None
with open("Makefile") as f:
for line in f:
if line.startswith("host = "):
host = line.strip()[7:]
break
if not host or not exists(host):
raise RuntimeError("failed to find build output! ({})".format(host))
return host
def get_host(self, arch):
with current_directory(self.get_build_dir(arch.arch)):
host = None
with open('Makefile') as f:
for line in f:
if line.startswith('host = '):
host = line.strip()[7:]
break
if not host or not exists(host):
raise RuntimeError(
'failed to find build output! ({})'.format(host))
return host
def run_distribute(self):
# print('Default bootstrap being used doesn\'t know how '
# 'to distribute...failing.')
# exit(1)
with current_directory(self.dist_dir):
info('Saving distribution info')
with open('dist_info.json', 'w') as fileh:
json.dump(
{
'dist_name': self.ctx.dist_name,
'bootstrap': self.ctx.bootstrap.name,
'archs': [arch.arch for arch in self.ctx.archs],
'recipes': self.ctx.recipe_build_order
}, fileh)
def unpack(self, arch):
"""Only overwritten to keep the .git directory..."""
info_main('Unpacking {} for {}'.format(self.name, arch))
build_dir = self.get_build_container_dir(arch)
user_dir = os.environ.get('P4A_{}_DIR'.format(self.name.lower()))
if user_dir is not None:
info('P4A_{}_DIR exists, symlinking instead'.format(
self.name.lower()))
if exists(self.get_build_dir(arch)):
return
shprint(sh.rm, '-rf', build_dir)
shprint(sh.mkdir, '-p', build_dir)
shprint(sh.rmdir, build_dir)
ensure_dir(build_dir)
shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
return
if self.url is None:
info('Skipping {} unpack as no URL is set'.format(self.name))
return
filename = shprint(sh.basename,
self.versioned_url).stdout[:-1].decode('utf-8')
ma = match(u'^(.+)#[a-z0-9_]{3,}=([0-9a-f]{32,})$', filename)
if ma: # fragmented URL?
filename = ma.group(1)
with current_directory(build_dir):
directory_name = self.get_build_dir(arch)
if not exists(directory_name) or not isdir(directory_name):
extraction_filename = join(self.ctx.packages_path, self.name,
filename)
if isfile(extraction_filename):
raise NotImplementedError(
"Not implemented, remove custom unpack.")
elif isdir(extraction_filename):
os.mkdir(directory_name)
for entry in listdir(extraction_filename):
shprint(sh.cp, '-Rv', join(extraction_filename, entry),
directory_name)
else:
raise Exception(
'Given path is neither a file nor a directory: {}'.
format(extraction_filename))
else:
info('{} is already unpacked, skipping'.format(self.name))
def prepare_build_dir(self):
"""Ensure that a build dir exists for the recipe. This same single
dir will be used for building all different archs."""
bootstrap_dirs = self.get_bootstrap_dirs()
# now do a cumulative copy of all bootstrap dirs
self.build_dir = self.get_build_dir()
for bootstrap_dir in bootstrap_dirs:
copy_files(join(bootstrap_dir, 'build'),
self.build_dir,
symlink=self.ctx.symlink_bootstrap_files)
with current_directory(self.build_dir):
with open('project.properties', 'w') as fileh:
fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
try:
shprint(hostpython,
'setup.py',
'build_ext',
_env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
info('Running cython where appropriate')
shprint(sh.find,
self.get_build_dir(arch.arch),
'-iname',
'*.pyx',
'-exec',
self.ctx.cython,
'{}',
';',
_env=env)
info('ran cython')
shprint(hostpython,
'setup.py',
'build_ext',
'-v',
_env=env,
_tail=20,
_critical=True,
*self.setup_extra_args)
print('stripping')
build_lib = glob.glob('./build/lib*')
shprint(sh.find,
build_lib[0],
'-name',
'*.o',
'-exec',
env['STRIP'],
'{}',
';',
_env=env)
print('stripped!?')
def build_arch(self, arch):
with current_directory(self.get_build_dir(arch.arch)):
env = self.get_recipe_env(arch)
shprint(sh.Command('./bootstrap'))
configure = sh.Command('./configure')
shprint(configure,
f'--host={arch.command_prefix}',
'--enable-pic',
'--disable-static',
'--enable-shared',
f'--prefix={realpath(".")}',
_env=env)
shprint(sh.make, '-j', str(cpu_count()), _env=env)
shprint(sh.make, 'install', _env=env)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
# Build libproto.a
with current_directory(self.get_build_dir(arch.arch)):
env['HOSTARCH'] = 'arm-eabi'
env['BUILDARCH'] = shprint(
sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('./configure'),
'--host={}'.format(env['HOSTARCH']),
'--enable-shared',
_env=env)
with current_directory(join(self.get_build_dir(arch.arch), 'src')):
shprint(sh.make,
'libprotobuf.la',
'-j' + str(cpu_count()),
_env=env)
shprint(
sh.cp, '.libs/libprotobuf.a',
join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a'))
# Build python bindings and _message.so
with current_directory(join(self.get_build_dir(arch.arch), 'python')):
hostpython = sh.Command(self.hostpython_location)
shprint(hostpython,
'setup.py',
'build_ext',
'--cpp_implementation',
_env=env)
# Install python bindings
self.install_python_package(arch)
def prebuild_arch(self, arch):
super(AndroidRecipe, self).prebuild_arch(arch)
tpxi = 'DEF {} = {}\n'
th = '#define {} {}\n'
tpy = '{} = {}\n'
bootstrap = bootstrap_name = self.ctx.bootstrap.name
is_sdl2 = bootstrap_name in ('sdl2', 'sdl2python3')
is_pygame = bootstrap_name in ('pygame',)
is_webview = bootstrap_name in ('webview',)
if is_sdl2 or is_webview:
if is_sdl2:
bootstrap = 'sdl2'
java_ns = 'org.kivy.android'
jni_ns = 'org/kivy/android'
elif is_pygame:
java_ns = 'org.renpy.android'
jni_ns = 'org/renpy/android'
else:
logger.error('unsupported bootstrap for android recipe: {}'.format(bootstrap_name))
exit(1)
config = {
'BOOTSTRAP': bootstrap,
'IS_SDL2': int(is_sdl2),
'IS_PYGAME': int(is_pygame),
'PY2': int(will_build('python2')(self)),
'JAVA_NAMESPACE': java_ns,
'JNI_NAMESPACE': jni_ns,
}
with current_directory(self.get_build_dir(arch.arch)):
with open(join('android', 'config.pxi'), 'w') as fpxi:
with open(join('android', 'config.h'), 'w') as fh:
with open(join('android', 'config.py'), 'w') as fpy:
for key, value in config.items():
fpxi.write(tpxi.format(key, repr(value)))
fpy.write(tpy.format(key, repr(value)))
fh.write(th.format(key, value if isinstance(value, int)
else '"{}"'.format(value)))
self.config_env[key] = str(value)
if is_sdl2:
fh.write('JNIEnv *SDL_AndroidGetJNIEnv(void);\n')
fh.write('#define SDL_ANDROID_GetJNIEnv SDL_AndroidGetJNIEnv\n')
elif is_pygame:
fh.write('JNIEnv *SDL_ANDROID_GetJNIEnv(void);\n')
def build_arch(self, arch, with_harfbuzz=False):
env = self.get_recipe_env(arch, with_harfbuzz=with_harfbuzz)
harfbuzz_in_recipes = 'harfbuzz' in self.ctx.recipe_build_order
prefix_path = self.get_build_dir(arch.arch)
if harfbuzz_in_recipes and not with_harfbuzz:
# This is the first time we build freetype and we modify `prefix`,
# because we will install the compiled library so later we can
# build harfbuzz (with freetype support) using this freetype
# installation
prefix_path = join(prefix_path, 'install')
# Configure freetype library
config_args = {
'--host={}'.format(arch.command_prefix),
'--prefix={}'.format(prefix_path),
'--without-zlib',
'--without-bzip2',
'--with-png=no',
}
if not harfbuzz_in_recipes:
info('Build freetype (without harfbuzz)')
config_args = config_args.union(
{'--disable-static', '--enable-shared', '--with-harfbuzz=no'})
elif not with_harfbuzz:
info('Build freetype for First time (without harfbuzz)')
# This time we will build our freetype library as static because we
# want that the harfbuzz library to have the necessary freetype
# symbols/functions, so we avoid to have two freetype shared
# libraries which will be confusing and harder to link with them
config_args = config_args.union(
{'--disable-shared', '--with-harfbuzz=no'})
else:
info('Build freetype for Second time (with harfbuzz)')
config_args = config_args.union(
{'--disable-static', '--enable-shared', '--with-harfbuzz=yes'})
info('Configure args are:\n\t-{}'.format('\n\t-'.join(config_args)))
# Build freetype library
with current_directory(self.get_build_dir(arch.arch)):
configure = sh.Command('./configure')
shprint(configure, *config_args, _env=env)
shprint(sh.make, '-j', str(cpu_count()), _env=env)
if not with_harfbuzz and harfbuzz_in_recipes:
info('Installing freetype (first time build without harfbuzz)')
# First build, install the compiled lib, and clean build env
shprint(sh.make, 'install', _env=env)
shprint(sh.make, 'distclean', _env=env)
def prepare_build_dir(self):
'''Ensure that a build dir exists for the recipe. This same single
dir will be used for building all different archs.'''
self.build_dir = self.get_build_dir()
shprint(
sh.cp,
'-r',
join(self.bootstrap_dir, 'build'),
# join(self.ctx.root_dir,
# 'bootstrap_templates',
# self.name),
self.build_dir)
with current_directory(self.build_dir):
with open('project.properties', 'w') as fileh:
fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_arch(self, arch):
env = self.get_recipe_env(arch)
# Build libproto.so
with current_directory(self.get_build_dir(arch.arch)):
build_arch = (shprint(
sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0])
if not exists('configure'):
shprint(sh.Command('./autogen.sh'), _env=env)
shprint(sh.Command('./configure'),
'--build={}'.format(build_arch),
'--host={}'.format(arch.command_prefix),
'--target={}'.format(arch.command_prefix),
'--disable-static',
'--enable-shared',
_env=env)
with current_directory(join(self.get_build_dir(arch.arch), 'src')):
shprint(sh.make,
'libprotobuf.la',
'-j' + str(cpu_count()),
_env=env)
def test_current_directory_exception(self):
"""
Another test for method
:meth:`~pythonforandroid.util.current_directory`, but here we check
that using the method with a non-existing-directory raises an `OSError`
exception.
.. note:: test chdir to non-existing directory, should raise error,
for py3 the exception is FileNotFoundError and IOError for py2, to
avoid introduce conditions, we test with a more generic exception
"""
with self.assertRaises(OSError), util.current_directory(
"/fake/directory"
):
pass
def build_arch(self, arch):
if self.ctx.ndk_api < self.MIN_NDK_API:
raise BuildInterruptingException(
'Target ndk-api is {}, but the python3 recipe supports only'
' {}+'.format(self.ctx.ndk_api, self.MIN_NDK_API))
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, 'android-build')
ensure_dir(build_dir)
# TODO: Get these dynamically, like bpo-30386 does
sys_prefix = '/usr/local'
sys_exec_prefix = '/usr/local'
with current_directory(build_dir):
env = self.get_recipe_env(arch)
env = self.set_libs_flags(env, arch)
android_build = sh.Command(
join(recipe_build_dir,
'config.guess'))().stdout.strip().decode('utf-8')
if not exists('config.status'):
shprint(sh.Command(join(recipe_build_dir, 'configure')),
*(' '.join(self.configure_args).format(
android_host=env['HOSTARCH'],
android_build=android_build,
prefix=sys_prefix,
exec_prefix=sys_exec_prefix)).split(' '),
_env=env)
if not exists('python'):
py_version = self.major_minor_version_string
if self.major_minor_version_string[0] == '3':
py_version += 'm'
shprint(sh.make,
'all',
'-j',
str(cpu_count()),
'INSTSONAME=libpython{version}.so'.format(
version=py_version),
_env=env)
# TODO: Look into passing the path to pyconfig.h in a
# better way, although this is probably acceptable
sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
def test_current_directory(self, moch_chdir):
"""
Basic test for method :meth:`~pythonforandroid.util.current_directory`.
We `mock` chdir and we check that the command is executed once we are
inside a python's `with` statement. Then we check that `chdir has been
called with the proper arguments inside this `with` statement and also
that, once we leave the `with` statement, is called again with the
current working path.
"""
chdir_dir = "/temp/any_directory"
# test chdir to existing directory
with util.current_directory(chdir_dir):
moch_chdir.assert_called_once_with("/temp/any_directory")
moch_chdir.assert_has_calls(
[mock.call("/temp/any_directory"),
mock.call(os.getcwd())])
def prepare_build_dir(self):
'''Ensure that a build dir exists for the recipe. This same single
dir will be used for building all different archs.'''
self.build_dir = self.get_build_dir()
shprint(sh.cp, '-r', join(self.bootstrap_dir, 'build'), self.build_dir)
if self.ctx.symlink_java_src:
info('Symlinking java src instead of copying')
shprint(sh.rm, '-r', join(self.build_dir, 'src'))
shprint(sh.mkdir, join(self.build_dir, 'src'))
for dirn in listdir(join(self.bootstrap_dir, 'build', 'src')):
shprint(sh.ln, '-s',
join(self.bootstrap_dir, 'build', 'src', dirn),
join(self.build_dir, 'src'))
with current_directory(self.build_dir):
with open('project.properties', 'w') as fileh:
fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_arch(self, arch):
if self.ctx.ndk_api < self.MIN_NDK_API:
raise BuildInterruptingException(
NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format(
ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API), )
recipe_build_dir = self.get_build_dir(arch.arch)
# Create a subdirectory to actually perform the build
build_dir = join(recipe_build_dir, "android-build")
ensure_dir(build_dir)
# TODO: Get these dynamically, like bpo-30386 does
sys_prefix = "/usr/local"
sys_exec_prefix = "/usr/local"
env = self.get_recipe_env(arch)
env = self.set_libs_flags(env, arch)
android_build = sh.Command(join(
recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8')
with current_directory(build_dir):
if not exists('config.status'):
shprint(
sh.Command(join(recipe_build_dir, "configure")),
*(" ".join(self.configure_args).format(
android_host=env["HOSTARCH"],
android_build=android_build,
prefix=sys_prefix,
exec_prefix=sys_exec_prefix,
)).split(" "),
_env=env,
)
shprint(
sh.make,
"all",
"-j",
str(cpu_count()),
"INSTSONAME={lib_name}".format(lib_name=self._libpython),
_env=env,
)
# TODO: Look into passing the path to pyconfig.h in a
# better way, although this is probably acceptable
sh.cp("pyconfig.h", join(recipe_build_dir, "Include"))
def prebuild_arch(self, arch):
super(ProtobufCppRecipe, self).prebuild_arch(arch)
patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched')
if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark):
self.apply_patch('fix-python3-compatibility.patch', arch.arch)
shprint(sh.touch, patch_mark)
# During building, host needs to transpile .proto files to .py
# ideally with the same version as protobuf runtime, or with an older one.
# Because protoc is compiled for target (i.e. Android), we need an other binary
# which can be run by host.
# To make it easier, we download prebuild protoc binary adapted to the platform
info_notify("Downloading protoc compiler for your platform")
url_prefix = "https://github.com/protocolbuffers/protobuf/releases/download/v{version}".format(
version=self.version)
if sys.platform.startswith('linux'):
info_notify("GNU/Linux detected")
filename = "protoc-{version}-linux-x86_64.zip".format(
version=self.version)
elif sys.platform.startswith('darwin'):
info_notify("Mac OS X detected")
filename = "protoc-{version}-osx-x86_64.zip".format(
version=self.version)
else:
info_notify("Your platform is not supported, but recipe can still "
"be built if you have a valid protoc (<={version}) in "
"your path".format(version=self.version))
return
protoc_url = join(url_prefix, filename)
self.protoc_dir = join(self.ctx.build_dir, "tools", "protoc")
if os.path.exists(join(self.protoc_dir, "bin", "protoc")):
info_notify("protoc found, no download needed")
return
try:
os.makedirs(self.protoc_dir)
except OSError as e:
# if dir already exists (errno 17), we ignore the error
if e.errno != 17:
raise e
info_notify(
"Will download into {dest_dir}".format(dest_dir=self.protoc_dir))
self.download_file(protoc_url, join(self.protoc_dir, filename))
with current_directory(self.protoc_dir):
shprint(sh.unzip, join(self.protoc_dir, filename))
def strip_object_files(self, arch, env, build_dir=None):
if build_dir is None:
build_dir = self.get_build_dir(arch.arch)
with current_directory(build_dir):
info('Stripping object files')
if self.ctx.python_recipe.name == 'python2legacy':
info('Stripping object files')
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
else:
shprint(sh.find, '.', '-iname', '*.so', '-exec',
'/usr/bin/echo', '{}', ';', _env=env)
shprint(sh.find, '.', '-iname', '*.so', '-exec',
env['STRIP'].split(' ')[0], '--strip-unneeded',
# '/usr/bin/strip', '--strip-unneeded',
'{}', ';', _env=env)
def biglink(ctx, arch):
# First, collate object files from each recipe
info("Collating object files from each recipe")
obj_dir = join(ctx.bootstrap.build_dir, "collated_objects")
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
"objects_{}".format(recipe.name))
if not exists(recipe_obj_dir):
info("{} recipe has no biglinkable files dir, skipping".format(
recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, "*"))
if not len(files):
info("{} recipe has no biglinkable files, skipping".format(
recipe.name))
continue
info("{} recipe has object files, copying".format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, "-r", *files)
env = arch.get_env()
env["LDFLAGS"] = env["LDFLAGS"] + " -L{}".format(
join(ctx.bootstrap.build_dir, "obj", "local", arch.arch))
if not len(glob.glob(join(obj_dir, "*"))):
info("There seem to be no libraries to biglink, skipping.")
return
info("Biglinking")
info("target {}".format(
join(ctx.get_libs_dir(arch.arch), "libpymodules.so")))
do_biglink = copylibs_function if ctx.copy_libs else biglink_function
# Move to the directory containing crtstart_so.o and crtend_so.o
# This is necessary with newer NDKs? A gcc bug?
with current_directory(join(ctx.ndk_platform, "usr", "lib")):
do_biglink(
join(ctx.get_libs_dir(arch.arch), "libpymodules.so"),
obj_dir.split(" "),
extra_link_dirs=[
join(ctx.bootstrap.build_dir, "obj", "local", arch.arch),
os.path.abspath("."),
],
env=env,
)
def build_cython_components(self, arch):
info('Cythonizing anything necessary in {}'.format(self.name))
env = self.get_recipe_env(arch)
if self.ctx.python_recipe.from_crystax:
command = sh.Command('python{}'.format(self.ctx.python_recipe.version))
site_packages_dirs = command(
'-c', 'import site; print("\\n".join(site.getsitepackages()))')
site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n')
# env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
if 'PYTHONPATH' in env:
env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs))
else:
env['PYTHONPATH'] = ':'.join(site_packages_dirs)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
# hostpython = sh.Command('python3.5')
shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
print('cwd is', realpath(curdir))
info('Trying first build of {} to get cython files: this is '
'expected to fail'.format(self.name))
manually_cythonise = False
try:
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
*self.setup_extra_args)
except sh.ErrorReturnCode_1:
print()
info('{} first build failed (as expected)'.format(self.name))
manually_cythonise = True
if manually_cythonise:
self.cythonize_build(env=env)
shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
_tail=20, _critical=True, *self.setup_extra_args)
else:
info('First build appeared to complete correctly, skipping manual'
'cythonising.')
print('stripping')
build_lib = glob.glob('./build/lib*')
shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
env['STRIP'], '{}', ';', _env=env)
print('stripped!?')
def build_arch(self, arch, *extra_args):
super().build_arch(arch)
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
shprint(
sh.cmake,
f'-DCMAKE_TOOLCHAIN_FILE={self.ctx.ndk_dir}/build/cmake/android.toolchain.cmake',
f'-DANDROID_ABI={arch}',
f'-DANDROID_NATIVE_API_LEVEL={self.ctx.ndk_api}',
f'-DCMAKE_BUILD_TYPE=Release CMakeLists.txt',
*extra_args,
_env=env)
shprint(sh.cmake, '--build', '.', _env=env)
# a bit of a dirty workaround
move(join(self.get_build_dir(arch.arch), "armips"),
join(self.get_build_dir(arch.arch), "libarmips.so"))
def build_arch(self, arch):
build_dir = self.get_build_dir(arch.arch)
with current_directory(build_dir):
env = self.get_recipe_env(arch)
build_arch = (shprint(
sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0])
shprint(
sh.Command('./configure'),
'--build=' + build_arch,
'--host=' + arch.command_prefix,
'--target=' + arch.command_prefix,
'--disable-static',
'--enable-shared',
'--prefix={}/install'.format(self.get_build_dir(arch.arch)),
_env=env,
)
shprint(sh.make, '-j', str(cpu_count()), _env=env)
def prebuild_arch(self, arch):
if not self.is_patched(arch):
super().prebuild_arch(arch)
recipe_dir = self.get_build_dir(arch.arch)
# Some versions of reportlab ship with a GPL-licensed font.
# Remove it, since this is problematic in .apks unless the
# entire app is GPL:
font_dir = os.path.join(recipe_dir, "src", "reportlab", "fonts")
if os.path.exists(font_dir):
for file in os.listdir(font_dir):
if file.lower().startswith('darkgarden'):
os.remove(os.path.join(font_dir, file))
# Apply patches:
self.apply_patch('patches/fix-setup.patch', arch.arch)
shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
ft = self.get_recipe('freetype', self.ctx)
ft_dir = ft.get_build_dir(arch.arch)
ft_lib_dir = os.environ.get('_FT_LIB_',
os.path.join(ft_dir, 'objs', '.libs'))
ft_inc_dir = os.environ.get('_FT_INC_',
os.path.join(ft_dir, 'include'))
tmp_dir = os.path.normpath(
os.path.join(recipe_dir, "..", "..", "tmp"))
info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
info('reportlab recipe: ft_dir={}'.format(ft_dir))
info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
with current_directory(recipe_dir):
ensure_dir(tmp_dir)
pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
if not os.path.isfile(pfbfile):
sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip",
"-O", pfbfile)
sh.unzip("-u", "-d",
os.path.join(recipe_dir, "src", "reportlab", "fonts"),
pfbfile)
if os.path.isfile("setup.py"):
with open('setup.py', 'r') as f:
text = f.read().replace('_FT_LIB_',
ft_lib_dir).replace(
'_FT_INC_', ft_inc_dir)
with open('setup.py', 'w') as f:
f.write(text)
def biglink(ctx, arch):
# First, collate object files from each recipe
info('Collating object files from each recipe')
obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
ensure_dir(obj_dir)
recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
for recipe in recipes:
recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
'objects_{}'.format(recipe.name))
if not exists(recipe_obj_dir):
info('{} recipe has no biglinkable files dir, skipping'.format(
recipe.name))
continue
files = glob.glob(join(recipe_obj_dir, '*'))
if not len(files):
info('{} recipe has no biglinkable files, skipping'.format(
recipe.name))
continue
info('{} recipe has object files, copying'.format(recipe.name))
files.append(obj_dir)
shprint(sh.cp, '-r', *files)
env = arch.get_env()
env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))
if not len(glob.glob(join(obj_dir, '*'))):
info('There seem to be no libraries to biglink, skipping.')
return
info('Biglinking')
info('target {}'.format(
join(ctx.get_libs_dir(arch.arch), 'libpymodules.so')))
do_biglink = copylibs_function if ctx.copy_libs else biglink_function
# Move to the directory containing crtstart_so.o and crtend_so.o
# This is necessary with newer NDKs? A gcc bug?
with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):
do_biglink(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
obj_dir.split(' '),
extra_link_dirs=[
join(ctx.bootstrap.build_dir, 'obj', 'local',
arch.arch),
os.path.abspath('.')
],
env=env)
def download(self):
if self.url is None:
info('Skipping {} download as no URL is set'.format(self.name))
return
url = self.versioned_url
shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name))
with current_directory(join(self.ctx.packages_path, self.name)):
filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')
do_download = True
marker_filename = '.mark-{}'.format(filename)
if exists(filename) and isfile(filename):
if not exists(marker_filename):
shprint(sh.rm, filename)
elif self.md5sum:
current_md5 = shprint(sh.md5sum, filename)
print('downloaded md5: {}'.format(current_md5))
print('expected md5: {}'.format(self.md5sum))
print('md5 not handled yet, exiting')
exit(1)
else:
do_download = False
info('{} download already cached, skipping'.format(
self.name))
# Should check headers here!
warning('Should check headers here! Skipping for now.')
# If we got this far, we will download
if do_download:
print('Downloading {} from {}'.format(self.name, url))
shprint(sh.rm, '-f', marker_filename)
self.download_file(url, filename)
shprint(sh.touch, marker_filename)
if self.md5sum is not None:
print('downloaded md5: {}'.format(current_md5))
print('expected md5: {}'.format(self.md5sum))
print('md5 not handled yet, exiting')
exit(1)
def install_python_package(self, arch):
env = self.get_recipe_env(arch)
info('Installing {} into site-packages'.format(self.name))
with current_directory(join(self.get_build_dir(arch.arch), 'python')):
hostpython = sh.Command(self.hostpython_location)
hpenv = env.copy()
shprint(hostpython,
'setup.py',
'install',
'-O2',
'--root={}'.format(self.ctx.get_python_install_dir()),
'--install-lib=.',
'--cpp_implementation',
_env=hpenv,
*self.setup_extra_args)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(self.get_build_dir(arch.arch)):
# sh fails with code 255 trying to execute ./Configure
# so instead we manually run perl passing in Configure
perl = sh.Command('perl')
buildarch = self.select_build_arch(arch)
config_args = [
'shared',
'no-dso',
'no-asm',
buildarch,
'-D__ANDROID_API__={}'.format(self.ctx.ndk_api),
]
shprint(perl, 'Configure', *config_args, _env=env)
self.apply_patch('disable-sover.patch', arch.arch)
shprint(sh.make, 'build_libs', _env=env)
def save_info(self, dirn):
'''
Save information about the distribution in its dist_dir.
'''
with current_directory(dirn):
info('Saving distribution info')
with open('dist_info.json', 'w') as fileh:
json.dump(
{
'dist_name':
self.ctx.dist_name,
'bootstrap':
self.ctx.bootstrap.name,
'archs': [arch.arch for arch in self.ctx.archs],
'ndk_api':
self.ctx.ndk_api,
'recipes':
self.ctx.recipe_build_order + self.ctx.python_modules
}, fileh)
def build_arch(self, arch):
env = self.get_recipe_env(arch)
with current_directory(
join(self.get_build_dir(arch.arch), 'libmysqlclient')):
shprint(sh.cp, '-t', '.', join(self.get_recipe_dir(), 'p4a.cmake'))
# shprint(sh.mkdir, 'Platform')
# shprint(sh.cp, '-t', 'Platform', join(self.get_recipe_dir(), 'Linux.cmake'))
shprint(sh.rm, '-f', 'CMakeCache.txt')
shprint(
sh.cmake,
'-G',
'Unix Makefiles',
# '-DCMAKE_MODULE_PATH=' + join(self.get_build_dir(arch.arch), 'libmysqlclient'),
'-DCMAKE_INSTALL_PREFIX=./install',
'-DCMAKE_TOOLCHAIN_FILE=p4a.cmake',
_env=env)
shprint(sh.make, _env=env)
self.install_libs(arch, join('libmysql', 'libmysql.so'))
def build_arch(self, arch):
env = self.get_recipe_env(arch)
openssl_recipe = self.get_recipe('openssl', self.ctx)
openssl_dir = openssl_recipe.get_build_dir(arch.arch)
env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch)
env['LIBS'] = env.get('LIBS', '') + openssl_recipe.link_libs_flags()
with current_directory(self.get_build_dir(arch.arch)):
dst_dir = join(self.get_build_dir(arch.arch), 'dist')
shprint(sh.Command('./configure'),
'--host=arm-linux-androideabi',
'--enable-shared',
'--with-ssl={}'.format(openssl_dir),
'--prefix={}'.format(dst_dir),
_env=env)
shprint(sh.make, '-j', str(cpu_count()), _env=env)
shprint(sh.make, 'install', _env=env)