def build_arch(self, arch): env = self.get_recipe_env(arch) # Build libproto.a with current_directory(self.get_build_dir(arch.arch)): env['HOSTARCH'] = 'arm-eabi' env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0] if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('./configure'), '--host={}'.format(env['HOSTARCH']), '--enable-shared', _env=env) with current_directory(join(self.get_build_dir(arch.arch), 'src')): shprint(sh.make, 'libprotobuf.la', '-j'+str(cpu_count()), _env=env) shprint(sh.cp, '.libs/libprotobuf.a', join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a')) # Copy stl library shutil.copyfile( self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so', join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so')) # Build python bindings and _message.so with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) shprint(hostpython, 'setup.py', 'build_ext', '--cpp_implementation', _env=env) # Install python bindings self.install_python_package(arch)
def build_arch(self, arch): recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, self.build_subdir) ensure_dir(build_dir) if not exists(join(build_dir, 'python')): with current_directory(recipe_build_dir): # Configure the build with current_directory(build_dir): if not exists('config.status'): shprint( sh.Command(join(recipe_build_dir, 'configure'))) # Create the Setup file. This copying from Setup.dist # seems to be the normal and expected procedure. shprint(sh.cp, join('Modules', 'Setup.dist'), join(build_dir, 'Modules', 'Setup')) result = shprint(sh.make, '-C', build_dir) else: info('Skipping {name} ({version}) build, as it has already ' 'been completed'.format(name=self.name, version=self.version)) self.ctx.hostpython = join(build_dir, 'python')
def download_file(self, url, target, cwd=None): """ (internal) Download an ``url`` to a ``target``. """ if not url: return info('Downloading {} from {}'.format(self.name, url)) if cwd: target = join(cwd, target) parsed_url = urlparse(url) if parsed_url.scheme in ('http', 'https'): def report_hook(index, blksize, size): if size <= 0: progression = '{0} bytes'.format(index * blksize) else: progression = '{0:.2f}%'.format( index * blksize * 100. / float(size)) if "CI" not in environ: stdout.write('- Download {}\r'.format(progression)) stdout.flush() if exists(target): unlink(target) # Download item with multiple attempts (for bad connections): attempts = 0 while True: try: urlretrieve(url, target, report_hook) except OSError as e: attempts += 1 if attempts >= 5: raise e stdout.write('Download failed retrying in a second...') time.sleep(1) continue break return target elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http', 'git+https'): if isdir(target): with current_directory(target): shprint(sh.git, 'fetch', '--tags') if self.version: shprint(sh.git, 'checkout', self.version) shprint(sh.git, 'pull') shprint(sh.git, 'pull', '--recurse-submodules') shprint(sh.git, 'submodule', 'update', '--recursive') else: if url.startswith('git+'): url = url[4:] shprint(sh.git, 'clone', '--recursive', url, target) if self.version: with current_directory(target): shprint(sh.git, 'checkout', self.version) shprint(sh.git, 'submodule', 'update', '--recursive') return target
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('autoreconf'), '-vif', _env=env) shprint(sh.Command('./configure'), '--host=' + arch.command_prefix, '--prefix=' + self.get_build_dir(arch.arch), '--disable-builddir', '--enable-shared', _env=env) # '--with-sysroot={}'.format(self.ctx.ndk_platform), # '--target={}'.format(arch.toolchain_prefix), # ndk 15 introduces unified headers required --sysroot and # -isysroot for libraries and headers. libtool's head explodes # trying to weave them into it's own magic. The result is a link # failure trying to link libc. We call make to compile the bits # and manually link... try: shprint(sh.make, '-j5', 'libffi.la', _env=env) except sh.ErrorReturnCode_2: info("make libffi.la failed as expected") cc = sh.Command(env['CC'].split()[0]) cflags = env['CC'].split()[1:] host_build = self.get_build_dir(arch.arch) arch_flags = '' if '-march=' in env['CFLAGS']: arch_flags = '-march={}'.format(env['CFLAGS'].split('-march=')[1]) src_arch = arch.command_prefix.split('-')[0] if src_arch == 'x86_64': # libffi has not specific arch files for x86_64...so...using # the ones from x86 which seems to build fine... src_arch = 'x86' cflags.extend(arch_flags.split()) cflags.extend(['-shared', '-fPIC', '-DPIC']) cflags.extend(glob(join(host_build, 'src/.libs/*.o'))) cflags.extend(glob(join(host_build, 'src', src_arch, '.libs/*.o'))) ldflags = env['LDFLAGS'].split() cflags.extend(ldflags) cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o', '.libs/libffi.so']) with current_directory(host_build): shprint(cc, *cflags, _env=env) ensure_dir(self.ctx.get_libs_dir(arch.arch)) shprint(sh.cp, join(host_build, '.libs', 'libffi.so'), self.ctx.get_libs_dir(arch.arch))
def build_arch(self, arch): env = self.get_recipe_env(arch) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format( jni_path=join(self.ctx.bootstrap.build_dir, 'jni')) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl/include -I{jni_path}/sdl_mixer'.format( jni_path=join(self.ctx.bootstrap.build_dir, 'jni')) env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl_ttf -I{jni_path}/sdl_image'.format( jni_path=join(self.ctx.bootstrap.build_dir, 'jni')) debug('pygame cflags', env['CFLAGS']) env['LDFLAGS'] = env['LDFLAGS'] + ' -L{libs_path} -L{src_path}/obj/local/{arch} -lm -lz'.format( libs_path=self.ctx.libs_dir, src_path=self.ctx.bootstrap.build_dir, arch=env['ARCH']) env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink') with current_directory(self.get_build_dir(arch.arch)): info('hostpython is ' + self.ctx.hostpython) hostpython = sh.Command(self.ctx.hostpython) shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, _tail=10, _critical=True) info('strip is ' + env['STRIP']) build_lib = glob.glob('./build/lib*') assert len(build_lib) == 1 print('stripping pygame') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';') python_install_path = join(self.ctx.build_dir, 'python-install') warning('Should remove pygame tests etc. here, but skipping for now')
def prebuild_arch(self, arch): if not self.is_patched(arch): super(ReportLabRecipe, self).prebuild_arch(arch) self.apply_patch('patches/fix-setup.patch', arch.arch) recipe_dir = self.get_build_dir(arch.arch) shprint(sh.touch, os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include')) tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp")) info('reportlab recipe: recipe_dir={}'.format(recipe_dir)) info('reportlab recipe: tmp_dir={}'.format(tmp_dir)) info('reportlab recipe: ft_dir={}'.format(ft_dir)) info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir)) info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir)) with current_directory(recipe_dir): sh.ls('-lathr') ensure_dir(tmp_dir) pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip") if not os.path.isfile(pfbfile): sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile) sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile) if os.path.isfile("setup.py"): with open('setup.py', 'rb') as f: text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir) with open('setup.py', 'wb') as f: f.write(text)
def apk(self, args): '''Create an APK using the given distribution.''' # AND: Need to add a parser here for any extra options # parser = argparse.ArgumentParser( # description='Build an APK') # args = parser.parse_args(args) ctx = self.ctx dist = self._dist # Manually fixing these arguments at the string stage is # unsatisfactory and should probably be changed somehow, but # we can't leave it until later as the build.py scripts assume # they are in the current directory. for i, arg in enumerate(args[:-1]): if arg in ('--dir', '--private'): args[i+1] = realpath(expanduser(args[i+1])) build = imp.load_source('build', join(dist.dist_dir, 'build.py')) with current_directory(dist.dist_dir): build.parse_args(args) shprint(sh.ant, 'debug', _tail=20, _critical=True) # AND: This is very crude, needs improving. Also only works # for debug for now. info_main('# Copying APK to current directory') apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-debug.apk')) if len(apks) == 0: raise ValueError('Couldn\'t find the built APK') if len(apks) > 1: info('More than one built APK found...guessing you ' 'just built {}'.format(apks[-1])) shprint(sh.cp, apks[-1], './')
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env) debug('cwd is {}'.format(realpath(curdir))) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) manually_cythonise = False try: shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) manually_cythonise = True if manually_cythonise: self.cythonize_build(env=env) shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) else: info('First build appeared to complete correctly, skipping manual' 'cythonising.') self.strip_object_files(arch, env)
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('./configure'), '--host=' + arch.toolchain_prefix, '--prefix=' + self.ctx.get_python_install_dir(), '--enable-shared', _env=env) shprint(sh.make, '-j5', 'libffi.la', _env=env) # dlname = None # with open(join(host, 'libffi.la')) as f: # for line in f: # if line.startswith('dlname='): # dlname = line.strip()[8:-1] # break # # if not dlname or not exists(join(host, '.libs', dlname)): # raise RuntimeError('failed to locate shared object! ({})' # .format(dlname)) # shprint(sh.sed, '-i', 's/^dlname=.*$/dlname=\'libffi.so\'/', join(host, 'libffi.la')) shprint(sh.cp, '-t', self.ctx.get_libs_dir(arch.arch), join(self.get_host(arch), '.libs', 'libffi.so')) #,
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) try: shprint(hostpython, 'setup.py', 'build_ext', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) info('Running cython where appropriate') shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx', '-exec', self.ctx.cython, '{}', ';', _env=env) info('ran cython') shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) print('stripping') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) print('stripped!?')
def run_pymodules_install(ctx, modules): modules = filter(ctx.not_has_package, modules) if not modules: info('There are no Python modules to install, skipping') return info('The requirements ({}) don\'t have recipes, attempting to install ' 'them with pip'.format(', '.join(modules))) info('If this fails, it may mean that the module has compiled ' 'components and needs a recipe.') venv = sh.Command(ctx.virtualenv) with current_directory(join(ctx.build_dir)): shprint(venv, '--python=python2.7', 'venv') info('Creating a requirements.txt file for the Python modules') with open('requirements.txt', 'w') as fileh: for module in modules: fileh.write('{}\n'.format(module)) info('Installing Python modules with pip') info('If this fails with a message about /bin/false, this ' 'probably means the package cannot be installed with ' 'pip as it needs a compilation recipe.') # This bash method is what old-p4a used # It works but should be replaced with something better shprint(sh.bash, '-c', ( "source venv/bin/activate && env CC=/bin/false CXX=/bin/false " "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt" ).format(ctx.get_site_packages_dir()))
def install_python_package(self, arch): env = self.get_recipe_env(arch) info('Installing {} into site-packages'.format(self.name)) with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) if self.ctx.python_recipe.from_crystax: hpenv = env.copy() shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', '--cpp_implementation', _env=hpenv, *self.setup_extra_args) else: hppath = join(dirname(self.hostpython_location), 'Lib', 'site-packages') hpenv = env.copy() if 'PYTHONPATH' in hpenv: hpenv['PYTHONPATH'] = ':'.join([hppath] + hpenv['PYTHONPATH'].split(':')) else: hpenv['PYTHONPATH'] = hppath shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=lib/python2.7/site-packages', '--cpp_implementation', _env=hpenv, *self.setup_extra_args)
def prebuild_arch(self, arch): super(AndroidRecipe, self).prebuild_arch(arch) ctx_bootstrap = self.ctx.bootstrap.name # define macros for Cython, C, Python tpxi = 'DEF {} = {}\n' th = '#define {} {}\n' tpy = '{} = {}\n' # make sure bootstrap name is in unicode if isinstance(ctx_bootstrap, bytes): ctx_bootstrap = ctx_bootstrap.decode('utf-8') bootstrap = bootstrap_name = ctx_bootstrap is_sdl2 = bootstrap_name in ('sdl2', 'sdl2python3', 'sdl2_gradle') is_webview = bootstrap_name in ('webview',) if is_sdl2 or is_webview: if is_sdl2: bootstrap = 'sdl2' java_ns = u'org.kivy.android' jni_ns = u'org/kivy/android' else: logger.error(( 'unsupported bootstrap for android recipe: {}' ''.format(bootstrap_name) )) exit(1) config = { 'BOOTSTRAP': bootstrap, 'IS_SDL2': int(is_sdl2), 'PY2': int(will_build('python2')(self)), 'JAVA_NAMESPACE': java_ns, 'JNI_NAMESPACE': jni_ns, } # create config files for Cython, C and Python with ( current_directory(self.get_build_dir(arch.arch))), ( open(join('android', 'config.pxi'), 'w')) as fpxi, ( open(join('android', 'config.h'), 'w')) as fh, ( open(join('android', 'config.py'), 'w')) as fpy: for key, value in config.items(): fpxi.write(tpxi.format(key, repr(value))) fpy.write(tpy.format(key, repr(value))) fh.write(th.format( key, value if isinstance(value, int) else '"{}"'.format(value) )) self.config_env[key] = str(value) if is_sdl2: fh.write('JNIEnv *SDL_AndroidGetJNIEnv(void);\n') fh.write( '#define SDL_ANDROID_GetJNIEnv SDL_AndroidGetJNIEnv\n' )
def install_python_package(self, arch, name=None, env=None, is_dir=True): '''Automate the installation of a Python package (or a cython package where the cython components are pre-built).''' # arch = self.filtered_archs[0] # old kivy-ios way if name is None: name = self.name if env is None: env = self.get_recipe_env(arch) info('Installing {} into site-packages'.format(self.name)) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) # hostpython = sh.Command('python3.5') if self.ctx.python_recipe.from_crystax: # hppath = join(dirname(self.hostpython_location), 'Lib', # 'site-packages') hpenv = env.copy() # if 'PYTHONPATH' in hpenv: # hpenv['PYTHONPATH'] = ':'.join([hppath] + # hpenv['PYTHONPATH'].split(':')) # else: # hpenv['PYTHONPATH'] = hppath # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir() # shprint(hostpython, 'setup.py', 'build', # _env=hpenv, *self.setup_extra_args) shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions) _env=hpenv, *self.setup_extra_args) # site_packages_dir = self.ctx.get_site_packages_dir() # built_files = glob.glob(join('build', 'lib*', '*')) # for filen in built_files: # shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1])) elif self.call_hostpython_via_targetpython: shprint(hostpython, 'setup.py', 'install', '-O2', _env=env, *self.setup_extra_args) else: hppath = join(dirname(self.hostpython_location), 'Lib', 'site-packages') hpenv = env.copy() if 'PYTHONPATH' in hpenv: hpenv['PYTHONPATH'] = ':'.join([hppath] + hpenv['PYTHONPATH'].split(':')) else: hpenv['PYTHONPATH'] = hppath shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=lib/python2.7/site-packages', _env=hpenv, *self.setup_extra_args) # AND: Hardcoded python2.7 needs fixing # If asked, also install in the hostpython build dir if self.install_in_hostpython: self.install_hostpython_package(arch)
def download(self): if self.url is None: info('Skipping {} download as no URL is set'.format(self.name)) return url = self.versioned_url ma = match(u'^(.+)#md5=([0-9a-f]{32})$', url) if ma: # fragmented URL? if self.md5sum: raise ValueError( ('Received md5sum from both the {} recipe ' 'and its url').format(self.name)) url = ma.group(1) expected_md5 = ma.group(2) else: expected_md5 = self.md5sum shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name)) with current_directory(join(self.ctx.packages_path, self.name)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') do_download = True marker_filename = '.mark-{}'.format(filename) if exists(filename) and isfile(filename): if not exists(marker_filename): shprint(sh.rm, filename) elif expected_md5: current_md5 = md5sum(filename) if current_md5 != expected_md5: debug('* Generated md5sum: {}'.format(current_md5)) debug('* Expected md5sum: {}'.format(expected_md5)) raise ValueError( ('Generated md5sum does not match expected md5sum ' 'for {} recipe').format(self.name)) do_download = False else: do_download = False # If we got this far, we will download if do_download: debug('Downloading {} from {}'.format(self.name, url)) shprint(sh.rm, '-f', marker_filename) self.download_file(self.versioned_url, filename) shprint(sh.touch, marker_filename) if exists(filename) and isfile(filename) and expected_md5: current_md5 = md5sum(filename) if expected_md5 is not None: if current_md5 != expected_md5: debug('* Generated md5sum: {}'.format(current_md5)) debug('* Expected md5sum: {}'.format(expected_md5)) raise ValueError( ('Generated md5sum does not match expected md5sum ' 'for {} recipe').format(self.name)) else: info('{} download already cached, skipping'.format(self.name))
def download_file(self, url, target, cwd=None): """ (internal) Download an ``url`` to a ``target``. """ if not url: return info('Downloading {} from {}'.format(self.name, url)) if cwd: target = join(cwd, target) parsed_url = urlparse(url) if parsed_url.scheme in ('http', 'https'): def report_hook(index, blksize, size): if size <= 0: progression = '{0} bytes'.format(index * blksize) else: progression = '{0:.2f}%'.format( index * blksize * 100. / float(size)) stdout.write('- Download {}\r'.format(progression)) stdout.flush() if exists(target): unlink(target) urlretrieve(url, target, report_hook) return target elif parsed_url.scheme in ('git', 'git+ssh', 'git+http', 'git+https'): if isdir(target): with current_directory(target): shprint(sh.git, 'fetch', '--tags') if self.version: shprint(sh.git, 'checkout', self.version) shprint(sh.git, 'pull') shprint(sh.git, 'pull', '--recurse-submodules') shprint(sh.git, 'submodule', 'update', '--recursive') else: if url.startswith('git+'): url = url[4:] shprint(sh.git, 'clone', '--recursive', url, target) if self.version: with current_directory(target): shprint(sh.git, 'checkout', self.version) shprint(sh.git, 'submodule', 'update', '--recursive') return target
def download_file(self, url, target, cwd=None): """ (internal) Download an ``url`` to a ``target``. """ if not url: return info("Downloading {} from {}".format(self.name, url)) if cwd: target = join(cwd, target) parsed_url = urlparse(url) if parsed_url.scheme in ("http", "https"): def report_hook(index, blksize, size): if size <= 0: progression = "{0} bytes".format(index * blksize) else: progression = "{0:.2f}%".format(index * blksize * 100.0 / float(size)) stdout.write("- Download {}\r".format(progression)) stdout.flush() if exists(target): unlink(target) urlretrieve(url, target, report_hook) return target elif parsed_url.scheme in ("git", "git+ssh", "git+http", "git+https"): if isdir(target): with current_directory(target): shprint(sh.git, "fetch", "--tags") if self.version: shprint(sh.git, "checkout", self.version) shprint(sh.git, "pull") shprint(sh.git, "pull", "--recurse-submodules") shprint(sh.git, "submodule", "update", "--recursive") else: if url.startswith("git+"): url = url[4:] shprint(sh.git, "clone", "--recursive", url, target) if self.version: with current_directory(target): shprint(sh.git, "checkout", self.version) shprint(sh.git, "submodule", "update", "--recursive") return target
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) if self.ctx.python_recipe.from_crystax: command = sh.Command('python{}'.format(self.ctx.python_recipe.version)) site_packages_dirs = command( '-c', 'import site; print("\\n".join(site.getsitepackages()))') site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n') # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5' if 'PYTHONPATH' in env: env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs)) else: env['PYTHONPATH'] = ':'.join(site_packages_dirs) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) # hostpython = sh.Command('python3.5') shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env) print('cwd is', realpath(curdir)) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) manually_cythonise = False try: shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) manually_cythonise = True if manually_cythonise: info('Running cython where appropriate') cyenv = env.copy() if 'CYTHONPATH' in cyenv: cyenv['PYTHONPATH'] = cyenv['CYTHONPATH'] elif 'PYTHONPATH' in cyenv: del cyenv['PYTHONPATH'] cython = 'cython' if self.ctx.python_recipe.from_crystax else self.ctx.cython cython_cmd = 'find "{}" -iname *.pyx | xargs "{}"'.format( self.get_build_dir(arch.arch), cython) shprint(sh.sh, '-c', cython_cmd, _env=cyenv) info('ran cython') shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) else: info('First build appeared to complete correctly, skipping manual' 'cythonising.') print('stripping') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) print('stripped!?')
def build_compiled_components(self,arch): super(CppCompiledComponentsPythonRecipe, self).build_compiled_components(arch) # Copy libgnustl_shared.so with current_directory(self.get_build_dir(arch.arch)): sh.cp( "{ctx.ndk_dir}/sources/cxx-stl/gnu-libstdc++/{ctx.toolchain_version}/libs/{arch.arch}/libgnustl_shared.so".format(ctx=self.ctx,arch=arch), self.ctx.get_libs_dir(arch.arch) )
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('autoreconf'), '-vif', _env=env) shprint(sh.Command('./configure'), '--host=' + arch.toolchain_prefix, '--prefix=' + self.ctx.get_python_install_dir(), '--enable-shared', _env=env) #'--with-sysroot={}'.format(self.ctx.ndk_platform), #'--target={}'.format(arch.toolchain_prefix), # ndk 15 introduces unified headers required --sysroot and # -isysroot for libraries and headers. libtool's head explodes # trying to weave them into it's own magic. The result is a link # failure tryng to link libc. We call make to compile the bits # and manually link... try: shprint(sh.make, '-j5', 'libffi.la', _env=env) except sh.ErrorReturnCode_2: info("make libffi.la failed as expected") cc = sh.Command(env['CC'].split()[0]) cflags = env['CC'].split()[1:] cflags.extend(['-march=armv7-a', '-mfloat-abi=softfp', '-mfpu=vfp', '-mthumb', '-shared', '-fPIC', '-DPIC', 'src/.libs/prep_cif.o', 'src/.libs/types.o', 'src/.libs/raw_api.o', 'src/.libs/java_raw_api.o', 'src/.libs/closures.o', 'src/arm/.libs/sysv.o', 'src/arm/.libs/ffi.o', ] ) ldflags = env['LDFLAGS'].split() cflags.extend(ldflags) cflags.extend(['-Wl,-soname', '-Wl,libffi.so', '-o', '.libs/libffi.so']) with current_directory(self.get_host(arch)): shprint(cc, *cflags, _env=env) shprint(sh.cp, '-t', self.ctx.get_libs_dir(arch.arch), join(self.get_host(arch), '.libs', 'libffi.so'))
def load_info(self): '''Load information about the dist from the info file that p4a automatically creates.''' with current_directory(self.dist_dir): filen = 'dist_info.json' if not exists(filen): return None with open('dist_info.json', 'r') as fileh: dist_info = json.load(fileh) return dist_info
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) if self.ctx.python_recipe.from_crystax: command = sh.Command('python{}'.format(self.ctx.python_recipe.version)) site_packages_dirs = command( '-c', 'import site; print("\\n".join(site.getsitepackages()))') site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n') # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5' if 'PYTHONPATH' in env: env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs)) else: env['PYTHONPATH'] = ':'.join(site_packages_dirs) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) # hostpython = sh.Command('python3.5') shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env) print('cwd is', realpath(curdir)) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) manually_cythonise = False try: shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) manually_cythonise = True if manually_cythonise: self.cythonize_build(env=env) shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) else: info('First build appeared to complete correctly, skipping manual' 'cythonising.') if 'python2' in self.ctx.recipe_build_order: info('Stripping object files') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) if 'python3crystax' in self.ctx.recipe_build_order: info('Stripping object files') shprint(sh.find, '.', '-iname', '*.so', '-exec', '/usr/bin/echo', '{}', ';', _env=env) shprint(sh.find, '.', '-iname', '*.so', '-exec', env['STRIP'].split(' ')[0], '--strip-unneeded', # '/usr/bin/strip', '--strip-unneeded', '{}', ';', _env=env)
def build_compiled_components(self, arch): info("Building compiled components in {}".format(self.name)) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) if self.install_in_hostpython: shprint(hostpython, "setup.py", "clean", "--all", _env=env) shprint(hostpython, "setup.py", self.build_cmd, "-v", _env=env, *self.setup_extra_args) build_dir = glob.glob("build/lib.*")[0] shprint(sh.find, build_dir, "-name", '"*.o"', "-exec", env["STRIP"], "{}", ";", _env=env)
def save_info(self): ''' Save information about the distribution in its dist_dir. ''' with current_directory(self.dist_dir): info('Saving distribution info') with open('dist_info.json', 'w') as fileh: json.dump({'dist_name': self.name, 'archs': [arch.arch for arch in self.ctx.archs], 'recipes': self.ctx.recipe_build_order}, fileh)
def build_arch(self, arch, *extra_args): super(NDKRecipe, self).build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): shprint( sh.ndk_build, 'V=1', 'APP_PLATFORM=android-' + str(self.ctx.ndk_api), 'APP_ABI=' + arch.arch, *extra_args, _env=env )
def run_distribute(self): # print('Default bootstrap being used doesn\'t know how ' # 'to distribute...failing.') # exit(1) with current_directory(self.dist_dir): info('Saving distribution info') with open('dist_info.json', 'w') as fileh: json.dump({'dist_name': self.ctx.dist_name, 'bootstrap': self.ctx.bootstrap.name, 'archs': [arch.arch for arch in self.ctx.archs], 'recipes': self.ctx.recipe_build_order + self.ctx.python_modules}, fileh)
def build_arch(self, arch): env = self.get_recipe_env(arch) # Build libproto.a with current_directory(self.get_build_dir(arch.arch)): env['HOSTARCH'] = 'arm-eabi' env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0] if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('./configure'), '--host={}'.format(env['HOSTARCH']), '--enable-shared', _env=env) with current_directory(join(self.get_build_dir(arch.arch), 'src')): shprint(sh.make, 'libprotobuf.la', '-j'+str(cpu_count()), _env=env) shprint(sh.cp, '.libs/libprotobuf.a', join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a')) # Copy stl library shutil.copyfile( self.ctx.ndk_dir + '/sources/cxx-stl/gnu-libstdc++/' + self.ctx.toolchain_version + '/libs/' + arch.arch + '/libgnustl_shared.so', join(self.ctx.get_libs_dir(arch.arch), 'libgnustl_shared.so')) # Build python bindings and _message.so with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) shprint(hostpython, 'setup.py', 'build_ext', '--cpp_implementation', _env=env) # Install python bindings self.install_python_package(arch) # Create __init__.py which is missing (cf. https://github.com/protocolbuffers/protobuf/issues/1296 # and https://stackoverflow.com/questions/13862562/google-protocol-buffers-not-found-when-trying-to-freeze-python-app) open(join(self.ctx.get_site_packages_dir(), 'google', '__init__.py'), 'a').close()
def prepare_build_dir(self): '''Ensure that a build dir exists for the recipe. This same single dir will be used for building all different archs.''' self.build_dir = self.get_build_dir() shprint(sh.cp, '-r', join(self.bootstrap_dir, 'build'), # join(self.ctx.root_dir, # 'bootstrap_templates', # self.name), self.build_dir) with current_directory(self.build_dir): with open('project.properties', 'w') as fileh: fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_compiled_components(self, arch): info('Building compiled components in {}'.format(self.name)) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.hostpython_location) if self.install_in_hostpython: shprint(hostpython, 'setup.py', 'clean', '--all', _env=env) shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env, *self.setup_extra_args) build_dir = glob.glob('build/lib.*')[0] shprint(sh.find, build_dir, '-name', '"*.o"', '-exec', env['STRIP'], '{}', ';', _env=env)
def get_host(self, arch): with current_directory(self.get_build_dir(arch.arch)): host = None with open("Makefile") as f: for line in f: if line.startswith("host = "): host = line.strip()[7:] break if not host or not exists(host): raise RuntimeError("failed to find build output! ({})".format(host)) return host
def get_host(self, arch): with current_directory(self.get_build_dir(arch.arch)): host = None with open('Makefile') as f: for line in f: if line.startswith('host = '): host = line.strip()[7:] break if not host or not exists(host): raise RuntimeError( 'failed to find build output! ({})'.format(host)) return host
def run_distribute(self): # print('Default bootstrap being used doesn\'t know how ' # 'to distribute...failing.') # exit(1) with current_directory(self.dist_dir): info('Saving distribution info') with open('dist_info.json', 'w') as fileh: json.dump( { 'dist_name': self.ctx.dist_name, 'bootstrap': self.ctx.bootstrap.name, 'archs': [arch.arch for arch in self.ctx.archs], 'recipes': self.ctx.recipe_build_order }, fileh)
def unpack(self, arch): """Only overwritten to keep the .git directory...""" info_main('Unpacking {} for {}'.format(self.name, arch)) build_dir = self.get_build_container_dir(arch) user_dir = os.environ.get('P4A_{}_DIR'.format(self.name.lower())) if user_dir is not None: info('P4A_{}_DIR exists, symlinking instead'.format( self.name.lower())) if exists(self.get_build_dir(arch)): return shprint(sh.rm, '-rf', build_dir) shprint(sh.mkdir, '-p', build_dir) shprint(sh.rmdir, build_dir) ensure_dir(build_dir) shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch)) return if self.url is None: info('Skipping {} unpack as no URL is set'.format(self.name)) return filename = shprint(sh.basename, self.versioned_url).stdout[:-1].decode('utf-8') ma = match(u'^(.+)#[a-z0-9_]{3,}=([0-9a-f]{32,})$', filename) if ma: # fragmented URL? filename = ma.group(1) with current_directory(build_dir): directory_name = self.get_build_dir(arch) if not exists(directory_name) or not isdir(directory_name): extraction_filename = join(self.ctx.packages_path, self.name, filename) if isfile(extraction_filename): raise NotImplementedError( "Not implemented, remove custom unpack.") elif isdir(extraction_filename): os.mkdir(directory_name) for entry in listdir(extraction_filename): shprint(sh.cp, '-Rv', join(extraction_filename, entry), directory_name) else: raise Exception( 'Given path is neither a file nor a directory: {}'. format(extraction_filename)) else: info('{} is already unpacked, skipping'.format(self.name))
def prepare_build_dir(self): """Ensure that a build dir exists for the recipe. This same single dir will be used for building all different archs.""" bootstrap_dirs = self.get_bootstrap_dirs() # now do a cumulative copy of all bootstrap dirs self.build_dir = self.get_build_dir() for bootstrap_dir in bootstrap_dirs: copy_files(join(bootstrap_dir, 'build'), self.build_dir, symlink=self.ctx.symlink_bootstrap_files) with current_directory(self.build_dir): with open('project.properties', 'w') as fileh: fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) shprint(sh.Command('./bootstrap')) configure = sh.Command('./configure') shprint(configure, f'--host={arch.command_prefix}', '--enable-pic', '--disable-static', '--enable-shared', f'--prefix={realpath(".")}', _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env)
def build_arch(self, arch): env = self.get_recipe_env(arch) # Build libproto.a with current_directory(self.get_build_dir(arch.arch)): env['HOSTARCH'] = 'arm-eabi' env['BUILDARCH'] = shprint( sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0] if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('./configure'), '--host={}'.format(env['HOSTARCH']), '--enable-shared', _env=env) with current_directory(join(self.get_build_dir(arch.arch), 'src')): shprint(sh.make, 'libprotobuf.la', '-j' + str(cpu_count()), _env=env) shprint( sh.cp, '.libs/libprotobuf.a', join(self.ctx.get_libs_dir(arch.arch), 'libprotobuf.a')) # Build python bindings and _message.so with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) shprint(hostpython, 'setup.py', 'build_ext', '--cpp_implementation', _env=env) # Install python bindings self.install_python_package(arch)
def prebuild_arch(self, arch): super(AndroidRecipe, self).prebuild_arch(arch) tpxi = 'DEF {} = {}\n' th = '#define {} {}\n' tpy = '{} = {}\n' bootstrap = bootstrap_name = self.ctx.bootstrap.name is_sdl2 = bootstrap_name in ('sdl2', 'sdl2python3') is_pygame = bootstrap_name in ('pygame',) is_webview = bootstrap_name in ('webview',) if is_sdl2 or is_webview: if is_sdl2: bootstrap = 'sdl2' java_ns = 'org.kivy.android' jni_ns = 'org/kivy/android' elif is_pygame: java_ns = 'org.renpy.android' jni_ns = 'org/renpy/android' else: logger.error('unsupported bootstrap for android recipe: {}'.format(bootstrap_name)) exit(1) config = { 'BOOTSTRAP': bootstrap, 'IS_SDL2': int(is_sdl2), 'IS_PYGAME': int(is_pygame), 'PY2': int(will_build('python2')(self)), 'JAVA_NAMESPACE': java_ns, 'JNI_NAMESPACE': jni_ns, } with current_directory(self.get_build_dir(arch.arch)): with open(join('android', 'config.pxi'), 'w') as fpxi: with open(join('android', 'config.h'), 'w') as fh: with open(join('android', 'config.py'), 'w') as fpy: for key, value in config.items(): fpxi.write(tpxi.format(key, repr(value))) fpy.write(tpy.format(key, repr(value))) fh.write(th.format(key, value if isinstance(value, int) else '"{}"'.format(value))) self.config_env[key] = str(value) if is_sdl2: fh.write('JNIEnv *SDL_AndroidGetJNIEnv(void);\n') fh.write('#define SDL_ANDROID_GetJNIEnv SDL_AndroidGetJNIEnv\n') elif is_pygame: fh.write('JNIEnv *SDL_ANDROID_GetJNIEnv(void);\n')
def build_arch(self, arch, with_harfbuzz=False): env = self.get_recipe_env(arch, with_harfbuzz=with_harfbuzz) harfbuzz_in_recipes = 'harfbuzz' in self.ctx.recipe_build_order prefix_path = self.get_build_dir(arch.arch) if harfbuzz_in_recipes and not with_harfbuzz: # This is the first time we build freetype and we modify `prefix`, # because we will install the compiled library so later we can # build harfbuzz (with freetype support) using this freetype # installation prefix_path = join(prefix_path, 'install') # Configure freetype library config_args = { '--host={}'.format(arch.command_prefix), '--prefix={}'.format(prefix_path), '--without-zlib', '--without-bzip2', '--with-png=no', } if not harfbuzz_in_recipes: info('Build freetype (without harfbuzz)') config_args = config_args.union( {'--disable-static', '--enable-shared', '--with-harfbuzz=no'}) elif not with_harfbuzz: info('Build freetype for First time (without harfbuzz)') # This time we will build our freetype library as static because we # want that the harfbuzz library to have the necessary freetype # symbols/functions, so we avoid to have two freetype shared # libraries which will be confusing and harder to link with them config_args = config_args.union( {'--disable-shared', '--with-harfbuzz=no'}) else: info('Build freetype for Second time (with harfbuzz)') config_args = config_args.union( {'--disable-static', '--enable-shared', '--with-harfbuzz=yes'}) info('Configure args are:\n\t-{}'.format('\n\t-'.join(config_args))) # Build freetype library with current_directory(self.get_build_dir(arch.arch)): configure = sh.Command('./configure') shprint(configure, *config_args, _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) if not with_harfbuzz and harfbuzz_in_recipes: info('Installing freetype (first time build without harfbuzz)') # First build, install the compiled lib, and clean build env shprint(sh.make, 'install', _env=env) shprint(sh.make, 'distclean', _env=env)
def prepare_build_dir(self): '''Ensure that a build dir exists for the recipe. This same single dir will be used for building all different archs.''' self.build_dir = self.get_build_dir() shprint( sh.cp, '-r', join(self.bootstrap_dir, 'build'), # join(self.ctx.root_dir, # 'bootstrap_templates', # self.name), self.build_dir) with current_directory(self.build_dir): with open('project.properties', 'w') as fileh: fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_arch(self, arch): env = self.get_recipe_env(arch) # Build libproto.so with current_directory(self.get_build_dir(arch.arch)): build_arch = (shprint( sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]) if not exists('configure'): shprint(sh.Command('./autogen.sh'), _env=env) shprint(sh.Command('./configure'), '--build={}'.format(build_arch), '--host={}'.format(arch.command_prefix), '--target={}'.format(arch.command_prefix), '--disable-static', '--enable-shared', _env=env) with current_directory(join(self.get_build_dir(arch.arch), 'src')): shprint(sh.make, 'libprotobuf.la', '-j' + str(cpu_count()), _env=env)
def test_current_directory_exception(self): """ Another test for method :meth:`~pythonforandroid.util.current_directory`, but here we check that using the method with a non-existing-directory raises an `OSError` exception. .. note:: test chdir to non-existing directory, should raise error, for py3 the exception is FileNotFoundError and IOError for py2, to avoid introduce conditions, we test with a more generic exception """ with self.assertRaises(OSError), util.current_directory( "/fake/directory" ): pass
def build_arch(self, arch): if self.ctx.ndk_api < self.MIN_NDK_API: raise BuildInterruptingException( 'Target ndk-api is {}, but the python3 recipe supports only' ' {}+'.format(self.ctx.ndk_api, self.MIN_NDK_API)) recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, 'android-build') ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does sys_prefix = '/usr/local' sys_exec_prefix = '/usr/local' with current_directory(build_dir): env = self.get_recipe_env(arch) env = self.set_libs_flags(env, arch) android_build = sh.Command( join(recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8') if not exists('config.status'): shprint(sh.Command(join(recipe_build_dir, 'configure')), *(' '.join(self.configure_args).format( android_host=env['HOSTARCH'], android_build=android_build, prefix=sys_prefix, exec_prefix=sys_exec_prefix)).split(' '), _env=env) if not exists('python'): py_version = self.major_minor_version_string if self.major_minor_version_string[0] == '3': py_version += 'm' shprint(sh.make, 'all', '-j', str(cpu_count()), 'INSTSONAME=libpython{version}.so'.format( version=py_version), _env=env) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
def test_current_directory(self, moch_chdir): """ Basic test for method :meth:`~pythonforandroid.util.current_directory`. We `mock` chdir and we check that the command is executed once we are inside a python's `with` statement. Then we check that `chdir has been called with the proper arguments inside this `with` statement and also that, once we leave the `with` statement, is called again with the current working path. """ chdir_dir = "/temp/any_directory" # test chdir to existing directory with util.current_directory(chdir_dir): moch_chdir.assert_called_once_with("/temp/any_directory") moch_chdir.assert_has_calls( [mock.call("/temp/any_directory"), mock.call(os.getcwd())])
def prepare_build_dir(self): '''Ensure that a build dir exists for the recipe. This same single dir will be used for building all different archs.''' self.build_dir = self.get_build_dir() shprint(sh.cp, '-r', join(self.bootstrap_dir, 'build'), self.build_dir) if self.ctx.symlink_java_src: info('Symlinking java src instead of copying') shprint(sh.rm, '-r', join(self.build_dir, 'src')) shprint(sh.mkdir, join(self.build_dir, 'src')) for dirn in listdir(join(self.bootstrap_dir, 'build', 'src')): shprint(sh.ln, '-s', join(self.bootstrap_dir, 'build', 'src', dirn), join(self.build_dir, 'src')) with current_directory(self.build_dir): with open('project.properties', 'w') as fileh: fileh.write('target=android-{}'.format(self.ctx.android_api))
def build_arch(self, arch): if self.ctx.ndk_api < self.MIN_NDK_API: raise BuildInterruptingException( NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format( ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API), ) recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, "android-build") ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does sys_prefix = "/usr/local" sys_exec_prefix = "/usr/local" env = self.get_recipe_env(arch) env = self.set_libs_flags(env, arch) android_build = sh.Command(join( recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8') with current_directory(build_dir): if not exists('config.status'): shprint( sh.Command(join(recipe_build_dir, "configure")), *(" ".join(self.configure_args).format( android_host=env["HOSTARCH"], android_build=android_build, prefix=sys_prefix, exec_prefix=sys_exec_prefix, )).split(" "), _env=env, ) shprint( sh.make, "all", "-j", str(cpu_count()), "INSTSONAME={lib_name}".format(lib_name=self._libpython), _env=env, ) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable sh.cp("pyconfig.h", join(recipe_build_dir, "Include"))
def prebuild_arch(self, arch): super(ProtobufCppRecipe, self).prebuild_arch(arch) patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched') if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark): self.apply_patch('fix-python3-compatibility.patch', arch.arch) shprint(sh.touch, patch_mark) # During building, host needs to transpile .proto files to .py # ideally with the same version as protobuf runtime, or with an older one. # Because protoc is compiled for target (i.e. Android), we need an other binary # which can be run by host. # To make it easier, we download prebuild protoc binary adapted to the platform info_notify("Downloading protoc compiler for your platform") url_prefix = "https://github.com/protocolbuffers/protobuf/releases/download/v{version}".format( version=self.version) if sys.platform.startswith('linux'): info_notify("GNU/Linux detected") filename = "protoc-{version}-linux-x86_64.zip".format( version=self.version) elif sys.platform.startswith('darwin'): info_notify("Mac OS X detected") filename = "protoc-{version}-osx-x86_64.zip".format( version=self.version) else: info_notify("Your platform is not supported, but recipe can still " "be built if you have a valid protoc (<={version}) in " "your path".format(version=self.version)) return protoc_url = join(url_prefix, filename) self.protoc_dir = join(self.ctx.build_dir, "tools", "protoc") if os.path.exists(join(self.protoc_dir, "bin", "protoc")): info_notify("protoc found, no download needed") return try: os.makedirs(self.protoc_dir) except OSError as e: # if dir already exists (errno 17), we ignore the error if e.errno != 17: raise e info_notify( "Will download into {dest_dir}".format(dest_dir=self.protoc_dir)) self.download_file(protoc_url, join(self.protoc_dir, filename)) with current_directory(self.protoc_dir): shprint(sh.unzip, join(self.protoc_dir, filename))
def strip_object_files(self, arch, env, build_dir=None): if build_dir is None: build_dir = self.get_build_dir(arch.arch) with current_directory(build_dir): info('Stripping object files') if self.ctx.python_recipe.name == 'python2legacy': info('Stripping object files') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) else: shprint(sh.find, '.', '-iname', '*.so', '-exec', '/usr/bin/echo', '{}', ';', _env=env) shprint(sh.find, '.', '-iname', '*.so', '-exec', env['STRIP'].split(' ')[0], '--strip-unneeded', # '/usr/bin/strip', '--strip-unneeded', '{}', ';', _env=env)
def biglink(ctx, arch): # First, collate object files from each recipe info("Collating object files from each recipe") obj_dir = join(ctx.bootstrap.build_dir, "collated_objects") ensure_dir(obj_dir) recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order] for recipe in recipes: recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch), "objects_{}".format(recipe.name)) if not exists(recipe_obj_dir): info("{} recipe has no biglinkable files dir, skipping".format( recipe.name)) continue files = glob.glob(join(recipe_obj_dir, "*")) if not len(files): info("{} recipe has no biglinkable files, skipping".format( recipe.name)) continue info("{} recipe has object files, copying".format(recipe.name)) files.append(obj_dir) shprint(sh.cp, "-r", *files) env = arch.get_env() env["LDFLAGS"] = env["LDFLAGS"] + " -L{}".format( join(ctx.bootstrap.build_dir, "obj", "local", arch.arch)) if not len(glob.glob(join(obj_dir, "*"))): info("There seem to be no libraries to biglink, skipping.") return info("Biglinking") info("target {}".format( join(ctx.get_libs_dir(arch.arch), "libpymodules.so"))) do_biglink = copylibs_function if ctx.copy_libs else biglink_function # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? with current_directory(join(ctx.ndk_platform, "usr", "lib")): do_biglink( join(ctx.get_libs_dir(arch.arch), "libpymodules.so"), obj_dir.split(" "), extra_link_dirs=[ join(ctx.bootstrap.build_dir, "obj", "local", arch.arch), os.path.abspath("."), ], env=env, )
def build_cython_components(self, arch): info('Cythonizing anything necessary in {}'.format(self.name)) env = self.get_recipe_env(arch) if self.ctx.python_recipe.from_crystax: command = sh.Command('python{}'.format(self.ctx.python_recipe.version)) site_packages_dirs = command( '-c', 'import site; print("\\n".join(site.getsitepackages()))') site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n') # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5' if 'PYTHONPATH' in env: env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs)) else: env['PYTHONPATH'] = ':'.join(site_packages_dirs) with current_directory(self.get_build_dir(arch.arch)): hostpython = sh.Command(self.ctx.hostpython) # hostpython = sh.Command('python3.5') shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env) print('cwd is', realpath(curdir)) info('Trying first build of {} to get cython files: this is ' 'expected to fail'.format(self.name)) manually_cythonise = False try: shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, *self.setup_extra_args) except sh.ErrorReturnCode_1: print() info('{} first build failed (as expected)'.format(self.name)) manually_cythonise = True if manually_cythonise: self.cythonize_build(env=env) shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env, _tail=20, _critical=True, *self.setup_extra_args) else: info('First build appeared to complete correctly, skipping manual' 'cythonising.') print('stripping') build_lib = glob.glob('./build/lib*') shprint(sh.find, build_lib[0], '-name', '*.o', '-exec', env['STRIP'], '{}', ';', _env=env) print('stripped!?')
def build_arch(self, arch, *extra_args): super().build_arch(arch) env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): shprint( sh.cmake, f'-DCMAKE_TOOLCHAIN_FILE={self.ctx.ndk_dir}/build/cmake/android.toolchain.cmake', f'-DANDROID_ABI={arch}', f'-DANDROID_NATIVE_API_LEVEL={self.ctx.ndk_api}', f'-DCMAKE_BUILD_TYPE=Release CMakeLists.txt', *extra_args, _env=env) shprint(sh.cmake, '--build', '.', _env=env) # a bit of a dirty workaround move(join(self.get_build_dir(arch.arch), "armips"), join(self.get_build_dir(arch.arch), "libarmips.so"))
def build_arch(self, arch): build_dir = self.get_build_dir(arch.arch) with current_directory(build_dir): env = self.get_recipe_env(arch) build_arch = (shprint( sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]) shprint( sh.Command('./configure'), '--build=' + build_arch, '--host=' + arch.command_prefix, '--target=' + arch.command_prefix, '--disable-static', '--enable-shared', '--prefix={}/install'.format(self.get_build_dir(arch.arch)), _env=env, ) shprint(sh.make, '-j', str(cpu_count()), _env=env)
def prebuild_arch(self, arch): if not self.is_patched(arch): super().prebuild_arch(arch) recipe_dir = self.get_build_dir(arch.arch) # Some versions of reportlab ship with a GPL-licensed font. # Remove it, since this is problematic in .apks unless the # entire app is GPL: font_dir = os.path.join(recipe_dir, "src", "reportlab", "fonts") if os.path.exists(font_dir): for file in os.listdir(font_dir): if file.lower().startswith('darkgarden'): os.remove(os.path.join(font_dir, file)) # Apply patches: self.apply_patch('patches/fix-setup.patch', arch.arch) shprint(sh.touch, os.path.join(recipe_dir, '.patched')) ft = self.get_recipe('freetype', self.ctx) ft_dir = ft.get_build_dir(arch.arch) ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs')) ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include')) tmp_dir = os.path.normpath( os.path.join(recipe_dir, "..", "..", "tmp")) info('reportlab recipe: recipe_dir={}'.format(recipe_dir)) info('reportlab recipe: tmp_dir={}'.format(tmp_dir)) info('reportlab recipe: ft_dir={}'.format(ft_dir)) info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir)) info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir)) with current_directory(recipe_dir): ensure_dir(tmp_dir) pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip") if not os.path.isfile(pfbfile): sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile) sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile) if os.path.isfile("setup.py"): with open('setup.py', 'r') as f: text = f.read().replace('_FT_LIB_', ft_lib_dir).replace( '_FT_INC_', ft_inc_dir) with open('setup.py', 'w') as f: f.write(text)
def biglink(ctx, arch): # First, collate object files from each recipe info('Collating object files from each recipe') obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects') ensure_dir(obj_dir) recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order] for recipe in recipes: recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch), 'objects_{}'.format(recipe.name)) if not exists(recipe_obj_dir): info('{} recipe has no biglinkable files dir, skipping'.format( recipe.name)) continue files = glob.glob(join(recipe_obj_dir, '*')) if not len(files): info('{} recipe has no biglinkable files, skipping'.format( recipe.name)) continue info('{} recipe has object files, copying'.format(recipe.name)) files.append(obj_dir) shprint(sh.cp, '-r', *files) env = arch.get_env() env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format( join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch)) if not len(glob.glob(join(obj_dir, '*'))): info('There seem to be no libraries to biglink, skipping.') return info('Biglinking') info('target {}'.format( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'))) do_biglink = copylibs_function if ctx.copy_libs else biglink_function # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? with current_directory(join(ctx.ndk_platform, 'usr', 'lib')): do_biglink(join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), extra_link_dirs=[ join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch), os.path.abspath('.') ], env=env)
def download(self): if self.url is None: info('Skipping {} download as no URL is set'.format(self.name)) return url = self.versioned_url shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name)) with current_directory(join(self.ctx.packages_path, self.name)): filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8') do_download = True marker_filename = '.mark-{}'.format(filename) if exists(filename) and isfile(filename): if not exists(marker_filename): shprint(sh.rm, filename) elif self.md5sum: current_md5 = shprint(sh.md5sum, filename) print('downloaded md5: {}'.format(current_md5)) print('expected md5: {}'.format(self.md5sum)) print('md5 not handled yet, exiting') exit(1) else: do_download = False info('{} download already cached, skipping'.format( self.name)) # Should check headers here! warning('Should check headers here! Skipping for now.') # If we got this far, we will download if do_download: print('Downloading {} from {}'.format(self.name, url)) shprint(sh.rm, '-f', marker_filename) self.download_file(url, filename) shprint(sh.touch, marker_filename) if self.md5sum is not None: print('downloaded md5: {}'.format(current_md5)) print('expected md5: {}'.format(self.md5sum)) print('md5 not handled yet, exiting') exit(1)
def install_python_package(self, arch): env = self.get_recipe_env(arch) info('Installing {} into site-packages'.format(self.name)) with current_directory(join(self.get_build_dir(arch.arch), 'python')): hostpython = sh.Command(self.hostpython_location) hpenv = env.copy() shprint(hostpython, 'setup.py', 'install', '-O2', '--root={}'.format(self.ctx.get_python_install_dir()), '--install-lib=.', '--cpp_implementation', _env=hpenv, *self.setup_extra_args)
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory(self.get_build_dir(arch.arch)): # sh fails with code 255 trying to execute ./Configure # so instead we manually run perl passing in Configure perl = sh.Command('perl') buildarch = self.select_build_arch(arch) config_args = [ 'shared', 'no-dso', 'no-asm', buildarch, '-D__ANDROID_API__={}'.format(self.ctx.ndk_api), ] shprint(perl, 'Configure', *config_args, _env=env) self.apply_patch('disable-sover.patch', arch.arch) shprint(sh.make, 'build_libs', _env=env)
def save_info(self, dirn): ''' Save information about the distribution in its dist_dir. ''' with current_directory(dirn): info('Saving distribution info') with open('dist_info.json', 'w') as fileh: json.dump( { 'dist_name': self.ctx.dist_name, 'bootstrap': self.ctx.bootstrap.name, 'archs': [arch.arch for arch in self.ctx.archs], 'ndk_api': self.ctx.ndk_api, 'recipes': self.ctx.recipe_build_order + self.ctx.python_modules }, fileh)
def build_arch(self, arch): env = self.get_recipe_env(arch) with current_directory( join(self.get_build_dir(arch.arch), 'libmysqlclient')): shprint(sh.cp, '-t', '.', join(self.get_recipe_dir(), 'p4a.cmake')) # shprint(sh.mkdir, 'Platform') # shprint(sh.cp, '-t', 'Platform', join(self.get_recipe_dir(), 'Linux.cmake')) shprint(sh.rm, '-f', 'CMakeCache.txt') shprint( sh.cmake, '-G', 'Unix Makefiles', # '-DCMAKE_MODULE_PATH=' + join(self.get_build_dir(arch.arch), 'libmysqlclient'), '-DCMAKE_INSTALL_PREFIX=./install', '-DCMAKE_TOOLCHAIN_FILE=p4a.cmake', _env=env) shprint(sh.make, _env=env) self.install_libs(arch, join('libmysql', 'libmysql.so'))
def build_arch(self, arch): env = self.get_recipe_env(arch) openssl_recipe = self.get_recipe('openssl', self.ctx) openssl_dir = openssl_recipe.get_build_dir(arch.arch) env['LDFLAGS'] += openssl_recipe.link_dirs_flags(arch) env['LIBS'] = env.get('LIBS', '') + openssl_recipe.link_libs_flags() with current_directory(self.get_build_dir(arch.arch)): dst_dir = join(self.get_build_dir(arch.arch), 'dist') shprint(sh.Command('./configure'), '--host=arm-linux-androideabi', '--enable-shared', '--with-ssl={}'.format(openssl_dir), '--prefix={}'.format(dst_dir), _env=env) shprint(sh.make, '-j', str(cpu_count()), _env=env) shprint(sh.make, 'install', _env=env)