Exemplo n.º 1
0
 def prepare_build_dir(self, arch):
     if self.src_filename is None:
         print('IncludedFilesBehaviour failed: no src_filename specified')
         exit(1)
     shprint(sh.rm, '-rf', self.get_build_dir(arch))
     shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename),
             self.get_build_dir(arch))
Exemplo n.º 2
0
 def install_hostpython_package(self, arch):
     env = self.get_hostrecipe_env(arch)
     real_hostpython = sh.Command(self.real_hostpython_location)
     shprint(real_hostpython, 'setup.py', 'install', '-O2',
             '--root={}'.format(dirname(self.real_hostpython_location)),
             '--install-lib=Lib/site-packages',
             _env=env, *self.setup_extra_args)
Exemplo n.º 3
0
    def build_arch(self, arch):
        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, self.build_subdir)
        ensure_dir(build_dir)

        if not exists(join(build_dir, 'python')):
            with current_directory(recipe_build_dir):
                # Configure the build
                with current_directory(build_dir):
                    if not exists('config.status'):
                        shprint(
                            sh.Command(join(recipe_build_dir, 'configure')))

                # Create the Setup file. This copying from Setup.dist
                # seems to be the normal and expected procedure.
                shprint(sh.cp, join('Modules', 'Setup.dist'),
                        join(build_dir, 'Modules', 'Setup'))

                result = shprint(sh.make, '-C', build_dir)
        else:
            info('Skipping {name} ({version}) build, as it has already '
                 'been completed'.format(name=self.name, version=self.version))

        self.ctx.hostpython = join(build_dir, 'python')
Exemplo n.º 4
0
    def rebuild_compiled_components(self, arch, env):
        info('Rebuilding compiled components in {}'.format(self.name))

        hostpython = sh.Command(self.real_hostpython_location)
        shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
                *self.setup_extra_args)
Exemplo n.º 5
0
 def install_libs(self, arch, *libs):
     libs_dir = self.ctx.get_libs_dir(arch.arch)
     if not libs:
         warning('install_libs called with no libraries to install!')
         return
     args = libs + (libs_dir,)
     shprint(sh.cp, *args)
Exemplo n.º 6
0
 def prebuild_arch(self, arch):
     super(VlcRecipe, self).prebuild_arch(arch)
     build_dir = self.get_build_dir(arch.arch)
     port_dir = join(build_dir, 'vlc-port-android')
     if self.ENV_LIBVLC_AAR in environ:
         aar = environ.get(self.ENV_LIBVLC_AAR)
         if isdir(aar):
             aar = join(aar, 'libvlc-{}.aar'.format(self.version))
         if not isfile(aar):
             warning("Error: {} is not valid libvlc-<ver>.aar bundle".format(aar))
             info("check {} environment!".format(self.ENV_LIBVLC_AAR))
             exit(1)
         self.aars[arch] = aar
     else:
         aar_path = join(port_dir, 'libvlc', 'build', 'outputs', 'aar')
         self.aars[arch] = aar = join(aar_path, 'libvlc-{}.aar'.format(self.version))
         warning("HINT: set path to precompiled libvlc-<ver>.aar bundle "
                 "in {} environment!".format(self.ENV_LIBVLC_AAR))
         info("libvlc-<ver>.aar should build "
              "from sources at {}".format(port_dir))
         if not isfile(join(port_dir, 'compile.sh')):
             info("clone vlc port for android sources from {}".format(
                         self.port_git))
             shprint(sh.git, 'clone', self.port_git, port_dir,
                     _tail=20, _critical=True)
Exemplo n.º 7
0
 def prepare_build_dir(self, arch):
     if self.src_filename is None:
         raise BuildInterruptingException(
             'IncludedFilesBehaviour failed: no src_filename specified')
     shprint(sh.rm, '-rf', self.get_build_dir(arch))
     shprint(sh.cp, '-a', join(self.get_recipe_dir(), self.src_filename),
             self.get_build_dir(arch))
Exemplo n.º 8
0
def run_pymodules_install(ctx, modules):
    modules = filter(ctx.not_has_package, modules)

    if not modules:
        info('There are no Python modules to install, skipping')
        return

    info('The requirements ({}) don\'t have recipes, attempting to install '
         'them with pip'.format(', '.join(modules)))
    info('If this fails, it may mean that the module has compiled '
         'components and needs a recipe.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(venv, '--python=python2.7', 'venv')

        info('Creating a requirements.txt file for the Python modules')
        with open('requirements.txt', 'w') as fileh:
            for module in modules:
                fileh.write('{}\n'.format(module))

        info('Installing Python modules with pip')
        info('If this fails with a message about /bin/false, this '
             'probably means the package cannot be installed with '
             'pip as it needs a compilation recipe.')

        # This bash method is what old-p4a used
        # It works but should be replaced with something better
        shprint(sh.bash, '-c', (
            "source venv/bin/activate && env CC=/bin/false CXX=/bin/false "
            "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt"
        ).format(ctx.get_site_packages_dir()))
Exemplo n.º 9
0
    def build_arch(self, arch):
        # Create a temporary folder to add to link path with a fake librt.so:
        fake_librt_temp_folder = join(
            self.get_build_dir(arch.arch),
            "p4a-librt-recipe-tempdir"
        )
        if not exists(fake_librt_temp_folder):
            makedirs(fake_librt_temp_folder)

        # Set symlinks, and make sure to update them on every build run:
        if exists(join(fake_librt_temp_folder, "librt.so")):
            remove(join(fake_librt_temp_folder, "librt.so"))
        shprint(sh.ln, '-sf',
                self.libc_path + '.so',
                join(fake_librt_temp_folder, "librt.so"),
                )
        if exists(join(fake_librt_temp_folder, "librt.a")):
            remove(join(fake_librt_temp_folder, "librt.a"))
        shprint(sh.ln, '-sf',
                self.libc_path + '.a',
                join(fake_librt_temp_folder, "librt.a"),
               )

        # Add folder as -L link option for all recipes if not done yet:
        if fake_librt_temp_folder not in arch.extra_global_link_paths:
            arch.extra_global_link_paths.append(
                fake_librt_temp_folder
            )
Exemplo n.º 10
0
 def prebuild_arch(self, arch):
     if not self.is_patched(arch):
         super(ReportLabRecipe, self).prebuild_arch(arch)
         self.apply_patch('patches/fix-setup.patch', arch.arch)
         recipe_dir = self.get_build_dir(arch.arch)
         shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
         ft = self.get_recipe('freetype', self.ctx)
         ft_dir = ft.get_build_dir(arch.arch)
         ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
         ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
         tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
         info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
         info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
         info('reportlab recipe: ft_dir={}'.format(ft_dir))
         info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
         info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
         with current_directory(recipe_dir):
             sh.ls('-lathr')
             ensure_dir(tmp_dir)
             pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
             if not os.path.isfile(pfbfile):
                 sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
             sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
             if os.path.isfile("setup.py"):
                 with open('setup.py', 'rb') as f:
                     text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
                 with open('setup.py', 'wb') as f:
                     f.write(text)
Exemplo n.º 11
0
    def build_arch(self, arch):
        env = self.get_recipe_env(arch)
        
        env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format(
            jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
        env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl/include -I{jni_path}/sdl_mixer'.format(
            jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
        env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl_ttf -I{jni_path}/sdl_image'.format(
            jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
        debug('pygame cflags', env['CFLAGS'])

        
        env['LDFLAGS'] = env['LDFLAGS'] + ' -L{libs_path} -L{src_path}/obj/local/{arch} -lm -lz'.format(
            libs_path=self.ctx.libs_dir, src_path=self.ctx.bootstrap.build_dir, arch=env['ARCH'])

        env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')

        with current_directory(self.get_build_dir(arch.arch)):
            info('hostpython is ' + self.ctx.hostpython)
            hostpython = sh.Command(self.ctx.hostpython)
            shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
                    _tail=10, _critical=True)

            info('strip is ' + env['STRIP'])
            build_lib = glob.glob('./build/lib*')
            assert len(build_lib) == 1
            print('stripping pygame')
            shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                    env['STRIP'], '{}', ';')

        python_install_path = join(self.ctx.build_dir, 'python-install')
        warning('Should remove pygame tests etc. here, but skipping for now')
Exemplo n.º 12
0
    def strip_libraries(self, arch):
        info('Stripping libraries')
        if self.ctx.python_recipe.from_crystax:
            info('Python was loaded from CrystaX, skipping strip')
            return
        env = arch.get_env()
        strip = which('arm-linux-androideabi-strip', env['PATH'])
        if strip is None:
            warning('Can\'t find strip in PATH...')
            return
        strip = sh.Command(strip)

        if self.ctx.python_recipe.name == 'python2':
            filens = shprint(sh.find, join(self.dist_dir, 'private'),
                             join(self.dist_dir, 'libs'),
                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
        else:
            filens = shprint(sh.find, join(self.dist_dir, '_python_bundle', '_python_bundle', 'modules'),
                             join(self.dist_dir, 'libs'),
                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
        logger.info('Stripping libraries in private dir')
        for filen in filens.split('\n'):
            try:
                strip(filen, _env=env)
            except sh.ErrorReturnCode_1:
                logger.debug('Failed to strip ' + filen)
Exemplo n.º 13
0
    def install_python_package(self, arch):
        env = self.get_recipe_env(arch)

        info('Installing {} into site-packages'.format(self.name))

        with current_directory(join(self.get_build_dir(arch.arch), 'python')):
            hostpython = sh.Command(self.hostpython_location)

            if self.ctx.python_recipe.from_crystax:
                hpenv = env.copy()
                shprint(hostpython, 'setup.py', 'install', '-O2',
                        '--root={}'.format(self.ctx.get_python_install_dir()),
                        '--install-lib=.',
                        '--cpp_implementation',
                        _env=hpenv, *self.setup_extra_args)
            else:
                hppath = join(dirname(self.hostpython_location), 'Lib',
                              'site-packages')
                hpenv = env.copy()
                if 'PYTHONPATH' in hpenv:
                    hpenv['PYTHONPATH'] = ':'.join([hppath] +
                                                   hpenv['PYTHONPATH'].split(':'))
                else:
                    hpenv['PYTHONPATH'] = hppath
                shprint(hostpython, 'setup.py', 'install', '-O2',
                        '--root={}'.format(self.ctx.get_python_install_dir()),
                        '--install-lib=lib/python2.7/site-packages',
                        '--cpp_implementation',
                        _env=hpenv, *self.setup_extra_args)
Exemplo n.º 14
0
	def build_arch(self, arch):
		env = self.get_recipe_env(arch)
		with current_directory(self.get_build_dir(arch.arch)):
			if not exists('configure'):
				shprint(sh.Command('./autogen.sh'), _env=env)
			shprint(sh.Command('autoreconf -vif'), _env=env)
			shprint(sh.Command('./configure'), '--host=' + arch.toolchain_prefix,
			        '--prefix=' + self.ctx.get_python_install_dir(),
			        '--enable-shared', _env=env)
			shprint(sh.make, '-j5', 'libffi.la', _env=env)


			# dlname = None
			# with open(join(host, 'libffi.la')) as f:
			# 	for line in f:
			# 		if line.startswith('dlname='):
			# 			dlname = line.strip()[8:-1]
			# 			break
			# 
			# if not dlname or not exists(join(host, '.libs', dlname)):
			# 	raise RuntimeError('failed to locate shared object! ({})'
			# 	                   .format(dlname))

			# shprint(sh.sed, '-i', 's/^dlname=.*$/dlname=\'libffi.so\'/', join(host, 'libffi.la'))

			shprint(sh.cp, '-t', self.ctx.get_libs_dir(arch.arch),
			        join(self.get_host(arch), '.libs', 'libffi.so')) #,
Exemplo n.º 15
0
    def apply_patches(self, arch, build_dir=None):
        '''Apply any patches for the Recipe.

        .. versionchanged:: 0.6.0
            Add ability to apply patches from any dir via kwarg `build_dir`'''
        if self.patches:
            info_main('Applying patches for {}[{}]'
                      .format(self.name, arch.arch))

            if self.is_patched(arch):
                info_main('{} already patched, skipping'.format(self.name))
                return

            build_dir = build_dir if build_dir else self.get_build_dir(arch.arch)
            for patch in self.patches:
                if isinstance(patch, (tuple, list)):
                    patch, patch_check = patch
                    if not patch_check(arch=arch, recipe=self):
                        continue

                self.apply_patch(
                        patch.format(version=self.version, arch=arch.arch),
                        arch.arch, build_dir=build_dir)

            shprint(sh.touch, join(build_dir, '.patched'))
Exemplo n.º 16
0
    def apk(self, args):
        '''Create an APK using the given distribution.'''

        # AND: Need to add a parser here for any extra options
        # parser = argparse.ArgumentParser(
        #     description='Build an APK')
        # args = parser.parse_args(args)

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        for i, arg in enumerate(args[:-1]):
            if arg in ('--dir', '--private'):
                args[i+1] = realpath(expanduser(args[i+1]))

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            build.parse_args(args)
            shprint(sh.ant, 'debug', _tail=20, _critical=True)

        # AND: This is very crude, needs improving. Also only works
        # for debug for now.
        info_main('# Copying APK to current directory')
        apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-debug.apk'))
        if len(apks) == 0:
            raise ValueError('Couldn\'t find the built APK')
        if len(apks) > 1:
            info('More than one built APK found...guessing you '
                 'just built {}'.format(apks[-1]))
        shprint(sh.cp, apks[-1], './')
Exemplo n.º 17
0
 def apply_patch(self, filename, arch):
     """
     Apply a patch from the current recipe directory into the current
     build directory.
     """
     info("Applying patch {}".format(filename))
     filename = join(self.recipe_dir, filename)
     shprint(sh.patch, "-t", "-d", self.get_build_dir(arch), "-p1", "-i", filename, _tail=10)
Exemplo n.º 18
0
 def distribute_libs(self, arch, src_dirs, wildcard='*', dest_dir="libs"):
     '''Copy existing arch libs from build dirs to current dist dir.'''
     info('Copying libs')
     tgt_dir = join(dest_dir, arch.arch)
     ensure_dir(tgt_dir)
     for src_dir in src_dirs:
         for lib in glob.glob(join(src_dir, wildcard)):
             shprint(sh.cp, '-a', lib, tgt_dir)
Exemplo n.º 19
0
    def build_cython_components(self, arch):
        info('Cythonizing anything necessary in {}'.format(self.name))

        env = self.get_recipe_env(arch)

        if self.ctx.python_recipe.from_crystax:
            command = sh.Command('python{}'.format(self.ctx.python_recipe.version))
            site_packages_dirs = command(
                '-c', 'import site; print("\\n".join(site.getsitepackages()))')
            site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n')
            # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
            if 'PYTHONPATH' in env:
                env['PYTHONPATH'] = env + ':{}'.format(':'.join(site_packages_dirs))
            else:
                env['PYTHONPATH'] = ':'.join(site_packages_dirs)

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.ctx.hostpython)
            # hostpython = sh.Command('python3.5')
            shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
            print('cwd is', realpath(curdir))
            info('Trying first build of {} to get cython files: this is '
                 'expected to fail'.format(self.name))

            manually_cythonise = False
            try:
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
                        *self.setup_extra_args)
            except sh.ErrorReturnCode_1:
                print()
                info('{} first build failed (as expected)'.format(self.name))
                manually_cythonise = True

            if manually_cythonise:
                info('Running cython where appropriate')
                cyenv = env.copy()
                if 'CYTHONPATH' in cyenv:
                    cyenv['PYTHONPATH'] = cyenv['CYTHONPATH']
                elif 'PYTHONPATH' in cyenv:
                    del cyenv['PYTHONPATH']
                cython = 'cython' if self.ctx.python_recipe.from_crystax else self.ctx.cython
                cython_cmd = 'find "{}" -iname *.pyx | xargs "{}"'.format(
                        self.get_build_dir(arch.arch), cython)
                shprint(sh.sh, '-c', cython_cmd, _env=cyenv)
                info('ran cython')

                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
                        _tail=20, _critical=True, *self.setup_extra_args)
            else:
                info('First build appeared to complete correctly, skipping manual'
                     'cythonising.')

            print('stripping')
            build_lib = glob.glob('./build/lib*')
            shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                    env['STRIP'], '{}', ';', _env=env)
            print('stripped!?')
Exemplo n.º 20
0
 def fry_eggs(self, sitepackages):
     info('Frying eggs in {}'.format(sitepackages))
     for d in listdir(sitepackages):
         rd = join(sitepackages, d)
         if isdir(rd) and d.endswith('.egg'):
             info('  ' + d)
             files = [join(rd, f) for f in listdir(rd) if f != 'EGG-INFO']
             shprint(sh.mv, '-t', sitepackages, *files)
             shprint(sh.rm, '-rf', d)
Exemplo n.º 21
0
    def prebuild_arch(self, arch):
        super(JpegRecipe, self).prebuild_arch(arch)

        build_dir = self.get_build_dir(arch.arch)
        app_mk = join(build_dir, 'Application.mk')
        if not exists(app_mk):
            shprint(sh.cp, join(self.get_recipe_dir(), 'Application.mk'), app_mk)
        jni_ln = join(build_dir, 'jni')
        if not exists(jni_ln):
            shprint(sh.ln, '-s', build_dir, jni_ln)
Exemplo n.º 22
0
    def build_arch(self, arch, *extra_args):
        super(NDKRecipe, self).build_arch(arch)

        env = self.get_recipe_env(arch)
        with current_directory(self.get_build_dir(arch.arch)):
            shprint(
                sh.ndk_build,
                'V=1',
                'APP_PLATFORM=android-' + str(self.ctx.ndk_api),
                'APP_ABI=' + arch.arch,
                *extra_args, _env=env
            )
Exemplo n.º 23
0
    def download(self):
        if self.url is None:
            info('Skipping {} download as no URL is set'.format(self.name))
            return

        url = self.versioned_url
        ma = match(u'^(.+)#md5=([0-9a-f]{32})$', url)
        if ma:                  # fragmented URL?
            if self.md5sum:
                raise ValueError(
                    ('Received md5sum from both the {} recipe '
                     'and its url').format(self.name))
            url = ma.group(1)
            expected_md5 = ma.group(2)
        else:
            expected_md5 = self.md5sum

        shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name))

        with current_directory(join(self.ctx.packages_path, self.name)):
            filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')

            do_download = True
            marker_filename = '.mark-{}'.format(filename)
            if exists(filename) and isfile(filename):
                if not exists(marker_filename):
                    shprint(sh.rm, filename)
                elif expected_md5:
                    current_md5 = md5sum(filename)
                    if current_md5 != expected_md5:
                        debug('* Generated md5sum: {}'.format(current_md5))
                        debug('* Expected md5sum: {}'.format(expected_md5))
                        raise ValueError(
                            ('Generated md5sum does not match expected md5sum '
                             'for {} recipe').format(self.name))
                    do_download = False
                else:
                    do_download = False

            # If we got this far, we will download
            if do_download:
                debug('Downloading {} from {}'.format(self.name, url))

                shprint(sh.rm, '-f', marker_filename)
                self.download_file(self.versioned_url, filename)
                shprint(sh.touch, marker_filename)

                if exists(filename) and isfile(filename) and expected_md5:
                    current_md5 = md5sum(filename)
                    if expected_md5 is not None:
                        if current_md5 != expected_md5:
                            debug('* Generated md5sum: {}'.format(current_md5))
                            debug('* Expected md5sum: {}'.format(expected_md5))
                            raise ValueError(
                                ('Generated md5sum does not match expected md5sum '
                                 'for {} recipe').format(self.name))
            else:
                info('{} download already cached, skipping'.format(self.name))
Exemplo n.º 24
0
 def prepare_build_dir(self):
     '''Ensure that a build dir exists for the recipe. This same single
     dir will be used for building all different archs.'''
     self.build_dir = self.get_build_dir()
     shprint(sh.cp, '-r',
             join(self.bootstrap_dir, 'build'),
             # join(self.ctx.root_dir,
             #      'bootstrap_templates',
             #      self.name),
             self.build_dir)
     with current_directory(self.build_dir):
         with open('project.properties', 'w') as fileh:
             fileh.write('target=android-{}'.format(self.ctx.android_api))
Exemplo n.º 25
0
 def cythonize_file(self, env, build_dir, filename):
     short_filename = filename
     if filename.startswith(build_dir):
         short_filename = filename[len(build_dir) + 1:]
     info(u"Cythonize {}".format(short_filename))
     cyenv = env.copy()
     if 'CYTHONPATH' in cyenv:
         cyenv['PYTHONPATH'] = cyenv['CYTHONPATH']
     elif 'PYTHONPATH' in cyenv:
         del cyenv['PYTHONPATH']
     cython = 'cython' if self.ctx.python_recipe.from_crystax else self.ctx.cython
     cython_command = sh.Command(cython)
     shprint(cython_command, filename, *self.cython_args, _env=cyenv)
Exemplo n.º 26
0
 def install_hostpython_package(self, arch):
     env = self.get_hostrecipe_env(arch)
     real_hostpython = sh.Command(self.real_hostpython_location)
     shprint(
         real_hostpython,
         "setup.py",
         "install",
         "-O2",
         "--root={}".format(dirname(self.real_hostpython_location)),
         "--install-lib=Lib/site-packages",
         _env=env,
         *self.setup_extra_args
     )
Exemplo n.º 27
0
 def reduce_object_file_names(self, dirn):
     """Recursively renames all files named XXX.cpython-...-linux-gnu.so"
     to "XXX.so", i.e. removing the erroneous architecture name
     coming from the local system.
     """
     py_so_files = shprint(sh.find, dirn, '-iname', '*.so')
     filens = py_so_files.stdout.decode('utf-8').split('\n')[:-1]
     for filen in filens:
         file_dirname, file_basename = split(filen)
         parts = file_basename.split('.')
         if len(parts) <= 2:
             continue
         shprint(sh.mv, filen, join(file_dirname, parts[0] + '.so'))
Exemplo n.º 28
0
    def apply_patch(self, filename, arch, build_dir=None):
        """
        Apply a patch from the current recipe directory into the current
        build directory.

        .. versionchanged:: 0.6.0
            Add ability to apply patch from any dir via kwarg `build_dir`'''
        """
        info("Applying patch {}".format(filename))
        build_dir = build_dir if build_dir else self.get_build_dir(arch)
        filename = join(self.get_recipe_dir(), filename)
        shprint(sh.patch, "-t", "-d", build_dir, "-p1",
                "-i", filename, _tail=10)
Exemplo n.º 29
0
    def build_cython_components(self, arch):
        info('Cythonizing anything necessary in {}'.format(self.name))
        env = self.get_recipe_env(arch)
        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.ctx.hostpython)
            info('Trying first build of {} to get cython files: this is '
                 'expected to fail'.format(self.name))
            try:
                shprint(hostpython, 'setup.py', 'build_ext', _env=env,
                        *self.setup_extra_args)
            except sh.ErrorReturnCode_1:
                print()
                info('{} first build failed (as expected)'.format(self.name))

            info('Running cython where appropriate')
            shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx',
                    '-exec', self.ctx.cython, '{}', ';', _env=env)
            info('ran cython')

            shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
                    _tail=20, _critical=True, *self.setup_extra_args)

            print('stripping')
            build_lib = glob.glob('./build/lib*')
            shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                    env['STRIP'], '{}', ';', _env=env)
            print('stripped!?')
Exemplo n.º 30
0
    def install_python_package(self, arch):
        env = self.get_recipe_env(arch)

        info('Installing {} into site-packages'.format(self.name))

        with current_directory(join(self.get_build_dir(arch.arch), 'python')):
            hostpython = sh.Command(self.hostpython_location)

            hpenv = env.copy()
            shprint(hostpython, 'setup.py', 'install', '-O2',
                    '--root={}'.format(self.ctx.get_python_install_dir()),
                    '--install-lib=.',
                    '--cpp_implementation',
                    _env=hpenv, *self.setup_extra_args)
Exemplo n.º 31
0
 def prebuild_arch(self, arch):
     super(Python2Recipe, self).prebuild_arch(arch)
     patch_mark = join(self.get_build_dir(arch.arch), '.openssl-patched')
     if 'openssl' in self.ctx.recipe_build_order and not exists(patch_mark):
         self.apply_patch(join('patches', 'enable-openssl.patch'), arch.arch)
         shprint(sh.touch, patch_mark)
Exemplo n.º 32
0
    def build_cython_components(self, arch):
        info('Cythonizing anything necessary in {}'.format(self.name))

        env = self.get_recipe_env(arch)

        if self.ctx.python_recipe.from_crystax:
            command = sh.Command('python{}'.format(self.ctx.python_recipe.version))

            if hasattr(sys, 'real_prefix'):
                #: If within a virtualenv
                #: See https://github.com/pypa/virtualenv/issues/355
                site_packages_dirs = command(
                    '-c', 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
            else:
                site_packages_dirs = command(
                    '-c', 'import site; print("\\n".join(site.getsitepackages()))')
            site_packages_dirs = site_packages_dirs.stdout.decode('utf-8').split('\n')
            if 'PYTHONPATH' in env:
                env['PYTHONPATH'] += ':{}'.format(':'.join(site_packages_dirs))
            else:
                env['PYTHONPATH'] = ':'.join(site_packages_dirs)

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.ctx.hostpython)
            shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
            print('cwd is', realpath(curdir))
            info('Trying first build of {} to get cython files: this is '
                 'expected to fail'.format(self.name))

            manually_cythonise = False
            try:
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
                        *self.setup_extra_args)
            except sh.ErrorReturnCode_1:
                print()
                info('{} first build failed (as expected)'.format(self.name))
                manually_cythonise = True

            if manually_cythonise:
                self.cythonize_build(env=env)
                shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
                        _tail=20, _critical=True, *self.setup_extra_args)
            else:
                info('First build appeared to complete correctly, skipping manual'
                     'cythonising.')
            if not '--gdb' in self.cython_args:
                # Then we don't want debug info...

                if 'python2' in self.ctx.recipe_build_order:
                    info('Stripping object files')
                    build_lib = glob.glob('./build/lib*')
                    shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                            env['STRIP'], '{}', ';', _env=env)

                if self.ctx.python_recipe.from_crystax:
                    info('Stripping object files')
                    shprint(sh.find, '.', '-iname', '*.so', '-exec',
                            '/usr/bin/echo', '{}', ';', _env=env)
                    shprint(sh.find, '.', '-iname', '*.so', '-exec',
                            env['STRIP'].split(' ')[0], '--strip-unneeded',
                            # '/usr/bin/strip', '--strip-unneeded',
                            '{}', ';', _env=env)
Exemplo n.º 33
0
def run_pymodules_install(ctx, modules):
    modules = list(filter(ctx.not_has_package, modules))

    if not modules:
        info('There are no Python modules to install, skipping')
        return

    info('The requirements ({}) don\'t have recipes, attempting to install '
         'them with pip'.format(', '.join(modules)))
    info('If this fails, it may mean that the module has compiled '
         'components and needs a recipe.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(
            venv, '--python=python{}.{}'.format(
                ctx.python_recipe.major_minor_version_string.partition(".")[0],
                ctx.python_recipe.major_minor_version_string.partition(".")
                [2]), 'venv')

        info('Creating a requirements.txt file for the Python modules')
        with open('requirements.txt', 'w') as fileh:
            for module in modules:
                key = 'VERSION_' + module
                if key in environ:
                    line = '{}=={}\n'.format(module, environ[key])
                else:
                    line = '{}\n'.format(module)
                fileh.write(line)

        # Prepare base environment and upgrade pip:
        base_env = copy.copy(os.environ)
        base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
        info('Upgrade pip to latest version')
        shprint(sh.bash,
                '-c', ("source venv/bin/activate && pip install -U pip"),
                _env=copy.copy(base_env))

        # Install Cython in case modules need it to build:
        info('Install Cython in case one of the modules needs it to build')
        shprint(sh.bash,
                '-c', ("venv/bin/pip install Cython"),
                _env=copy.copy(base_env))

        # Get environment variables for build (with CC/compiler set):
        standard_recipe = CythonRecipe()
        standard_recipe.ctx = ctx
        # (note: following line enables explicit -lpython... linker options)
        standard_recipe.call_hostpython_via_targetpython = False
        recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
        env = copy.copy(base_env)
        env.update(recipe_env)

        info('Installing Python modules with pip')
        info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
             'A reason for this is often modules compiling '
             'native code that is unaware of Android cross-compilation '
             'and does not work without additional '
             'changes / workarounds.')

        # Make sure our build package dir is available, and the virtualenv
        # site packages come FIRST (so the proper pip version is used):
        env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
        env["PYTHONPATH"] = os.path.abspath(
            join(ctx.build_dir, "venv", "lib",
                 "python" + ctx.python_recipe.major_minor_version_string,
                 "site-packages")) + ":" + env["PYTHONPATH"]
        '''
        # Do actual install:
        shprint(sh.bash, '-c', (
            "venv/bin/pip " +
            "install -v --target '{0}' --no-deps -r requirements.txt"
        ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
                _env=copy.copy(env))
        '''

        # use old install script
        shprint(sh.bash, '-c', (
            "source venv/bin/activate && env CC=/bin/false CXX=/bin/false "
            "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt"
        ).format(ctx.get_site_packages_dir()))

        # Strip object files after potential Cython or native code builds:
        standard_recipe.strip_object_files(ctx.archs[0],
                                           env,
                                           build_dir=ctx.build_dir)
    def download(self):
        if self.url is None:
            info('Skipping {} download as no URL is set'.format(self.name))
            return

        url = self.versioned_url
        expected_digests = {}
        for alg in set(hashlib.algorithms_guaranteed) | set(
            ('md5', 'sha512', 'blake2b')):
            expected_digest = getattr(self, alg +
                                      'sum') if hasattr(self, alg +
                                                        'sum') else None
            ma = match(u'^(.+)#' + alg + u'=([0-9a-f]{32,})$', url)
            if ma:  # fragmented URL?
                if expected_digest:
                    raise ValueError(('Received {}sum from both the {} recipe '
                                      'and its url').format(alg, self.name))
                url = ma.group(1)
                expected_digest = ma.group(2)
            if expected_digest:
                expected_digests[alg] = expected_digest

        shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name))

        with current_directory(join(self.ctx.packages_path, self.name)):
            filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')

            do_download = True
            marker_filename = '.mark-{}'.format(filename)
            if exists(filename) and isfile(filename):
                if not exists(marker_filename):
                    shprint(sh.rm, filename)
                else:
                    for alg, expected_digest in expected_digests.items():
                        current_digest = algsum(alg, filename)
                        if current_digest != expected_digest:
                            debug('* Generated {}sum: {}'.format(
                                alg, current_digest))
                            debug('* Expected {}sum: {}'.format(
                                alg, expected_digest))
                            raise ValueError((
                                'Generated {0}sum does not match expected {0}sum '
                                'for {1} recipe').format(alg, self.name))
                    do_download = False

            # If we got this far, we will download
            if do_download:
                debug('Downloading {} from {}'.format(self.name, url))

                shprint(sh.rm, '-f', marker_filename)
                self.download_file(self.versioned_url, filename)
                shprint(sh.touch, marker_filename)

                if exists(filename) and isfile(filename):
                    for alg, expected_digest in expected_digests.items():
                        current_digest = algsum(alg, filename)
                        if current_digest != expected_digest:
                            debug('* Generated {}sum: {}'.format(
                                alg, current_digest))
                            debug('* Expected {}sum: {}'.format(
                                alg, expected_digest))
                            raise ValueError((
                                'Generated {0}sum does not match expected {0}sum '
                                'for {1} recipe').format(alg, self.name))
            else:
                info('{} download already cached, skipping'.format(self.name))
Exemplo n.º 35
0
def run_pymodules_install(ctx,
                          modules,
                          project_dir=None,
                          ignore_setup_py=False):
    """ This function will take care of all non-recipe things, by:

        1. Processing them from --requirements (the modules argument)
           and installing them

        2. Installing the user project/app itself via setup.py if
           ignore_setup_py=True

    """

    info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
    modules = list(filter(ctx.not_has_package, modules))

    # We change current working directory later, so this has to be an absolute
    # path or `None` in case that we didn't supply the `project_dir` via kwargs
    project_dir = abspath(project_dir) if project_dir else None

    # Bail out if no python deps and no setup.py to process:
    if not modules and (ignore_setup_py or project_dir is None
                        or not project_has_setup_py(project_dir)):
        info('No Python modules and no setup.py to process, skipping')
        return

    # Output messages about what we're going to do:
    if modules:
        info('The requirements ({}) don\'t have recipes, attempting to '
             'install them with pip'.format(', '.join(modules)))
        info('If this fails, it may mean that the module has compiled '
             'components and needs a recipe.')
    if project_dir is not None and \
            project_has_setup_py(project_dir) and not ignore_setup_py:
        info('Will process project install, if it fails then the '
             'project may not be compatible for Android install.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(
            venv, '--python=python{}'.format(
                ctx.python_recipe.major_minor_version_string.partition(".")
                [0]), 'venv')

        # Prepare base environment and upgrade pip:
        base_env = copy.copy(os.environ)
        base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
        info('Upgrade pip to latest version')
        shprint(sh.bash,
                '-c', ("source venv/bin/activate && pip install -U pip"),
                _env=copy.copy(base_env))

        # Install Cython in case modules need it to build:
        info('Install Cython in case one of the modules needs it to build')
        shprint(sh.bash,
                '-c', ("venv/bin/pip install Cython"),
                _env=copy.copy(base_env))

        # Get environment variables for build (with CC/compiler set):
        standard_recipe = CythonRecipe()
        standard_recipe.ctx = ctx
        # (note: following line enables explicit -lpython... linker options)
        standard_recipe.call_hostpython_via_targetpython = False
        recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
        env = copy.copy(base_env)
        env.update(recipe_env)

        # Make sure our build package dir is available, and the virtualenv
        # site packages come FIRST (so the proper pip version is used):
        env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
        env["PYTHONPATH"] = os.path.abspath(
            join(ctx.build_dir, "venv", "lib",
                 "python" + ctx.python_recipe.major_minor_version_string,
                 "site-packages")) + ":" + env["PYTHONPATH"]

        # Install the manually specified requirements first:
        if not modules:
            info('There are no Python modules to install, skipping')
        else:
            info('Creating a requirements.txt file for the Python modules')
            with open('requirements.txt', 'w') as fileh:
                for module in modules:
                    key = 'VERSION_' + module
                    if key in environ:
                        line = '{}=={}\n'.format(module, environ[key])
                    else:
                        line = '{}\n'.format(module)
                    fileh.write(line)

            info('Installing Python modules with pip')
            info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
                 'A reason for this is often modules compiling '
                 'native code that is unaware of Android cross-compilation '
                 'and does not work without additional '
                 'changes / workarounds.')

            shprint(sh.bash,
                    '-c',
                    ("venv/bin/pip " +
                     "install -v --target '{0}' --no-deps -r requirements.txt"
                     ).format(ctx.get_site_packages_dir().replace(
                         "'", "'\"'\"'")),
                    _env=copy.copy(env))

        # Afterwards, run setup.py if present:
        if project_dir is not None and (project_has_setup_py(project_dir)
                                        and not ignore_setup_py):
            run_setuppy_install(ctx, project_dir, env)
        elif not ignore_setup_py:
            info("No setup.py found in project directory: " + str(project_dir))

        # Strip object files after potential Cython or native code builds:
        standard_recipe.strip_object_files(ctx.archs[0],
                                           env,
                                           build_dir=ctx.build_dir)
Exemplo n.º 36
0
    def unpack(self, arch):
        info_main('Unpacking {} for {}'.format(self.name, arch))

        build_dir = self.get_build_container_dir(arch)

        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
        if user_dir is not None:
            info('P4A_{}_DIR exists, symlinking instead'.format(
                self.name.lower()))
            if exists(self.get_build_dir(arch)):
                return
            shprint(sh.rm, '-rf', build_dir)
            shprint(sh.mkdir, '-p', build_dir)
            shprint(sh.rmdir, build_dir)
            ensure_dir(build_dir)
            shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
            return

        if self.url is None:
            info('Skipping {} unpack as no URL is set'.format(self.name))
            return

        filename = shprint(sh.basename,
                           self.versioned_url).stdout[:-1].decode('utf-8')
        ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename)
        if ma:  # fragmented URL?
            filename = ma.group(1)

        with current_directory(build_dir):
            directory_name = self.get_build_dir(arch)

            if not exists(directory_name) or not isdir(directory_name):
                extraction_filename = join(self.ctx.packages_path, self.name,
                                           filename)
                if isfile(extraction_filename):
                    if extraction_filename.endswith('.zip'):
                        try:
                            sh.unzip(extraction_filename)
                        except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2):
                            # return code 1 means unzipping had
                            # warnings but did complete,
                            # apparently happens sometimes with
                            # github zips
                            pass
                        import zipfile
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
                        root_directory = fileh.filelist[0].filename.split(
                            '/')[0]
                        if root_directory != basename(directory_name):
                            shprint(sh.mv, root_directory, directory_name)
                    elif extraction_filename.endswith(
                        ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz',
                         '.txz')):
                        sh.tar('xf', extraction_filename)
                        root_directory = sh.tar(
                            'tf', extraction_filename).stdout.decode(
                                'utf-8').split('\n')[0].split('/')[0]
                        if root_directory != basename(directory_name):
                            shprint(sh.mv, root_directory, directory_name)
                    else:
                        raise Exception(
                            'Could not extract {} download, it must be .zip, '
                            '.tar.gz or .tar.bz2 or .tar.xz'.format(
                                extraction_filename))
                elif isdir(extraction_filename):
                    mkdir(directory_name)
                    for entry in listdir(extraction_filename):
                        if entry not in ('.git', ):
                            shprint(sh.cp, '-Rv',
                                    join(extraction_filename, entry),
                                    directory_name)
                else:
                    raise Exception(
                        'Given path is neither a file nor a directory: {}'.
                        format(extraction_filename))

            else:
                info('{} is already unpacked, skipping'.format(self.name))
Exemplo n.º 37
0
 def prebuild_arch(self, arch):
     # Override hostpython Setup?
     shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'),
             join(self.get_build_dir(), 'Modules', 'Setup'))
Exemplo n.º 38
0
    def build_arch(self, arch):
        build_dir = join(self.get_build_dir(arch.arch), 'build')
        shprint(sh.mkdir, '-p', build_dir)

        opencv_extras = []
        if 'opencv_extras' in self.ctx.recipe_build_order:
            opencv_extras_dir = self.get_recipe(
                'opencv_extras', self.ctx).get_build_dir(arch.arch)
            opencv_extras = [
                f'-DOPENCV_EXTRA_MODULES_PATH={opencv_extras_dir}/modules',
                '-DBUILD_opencv_legacy=OFF',
            ]

        with current_directory(build_dir):
            env = self.get_recipe_env(arch)

            python_major = self.ctx.python_recipe.version[0]
            python_include_root = self.ctx.python_recipe.include_root(
                arch.arch)
            python_site_packages = self.ctx.get_site_packages_dir()
            python_link_root = self.ctx.python_recipe.link_root(arch.arch)
            python_link_version = self.ctx.python_recipe.major_minor_version_string
            if 'python3' in self.ctx.python_recipe.name:
                python_link_version += 'm'
            python_library = join(python_link_root,
                                  'libpython{}.so'.format(python_link_version))
            python_include_numpy = join(python_site_packages, 'numpy', 'core',
                                        'include')

            shprint(
                sh.cmake,
                '-DP4A=ON',
                '-DANDROID_ABI={}'.format(arch.arch),
                '-DANDROID_STANDALONE_TOOLCHAIN={}'.format(self.ctx.ndk_dir),
                '-DANDROID_NATIVE_API_LEVEL={}'.format(self.ctx.ndk_api),
                '-DANDROID_EXECUTABLE={}/tools/android'.format(
                    env['ANDROID_SDK']),
                '-DCMAKE_TOOLCHAIN_FILE={}'.format(
                    join(self.ctx.ndk_dir, 'build', 'cmake',
                         'android.toolchain.cmake')),
                # Make the linkage with our python library, otherwise we
                # will get dlopen error when trying to import cv2's module.
                '-DCMAKE_SHARED_LINKER_FLAGS=-L{path} -lpython{version}'.
                format(path=python_link_root, version=python_link_version),
                '-DBUILD_WITH_STANDALONE_TOOLCHAIN=ON',
                # Force to build as shared libraries the cv2's dependant
                # libs or we will not be able to link with our python
                '-DBUILD_SHARED_LIBS=ON',
                '-DBUILD_STATIC_LIBS=OFF',

                # Disable some opencv's features
                '-DBUILD_opencv_java=OFF',
                '-DBUILD_opencv_java_bindings_generator=OFF',
                # '-DBUILD_opencv_highgui=OFF',
                # '-DBUILD_opencv_imgproc=OFF',
                # '-DBUILD_opencv_flann=OFF',
                '-DBUILD_TESTS=OFF',
                '-DBUILD_PERF_TESTS=OFF',
                '-DENABLE_TESTING=OFF',
                '-DBUILD_EXAMPLES=OFF',
                '-DBUILD_ANDROID_EXAMPLES=OFF',

                # Force to only build our version of python
                '-DBUILD_OPENCV_PYTHON{major}=ON'.format(major=python_major),
                '-DBUILD_OPENCV_PYTHON{major}=OFF'.format(
                    major='2' if python_major == '3' else '3'),

                # Force to install the `cv2.so` library directly into
                # python's site packages (otherwise the cv2's loader fails
                # on finding the cv2.so library)
                '-DOPENCV_SKIP_PYTHON_LOADER=ON',
                '-DOPENCV_PYTHON{major}_INSTALL_PATH={site_packages}'.format(
                    major=python_major, site_packages=python_site_packages),

                # Define python's paths for: exe, lib, includes, numpy...
                '-DPYTHON_DEFAULT_EXECUTABLE={}'.format(self.ctx.hostpython),
                '-DPYTHON{major}_EXECUTABLE={host_python}'.format(
                    major=python_major, host_python=self.ctx.hostpython),
                '-DPYTHON{major}_INCLUDE_PATH={include_path}'.format(
                    major=python_major, include_path=python_include_root),
                '-DPYTHON{major}_LIBRARIES={python_lib}'.format(
                    major=python_major, python_lib=python_library),
                '-DPYTHON{major}_NUMPY_INCLUDE_DIRS={numpy_include}'.format(
                    major=python_major, numpy_include=python_include_numpy),
                '-DPYTHON{major}_PACKAGES_PATH={site_packages}'.format(
                    major=python_major, site_packages=python_site_packages),
                *opencv_extras,
                self.get_build_dir(arch.arch),
                _env=env)
            shprint(sh.make, '-j' + str(cpu_count()),
                    'opencv_python' + python_major)
            # Install python bindings (cv2.so)
            shprint(sh.cmake, '-DCOMPONENT=python', '-P',
                    './cmake_install.cmake')
            # Copy third party shared libs that we need in our final apk
            sh.cp('-a', sh.glob('./lib/{}/lib*.so'.format(arch.arch)),
                  self.ctx.get_libs_dir(arch.arch))
Exemplo n.º 39
0
    def download_file(self, url, target, cwd=None):
        """
        (internal) Download an ``url`` to a ``target``.
        """
        if not url:
            return
        info('Downloading {} from {}'.format(self.name, url))

        if cwd:
            target = join(cwd, target)

        parsed_url = urlparse(url)
        if parsed_url.scheme in ('http', 'https'):

            def report_hook(index, blksize, size):
                if size <= 0:
                    progression = '{0} bytes'.format(index * blksize)
                else:
                    progression = '{0:.2f}%'.format(index * blksize * 100. /
                                                    float(size))
                if "CI" not in environ:
                    stdout.write('- Download {}\r'.format(progression))
                    stdout.flush()

            if exists(target):
                unlink(target)

            # Download item with multiple attempts (for bad connections):
            attempts = 0
            while True:
                try:
                    urlretrieve(url, target, report_hook)
                except OSError:
                    attempts += 1
                    if attempts >= 5:
                        raise
                    stdout.write('Download failed retrying in a second...')
                    time.sleep(1)
                    continue
                break
            return target
        elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http',
                                   'git+https'):
            if isdir(target):
                with current_directory(target):
                    shprint(sh.git, 'fetch', '--tags')
                    if self.version:
                        shprint(sh.git, 'checkout', self.version)
                    shprint(sh.git, 'pull')
                    shprint(sh.git, 'pull', '--recurse-submodules')
                    shprint(sh.git, 'submodule', 'update', '--recursive')
            else:
                if url.startswith('git+'):
                    url = url[4:]
                shprint(sh.git, 'clone', '--recursive', url, target)
                if self.version:
                    with current_directory(target):
                        shprint(sh.git, 'checkout', self.version)
                        shprint(sh.git, 'submodule', 'update', '--recursive')
            return target
def copylibs_function(soname, objs_paths, extra_link_dirs=[], env=None):
    print('objs_paths are', objs_paths)

    re_needso = re.compile(
        r'^.*\(NEEDED\)\s+Shared library: \[lib(.*)\.so\]\s*$')
    blacklist_libs = (
        'c',
        'stdc++',
        'dl',
        'python2.7',
        'sdl',
        'sdl_image',
        'sdl_ttf',
        'z',
        'm',
        'GLESv2',
        'jpeg',
        'png',
        'log',

        # bootstrap takes care of sdl2 libs (if applicable)
        'SDL2',
        'SDL2_ttf',
        'SDL2_image',
        'SDL2_mixer',
    )
    found_libs = []
    sofiles = []
    if env and 'READELF' in env:
        readelf = env['READELF']
    elif 'READELF' in os.environ:
        readelf = os.environ['READELF']
    else:
        readelf = sh.which('readelf').strip()
    readelf = sh.Command(readelf).bake('-d')

    dest = dirname(soname)

    for directory in objs_paths:
        for fn in os.listdir(directory):
            fn = join(directory, fn)

            if not fn.endswith('.libs'):
                continue

            dirfn = fn[:-1] + 'dirs'
            if not exists(dirfn):
                continue

            with open(fn) as f:
                libs = f.read().strip().split(' ')
                needed_libs = [
                    lib for lib in libs if lib and lib not in blacklist_libs
                    and lib not in found_libs
                ]

            while needed_libs:
                print('need libs:\n\t' + '\n\t'.join(needed_libs))

                start_needed_libs = needed_libs[:]
                found_sofiles = []

                with open(dirfn) as f:
                    libdirs = f.read().split()
                    for libdir in libdirs:
                        if not needed_libs:
                            break

                        if libdir == dest:
                            # don't need to copy from dest to dest!
                            continue

                        libdir = libdir.strip()
                        print('scanning', libdir)
                        for lib in needed_libs[:]:
                            if lib in found_libs:
                                continue

                            if lib.endswith('.a'):
                                needed_libs.remove(lib)
                                found_libs.append(lib)
                                continue

                            lib_a = 'lib' + lib + '.a'
                            libpath_a = join(libdir, lib_a)
                            lib_so = 'lib' + lib + '.so'
                            libpath_so = join(libdir, lib_so)
                            plain_so = lib + '.so'
                            plainpath_so = join(libdir, plain_so)

                            sopath = None
                            if exists(libpath_so):
                                sopath = libpath_so
                            elif exists(plainpath_so):
                                sopath = plainpath_so

                            if sopath:
                                print('found', lib, 'in', libdir)
                                found_sofiles.append(sopath)
                                needed_libs.remove(lib)
                                found_libs.append(lib)
                                continue

                            if exists(libpath_a):
                                print('found', lib, '(static) in', libdir)
                                needed_libs.remove(lib)
                                found_libs.append(lib)
                                continue

                for sofile in found_sofiles:
                    print('scanning dependencies for', sofile)
                    out = readelf(sofile)
                    for line in out.splitlines():
                        needso = re_needso.match(line)
                        if needso:
                            lib = needso.group(1)
                            if (lib not in needed_libs
                                    and lib not in found_libs
                                    and lib not in blacklist_libs):
                                needed_libs.append(needso.group(1))

                sofiles += found_sofiles

                if needed_libs == start_needed_libs:
                    raise RuntimeError(
                        'Failed to locate needed libraries!\n\t' +
                        '\n\t'.join(needed_libs))

    print('Copying libraries')
    for lib in sofiles:
        shprint(sh.cp, lib, dest)
Exemplo n.º 41
0
    def apk(self, args):
        """Create an APK using the given distribution."""

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        fix_args = ('--dir', '--private', '--add-jar', '--add-source',
                    '--whitelist', '--blacklist', '--presplash', '--icon')
        unknown_args = args.unknown_args
        for i, arg in enumerate(unknown_args):
            argx = arg.split('=')
            if argx[0] in fix_args:
                if len(argx) > 1:
                    unknown_args[i] = '='.join(
                        (argx[0], realpath(expanduser(argx[1]))))
                else:
                    unknown_args[i+1] = realpath(expanduser(unknown_args[i+1]))

        env = os.environ.copy()
        if args.build_mode == 'release':
            if args.keystore:
                env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore))
            if args.signkey:
                env['P4A_RELEASE_KEYALIAS'] = args.signkey
            if args.keystorepw:
                env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
            if args.signkeypw:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
            elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            self.hook("before_apk_build")
            os.environ["ANDROID_API"] = str(self.ctx.android_api)
            build_args = build.parse_args(args.unknown_args)
            self.hook("after_apk_build")
            self.hook("before_apk_assemble")

            build_type = ctx.java_build_tool
            if build_type == 'auto':
                info('Selecting java build tool:')

                build_tools_versions = os.listdir(join(ctx.sdk_dir,
                                                       'build-tools'))
                build_tools_versions = sorted(build_tools_versions,
                                              key=LooseVersion)
                build_tools_version = build_tools_versions[-1]
                info(('Detected highest available build tools '
                      'version to be {}').format(build_tools_version))

                if build_tools_version >= '25.0' and exists('gradlew'):
                    build_type = 'gradle'
                    info('    Building with gradle, as gradle executable is '
                         'present')
                else:
                    build_type = 'ant'
                    if build_tools_version < '25.0':
                        info(('    Building with ant, as the highest '
                              'build-tools-version is only {}').format(
                            build_tools_version))
                    else:
                        info('    Building with ant, as no gradle executable '
                             'detected')

            if build_type == 'gradle':
                # gradle-based build
                env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir
                env["ANDROID_HOME"] = self.ctx.sdk_dir

                gradlew = sh.Command('./gradlew')
                if exists('/usr/bin/dos2unix'):
                    # .../dists/bdisttest_python3/gradlew
                    # .../build/bootstrap_builds/sdl2-python3crystax/gradlew
                    # if docker on windows, gradle contains CRLF
                    output = shprint(
                        sh.Command('dos2unix'), gradlew._path.decode('utf8'),
                        _tail=20, _critical=True, _env=env
                    )
                if args.build_mode == "debug":
                    gradle_task = "assembleDebug"
                elif args.build_mode == "release":
                    gradle_task = "assembleRelease"
                else:
                    error("Unknown build mode {} for apk()".format(
                        args.build_mode))
                    exit(1)
                output = shprint(gradlew, gradle_task, _tail=20,
                                 _critical=True, _env=env)

                # gradle output apks somewhere else
                # and don't have version in file
                apk_dir = join(dist.dist_dir,
                               "build", "outputs", "apk",
                               args.build_mode)
                apk_glob = "*-{}.apk"
                apk_add_version = True

            else:
                # ant-based build
                try:
                    ant = sh.Command('ant')
                except sh.CommandNotFound:
                    error('Could not find ant binary, please install it '
                          'and make sure it is in your $PATH.')
                    exit(1)
                output = shprint(ant, args.build_mode, _tail=20,
                                 _critical=True, _env=env)
                apk_dir = join(dist.dist_dir, "bin")
                apk_glob = "*-*-{}.apk"
                apk_add_version = False

            self.hook("after_apk_assemble")

        info_main('# Copying APK to current directory')

        apk_re = re.compile(r'.*Package: (.*\.apk)$')
        apk_file = None
        for line in reversed(output.splitlines()):
            m = apk_re.match(line)
            if m:
                apk_file = m.groups()[0]
                break

        if not apk_file:
            info_main('# APK filename not found in build output. Guessing...')
            if args.build_mode == "release":
                suffixes = ("release", "release-unsigned")
            else:
                suffixes = ("debug", )
            for suffix in suffixes:
                apks = glob.glob(join(apk_dir, apk_glob.format(suffix)))
                if apks:
                    if len(apks) > 1:
                        info('More than one built APK found... guessing you '
                             'just built {}'.format(apks[-1]))
                    apk_file = apks[-1]
                    break
            else:
                raise ValueError('Couldn\'t find the built APK')

        info_main('# Found APK file: {}'.format(apk_file))
        if apk_add_version:
            info('# Add version number to APK')
            apk_name, apk_suffix = basename(apk_file).split("-", 1)
            apk_file_dest = "{}-{}-{}".format(
                apk_name, build_args.version, apk_suffix)
            info('# APK renamed to {}'.format(apk_file_dest))
            shprint(sh.cp, apk_file, apk_file_dest)
        else:
            shprint(sh.cp, apk_file, './')
Exemplo n.º 42
0
    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        # Todo: find a better way to find the build libs folder
        modules_build_dir = join(
            self.get_build_dir(arch.arch), 'android-build', 'build',
            'lib.linux{}-{}-{}'.format('2' if self.version[0] == '2' else '',
                                       arch.command_prefix.split('-')[0],
                                       self.major_minor_version_string))

        # Compile to *.pyc/*.pyo the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc/*.pyo the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc/*.pyo the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir())

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(
                walk_valid_filens('.', self.stdlib_dir_blacklist,
                                  self.stdlib_filen_blacklist))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir())
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir()):
            filens = list(
                walk_valid_filens('.', self.site_packages_dir_blacklist,
                                  self.site_packages_filen_blacklist))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch), 'android-build')
        python_lib_name = 'libpython' + self.major_minor_version_string
        if self.major_minor_version_string[0] == '3':
            python_lib_name += 'm'
        shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'),
                join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch))

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')
Exemplo n.º 43
0
 def prebuild_arch(self, arch):
     if self.is_patched(arch):
         return
     shprint(sh.cp, join(self.get_recipe_dir(), 'Setup'),
             join(self.get_build_dir(arch.arch), 'Setup'))
Exemplo n.º 44
0
def test_init_package():
    if os.path.exists('tmp/enaml-native-test'):
        sh.rm('-R', 'tmp/enaml-native-test')
    cmd = sh.Command('enaml-native')
    shprint(cmd, 'init-package', 'enaml-native-test', 'tmp/', _debug=True)
Exemplo n.º 45
0
    def apk(self, args):
        '''Create an APK using the given distribution.'''

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        fix_args = ('--dir', '--private', '--add-jar', '--add-source',
                    '--whitelist', '--blacklist', '--presplash', '--icon')
        unknown_args = args.unknown_args
        for i, arg in enumerate(unknown_args[:-1]):
            argx = arg.split('=')
            if argx[0] in fix_args:
                if len(argx) > 1:
                    unknown_args[i] = '='.join(
                        (argx[0], realpath(expanduser(argx[1]))))
                else:
                    unknown_args[i + 1] = realpath(
                        expanduser(unknown_args[i + 1]))

        env = os.environ.copy()
        if args.build_mode == 'release':
            if args.keystore:
                env['P4A_RELEASE_KEYSTORE'] = realpath(
                    expanduser(args.keystore))
            if args.signkey:
                env['P4A_RELEASE_KEYALIAS'] = args.signkey
            if args.keystorepw:
                env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
            if args.signkeypw:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
            elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            build_args = build.parse_args(args.unknown_args)
            output = shprint(sh.ant,
                             args.build_mode,
                             _tail=20,
                             _critical=True,
                             _env=env)

        info_main('# Copying APK to current directory')

        apk_re = re.compile(r'.*Package: (.*\.apk)$')
        apk_file = None
        for line in reversed(output.splitlines()):
            m = apk_re.match(line)
            if m:
                apk_file = m.groups()[0]
                break

        if not apk_file:
            info_main(
                '# APK filename not found in build output, trying to guess')
            apks = glob.glob(
                join(dist.dist_dir, 'bin',
                     '*-*-{}.apk'.format(args.build_mode)))
            if len(apks) == 0:
                raise ValueError('Couldn\'t find the built APK')
            if len(apks) > 1:
                info('More than one built APK found...guessing you '
                     'just built {}'.format(apks[-1]))
            apk_file = apks[-1]

        info_main('# Found APK file: {}'.format(apk_file))
        shprint(sh.cp, apk_file, './')
Exemplo n.º 46
0
    def download_file(self, url, target, cwd=None):
        """
        (internal) Download an ``url`` to a ``target``.
        """
        if not url:
            return
        info('Downloading {} from {}'.format(self.name, url))

        if cwd:
            target = join(cwd, target)

        parsed_url = urlparse(url)
        if parsed_url.scheme in ('http', 'https'):

            def report_hook(index, blksize, size):
                if size <= 0:
                    progression = '{0} bytes'.format(index * blksize)
                else:
                    progression = '{0:.2f}%'.format(index * blksize * 100. /
                                                    float(size))
                stdout.write('- Download {}\r'.format(progression))
                stdout.flush()

            if exists(target):
                unlink(target)

            urlretrieve(url, target, report_hook)
            return target
        elif parsed_url.scheme in ('git', 'git+ssh', 'git+http', 'git+https'):
            if isdir(target):
                with current_directory(target):
                    shprint(sh.git, 'fetch', '--tags')
                    if self.version:
                        shprint(sh.git, 'checkout', self.version)
                    shprint(sh.git, 'pull')
                    shprint(sh.git, 'pull', '--recurse-submodules')
                    shprint(sh.git, 'submodule', 'update', '--recursive')
            else:
                if url.startswith('git+'):
                    url = url[4:]
                shprint(sh.git, 'clone', '--recursive', url, target)
                if self.version:
                    with current_directory(target):
                        shprint(sh.git, 'checkout', self.version)
                        shprint(sh.git, 'submodule', 'update', '--recursive')
            return target
Exemplo n.º 47
0
    def download(self):
        if self.url is None:
            info('Skipping {} download as no URL is set'.format(self.name))
            return

        url = self.versioned_url

        shprint(sh.mkdir, '-p', join(self.ctx.packages_path, self.name))

        with current_directory(join(self.ctx.packages_path, self.name)):
            filename = shprint(sh.basename, url).stdout[:-1].decode('utf-8')

            do_download = True

            marker_filename = '.mark-{}'.format(filename)
            if exists(filename) and isfile(filename):
                if not exists(marker_filename):
                    shprint(sh.rm, filename)
                elif self.md5sum:
                    current_md5 = shprint(sh.md5sum, filename).split()[0]
                    if current_md5 == self.md5sum:
                        debug('Downloaded expected content!')
                        do_download = False
                    else:
                        info('Downloaded unexpected content...')
                        debug('* Generated md5sum: {}'.format(current_md5))
                        debug('* Expected md5sum: {}'.format(self.md5sum))

                else:
                    do_download = False
                    info('{} download already cached, skipping'.format(
                        self.name))

            # Should check headers here!
            warning('Should check headers here! Skipping for now.')

            # If we got this far, we will download
            if do_download:
                debug('Downloading {} from {}'.format(self.name, url))

                shprint(sh.rm, '-f', marker_filename)
                self.download_file(url, filename)
                shprint(sh.touch, marker_filename)

                if exists(filename) and isfile(filename) and self.md5sum:
                    current_md5 = shprint(sh.md5sum, filename).split()[0]
                    if self.md5sum is not None:
                        if current_md5 == self.md5sum:
                            debug('Downloaded expected content!')
                        else:
                            info('Downloaded unexpected content...')
                            debug('* Generated md5sum: {}'.format(current_md5))
                            debug('* Expected md5sum: {}'.format(self.md5sum))
                            exit(1)
Exemplo n.º 48
0
    def unpack(self, arch):
        info_main('Unpacking {} for {}'.format(self.name, arch))

        build_dir = self.get_build_container_dir(arch)

        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
        if user_dir is not None:
            info('P4A_{}_DIR exists, symlinking instead'.format(
                self.name.lower()))
            # AND: Currently there's something wrong if I use ln, fix this
            warning('Using cp -a instead of symlink...fix this!')
            if exists(self.get_build_dir(arch)):
                return
            shprint(sh.rm, '-rf', build_dir)
            shprint(sh.mkdir, '-p', build_dir)
            shprint(sh.rmdir, build_dir)
            ensure_dir(build_dir)
            shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
            return

        if self.url is None:
            info('Skipping {} unpack as no URL is set'.format(self.name))
            return

        filename = shprint(sh.basename,
                           self.versioned_url).stdout[:-1].decode('utf-8')

        with current_directory(build_dir):
            directory_name = self.get_build_dir(arch)

            # AND: Could use tito's get_archive_rootdir here
            if not exists(directory_name) or not isdir(directory_name):
                extraction_filename = join(self.ctx.packages_path, self.name,
                                           filename)
                if isfile(extraction_filename):
                    if extraction_filename.endswith('.zip'):
                        sh.unzip(extraction_filename)
                        import zipfile
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
                        root_directory = fileh.filelist[0].filename.split(
                            '/')[0]
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    elif (extraction_filename.endswith('.tar.gz')
                          or extraction_filename.endswith('.tgz')
                          or extraction_filename.endswith('.tar.bz2')
                          or extraction_filename.endswith('.tbz2')
                          or extraction_filename.endswith('.tar.xz')
                          or extraction_filename.endswith('.txz')):
                        sh.tar('xf', extraction_filename)
                        root_directory = shprint(
                            sh.tar, 'tf', extraction_filename).stdout.decode(
                                'utf-8').split('\n')[0].split('/')[0]
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    else:
                        raise Exception(
                            'Could not extract {} download, it must be .zip, '
                            '.tar.gz or .tar.bz2 or .tar.xz')
                elif isdir(extraction_filename):
                    mkdir(directory_name)
                    for entry in listdir(extraction_filename):
                        if entry not in ('.git', ):
                            shprint(sh.cp, '-Rv',
                                    join(extraction_filename, entry),
                                    directory_name)
                else:
                    raise Exception(
                        'Given path is neither a file nor a directory: {}'.
                        format(extraction_filename))

            else:
                info('{} is already unpacked, skipping'.format(self.name))
Exemplo n.º 49
0
 def build_arch(self, arch):
     super(JpegRecipe, self).build_arch(arch)
     with current_directory(self.get_lib_dir(arch)):
         shprint(sh.mv, 'libjpeg.a', 'libjpeg-orig.a')
         shprint(sh.ar, '-rcT', 'libjpeg.a', 'libjpeg-orig.a', 'libsimd.a')
Exemplo n.º 50
0
def run_pymodules_install(ctx,
                          modules,
                          project_dir=None,
                          ignore_setup_py=False):
    """ This function will take care of all non-recipe things, by:

        1. Processing them from --requirements (the modules argument)
           and installing them

        2. Installing the user project/app itself via setup.py if
           ignore_setup_py=True

    """

    info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
    modules = list(filter(ctx.not_has_package, modules))

    # Bail out if no python deps and no setup.py to process:
    if not modules and (ignore_setup_py or project_dir is None
                        or not project_has_setup_py(project_dir)):
        info('No Python modules and no setup.py to process, skipping')
        return

    # Output messages about what we're going to do:
    if modules:
        info('The requirements ({}) don\'t have recipes, attempting to '
             'install them with pip'.format(', '.join(modules)))
        info('If this fails, it may mean that the module has compiled '
             'components and needs a recipe.')
    if project_dir is not None and \
            project_has_setup_py(project_dir) and not ignore_setup_py:
        info('Will process project install, if it fails then the '
             'project may not be compatible for Android install.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(
            venv, '--python=python{}'.format(
                ctx.python_recipe.major_minor_version_string.partition(".")
                [0]), 'venv')

        # Prepare base environment and upgrade pip:
        base_env = copy.copy(os.environ)
        base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
        info('Upgrade pip to latest version')
        shprint(sh.bash,
                '-c', ("source venv/bin/activate && pip install -U pip"),
                _env=copy.copy(base_env))

        # Install Cython in case modules need it to build:
        info('Install Cython in case one of the modules needs it to build')
        shprint(sh.bash,
                '-c', ("venv/bin/pip install Cython"),
                _env=copy.copy(base_env))

        # Get environment variables for build (with CC/compiler set):
        standard_recipe = CythonRecipe()
        standard_recipe.ctx = ctx
        # (note: following line enables explicit -lpython... linker options)
        standard_recipe.call_hostpython_via_targetpython = False
        recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
        env = copy.copy(base_env)
        env.update(recipe_env)

        # Make sure our build package dir is available, and the virtualenv
        # site packages come FIRST (so the proper pip version is used):
        env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
        env["PYTHONPATH"] = os.path.abspath(
            join(ctx.build_dir, "venv", "lib",
                 "python" + ctx.python_recipe.major_minor_version_string,
                 "site-packages")) + ":" + env["PYTHONPATH"]

        # Install the manually specified requirements first:
        if not modules:
            info('There are no Python modules to install, skipping')
        else:
            info('Creating a requirements.txt file for the Python modules')
            with open('requirements.txt', 'w') as fileh:
                for module in modules:
                    key = 'VERSION_' + module
                    if key in environ:
                        line = '{}=={}\n'.format(module, environ[key])
                    else:
                        line = '{}\n'.format(module)
                    fileh.write(line)

            info('Installing Python modules with pip')
            info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
                 'A reason for this is often modules compiling '
                 'native code that is unaware of Android cross-compilation '
                 'and does not work without additional '
                 'changes / workarounds.')

            shprint(sh.bash,
                    '-c',
                    ("venv/bin/pip " +
                     "install -v --target '{0}' --no-deps -r requirements.txt"
                     ).format(ctx.get_site_packages_dir().replace(
                         "'", "'\"'\"'")),
                    _env=copy.copy(env))

        # Afterwards, run setup.py if present:
        if project_dir is not None and (project_has_setup_py(project_dir)
                                        and not ignore_setup_py):
            with current_directory(project_dir):
                info('got setup.py or similar, running project install. ' +
                     '(disable this behavior with --ignore-setup-py)')

                # Compute & output the constraints we will use:
                info('Contents that will be used for constraints.txt:')
                constraints = subprocess.check_output(
                    [join(ctx.build_dir, "venv", "bin", "pip"), "freeze"],
                    env=copy.copy(env))
                try:
                    constraints = constraints.decode("utf-8", "replace")
                except AttributeError:
                    pass
                info(constraints)

                # Make sure all packages found are fixed in version
                # by writing a constraint file, to avoid recipes being
                # upgraded & reinstalled:
                with open('constraints.txt', 'wb') as fileh:
                    fileh.write(constraints.encode("utf-8", "replace"))

                info('Populating venv\'s site-packages with '
                     'ctx.get_site_packages_dir()...')

                # Copy dist contents into site-packages for discovery.
                # Why this is needed:
                # --target is somewhat evil and messes with discovery of
                # packages in PYTHONPATH if that also includes the target
                # folder. So we need to use the regular virtualenv
                # site-packages folder instead.
                # Reference:
                # https://github.com/pypa/pip/issues/6223
                ctx_site_packages_dir = os.path.normpath(
                    os.path.abspath(ctx.get_site_packages_dir()))
                venv_site_packages_dir = os.path.normpath(
                    os.path.join(ctx.build_dir, "venv", "lib", [
                        f for f in os.listdir(
                            os.path.join(ctx.build_dir, "venv", "lib"))
                        if f.startswith("python")
                    ][0], "site-packages"))
                copied_over_contents = []
                for f in os.listdir(ctx_site_packages_dir):
                    full_path = os.path.join(ctx_site_packages_dir, f)
                    if not os.path.exists(
                            os.path.join(venv_site_packages_dir, f)):
                        if os.path.isdir(full_path):
                            shutil.copytree(
                                full_path,
                                os.path.join(venv_site_packages_dir, f))
                        else:
                            shutil.copy2(
                                full_path,
                                os.path.join(venv_site_packages_dir, f))
                        copied_over_contents.append(f)

                # Get listing of virtualenv's site-packages, to see the
                # newly added things afterwards & copy them back into
                # the distribution folder / build context site-packages:
                previous_venv_contents = os.listdir(venv_site_packages_dir)

                # Actually run setup.py:
                info('Launching package install...')
                shprint(sh.bash,
                        '-c',
                        ("'" + join(ctx.build_dir, "venv", "bin",
                                    "pip").replace("'", "'\"'\"'") + "' " +
                         "install -c constraints.txt -v .").format(
                             ctx.get_site_packages_dir().replace(
                                 "'", "'\"'\"'")),
                        _env=copy.copy(env))

                # Go over all new additions and copy them back:
                info('Copying additions resulting from setup.py back ' +
                     'into ctx.get_site_packages_dir()...')
                new_venv_additions = []
                for f in (set(os.listdir(venv_site_packages_dir)) -
                          set(previous_venv_contents)):
                    new_venv_additions.append(f)
                    full_path = os.path.join(venv_site_packages_dir, f)
                    if os.path.isdir(full_path):
                        shutil.copytree(full_path,
                                        os.path.join(ctx_site_packages_dir, f))
                    else:
                        shutil.copy2(full_path,
                                     os.path.join(ctx_site_packages_dir, f))

                # Undo all the changes we did to the venv-site packages:
                info('Reverting additions to virtualenv\'s site-packages...')
                for f in set(copied_over_contents + new_venv_additions):
                    full_path = os.path.join(venv_site_packages_dir, f)
                    if os.path.isdir(full_path):
                        shutil.rmtree(full_path)
                    else:
                        os.remove(full_path)
        elif not ignore_setup_py:
            info("No setup.py found in project directory: " + str(project_dir))

        # Strip object files after potential Cython or native code builds:
        standard_recipe.strip_object_files(ctx.archs[0],
                                           env,
                                           build_dir=ctx.build_dir)
Exemplo n.º 51
0
def run_setuppy_install(ctx, project_dir, env=None):
    if env is None:
        env = dict()

    with current_directory(project_dir):
        info('got setup.py or similar, running project install. ' +
             '(disable this behavior with --ignore-setup-py)')

        # Compute & output the constraints we will use:
        info('Contents that will be used for constraints.txt:')
        constraints = subprocess.check_output(
            [join(ctx.build_dir, "venv", "bin", "pip"), "freeze"],
            env=copy.copy(env))
        try:
            constraints = constraints.decode("utf-8", "replace")
        except AttributeError:
            pass
        info(constraints)

        # Make sure all packages found are fixed in version
        # by writing a constraint file, to avoid recipes being
        # upgraded & reinstalled:
        with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh:
            fileh.write(constraints.encode("utf-8", "replace"))
        try:

            info('Populating venv\'s site-packages with '
                 'ctx.get_site_packages_dir()...')

            # Copy dist contents into site-packages for discovery.
            # Why this is needed:
            # --target is somewhat evil and messes with discovery of
            # packages in PYTHONPATH if that also includes the target
            # folder. So we need to use the regular virtualenv
            # site-packages folder instead.
            # Reference:
            # https://github.com/pypa/pip/issues/6223
            ctx_site_packages_dir = os.path.normpath(
                os.path.abspath(ctx.get_site_packages_dir()))
            venv_site_packages_dir = os.path.normpath(
                os.path.join(ctx.build_dir, "venv", "lib", [
                    f for f in os.listdir(
                        os.path.join(ctx.build_dir, "venv", "lib"))
                    if f.startswith("python")
                ][0], "site-packages"))
            copied_over_contents = []
            for f in os.listdir(ctx_site_packages_dir):
                full_path = os.path.join(ctx_site_packages_dir, f)
                if not os.path.exists(os.path.join(venv_site_packages_dir, f)):
                    if os.path.isdir(full_path):
                        shutil.copytree(
                            full_path, os.path.join(venv_site_packages_dir, f))
                    else:
                        shutil.copy2(full_path,
                                     os.path.join(venv_site_packages_dir, f))
                    copied_over_contents.append(f)

            # Get listing of virtualenv's site-packages, to see the
            # newly added things afterwards & copy them back into
            # the distribution folder / build context site-packages:
            previous_venv_contents = os.listdir(venv_site_packages_dir)

            # Actually run setup.py:
            info('Launching package install...')
            shprint(
                sh.bash,
                '-c',
                ("'" + join(ctx.build_dir, "venv", "bin", "pip").replace(
                    "'", "'\"'\"'") + "' " +
                 "install -c ._tmp_p4a_recipe_constraints.txt -v .").format(
                     ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
                _env=copy.copy(env))

            # Go over all new additions and copy them back:
            info('Copying additions resulting from setup.py back '
                 'into ctx.get_site_packages_dir()...')
            new_venv_additions = []
            for f in (set(os.listdir(venv_site_packages_dir)) -
                      set(previous_venv_contents)):
                new_venv_additions.append(f)
                full_path = os.path.join(venv_site_packages_dir, f)
                if os.path.isdir(full_path):
                    shutil.copytree(full_path,
                                    os.path.join(ctx_site_packages_dir, f))
                else:
                    shutil.copy2(full_path,
                                 os.path.join(ctx_site_packages_dir, f))

            # Undo all the changes we did to the venv-site packages:
            info('Reverting additions to ' 'virtualenv\'s site-packages...')
            for f in set(copied_over_contents + new_venv_additions):
                full_path = os.path.join(venv_site_packages_dir, f)
                if os.path.isdir(full_path):
                    shutil.rmtree(full_path)
                else:
                    os.remove(full_path)
        finally:
            os.remove("._tmp_p4a_recipe_constraints.txt")
Exemplo n.º 52
0
    def build_cython_components(self, arch):
        info('Cythonizing anything necessary in {}'.format(self.name))

        env = self.get_recipe_env(arch)

        if self.ctx.python_recipe.from_crystax:
            command = sh.Command('python{}'.format(
                self.ctx.python_recipe.version))
            site_packages_dirs = command(
                '-c', 'import site; print("\\n".join(site.getsitepackages()))')
            site_packages_dirs = site_packages_dirs.stdout.decode(
                'utf-8').split('\n')
            # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
            if 'PYTHONPATH' in env:
                env['PYTHONPATH'] = env + ':{}'.format(
                    ':'.join(site_packages_dirs))
            else:
                env['PYTHONPATH'] = ':'.join(site_packages_dirs)

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.ctx.hostpython)
            # hostpython = sh.Command('python3.5')
            shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
            print('cwd is', realpath(curdir))
            info('Trying first build of {} to get cython files: this is '
                 'expected to fail'.format(self.name))

            manually_cythonise = False
            try:
                shprint(hostpython,
                        'setup.py',
                        'build_ext',
                        '-v',
                        _env=env,
                        *self.setup_extra_args)
            except sh.ErrorReturnCode_1:
                print()
                info('{} first build failed (as expected)'.format(self.name))
                manually_cythonise = True

            if manually_cythonise:
                self.cythonize_build(env=env)
                shprint(hostpython,
                        'setup.py',
                        'build_ext',
                        '-v',
                        _env=env,
                        _tail=20,
                        _critical=True,
                        *self.setup_extra_args)
            else:
                info(
                    'First build appeared to complete correctly, skipping manual'
                    'cythonising.')

            print('stripping')
            build_lib = glob.glob('./build/lib*')
            shprint(sh.find,
                    build_lib[0],
                    '-name',
                    '*.o',
                    '-exec',
                    env['STRIP'],
                    '{}',
                    ';',
                    _env=env)
            print('stripped!?')
    def download_file(self, url, target, cwd=None):
        """
        (internal) Download an ``url`` to a ``target``.
        """
        if not url:
            return
        info('Downloading {} from {}'.format(self.name, url))

        if cwd:
            target = join(cwd, target)

        parsed_url = urlparse(url)
        if parsed_url.scheme in ('http', 'https'):

            def report_hook(index, blksize, size):
                if size <= 0:
                    progression = '{0} bytes'.format(index * blksize)
                else:
                    progression = '{0:.2f}%'.format(index * blksize * 100. /
                                                    float(size))
                if "CI" not in environ:
                    stdout.write('- Download {}\r'.format(progression))
                    stdout.flush()

            if exists(target):
                unlink(target)

            # Download item with multiple attempts (for bad connections):
            attempts = 0
            seconds = 1
            while True:
                try:
                    # jqueryui.com returns a 403 w/ the default user agent
                    # Mozilla/5.0 doesnt handle redirection for liblzma
                    url_opener.addheaders = [('User-agent', 'Wget/1.0')]
                    urlretrieve(url, target, report_hook)
                except OSError as e:
                    attempts += 1
                    if attempts >= 5:
                        raise
                    stdout.write(
                        'Download failed: {}; retrying in {} second(s)...'.
                        format(e, seconds))
                    time.sleep(seconds)
                    seconds *= 2
                    continue
                finally:
                    url_opener.addheaders = url_orig_headers
                break
            return target
        elif parsed_url.scheme in ('git', 'git+file', 'git+ssh', 'git+http',
                                   'git+https'):
            if isdir(target):
                with current_directory(target):
                    shprint(sh.git, 'fetch', '--tags', '--recurse-submodules')
                    if self.version:
                        shprint(sh.git, 'checkout', self.version)
                    branch = sh.git('branch', '--show-current')
                    if branch:
                        shprint(sh.git, 'pull')
                        shprint(sh.git, 'pull', '--recurse-submodules')
                    shprint(sh.git, 'submodule', 'update', '--recursive')
            else:
                if url.startswith('git+'):
                    url = url[4:]
                shprint(sh.git, 'clone', '--recursive', url, target)
                if self.version:
                    with current_directory(target):
                        shprint(sh.git, 'checkout', self.version)
                        shprint(sh.git, 'submodule', 'update', '--recursive')
            return target
Exemplo n.º 54
0
 def prebuild_arch(self, arch):
     """Make the build and target directories"""
     path = self.get_build_dir(arch.arch)
     if not exists(path):
         info("creating {}".format(path))
         shprint(sh.mkdir, '-p', path)
Exemplo n.º 55
0
    def build_arch(self, arch, *extra_args):
        super(NDKRecipe, self).build_arch(arch)

        env = self.get_recipe_env(arch)
        with current_directory(self.get_build_dir(arch.arch)):
            shprint(sh.ndk_build, 'V=1', 'APP_ABI=' + arch.arch, *extra_args, _env=env)
Exemplo n.º 56
0
    def build_arch(self, arch):
        """simple shared compile"""
        env = self.get_recipe_env(arch, with_flags_in_cc=False)
        for path in (
                self.get_build_dir(arch.arch),
                join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'),
                join(self.ctx.python_recipe.get_build_dir(arch.arch),
                     'Include'),
        ):
            if not exists(path):
                info("creating {}".format(path))
                shprint(sh.mkdir, '-p', path)
        cli = env['CC'].split()
        cc = sh.Command(cli[0])

        with current_directory(self.get_build_dir(arch.arch)):
            cflags = env['CFLAGS'].split()
            cflags.extend(['-I.', '-c', '-l.', 'ifaddrs.c', '-I.'])
            shprint(cc, *cflags, _env=env)

            cflags = env['CFLAGS'].split()
            cflags.extend(
                ['-shared', '-I.', 'ifaddrs.o', '-o', 'libifaddrs.so'])
            cflags.extend(env['LDFLAGS'].split())
            shprint(cc, *cflags, _env=env)

            shprint(sh.cp, 'libifaddrs.so', self.ctx.get_libs_dir(arch.arch))
            shprint(sh.cp, "libifaddrs.so",
                    join(self.ctx.get_python_install_dir(), 'lib'))
            # drop header in to the Python include directory
            shprint(
                sh.cp, "ifaddrs.h",
                join(
                    self.ctx.get_python_install_dir(),
                    'include/python{}'.format(
                        self.ctx.python_recipe.version[0:3])))
            include_path = join(
                self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')
            shprint(sh.cp, "ifaddrs.h", include_path)
Exemplo n.º 57
0
    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
        if self.from_crystax:
            return super(GuestPythonRecipe, self).get_recipe_env(
                arch=arch, with_flags_in_cc=with_flags_in_cc)

        env = environ.copy()

        android_host = env['HOSTARCH'] = arch.command_prefix
        toolchain = '{toolchain_prefix}-{toolchain_version}'.format(
            toolchain_prefix=self.ctx.toolchain_prefix,
            toolchain_version=self.ctx.toolchain_version)
        toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt',
                         'linux-x86_64')

        env['CC'] = (
            '{clang} -target {target} -gcc-toolchain {toolchain}').format(
                clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt',
                           'linux-x86_64', 'bin', 'clang'),
                target=arch.target,
                toolchain=toolchain)
        env['AR'] = join(toolchain, 'bin', android_host) + '-ar'
        env['LD'] = join(toolchain, 'bin', android_host) + '-ld'
        env['RANLIB'] = join(toolchain, 'bin', android_host) + '-ranlib'
        env['READELF'] = join(toolchain, 'bin', android_host) + '-readelf'
        env['STRIP'] = join(toolchain, 'bin', android_host) + '-strip'
        env['STRIP'] += ' --strip-debug --strip-unneeded'

        env['PATH'] = ('{hostpython_dir}:{old_path}').format(
            hostpython_dir=self.get_recipe('host' + self.name,
                                           self.ctx).get_path_to_python(),
            old_path=env['PATH'])

        ndk_flags = (
            '-fPIC --sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} '
            '-isystem {ndk_android_host} -I{ndk_include}').format(
                ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'),
                android_api=self.ctx.ndk_api,
                ndk_android_host=join(self.ctx.ndk_dir, 'sysroot', 'usr',
                                      'include', android_host),
                ndk_include=join(self.ctx.ndk_dir, 'sysroot', 'usr',
                                 'include'))
        sysroot = self.ctx.ndk_platform
        env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags
        env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags
        env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format(
            sysroot, join(sysroot, 'usr', 'lib'))

        # Manually add the libs directory, and copy some object
        # files to the current directory otherwise they aren't
        # picked up. This seems necessary because the --sysroot
        # setting in LDFLAGS is overridden by the other flags.
        # TODO: Work out why this doesn't happen in the original
        # bpo-30386 Makefile system.
        logger.warning('Doing some hacky stuff to link properly')
        lib_dir = join(sysroot, 'usr', 'lib')
        if arch.arch == 'x86_64':
            lib_dir = join(sysroot, 'usr', 'lib64')
        env['LDFLAGS'] += ' -L{}'.format(lib_dir)
        shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './')
        shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './')

        env['SYSROOT'] = sysroot

        return env
Exemplo n.º 58
0
 def distribute_javaclasses(self, javaclass_dir):
     '''Copy existing javaclasses from build dir to current dist dir.'''
     info('Copying java files')
     for filename in glob.glob(javaclass_dir):
         shprint(sh.cp, '-a', filename, 'src')
Exemplo n.º 59
0
    def install_python_package(self, arch, name=None, env=None, is_dir=True):
        '''Automate the installation of a Python package (or a cython
        package where the cython components are pre-built).'''
        # arch = self.filtered_archs[0]  # old kivy-ios way
        if name is None:
            name = self.name
        if env is None:
            env = self.get_recipe_env(arch)

        info('Installing {} into site-packages'.format(self.name))

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.hostpython_location)
            # hostpython = sh.Command('python3.5')

            if self.ctx.python_recipe.from_crystax:
                # hppath = join(dirname(self.hostpython_location), 'Lib',
                #               'site-packages')
                hpenv = env.copy()
                # if 'PYTHONPATH' in hpenv:
                #     hpenv['PYTHONPATH'] = ':'.join([hppath] +
                #                                    hpenv['PYTHONPATH'].split(':'))
                # else:
                #     hpenv['PYTHONPATH'] = hppath
                # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
                # shprint(hostpython, 'setup.py', 'build',
                #         _env=hpenv, *self.setup_extra_args)
                shprint(
                    hostpython,
                    'setup.py',
                    'install',
                    '-O2',
                    '--root={}'.format(self.ctx.get_python_install_dir()),
                    '--install-lib=.',
                    # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
                    _env=hpenv,
                    *self.setup_extra_args)
                # site_packages_dir = self.ctx.get_site_packages_dir()
                # built_files = glob.glob(join('build', 'lib*', '*'))
                # for filen in built_files:
                #     shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
            elif self.call_hostpython_via_targetpython:
                shprint(hostpython,
                        'setup.py',
                        'install',
                        '-O2',
                        _env=env,
                        *self.setup_extra_args)
            else:
                hppath = join(dirname(self.hostpython_location), 'Lib',
                              'site-packages')
                hpenv = env.copy()
                if 'PYTHONPATH' in hpenv:
                    hpenv['PYTHONPATH'] = ':'.join(
                        [hppath] + hpenv['PYTHONPATH'].split(':'))
                else:
                    hpenv['PYTHONPATH'] = hppath
                shprint(hostpython,
                        'setup.py',
                        'install',
                        '-O2',
                        '--root={}'.format(self.ctx.get_python_install_dir()),
                        '--install-lib=lib/python2.7/site-packages',
                        _env=hpenv,
                        *self.setup_extra_args)
                # AND: Hardcoded python2.7 needs fixing

            # If asked, also install in the hostpython build dir
            if self.install_in_hostpython:
                self.install_hostpython_package(arch)
Exemplo n.º 60
0
 def distribute_javaclasses(self, javaclass_dir, dest_dir="src"):
     '''Copy existing javaclasses from build dir to current dist dir.'''
     info('Copying java files')
     ensure_dir(dest_dir)
     filenames = glob.glob(javaclass_dir)
     shprint(sh.cp, '-a', *filenames, dest_dir)