Exemplo n.º 1
0
    def build_arch(self, arch):
        recipe_build_dir = self.get_build_dir(arch.arch)

        # Create a subdirectory to actually perform the build
        build_dir = join(recipe_build_dir, self.build_subdir)
        ensure_dir(build_dir)

        if not exists(join(build_dir, 'python')):
            with current_directory(recipe_build_dir):
                # Configure the build
                with current_directory(build_dir):
                    if not exists('config.status'):
                        shprint(
                            sh.Command(join(recipe_build_dir, 'configure')))

                # Create the Setup file. This copying from Setup.dist
                # seems to be the normal and expected procedure.
                shprint(sh.cp, join('Modules', 'Setup.dist'),
                        join(build_dir, 'Modules', 'Setup'))

                result = shprint(sh.make, '-C', build_dir)
        else:
            info('Skipping {name} ({version}) build, as it has already '
                 'been completed'.format(name=self.name, version=self.version))

        self.ctx.hostpython = join(build_dir, 'python')
Exemplo n.º 2
0
    def build_arch(self, arch):
        info("Extracting CrystaX python3 from NDK package")

        dirn = self.ctx.get_python_install_dir()
        ensure_dir(dirn)

        self.ctx.hostpython = "python{}".format(self.version)
Exemplo n.º 3
0
 def clean_bootstrap_builds(self, args):
     '''Delete all the bootstrap builds.'''
     for bs in Bootstrap.list_bootstraps():
         bs = Bootstrap.get_bootstrap(bs, self.ctx)
         if bs.build_dir and exists(bs.build_dir):
             info('Cleaning build for {} bootstrap.'.format(bs.name))
             shutil.rmtree(bs.build_dir)
Exemplo n.º 4
0
    def clean_build(self, arch=None):
        """Deletes all the build information of the recipe.

        If arch is not None, only this arch dir is deleted. Otherwise
        (the default) all builds for all archs are deleted.

        By default, this just deletes the main build dir. If the
        recipe has e.g. object files biglinked, or .so files stored
        elsewhere, you should override this method.

        This method is intended for testing purposes, it may have
        strange results. Rebuild everything if this seems to happen.

        """
        if arch is None:
            base_dir = join(self.ctx.build_dir, "other_builds", self.name)
        else:
            base_dir = self.get_build_container_dir(arch)
        dirs = glob.glob(base_dir + "-*")
        if exists(base_dir):
            dirs.append(base_dir)
        if not dirs:
            warning(("Attempted to clean build for {} but found no existing " "build dirs").format(self.name))

        for directory in dirs:
            if exists(directory):
                info("Deleting {}".format(directory))
                shutil.rmtree(directory)
Exemplo n.º 5
0
 def download_if_necessary(self):
     info_main("Downloading {}".format(self.name))
     user_dir = environ.get("P4A_{}_DIR".format(self.name.lower()))
     if user_dir is not None:
         info("P4A_{}_DIR is set, skipping download for {}".format(self.name, self.name))
         return
     self.download()
Exemplo n.º 6
0
def build_recipes(build_order, python_modules, ctx):
    # Put recipes in correct build order
    bs = ctx.bootstrap
    info_notify("Recipe build order is {}".format(build_order))
    if python_modules:
        python_modules = sorted(set(python_modules))
        info_notify(
            ('The requirements ({}) were not found as recipes, they will be '
             'installed with pip.').format(', '.join(python_modules)))

    recipes = [Recipe.get_recipe(name, ctx) for name in build_order]

    # download is arch independent
    info_main('# Downloading recipes ')
    for recipe in recipes:
        recipe.download_if_necessary()

    for arch in ctx.archs:
        info_main('# Building all recipes for arch {}'.format(arch.arch))

        info_main('# Unpacking recipes')
        for recipe in recipes:
            ensure_dir(recipe.get_build_container_dir(arch.arch))
            recipe.prepare_build_dir(arch.arch)

        info_main('# Prebuilding recipes')
        # 2) prebuild packages
        for recipe in recipes:
            info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
            recipe.prebuild_arch(arch)
            recipe.apply_patches(arch)

        # 3) build packages
        info_main('# Building recipes')
        for recipe in recipes:
            info_main('Building {} for {}'.format(recipe.name, arch.arch))
            if recipe.should_build(arch):
                recipe.build_arch(arch)
            else:
                info('{} said it is already built, skipping'
                     .format(recipe.name))

        # 4) biglink everything
        # AND: Should make this optional
        info_main('# Biglinking object files')
        if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
            biglink(ctx, arch)
        else:
            info('NDK is crystax, skipping biglink (will this work?)')

        # 5) postbuild packages
        info_main('# Postbuilding recipes')
        for recipe in recipes:
            info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
            recipe.postbuild_arch(arch)

    info_main('# Installing pure Python modules')
    run_pymodules_install(ctx, python_modules)

    return
Exemplo n.º 7
0
    def build_cython_components(self, arch):
        info('Cythonizing anything necessary in {}'.format(self.name))
        env = self.get_recipe_env(arch)
        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.ctx.hostpython)
            info('Trying first build of {} to get cython files: this is '
                 'expected to fail'.format(self.name))
            try:
                shprint(hostpython, 'setup.py', 'build_ext', _env=env,
                        *self.setup_extra_args)
            except sh.ErrorReturnCode_1:
                print()
                info('{} first build failed (as expected)'.format(self.name))

            info('Running cython where appropriate')
            shprint(sh.find, self.get_build_dir(arch.arch), '-iname', '*.pyx',
                    '-exec', self.ctx.cython, '{}', ';', _env=env)
            info('ran cython')

            shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env,
                    _tail=20, _critical=True, *self.setup_extra_args)

            print('stripping')
            build_lib = glob.glob('./build/lib*')
            shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                    env['STRIP'], '{}', ';', _env=env)
            print('stripped!?')
Exemplo n.º 8
0
    def extract_source(self, source, cwd):
        """
        (internal) Extract the `source` into the directory `cwd`.
        """
        if not source:
            return
        if isfile(source):
            info("Extract {} into {}".format(source, cwd))

            if source.endswith(".tgz") or source.endswith(".tar.gz"):
                shprint(sh.tar, "-C", cwd, "-xvzf", source)

            elif source.endswith(".tbz2") or source.endswith(".tar.bz2"):
                shprint(sh.tar, "-C", cwd, "-xvjf", source)

            elif source.endswith(".zip"):
                zf = zipfile.ZipFile(source)
                zf.extractall(path=cwd)
                zf.close()

            else:
                warning("Error: cannot extract, unrecognized extension for {}".format(source))
                raise Exception()

        elif isdir(source):
            info("Copying {} into {}".format(source, cwd))

            shprint(sh.cp, "-a", source, cwd)

        else:
            warning("Error: cannot extract or copy, unrecognized path {}".format(source))
            raise Exception()
Exemplo n.º 9
0
    def clean_build(self, arch=None):
        '''Deletes all the build information of the recipe.

        If arch is not None, only this arch dir is deleted. Otherwise
        (the default) all builds for all archs are deleted.

        By default, this just deletes the main build dir. If the
        recipe has e.g. object files biglinked, or .so files stored
        elsewhere, you should override this method.

        This method is intended for testing purposes, it may have
        strange results. Rebuild everything if this seems to happen.

        '''
        if arch is None:
            base_dir = join(self.ctx.build_dir, 'other_builds', self.name)
        else:
            base_dir = self.get_build_container_dir(arch)
        dirs = glob.glob(base_dir + '-*')
        if exists(base_dir):
            dirs.append(base_dir)
        if not dirs:
             warning(('Attempted to clean build for {} but found no existing '
                      'build dirs').format(self.name))

        for directory in dirs:
            if exists(directory):
                info('Deleting {}'.format(directory))
                shutil.rmtree(directory)

        # Delete any Python distributions to ensure the recipe build
        # doesn't persist in site-packages
        shutil.rmtree(self.ctx.python_installs_dir)
Exemplo n.º 10
0
    def rebuild_compiled_components(self, arch, env):
        info('Rebuilding compiled components in {}'.format(self.name))

        hostpython = sh.Command(self.real_hostpython_location)
        shprint(hostpython, 'setup.py', 'clean', '--all', _env=env)
        shprint(hostpython, 'setup.py', self.build_cmd, '-v', _env=env,
                *self.setup_extra_args)
Exemplo n.º 11
0
 def should_build(self, arch):
     name = self.folder_name
     if self.ctx.has_package(name):
         info('Python package already exists in site-packages')
         return False
     info('{} apparently isn\'t already in site-packages'.format(name))
     return True
Exemplo n.º 12
0
def build_dist_from_args(ctx, dist, args):
    '''Parses out any bootstrap related arguments, and uses them to build
    a dist.'''
    bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
    build_order, python_modules, bs \
        = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
    ctx.recipe_build_order = build_order

    info('The selected bootstrap is {}'.format(bs.name))
    info_main('# Creating dist with {} bootstrap'.format(bs.name))
    bs.distribution = dist
    info_notify('Dist will have name {} and recipes ({})'.format(
        dist.name, ', '.join(dist.recipes)))

    ctx.dist_name = bs.distribution.name
    ctx.prepare_bootstrap(bs)
    ctx.prepare_dist(ctx.dist_name)

    build_recipes(build_order, python_modules, ctx)

    ctx.bootstrap.run_distribute()

    info_main('# Your distribution was created successfully, exiting.')
    info('Dist can be found at (for now) {}'
         .format(join(ctx.dist_dir, ctx.dist_name)))
Exemplo n.º 13
0
 def prebuild_arch(self, arch):
     if not self.is_patched(arch):
         super(ReportLabRecipe, self).prebuild_arch(arch)
         self.apply_patch('patches/fix-setup.patch', arch.arch)
         recipe_dir = self.get_build_dir(arch.arch)
         shprint(sh.touch, os.path.join(recipe_dir, '.patched'))
         ft = self.get_recipe('freetype', self.ctx)
         ft_dir = ft.get_build_dir(arch.arch)
         ft_lib_dir = os.environ.get('_FT_LIB_', os.path.join(ft_dir, 'objs', '.libs'))
         ft_inc_dir = os.environ.get('_FT_INC_', os.path.join(ft_dir, 'include'))
         tmp_dir = os.path.normpath(os.path.join(recipe_dir, "..", "..", "tmp"))
         info('reportlab recipe: recipe_dir={}'.format(recipe_dir))
         info('reportlab recipe: tmp_dir={}'.format(tmp_dir))
         info('reportlab recipe: ft_dir={}'.format(ft_dir))
         info('reportlab recipe: ft_lib_dir={}'.format(ft_lib_dir))
         info('reportlab recipe: ft_inc_dir={}'.format(ft_inc_dir))
         with current_directory(recipe_dir):
             sh.ls('-lathr')
             ensure_dir(tmp_dir)
             pfbfile = os.path.join(tmp_dir, "pfbfer-20070710.zip")
             if not os.path.isfile(pfbfile):
                 sh.wget("http://www.reportlab.com/ftp/pfbfer-20070710.zip", "-O", pfbfile)
             sh.unzip("-u", "-d", os.path.join(recipe_dir, "src", "reportlab", "fonts"), pfbfile)
             if os.path.isfile("setup.py"):
                 with open('setup.py', 'rb') as f:
                     text = f.read().replace('_FT_LIB_', ft_lib_dir).replace('_FT_INC_', ft_inc_dir)
                 with open('setup.py', 'wb') as f:
                     f.write(text)
Exemplo n.º 14
0
    def build_arch(self, arch):
        """simple shared compile"""
        env = self.get_recipe_env(arch, with_flags_in_cc=False)
        for path in (
                self.get_build_dir(arch.arch),
                join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'),
                join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')):
            if not exists(path):
                info("creating {}".format(path))
                shprint(sh.mkdir, '-p', path)
        cli = env['CC'].split()[0]
        # makes sure first CC command is the compiler rather than ccache, refs:
        # https://github.com/kivy/python-for-android/issues/1399
        if 'ccache' in cli:
            cli = env['CC'].split()[1]
        cc = sh.Command(cli)

        with current_directory(self.get_build_dir(arch.arch)):
            cflags = env['CFLAGS'].split()
            cflags.extend(['-I.', '-c', '-l.', 'glob.c', '-I.'])
            shprint(cc, *cflags, _env=env)
            cflags = env['CFLAGS'].split()
            cflags.extend(['-shared', '-I.', 'glob.o', '-o', 'libglob.so'])
            cflags.extend(env['LDFLAGS'].split())
            shprint(cc, *cflags, _env=env)
            shprint(sh.cp, 'libglob.so', join(self.ctx.libs_dir, arch.arch))
Exemplo n.º 15
0
    def strip_libraries(self, arch):
        info('Stripping libraries')
        if self.ctx.python_recipe.from_crystax:
            info('Python was loaded from CrystaX, skipping strip')
            return
        env = arch.get_env()
        tokens = shlex.split(env['STRIP'])
        strip = sh.Command(tokens[0])
        if len(tokens) > 1:
            strip = strip.bake(tokens[1:])

        libs_dir = join(self.dist_dir, '_python_bundle',
                        '_python_bundle', 'modules')
        if self.ctx.python_recipe.name == 'python2legacy':
            libs_dir = join(self.dist_dir, 'private')
        filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'),
                         '-iname', '*.so', _env=env).stdout.decode('utf-8')

        logger.info('Stripping libraries in private dir')
        for filen in filens.split('\n'):
            if not filen:
                continue  # skip the last ''
            try:
                strip(filen, _env=env)
            except sh.ErrorReturnCode_1:
                logger.debug('Failed to strip ' + filen)
Exemplo n.º 16
0
    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        if 'libffi' in self.ctx.recipe_build_order:
            info('Activating flags for libffi')
            recipe = Recipe.get_recipe('libffi', self.ctx)
            include = ' -I' + ' -I'.join(recipe.get_include_dirs(arch))
            ldflag = ' -L' + join(recipe.get_build_dir(arch.arch),
                                  recipe.get_host(arch), '.libs') + ' -lffi'
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include
            env['LDFLAGS'] = env.get('LDFLAGS', '') + ldflag

        if 'openssl' in self.ctx.recipe_build_order:
            recipe = Recipe.get_recipe('openssl', self.ctx)
            openssl_build_dir = recipe.get_build_dir(arch.arch)
            setuplocal = join('Modules', 'Setup.local')
            shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal)
            shprint(sh.sed, '-i.backup', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal)
            env['OPENSSL_VERSION'] = recipe.version

        if 'sqlite3' in self.ctx.recipe_build_order:
            # Include sqlite3 in python2 build
            recipe = Recipe.get_recipe('sqlite3', self.ctx)
            include = ' -I' + recipe.get_build_dir(arch.arch)
            lib = ' -L' + recipe.get_lib_dir(arch) + ' -lsqlite3'
            # Insert or append to env
            flag = 'CPPFLAGS'
            env[flag] = env[flag] + include if flag in env else include
            flag = 'LDFLAGS'
            env[flag] = env[flag] + lib if flag in env else lib
            
        return env
Exemplo n.º 17
0
    def apk(self, args):
        '''Create an APK using the given distribution.'''

        # AND: Need to add a parser here for any extra options
        # parser = argparse.ArgumentParser(
        #     description='Build an APK')
        # args = parser.parse_args(args)

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        for i, arg in enumerate(args[:-1]):
            if arg in ('--dir', '--private'):
                args[i+1] = realpath(expanduser(args[i+1]))

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            build.parse_args(args)
            shprint(sh.ant, 'debug', _tail=20, _critical=True)

        # AND: This is very crude, needs improving. Also only works
        # for debug for now.
        info_main('# Copying APK to current directory')
        apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-debug.apk'))
        if len(apks) == 0:
            raise ValueError('Couldn\'t find the built APK')
        if len(apks) > 1:
            info('More than one built APK found...guessing you '
                 'just built {}'.format(apks[-1]))
        shprint(sh.cp, apks[-1], './')
Exemplo n.º 18
0
    def build_arch(self, arch):
        env = self.get_recipe_env(arch)
        
        env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/png -I{jni_path}/jpeg'.format(
            jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
        env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl/include -I{jni_path}/sdl_mixer'.format(
            jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
        env['CFLAGS'] = env['CFLAGS'] + ' -I{jni_path}/sdl_ttf -I{jni_path}/sdl_image'.format(
            jni_path=join(self.ctx.bootstrap.build_dir, 'jni'))
        debug('pygame cflags', env['CFLAGS'])

        
        env['LDFLAGS'] = env['LDFLAGS'] + ' -L{libs_path} -L{src_path}/obj/local/{arch} -lm -lz'.format(
            libs_path=self.ctx.libs_dir, src_path=self.ctx.bootstrap.build_dir, arch=env['ARCH'])

        env['LDSHARED'] = join(self.ctx.root_dir, 'tools', 'liblink')

        with current_directory(self.get_build_dir(arch.arch)):
            info('hostpython is ' + self.ctx.hostpython)
            hostpython = sh.Command(self.ctx.hostpython)
            shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
                    _tail=10, _critical=True)

            info('strip is ' + env['STRIP'])
            build_lib = glob.glob('./build/lib*')
            assert len(build_lib) == 1
            print('stripping pygame')
            shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                    env['STRIP'], '{}', ';')

        python_install_path = join(self.ctx.build_dir, 'python-install')
        warning('Should remove pygame tests etc. here, but skipping for now')
Exemplo n.º 19
0
 def delete_dist(self, _args):
     dist = self._dist
     if not dist.folder_exists():
         info('No dist exists that matches your specifications, '
              'exiting without deleting.')
         return
     dist.delete()
Exemplo n.º 20
0
    def build_cython_components(self, arch):
        env = self.get_recipe_env(arch)
        with current_directory(self.get_build_dir(arch.arch)):
            info('hostpython is ' + self.ctx.hostpython)
            hostpython = sh.Command(self.ctx.hostpython)

            app_mk = join(self.get_build_dir(arch.arch), 'Application.mk')
            if not exists(app_mk):
                shprint(sh.cp, join(self.get_recipe_dir(), 'Application.mk'), app_mk)
            app_setup = join(self.get_build_dir(arch.arch), 'setup.py')
            if not exists(app_setup):
                shprint(sh.cp, join(self.get_recipe_dir(), 'setup.py'), app_setup)

            # This first attempt *will* fail, because cython isn't
            # installed in the hostpython
            try:
                shprint(hostpython, 'setup.py', 'build_ext', _env=env)
            except sh.ErrorReturnCode_1:
                pass

            # ...so we manually run cython from the user's system
            shprint(sh.find, self.get_build_dir('armeabi'), '-iname', '*.pyx', '-exec',
                    self.ctx.cython, '{}', ';', _env=env)

            # now cython has already been run so the build works
            shprint(hostpython, 'setup.py', 'build_ext', '-v', _env=env)

            # stripping debug symbols lowers the file size a lot
            build_lib = glob.glob('./build/lib*')
            shprint(sh.find, build_lib[0], '-name', '*.o', '-exec',
                    env['STRIP'], '{}', ';', _env=env)
Exemplo n.º 21
0
    def prebuild_arch(self, arch):
        super(MobileInsightRecipe, self).prebuild_arch(arch)

        build_dir = self.get_build_dir(arch.arch)
        tmp_dir = join(build_dir, 'mi_tmp')
        info("Cleaning old MobileInsight-core sources at {}".format(build_dir))
        try:
            shprint(sh.rm, '-r',
                    build_dir,
                    _tail     = 20,
                    _critical = True)
        except:
            pass

        if LOCAL_DEBUG is False:
            info("Cloning MobileInsight-core sources from {}".format(self.mi_git))
            shprint(sh.git,
                    'clone', '-b',
                    self.mi_branch,
                    '--depth=1',
                    self.mi_git,
                    tmp_dir,
                    _tail     = 20,
                    _critical = True)
        else:
            warning("Debugging using local sources of MobileInsight at {}".format(self.local_src))
            shprint(sh.mkdir,
                    build_dir,
                    _tail     = 20,
                    _critical = True)
            shprint(sh.mkdir,
                    tmp_dir,
                    _tail     = 20,
                    _critical = True)
            shprint(sh.cp,
                    '-fr',
                    self.local_src,
                    tmp_dir,
                    _tail     = 20,
                    _critical = True)
            tmp_dir = join(tmp_dir, 'MobileInsight-core')

        shprint(sh.mv,
                join(tmp_dir, 'mobile_insight'),
                build_dir,
                _tail     = 20,
                _critical = True)

        shprint(sh.mv,
                join(tmp_dir, 'dm_collector_c'),
                build_dir,
                _tail     = 20,
                _critical = True)

        # remove unnecessary codes
        shprint(sh.rm, '-r', tmp_dir,
                _tail     = 20,
                _critical = True)

        self.get_newest_toolchain(arch)
Exemplo n.º 22
0
def build_dist_from_args(ctx, dist, args):
    '''Parses out any bootstrap related arguments, and uses them to build
    a dist.'''
    bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
    build_order, python_modules, bs \
        = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
    ctx.recipe_build_order = build_order
    ctx.python_modules = python_modules

    if python_modules and hasattr(sys, 'real_prefix'):
        error('virtualenv is needed to install pure-Python modules, but')
        error('virtualenv does not support nesting, and you are running')
        error('python-for-android in one. Please run p4a outside of a')
        error('virtualenv instead.')
        exit(1)

    info('The selected bootstrap is {}'.format(bs.name))
    info_main('# Creating dist with {} bootstrap'.format(bs.name))
    bs.distribution = dist
    info_notify('Dist will have name {} and recipes ({})'.format(
        dist.name, ', '.join(dist.recipes)))

    ctx.dist_name = bs.distribution.name
    ctx.prepare_bootstrap(bs)
    ctx.prepare_dist(ctx.dist_name)

    build_recipes(build_order, python_modules, ctx)

    ctx.bootstrap.run_distribute()

    info_main('# Your distribution was created successfully, exiting.')
    info('Dist can be found at (for now) {}'
         .format(join(ctx.dist_dir, ctx.dist_name)))
Exemplo n.º 23
0
    def _unpack_aar(self, aar, arch):
        '''Unpack content of .aar bundle and copy to current dist dir.'''
        with temp_directory() as temp_dir:
            name = splitext(basename(aar))[0]
            jar_name = name + '.jar'
            info("unpack {} aar".format(name))
            debug("  from {}".format(aar))
            debug("  to {}".format(temp_dir))
            shprint(sh.unzip, '-o', aar, '-d', temp_dir)

            jar_src = join(temp_dir, 'classes.jar')
            jar_tgt = join('libs', jar_name)
            debug("copy {} jar".format(name))
            debug("  from {}".format(jar_src))
            debug("  to {}".format(jar_tgt))
            ensure_dir('libs')
            shprint(sh.cp, '-a', jar_src, jar_tgt)

            so_src_dir = join(temp_dir, 'jni', arch.arch)
            so_tgt_dir = join('libs', arch.arch)
            debug("copy {} .so".format(name))
            debug("  from {}".format(so_src_dir))
            debug("  to {}".format(so_tgt_dir))
            ensure_dir(so_tgt_dir)
            so_files = glob.glob(join(so_src_dir, '*.so'))
            for f in so_files:
                shprint(sh.cp, '-a', f, so_tgt_dir)
Exemplo n.º 24
0
    def build_arch(self, arch):
        """simple shared compile"""
        env = self.get_recipe_env(arch, with_flags_in_cc=False)
        for path in (
                self.get_build_dir(arch.arch),
                join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Lib'),
                join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')):
            if not exists(path):
                info("creating {}".format(path))
                shprint(sh.mkdir, '-p', path)
        cli = env['CC'].split()
        cc = sh.Command(cli[0])

        with current_directory(self.get_build_dir(arch.arch)):
            cflags = env['CFLAGS'].split()
            cflags.extend(['-I.', '-c', '-l.', 'ifaddrs.c', '-I.'])
            shprint(cc, *cflags, _env=env)

            cflags = env['CFLAGS'].split()
            cflags.extend(['-shared', '-I.', 'ifaddrs.o', '-o', 'libifaddrs.so'])
            cflags.extend(env['LDFLAGS'].split())
            shprint(cc, *cflags, _env=env)

            shprint(sh.cp, 'libifaddrs.so', self.ctx.get_libs_dir(arch.arch))
            shprint(sh.cp, "libifaddrs.so", join(self.ctx.get_python_install_dir(), 'lib'))
            # drop header in to the Python include directory
            python_version = self.ctx.python_recipe.version[0:3]
            shprint(sh.cp, "ifaddrs.h",
                    join(
                        self.ctx.get_python_install_dir(),
                        'include/python{}'.format(python_version))
           )
            include_path = join(self.ctx.python_recipe.get_build_dir(arch.arch), 'Include')
            shprint(sh.cp, "ifaddrs.h", include_path)
Exemplo n.º 25
0
    def strip_libraries(self, arch):
        info('Stripping libraries')
        if self.ctx.python_recipe.from_crystax:
            info('Python was loaded from CrystaX, skipping strip')
            return
        env = arch.get_env()
        strip = which('arm-linux-androideabi-strip', env['PATH'])
        if strip is None:
            warning('Can\'t find strip in PATH...')
            return
        strip = sh.Command(strip)

        libs_dir = join(self.dist_dir, '_python_bundle',
                        '_python_bundle', 'modules')
        if self.ctx.python_recipe.name == 'python2legacy':
            libs_dir = join(self.dist_dir, 'private')
        filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'),
                         '-iname', '*.so', _env=env).stdout.decode('utf-8')

        logger.info('Stripping libraries in private dir')
        for filen in filens.split('\n'):
            try:
                strip(filen, _env=env)
            except sh.ErrorReturnCode_1:
                logger.debug('Failed to strip ' + filen)
Exemplo n.º 26
0
 def cythonize_build(self, env, build_dir="."):
     if not self.cythonize:
         info('Running cython cancelled per recipe setting')
         return
     info('Running cython where appropriate')
     for root, dirnames, filenames in walk("."):
         for filename in fnmatch.filter(filenames, "*.pyx"):
             self.cythonize_file(env, build_dir, join(root, filename))
Exemplo n.º 27
0
 def distribute_libs(self, arch, src_dirs, wildcard='*', dest_dir="libs"):
     '''Copy existing arch libs from build dirs to current dist dir.'''
     info('Copying libs')
     tgt_dir = join(dest_dir, arch.arch)
     ensure_dir(tgt_dir)
     for src_dir in src_dirs:
         for lib in glob.glob(join(src_dir, wildcard)):
             shprint(sh.cp, '-a', lib, tgt_dir)
Exemplo n.º 28
0
 def append_file(self, filename, dest):
     info("Append {} to {}".format(filename, dest))
     filename = join(self.recipe_dir, filename)
     dest = join(self.build_dir, dest)
     with open(filename, "rb") as fd:
         data = fd.read()
     with open(dest, "ab") as fd:
         fd.write(data)
Exemplo n.º 29
0
 def apply_patch(self, filename, arch):
     """
     Apply a patch from the current recipe directory into the current
     build directory.
     """
     info("Applying patch {}".format(filename))
     filename = join(self.recipe_dir, filename)
     shprint(sh.patch, "-t", "-d", self.get_build_dir(arch), "-p1", "-i", filename, _tail=10)
Exemplo n.º 30
0
    def install_python_package(self, arch, name=None, env=None, is_dir=True):
        '''Automate the installation of a Python package (or a cython
        package where the cython components are pre-built).'''
        # arch = self.filtered_archs[0]  # old kivy-ios way
        if name is None:
            name = self.name
        if env is None:
            env = self.get_recipe_env(arch)

        info('Installing {} into site-packages'.format(self.name))

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.hostpython_location)
            # hostpython = sh.Command('python3.5')


            if self.ctx.python_recipe.from_crystax:
                # hppath = join(dirname(self.hostpython_location), 'Lib',
                #               'site-packages')
                hpenv = env.copy()
                # if 'PYTHONPATH' in hpenv:
                #     hpenv['PYTHONPATH'] = ':'.join([hppath] +
                #                                    hpenv['PYTHONPATH'].split(':'))
                # else:
                #     hpenv['PYTHONPATH'] = hppath
                # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
                # shprint(hostpython, 'setup.py', 'build',
                #         _env=hpenv, *self.setup_extra_args)
                shprint(hostpython, 'setup.py', 'install', '-O2',
                        '--root={}'.format(self.ctx.get_python_install_dir()),
                        '--install-lib=.',
                        # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
                        _env=hpenv, *self.setup_extra_args)
                # site_packages_dir = self.ctx.get_site_packages_dir()
                # built_files = glob.glob(join('build', 'lib*', '*'))
                # for filen in built_files:
                #     shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
            elif self.call_hostpython_via_targetpython:
                shprint(hostpython, 'setup.py', 'install', '-O2', _env=env,
                        *self.setup_extra_args)
            else:
                hppath = join(dirname(self.hostpython_location), 'Lib',
                              'site-packages')
                hpenv = env.copy()
                if 'PYTHONPATH' in hpenv:
                    hpenv['PYTHONPATH'] = ':'.join([hppath] +
                                                   hpenv['PYTHONPATH'].split(':'))
                else:
                    hpenv['PYTHONPATH'] = hppath
                shprint(hostpython, 'setup.py', 'install', '-O2',
                        '--root={}'.format(self.ctx.get_python_install_dir()),
                        '--install-lib=lib/python2.7/site-packages',
                        _env=hpenv, *self.setup_extra_args)
                # AND: Hardcoded python2.7 needs fixing

            # If asked, also install in the hostpython build dir
            if self.install_in_hostpython:
                self.install_hostpython_package(arch)
Exemplo n.º 31
0
 def delete_dist(self, args):
     dist = self._dist
     if dist.needs_build:
         info('No dist exists that matches your specifications, '
              'exiting without deleting.')
     shutil.rmtree(dist.dist_dir)
Exemplo n.º 32
0
                                     Out_Fore, Err_Style, Err_Fore,
                                     info_notify, info_main, shprint,
                                     Null_Fore, Null_Style)
from pythonforandroid.util import current_directory, ensure_dir
from pythonforandroid.bootstrap import Bootstrap
from pythonforandroid.distribution import Distribution, pretty_log_dists
from pythonforandroid.graph import get_recipe_order_and_bootstrap
from pythonforandroid.build import Context, build_recipes

user_dir = dirname(realpath(os.path.curdir))
toolchain_dir = dirname(__file__)
sys.path.insert(0, join(toolchain_dir, "tools", "external"))

info(''.join([
    Err_Style.BRIGHT, Err_Fore.RED,
    'This python-for-android revamp is an experimental alpha release!',
    Err_Style.RESET_ALL
]))
info(''.join([
    Err_Fore.RED,
    ('It should work (mostly), but you may experience '
     'missing features or bugs.'), Err_Style.RESET_ALL
]))


def add_boolean_option(parser,
                       names,
                       no_names=None,
                       default=True,
                       dest=None,
                       description=None):
Exemplo n.º 33
0
 def distribute_javaclasses(self, javaclass_dir, dest_dir="src"):
     '''Copy existing javaclasses from build dir to current dist dir.'''
     info('Copying java files')
     ensure_dir(dest_dir)
     filenames = glob.glob(javaclass_dir)
     shprint(sh.cp, '-a', *filenames, dest_dir)
Exemplo n.º 34
0
    def apk(self, args):
        '''Create an APK using the given distribution.'''

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        fix_args = ('--dir', '--private', '--add-jar', '--add-source',
                    '--whitelist', '--blacklist', '--presplash', '--icon')
        unknown_args = args.unknown_args
        for i, arg in enumerate(unknown_args[:-1]):
            argx = arg.split('=')
            if argx[0] in fix_args:
                if len(argx) > 1:
                    unknown_args[i] = '='.join((argx[0],
                                        realpath(expanduser(argx[1]))))
                else:
                    unknown_args[i+1] = realpath(expanduser(unknown_args[i+1]))

        env = os.environ.copy()
        if args.build_mode == 'release':
            if args.keystore:
                env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore))
            if args.signkey:
                env['P4A_RELEASE_KEYALIAS'] = args.signkey
            if args.keystorepw:
                env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
            if args.signkeypw:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
            elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            build_args = build.parse_args(args.unknown_args)
            output = shprint(sh.ant, args.build_mode, _tail=20, _critical=True, _env=env)

        info_main('# Copying APK to current directory')

        apk_re = re.compile(r'.*Package: (.*\.apk)$')
        apk_file = None
        for line in reversed(output.splitlines()):
            m = apk_re.match(line)
            if m:
                apk_file = m.groups()[0]
                break

        if not apk_file:
            info_main('# APK filename not found in build output, trying to guess')
            suffix = args.build_mode
            if suffix == 'release':
                suffix = suffix + '-unsigned'
            apks = glob.glob(join(dist.dist_dir, 'bin', '*-*-{}.apk'.format(suffix)))
            if len(apks) == 0:
                raise ValueError('Couldn\'t find the built APK')
            if len(apks) > 1:
                info('More than one built APK found...guessing you '
                     'just built {}'.format(apks[-1]))
            apk_file = apks[-1]

        info_main('# Found APK file: {}'.format(apk_file))
        shprint(sh.cp, apk_file, './')
Exemplo n.º 35
0
    def unpack(self, arch):
        info_main('Unpacking {} for {}'.format(self.name, arch))

        build_dir = self.get_build_container_dir(arch)

        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
        if user_dir is not None:
            info('P4A_{}_DIR exists, symlinking instead'.format(
                self.name.lower()))
            # AND: Currently there's something wrong if I use ln, fix this
            warning('Using cp -a instead of symlink...fix this!')
            if exists(self.get_build_dir(arch)):
                return
            shprint(sh.rm, '-rf', build_dir)
            shprint(sh.mkdir, '-p', build_dir)
            shprint(sh.rmdir, build_dir)
            ensure_dir(build_dir)
            shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
            return

        if self.url is None:
            info('Skipping {} unpack as no URL is set'.format(self.name))
            return

        filename = shprint(sh.basename,
                           self.versioned_url).stdout[:-1].decode('utf-8')

        with current_directory(build_dir):
            directory_name = self.get_build_dir(arch)

            # AND: Could use tito's get_archive_rootdir here
            if not exists(directory_name) or not isdir(directory_name):
                extraction_filename = join(self.ctx.packages_path, self.name,
                                           filename)
                if isfile(extraction_filename):
                    if extraction_filename.endswith('.zip'):
                        sh.unzip(extraction_filename)
                        import zipfile
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
                        root_directory = fileh.filelist[0].filename.split(
                            '/')[0]
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    elif (extraction_filename.endswith('.tar.gz')
                          or extraction_filename.endswith('.tgz')
                          or extraction_filename.endswith('.tar.bz2')
                          or extraction_filename.endswith('.tbz2')
                          or extraction_filename.endswith('.tar.xz')
                          or extraction_filename.endswith('.txz')):
                        sh.tar('xf', extraction_filename)
                        root_directory = shprint(
                            sh.tar, 'tf', extraction_filename).stdout.decode(
                                'utf-8').split('\n')[0].split('/')[0]
                        if root_directory != directory_name:
                            shprint(sh.mv, root_directory, directory_name)
                    else:
                        raise Exception(
                            'Could not extract {} download, it must be .zip, '
                            '.tar.gz or .tar.bz2 or .tar.xz')
                elif isdir(extraction_filename):
                    mkdir(directory_name)
                    for entry in listdir(extraction_filename):
                        if entry not in ('.git', ):
                            shprint(sh.cp, '-Rv',
                                    join(extraction_filename, entry),
                                    directory_name)
                else:
                    raise Exception(
                        'Given path is neither a file nor a directory: {}'.
                        format(extraction_filename))

            else:
                info('{} is already unpacked, skipping'.format(self.name))
Exemplo n.º 36
0
def run_pymodules_install(ctx,
                          modules,
                          project_dir=None,
                          ignore_setup_py=False):
    """ This function will take care of all non-recipe things, by:

        1. Processing them from --requirements (the modules argument)
           and installing them

        2. Installing the user project/app itself via setup.py if
           ignore_setup_py=True

    """

    info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
    modules = list(filter(ctx.not_has_package, modules))

    # Bail out if no python deps and no setup.py to process:
    if not modules and (ignore_setup_py or project_dir is None
                        or not project_has_setup_py(project_dir)):
        info('No Python modules and no setup.py to process, skipping')
        return

    # Output messages about what we're going to do:
    if modules:
        info('The requirements ({}) don\'t have recipes, attempting to '
             'install them with pip'.format(', '.join(modules)))
        info('If this fails, it may mean that the module has compiled '
             'components and needs a recipe.')
    if project_dir is not None and \
            project_has_setup_py(project_dir) and not ignore_setup_py:
        info('Will process project install, if it fails then the '
             'project may not be compatible for Android install.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(
            venv, '--python=python{}'.format(
                ctx.python_recipe.major_minor_version_string.partition(".")
                [0]), 'venv')

        # Prepare base environment and upgrade pip:
        base_env = copy.copy(os.environ)
        base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
        info('Upgrade pip to latest version')
        shprint(sh.bash,
                '-c', ("source venv/bin/activate && pip install -U pip"),
                _env=copy.copy(base_env))

        # Install Cython in case modules need it to build:
        info('Install Cython in case one of the modules needs it to build')
        shprint(sh.bash,
                '-c', ("venv/bin/pip install Cython"),
                _env=copy.copy(base_env))

        # Get environment variables for build (with CC/compiler set):
        standard_recipe = CythonRecipe()
        standard_recipe.ctx = ctx
        # (note: following line enables explicit -lpython... linker options)
        standard_recipe.call_hostpython_via_targetpython = False
        recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
        env = copy.copy(base_env)
        env.update(recipe_env)

        # Make sure our build package dir is available, and the virtualenv
        # site packages come FIRST (so the proper pip version is used):
        env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
        env["PYTHONPATH"] = os.path.abspath(
            join(ctx.build_dir, "venv", "lib",
                 "python" + ctx.python_recipe.major_minor_version_string,
                 "site-packages")) + ":" + env["PYTHONPATH"]

        # Install the manually specified requirements first:
        if not modules:
            info('There are no Python modules to install, skipping')
        else:
            info('Creating a requirements.txt file for the Python modules')
            with open('requirements.txt', 'w') as fileh:
                for module in modules:
                    key = 'VERSION_' + module
                    if key in environ:
                        line = '{}=={}\n'.format(module, environ[key])
                    else:
                        line = '{}\n'.format(module)
                    fileh.write(line)

            info('Installing Python modules with pip')
            info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
                 'A reason for this is often modules compiling '
                 'native code that is unaware of Android cross-compilation '
                 'and does not work without additional '
                 'changes / workarounds.')

            shprint(sh.bash,
                    '-c',
                    ("venv/bin/pip " +
                     "install -v --target '{0}' --no-deps -r requirements.txt"
                     ).format(ctx.get_site_packages_dir().replace(
                         "'", "'\"'\"'")),
                    _env=copy.copy(env))

        # Afterwards, run setup.py if present:
        if project_dir is not None and (project_has_setup_py(project_dir)
                                        and not ignore_setup_py):
            with current_directory(project_dir):
                info('got setup.py or similar, running project install. ' +
                     '(disable this behavior with --ignore-setup-py)')

                # Compute & output the constraints we will use:
                info('Contents that will be used for constraints.txt:')
                constraints = subprocess.check_output(
                    [join(ctx.build_dir, "venv", "bin", "pip"), "freeze"],
                    env=copy.copy(env))
                try:
                    constraints = constraints.decode("utf-8", "replace")
                except AttributeError:
                    pass
                info(constraints)

                # Make sure all packages found are fixed in version
                # by writing a constraint file, to avoid recipes being
                # upgraded & reinstalled:
                with open('constraints.txt', 'wb') as fileh:
                    fileh.write(constraints.encode("utf-8", "replace"))

                info('Populating venv\'s site-packages with '
                     'ctx.get_site_packages_dir()...')

                # Copy dist contents into site-packages for discovery.
                # Why this is needed:
                # --target is somewhat evil and messes with discovery of
                # packages in PYTHONPATH if that also includes the target
                # folder. So we need to use the regular virtualenv
                # site-packages folder instead.
                # Reference:
                # https://github.com/pypa/pip/issues/6223
                ctx_site_packages_dir = os.path.normpath(
                    os.path.abspath(ctx.get_site_packages_dir()))
                venv_site_packages_dir = os.path.normpath(
                    os.path.join(ctx.build_dir, "venv", "lib", [
                        f for f in os.listdir(
                            os.path.join(ctx.build_dir, "venv", "lib"))
                        if f.startswith("python")
                    ][0], "site-packages"))
                copied_over_contents = []
                for f in os.listdir(ctx_site_packages_dir):
                    full_path = os.path.join(ctx_site_packages_dir, f)
                    if not os.path.exists(
                            os.path.join(venv_site_packages_dir, f)):
                        if os.path.isdir(full_path):
                            shutil.copytree(
                                full_path,
                                os.path.join(venv_site_packages_dir, f))
                        else:
                            shutil.copy2(
                                full_path,
                                os.path.join(venv_site_packages_dir, f))
                        copied_over_contents.append(f)

                # Get listing of virtualenv's site-packages, to see the
                # newly added things afterwards & copy them back into
                # the distribution folder / build context site-packages:
                previous_venv_contents = os.listdir(venv_site_packages_dir)

                # Actually run setup.py:
                info('Launching package install...')
                shprint(sh.bash,
                        '-c',
                        ("'" + join(ctx.build_dir, "venv", "bin",
                                    "pip").replace("'", "'\"'\"'") + "' " +
                         "install -c constraints.txt -v .").format(
                             ctx.get_site_packages_dir().replace(
                                 "'", "'\"'\"'")),
                        _env=copy.copy(env))

                # Go over all new additions and copy them back:
                info('Copying additions resulting from setup.py back ' +
                     'into ctx.get_site_packages_dir()...')
                new_venv_additions = []
                for f in (set(os.listdir(venv_site_packages_dir)) -
                          set(previous_venv_contents)):
                    new_venv_additions.append(f)
                    full_path = os.path.join(venv_site_packages_dir, f)
                    if os.path.isdir(full_path):
                        shutil.copytree(full_path,
                                        os.path.join(ctx_site_packages_dir, f))
                    else:
                        shutil.copy2(full_path,
                                     os.path.join(ctx_site_packages_dir, f))

                # Undo all the changes we did to the venv-site packages:
                info('Reverting additions to virtualenv\'s site-packages...')
                for f in set(copied_over_contents + new_venv_additions):
                    full_path = os.path.join(venv_site_packages_dir, f)
                    if os.path.isdir(full_path):
                        shutil.rmtree(full_path)
                    else:
                        os.remove(full_path)
        elif not ignore_setup_py:
            info("No setup.py found in project directory: " + str(project_dir))

        # Strip object files after potential Cython or native code builds:
        standard_recipe.strip_object_files(ctx.archs[0],
                                           env,
                                           build_dir=ctx.build_dir)
Exemplo n.º 37
0
def get_recipe_order_and_bootstrap(ctx, names, bs=None):
    recipes_to_load = set(names)
    if bs is not None and bs.recipe_depends:
        recipes_to_load = recipes_to_load.union(set(bs.recipe_depends))

    possible_orders = []

    # get all possible order graphs, as names may include tuples/lists
    # of alternative dependencies
    names = [([name] if not isinstance(name, (list, tuple)) else name)
             for name in names]
    for name_set in product(*names):
        new_possible_orders = [RecipeOrder(ctx)]
        for name in name_set:
            new_possible_orders = recursively_collect_orders(
                name, ctx, orders=new_possible_orders)
        possible_orders.extend(new_possible_orders)

    # turn each order graph into a linear list if possible
    orders = []
    for possible_order in possible_orders:
        try:
            order = find_order(possible_order)
        except ValueError:  # a circular dependency was found
            info('Circular dependency found in graph {}, skipping it.'.format(
                possible_order))
            continue
        except:
            warning('Failed to import recipe named {}; the recipe exists '
                    'but appears broken.'.format(name))
            warning('Exception was:')
            raise
        orders.append(list(order))

    # prefer python2 and SDL2 if available
    orders = sorted(orders,
                    key=lambda order: -('python2' in order) -
                    ('sdl2' in order))

    if not orders:
        error('Didn\'t find any valid dependency graphs.')
        error('This means that some of your requirements pull in '
              'conflicting dependencies.')
        error('Exiting.')
        exit(1)
    # It would be better to check against possible orders other
    # than the first one, but in practice clashes will be rare,
    # and can be resolved by specifying more parameters
    chosen_order = orders[0]
    if len(orders) > 1:
        info('Found multiple valid dependency orders:')
        for order in orders:
            info('    {}'.format(order))
        info('Using the first of these: {}'.format(chosen_order))
    else:
        info('Found a single valid recipe set: {}'.format(chosen_order))

    if bs is None:
        bs = Bootstrap.get_bootstrap_from_recipes(chosen_order, ctx)
        recipes, python_modules, bs = get_recipe_order_and_bootstrap(
            ctx, chosen_order, bs=bs)
    else:
        # check if each requirement has a recipe
        recipes = []
        python_modules = []
        for name in chosen_order:
            try:
                Recipe.get_recipe(name, ctx)
            except IOError:
                python_modules.append(name)
            else:
                recipes.append(name)

    return recipes, python_modules, bs
Exemplo n.º 38
0
def run_pymodules_install(ctx, modules):
    modules = list(filter(ctx.not_has_package, modules))

    if not modules:
        info('There are no Python modules to install, skipping')
        return

    info('The requirements ({}) don\'t have recipes, attempting to install '
         'them with pip'.format(', '.join(modules)))
    info('If this fails, it may mean that the module has compiled '
         'components and needs a recipe.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(venv,
                '--python=python{}'.format(ctx.python_recipe.major_minor_version_string),
                'venv')

        info('Creating a requirements.txt file for the Python modules')
        with open('requirements.txt', 'w') as fileh:
            for module in modules:
                key = 'VERSION_' + module
                if key in environ:
                    line = '{}=={}\n'.format(module, environ[key])
                else:
                    line = '{}\n'.format(module)
                fileh.write(line)

        info('Installing Python modules with pip')
        info('If this fails with a message about /bin/false, this '
             'probably means the package cannot be installed with '
             'pip as it needs a compilation recipe.')

        # This bash method is what old-p4a used
        # It works but should be replaced with something better
        shprint(sh.bash, '-c', (
            "env CC=/bin/false CXX=/bin/false "
            "PYTHONPATH={0} venv/bin/pip install --target '{0}' --no-deps -r requirements.txt"
        ).format(ctx.get_site_packages_dir()))
Exemplo n.º 39
0
    def prepare_build_environment(self,
                                  user_sdk_dir,
                                  user_ndk_dir,
                                  user_android_api,
                                  user_ndk_ver,
                                  user_ndk_api):
        '''Checks that build dependencies exist and sets internal variables
        for the Android SDK etc.

        ..warning:: This *must* be called before trying any build stuff

        '''

        self.ensure_dirs()

        if self._build_env_prepared:
            return

        ok = True

        # Work out where the Android SDK is
        sdk_dir = None
        if user_sdk_dir:
            sdk_dir = user_sdk_dir
        # This is the old P4A-specific var
        if sdk_dir is None:
            sdk_dir = environ.get('ANDROIDSDK', None)
        # This seems used more conventionally
        if sdk_dir is None:
            sdk_dir = environ.get('ANDROID_HOME', None)
        # Checks in the buildozer SDK dir, useful for debug tests of p4a
        if sdk_dir is None:
            possible_dirs = glob.glob(expanduser(join(
                '~', '.buildozer', 'android', 'platform', 'android-sdk-*')))
            possible_dirs = [d for d in possible_dirs if not
                             (d.endswith('.bz2') or d.endswith('.gz'))]
            if possible_dirs:
                info('Found possible SDK dirs in buildozer dir: {}'.format(
                    ', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
                info('Will attempt to use SDK at {}'.format(possible_dirs[0]))
                warning('This SDK lookup is intended for debug only, if you '
                        'use python-for-android much you should probably '
                        'maintain your own SDK download.')
                sdk_dir = possible_dirs[0]
        if sdk_dir is None:
            raise BuildInterruptingException('Android SDK dir was not specified, exiting.')
        self.sdk_dir = realpath(sdk_dir)

        # Check what Android API we're using
        android_api = None
        if user_android_api:
            android_api = user_android_api
            info('Getting Android API version from user argument: {}'.format(android_api))
        elif 'ANDROIDAPI' in environ:
            android_api = environ['ANDROIDAPI']
            info('Found Android API target in $ANDROIDAPI: {}'.format(android_api))
        else:
            info('Android API target was not set manually, using '
                 'the default of {}'.format(DEFAULT_ANDROID_API))
            android_api = DEFAULT_ANDROID_API
        android_api = int(android_api)
        self.android_api = android_api

        if self.android_api >= 21 and self.archs[0].arch == 'armeabi':
            raise BuildInterruptingException(
                'Asked to build for armeabi architecture with API '
                '{}, but API 21 or greater does not support armeabi'.format(
                    self.android_api),
                instructions='You probably want to build with --arch=armeabi-v7a instead')

        info('Checking SDK: {}'.format(sdk_dir))
        if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):
            info('valid adv manager:')
            avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager'))
            targets = avdmanager('list', 'target').stdout.decode('utf-8').split('\n')
        elif exists(join(sdk_dir, 'tools', 'android')):
            info('valid android:')
            android = sh.Command(join(sdk_dir, 'tools', 'android'))
            targets = android('list').stdout.decode('utf-8').split('\n')
        else:
            info('Could not find `android` or `sdkmanager`')
            raise BuildInterruptingException(
                'Could not find `android` or `sdkmanager` binaries in Android SDK',
                instructions='Make sure the path to the Android SDK is correct')
        apis = [s for s in targets if re.match(r'^ *API level: ', s)]
        apis = [re.findall(r'[0-9]+', s) for s in apis]
        apis = [int(s[0]) for s in apis if s]
        info('Available Android APIs are ({})'.format(
            ', '.join(map(str, apis))))
        if android_api in apis:
            info(('Requested API target {} is available, '
                  'continuing.').format(android_api))
        else:
            raise BuildInterruptingException(
                ('Requested API target {} is not available, install '
                 'it with the SDK android tool.').format(android_api))

        # Find the Android NDK
        # Could also use ANDROID_NDK, but doesn't look like many tools use this
        ndk_dir = None
        if user_ndk_dir:
            ndk_dir = user_ndk_dir
            info('Getting NDK dir from from user argument')
        if ndk_dir is None:  # The old P4A-specific dir
            ndk_dir = environ.get('ANDROIDNDK', None)
            if ndk_dir is not None:
                info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir))
        if ndk_dir is None:  # Apparently the most common convention
            ndk_dir = environ.get('NDK_HOME', None)
            if ndk_dir is not None:
                info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir))
        if ndk_dir is None:  # Another convention (with maven?)
            ndk_dir = environ.get('ANDROID_NDK_HOME', None)
            if ndk_dir is not None:
                info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir))
        if ndk_dir is None:  # Checks in the buildozer NDK dir, useful
            #                # for debug tests of p4a
            possible_dirs = glob.glob(expanduser(join(
                '~', '.buildozer', 'android', 'platform', 'android-ndk-r*')))
            if possible_dirs:
                info('Found possible NDK dirs in buildozer dir: {}'.format(
                    ', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
                info('Will attempt to use NDK at {}'.format(possible_dirs[0]))
                warning('This NDK lookup is intended for debug only, if you '
                        'use python-for-android much you should probably '
                        'maintain your own NDK download.')
                ndk_dir = possible_dirs[0]
        if ndk_dir is None:
            raise BuildInterruptingException('Android NDK dir was not specified')
        self.ndk_dir = realpath(ndk_dir)

        # Find the NDK version, and check it against what the NDK dir
        # seems to report
        ndk_ver = None
        if user_ndk_ver:
            ndk_ver = user_ndk_ver
            if ndk_dir is not None:
                info('Got NDK version from from user argument: {}'.format(ndk_ver))
        if ndk_ver is None:
            ndk_ver = environ.get('ANDROIDNDKVER', None)
            if ndk_ver is not None:
                info('Got NDK version from $ANDROIDNDKVER: {}'.format(ndk_ver))

        self.ndk = 'google'

        try:
            with open(join(ndk_dir, 'RELEASE.TXT')) as fileh:
                reported_ndk_ver = fileh.read().split(' ')[0].strip()
        except IOError:
            pass
        else:
            if reported_ndk_ver.startswith('crystax-ndk-'):
                reported_ndk_ver = reported_ndk_ver[12:]
                self.ndk = 'crystax'
            if ndk_ver is None:
                ndk_ver = reported_ndk_ver
                info(('Got Android NDK version from the NDK dir: {}').format(ndk_ver))
            else:
                if ndk_ver != reported_ndk_ver:
                    warning('NDK version was set as {}, but checking '
                            'the NDK dir claims it is {}.'.format(
                                ndk_ver, reported_ndk_ver))
                    warning('The build will try to continue, but it may '
                            'fail and you should check '
                            'that your setting is correct.')
                    warning('If the NDK dir result is correct, you don\'t '
                            'need to manually set the NDK ver.')
        if ndk_ver is None:
            warning('Android NDK version could not be found. This probably'
                    'won\'t cause any problems, but if necessary you can'
                    'set it with `--ndk-version=...`.')
        self.ndk_ver = ndk_ver

        ndk_api = None
        if user_ndk_api:
            ndk_api = user_ndk_api
            info('Getting NDK API version (i.e. minimum supported API) from user argument')
        elif 'NDKAPI' in environ:
            ndk_api = environ.get('NDKAPI', None)
            info('Found Android API target in $NDKAPI')
        else:
            ndk_api = min(self.android_api, DEFAULT_NDK_API)
            warning('NDK API target was not set manually, using '
                    'the default of {} = min(android-api={}, default ndk-api={})'.format(
                        ndk_api, self.android_api, DEFAULT_NDK_API))
        ndk_api = int(ndk_api)
        self.ndk_api = ndk_api

        if self.ndk_api > self.android_api:
            raise BuildInterruptingException(
                'Target NDK API is {}, higher than the target Android API {}.'.format(
                    self.ndk_api, self.android_api),
                instructions=('The NDK API is a minimum supported API number and must be lower '
                              'than the target Android API'))

        info('Using {} NDK {}'.format(self.ndk.capitalize(), self.ndk_ver))

        virtualenv = None
        if virtualenv is None:
            virtualenv = sh.which('virtualenv2')
        if virtualenv is None:
            virtualenv = sh.which('virtualenv-2.7')
        if virtualenv is None:
            virtualenv = sh.which('virtualenv')
        if virtualenv is None:
            raise IOError('Couldn\'t find a virtualenv executable, '
                          'you must install this to use p4a.')
        self.virtualenv = virtualenv
        info('Found virtualenv at {}'.format(virtualenv))

        # path to some tools
        self.ccache = sh.which("ccache")
        if not self.ccache:
            info('ccache is missing, the build will not be optimized in the '
                 'future.')
        for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"):
            cython = sh.which(cython_fn)
            if cython:
                self.cython = cython
                break
        else:
            raise BuildInterruptingException('No cython binary found.')
        if not self.cython:
            ok = False
            warning("Missing requirement: cython is not installed")

        # This would need to be changed if supporting multiarch APKs
        arch = self.archs[0]
        platform_dir = arch.platform_dir
        toolchain_prefix = arch.toolchain_prefix
        toolchain_version = None
        self.ndk_platform = join(
            self.ndk_dir,
            'platforms',
            'android-{}'.format(self.ndk_api),
            platform_dir)
        if not exists(self.ndk_platform):
            warning('ndk_platform doesn\'t exist: {}'.format(
                self.ndk_platform))
            ok = False

        py_platform = sys.platform
        if py_platform in ['linux2', 'linux3']:
            py_platform = 'linux'

        toolchain_versions = []
        toolchain_path = join(self.ndk_dir, 'toolchains')
        if isdir(toolchain_path):
            toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path,
                                                            toolchain_prefix))
            toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:]
                                  for path in toolchain_contents]
        else:
            warning('Could not find toolchain subdirectory!')
            ok = False
        toolchain_versions.sort()

        toolchain_versions_gcc = []
        for toolchain_version in toolchain_versions:
            if toolchain_version[0].isdigit():
                # GCC toolchains begin with a number
                toolchain_versions_gcc.append(toolchain_version)

        if toolchain_versions:
            info('Found the following toolchain versions: {}'.format(
                toolchain_versions))
            info('Picking the latest gcc toolchain, here {}'.format(
                toolchain_versions_gcc[-1]))
            toolchain_version = toolchain_versions_gcc[-1]
        else:
            warning('Could not find any toolchain for {}!'.format(
                toolchain_prefix))
            ok = False

        self.toolchain_prefix = toolchain_prefix
        self.toolchain_version = toolchain_version
        # Modify the path so that sh finds modules appropriately
        environ['PATH'] = (
            '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/'
            'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/'
            '{toolchain_prefix}-{toolchain_version}/prebuilt/'
            '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/'
            'tools:{path}').format(
                sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir,
                toolchain_prefix=toolchain_prefix,
                toolchain_version=toolchain_version,
                py_platform=py_platform, path=environ.get('PATH'))

        for executable in ("pkg-config", "autoconf", "automake", "libtoolize",
                           "tar", "bzip2", "unzip", "make", "gcc", "g++"):
            if not sh.which(executable):
                warning("Missing executable: {} is not installed".format(
                    executable))

        if not ok:
            raise BuildInterruptingException(
                'python-for-android cannot continue due to the missing executables above')
Exemplo n.º 40
0
    def get_distribution(
            cls,
            ctx,
            *,
            arch_name,  # required keyword argument: there is no sensible default
            name=None,
            recipes=[],
            ndk_api=None,
            force_build=False,
            extra_dist_dirs=[],
            require_perfect_match=False,
            allow_replace_dist=True):
        '''Takes information about the distribution, and decides what kind of
        distribution it will be.

        If parameters conflict (e.g. a dist with that name already
        exists, but doesn't have the right set of recipes),
        an error is thrown.

        Parameters
        ----------
        name : str
            The name of the distribution. If a dist with this name already '
            exists, it will be used.
        ndk_api : int
            The NDK API to compile against, included in the dist because it cannot
            be changed later during APK packaging.
        arch_name : str
            The target architecture name to compile against, included in the dist because
            it cannot be changed later during APK packaging.
        recipes : list
            The recipes that the distribution must contain.
        force_download: bool
            If True, only downloaded dists are considered.
        force_build : bool
            If True, the dist is forced to be built locally.
        extra_dist_dirs : list
            Any extra directories in which to search for dists.
        require_perfect_match : bool
            If True, will only match distributions with precisely the
            correct set of recipes.
        allow_replace_dist : bool
            If True, will allow an existing dist with the specified
            name but incompatible requirements to be overwritten by
            a new one with the current requirements.
        '''

        possible_dists = Distribution.get_distributions(ctx)

        # Will hold dists that would be built in the same folder as an existing dist
        folder_match_dist = None

        # 0) Check if a dist with that name and architecture already exists
        if name is not None and name:
            possible_dists = [
                d for d in possible_dists
                if (d.name == name) and (arch_name in d.archs)
            ]

            if possible_dists:
                # There should only be one folder with a given dist name *and* arch.
                # We could check that here, but for compatibility let's let it slide
                # and just record the details of one of them. We only use this data to
                # possibly fail the build later, so it doesn't really matter if there
                # was more than one clash.
                folder_match_dist = possible_dists[0]

        # 1) Check if any existing dists meet the requirements
        _possible_dists = []
        for dist in possible_dists:
            if (ndk_api is not None
                    and dist.ndk_api != ndk_api) or dist.ndk_api is None:
                continue
            for recipe in recipes:
                if recipe not in dist.recipes:
                    break
            else:
                _possible_dists.append(dist)
        possible_dists = _possible_dists

        if possible_dists:
            info('Of the existing distributions, the following meet '
                 'the given requirements:')
            pretty_log_dists(possible_dists)
        else:
            info('No existing dists meet the given requirements!')

        # If any dist has perfect recipes, arch and NDK API, return it
        for dist in possible_dists:
            if force_build:
                continue
            if ndk_api is not None and dist.ndk_api != ndk_api:
                continue
            if arch_name is not None and arch_name not in dist.archs:
                continue
            if (set(dist.recipes) == set(recipes)
                    or (set(recipes).issubset(set(dist.recipes))
                        and not require_perfect_match)):
                info_notify('{} has compatible recipes, using this one'.format(
                    dist.name))
                return dist

        # If there was a name match but we didn't already choose it,
        # then the existing dist is incompatible with the requested
        # configuration and the build cannot continue
        if folder_match_dist is not None and not allow_replace_dist:
            raise BuildInterruptingException(
                'Asked for dist with name {name} with recipes ({req_recipes}) and '
                'NDK API {req_ndk_api}, but a dist '
                'with this name already exists and has either incompatible recipes '
                '({dist_recipes}) or NDK API {dist_ndk_api}'.format(
                    name=name,
                    req_ndk_api=ndk_api,
                    dist_ndk_api=folder_match_dist.ndk_api,
                    req_recipes=', '.join(recipes),
                    dist_recipes=', '.join(folder_match_dist.recipes)))

        assert len(possible_dists) < 2

        # If we got this far, we need to build a new dist
        dist = Distribution(ctx)
        dist.needs_build = True

        if not name:
            filen = 'unnamed_dist_{}'
            i = 1
            while exists(join(ctx.dist_dir, filen.format(i))):
                i += 1
            name = filen.format(i)

        dist.name = name
        dist.dist_dir = join(
            ctx.dist_dir,
            generate_dist_folder_name(
                name,
                [arch_name] if arch_name is not None else None,
            ))
        dist.recipes = recipes
        dist.ndk_api = ctx.ndk_api
        dist.archs = [arch_name]

        return dist
Exemplo n.º 41
0
 def install_python_package(self, arch, name=None, env=None, is_dir=True):
     for package in packages:
         #srcdir = os.path.join(self.get_recipe_dir(), "..", "..", "..", "..", "tribler", "src", package)
         srcdir = os.path.join("/work/tribler/src", package)
         info('Installing {} into site-packages'.format(package))
         copytree(srcdir, self.ctx.get_python_install_dir())
Exemplo n.º 42
0
    def build_cython_components(self, arch):
        info('Cythonizing anything necessary in {}'.format(self.name))

        env = self.get_recipe_env(arch)

        if self.ctx.python_recipe.from_crystax:
            command = sh.Command('python{}'.format(
                self.ctx.python_recipe.version))
            site_packages_dirs = command(
                '-c', 'import site; print("\\n".join(site.getsitepackages()))')
            site_packages_dirs = site_packages_dirs.stdout.decode(
                'utf-8').split('\n')
            # env['PYTHONPATH'] = '/usr/lib/python3.5/site-packages/:/usr/lib/python3.5'
            if 'PYTHONPATH' in env:
                env['PYTHONPATH'] = env + ':{}'.format(
                    ':'.join(site_packages_dirs))
            else:
                env['PYTHONPATH'] = ':'.join(site_packages_dirs)

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.ctx.hostpython)
            # hostpython = sh.Command('python3.5')
            shprint(hostpython, '-c', 'import sys; print(sys.path)', _env=env)
            print('cwd is', realpath(curdir))
            info('Trying first build of {} to get cython files: this is '
                 'expected to fail'.format(self.name))

            manually_cythonise = False
            try:
                shprint(hostpython,
                        'setup.py',
                        'build_ext',
                        '-v',
                        _env=env,
                        *self.setup_extra_args)
            except sh.ErrorReturnCode_1:
                print()
                info('{} first build failed (as expected)'.format(self.name))
                manually_cythonise = True

            if manually_cythonise:
                self.cythonize_build(env=env)
                shprint(hostpython,
                        'setup.py',
                        'build_ext',
                        '-v',
                        _env=env,
                        _tail=20,
                        _critical=True,
                        *self.setup_extra_args)
            else:
                info(
                    'First build appeared to complete correctly, skipping manual'
                    'cythonising.')

            print('stripping')
            build_lib = glob.glob('./build/lib*')
            shprint(sh.find,
                    build_lib[0],
                    '-name',
                    '*.o',
                    '-exec',
                    env['STRIP'],
                    '{}',
                    ';',
                    _env=env)
            print('stripped!?')
Exemplo n.º 43
0
    def create_python_bundle(self, dirn, arch):
        """
        Create a packaged python bundle in the target directory, by
        copying all the modules and standard library to the right
        place.
        """
        # Todo: find a better way to find the build libs folder
        modules_build_dir = join(
            self.get_build_dir(arch.arch), 'android-build', 'build',
            'lib.linux{}-{}-{}'.format('2' if self.version[0] == '2' else '',
                                       arch.command_prefix.split('-')[0],
                                       self.major_minor_version_string))

        # Compile to *.pyc/*.pyo the python modules
        self.compile_python_files(modules_build_dir)
        # Compile to *.pyc/*.pyo the standard python library
        self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib'))
        # Compile to *.pyc/*.pyo the other python packages (site-packages)
        self.compile_python_files(self.ctx.get_python_install_dir())

        # Bundle compiled python modules to a folder
        modules_dir = join(dirn, 'modules')
        c_ext = self.compiled_extension
        ensure_dir(modules_dir)
        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
                         glob.glob(join(modules_build_dir, '*' + c_ext)))
        info("Copy {} files into the bundle".format(len(module_filens)))
        for filen in module_filens:
            info(" - copy {}".format(filen))
            copy2(filen, modules_dir)

        # zip up the standard library
        stdlib_zip = join(dirn, 'stdlib.zip')
        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
            stdlib_filens = list(
                walk_valid_filens('.', self.stdlib_dir_blacklist,
                                  self.stdlib_filen_blacklist))
            info("Zip {} files into the bundle".format(len(stdlib_filens)))
            shprint(sh.zip, stdlib_zip, *stdlib_filens)

        # copy the site-packages into place
        ensure_dir(join(dirn, 'site-packages'))
        ensure_dir(self.ctx.get_python_install_dir())
        # TODO: Improve the API around walking and copying the files
        with current_directory(self.ctx.get_python_install_dir()):
            filens = list(
                walk_valid_filens('.', self.site_packages_dir_blacklist,
                                  self.site_packages_filen_blacklist))
            info("Copy {} files into the site-packages".format(len(filens)))
            for filen in filens:
                info(" - copy {}".format(filen))
                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
                copy2(filen, join(dirn, 'site-packages', filen))

        # copy the python .so files into place
        python_build_dir = join(self.get_build_dir(arch.arch), 'android-build')
        python_lib_name = 'libpython' + self.major_minor_version_string
        if self.major_minor_version_string[0] == '3':
            python_lib_name += 'm'
        shprint(sh.cp, join(python_build_dir, python_lib_name + '.so'),
                join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch))

        info('Renaming .so files to reflect cross-compile')
        self.reduce_object_file_names(join(dirn, 'site-packages'))

        return join(dirn, 'site-packages')
Exemplo n.º 44
0
def biglink(ctx, arch):
    # First, collate object files from each recipe
    info('Collating object files from each recipe')
    obj_dir = join(ctx.bootstrap.build_dir, 'collated_objects')
    ensure_dir(obj_dir)
    recipes = [Recipe.get_recipe(name, ctx) for name in ctx.recipe_build_order]
    for recipe in recipes:
        recipe_obj_dir = join(recipe.get_build_container_dir(arch.arch),
                              'objects_{}'.format(recipe.name))
        if not exists(recipe_obj_dir):
            info('{} recipe has no biglinkable files dir, skipping'
                 .format(recipe.name))
            continue
        files = glob.glob(join(recipe_obj_dir, '*'))
        if not len(files):
            info('{} recipe has no biglinkable files, skipping'
                 .format(recipe.name))
            continue
        info('{} recipe has object files, copying'.format(recipe.name))
        files.append(obj_dir)
        shprint(sh.cp, '-r', *files)

    env = arch.get_env()
    env['LDFLAGS'] = env['LDFLAGS'] + ' -L{}'.format(
        join(ctx.bootstrap.build_dir, 'obj', 'local', arch.arch))

    if not len(glob.glob(join(obj_dir, '*'))):
        info('There seem to be no libraries to biglink, skipping.')
        return
    info('Biglinking')
    info('target {}'.format(join(ctx.get_libs_dir(arch.arch),
                                 'libpymodules.so')))
    do_biglink = copylibs_function if ctx.copy_libs else biglink_function

    # Move to the directory containing crtstart_so.o and crtend_so.o
    # This is necessary with newer NDKs? A gcc bug?
    with current_directory(join(ctx.ndk_platform, 'usr', 'lib')):
        do_biglink(
            join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'),
            obj_dir.split(' '),
            extra_link_dirs=[join(ctx.bootstrap.build_dir,
                                  'obj', 'local', arch.arch),
                             os.path.abspath('.')],
            env=env)
Exemplo n.º 45
0
    def set_libs_flags(self, env, arch):
        '''Takes care to properly link libraries with python depending on our
        requirements and the attribute :attr:`opt_depends`.
        '''
        def add_flags(include_flags, link_dirs, link_libs):
            env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags
            env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs
            env['LIBS'] = env.get('LIBS', '') + link_libs

        if 'sqlite3' in self.ctx.recipe_build_order:
            info('Activating flags for sqlite3')
            recipe = Recipe.get_recipe('sqlite3', self.ctx)
            add_flags(' -I' + recipe.get_build_dir(arch.arch),
                      ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3')

        if 'libffi' in self.ctx.recipe_build_order:
            info('Activating flags for libffi')
            recipe = Recipe.get_recipe('libffi', self.ctx)
            # In order to force the correct linkage for our libffi library, we
            # set the following variable to point where is our libffi.pc file,
            # because the python build system uses pkg-config to configure it.
            env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch)
            add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)),
                      ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'),
                      ' -lffi')

        if 'openssl' in self.ctx.recipe_build_order:
            info('Activating flags for openssl')
            recipe = Recipe.get_recipe('openssl', self.ctx)
            self.configure_args += \
                ('--with-openssl=' + recipe.get_build_dir(arch.arch),)
            add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch),
                      recipe.link_libs_flags())

        for library_name in {'libbz2', 'liblzma'}:
            if library_name in self.ctx.recipe_build_order:
                info(f'Activating flags for {library_name}')
                recipe = Recipe.get_recipe(library_name, self.ctx)
                add_flags(recipe.get_library_includes(arch),
                          recipe.get_library_ldflags(arch),
                          recipe.get_library_libs_flag())

        # python build system contains hardcoded zlib version which prevents
        # the build of zlib module, here we search for android's zlib version
        # and sets the right flags, so python can be build with android's zlib
        info("Activating flags for android's zlib")
        zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib')
        zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
        zlib_h = join(zlib_includes, 'zlib.h')
        try:
            with open(zlib_h) as fileh:
                zlib_data = fileh.read()
        except IOError:
            raise BuildInterruptingException(
                "Could not determine android's zlib version, no zlib.h ({}) in"
                " the NDK dir includes".format(zlib_h))
        for line in zlib_data.split('\n'):
            if line.startswith('#define ZLIB_VERSION '):
                break
        else:
            raise BuildInterruptingException(
                'Could not parse zlib.h...so we cannot find zlib version,'
                'required by python build,')
        env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '')
        add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz')

        return env
Exemplo n.º 46
0
def build_recipes(build_order,
                  python_modules,
                  ctx,
                  project_dir,
                  ignore_project_setup_py=False):
    # Put recipes in correct build order
    info_notify("Recipe build order is {}".format(build_order))
    if python_modules:
        python_modules = sorted(set(python_modules))
        info_notify(
            ('The requirements ({}) were not found as recipes, they will be '
             'installed with pip.').format(', '.join(python_modules)))

    recipes = [Recipe.get_recipe(name, ctx) for name in build_order]

    # download is arch independent
    info_main('# Downloading recipes ')
    for recipe in recipes:
        recipe.download_if_necessary()

    for arch in ctx.archs:
        info_main('# Building all recipes for arch {}'.format(arch.arch))

        info_main('# Unpacking recipes')
        for recipe in recipes:
            ensure_dir(recipe.get_build_container_dir(arch.arch))
            recipe.prepare_build_dir(arch.arch)

        info_main('# Prebuilding recipes')
        # 2) prebuild packages
        for recipe in recipes:
            info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
            recipe.prebuild_arch(arch)
            recipe.apply_patches(arch)

        # 3) build packages
        info_main('# Building recipes')
        for recipe in recipes:
            info_main('Building {} for {}'.format(recipe.name, arch.arch))
            if recipe.should_build(arch):
                recipe.build_arch(arch)
            else:
                info('{} said it is already built, skipping'.format(
                    recipe.name))

        # 4) biglink everything
        info_main('# Biglinking object files')
        if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
            biglink(ctx, arch)
        else:
            info('NDK is crystax, skipping biglink (will this work?)')

        # 5) postbuild packages
        info_main('# Postbuilding recipes')
        for recipe in recipes:
            info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
            recipe.postbuild_arch(arch)

    info_main('# Installing pure Python modules')
    run_pymodules_install(ctx,
                          python_modules,
                          project_dir,
                          ignore_setup_py=ignore_project_setup_py)

    return
Exemplo n.º 47
0
    def apk(self, args):
        """Create an APK using the given distribution."""

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        fix_args = ('--dir', '--private', '--add-jar', '--add-source',
                    '--whitelist', '--blacklist', '--presplash', '--icon')
        unknown_args = args.unknown_args
        for i, arg in enumerate(unknown_args):
            argx = arg.split('=')
            if argx[0] in fix_args:
                if len(argx) > 1:
                    unknown_args[i] = '='.join(
                        (argx[0], realpath(expanduser(argx[1]))))
                else:
                    unknown_args[i+1] = realpath(expanduser(unknown_args[i+1]))

        env = os.environ.copy()
        if args.build_mode == 'release':
            if args.keystore:
                env['P4A_RELEASE_KEYSTORE'] = realpath(expanduser(args.keystore))
            if args.signkey:
                env['P4A_RELEASE_KEYALIAS'] = args.signkey
            if args.keystorepw:
                env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
            if args.signkeypw:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
            elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            self.hook("before_apk_build")
            os.environ["ANDROID_API"] = str(self.ctx.android_api)
            build_args = build.parse_args(args.unknown_args)
            self.hook("after_apk_build")
            self.hook("before_apk_assemble")

            build_type = ctx.java_build_tool
            if build_type == 'auto':
                info('Selecting java build tool:')

                build_tools_versions = os.listdir(join(ctx.sdk_dir,
                                                       'build-tools'))
                build_tools_versions = sorted(build_tools_versions,
                                              key=LooseVersion)
                build_tools_version = build_tools_versions[-1]
                info(('Detected highest available build tools '
                      'version to be {}').format(build_tools_version))

                if build_tools_version >= '25.0' and exists('gradlew'):
                    build_type = 'gradle'
                    info('    Building with gradle, as gradle executable is '
                         'present')
                else:
                    build_type = 'ant'
                    if build_tools_version < '25.0':
                        info(('    Building with ant, as the highest '
                              'build-tools-version is only {}').format(
                            build_tools_version))
                    else:
                        info('    Building with ant, as no gradle executable '
                             'detected')

            if build_type == 'gradle':
                # gradle-based build
                env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir
                env["ANDROID_HOME"] = self.ctx.sdk_dir

                gradlew = sh.Command('./gradlew')
                if exists('/usr/bin/dos2unix'):
                    # .../dists/bdisttest_python3/gradlew
                    # .../build/bootstrap_builds/sdl2-python3crystax/gradlew
                    # if docker on windows, gradle contains CRLF
                    output = shprint(
                        sh.Command('dos2unix'), gradlew._path.decode('utf8'),
                        _tail=20, _critical=True, _env=env
                    )
                if args.build_mode == "debug":
                    gradle_task = "assembleDebug"
                elif args.build_mode == "release":
                    gradle_task = "assembleRelease"
                else:
                    error("Unknown build mode {} for apk()".format(
                        args.build_mode))
                    exit(1)
                output = shprint(gradlew, gradle_task, _tail=20,
                                 _critical=True, _env=env)

                # gradle output apks somewhere else
                # and don't have version in file
                apk_dir = join(dist.dist_dir,
                               "build", "outputs", "apk",
                               args.build_mode)
                apk_glob = "*-{}.apk"
                apk_add_version = True

            else:
                # ant-based build
                try:
                    ant = sh.Command('ant')
                except sh.CommandNotFound:
                    error('Could not find ant binary, please install it '
                          'and make sure it is in your $PATH.')
                    exit(1)
                output = shprint(ant, args.build_mode, _tail=20,
                                 _critical=True, _env=env)
                apk_dir = join(dist.dist_dir, "bin")
                apk_glob = "*-*-{}.apk"
                apk_add_version = False

            self.hook("after_apk_assemble")

        info_main('# Copying APK to current directory')

        apk_re = re.compile(r'.*Package: (.*\.apk)$')
        apk_file = None
        for line in reversed(output.splitlines()):
            m = apk_re.match(line)
            if m:
                apk_file = m.groups()[0]
                break

        if not apk_file:
            info_main('# APK filename not found in build output. Guessing...')
            if args.build_mode == "release":
                suffixes = ("release", "release-unsigned")
            else:
                suffixes = ("debug", )
            for suffix in suffixes:
                apks = glob.glob(join(apk_dir, apk_glob.format(suffix)))
                if apks:
                    if len(apks) > 1:
                        info('More than one built APK found... guessing you '
                             'just built {}'.format(apks[-1]))
                    apk_file = apks[-1]
                    break
            else:
                raise ValueError('Couldn\'t find the built APK')

        info_main('# Found APK file: {}'.format(apk_file))
        if apk_add_version:
            info('# Add version number to APK')
            apk_name, apk_suffix = basename(apk_file).split("-", 1)
            apk_file_dest = "{}-{}-{}".format(
                apk_name, build_args.version, apk_suffix)
            info('# APK renamed to {}'.format(apk_file_dest))
            shprint(sh.cp, apk_file, apk_file_dest)
        else:
            shprint(sh.cp, apk_file, './')
Exemplo n.º 48
0
def check_ndk_version(ndk_dir):
    """
    Check the NDK version against what is currently recommended and raise an
    exception of :class:`~pythonforandroid.util.BuildInterruptingException` in
    case that the user tries to use an NDK lower than minimum supported,
    specified via attribute `MIN_NDK_VERSION`.

    .. versionchanged:: 2019.06.06.1.dev0
        Added the ability to get android's NDK `letter version` and also
        rewrote to raise an exception in case that an NDK version lower than
        the minimum supported is detected.
    """
    version = read_ndk_version(ndk_dir)

    if version is None:
        warning(READ_ERROR_NDK_MESSAGE.format(ndk_dir=ndk_dir))
        warning(
            ENSURE_RIGHT_NDK_MESSAGE.format(
                min_supported=MIN_NDK_VERSION,
                rec_version=RECOMMENDED_NDK_VERSION,
                ndk_url=NDK_DOWNLOAD_URL,
            )
        )
        return

    # create a dictionary which will describe the relationship of the android's
    # NDK minor version with the `human readable` letter version, egs:
    # Pkg.Revision = 17.1.4828580 => ndk-17b
    # Pkg.Revision = 17.2.4988734 => ndk-17c
    # Pkg.Revision = 19.0.5232133 => ndk-19 (No letter)
    minor_to_letter = {0: ''}
    minor_to_letter.update(
        {n + 1: chr(i) for n, i in enumerate(range(ord('b'), ord('b') + 25))}
    )

    major_version = version.version[0]
    letter_version = minor_to_letter[version.version[1]]
    string_version = '{major_version}{letter_version}'.format(
        major_version=major_version, letter_version=letter_version
    )

    info(CURRENT_NDK_VERSION_MESSAGE.format(ndk_version=string_version))

    if major_version < MIN_NDK_VERSION:
        raise BuildInterruptingException(
            NDK_LOWER_THAN_SUPPORTED_MESSAGE.format(
                min_supported=MIN_NDK_VERSION, ndk_url=NDK_DOWNLOAD_URL
            ),
            instructions=(
                'Please, go to the android NDK page ({ndk_url}) and download a'
                ' supported version.\n*** The currently recommended NDK'
                ' version is {rec_version} ***'.format(
                    ndk_url=NDK_DOWNLOAD_URL,
                    rec_version=RECOMMENDED_NDK_VERSION,
                )
            ),
        )
    elif major_version > MAX_NDK_VERSION:
        warning(
            RECOMMENDED_NDK_VERSION_MESSAGE.format(
                recommended_ndk_version=RECOMMENDED_NDK_VERSION
            )
        )
        warning(NEW_NDK_MESSAGE)
Exemplo n.º 49
0
    def build_arch(self, arch):
        # If openssl is needed we may have to recompile cPython to get the
        # ssl.py module working properly
        if self.from_crystax and 'openssl' in self.ctx.recipe_build_order:
            info('Openssl and crystax-python combination may require '
                 'recompilation of python...')
            ssl_recipe = self.get_recipe('openssl', self.ctx)
            stage, msg = self.check_for_sslso(ssl_recipe, arch)
            stage = 0 if stage < 5 else stage
            info(msg)
            openssl_build_dir = ssl_recipe.get_build_dir(arch.arch)
            openssl_ndk_dir = join(self.ctx.ndk_dir, 'sources', 'openssl',
                                   ssl_recipe.version)

            if stage < 2:
                info('Copying openssl headers and Android.mk to ndk')
                ensure_dir(openssl_ndk_dir)
                if stage < 1.2:
                    # copy include folder and Android.mk to ndk
                    mk_path = self.find_Android_mk()
                    if mk_path is None:
                        raise IOError('Android.mk file could not be found in '
                                      'any versions in ndk->sources->openssl')
                    shprint(sh.cp, mk_path, openssl_ndk_dir)

                include_dir = join(openssl_build_dir, 'include')
                if stage < 1.3:
                    ndk_include_dir = join(openssl_ndk_dir, 'include',
                                           'openssl')
                    self.copy_include_dir(join(include_dir, 'openssl'),
                                          ndk_include_dir)

                    target_conf = join(openssl_ndk_dir, 'include', 'openssl',
                                       'opensslconf.h')
                    shprint(sh.rm, '-f', target_conf)
                    # overwrite opensslconf.h
                    with open(target_conf, 'w') as fp:
                        fp.write(OPENSSLCONF)

                if stage < 1.4:
                    # move current conf to arch specific conf in ndk
                    under_scored_arch = arch.arch.replace('-', '_')
                    shprint(
                        sh.ln, '-sf',
                        realpath(join(include_dir, 'openssl',
                                      'opensslconf.h')),
                        join(openssl_ndk_dir, 'include', 'openssl',
                             'opensslconf_{}.h'.format(under_scored_arch)))

            if stage < 3:
                info('Copying openssl libs to ndk')
                arch_ndk_lib = join(openssl_ndk_dir, 'libs', arch.arch)
                ensure_dir(arch_ndk_lib)
                shprint(
                    sh.ln, '-sf',
                    realpath(
                        join(openssl_build_dir,
                             'libcrypto{}.so'.format(ssl_recipe.version))),
                    join(openssl_build_dir, 'libcrypto.so'))
                shprint(
                    sh.ln, '-sf',
                    realpath(
                        join(openssl_build_dir,
                             'libssl{}.so'.format(ssl_recipe.version))),
                    join(openssl_build_dir, 'libssl.so'))
                libs = ['libcrypto.a', 'libcrypto.so', 'libssl.a', 'libssl.so']
                cmd = [join(openssl_build_dir, lib)
                       for lib in libs] + [arch_ndk_lib]
                shprint(sh.cp, '-f', *cmd)

            if stage < 10:
                info('Recompiling python-crystax')
                self.patch_dev_defaults(ssl_recipe)
                build_script = join(self.ctx.ndk_dir, 'build', 'tools',
                                    'build-target-python.sh')

                shprint(Command(build_script),
                        '--ndk-dir={}'.format(self.ctx.ndk_dir),
                        '--abis={}'.format(arch.arch), '-j5', '--verbose',
                        self.get_build_dir(arch.arch))

        info('Extracting CrystaX python3 from NDK package')
        dirn = self.ctx.get_python_install_dir()
        ensure_dir(dirn)
        self.ctx.hostpython = 'python{}'.format(self.version)
Exemplo n.º 50
0
 def copy_file(self, filename, dest):
     info("Copy {} to {}".format(filename, dest))
     filename = join(self.recipe_dir, filename)
     dest = join(self.build_dir, dest)
     shutil.copy(filename, dest)
Exemplo n.º 51
0
    def __init__(self):

        argv = sys.argv
        # Buildozer used to pass these arguments in a now-invalid order
        # If that happens, apply this fix
        # This fix will be removed once a fixed buildozer is released
        if (len(argv) > 2 and
            argv[1].startswith('--color') and
            argv[2].startswith('--storage-dir')):
            argv.append(argv.pop(1))  # the --color arg
            argv.append(argv.pop(1))  # the --storage-dir arg

        parser = NoAbbrevParser(
            description=('A packaging tool for turning Python scripts and apps '
                         'into Android APKs'))

        generic_parser = argparse.ArgumentParser(
            add_help=False,
            description=('Generic arguments applied to all commands'))
        dist_parser = argparse.ArgumentParser(
            add_help=False,
            description=('Arguments for dist building'))

        generic_parser.add_argument(
            '--debug', dest='debug', action='store_true',
            default=False,
            help='Display debug output and all build info')
        generic_parser.add_argument(
            '--color', dest='color', choices=['always', 'never', 'auto'],
            help='Enable or disable color output (default enabled on tty)')
        generic_parser.add_argument(
            '--sdk-dir', '--sdk_dir', dest='sdk_dir', default='',
            help='The filepath where the Android SDK is installed')
        generic_parser.add_argument(
            '--ndk-dir', '--ndk_dir', dest='ndk_dir', default='',
            help='The filepath where the Android NDK is installed')
        generic_parser.add_argument(
            '--android-api', '--android_api', dest='android_api', default=0, type=int,
            help='The Android API level to build against.')
        generic_parser.add_argument(
            '--ndk-version', '--ndk_version', dest='ndk_version', default='',
            help=('The version of the Android NDK. This is optional, '
                  'we try to work it out automatically from the ndk_dir.'))
        generic_parser.add_argument(
            '--symlink-java-src', '--symlink_java_src',
            action='store_true',
            dest='symlink_java_src',
            default=False,
            help=('If True, symlinks the java src folder during build and dist '
                  'creation. This is useful for development only, it could also '
                  'cause weird problems.'))

        default_storage_dir = user_data_dir('python-for-android')
        if ' ' in default_storage_dir:
            default_storage_dir = '~/.python-for-android'
        generic_parser.add_argument(
            '--storage-dir', dest='storage_dir',
            default=default_storage_dir,
            help=('Primary storage directory for downloads and builds '
                  '(default: {})'.format(default_storage_dir)))

        # AND: This option doesn't really fit in the other categories, the
        # arg structure needs a rethink
        generic_parser.add_argument(
            '--arch',
            help='The archs to build for, separated by commas.',
            default='armeabi')

        # Options for specifying the Distribution
        generic_parser.add_argument(
            '--dist-name', '--dist_name',
            help='The name of the distribution to use or create',
            default='')

        generic_parser.add_argument(
            '--requirements',
            help=('Dependencies of your app, should be recipe names or '
                  'Python modules'),
            default='')
        
        generic_parser.add_argument(
            '--bootstrap',
            help='The bootstrap to build with. Leave unset to choose automatically.',
            default=None)

        add_boolean_option(
            generic_parser, ["force-build"],
            default=False,
            description='Whether to force compilation of a new distribution:')

        generic_parser.add_argument(
            '--extra-dist-dirs', '--extra_dist_dirs',
            dest='extra_dist_dirs', default='',
            help='Directories in which to look for distributions')

        add_boolean_option(
            generic_parser, ["require-perfect-match"],
            default=False,
            description=('Whether the dist recipes must perfectly match '
                         'those requested'))

        generic_parser.add_argument(
            '--local-recipes', '--local_recipes',
            dest='local_recipes', default='./p4a-recipes',
            help='Directory to look for local recipes')

        add_boolean_option(
            generic_parser, ['copy-libs'],
            default=False,
            description='Copy libraries instead of using biglink (Android 4.3+)')

        self._read_configuration()

        subparsers = parser.add_subparsers(dest='subparser_name',
                                           help='The command to run')

        def add_parser(subparsers, *args, **kwargs):
            '''
            argparse in python2 doesn't support the aliases option,
            so we just don't provide the aliases there.
            '''
            if 'aliases' in kwargs and sys.version_info.major < 3:
                kwargs.pop('aliases')
            return subparsers.add_parser(*args, **kwargs)

        parser_recipes = add_parser(subparsers,
            'recipes',
            parents=[generic_parser],
            help='List the available recipes')
        parser_recipes.add_argument(
                "--compact", action="store_true", default=False,
                help="Produce a compact list suitable for scripting")

        parser_bootstraps = add_parser(subparsers,
            'bootstraps', help='List the available bootstraps',
            parents=[generic_parser])
        parser_clean_all = add_parser(subparsers,
            'clean_all', aliases=['clean-all'],
            help='Delete all builds, dists and caches',
            parents=[generic_parser])
        parser_clean_dists = add_parser(subparsers,
            'clean_dists', aliases=['clean-dists'],
            help='Delete all dists',
            parents=[generic_parser])
        parser_clean_bootstrap_builds = add_parser(subparsers,
            'clean_bootstrap_builds', aliases=['clean-bootstrap-builds'],
            help='Delete all bootstrap builds',
            parents=[generic_parser])
        parser_clean_builds = add_parser(subparsers,
            'clean_builds', aliases=['clean-builds'],
            help='Delete all builds',
            parents=[generic_parser])

        parser_clean_recipe_build = add_parser(subparsers,
            'clean_recipe_build', aliases=['clean-recipe-build'],
            help=('Delete the build components of the given recipe. '
                  'By default this will also delete built dists'),
            parents=[generic_parser])
        parser_clean_recipe_build.add_argument('recipe', help='The recipe name')
        parser_clean_recipe_build.add_argument('--no-clean-dists', default=False,
                                               dest='no_clean_dists',
                                               action='store_true',
                                               help='If passed, do not delete existing dists')

        parser_clean_download_cache= add_parser(subparsers,
            'clean_download_cache', aliases=['clean-download-cache'],
            help='Delete cached downloads for requirement builds',
            parents=[generic_parser])
        parser_clean_download_cache.add_argument(
            'recipes', nargs='*',
            help=('The recipes to clean (space-separated). If no recipe name is '
                  'provided, the entire cache is cleared.'))

        parser_export_dist = add_parser(subparsers,
            'export_dist', aliases=['export-dist'],
            help='Copy the named dist to the given path',
            parents=[generic_parser])
        parser_export_dist.add_argument('output_dir', help=('The output dir to copy to'))
        parser_export_dist.add_argument('--symlink', action='store_true',
                                        help=('Symlink the dist instead of copying'))

        parser_apk = add_parser(subparsers,
            'apk', help='Build an APK',
            parents=[generic_parser])
        parser_apk.add_argument('--release', dest='build_mode', action='store_const',
                        const='release', default='debug',
                        help='Build the PARSER_APK. in Release mode')
        parser_apk.add_argument('--keystore', dest='keystore', action='store', default=None,
                        help=('Keystore for JAR signing key, will use jarsigner '
                              'default if not specified (release build only)'))
        parser_apk.add_argument('--signkey', dest='signkey', action='store', default=None,
                        help='Key alias to sign PARSER_APK. with (release build only)')
        parser_apk.add_argument('--keystorepw', dest='keystorepw', action='store', default=None,
                        help='Password for keystore')
        parser_apk.add_argument('--signkeypw', dest='signkeypw', action='store', default=None,
                        help='Password for key alias')

        parser_create = add_parser(subparsers,
            'create', help='Compile a set of requirements into a dist',
            parents=[generic_parser])
        parser_archs = add_parser(subparsers,
            'archs', help='List the available target architectures',
            parents=[generic_parser])
        parser_distributions = add_parser(subparsers,
            'distributions', aliases=['dists'],
            help='List the currently available (compiled) dists',
            parents=[generic_parser])
        parser_delete_dist = add_parser(subparsers,
            'delete_dist', aliases=['delete-dist'], help='Delete a compiled dist',
            parents=[generic_parser])

        parser_sdk_tools = add_parser(subparsers,
            'sdk_tools', aliases=['sdk-tools'],
            help='Run the given binary from the SDK tools dis',
            parents=[generic_parser])
        parser_sdk_tools.add_argument(
            'tool', help=('The tool binary name to run'))

        parser_adb = add_parser(subparsers,
            'adb', help='Run adb from the given SDK',
            parents=[generic_parser])
        parser_logcat = add_parser(subparsers,
            'logcat', help='Run logcat from the given SDK',
            parents=[generic_parser])
        parser_build_status = add_parser(subparsers,
            'build_status', aliases=['build-status'],
            help='Print some debug information about current built components',
            parents=[generic_parser])

        args, unknown = parser.parse_known_args(sys.argv[1:])
        args.unknown_args = unknown

        self.args = args

        setup_color(args.color)

        if args.debug:
            logger.setLevel(logging.DEBUG)

        # strip version from requirements, and put them in environ
        if hasattr(args, 'requirements'):
            requirements = []
            for requirement in split_argument_list(args.requirements):
                if "==" in requirement:
                    requirement, version = requirement.split(u"==", 1)
                    os.environ["VERSION_{}".format(requirement)] = version
                    info('Recipe {}: version "{}" requested'.format(
                        requirement, version))
                requirements.append(requirement)
            args.requirements = u",".join(requirements)

        self.ctx = Context()
        self.storage_dir = args.storage_dir
        self.ctx.setup_dirs(self.storage_dir)
        self.sdk_dir = args.sdk_dir
        self.ndk_dir = args.ndk_dir
        self.android_api = args.android_api
        self.ndk_version = args.ndk_version
        self.ctx.symlink_java_src = args.symlink_java_src

        self._archs = split_argument_list(args.arch)

        # AND: Fail nicely if the args aren't handled yet
        if args.extra_dist_dirs:
            warning('Received --extra_dist_dirs but this arg currently is not '
                    'handled, exiting.')
            exit(1)

        self.ctx.local_recipes = args.local_recipes
        self.ctx.copy_libs = args.copy_libs

        # Each subparser corresponds to a method
        getattr(self, args.subparser_name.replace('-', '_'))(args)
Exemplo n.º 52
0
    def unpack(self, arch):
        info_main('Unpacking {} for {}'.format(self.name, arch))

        build_dir = self.get_build_container_dir(arch)

        user_dir = environ.get('P4A_{}_DIR'.format(self.name.lower()))
        if user_dir is not None:
            info('P4A_{}_DIR exists, symlinking instead'.format(
                self.name.lower()))
            if exists(self.get_build_dir(arch)):
                return
            shprint(sh.rm, '-rf', build_dir)
            shprint(sh.mkdir, '-p', build_dir)
            shprint(sh.rmdir, build_dir)
            ensure_dir(build_dir)
            shprint(sh.cp, '-a', user_dir, self.get_build_dir(arch))
            return

        if self.url is None:
            info('Skipping {} unpack as no URL is set'.format(self.name))
            return

        filename = shprint(sh.basename,
                           self.versioned_url).stdout[:-1].decode('utf-8')
        ma = match(u'^(.+)#md5=([0-9a-f]{32})$', filename)
        if ma:  # fragmented URL?
            filename = ma.group(1)

        with current_directory(build_dir):
            directory_name = self.get_build_dir(arch)

            if not exists(directory_name) or not isdir(directory_name):
                extraction_filename = join(self.ctx.packages_path, self.name,
                                           filename)
                if isfile(extraction_filename):
                    if extraction_filename.endswith('.zip'):
                        try:
                            sh.unzip(extraction_filename)
                        except (sh.ErrorReturnCode_1, sh.ErrorReturnCode_2):
                            # return code 1 means unzipping had
                            # warnings but did complete,
                            # apparently happens sometimes with
                            # github zips
                            pass
                        import zipfile
                        fileh = zipfile.ZipFile(extraction_filename, 'r')
                        root_directory = fileh.filelist[0].filename.split(
                            '/')[0]
                        if root_directory != basename(directory_name):
                            shprint(sh.mv, root_directory, directory_name)
                    elif extraction_filename.endswith(
                        ('.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz',
                         '.txz')):
                        sh.tar('xf', extraction_filename)
                        root_directory = sh.tar(
                            'tf', extraction_filename).stdout.decode(
                                'utf-8').split('\n')[0].split('/')[0]
                        if root_directory != basename(directory_name):
                            shprint(sh.mv, root_directory, directory_name)
                    else:
                        raise Exception(
                            'Could not extract {} download, it must be .zip, '
                            '.tar.gz or .tar.bz2 or .tar.xz'.format(
                                extraction_filename))
                elif isdir(extraction_filename):
                    mkdir(directory_name)
                    for entry in listdir(extraction_filename):
                        if entry not in ('.git', ):
                            shprint(sh.cp, '-Rv',
                                    join(extraction_filename, entry),
                                    directory_name)
                else:
                    raise Exception(
                        'Given path is neither a file nor a directory: {}'.
                        format(extraction_filename))

            else:
                info('{} is already unpacked, skipping'.format(self.name))
Exemplo n.º 53
0
    def get_distribution(cls, ctx, name=None, recipes=[],
                         ndk_api=None,
                         force_build=False,
                         extra_dist_dirs=[],
                         require_perfect_match=False,
                         allow_replace_dist=True):
        '''Takes information about the distribution, and decides what kind of
        distribution it will be.

        If parameters conflict (e.g. a dist with that name already
        exists, but doesn't have the right set of recipes),
        an error is thrown.

        Parameters
        ----------
        name : str
            The name of the distribution. If a dist with this name already '
            exists, it will be used.
        recipes : list
            The recipes that the distribution must contain.
        force_download: bool
            If True, only downloaded dists are considered.
        force_build : bool
            If True, the dist is forced to be built locally.
        extra_dist_dirs : list
            Any extra directories in which to search for dists.
        require_perfect_match : bool
            If True, will only match distributions with precisely the
            correct set of recipes.
        allow_replace_dist : bool
            If True, will allow an existing dist with the specified
            name but incompatible requirements to be overwritten by
            a new one with the current requirements.
        '''

        existing_dists = Distribution.get_distributions(ctx)

        needs_build = True  # whether the dist needs building, will be returned

        possible_dists = existing_dists

        name_match_dist = None

        # 0) Check if a dist with that name already exists
        if name is not None and name:
            possible_dists = [d for d in possible_dists if d.name == name]
            if possible_dists:
                name_match_dist = possible_dists[0]

        # 1) Check if any existing dists meet the requirements
        _possible_dists = []
        for dist in possible_dists:
            if (
                ndk_api is not None and dist.ndk_api != ndk_api
            ) or dist.ndk_api is None:
                continue
            for recipe in recipes:
                if recipe not in dist.recipes:
                    break
            else:
                _possible_dists.append(dist)
        possible_dists = _possible_dists

        if possible_dists:
            info('Of the existing distributions, the following meet '
                 'the given requirements:')
            pretty_log_dists(possible_dists)
        else:
            info('No existing dists meet the given requirements!')

        # If any dist has perfect recipes and ndk API, return it
        for dist in possible_dists:
            if force_build:
                continue
            if ndk_api is not None and dist.ndk_api != ndk_api:
                continue
            if (set(dist.recipes) == set(recipes) or
                (set(recipes).issubset(set(dist.recipes)) and
                 not require_perfect_match)):
                info_notify('{} has compatible recipes, using this one'
                            .format(dist.name))
                return dist

        assert len(possible_dists) < 2

        # If there was a name match but we didn't already choose it,
        # then the existing dist is incompatible with the requested
        # configuration and the build cannot continue
        if name_match_dist is not None and not allow_replace_dist:
            raise BuildInterruptingException(
                'Asked for dist with name {name} with recipes ({req_recipes}) and '
                'NDK API {req_ndk_api}, but a dist '
                'with this name already exists and has either incompatible recipes '
                '({dist_recipes}) or NDK API {dist_ndk_api}'.format(
                    name=name,
                    req_ndk_api=ndk_api,
                    dist_ndk_api=name_match_dist.ndk_api,
                    req_recipes=', '.join(recipes),
                    dist_recipes=', '.join(name_match_dist.recipes)))

        # If we got this far, we need to build a new dist
        dist = Distribution(ctx)
        dist.needs_build = True

        if not name:
            filen = 'unnamed_dist_{}'
            i = 1
            while exists(join(ctx.dist_dir, filen.format(i))):
                i += 1
            name = filen.format(i)

        dist.name = name
        dist.dist_dir = join(ctx.dist_dir, dist.name)
        dist.recipes = recipes
        dist.ndk_api = ctx.ndk_api

        return dist
Exemplo n.º 54
0
    def get_distribution(cls,
                         ctx,
                         name=None,
                         recipes=[],
                         force_build=False,
                         extra_dist_dirs=[],
                         require_perfect_match=False):
        '''Takes information about the distribution, and decides what kind of
        distribution it will be.

        If parameters conflict (e.g. a dist with that name already
        exists, but doesn't have the right set of recipes),
        an error is thrown.

        Parameters
        ----------
        name : str
            The name of the distribution. If a dist with this name already '
            exists, it will be used.
        recipes : list
            The recipes that the distribution must contain.
        force_download: bool
            If True, only downloaded dists are considered.
        force_build : bool
            If True, the dist is forced to be built locally.
        extra_dist_dirs : list
            Any extra directories in which to search for dists.
        require_perfect_match : bool
            If True, will only match distributions with precisely the
            correct set of recipes.
        '''

        # AND: This whole function is a bit hacky, it needs checking
        # properly to make sure it follows logically correct
        # possibilities

        existing_dists = Distribution.get_distributions(ctx)

        needs_build = True  # whether the dist needs building, will be returned

        possible_dists = existing_dists

        # 0) Check if a dist with that name already exists
        if name is not None and name:
            possible_dists = [d for d in possible_dists if d.name == name]

        # 1) Check if any existing dists meet the requirements
        _possible_dists = []
        for dist in possible_dists:
            for recipe in recipes:
                if recipe not in dist.recipes:
                    break
            else:
                _possible_dists.append(dist)
        possible_dists = _possible_dists

        if possible_dists:
            info('Of the existing distributions, the following meet '
                 'the given requirements:')
            pretty_log_dists(possible_dists)
        else:
            info('No existing dists meet the given requirements!')

        # If any dist has perfect recipes, return it
        P4A_force_build = False
        for dist in possible_dists:
            if (set(dist.recipes) == set(recipes)
                    or (set(recipes).issubset(set(dist.recipes))
                        and not require_perfect_match)):
                if force_build:
                    #  distribution exists, rebuild forced
                    P4A_force_build = True
                    continue
                else:
                    #  existing distribution returned, no build required
                    info_notify(
                        '{} has compatible recipes, using this one'.format(
                            dist.name))
                    return dist

        assert len(possible_dists) < 2

        if not name and possible_dists:
            info('Asked for dist with name {} with recipes ({}), but a dist '
                 'with this name already exists and has incompatible recipes '
                 '({})'.format(name, ', '.join(recipes),
                               ', '.join(possible_dists[0].recipes)))
            info('No compatible dist found, so exiting.')
            exit(1)

        # # 2) Check if any downloadable dists meet the requirements

        # online_dists = [('testsdl2', ['hostpython2', 'sdl2_image',
        #                               'sdl2_mixer', 'sdl2_ttf',
        #                               'python2', 'sdl2',
        #                               'pyjniussdl2', 'kivysdl2'],
        #                  'https://github.com/inclement/sdl2-example-dist/archive/master.zip'),
        #                  ]
        # _possible_dists = []
        # for dist_name, dist_recipes, dist_url in online_dists:
        #     for recipe in recipes:
        #         if recipe not in dist_recipes:
        #             break
        #     else:
        #         dist = Distribution(ctx)
        #         dist.name = dist_name
        #         dist.url = dist_url
        #         _possible_dists.append(dist)
        # # if _possible_dists

        # If we got this far, we need to build a new dist
        dist = Distribution(ctx)
        dist.needs_build = True
        """Locally modified recipes will be forced to build, others are reused
        environmental variable P4A_{recipe_name}_DIR points to local recipe
        P4A_force_build affects all local recipes"""

        dist.P4A_force_build = P4A_force_build and os.environ.get(
            'P4A_force_build')

        if not name:
            filen = 'unnamed_dist_{}'
            i = 1
            while exists(join(ctx.dist_dir, filen.format(i))):
                i += 1
            name = filen.format(i)

        dist.name = name
        dist.dist_dir = join(ctx.dist_dir, dist.name)
        dist.recipes = recipes

        return dist
Exemplo n.º 55
0
 def prebuild_arch(self, arch):
     """Make the build and target directories"""
     path = self.get_build_dir(arch.arch)
     if not exists(path):
         info("creating {}".format(path))
         shprint(sh.mkdir, '-p', path)
Exemplo n.º 56
0
def run_pymodules_install(ctx, modules):
    modules = list(filter(ctx.not_has_package, modules))

    if not modules:
        info('There are no Python modules to install, skipping')
        return

    info('The requirements ({}) don\'t have recipes, attempting to install '
         'them with pip'.format(', '.join(modules)))
    info('If this fails, it may mean that the module has compiled '
         'components and needs a recipe.')

    venv = sh.Command(ctx.virtualenv)
    with current_directory(join(ctx.build_dir)):
        shprint(
            venv, '--python=python{}.{}'.format(
                ctx.python_recipe.major_minor_version_string.partition(".")[0],
                ctx.python_recipe.major_minor_version_string.partition(".")
                [2]), 'venv')

        info('Creating a requirements.txt file for the Python modules')
        with open('requirements.txt', 'w') as fileh:
            for module in modules:
                key = 'VERSION_' + module
                if key in environ:
                    line = '{}=={}\n'.format(module, environ[key])
                else:
                    line = '{}\n'.format(module)
                fileh.write(line)

        # Prepare base environment and upgrade pip:
        base_env = copy.copy(os.environ)
        base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
        info('Upgrade pip to latest version')
        shprint(sh.bash,
                '-c', ("source venv/bin/activate && pip install -U pip"),
                _env=copy.copy(base_env))

        # Install Cython in case modules need it to build:
        info('Install Cython in case one of the modules needs it to build')
        shprint(sh.bash,
                '-c', ("venv/bin/pip install Cython"),
                _env=copy.copy(base_env))

        # Get environment variables for build (with CC/compiler set):
        standard_recipe = CythonRecipe()
        standard_recipe.ctx = ctx
        # (note: following line enables explicit -lpython... linker options)
        standard_recipe.call_hostpython_via_targetpython = False
        recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
        env = copy.copy(base_env)
        env.update(recipe_env)

        info('Installing Python modules with pip')
        info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
             'A reason for this is often modules compiling '
             'native code that is unaware of Android cross-compilation '
             'and does not work without additional '
             'changes / workarounds.')

        # Make sure our build package dir is available, and the virtualenv
        # site packages come FIRST (so the proper pip version is used):
        env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
        env["PYTHONPATH"] = os.path.abspath(
            join(ctx.build_dir, "venv", "lib",
                 "python" + ctx.python_recipe.major_minor_version_string,
                 "site-packages")) + ":" + env["PYTHONPATH"]
        '''
        # Do actual install:
        shprint(sh.bash, '-c', (
            "venv/bin/pip " +
            "install -v --target '{0}' --no-deps -r requirements.txt"
        ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
                _env=copy.copy(env))
        '''

        # use old install script
        shprint(sh.bash, '-c', (
            "source venv/bin/activate && env CC=/bin/false CXX=/bin/false "
            "PYTHONPATH={0} pip install --target '{0}' --no-deps -r requirements.txt"
        ).format(ctx.get_site_packages_dir()))

        # Strip object files after potential Cython or native code builds:
        standard_recipe.strip_object_files(ctx.archs[0],
                                           env,
                                           build_dir=ctx.build_dir)
Exemplo n.º 57
0
 def distribute_javaclasses(self, javaclass_dir):
     '''Copy existing javaclasses from build dir to current dist dir.'''
     info('Copying java files')
     for filename in glob.glob(javaclass_dir):
         shprint(sh.cp, '-a', filename, 'src')
Exemplo n.º 58
0
 def distribute_aars(self, arch):
     '''Process existing .aar bundles and copy to current dist dir.'''
     info('Unpacking aars')
     for aar in glob.glob(join(self.ctx.aars_dir, '*.aar')):
         self._unpack_aar(aar, arch)
Exemplo n.º 59
0
    def install_python_package(self, arch, name=None, env=None, is_dir=True):
        '''Automate the installation of a Python package (or a cython
        package where the cython components are pre-built).'''
        # arch = self.filtered_archs[0]  # old kivy-ios way
        if name is None:
            name = self.name
        if env is None:
            env = self.get_recipe_env(arch)

        info('Installing {} into site-packages'.format(self.name))

        with current_directory(self.get_build_dir(arch.arch)):
            hostpython = sh.Command(self.hostpython_location)
            # hostpython = sh.Command('python3.5')

            if self.ctx.python_recipe.from_crystax:
                # hppath = join(dirname(self.hostpython_location), 'Lib',
                #               'site-packages')
                hpenv = env.copy()
                # if 'PYTHONPATH' in hpenv:
                #     hpenv['PYTHONPATH'] = ':'.join([hppath] +
                #                                    hpenv['PYTHONPATH'].split(':'))
                # else:
                #     hpenv['PYTHONPATH'] = hppath
                # hpenv['PYTHONHOME'] = self.ctx.get_python_install_dir()
                # shprint(hostpython, 'setup.py', 'build',
                #         _env=hpenv, *self.setup_extra_args)
                shprint(
                    hostpython,
                    'setup.py',
                    'install',
                    '-O2',
                    '--root={}'.format(self.ctx.get_python_install_dir()),
                    '--install-lib=.',
                    # AND: will need to unhardcode the 3.5 when adding 2.7 (and other crystax supported versions)
                    _env=hpenv,
                    *self.setup_extra_args)
                # site_packages_dir = self.ctx.get_site_packages_dir()
                # built_files = glob.glob(join('build', 'lib*', '*'))
                # for filen in built_files:
                #     shprint(sh.cp, '-r', filen, join(site_packages_dir, split(filen)[-1]))
            elif self.call_hostpython_via_targetpython:
                shprint(hostpython,
                        'setup.py',
                        'install',
                        '-O2',
                        _env=env,
                        *self.setup_extra_args)
            else:
                hppath = join(dirname(self.hostpython_location), 'Lib',
                              'site-packages')
                hpenv = env.copy()
                if 'PYTHONPATH' in hpenv:
                    hpenv['PYTHONPATH'] = ':'.join(
                        [hppath] + hpenv['PYTHONPATH'].split(':'))
                else:
                    hpenv['PYTHONPATH'] = hppath
                shprint(hostpython,
                        'setup.py',
                        'install',
                        '-O2',
                        '--root={}'.format(self.ctx.get_python_install_dir()),
                        '--install-lib=lib/python2.7/site-packages',
                        _env=hpenv,
                        *self.setup_extra_args)
                # AND: Hardcoded python2.7 needs fixing

            # If asked, also install in the hostpython build dir
            if self.install_in_hostpython:
                self.install_hostpython_package(arch)
Exemplo n.º 60
0
    def apk(self, args):
        '''Create an APK using the given distribution.'''

        ap = argparse.ArgumentParser(description='Build an APK')
        ap.add_argument('--release',
                        dest='build_mode',
                        action='store_const',
                        const='release',
                        default='debug',
                        help='Build the APK in Release mode')
        ap.add_argument(
            '--keystore',
            dest='keystore',
            action='store',
            default=None,
            help=('Keystore for JAR signing key, will use jarsigner '
                  'default if not specified (release build only)'))
        ap.add_argument('--signkey',
                        dest='signkey',
                        action='store',
                        default=None,
                        help='Key alias to sign APK with (release build only)')
        ap.add_argument('--keystorepw',
                        dest='keystorepw',
                        action='store',
                        default=None,
                        help='Password for keystore')
        ap.add_argument('--signkeypw',
                        dest='signkeypw',
                        action='store',
                        default=None,
                        help='Password for key alias')
        apk_args, args = ap.parse_known_args(args)

        ctx = self.ctx
        dist = self._dist

        # Manually fixing these arguments at the string stage is
        # unsatisfactory and should probably be changed somehow, but
        # we can't leave it until later as the build.py scripts assume
        # they are in the current directory.
        fix_args = ('--dir', '--private', '--add-jar', '--add-source',
                    '--whitelist', '--blacklist', '--presplash', '--icon')
        for i, arg in enumerate(args[:-1]):
            argx = arg.split('=')
            if argx[0] in fix_args:
                if len(argx) > 1:
                    args[i] = '='.join(
                        (argx[0], realpath(expanduser(argx[1]))))
                else:
                    args[i + 1] = realpath(expanduser(args[i + 1]))

        env = os.environ.copy()
        if apk_args.build_mode == 'release':
            if apk_args.keystore:
                env['P4A_RELEASE_KEYSTORE'] = realpath(
                    expanduser(apk_args.keystore))
            if apk_args.signkey:
                env['P4A_RELEASE_KEYALIAS'] = apk_args.signkey
            if apk_args.keystorepw:
                env['P4A_RELEASE_KEYSTORE_PASSWD'] = apk_args.keystorepw
            if apk_args.signkeypw:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = apk_args.signkeypw
            elif apk_args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
                env['P4A_RELEASE_KEYALIAS_PASSWD'] = apk_args.keystorepw

        build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
        with current_directory(dist.dist_dir):
            build_args = build.parse_args(args)
            output = shprint(sh.ant,
                             apk_args.build_mode,
                             _tail=20,
                             _critical=True,
                             _env=env)

        info_main('# Copying APK to current directory')

        apk_re = re.compile(r'.*Please sign (/.*\.apk) manually$')
        apk_file = None
        for line in reversed(output.splitlines()):
            m = apk_re.match(line)
            if m:
                apk_file = m.groups()[0]
                break

        if not apk_file:
            info_main(
                '# APK filename not found in build output, trying to guess')
            apks = glob.glob(
                join(dist.dist_dir, 'bin',
                     '*-*-{}.apk'.format(apk_args.build_mode)))
            if len(apks) == 0:
                raise ValueError('Couldn\'t find the built APK')
            if len(apks) > 1:
                info('More than one built APK found...guessing you '
                     'just built {}'.format(apks[-1]))
            apk_file = apks[-1]

        info_main('# Found APK file: {}'.format(apk_file))
        shprint(sh.cp, apk_file, './')