def build_recipes(build_order, python_modules, ctx): # Put recipes in correct build order bs = ctx.bootstrap info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping' .format(recipe.name)) # 4) biglink everything # AND: Should make this optional info_main('# Biglinking object files') if not ctx.python_recipe or not ctx.python_recipe.from_crystax: biglink(ctx, arch) else: info('NDK is crystax, skipping biglink (will this work?)') # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules) return
def build_dist_from_args(ctx, dist, args): '''Parses out any bootstrap related arguments, and uses them to build a dist.''' bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) build_order, python_modules, bs \ = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs) ctx.recipe_build_order = build_order ctx.python_modules = python_modules if python_modules and hasattr(sys, 'real_prefix'): error('virtualenv is needed to install pure-Python modules, but') error('virtualenv does not support nesting, and you are running') error('python-for-android in one. Please run p4a outside of a') error('virtualenv instead.') exit(1) info('The selected bootstrap is {}'.format(bs.name)) info_main('# Creating dist with {} bootstrap'.format(bs.name)) bs.distribution = dist info_notify('Dist will have name {} and recipes ({})'.format( dist.name, ', '.join(dist.recipes))) ctx.dist_name = bs.distribution.name ctx.prepare_bootstrap(bs) ctx.prepare_dist(ctx.dist_name) build_recipes(build_order, python_modules, ctx) ctx.bootstrap.run_distribute() info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' .format(join(ctx.dist_dir, ctx.dist_name)))
def build_dist_from_args(ctx, dist, args): '''Parses out any bootstrap related arguments, and uses them to build a dist.''' bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) build_order, python_modules, bs \ = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs) ctx.recipe_build_order = build_order info('The selected bootstrap is {}'.format(bs.name)) info_main('# Creating dist with {} bootstrap'.format(bs.name)) bs.distribution = dist info_notify('Dist will have name {} and recipes ({})'.format( dist.name, ', '.join(dist.recipes))) ctx.dist_name = bs.distribution.name ctx.prepare_bootstrap(bs) ctx.prepare_dist(ctx.dist_name) build_recipes(build_order, python_modules, ctx) ctx.bootstrap.run_distribute() info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' .format(join(ctx.dist_dir, ctx.dist_name)))
def build_recipes(build_order, python_modules, ctx): # Put recipes in correct build order bs = ctx.bootstrap info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ('The requirements ({}) were not found as recipes, they will be ' 'installed with pip.').format(', '.join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main('# Downloading recipes ') for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main('# Building all recipes for arch {}'.format(arch.arch)) info_main('# Unpacking recipes') for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main('# Prebuilding recipes') # 2) prebuild packages for recipe in recipes: info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main('# Building recipes') for recipe in recipes: info_main('Building {} for {}'.format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info('{} said it is already built, skipping'.format( recipe.name)) # 4) biglink everything info_main('# Biglinking object files') if not ctx.python_recipe or not ctx.python_recipe.from_crystax: biglink(ctx, arch) else: info('NDK is crystax, skipping biglink (will this work?)') # 5) postbuild packages info_main('# Postbuilding recipes') for recipe in recipes: info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') run_pymodules_install(ctx, python_modules) return
def wrapper_func(self, args): ctx = self.ctx ctx.set_archs(self._archs) ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, user_ndk_ver=self.ndk_version) dist = self._dist if dist.needs_build: info_notify('No dist exists that meets your requirements, ' 'so one will be built.') build_dist_from_args(ctx, dist, args) func(self, args)
def wrapper_func(self, args): ctx = self.ctx ctx.set_archs(self._archs) ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, user_ndk_api=self.ndk_api) dist = self._dist if dist.needs_build: if dist.folder_exists(): # possible if the dist is being replaced dist.delete() info_notify('No dist exists that meets your requirements, ' 'so one will be built.') build_dist_from_args(ctx, dist, args) func(self, args)
def wrapper_func(self, args): ctx = self.ctx ctx.set_archs(self._archs) ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, user_ndk_ver=self.ndk_version, user_ndk_api=self.ndk_api) dist = self._dist if dist.needs_build: if dist.folder_exists(): # possible if the dist is being replaced dist.delete() info_notify('No dist exists that meets your requirements, ' 'so one will be built.') build_dist_from_args(ctx, dist, args) func(self, args)
def _adb(self, commands): '''Call the adb executable from the SDK, passing the given commands as arguments.''' ctx = self.ctx ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, user_ndk_ver=self.ndk_version) if platform in ('win32', 'cygwin'): adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb.exe')) else: adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb')) info_notify('Starting adb...') output = adb(*commands, _iter=True, _out_bufsize=1, _err_to_out=True) for line in output: sys.stdout.write(line) sys.stdout.flush()
def build_dist_from_args(ctx, dist, args): """Parses out any bootstrap related arguments, and uses them to build a dist.""" bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) blacklist = getattr(args, "blacklist_requirements", "").split(",") if len(blacklist) == 1 and blacklist[0] == "": blacklist = [] build_order, python_modules, bs = ( get_recipe_order_and_bootstrap( ctx, dist.recipes, bs, blacklist=blacklist )) assert set(build_order).intersection(set(python_modules)) == set() ctx.recipe_build_order = build_order ctx.python_modules = python_modules info('The selected bootstrap is {}'.format(bs.name)) info_main('# Creating dist with {} bootstrap'.format(bs.name)) bs.distribution = dist info_notify('Dist will have name {} and requirements ({})'.format( dist.name, ', '.join(dist.recipes))) info('Dist contains the following requirements as recipes: {}'.format( ctx.recipe_build_order)) info('Dist will also contain modules ({}) installed from pip'.format( ', '.join(ctx.python_modules))) ctx.distribution = dist ctx.prepare_bootstrap(bs) if dist.needs_build: ctx.prepare_dist() build_recipes(build_order, python_modules, ctx, getattr(args, "private", None), ignore_project_setup_py=getattr( args, "ignore_setup_py", False ), ) ctx.bootstrap.run_distribute() info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' .format(join(ctx.dist_dir, ctx.distribution.dist_dir)))
def adb(self, args): '''Runs the adb binary from the detected SDK directory, passing all arguments straight to it. This is intended as a convenience function if adb is not in your $PATH. ''' ctx = self.ctx ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, user_ndk_ver=self.ndk_version) if platform in ('win32', 'cygwin'): adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb.exe')) else: adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb')) info_notify('Starting adb...') output = adb(args, _iter=True, _out_bufsize=1, _err_to_out=True) for line in output: sys.stdout.write(line) sys.stdout.flush()
def wrapper_func(self, args): ctx = self.ctx ctx.set_archs(self._archs) ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir, user_ndk_dir=self.ndk_dir, user_android_api=self.android_api, user_ndk_ver=self.ndk_version, user_ndk_api=self.ndk_api) dist = self._dist bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) # recipes rarely change, but during dev, bootstraps can change from # build to build, so run prepare_bootstrap even if needs_build is false if dist.needs_build: if dist.folder_exists(): # possible if the dist is being replaced dist.delete() info_notify('No dist exists that meets your requirements, ' 'so one will be built.') build_dist_from_args(ctx, dist, args) func(self, args)
def download_web_backend_dependencies(self, arch): """ During building, host needs to download the jquery-ui package (in order to make it work the mpl web backend). This operation seems to fail in our CI tests, so we download this package at the expected location by the mpl install script which is defined by the environ variable `XDG_CACHE_HOME` (we modify that one in our `get_recipe_env` so it will be the same regardless of the host platform). """ env = self.get_recipe_env(arch) info_notify('Downloading jquery-ui for matplatlib web backend') # We use the same jquery-ui version than mpl's setup.py script, # inside function `_download_jquery_to` jquery_sha = ( 'f8233674366ab36b2c34c577ec77a3d70cac75d2e387d8587f3836345c0f624d') url = f'https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip' target_file = join(env['XDG_CACHE_HOME'], 'matplotlib', jquery_sha) info_notify(f'Will download into {env["XDG_CACHE_HOME"]}') ensure_dir(join(env['XDG_CACHE_HOME'], 'matplotlib')) self.download_file(url, target_file)
def build_dist_from_args(ctx, dist, args_list): '''Parses out any bootstrap related arguments, and uses them to build a dist.''' parser = argparse.ArgumentParser( description='Create a newAndroid project') parser.add_argument( '--bootstrap', help=('The name of the bootstrap type, \'pygame\' ' 'or \'sdl2\', or leave empty to let a ' 'bootstrap be chosen automatically from your ' 'requirements.'), default=None) args, unknown = parser.parse_known_args(args_list) bs = Bootstrap.get_bootstrap(args.bootstrap, ctx) build_order, python_modules, bs \ = get_recipe_order_and_bootstrap(ctx, dist.recipes, bs) info('The selected bootstrap is {}'.format(bs.name)) info_main('# Creating dist with {} bootstrap'.format(bs.name)) bs.distribution = dist info_notify('Dist will have name {} and recipes ({})'.format( dist.name, ', '.join(dist.recipes))) ctx.dist_name = bs.distribution.name ctx.prepare_bootstrap(bs) ctx.prepare_dist(ctx.dist_name) build_recipes(build_order, python_modules, ctx) ctx.bootstrap.run_distribute() info_main('# Your distribution was created successfully, exiting.') info('Dist can be found at (for now) {}' .format(join(ctx.dist_dir, ctx.dist_name))) return unknown
def get_distribution(cls, ctx, name=None, recipes=[], force_build=False, extra_dist_dirs=[], require_perfect_match=False): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. ''' existing_dists = Distribution.get_distributions(ctx) needs_build = True # whether the dist needs building, will be returned possible_dists = existing_dists # 0) Check if a dist with that name already exists if name is not None and name: possible_dists = [d for d in possible_dists if d.name == name] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes, return it for dist in possible_dists: if force_build: continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): info_notify('{} has compatible recipes, using this one' .format(dist.name)) return dist assert len(possible_dists) < 2 if not name and possible_dists: info('Asked for dist with name {} with recipes ({}), but a dist ' 'with this name already exists and has incompatible recipes ' '({})'.format(name, ', '.join(recipes), ', '.join(possible_dists[0].recipes))) info('No compatible dist found, so exiting.') exit(1) # # 2) Check if any downloadable dists meet the requirements # online_dists = [('testsdl2', ['hostpython2', 'sdl2_image', # 'sdl2_mixer', 'sdl2_ttf', # 'python2', 'sdl2', # 'pyjniussdl2', 'kivysdl2'], # 'https://github.com/inclement/sdl2-example-dist/archive/master.zip'), # ] # _possible_dists = [] # for dist_name, dist_recipes, dist_url in online_dists: # for recipe in recipes: # if recipe not in dist_recipes: # break # else: # dist = Distribution(ctx) # dist.name = dist_name # dist.url = dist_url # _possible_dists.append(dist) # # if _possible_dists # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join(ctx.dist_dir, dist.name) dist.recipes = recipes return dist
def get_distribution( cls, ctx, *, arch_name, # required keyword argument: there is no sensible default name=None, recipes=[], ndk_api=None, force_build=False, extra_dist_dirs=[], require_perfect_match=False, allow_replace_dist=True): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. ndk_api : int The NDK API to compile against, included in the dist because it cannot be changed later during APK packaging. arch_name : str The target architecture name to compile against, included in the dist because it cannot be changed later during APK packaging. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. allow_replace_dist : bool If True, will allow an existing dist with the specified name but incompatible requirements to be overwritten by a new one with the current requirements. ''' possible_dists = Distribution.get_distributions(ctx) # Will hold dists that would be built in the same folder as an existing dist folder_match_dist = None # 0) Check if a dist with that name and architecture already exists if name is not None and name: possible_dists = [ d for d in possible_dists if (d.name == name) and (arch_name in d.archs) ] if possible_dists: # There should only be one folder with a given dist name *and* arch. # We could check that here, but for compatibility let's let it slide # and just record the details of one of them. We only use this data to # possibly fail the build later, so it doesn't really matter if there # was more than one clash. folder_match_dist = possible_dists[0] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: if (ndk_api is not None and dist.ndk_api != ndk_api) or dist.ndk_api is None: continue for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes, arch and NDK API, return it for dist in possible_dists: if force_build: continue if ndk_api is not None and dist.ndk_api != ndk_api: continue if arch_name is not None and arch_name not in dist.archs: continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): info_notify('{} has compatible recipes, using this one'.format( dist.name)) return dist # If there was a name match but we didn't already choose it, # then the existing dist is incompatible with the requested # configuration and the build cannot continue if folder_match_dist is not None and not allow_replace_dist: raise BuildInterruptingException( 'Asked for dist with name {name} with recipes ({req_recipes}) and ' 'NDK API {req_ndk_api}, but a dist ' 'with this name already exists and has either incompatible recipes ' '({dist_recipes}) or NDK API {dist_ndk_api}'.format( name=name, req_ndk_api=ndk_api, dist_ndk_api=folder_match_dist.ndk_api, req_recipes=', '.join(recipes), dist_recipes=', '.join(folder_match_dist.recipes))) assert len(possible_dists) < 2 # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join( ctx.dist_dir, generate_dist_folder_name( name, [arch_name] if arch_name is not None else None, )) dist.recipes = recipes dist.ndk_api = ctx.ndk_api dist.archs = [arch_name] return dist
def build_recipes(build_order, python_modules, ctx, project_dir, ignore_project_setup_py=False): # Put recipes in correct build order info_notify("Recipe build order is {}".format(build_order)) if python_modules: python_modules = sorted(set(python_modules)) info_notify( ("The requirements ({}) were not found as recipes, they will be " "installed with pip.").format(", ".join(python_modules))) recipes = [Recipe.get_recipe(name, ctx) for name in build_order] # download is arch independent info_main("# Downloading recipes ") for recipe in recipes: recipe.download_if_necessary() for arch in ctx.archs: info_main("# Building all recipes for arch {}".format(arch.arch)) info_main("# Unpacking recipes") for recipe in recipes: ensure_dir(recipe.get_build_container_dir(arch.arch)) recipe.prepare_build_dir(arch.arch) info_main("# Prebuilding recipes") # 2) prebuild packages for recipe in recipes: info_main("Prebuilding {} for {}".format(recipe.name, arch.arch)) recipe.prebuild_arch(arch) recipe.apply_patches(arch) # 3) build packages info_main("# Building recipes") for recipe in recipes: info_main("Building {} for {}".format(recipe.name, arch.arch)) if recipe.should_build(arch): recipe.build_arch(arch) else: info("{} said it is already built, skipping".format( recipe.name)) recipe.install_libraries(arch) # 4) biglink everything info_main("# Biglinking object files") if not ctx.python_recipe: biglink(ctx, arch) else: warning("Context's python recipe found, " "skipping biglink (will this work?)") # 5) postbuild packages info_main("# Postbuilding recipes") for recipe in recipes: info_main("Postbuilding {} for {}".format(recipe.name, arch.arch)) recipe.postbuild_arch(arch) info_main("# Installing pure Python modules") run_pymodules_install(ctx, python_modules, project_dir, ignore_setup_py=ignore_project_setup_py)
def get_distribution(cls, ctx, name=None, recipes=[], force_build=False, extra_dist_dirs=[], require_perfect_match=False): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. ''' # AND: This whole function is a bit hacky, it needs checking # properly to make sure it follows logically correct # possibilities existing_dists = Distribution.get_distributions(ctx) needs_build = True # whether the dist needs building, will be returned possible_dists = existing_dists # 0) Check if a dist with that name already exists if name is not None and name: possible_dists = [d for d in possible_dists if d.name == name] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes, return it P4A_force_build = False for dist in possible_dists: if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): if force_build: # distribution exists, rebuild forced P4A_force_build = True continue else: # existing distribution returned, no build required info_notify( '{} has compatible recipes, using this one'.format( dist.name)) return dist assert len(possible_dists) < 2 if not name and possible_dists: info('Asked for dist with name {} with recipes ({}), but a dist ' 'with this name already exists and has incompatible recipes ' '({})'.format(name, ', '.join(recipes), ', '.join(possible_dists[0].recipes))) info('No compatible dist found, so exiting.') exit(1) # # 2) Check if any downloadable dists meet the requirements # online_dists = [('testsdl2', ['hostpython2', 'sdl2_image', # 'sdl2_mixer', 'sdl2_ttf', # 'python2', 'sdl2', # 'pyjniussdl2', 'kivysdl2'], # 'https://github.com/inclement/sdl2-example-dist/archive/master.zip'), # ] # _possible_dists = [] # for dist_name, dist_recipes, dist_url in online_dists: # for recipe in recipes: # if recipe not in dist_recipes: # break # else: # dist = Distribution(ctx) # dist.name = dist_name # dist.url = dist_url # _possible_dists.append(dist) # # if _possible_dists # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True """Locally modified recipes will be forced to build, others are reused environmental variable P4A_{recipe_name}_DIR points to local recipe P4A_force_build affects all local recipes""" dist.P4A_force_build = P4A_force_build and os.environ.get( 'P4A_force_build') if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join(ctx.dist_dir, dist.name) dist.recipes = recipes return dist
def get_distribution(cls, ctx, name=None, recipes=[], ndk_api=None, force_build=False, extra_dist_dirs=[], require_perfect_match=False, allow_replace_dist=True): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. allow_replace_dist : bool If True, will allow an existing dist with the specified name but incompatible requirements to be overwritten by a new one with the current requirements. ''' existing_dists = Distribution.get_distributions(ctx) needs_build = True # whether the dist needs building, will be returned possible_dists = existing_dists name_match_dist = None # 0) Check if a dist with that name already exists if name is not None and name: possible_dists = [d for d in possible_dists if d.name == name] if possible_dists: name_match_dist = possible_dists[0] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: if ( ndk_api is not None and dist.ndk_api != ndk_api ) or dist.ndk_api is None: continue for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes and ndk API, return it for dist in possible_dists: if force_build: continue if ndk_api is not None and dist.ndk_api != ndk_api: continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): info_notify('{} has compatible recipes, using this one' .format(dist.name)) return dist assert len(possible_dists) < 2 # If there was a name match but we didn't already choose it, # then the existing dist is incompatible with the requested # configuration and the build cannot continue if name_match_dist is not None and not allow_replace_dist: raise BuildInterruptingException( 'Asked for dist with name {name} with recipes ({req_recipes}) and ' 'NDK API {req_ndk_api}, but a dist ' 'with this name already exists and has either incompatible recipes ' '({dist_recipes}) or NDK API {dist_ndk_api}'.format( name=name, req_ndk_api=ndk_api, dist_ndk_api=name_match_dist.ndk_api, req_recipes=', '.join(recipes), dist_recipes=', '.join(name_match_dist.recipes))) # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join(ctx.dist_dir, dist.name) dist.recipes = recipes dist.ndk_api = ctx.ndk_api return dist
def get_distribution(cls, ctx, name=None, recipes=[], ndk_api=None, force_build=False, extra_dist_dirs=[], require_perfect_match=False, allow_replace_dist=True): '''Takes information about the distribution, and decides what kind of distribution it will be. If parameters conflict (e.g. a dist with that name already exists, but doesn't have the right set of recipes), an error is thrown. Parameters ---------- name : str The name of the distribution. If a dist with this name already ' exists, it will be used. recipes : list The recipes that the distribution must contain. force_download: bool If True, only downloaded dists are considered. force_build : bool If True, the dist is forced to be built locally. extra_dist_dirs : list Any extra directories in which to search for dists. require_perfect_match : bool If True, will only match distributions with precisely the correct set of recipes. allow_replace_dist : bool If True, will allow an existing dist with the specified name but incompatible requirements to be overwritten by a new one with the current requirements. ''' existing_dists = Distribution.get_distributions(ctx) possible_dists = existing_dists name_match_dist = None # 0) Check if a dist with that name already exists if name is not None and name: possible_dists = [d for d in possible_dists if d.name == name] if possible_dists: name_match_dist = possible_dists[0] # 1) Check if any existing dists meet the requirements _possible_dists = [] for dist in possible_dists: if ( ndk_api is not None and dist.ndk_api != ndk_api ) or dist.ndk_api is None: continue for recipe in recipes: if recipe not in dist.recipes: break else: _possible_dists.append(dist) possible_dists = _possible_dists if possible_dists: info('Of the existing distributions, the following meet ' 'the given requirements:') pretty_log_dists(possible_dists) else: info('No existing dists meet the given requirements!') # If any dist has perfect recipes and ndk API, return it for dist in possible_dists: if force_build: continue if ndk_api is not None and dist.ndk_api != ndk_api: continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and not require_perfect_match)): info_notify('{} has compatible recipes, using this one' .format(dist.name)) return dist assert len(possible_dists) < 2 # If there was a name match but we didn't already choose it, # then the existing dist is incompatible with the requested # configuration and the build cannot continue if name_match_dist is not None and not allow_replace_dist: raise BuildInterruptingException( 'Asked for dist with name {name} with recipes ({req_recipes}) and ' 'NDK API {req_ndk_api}, but a dist ' 'with this name already exists and has either incompatible recipes ' '({dist_recipes}) or NDK API {dist_ndk_api}'.format( name=name, req_ndk_api=ndk_api, dist_ndk_api=name_match_dist.ndk_api, req_recipes=', '.join(recipes), dist_recipes=', '.join(name_match_dist.recipes))) # If we got this far, we need to build a new dist dist = Distribution(ctx) dist.needs_build = True if not name: filen = 'unnamed_dist_{}' i = 1 while exists(join(ctx.dist_dir, filen.format(i))): i += 1 name = filen.format(i) dist.name = name dist.dist_dir = join(ctx.dist_dir, dist.name) dist.recipes = recipes dist.ndk_api = ctx.ndk_api return dist
def get_recipe_order_and_bootstrap(ctx, names, bs=None): '''Takes a list of recipe names and (optionally) a bootstrap. Then works out the dependency graph (including bootstrap recipes if necessary). Finally, if no bootstrap was initially selected, chooses one that supports all the recipes. ''' graph = Graph() recipes_to_load = set(names) if bs is not None and bs.recipe_depends: info_notify('Bootstrap requires recipes {}'.format(bs.recipe_depends)) recipes_to_load = recipes_to_load.union(set(bs.recipe_depends)) recipes_to_load = list(recipes_to_load) recipe_loaded = [] python_modules = [] while recipes_to_load: name = recipes_to_load.pop(0) if name in recipe_loaded or isinstance(name, (list, tuple)): continue try: recipe = Recipe.get_recipe(name, ctx) except IOError: info('No recipe named {}; will attempt to install with pip' .format(name)) python_modules.append(name) continue except (KeyboardInterrupt, SystemExit): raise except: warning('Failed to import recipe named {}; the recipe exists ' 'but appears broken.'.format(name)) warning('Exception was:') raise graph.add(name, name) info('Loaded recipe {} (depends on {}{})'.format( name, recipe.depends, ', conflicts {}'.format(recipe.conflicts) if recipe.conflicts else '')) for depend in recipe.depends: graph.add(name, depend) recipes_to_load += recipe.depends for conflict in recipe.conflicts: if graph.conflicts(conflict): warning( ('{} conflicts with {}, but both have been ' 'included or pulled into the requirements.' .format(recipe.name, conflict))) warning( 'Due to this conflict the build cannot continue, exiting.') exit(1) python_modules += recipe.python_depends recipe_loaded.append(name) graph.remove_remaining_conflicts(ctx) if len(graph.graphs) > 1: info('Found multiple valid recipe sets:') for g in graph.graphs: info(' {}'.format(g.keys())) info_notify('Using the first of these: {}' .format(graph.graphs[0].keys())) elif len(graph.graphs) == 0: warning('Didn\'t find any valid dependency graphs, exiting.') exit(1) else: info('Found a single valid recipe set (this is good)') build_order = list(graph.find_order(0)) if bs is None: # It would be better to check against possible # orders other than the first one, but in practice # there will rarely be clashes, and the user can # specify more parameters if necessary to resolve # them. bs = Bootstrap.get_bootstrap_from_recipes(build_order, ctx) if bs is None: info('Could not find a bootstrap compatible with the ' 'required recipes.') info('If you think such a combination should exist, try ' 'specifying the bootstrap manually with --bootstrap.') exit(1) info('{} bootstrap appears compatible with the required recipes.' .format(bs.name)) info('Checking this...') recipes_to_load = bs.recipe_depends # This code repeats the code from earlier! Should move to a function: while recipes_to_load: name = recipes_to_load.pop(0) if name in recipe_loaded or isinstance(name, (list, tuple)): continue try: recipe = Recipe.get_recipe(name, ctx) except ImportError: info('No recipe named {}; will attempt to install with pip' .format(name)) python_modules.append(name) continue graph.add(name, name) info('Loaded recipe {} (depends on {}{})'.format( name, recipe.depends, ', conflicts {}'.format(recipe.conflicts) if recipe.conflicts else '')) for depend in recipe.depends: graph.add(name, depend) recipes_to_load += recipe.depends for conflict in recipe.conflicts: if graph.conflicts(conflict): warning( ('{} conflicts with {}, but both have been ' 'included or pulled into the requirements.' .format(recipe.name, conflict))) warning('Due to this conflict the build cannot continue, ' 'exiting.') exit(1) recipe_loaded.append(name) graph.remove_remaining_conflicts(ctx) build_order = list(graph.find_order(0)) return build_order, python_modules, bs
def prebuild_arch(self, arch): super(ProtobufCppRecipe, self).prebuild_arch(arch) patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched') if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark): self.apply_patch('fix-python3-compatibility.patch', arch.arch) shprint(sh.touch, patch_mark) # During building, host needs to transpile .proto files to .py # ideally with the same version as protobuf runtime, or with an older one. # Because protoc is compiled for target (i.e. Android), we need an other binary # which can be run by host. # To make it easier, we download prebuild protoc binary adapted to the platform info_notify("Downloading protoc compiler for your platform") url_prefix = "https://github.com/protocolbuffers/protobuf/releases/download/v{version}".format(version=self.version) if sys.platform.startswith('linux'): info_notify("GNU/Linux detected") filename = "protoc-{version}-linux-x86_64.zip".format(version=self.version) elif sys.platform.startswith('darwin'): info_notify("Mac OS X detected") filename = "protoc-{version}-osx-x86_64.zip".format(version=self.version) else: info_notify("Your platform is not supported, but recipe can still " "be built if you have a valid protoc (<={version}) in " "your path".format(version=self.version)) return protoc_url = join(url_prefix, filename) self.protoc_dir = join(self.ctx.build_dir, "tools", "protoc") if os.path.exists(join(self.protoc_dir, "bin", "protoc")): info_notify("protoc found, no download needed") return try: os.makedirs(self.protoc_dir) except OSError as e: # if dir already exists (errno 17), we ignore the error if e.errno != 17: raise e info_notify("Will download into {dest_dir}".format(dest_dir=self.protoc_dir)) self.download_file(protoc_url, join(self.protoc_dir, filename)) with current_directory(self.protoc_dir): shprint(sh.unzip, join(self.protoc_dir, filename))
def prebuild_arch(self, arch): super(ProtobufCppRecipe, self).prebuild_arch(arch) # During building, host needs to transpile .proto files to .py # ideally with the same version as protobuf runtime, or with an older one. # Because protoc is compiled for target (i.e. Android), we need an other binary # which can be run by host. # To make it easier, we download prebuild protoc binary adapted to the platform info_notify("Downloading protoc compiler for your platform") url_prefix = "https://github.com/protocolbuffers/protobuf/releases/download/v{version}".format(version=self.version) if sys.platform.startswith('linux'): info_notify("GNU/Linux detected") filename = "protoc-{version}-linux-x86_64.zip".format(version=self.version) elif sys.platform.startswith('darwin'): info_notify("Mac OS X detected") filename = "protoc-{version}-osx-x86_64.zip".format(version=self.version) else: info_notify("Your platform is not supported, but recipe can still " "be built if you have a valid protoc (<={version}) in " "your path".format(version=self.version)) return protoc_url = join(url_prefix, filename) self.protoc_dir = join(self.ctx.build_dir, "tools", "protoc") if os.path.exists(join(self.protoc_dir, "bin", "protoc")): info_notify("protoc found, no download needed") return try: os.makedirs(self.protoc_dir) except OSError as e: # if dir already exists (errno 17), we ignore the error if e.errno != 17: raise e info_notify("Will download into {dest_dir}".format(dest_dir=self.protoc_dir)) self.download_file(protoc_url, join(self.protoc_dir, filename)) with current_directory(self.protoc_dir): shprint(sh.unzip, join(self.protoc_dir, filename))
def get_recipe_order_and_bootstrap(ctx, names, bs=None): '''Takes a list of recipe names and (optionally) a bootstrap. Then works out the dependency graph (including bootstrap recipes if necessary). Finally, if no bootstrap was initially selected, chooses one that supports all the recipes. ''' graph = Graph() recipes_to_load = set(names) if bs is not None and bs.recipe_depends: info_notify('Bootstrap requires recipes {}'.format(bs.recipe_depends)) recipes_to_load = recipes_to_load.union(set(bs.recipe_depends)) recipes_to_load = list(recipes_to_load) recipe_loaded = [] python_modules = [] while recipes_to_load: name = recipes_to_load.pop(0) if name in recipe_loaded or isinstance(name, (list, tuple)): continue try: recipe = Recipe.get_recipe(name, ctx) except IOError: info('No recipe named {}; will attempt to install with pip'.format( name)) python_modules.append(name) continue except (KeyboardInterrupt, SystemExit): raise except: warning('Failed to import recipe named {}; the recipe exists ' 'but appears broken.'.format(name)) warning('Exception was:') raise graph.add(name, name) info('Loaded recipe {} (depends on {}{})'.format( name, recipe.depends, ', conflicts {}'.format(recipe.conflicts) if recipe.conflicts else '')) for depend in recipe.depends: graph.add(name, depend) recipes_to_load += recipe.depends for conflict in recipe.conflicts: if graph.conflicts(conflict): warning(('{} conflicts with {}, but both have been ' 'included or pulled into the requirements.'.format( recipe.name, conflict))) warning( 'Due to this conflict the build cannot continue, exiting.') exit(1) python_modules += recipe.python_depends recipe_loaded.append(name) graph.remove_remaining_conflicts(ctx) if len(graph.graphs) > 1: info('Found multiple valid recipe sets:') for g in graph.graphs: info(' {}'.format(g.keys())) info_notify('Using the first of these: {}'.format( graph.graphs[0].keys())) elif len(graph.graphs) == 0: warning('Didn\'t find any valid dependency graphs, exiting.') exit(1) else: info('Found a single valid recipe set (this is good)') build_order = list(graph.find_order(0)) if bs is None: # It would be better to check against possible # orders other than the first one, but in practice # there will rarely be clashes, and the user can # specify more parameters if necessary to resolve # them. bs = Bootstrap.get_bootstrap_from_recipes(build_order, ctx) if bs is None: info('Could not find a bootstrap compatible with the ' 'required recipes.') info('If you think such a combination should exist, try ' 'specifying the bootstrap manually with --bootstrap.') exit(1) info('{} bootstrap appears compatible with the required recipes.'. format(bs.name)) info('Checking this...') recipes_to_load = bs.recipe_depends # This code repeats the code from earlier! Should move to a function: while recipes_to_load: name = recipes_to_load.pop(0) if name in recipe_loaded or isinstance(name, (list, tuple)): continue try: recipe = Recipe.get_recipe(name, ctx) except ImportError: info('No recipe named {}; will attempt to install with pip'. format(name)) python_modules.append(name) continue graph.add(name, name) info('Loaded recipe {} (depends on {}{})'.format( name, recipe.depends, ', conflicts {}'.format(recipe.conflicts) if recipe.conflicts else '')) for depend in recipe.depends: graph.add(name, depend) recipes_to_load += recipe.depends for conflict in recipe.conflicts: if graph.conflicts(conflict): warning( ('{} conflicts with {}, but both have been ' 'included or pulled into the requirements.'.format( recipe.name, conflict))) warning('Due to this conflict the build cannot continue, ' 'exiting.') exit(1) recipe_loaded.append(name) graph.remove_remaining_conflicts(ctx) build_order = list(graph.find_order(0)) return build_order, python_modules, bs