def build_recipes(build_order, python_modules, ctx):
# Put recipes in correct build order
bs = ctx.bootstrap
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'
.format(recipe.name))
# 4) biglink everything
# AND: Should make this optional
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx, python_modules)
return
def build_dist_from_args(ctx, dist, args):
'''Parses out any bootstrap related arguments, and uses them to build
a dist.'''
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
build_order, python_modules, bs \
= get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
ctx.recipe_build_order = build_order
ctx.python_modules = python_modules
if python_modules and hasattr(sys, 'real_prefix'):
error('virtualenv is needed to install pure-Python modules, but')
error('virtualenv does not support nesting, and you are running')
error('python-for-android in one. Please run p4a outside of a')
error('virtualenv instead.')
exit(1)
info('The selected bootstrap is {}'.format(bs.name))
info_main('# Creating dist with {} bootstrap'.format(bs.name))
bs.distribution = dist
info_notify('Dist will have name {} and recipes ({})'.format(
dist.name, ', '.join(dist.recipes)))
ctx.dist_name = bs.distribution.name
ctx.prepare_bootstrap(bs)
ctx.prepare_dist(ctx.dist_name)
build_recipes(build_order, python_modules, ctx)
ctx.bootstrap.run_distribute()
info_main('# Your distribution was created successfully, exiting.')
info('Dist can be found at (for now) {}'
.format(join(ctx.dist_dir, ctx.dist_name)))
def build_dist_from_args(ctx, dist, args):
'''Parses out any bootstrap related arguments, and uses them to build
a dist.'''
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
build_order, python_modules, bs \
= get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
ctx.recipe_build_order = build_order
info('The selected bootstrap is {}'.format(bs.name))
info_main('# Creating dist with {} bootstrap'.format(bs.name))
bs.distribution = dist
info_notify('Dist will have name {} and recipes ({})'.format(
dist.name, ', '.join(dist.recipes)))
ctx.dist_name = bs.distribution.name
ctx.prepare_bootstrap(bs)
ctx.prepare_dist(ctx.dist_name)
build_recipes(build_order, python_modules, ctx)
ctx.bootstrap.run_distribute()
info_main('# Your distribution was created successfully, exiting.')
info('Dist can be found at (for now) {}'
.format(join(ctx.dist_dir, ctx.dist_name)))
def build_recipes(build_order, python_modules, ctx):
# Put recipes in correct build order
bs = ctx.bootstrap
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
('The requirements ({}) were not found as recipes, they will be '
'installed with pip.').format(', '.join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main('# Downloading recipes ')
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main('# Building all recipes for arch {}'.format(arch.arch))
info_main('# Unpacking recipes')
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main('# Prebuilding recipes')
# 2) prebuild packages
for recipe in recipes:
info_main('Prebuilding {} for {}'.format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main('# Building recipes')
for recipe in recipes:
info_main('Building {} for {}'.format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info('{} said it is already built, skipping'.format(
recipe.name))
# 4) biglink everything
info_main('# Biglinking object files')
if not ctx.python_recipe or not ctx.python_recipe.from_crystax:
biglink(ctx, arch)
else:
info('NDK is crystax, skipping biglink (will this work?)')
# 5) postbuild packages
info_main('# Postbuilding recipes')
for recipe in recipes:
info_main('Postbuilding {} for {}'.format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main('# Installing pure Python modules')
run_pymodules_install(ctx, python_modules)
return
def wrapper_func(self, args):
ctx = self.ctx
ctx.set_archs(self._archs)
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version)
dist = self._dist
if dist.needs_build:
info_notify('No dist exists that meets your requirements, '
'so one will be built.')
build_dist_from_args(ctx, dist, args)
func(self, args)
def wrapper_func(self, args):
ctx = self.ctx
ctx.set_archs(self._archs)
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version)
dist = self._dist
if dist.needs_build:
info_notify('No dist exists that meets your requirements, '
'so one will be built.')
build_dist_from_args(ctx, dist, args)
func(self, args)
def wrapper_func(self, args):
ctx = self.ctx
ctx.set_archs(self._archs)
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_api=self.ndk_api)
dist = self._dist
if dist.needs_build:
if dist.folder_exists(): # possible if the dist is being replaced
dist.delete()
info_notify('No dist exists that meets your requirements, '
'so one will be built.')
build_dist_from_args(ctx, dist, args)
func(self, args)
def wrapper_func(self, args):
ctx = self.ctx
ctx.set_archs(self._archs)
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version,
user_ndk_api=self.ndk_api)
dist = self._dist
if dist.needs_build:
if dist.folder_exists(): # possible if the dist is being replaced
dist.delete()
info_notify('No dist exists that meets your requirements, '
'so one will be built.')
build_dist_from_args(ctx, dist, args)
func(self, args)
def _adb(self, commands):
'''Call the adb executable from the SDK, passing the given commands as
arguments.'''
ctx = self.ctx
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version)
if platform in ('win32', 'cygwin'):
adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb.exe'))
else:
adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb'))
info_notify('Starting adb...')
output = adb(*commands, _iter=True, _out_bufsize=1, _err_to_out=True)
for line in output:
sys.stdout.write(line)
sys.stdout.flush()
def _adb(self, commands):
'''Call the adb executable from the SDK, passing the given commands as
arguments.'''
ctx = self.ctx
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version)
if platform in ('win32', 'cygwin'):
adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb.exe'))
else:
adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb'))
info_notify('Starting adb...')
output = adb(*commands, _iter=True, _out_bufsize=1, _err_to_out=True)
for line in output:
sys.stdout.write(line)
sys.stdout.flush()
def build_dist_from_args(ctx, dist, args):
"""Parses out any bootstrap related arguments, and uses them to build
a dist."""
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
blacklist = getattr(args, "blacklist_requirements", "").split(",")
if len(blacklist) == 1 and blacklist[0] == "":
blacklist = []
build_order, python_modules, bs = (
get_recipe_order_and_bootstrap(
ctx, dist.recipes, bs,
blacklist=blacklist
))
assert set(build_order).intersection(set(python_modules)) == set()
ctx.recipe_build_order = build_order
ctx.python_modules = python_modules
info('The selected bootstrap is {}'.format(bs.name))
info_main('# Creating dist with {} bootstrap'.format(bs.name))
bs.distribution = dist
info_notify('Dist will have name {} and requirements ({})'.format(
dist.name, ', '.join(dist.recipes)))
info('Dist contains the following requirements as recipes: {}'.format(
ctx.recipe_build_order))
info('Dist will also contain modules ({}) installed from pip'.format(
', '.join(ctx.python_modules)))
ctx.distribution = dist
ctx.prepare_bootstrap(bs)
if dist.needs_build:
ctx.prepare_dist()
build_recipes(build_order, python_modules, ctx,
getattr(args, "private", None),
ignore_project_setup_py=getattr(
args, "ignore_setup_py", False
),
)
ctx.bootstrap.run_distribute()
info_main('# Your distribution was created successfully, exiting.')
info('Dist can be found at (for now) {}'
.format(join(ctx.dist_dir, ctx.distribution.dist_dir)))
def adb(self, args):
'''Runs the adb binary from the detected SDK directory, passing all
arguments straight to it. This is intended as a convenience
function if adb is not in your $PATH.
'''
ctx = self.ctx
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version)
if platform in ('win32', 'cygwin'):
adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb.exe'))
else:
adb = sh.Command(join(ctx.sdk_dir, 'platform-tools', 'adb'))
info_notify('Starting adb...')
output = adb(args, _iter=True, _out_bufsize=1, _err_to_out=True)
for line in output:
sys.stdout.write(line)
sys.stdout.flush()
def wrapper_func(self, args):
ctx = self.ctx
ctx.set_archs(self._archs)
ctx.prepare_build_environment(user_sdk_dir=self.sdk_dir,
user_ndk_dir=self.ndk_dir,
user_android_api=self.android_api,
user_ndk_ver=self.ndk_version,
user_ndk_api=self.ndk_api)
dist = self._dist
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
# recipes rarely change, but during dev, bootstraps can change from
# build to build, so run prepare_bootstrap even if needs_build is false
if dist.needs_build:
if dist.folder_exists(): # possible if the dist is being replaced
dist.delete()
info_notify('No dist exists that meets your requirements, '
'so one will be built.')
build_dist_from_args(ctx, dist, args)
func(self, args)
def download_web_backend_dependencies(self, arch):
"""
During building, host needs to download the jquery-ui package (in order
to make it work the mpl web backend). This operation seems to fail
in our CI tests, so we download this package at the expected location
by the mpl install script which is defined by the environ variable
`XDG_CACHE_HOME` (we modify that one in our `get_recipe_env` so it will
be the same regardless of the host platform).
"""
env = self.get_recipe_env(arch)
info_notify('Downloading jquery-ui for matplatlib web backend')
# We use the same jquery-ui version than mpl's setup.py script,
# inside function `_download_jquery_to`
jquery_sha = (
'f8233674366ab36b2c34c577ec77a3d70cac75d2e387d8587f3836345c0f624d')
url = f'https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip'
target_file = join(env['XDG_CACHE_HOME'], 'matplotlib', jquery_sha)
info_notify(f'Will download into {env["XDG_CACHE_HOME"]}')
ensure_dir(join(env['XDG_CACHE_HOME'], 'matplotlib'))
self.download_file(url, target_file)
def build_dist_from_args(ctx, dist, args_list):
'''Parses out any bootstrap related arguments, and uses them to build
a dist.'''
parser = argparse.ArgumentParser(
description='Create a newAndroid project')
parser.add_argument(
'--bootstrap',
help=('The name of the bootstrap type, \'pygame\' '
'or \'sdl2\', or leave empty to let a '
'bootstrap be chosen automatically from your '
'requirements.'),
default=None)
args, unknown = parser.parse_known_args(args_list)
bs = Bootstrap.get_bootstrap(args.bootstrap, ctx)
build_order, python_modules, bs \
= get_recipe_order_and_bootstrap(ctx, dist.recipes, bs)
info('The selected bootstrap is {}'.format(bs.name))
info_main('# Creating dist with {} bootstrap'.format(bs.name))
bs.distribution = dist
info_notify('Dist will have name {} and recipes ({})'.format(
dist.name, ', '.join(dist.recipes)))
ctx.dist_name = bs.distribution.name
ctx.prepare_bootstrap(bs)
ctx.prepare_dist(ctx.dist_name)
build_recipes(build_order, python_modules, ctx)
ctx.bootstrap.run_distribute()
info_main('# Your distribution was created successfully, exiting.')
info('Dist can be found at (for now) {}'
.format(join(ctx.dist_dir, ctx.dist_name)))
return unknown
def get_distribution(cls, ctx, name=None, recipes=[],
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
'''
existing_dists = Distribution.get_distributions(ctx)
needs_build = True # whether the dist needs building, will be returned
possible_dists = existing_dists
# 0) Check if a dist with that name already exists
if name is not None and name:
possible_dists = [d for d in possible_dists if d.name == name]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes, return it
for dist in possible_dists:
if force_build:
continue
if (set(dist.recipes) == set(recipes) or
(set(recipes).issubset(set(dist.recipes)) and
not require_perfect_match)):
info_notify('{} has compatible recipes, using this one'
.format(dist.name))
return dist
assert len(possible_dists) < 2
if not name and possible_dists:
info('Asked for dist with name {} with recipes ({}), but a dist '
'with this name already exists and has incompatible recipes '
'({})'.format(name, ', '.join(recipes),
', '.join(possible_dists[0].recipes)))
info('No compatible dist found, so exiting.')
exit(1)
# # 2) Check if any downloadable dists meet the requirements
# online_dists = [('testsdl2', ['hostpython2', 'sdl2_image',
# 'sdl2_mixer', 'sdl2_ttf',
# 'python2', 'sdl2',
# 'pyjniussdl2', 'kivysdl2'],
# 'https://github.com/inclement/sdl2-example-dist/archive/master.zip'),
# ]
# _possible_dists = []
# for dist_name, dist_recipes, dist_url in online_dists:
# for recipe in recipes:
# if recipe not in dist_recipes:
# break
# else:
# dist = Distribution(ctx)
# dist.name = dist_name
# dist.url = dist_url
# _possible_dists.append(dist)
# # if _possible_dists
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(ctx.dist_dir, dist.name)
dist.recipes = recipes
return dist
def get_distribution(
cls,
ctx,
*,
arch_name, # required keyword argument: there is no sensible default
name=None,
recipes=[],
ndk_api=None,
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False,
allow_replace_dist=True):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
ndk_api : int
The NDK API to compile against, included in the dist because it cannot
be changed later during APK packaging.
arch_name : str
The target architecture name to compile against, included in the dist because
it cannot be changed later during APK packaging.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
allow_replace_dist : bool
If True, will allow an existing dist with the specified
name but incompatible requirements to be overwritten by
a new one with the current requirements.
'''
possible_dists = Distribution.get_distributions(ctx)
# Will hold dists that would be built in the same folder as an existing dist
folder_match_dist = None
# 0) Check if a dist with that name and architecture already exists
if name is not None and name:
possible_dists = [
d for d in possible_dists
if (d.name == name) and (arch_name in d.archs)
]
if possible_dists:
# There should only be one folder with a given dist name *and* arch.
# We could check that here, but for compatibility let's let it slide
# and just record the details of one of them. We only use this data to
# possibly fail the build later, so it doesn't really matter if there
# was more than one clash.
folder_match_dist = possible_dists[0]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
if (ndk_api is not None
and dist.ndk_api != ndk_api) or dist.ndk_api is None:
continue
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes, arch and NDK API, return it
for dist in possible_dists:
if force_build:
continue
if ndk_api is not None and dist.ndk_api != ndk_api:
continue
if arch_name is not None and arch_name not in dist.archs:
continue
if (set(dist.recipes) == set(recipes)
or (set(recipes).issubset(set(dist.recipes))
and not require_perfect_match)):
info_notify('{} has compatible recipes, using this one'.format(
dist.name))
return dist
# If there was a name match but we didn't already choose it,
# then the existing dist is incompatible with the requested
# configuration and the build cannot continue
if folder_match_dist is not None and not allow_replace_dist:
raise BuildInterruptingException(
'Asked for dist with name {name} with recipes ({req_recipes}) and '
'NDK API {req_ndk_api}, but a dist '
'with this name already exists and has either incompatible recipes '
'({dist_recipes}) or NDK API {dist_ndk_api}'.format(
name=name,
req_ndk_api=ndk_api,
dist_ndk_api=folder_match_dist.ndk_api,
req_recipes=', '.join(recipes),
dist_recipes=', '.join(folder_match_dist.recipes)))
assert len(possible_dists) < 2
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(
ctx.dist_dir,
generate_dist_folder_name(
name,
[arch_name] if arch_name is not None else None,
))
dist.recipes = recipes
dist.ndk_api = ctx.ndk_api
dist.archs = [arch_name]
return dist
def build_recipes(build_order,
python_modules,
ctx,
project_dir,
ignore_project_setup_py=False):
# Put recipes in correct build order
info_notify("Recipe build order is {}".format(build_order))
if python_modules:
python_modules = sorted(set(python_modules))
info_notify(
("The requirements ({}) were not found as recipes, they will be "
"installed with pip.").format(", ".join(python_modules)))
recipes = [Recipe.get_recipe(name, ctx) for name in build_order]
# download is arch independent
info_main("# Downloading recipes ")
for recipe in recipes:
recipe.download_if_necessary()
for arch in ctx.archs:
info_main("# Building all recipes for arch {}".format(arch.arch))
info_main("# Unpacking recipes")
for recipe in recipes:
ensure_dir(recipe.get_build_container_dir(arch.arch))
recipe.prepare_build_dir(arch.arch)
info_main("# Prebuilding recipes")
# 2) prebuild packages
for recipe in recipes:
info_main("Prebuilding {} for {}".format(recipe.name, arch.arch))
recipe.prebuild_arch(arch)
recipe.apply_patches(arch)
# 3) build packages
info_main("# Building recipes")
for recipe in recipes:
info_main("Building {} for {}".format(recipe.name, arch.arch))
if recipe.should_build(arch):
recipe.build_arch(arch)
else:
info("{} said it is already built, skipping".format(
recipe.name))
recipe.install_libraries(arch)
# 4) biglink everything
info_main("# Biglinking object files")
if not ctx.python_recipe:
biglink(ctx, arch)
else:
warning("Context's python recipe found, "
"skipping biglink (will this work?)")
# 5) postbuild packages
info_main("# Postbuilding recipes")
for recipe in recipes:
info_main("Postbuilding {} for {}".format(recipe.name, arch.arch))
recipe.postbuild_arch(arch)
info_main("# Installing pure Python modules")
run_pymodules_install(ctx,
python_modules,
project_dir,
ignore_setup_py=ignore_project_setup_py)
def get_distribution(cls,
ctx,
name=None,
recipes=[],
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
'''
# AND: This whole function is a bit hacky, it needs checking
# properly to make sure it follows logically correct
# possibilities
existing_dists = Distribution.get_distributions(ctx)
needs_build = True # whether the dist needs building, will be returned
possible_dists = existing_dists
# 0) Check if a dist with that name already exists
if name is not None and name:
possible_dists = [d for d in possible_dists if d.name == name]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes, return it
P4A_force_build = False
for dist in possible_dists:
if (set(dist.recipes) == set(recipes)
or (set(recipes).issubset(set(dist.recipes))
and not require_perfect_match)):
if force_build:
# distribution exists, rebuild forced
P4A_force_build = True
continue
else:
# existing distribution returned, no build required
info_notify(
'{} has compatible recipes, using this one'.format(
dist.name))
return dist
assert len(possible_dists) < 2
if not name and possible_dists:
info('Asked for dist with name {} with recipes ({}), but a dist '
'with this name already exists and has incompatible recipes '
'({})'.format(name, ', '.join(recipes),
', '.join(possible_dists[0].recipes)))
info('No compatible dist found, so exiting.')
exit(1)
# # 2) Check if any downloadable dists meet the requirements
# online_dists = [('testsdl2', ['hostpython2', 'sdl2_image',
# 'sdl2_mixer', 'sdl2_ttf',
# 'python2', 'sdl2',
# 'pyjniussdl2', 'kivysdl2'],
# 'https://github.com/inclement/sdl2-example-dist/archive/master.zip'),
# ]
# _possible_dists = []
# for dist_name, dist_recipes, dist_url in online_dists:
# for recipe in recipes:
# if recipe not in dist_recipes:
# break
# else:
# dist = Distribution(ctx)
# dist.name = dist_name
# dist.url = dist_url
# _possible_dists.append(dist)
# # if _possible_dists
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
"""Locally modified recipes will be forced to build, others are reused
environmental variable P4A_{recipe_name}_DIR points to local recipe
P4A_force_build affects all local recipes"""
dist.P4A_force_build = P4A_force_build and os.environ.get(
'P4A_force_build')
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(ctx.dist_dir, dist.name)
dist.recipes = recipes
return dist
def get_distribution(cls, ctx, name=None, recipes=[],
ndk_api=None,
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False,
allow_replace_dist=True):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
allow_replace_dist : bool
If True, will allow an existing dist with the specified
name but incompatible requirements to be overwritten by
a new one with the current requirements.
'''
existing_dists = Distribution.get_distributions(ctx)
needs_build = True # whether the dist needs building, will be returned
possible_dists = existing_dists
name_match_dist = None
# 0) Check if a dist with that name already exists
if name is not None and name:
possible_dists = [d for d in possible_dists if d.name == name]
if possible_dists:
name_match_dist = possible_dists[0]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
if (
ndk_api is not None and dist.ndk_api != ndk_api
) or dist.ndk_api is None:
continue
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes and ndk API, return it
for dist in possible_dists:
if force_build:
continue
if ndk_api is not None and dist.ndk_api != ndk_api:
continue
if (set(dist.recipes) == set(recipes) or
(set(recipes).issubset(set(dist.recipes)) and
not require_perfect_match)):
info_notify('{} has compatible recipes, using this one'
.format(dist.name))
return dist
assert len(possible_dists) < 2
# If there was a name match but we didn't already choose it,
# then the existing dist is incompatible with the requested
# configuration and the build cannot continue
if name_match_dist is not None and not allow_replace_dist:
raise BuildInterruptingException(
'Asked for dist with name {name} with recipes ({req_recipes}) and '
'NDK API {req_ndk_api}, but a dist '
'with this name already exists and has either incompatible recipes '
'({dist_recipes}) or NDK API {dist_ndk_api}'.format(
name=name,
req_ndk_api=ndk_api,
dist_ndk_api=name_match_dist.ndk_api,
req_recipes=', '.join(recipes),
dist_recipes=', '.join(name_match_dist.recipes)))
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(ctx.dist_dir, dist.name)
dist.recipes = recipes
dist.ndk_api = ctx.ndk_api
return dist
def get_distribution(cls, ctx, name=None, recipes=[],
ndk_api=None,
force_build=False,
extra_dist_dirs=[],
require_perfect_match=False,
allow_replace_dist=True):
'''Takes information about the distribution, and decides what kind of
distribution it will be.
If parameters conflict (e.g. a dist with that name already
exists, but doesn't have the right set of recipes),
an error is thrown.
Parameters
----------
name : str
The name of the distribution. If a dist with this name already '
exists, it will be used.
recipes : list
The recipes that the distribution must contain.
force_download: bool
If True, only downloaded dists are considered.
force_build : bool
If True, the dist is forced to be built locally.
extra_dist_dirs : list
Any extra directories in which to search for dists.
require_perfect_match : bool
If True, will only match distributions with precisely the
correct set of recipes.
allow_replace_dist : bool
If True, will allow an existing dist with the specified
name but incompatible requirements to be overwritten by
a new one with the current requirements.
'''
existing_dists = Distribution.get_distributions(ctx)
possible_dists = existing_dists
name_match_dist = None
# 0) Check if a dist with that name already exists
if name is not None and name:
possible_dists = [d for d in possible_dists if d.name == name]
if possible_dists:
name_match_dist = possible_dists[0]
# 1) Check if any existing dists meet the requirements
_possible_dists = []
for dist in possible_dists:
if (
ndk_api is not None and dist.ndk_api != ndk_api
) or dist.ndk_api is None:
continue
for recipe in recipes:
if recipe not in dist.recipes:
break
else:
_possible_dists.append(dist)
possible_dists = _possible_dists
if possible_dists:
info('Of the existing distributions, the following meet '
'the given requirements:')
pretty_log_dists(possible_dists)
else:
info('No existing dists meet the given requirements!')
# If any dist has perfect recipes and ndk API, return it
for dist in possible_dists:
if force_build:
continue
if ndk_api is not None and dist.ndk_api != ndk_api:
continue
if (set(dist.recipes) == set(recipes) or
(set(recipes).issubset(set(dist.recipes)) and
not require_perfect_match)):
info_notify('{} has compatible recipes, using this one'
.format(dist.name))
return dist
assert len(possible_dists) < 2
# If there was a name match but we didn't already choose it,
# then the existing dist is incompatible with the requested
# configuration and the build cannot continue
if name_match_dist is not None and not allow_replace_dist:
raise BuildInterruptingException(
'Asked for dist with name {name} with recipes ({req_recipes}) and '
'NDK API {req_ndk_api}, but a dist '
'with this name already exists and has either incompatible recipes '
'({dist_recipes}) or NDK API {dist_ndk_api}'.format(
name=name,
req_ndk_api=ndk_api,
dist_ndk_api=name_match_dist.ndk_api,
req_recipes=', '.join(recipes),
dist_recipes=', '.join(name_match_dist.recipes)))
# If we got this far, we need to build a new dist
dist = Distribution(ctx)
dist.needs_build = True
if not name:
filen = 'unnamed_dist_{}'
i = 1
while exists(join(ctx.dist_dir, filen.format(i))):
i += 1
name = filen.format(i)
dist.name = name
dist.dist_dir = join(ctx.dist_dir, dist.name)
dist.recipes = recipes
dist.ndk_api = ctx.ndk_api
return dist
def get_recipe_order_and_bootstrap(ctx, names, bs=None):
'''Takes a list of recipe names and (optionally) a bootstrap. Then
works out the dependency graph (including bootstrap recipes if
necessary). Finally, if no bootstrap was initially selected,
chooses one that supports all the recipes.
'''
graph = Graph()
recipes_to_load = set(names)
if bs is not None and bs.recipe_depends:
info_notify('Bootstrap requires recipes {}'.format(bs.recipe_depends))
recipes_to_load = recipes_to_load.union(set(bs.recipe_depends))
recipes_to_load = list(recipes_to_load)
recipe_loaded = []
python_modules = []
while recipes_to_load:
name = recipes_to_load.pop(0)
if name in recipe_loaded or isinstance(name, (list, tuple)):
continue
try:
recipe = Recipe.get_recipe(name, ctx)
except IOError:
info('No recipe named {}; will attempt to install with pip'
.format(name))
python_modules.append(name)
continue
except (KeyboardInterrupt, SystemExit):
raise
except:
warning('Failed to import recipe named {}; the recipe exists '
'but appears broken.'.format(name))
warning('Exception was:')
raise
graph.add(name, name)
info('Loaded recipe {} (depends on {}{})'.format(
name, recipe.depends,
', conflicts {}'.format(recipe.conflicts) if recipe.conflicts
else ''))
for depend in recipe.depends:
graph.add(name, depend)
recipes_to_load += recipe.depends
for conflict in recipe.conflicts:
if graph.conflicts(conflict):
warning(
('{} conflicts with {}, but both have been '
'included or pulled into the requirements.'
.format(recipe.name, conflict)))
warning(
'Due to this conflict the build cannot continue, exiting.')
exit(1)
python_modules += recipe.python_depends
recipe_loaded.append(name)
graph.remove_remaining_conflicts(ctx)
if len(graph.graphs) > 1:
info('Found multiple valid recipe sets:')
for g in graph.graphs:
info(' {}'.format(g.keys()))
info_notify('Using the first of these: {}'
.format(graph.graphs[0].keys()))
elif len(graph.graphs) == 0:
warning('Didn\'t find any valid dependency graphs, exiting.')
exit(1)
else:
info('Found a single valid recipe set (this is good)')
build_order = list(graph.find_order(0))
if bs is None: # It would be better to check against possible
# orders other than the first one, but in practice
# there will rarely be clashes, and the user can
# specify more parameters if necessary to resolve
# them.
bs = Bootstrap.get_bootstrap_from_recipes(build_order, ctx)
if bs is None:
info('Could not find a bootstrap compatible with the '
'required recipes.')
info('If you think such a combination should exist, try '
'specifying the bootstrap manually with --bootstrap.')
exit(1)
info('{} bootstrap appears compatible with the required recipes.'
.format(bs.name))
info('Checking this...')
recipes_to_load = bs.recipe_depends
# This code repeats the code from earlier! Should move to a function:
while recipes_to_load:
name = recipes_to_load.pop(0)
if name in recipe_loaded or isinstance(name, (list, tuple)):
continue
try:
recipe = Recipe.get_recipe(name, ctx)
except ImportError:
info('No recipe named {}; will attempt to install with pip'
.format(name))
python_modules.append(name)
continue
graph.add(name, name)
info('Loaded recipe {} (depends on {}{})'.format(
name, recipe.depends,
', conflicts {}'.format(recipe.conflicts) if recipe.conflicts
else ''))
for depend in recipe.depends:
graph.add(name, depend)
recipes_to_load += recipe.depends
for conflict in recipe.conflicts:
if graph.conflicts(conflict):
warning(
('{} conflicts with {}, but both have been '
'included or pulled into the requirements.'
.format(recipe.name, conflict)))
warning('Due to this conflict the build cannot continue, '
'exiting.')
exit(1)
recipe_loaded.append(name)
graph.remove_remaining_conflicts(ctx)
build_order = list(graph.find_order(0))
return build_order, python_modules, bs
def prebuild_arch(self, arch):
super(ProtobufCppRecipe, self).prebuild_arch(arch)
patch_mark = join(self.get_build_dir(arch.arch), '.protobuf-patched')
if self.ctx.python_recipe.name == 'python3' and not exists(patch_mark):
self.apply_patch('fix-python3-compatibility.patch', arch.arch)
shprint(sh.touch, patch_mark)
# During building, host needs to transpile .proto files to .py
# ideally with the same version as protobuf runtime, or with an older one.
# Because protoc is compiled for target (i.e. Android), we need an other binary
# which can be run by host.
# To make it easier, we download prebuild protoc binary adapted to the platform
info_notify("Downloading protoc compiler for your platform")
url_prefix = "https://github.com/protocolbuffers/protobuf/releases/download/v{version}".format(version=self.version)
if sys.platform.startswith('linux'):
info_notify("GNU/Linux detected")
filename = "protoc-{version}-linux-x86_64.zip".format(version=self.version)
elif sys.platform.startswith('darwin'):
info_notify("Mac OS X detected")
filename = "protoc-{version}-osx-x86_64.zip".format(version=self.version)
else:
info_notify("Your platform is not supported, but recipe can still "
"be built if you have a valid protoc (<={version}) in "
"your path".format(version=self.version))
return
protoc_url = join(url_prefix, filename)
self.protoc_dir = join(self.ctx.build_dir, "tools", "protoc")
if os.path.exists(join(self.protoc_dir, "bin", "protoc")):
info_notify("protoc found, no download needed")
return
try:
os.makedirs(self.protoc_dir)
except OSError as e:
# if dir already exists (errno 17), we ignore the error
if e.errno != 17:
raise e
info_notify("Will download into {dest_dir}".format(dest_dir=self.protoc_dir))
self.download_file(protoc_url, join(self.protoc_dir, filename))
with current_directory(self.protoc_dir):
shprint(sh.unzip, join(self.protoc_dir, filename))
def prebuild_arch(self, arch):
super(ProtobufCppRecipe, self).prebuild_arch(arch)
# During building, host needs to transpile .proto files to .py
# ideally with the same version as protobuf runtime, or with an older one.
# Because protoc is compiled for target (i.e. Android), we need an other binary
# which can be run by host.
# To make it easier, we download prebuild protoc binary adapted to the platform
info_notify("Downloading protoc compiler for your platform")
url_prefix = "https://github.com/protocolbuffers/protobuf/releases/download/v{version}".format(version=self.version)
if sys.platform.startswith('linux'):
info_notify("GNU/Linux detected")
filename = "protoc-{version}-linux-x86_64.zip".format(version=self.version)
elif sys.platform.startswith('darwin'):
info_notify("Mac OS X detected")
filename = "protoc-{version}-osx-x86_64.zip".format(version=self.version)
else:
info_notify("Your platform is not supported, but recipe can still "
"be built if you have a valid protoc (<={version}) in "
"your path".format(version=self.version))
return
protoc_url = join(url_prefix, filename)
self.protoc_dir = join(self.ctx.build_dir, "tools", "protoc")
if os.path.exists(join(self.protoc_dir, "bin", "protoc")):
info_notify("protoc found, no download needed")
return
try:
os.makedirs(self.protoc_dir)
except OSError as e:
# if dir already exists (errno 17), we ignore the error
if e.errno != 17:
raise e
info_notify("Will download into {dest_dir}".format(dest_dir=self.protoc_dir))
self.download_file(protoc_url, join(self.protoc_dir, filename))
with current_directory(self.protoc_dir):
shprint(sh.unzip, join(self.protoc_dir, filename))
def get_recipe_order_and_bootstrap(ctx, names, bs=None):
'''Takes a list of recipe names and (optionally) a bootstrap. Then
works out the dependency graph (including bootstrap recipes if
necessary). Finally, if no bootstrap was initially selected,
chooses one that supports all the recipes.
'''
graph = Graph()
recipes_to_load = set(names)
if bs is not None and bs.recipe_depends:
info_notify('Bootstrap requires recipes {}'.format(bs.recipe_depends))
recipes_to_load = recipes_to_load.union(set(bs.recipe_depends))
recipes_to_load = list(recipes_to_load)
recipe_loaded = []
python_modules = []
while recipes_to_load:
name = recipes_to_load.pop(0)
if name in recipe_loaded or isinstance(name, (list, tuple)):
continue
try:
recipe = Recipe.get_recipe(name, ctx)
except IOError:
info('No recipe named {}; will attempt to install with pip'.format(
name))
python_modules.append(name)
continue
except (KeyboardInterrupt, SystemExit):
raise
except:
warning('Failed to import recipe named {}; the recipe exists '
'but appears broken.'.format(name))
warning('Exception was:')
raise
graph.add(name, name)
info('Loaded recipe {} (depends on {}{})'.format(
name, recipe.depends, ', conflicts {}'.format(recipe.conflicts)
if recipe.conflicts else ''))
for depend in recipe.depends:
graph.add(name, depend)
recipes_to_load += recipe.depends
for conflict in recipe.conflicts:
if graph.conflicts(conflict):
warning(('{} conflicts with {}, but both have been '
'included or pulled into the requirements.'.format(
recipe.name, conflict)))
warning(
'Due to this conflict the build cannot continue, exiting.')
exit(1)
python_modules += recipe.python_depends
recipe_loaded.append(name)
graph.remove_remaining_conflicts(ctx)
if len(graph.graphs) > 1:
info('Found multiple valid recipe sets:')
for g in graph.graphs:
info(' {}'.format(g.keys()))
info_notify('Using the first of these: {}'.format(
graph.graphs[0].keys()))
elif len(graph.graphs) == 0:
warning('Didn\'t find any valid dependency graphs, exiting.')
exit(1)
else:
info('Found a single valid recipe set (this is good)')
build_order = list(graph.find_order(0))
if bs is None: # It would be better to check against possible
# orders other than the first one, but in practice
# there will rarely be clashes, and the user can
# specify more parameters if necessary to resolve
# them.
bs = Bootstrap.get_bootstrap_from_recipes(build_order, ctx)
if bs is None:
info('Could not find a bootstrap compatible with the '
'required recipes.')
info('If you think such a combination should exist, try '
'specifying the bootstrap manually with --bootstrap.')
exit(1)
info('{} bootstrap appears compatible with the required recipes.'.
format(bs.name))
info('Checking this...')
recipes_to_load = bs.recipe_depends
# This code repeats the code from earlier! Should move to a function:
while recipes_to_load:
name = recipes_to_load.pop(0)
if name in recipe_loaded or isinstance(name, (list, tuple)):
continue
try:
recipe = Recipe.get_recipe(name, ctx)
except ImportError:
info('No recipe named {}; will attempt to install with pip'.
format(name))
python_modules.append(name)
continue
graph.add(name, name)
info('Loaded recipe {} (depends on {}{})'.format(
name, recipe.depends, ', conflicts {}'.format(recipe.conflicts)
if recipe.conflicts else ''))
for depend in recipe.depends:
graph.add(name, depend)
recipes_to_load += recipe.depends
for conflict in recipe.conflicts:
if graph.conflicts(conflict):
warning(
('{} conflicts with {}, but both have been '
'included or pulled into the requirements.'.format(
recipe.name, conflict)))
warning('Due to this conflict the build cannot continue, '
'exiting.')
exit(1)
recipe_loaded.append(name)
graph.remove_remaining_conflicts(ctx)
build_order = list(graph.find_order(0))
return build_order, python_modules, bs
