def fetch(self, cookbook, recipes, no_deps, reset_rdeps):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates (fetch_recipes)
m.message(_("Fetching the following recipes: %s") %
' '.join([x.name for x in fetch_recipes]))
to_rebuild = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
recipe.fetch()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
m.message(_("These recipes have been updated and will "
"be rebuilt:\n%s") %
'\n'.join([x.name for x in to_rebuild]))
def recipes_dependencies(self, use_devel=True):
deps = []
for dep in self.deps:
package = self.store.get_package(dep)
deps.extend(package.recipes_dependencies(use_devel))
if self.app_recipe is not None:
deps.append(self.app_recipe)
return remove_list_duplicates(deps)
def recipes_dependencies(self, use_devel=True):
deps = []
for dep in self.deps:
package = self.store.get_package(dep)
deps.extend(package.recipes_dependencies(use_devel))
if self.app_recipe is not None:
deps.append(self.app_recipe)
return remove_list_duplicates(deps)
def recipes_dependencies(self, use_devel=True):
deps = [x.split(':')[0] for x in self._files]
if use_devel:
deps.extend([x.split(':')[0] for x in self._files_devel])
for name in self.deps:
p = self.store.get_package(name)
deps += p.recipes_dependencies(use_devel)
return remove_list_duplicates(deps)
def recipes_dependencies(self, use_devel=True):
deps = [x.split(':')[0] for x in self._files]
if use_devel:
deps.extend([x.split(':')[0] for x in self._files_devel])
for name in self.deps:
p = self.store.get_package(name)
deps += p.recipes_dependencies(use_devel)
return remove_list_duplicates(deps)
def fetch(cookbook, recipes, no_deps, reset_rdeps, full_reset, print_only,
jobs):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates(fetch_recipes)
m.message(
_("Fetching the following recipes using %s async job(s): %s") %
(jobs, ' '.join([x.name for x in fetch_recipes])))
shell.set_max_non_cpu_bound_calls(jobs)
to_rebuild = []
tasks = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
if print_only:
# For now just print tarball URLs
if isinstance(recipe, Tarball):
m.message("TARBALL: {} {}".format(recipe.url,
recipe.tarball_name))
continue
stepfunc = getattr(recipe, 'fetch')
if asyncio.iscoroutinefunction(stepfunc):
tasks.append(stepfunc())
else:
m.build_step(i + 1, len(fetch_recipes), recipe,
'fetch started')
stepfunc()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
# On different versions, only reset recipe if:
# * forced
# * OR it was fully built already
if full_reset or not cookbook.recipe_needs_build(recipe.name):
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if recipe.library_type == LibraryType.STATIC or reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(
recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
run_until_complete(tasks)
m.message("All async fetch jobs finished")
if to_rebuild:
to_rebuild = sorted(list(set(to_rebuild)), key=lambda r: r.name)
m.message(
_("These recipes have been updated and will "
"be rebuilt:\n%s") % '\n'.join([x.name for x in to_rebuild]))
def fetch(cookbook, recipes, no_deps, reset_rdeps, full_reset, print_only):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates(fetch_recipes)
m.message(
_("Fetching the following recipes: %s") %
' '.join([x.name for x in fetch_recipes]))
to_rebuild = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
if print_only:
# For now just print tarball URLs
if isinstance(recipe, Tarball):
m.message("TARBALL: {} {}".format(recipe.url,
recipe.tarball_name))
continue
m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
stepfunc = getattr(recipe, 'fetch')
if asyncio.iscoroutinefunction(stepfunc):
loop = asyncio.get_event_loop()
loop.run_until_complete(stepfunc(recipe))
else:
stepfunc()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
# On different versions, only reset recipe if:
# * forced
# * OR it was fully built already
if full_reset or not cookbook.recipe_needs_build(recipe.name):
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(
recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
to_rebuild = sorted(list(set(to_rebuild)), key=lambda r: r.name)
m.message(
_("These recipes have been updated and will "
"be rebuilt:\n%s") % '\n'.join([x.name for x in to_rebuild]))
def _list_metapackage_deps(self, metapackage):
def get_package_deps(package_name, visited=[], depslist=[]):
if package_name in visited:
return
visited.append(package_name)
p = self.get_package(package_name)
depslist.append(p)
for p_name in p.deps:
get_package_deps(p_name, visited, depslist)
return depslist
deps = []
for p in metapackage.list_packages():
deps.extend(get_package_deps(p, [], []))
return remove_list_duplicates(deps)
def _list_metapackage_deps(self, metapackage):
def get_package_deps(package_name, visited=[], depslist=[]):
if package_name in visited:
return
visited.append(package_name)
p = self.get_package(package_name)
depslist.append(p)
for p_name in p.deps:
get_package_deps(p_name, visited, depslist)
return depslist
deps = []
for p in metapackage.list_packages():
deps.extend(get_package_deps(p, [], []))
return remove_list_duplicates(deps)
def get_package_deps(self, pkg, recursive=False):
'''
Gets the dependencies of a package
@param package: name of the package or package instance
@type package: L{cerbero.packages.package.Package}
@return: a list with the package dependencies
@rtype: list
'''
if isinstance(pkg, str):
pkg = self.get_package(pkg)
if isinstance(pkg, package.MetaPackage):
ret = self._list_metapackage_deps(pkg)
else:
ret = [self.get_package(x) for x in pkg.deps]
# get deps recursively
if recursive:
for p in ret:
ret.extend(self.get_package_deps(p, recursive))
return remove_list_duplicates(ret)
def fetch(self, cookbook, recipes, no_deps, reset_rdeps, full_reset, print_only):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates (fetch_recipes)
m.message(_("Fetching the following recipes: %s") %
' '.join([x.name for x in fetch_recipes]))
to_rebuild = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
if print_only:
# For now just print tarball URLs
if isinstance(recipe, Tarball):
m.message("TARBALL: {} {}".format(recipe.url, recipe.tarball_name))
continue
m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
recipe.fetch()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
# On different versions, only reset recipe if:
# * forced
# * OR it was fully built already
if full_reset or not cookbook.recipe_needs_build(recipe.name):
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
to_rebuild = sorted(list(set(to_rebuild)), key=lambda r:r.name)
m.message(_("These recipes have been updated and will "
"be rebuilt:\n%s") %
'\n'.join([x.name for x in to_rebuild]))
def fetch(self, cookbook, recipes, no_deps, reset_rdeps, full_reset):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates(fetch_recipes)
m.message(
_("Fetching the following recipes: %s") %
' '.join([x.name for x in fetch_recipes]))
to_rebuild = []
for i in range(len(fetch_recipes)):
recipe = fetch_recipes[i]
m.build_step(i + 1, len(fetch_recipes), recipe, 'Fetch')
recipe.fetch()
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
# On different versions, only reset recipe if:
# * forced
# * OR it was fully built already
if full_reset or not cookbook.recipe_needs_build(recipe.name):
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(
recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
to_rebuild = sorted(list(set(to_rebuild)), key=lambda r: r.name)
m.message(
_("These recipes have been updated and will "
"be rebuilt:\n%s") % '\n'.join([x.name for x in to_rebuild]))
def recipes_dependencies(self, use_devel=True):
deps = []
for package in self.store.get_package_deps(self.name, True):
deps.extend(package.recipes_dependencies(use_devel))
return remove_list_duplicates(deps)
def all_files_list(self):
files = self.files_list()
files.extend(self.devel_files_list())
return remove_list_duplicates(files)
def run(self, config, args):
packages = []
recipes = []
bundle_recipes = []
bundle_dirs = []
setup_args = ['sdist']
if not config.uninstalled:
m.error("Can only be run on cerbero-uninstalled")
store = PackagesStore(config)
cookbook = CookBook(config)
packages = list(args.bundlepackages)
for p in packages:
package = store.get_package(p)
if hasattr(package, 'list_packages'):
packages += package.list_packages()
packages = remove_list_duplicates(packages)
for p in packages:
package = store.get_package(p)
if hasattr(package, 'deps'):
packages += package.deps
packages = remove_list_duplicates(packages)
for p in packages:
package = store.get_package(p)
recipes += package.recipes_dependencies()
recipes += args.add_recipe
for r in recipes:
bundle_recipes += cookbook.list_recipe_deps(r)
bundle_recipes = remove_list_duplicates(bundle_recipes)
for p in packages:
setup_args.append('--package=' + p)
for r in bundle_recipes:
setup_args.append('--recipe=' + r.name)
if r.stype != SourceType.CUSTOM:
bundle_dirs.append(r.repo_dir)
if not args.no_bootstrap:
build_tools = BuildTools(config)
bs_recipes = build_tools.BUILD_TOOLS + \
build_tools.PLAT_BUILD_TOOLS.get(config.platform, [])
b_recipes = []
for r in bs_recipes:
b_recipes += cookbook.list_recipe_deps(r)
b_recipes = remove_list_duplicates(b_recipes)
for r in b_recipes:
if r.stype != SourceType.CUSTOM:
bundle_dirs.append(r.repo_dir)
setup_args.append('--source-dirs=' + ','.join(bundle_dirs))
command = str(config._relative_path('setup.py'))
run_setup(command, setup_args)
def all_files_list(self):
files = self.files_list()
files.extend(self.devel_files_list())
return remove_list_duplicates(files)
def run(self, config, args):
packages = []
recipes = []
bundle_recipes = []
bundle_dirs = []
setup_args = ['sdist']
if not config.uninstalled:
m.error("Can only be run on cerbero-uninstalled")
store = PackagesStore(config)
cookbook = CookBook(config)
packages = list(args.bundlepackages)
for p in packages:
package = store.get_package(p)
if hasattr(package, 'list_packages'):
packages += package.list_packages()
packages = remove_list_duplicates(packages)
for p in packages:
package = store.get_package(p)
if hasattr(package, 'deps'):
packages += package.deps
packages = remove_list_duplicates(packages)
for p in packages:
package = store.get_package(p)
recipes += package.recipes_dependencies()
recipes += args.add_recipe
for r in recipes:
bundle_recipes += cookbook.list_recipe_deps(r)
bundle_recipes = remove_list_duplicates(bundle_recipes)
for p in packages:
setup_args.append('--package=' + p)
for r in bundle_recipes:
setup_args.append('--recipe=' + r.name)
if r.stype != SourceType.CUSTOM:
bundle_dirs.append(r.repo_dir)
if not args.no_bootstrap:
build_tools = BuildTools(config)
bs_recipes = build_tools.BUILD_TOOLS + \
build_tools.PLAT_BUILD_TOOLS.get(config.platform, [])
b_recipes = []
for r in bs_recipes:
b_recipes += cookbook.list_recipe_deps(r)
b_recipes = remove_list_duplicates(b_recipes)
for r in b_recipes:
if r.stype != SourceType.CUSTOM:
bundle_dirs.append(r.repo_dir)
setup_args.append('--source-dirs=' + ','.join(bundle_dirs))
command = str(config._relative_path('setup.py'))
run_setup(command, setup_args)
def recipes_dependencies(self):
deps = []
for package in self.store.get_package_deps(self.name, True):
deps.extend(package.recipes_dependencies())
return remove_list_duplicates(deps)
def fetch(cookbook, recipes, no_deps, reset_rdeps, full_reset, print_only,
jobs):
fetch_recipes = []
if not recipes:
fetch_recipes = cookbook.get_recipes_list()
elif no_deps:
fetch_recipes = [cookbook.get_recipe(x) for x in recipes]
else:
for recipe in recipes:
fetch_recipes += cookbook.list_recipe_deps(recipe)
fetch_recipes = remove_list_duplicates(fetch_recipes)
m.message(
_("Fetching the following recipes using %s async job(s): %s") %
(jobs, ' '.join([x.name for x in fetch_recipes])))
shell.set_max_non_cpu_bound_calls(jobs)
to_rebuild = []
tasks = []
printer = BuildStatusPrinter(('fetch', ),
cookbook.get_config().interactive)
printer.total = len(fetch_recipes)
async def fetch_print_wrapper(recipe_name, stepfunc):
printer.update_recipe_step(printer.count, printer.total,
recipe_name, 'fetch')
await stepfunc()
printer.count += 1
printer.remove_recipe(recipe_name)
for recipe in fetch_recipes:
if print_only:
# For now just print tarball URLs
if isinstance(recipe, Tarball):
m.message("TARBALL: {} {}".format(recipe.url,
recipe.tarball_name))
continue
stepfunc = getattr(recipe, 'fetch')
if asyncio.iscoroutinefunction(stepfunc):
tasks.append(fetch_print_wrapper(recipe.name, stepfunc))
else:
printer.update_recipe_step(printer.count, printer.total,
recipe.name, 'fetch')
stepfunc()
printer.count += 1
printer.remove_recipe(recipe.name)
run_until_complete(tasks)
m.message("All async fetch jobs finished")
# Checking the current built version against the fetched one
# needs to be done *after* actually fetching
for recipe in fetch_recipes:
bv = cookbook.recipe_built_version(recipe.name)
cv = recipe.built_version()
if bv != cv:
# On different versions, only reset recipe if:
# * forced
# * OR it was fully built already
if full_reset or not cookbook.recipe_needs_build(recipe.name):
to_rebuild.append(recipe)
cookbook.reset_recipe_status(recipe.name)
if recipe.library_type == LibraryType.STATIC or reset_rdeps:
for r in cookbook.list_recipe_reverse_deps(
recipe.name):
to_rebuild.append(r)
cookbook.reset_recipe_status(r.name)
if to_rebuild:
to_rebuild = sorted(list(set(to_rebuild)), key=lambda r: r.name)
m.message(
_("These recipes have been updated and will "
"be rebuilt:\n%s") % '\n'.join([x.name for x in to_rebuild]))
