def _gen_recipe_for_package(distro, pkg_name, pkg, repo, ros_pkg,
pkg_rosinstall, tar_dir, md5_cache, sha256_cache,
skip_keys):
pkg_names = get_package_names(distro)
pkg_dep_walker = DependencyWalker(distro,
evaluate_condition_context=os.environ)
pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool")
pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build")
pkg_build_export_deps = pkg_dep_walker.get_depends(pkg_name,
"build_export")
pkg_buildtool_export_deps = pkg_dep_walker.get_depends(
pkg_name, "buildtool_export")
pkg_exec_deps = pkg_dep_walker.get_depends(pkg_name, "exec")
pkg_test_deps = pkg_dep_walker.get_depends(pkg_name, "test")
src_uri = pkg_rosinstall[0]['tar']['uri']
# parse through package xml
err_msg = 'Failed to fetch metadata for package {}'.format(pkg_name)
pkg_xml = retry_on_exception(ros_pkg.get_package_xml,
distro.name,
retry_msg='Could not get package xml!',
error_msg=err_msg)
pkg_recipe = yoctoRecipe(
pkg.repository_name,
len(ros_pkg.repository.package_names),
pkg_name,
pkg_xml,
distro,
src_uri,
tar_dir,
md5_cache,
sha256_cache,
skip_keys,
)
# add build dependencies
for bdep in pkg_build_deps:
pkg_recipe.add_build_depend(bdep, bdep in pkg_names[0])
# add build tool dependencies
for btdep in pkg_buildtool_deps:
pkg_recipe.add_buildtool_depend(btdep, btdep in pkg_names[0])
# add export dependencies
for edep in pkg_build_export_deps:
pkg_recipe.add_export_depend(edep, edep in pkg_names[0])
# add buildtool export dependencies
for btedep in pkg_buildtool_export_deps:
pkg_recipe.add_buildtool_export_depend(btedep, btedep in pkg_names[0])
# add exec dependencies
for xdep in pkg_exec_deps:
pkg_recipe.add_run_depend(xdep, xdep in pkg_names[0])
# add test dependencies
for tdep in pkg_test_deps:
pkg_recipe.add_test_depend(tdep, tdep in pkg_names[0])
return pkg_recipe
def _gen_ebuild_for_package(distro, pkg_name, pkg, repo, ros_pkg,
pkg_rosinstall):
pkg_ebuild = Ebuild()
pkg_ebuild.distro = distro.name
pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri']
pkg_names = get_package_names(distro)
pkg_dep_walker = DependencyWalker(distro)
pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool")
pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build")
pkg_run_deps = pkg_dep_walker.get_depends(pkg_name, "run")
pkg_keywords = ['x86', 'amd64', 'arm', 'arm64']
# add run dependencies
for rdep in pkg_run_deps:
pkg_ebuild.add_run_depend(rdep, rdep in pkg_names[0])
# add build dependencies
for bdep in pkg_build_deps:
pkg_ebuild.add_build_depend(bdep, bdep in pkg_names[0])
# add build tool dependencies
for tdep in pkg_buildtool_deps:
pkg_ebuild.add_build_depend(tdep, tdep in pkg_names[0])
# add keywords
for key in pkg_keywords:
pkg_ebuild.add_keyword(key)
# parse throught package xml
try:
pkg_xml = ros_pkg.get_package_xml(distro.name)
except Exception as e:
warn("fetch metadata for package {}".format(pkg_name))
return pkg_ebuild
pkg_fields = xmltodict.parse(pkg_xml)
pkg_ebuild.upstream_license = pkg_fields['package']['license']
pkg_ebuild.description = pkg_fields['package']['description']
if isinstance(pkg_ebuild.description, str):
pkg_ebuild.description = pkg_ebuild.description.replace('`', "")
if len(pkg_ebuild.description) > 80:
pkg_ebuild.description = pkg_ebuild.description[:80]
try:
if 'url' not in pkg_fields['package']:
warn("no website field for package {}".format(pkg_name))
elif isinstance(pkg_fields['package']['url'], str):
pkg_ebuild.homepage = pkg_fields['package']['url']
elif '@type' in pkg_fields['package']['url']:
if pkg_fields['package']['url']['@type'] == 'website':
if '#text' in pkg_fields['package']['url']:
pkg_ebuild.homepage = pkg_fields['package']['url']['#text']
else:
warn("failed to parse website for package {}".format(pkg_name))
except TypeError as e:
warn("failed to parse website package {}: {}".format(pkg_name, e))
return pkg_ebuild
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
ebuild_name =\
'/ros-{0}/{1}/{1}-{2}.ebuild'.format(distro.name, pkg, version)
ebuild_name = overlay.repo.repo_dir + ebuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing ebuild.
existing = glob.glob('{0}/ros-{1}/{2}/*.ebuild'.format(
overlay.repo.repo_dir, distro.name, pkg))
if preserve_existing and os.path.isfile(ebuild_name):
ok("ebuild for package '%s' up to date, skipping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg)
overlay.repo.remove_file(manifest_file)
try:
current = gentoo_installer(distro, pkg, has_patches)
current.ebuild.name = pkg
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
ebuild_text = current.ebuild_text()
metadata_text = current.metadata_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.ebuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.ebuild.get_unresolved()
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir("{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
try:
ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
overlay.repo.repo_dir, distro.name, pkg, version)
ebuild_file = open(ebuild_file, "w")
metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format(
overlay.repo.repo_dir, distro.name, pkg)
metadata_file = open(metadata_file, "w")
ebuild_file.write(ebuild_text)
metadata_file.write(metadata_text)
except Exception as e:
err("Failed to write ebuild/metadata to disk!")
raise e
return current, []
def regenerate_installer(overlay, pkg, distro, preserve_existing, tar_dir):
make_dir("{0}/recipes-ros-{1}".format(overlay.repo.repo_dir, distro.name))
version = get_pkg_version(distro, pkg)
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % pkg)
# check for an existing recipe
existing = glob.glob('{0}/recipes-ros-{1}/{2}/*.bb'.format(
overlay.repo.repo_dir, distro.name, pkg))
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skpping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
try:
current = oe_installer(distro, pkg, tar_dir)
current.recipe.name = pkg.replace('_', '-')
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
recipe_text = current.recipe_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.recipe.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.recipe.get_unresolved()
except NoPkgXml:
err("Could not fetch pkg!")
return None, []
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir("{0}/recipes-ros-{1}/{2}".format(overlay.repo.repo_dir,
distro.name,
pkg.replace('_', '-')))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_name = '{0}/recipes-ros-{1}/{2}/{2}_{3}.bb'.format(
overlay.repo.repo_dir, distro.name, pkg.replace('_', '-'), version)
try:
with open('{0}'.format(recipe_name), "w") as recipe_file:
recipe_file.write(recipe_text)
except Exception as e:
err("Failed to write recipe to disk!")
raise e
return current, []
def _expand_keywords(distro_name, keywords):
names = set([])
if ARG_ALL_PACKAGES in keywords:
wet_distro = get_wet_distro(distro_name)
released_package_names, _ = get_package_names(wet_distro)
names.update(released_package_names)
if distro_name == 'groovy':
dry_distro = get_dry_distro(distro_name)
released_stack_names, _ = get_stack_names(dry_distro)
names.update(released_stack_names)
if ARG_CURRENT_ENVIRONMENT in keywords:
names.update(_get_packages_in_environment())
return names
def _expand_keywords(distro_name, keywords):
names = set([])
if ARG_ALL_PACKAGES in keywords:
wet_distro = get_wet_distro(distro_name)
released_package_names, _ = get_package_names(wet_distro)
names.update(released_package_names)
if distro_name == 'groovy':
dry_distro = get_dry_distro(distro_name)
released_stack_names, _ = get_stack_names(dry_distro)
names.update(released_stack_names)
if ARG_CURRENT_ENVIRONMENT in keywords:
names.update(_get_packages_in_environment())
return names
def _gen_ebuild_for_package(
distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall
):
pkg_ebuild = Ebuild()
pkg_ebuild.distro = distro.name
pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri']
pkg_names = get_package_names(distro)
pkg_dep_walker = DependencyWalker(distro)
pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool")
pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build")
pkg_run_deps = pkg_dep_walker.get_depends(pkg_name, "run")
pkg_test_deps = pkg_dep_walker.get_depends(pkg_name, "test")
pkg_keywords = ['x86', 'amd64', 'arm', 'arm64']
# add run dependencies
for rdep in pkg_run_deps:
pkg_ebuild.add_run_depend(rdep, rdep in pkg_names[0])
# add build dependencies
for bdep in pkg_build_deps:
pkg_ebuild.add_build_depend(bdep, bdep in pkg_names[0])
# add build tool dependencies
for tdep in pkg_buildtool_deps:
pkg_ebuild.add_build_depend(tdep, tdep in pkg_names[0])
# add test dependencies
for test_dep in pkg_test_deps:
pkg_ebuild.add_test_depend(test_dep, test_dep in pkg_names[0])
# add keywords
for key in pkg_keywords:
pkg_ebuild.add_keyword(key)
# parse throught package xml
try:
pkg_xml = ros_pkg.get_package_xml(distro.name)
except Exception:
warn("fetch metadata for package {}".format(pkg_name))
return pkg_ebuild
pkg = PackageMetadata(pkg_xml)
pkg_ebuild.upstream_license = pkg.upstream_license
pkg_ebuild.description = pkg.description
pkg_ebuild.homepage = pkg.homepage
pkg_ebuild.build_type = pkg.build_type
return pkg_ebuild
def generate_installers(
distro, # ros distro
overlay, # repo instance
gen_pkg_func, # function to call for generating
preserve_existing=True, # don't regenerate if installer exists
*args, # any additional args for gen_pkg_func
**kwargs # any additional keyword arguments
):
distro_name = distro.name
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
bad_installers = []
succeeded = 0
failed = 0
info("Generating installers for distro '%s'" % distro_name)
for i, pkg in enumerate(sorted(pkg_names[0])):
if 'skip_keys' in kwargs and pkg in kwargs['skip_keys']:
warn("Package '%s' is in skip-keys list, skipping..." % pkg)
continue
version = get_pkg_version(distro, pkg)
percent = '%.1f' % (100 * (float(i) / total))
try:
current, current_info = gen_pkg_func(overlay, pkg, distro,
preserve_existing, *args)
if not current and current_info:
# we are missing dependencies
failed_msg = "{0}%: Failed to generate".format(percent)
failed_msg += " installer for package '%s'!" % pkg
err(failed_msg)
borkd_pkgs[pkg] = current_info
failed = failed + 1
continue
elif not current and preserve_existing:
# don't replace the installer
succeeded = succeeded + 1
continue
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1
if current_info:
changes.append('*{0} {1} --> {2}*'.format(
pkg, current_info, version))
else:
changes.append('*{0} {1}*'.format(pkg, version))
installers.append(pkg)
except UnknownBuildType as ub:
err("{0}%: Unknown Build type '{1}' for package '{2}'".format(
percent, str(ub), pkg))
failed = failed + 1
except KeyError:
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(pkg)
failed = failed + 1
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
info("------ {0} ------\n".format(results))
if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))
return installers, borkd_pkgs, changes
def analyse(self, root_pkg):
released_names, unreleased_names = get_package_names(self._distro)
pprint(
unreleased_names
#get_recursive_dependencies(self._distro, ['strands_apps'], source=True)
)
def generate_rosinstall(distro_name, names,
deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
excludes=None,
flat=False,
tar=False):
# classify package/stack names
names, keywords = _split_special_keywords(names)
names, unknown_names = _classify_names(distro_name, names)
if unknown_names:
logger.warn('The following not released packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn('The following not released packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
names.update(expanded_names)
if not names.wet_package_names and not names.dry_stack_names:
raise RuntimeError('No packages/stacks left after ignoring not released')
logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
# classify deps-up-to
deps_up_to_names, keywords = _split_special_keywords(deps_up_to or [])
deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names)
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
deps_up_to_names.update(expanded_names)
if deps_up_to:
logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))
# classify excludes
exclude_names, keywords = _split_special_keywords(excludes or [])
exclude_names, unknown_names = _classify_names(distro_name, exclude_names)
if unknown_names:
logger.warn("The following not released '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
exclude_names.update(expanded_names)
if excludes:
logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))
result = copy.deepcopy(names)
# clear wet packages if not requested
if dry_only:
result.wet_package_names.clear()
# clear dry packages if not requested and no dependencies
if wet_only and not deps and not deps_up_to:
result.dry_stack_names.clear()
# remove excluded names from the list of wet and dry names
result.wet_package_names -= exclude_names.wet_package_names
result.dry_stack_names -= exclude_names.dry_stack_names
if not result.wet_package_names and not result.dry_stack_names:
raise RuntimeError('No packages/stacks left after applying the exclusions')
if result.wet_package_names:
logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
if result.dry_stack_names:
logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
# extend the names with recursive dependencies
if deps or deps_up_to:
# add dry dependencies
if result.dry_stack_names:
dry_distro = get_dry_distro(distro_name)
_, unreleased_stack_names = get_stack_names(dry_distro)
excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
result.dry_stack_names |= dry_dependencies
if not dry_only:
# add wet dependencies of dry stuff
logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
for depend in wet_dependencies:
if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
continue
wet_distro = get_wet_distro(distro_name)
assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
result.wet_package_names.add(depend)
# add wet dependencies
if result.wet_package_names:
wet_distro = get_wet_distro(distro_name)
_, unreleased_package_names = get_package_names(wet_distro)
excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes, limit_depth=deps_depth)
logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect result with recursive dependencies on
if deps_up_to:
# intersect with wet dependencies on
if deps_up_to_names.wet_package_names:
wet_distro = get_wet_distro(distro_name)
# wet depends on do not include the names since they are excluded to stop recursion asap
wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names, limit=result.wet_package_names)
# keep all names which are already in the result set
wet_package_names |= result.wet_package_names & names.wet_package_names
result.wet_package_names = wet_package_names
else:
result.wet_package_names.clear()
logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect with dry dependencies on
dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
if dry_dependency_names and not wet_only:
dry_distro = get_dry_distro(distro_name)
# dry depends on do not include the names since they are excluded to stop recursion asap
dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
# keep all names which are already in the result set
dry_stack_names |= result.dry_stack_names & names.dry_stack_names
result.dry_stack_names = dry_stack_names
else:
result.dry_stack_names.clear()
logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))
# exclude passed in names
if deps_only:
result.wet_package_names -= set(names.wet_package_names)
result.dry_stack_names -= set(names.dry_stack_names)
# exclude wet packages based on build type
if catkin_only or non_catkin_only:
wet_distro = get_wet_distro(distro_name)
for pkg_name in list(result.wet_package_names):
pkg_xml = wet_distro.get_release_package_xml(pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
result.wet_package_names.remove(pkg_name)
continue
build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
if catkin_only ^ (build_type == 'catkin'):
result.wet_package_names.remove(pkg_name)
# get wet and/or dry rosinstall data
rosinstall_data = []
if not dry_only and result.wet_package_names:
logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
wet_distro = get_wet_distro(distro_name)
wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
rosinstall_data += wet_rosinstall_data
if not wet_only and result.dry_stack_names:
logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
dry_distro = get_dry_distro(distro_name)
dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
rosinstall_data += dry_rosinstall_data
return rosinstall_data
def test_get_package_names():
d = _get_test_dist()
assert set(get_package_names(d)[0]) == set(['archie', 'betty', 'veronica'])
d.repositories['riverdale'].release_repository.version = None
assert set(get_package_names(d)[1]) == set(['archie', 'betty', 'veronica'])
def main():
overlay = None
parser = get_parser('Generate OpenEmbedded recipes for ROS packages',
exclude_all=True,
require_rosdistro=True,
require_dryrun=True)
parser.add_argument('--tar-archive-dir',
help='location to store archived packages',
type=str)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = set(args.skip_keys) if args.skip_keys else set()
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
warn('"{0}" distro detected...'.format(args.ros_distro))
"""
No longer supporting generation for multiple targets, but left the code in
place to handle them in case it might be needed again in the future.
"""
selected_targets = [args.ros_distro]
preserve_existing = args.only
now = os.getenv('SUPERFLORE_GENERATION_DATETIME',
get_utcnow_timestamp_str())
repo_org = 'ros'
repo_name = 'meta-ros'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
branch=(('superflore/{}'.format(now))
if not args.no_branch else None),
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
)
if not args.only:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for all packages in ROS '
'distribution {}.\n'.format(selected_targets[0]))
else:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for package(s) {} in ROS '
'distribution {}.\n'.format(args.only, args.ros_distro))
# generate installers
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir
else:
srcrev_filename = None
with CacheManager(srcrev_filename) as srcrev_cache:
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
distro,
False, # preserve_existing
srcrev_cache,
skip_keys=skip_keys,
)
except KeyError:
err("No package to satisfy key '%s' available "
"packages in selected distro: %s" %
(pkg, get_package_names(distro)))
sys.exit(1)
# Commit changes and file pull request
title =\
'{{{0}}} Selected recipes generated from '\
'files/{0}/generated/cache.yaml '\
'as of {1}\n'.format(
args.ros_distro,
now)
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
delta = "Regenerated: '%s'\n" % args.only
overlay.add_generated_files(args.ros_distro)
commit_msg = '\n'.join([
get_pr_text(
title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
if args.dry_run:
save_pr(overlay, args.only, '', pr_comment, title=title)
sys.exit(0)
file_pr(overlay,
delta,
'',
pr_comment,
distro=args.ros_distro,
title=title)
ok('Successfully synchronized repositories!')
sys.exit(0)
overlay.clean_ros_recipe_dirs(args.ros_distro)
for adistro in selected_targets:
yoctoRecipe.reset()
distro = get_distro(adistro)
distro_installers, _, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_pkg,
preserve_existing,
srcrev_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
)
total_changes[adistro] = distro_changes
total_installers[adistro] = distro_installers
yoctoRecipe.generate_ros_distro_inc(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'meta-ros{0}-{1}/files/{1}/generated/cache.yaml'.
format(yoctoRecipe._get_ros_version(args.ros_distro),
args.ros_distro)), distro.release_platforms,
skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
overlay.add_generated_files(args.ros_distro)
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
summary = overlay.get_change_summary(args.ros_distro)
if len(summary) == 0:
info('Exiting...')
clean_up()
sys.exit(0)
else:
info('But there are some changes in other regenerated files:'
'%s' % summary)
# remove duplicates
delta = gen_delta_msg(total_changes, markup='')
# Commit changes and file pull request
title = '{{{0}}} Sync to files/{0}/generated/'\
'cache.yaml as of {1}\n'.format(
args.ros_distro,
now)
commit_msg = '\n'.join([
get_pr_text(title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
delta = gen_delta_msg(total_changes)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay,
delta,
'',
pr_comment,
title=title,
)
sys.exit(0)
file_pr(overlay, delta, '', comment=pr_comment, title=title)
clean_up()
ok('Successfully synchronized repositories!')
def regenerate_pkg(overlay, pkg, distro, preserve_existing, tar_dir, md5_cache,
sha256_cache, skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
glob_pattern = '{0}/generated-recipes-{1}/{2}/{3}*.bb'.format(
repo_dir, distro.name, component_name, recipe)
existing = glob.glob(glob_pattern)
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0], True)
try:
current = oe_installer(distro, pkg, tar_dir, md5_cache, sha256_cache,
skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
except Exception as e:
err('Failed generating installer for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
make_dir("{0}/generated-recipes-{1}/{2}".format(repo_dir, distro.name,
component_name))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/generated-recipes-{1}/{2}/{3}_{4}.bb'.format(
repo_dir, distro.name, component_name, recipe, version)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
return current, []
def generate_rosinstall(distro_name, names,
from_paths=None, repo_names=None,
deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
excludes=None, exclude_paths=None,
flat=False,
tar=False,
upstream_version_tag=False, upstream_source_version=False):
# classify package/stack names
names, pkg_keywords = _split_special_keywords(names)
# find packages recursively in include paths
if from_paths:
include_names_from_path = set([])
[include_names_from_path.update(_get_package_names(from_path)) for from_path in from_paths]
logger.debug("The following wet packages found in '--from-path' will be considered: %s" % ', '.join(sorted(include_names_from_path)))
names.update(include_names_from_path)
# Allow special keywords in repos
repo_names, repo_keywords = _split_special_keywords(repo_names or [])
if set(repo_keywords).difference(set([ARG_ALL_PACKAGES])):
raise RuntimeError('The only keyword supported by repos is %r' % (ARG_ALL_PACKAGES))
if ARG_ALL_PACKAGES in repo_keywords:
wet_distro = get_wet_distro(distro_name)
repo_names = wet_distro.repositories.keys()
# expand repository names into package names
repo_names, unknown_repo_names = _classify_repo_names(distro_name, repo_names)
if unknown_repo_names:
logger.warn('The following unknown repositories will be ignored: %s' % (', '.join(sorted(unknown_repo_names))))
wet_package_names, unreleased_repo_names = _get_packages_for_repos(distro_name, repo_names, source=upstream_source_version)
names.update(wet_package_names)
if unreleased_repo_names and not upstream_version_tag and not upstream_source_version:
logger.warn('The following unreleased repositories will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
if unreleased_repo_names and (deps or deps_up_to) and (upstream_version_tag or upstream_source_version):
logger.warn('The dependencies of the following unreleased repositories are unknown and will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
has_repos = ((repo_names - unreleased_repo_names) and (upstream_version_tag or upstream_source_version)) or (unreleased_repo_names and upstream_source_version)
names, unknown_names = _classify_names(distro_name, names, source=upstream_source_version)
if unknown_names:
logger.warn('The following unreleased packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
if pkg_keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, pkg_keywords), source=upstream_source_version)
if unknown_names:
logger.warn('The following unreleased packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
names.update(expanded_names)
if not names.wet_package_names and not names.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after ignoring unreleased')
if names.wet_package_names or names.dry_stack_names:
logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
if unreleased_repo_names:
logger.debug('Unreleased repositories: %s' % ', '.join(sorted(unreleased_repo_names)))
# classify deps-up-to
deps_up_to_names, deps_keywords = _split_special_keywords(deps_up_to or [])
deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names, source=upstream_source_version)
if unknown_names:
logger.warn("The following unreleased '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if deps_keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, deps_keywords), source=upstream_source_version)
if unknown_names:
logger.warn("The following unreleased '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
deps_up_to_names.update(expanded_names)
if deps_up_to:
logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))
# classify excludes
exclude_names, excludes_keywords = _split_special_keywords(excludes or [])
if exclude_paths:
exclude_names_from_path = set([])
[exclude_names_from_path.update(_get_package_names(exclude_path)) for exclude_path in exclude_paths]
logger.debug("The following wet packages found in '--exclude-path' will be excluded: %s" % ', '.join(sorted(exclude_names_from_path)))
exclude_names.update(exclude_names_from_path)
exclude_names, unknown_names = _classify_names(distro_name, exclude_names, source=upstream_source_version)
if unknown_names:
logger.warn("The following unreleased '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if excludes_keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, excludes_keywords), source=upstream_source_version)
exclude_names.update(expanded_names)
if excludes:
logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))
result = copy.deepcopy(names)
# clear wet packages if not requested
if dry_only:
result.wet_package_names.clear()
# clear dry packages if not requested and no dependencies
if wet_only and not deps and not deps_up_to:
result.dry_stack_names.clear()
# remove excluded names from the list of wet and dry names
result.wet_package_names -= exclude_names.wet_package_names
result.dry_stack_names -= exclude_names.dry_stack_names
if not result.wet_package_names and not result.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after applying the exclusions')
if result.wet_package_names:
logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
if result.dry_stack_names:
logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
# extend the names with recursive dependencies
if deps or deps_up_to:
# add dry dependencies
if result.dry_stack_names:
dry_distro = get_dry_distro(distro_name)
_, unreleased_stack_names = get_stack_names(dry_distro)
excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
result.dry_stack_names |= dry_dependencies
if not dry_only:
# add wet dependencies of dry stuff
logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
for depend in wet_dependencies:
if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
continue
wet_distro = get_wet_distro(distro_name)
assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
result.wet_package_names.add(depend)
# add wet dependencies
if result.wet_package_names:
wet_distro = get_wet_distro(distro_name)
_, unreleased_package_names = get_package_names(wet_distro)
excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes,
limit_depth=deps_depth, source=upstream_source_version)
logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect result with recursive dependencies on
if deps_up_to:
# intersect with wet dependencies on
if deps_up_to_names.wet_package_names:
wet_distro = get_wet_distro(distro_name)
# wet depends on do not include the names since they are excluded to stop recursion asap
wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names,
limit=result.wet_package_names, source=upstream_source_version)
# keep all names which are already in the result set
wet_package_names |= result.wet_package_names & names.wet_package_names
result.wet_package_names = wet_package_names
else:
result.wet_package_names.clear()
logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect with dry dependencies on
dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
if dry_dependency_names and not wet_only:
dry_distro = get_dry_distro(distro_name)
# dry depends on do not include the names since they are excluded to stop recursion asap
dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
# keep all names which are already in the result set
dry_stack_names |= result.dry_stack_names & names.dry_stack_names
result.dry_stack_names = dry_stack_names
else:
result.dry_stack_names.clear()
logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))
# exclude passed in names
if deps_only:
result.wet_package_names -= set(names.wet_package_names)
result.dry_stack_names -= set(names.dry_stack_names)
# exclude wet packages based on build type
if catkin_only or non_catkin_only:
wet_distro = get_wet_distro(distro_name)
for pkg_name in list(result.wet_package_names):
pkg_xml = wet_distro.get_release_package_xml(pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
result.wet_package_names.remove(pkg_name)
continue
build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
if catkin_only ^ (build_type == 'catkin'):
result.wet_package_names.remove(pkg_name)
# get wet and/or dry rosinstall data
rosinstall_data = []
if not dry_only and (result.wet_package_names or has_repos):
wet_distro = get_wet_distro(distro_name)
if upstream_version_tag or upstream_source_version:
# determine repositories based on package names and passed in repository names
repos = {}
for pkg_name in result.wet_package_names:
if upstream_source_version and wet_distro.source_packages:
pkg = wet_distro.source_packages[pkg_name]
repos[pkg.repository_name] = wet_distro.repositories[pkg.repository_name]
else:
pkg = wet_distro.release_packages[pkg_name]
if pkg.repository_name not in repos:
repo = wet_distro.repositories[pkg.repository_name]
release_repo = repo.release_repository
assert not upstream_version_tag or release_repo.version is not None, "Package '%s' in repository '%s' does not have a release version" % (pkg_name, pkg.repository_name)
repos[pkg.repository_name] = repo
# If asked to get upstream development then the release state doesn't matter
if upstream_source_version:
repo_names = repo_names.union(unreleased_repo_names)
for repo_name in repo_names:
if repo_name not in repos:
repos[repo_name] = wet_distro.repositories[repo_name]
# ignore repos which lack information
repos_without_source = [repo_name for repo_name, repo in repos.items() if not repo.source_repository]
if repos_without_source:
logger.warn('The following repositories with an unknown upstream will be ignored: %s' % ', '.join(sorted(repos_without_source)))
[repos.pop(repo_name) for repo_name in repos_without_source]
if upstream_version_tag:
repos_without_release = [repo_name for repo_name, repo in repos.items() if not repo.release_repository or not repo.release_repository.version]
if repos_without_release:
logger.warn('The following repositories without a release will be ignored: %s' % ', '.join(sorted(repos_without_release)))
[repos.pop(repo_name) for repo_name in repos_without_release]
logger.debug('Generate rosinstall entries for wet repositories: %s' % ', '.join(sorted(repos.keys())))
wet_rosinstall_data = generate_rosinstall_for_repos(repos, version_tag=upstream_version_tag, tar=tar)
rosinstall_data += wet_rosinstall_data
else:
logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
rosinstall_data += wet_rosinstall_data
if not wet_only and result.dry_stack_names:
logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
dry_distro = get_dry_distro(distro_name)
dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
rosinstall_data += dry_rosinstall_data
return rosinstall_data
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
pkgbuild_name =\
'/ros-{0}/{1}/{1}.pkgbuild'.format(distro.name, pkg)
pkgbuild_name = overlay.repo.repo_dir + pkgbuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
is_ros2 = get_distros()[distro.name]['distribution_type'] == 'ros2'
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]
patches = None
if os.path.exists(patch_path):
patches = [
f for f in glob.glob('%s/*.patch' % patch_path)
]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing pkgbuild.
prefix = '{0}/ros-{1}/{2}/'.format(overlay.repo.repo_dir, distro.name, pkg)
existing = glob.glob('%s*.pkgbuild' % prefix)
previous_version = None
if preserve_existing and os.path.isfile(pkgbuild_name):
ok("pkgbuild for package '%s' up to date, skipping..." % pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing[0])
previous_version = existing[0].lstrip(prefix).rstrip('.pkgbuild')
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg
)
overlay.repo.remove_file(manifest_file)
try:
current = arch_pkgbuild(distro, pkg, has_patches)
current.pkgbuild.name = pkg
current.pkgbuild.version = version
current.pkgbuild.patches = patches
current.pkgbuild.is_ros2 = is_ros2
except Exception as e:
err('Failed to generate pkgbuild for package {}!'.format(pkg))
raise e
try:
pkgbuild_text = current.pkgbuild_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.pkgbuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.pkgbuild.get_unresolved(), None
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir(
"{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg)
)
success_msg = 'Successfully generated pkgbuild for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
try:
pkgbuild_file = '{0}/ros-{1}/{2}/PKGBUILD'.format(
overlay.repo.repo_dir,
distro.name,
pkg,
)
ok(f"writing {pkgbuild_file}")
with open(pkgbuild_file, "w") as pkgbuild_file_f:
pkgbuild_file_f.write(pkgbuild_text)
except Exception as e:
err(f"Failed to write f{pkgbuild_file} to disk!")
raise e
return current, previous_version, pkg
def generate_rosinstall(distro_name, names,
from_paths=None, repo_names=None,
deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
excludes=None, exclude_paths=None,
flat=False,
tar=False,
upstream_version_tag=False, upstream_source_version=False):
# classify package/stack names
names, keywords = _split_special_keywords(names)
# find packages recursively in include paths
if from_paths:
include_names_from_path = set([])
[include_names_from_path.update(_get_package_names(from_path)) for from_path in from_paths]
logger.debug("The following wet packages found in '--from-path' will be considered: %s" % ', '.join(sorted(include_names_from_path)))
names.update(include_names_from_path)
# expand repository names into package names
repo_names, unknown_repo_names = _classify_repo_names(distro_name, repo_names)
if unknown_repo_names:
logger.warn('The following unknown repositories will be ignored: %s' % (', '.join(sorted(unknown_repo_names))))
wet_package_names, unreleased_repo_names = _get_packages_for_repos(distro_name, repo_names)
names.update(wet_package_names)
if unreleased_repo_names and not upstream_version_tag and not upstream_source_version:
logger.warn('The following unreleased repositories will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
if unreleased_repo_names and (deps or deps_up_to) and (upstream_version_tag or upstream_source_version):
logger.warn('The dependencies of the following unreleased repositories are unknown and will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
has_repos = ((repo_names - unreleased_repo_names) and (upstream_version_tag or upstream_source_version)) or (unreleased_repo_names and upstream_source_version)
names, unknown_names = _classify_names(distro_name, names)
if unknown_names:
logger.warn('The following not released packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn('The following not released packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
names.update(expanded_names)
if not names.wet_package_names and not names.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after ignoring not released')
if names.wet_package_names or names.dry_stack_names:
logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
if unreleased_repo_names:
logger.debug('Unreleased repositories: %s' % ', '.join(sorted(unreleased_repo_names)))
# classify deps-up-to
deps_up_to_names, keywords = _split_special_keywords(deps_up_to or [])
deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names)
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
deps_up_to_names.update(expanded_names)
if deps_up_to:
logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))
# classify excludes
exclude_names, keywords = _split_special_keywords(excludes or [])
if exclude_paths:
exclude_names_from_path = set([])
[exclude_names_from_path.update(_get_package_names(exclude_path)) for exclude_path in exclude_paths]
logger.debug("The following wet packages found in '--exclude-path' will be excluded: %s" % ', '.join(sorted(exclude_names_from_path)))
exclude_names.update(exclude_names_from_path)
exclude_names, unknown_names = _classify_names(distro_name, exclude_names)
if unknown_names:
logger.warn("The following not released '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
exclude_names.update(expanded_names)
if excludes:
logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))
result = copy.deepcopy(names)
# clear wet packages if not requested
if dry_only:
result.wet_package_names.clear()
# clear dry packages if not requested and no dependencies
if wet_only and not deps and not deps_up_to:
result.dry_stack_names.clear()
# remove excluded names from the list of wet and dry names
result.wet_package_names -= exclude_names.wet_package_names
result.dry_stack_names -= exclude_names.dry_stack_names
if not result.wet_package_names and not result.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after applying the exclusions')
if result.wet_package_names:
logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
if result.dry_stack_names:
logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
# extend the names with recursive dependencies
if deps or deps_up_to:
# add dry dependencies
if result.dry_stack_names:
dry_distro = get_dry_distro(distro_name)
_, unreleased_stack_names = get_stack_names(dry_distro)
excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
result.dry_stack_names |= dry_dependencies
if not dry_only:
# add wet dependencies of dry stuff
logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
for depend in wet_dependencies:
if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
continue
wet_distro = get_wet_distro(distro_name)
assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
result.wet_package_names.add(depend)
# add wet dependencies
if result.wet_package_names:
wet_distro = get_wet_distro(distro_name)
_, unreleased_package_names = get_package_names(wet_distro)
excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes, limit_depth=deps_depth)
logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect result with recursive dependencies on
if deps_up_to:
# intersect with wet dependencies on
if deps_up_to_names.wet_package_names:
wet_distro = get_wet_distro(distro_name)
# wet depends on do not include the names since they are excluded to stop recursion asap
wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names, limit=result.wet_package_names)
# keep all names which are already in the result set
wet_package_names |= result.wet_package_names & names.wet_package_names
result.wet_package_names = wet_package_names
else:
result.wet_package_names.clear()
logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect with dry dependencies on
dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
if dry_dependency_names and not wet_only:
dry_distro = get_dry_distro(distro_name)
# dry depends on do not include the names since they are excluded to stop recursion asap
dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
# keep all names which are already in the result set
dry_stack_names |= result.dry_stack_names & names.dry_stack_names
result.dry_stack_names = dry_stack_names
else:
result.dry_stack_names.clear()
logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))
# exclude passed in names
if deps_only:
result.wet_package_names -= set(names.wet_package_names)
result.dry_stack_names -= set(names.dry_stack_names)
# exclude wet packages based on build type
if catkin_only or non_catkin_only:
wet_distro = get_wet_distro(distro_name)
for pkg_name in list(result.wet_package_names):
pkg_xml = wet_distro.get_release_package_xml(pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
result.wet_package_names.remove(pkg_name)
continue
build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
if catkin_only ^ (build_type == 'catkin'):
result.wet_package_names.remove(pkg_name)
# get wet and/or dry rosinstall data
rosinstall_data = []
if not dry_only and (result.wet_package_names or has_repos):
wet_distro = get_wet_distro(distro_name)
if upstream_version_tag or upstream_source_version:
# determine repositories based on package names and passed in repository names
repos = {}
for pkg_name in result.wet_package_names:
pkg = wet_distro.release_packages[pkg_name]
if pkg.repository_name not in repos:
repo = wet_distro.repositories[pkg.repository_name]
release_repo = repo.release_repository
assert not upstream_version_tag or release_repo.version is not None, "Package '%s' in repository '%s' does not have a release version" % (pkg_name, pkg.repository_name)
repos[pkg.repository_name] = repo
for repo_name in repo_names:
if repo_name not in repos:
repos[repo_name] = wet_distro.repositories[repo_name]
# ignore repos which lack information
repos_without_source = [repo_name for repo_name, repo in repos.items() if not repo.source_repository]
if repos_without_source:
logger.warn('The following repositories with an unknown upstream will be ignored: %s' % ', '.join(sorted(repos_without_source)))
[repos.pop(repo_name) for repo_name in repos_without_source]
if upstream_version_tag:
repos_without_release = [repo_name for repo_name, repo in repos.items() if not repo.release_repository or not repo.release_repository.version]
if repos_without_release:
logger.warn('The following repositories without a release will be ignored: %s' % ', '.join(sorted(repos_without_release)))
[repos.pop(repo_name) for repo_name in repos_without_release]
logger.debug('Generate rosinstall entries for wet repositories: %s' % ', '.join(sorted(repos.keys())))
wet_rosinstall_data = generate_rosinstall_for_repos(repos, version_tag=upstream_version_tag, tar=tar)
rosinstall_data += wet_rosinstall_data
else:
logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
rosinstall_data += wet_rosinstall_data
if not wet_only and result.dry_stack_names:
logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
dry_distro = get_dry_distro(distro_name)
dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
rosinstall_data += dry_rosinstall_data
return rosinstall_data
def generate_installers(
distro_name, # ros distro name
overlay, # repo instance
gen_pkg_func, # function to call for generating
preserve_existing=True, # don't regenerate if installer exists
*args # any aditional args for gen_pkg_func
):
distro = get_distro(distro_name)
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
bad_installers = []
succeeded = 0
failed = 0
info("Generating installers for distro '%s'" % distro_name)
for i, pkg in enumerate(sorted(pkg_names[0])):
version = get_pkg_version(distro, pkg)
percent = '%.1f' % (100 * (float(i) / total))
try:
current, bad_deps = gen_pkg_func(overlay, pkg, distro,
preserve_existing, *args)
if not current and bad_deps:
# we are missing dependencies
failed_msg = "{0}%: Failed to generate".format(percent)
failed_msg += " installer for package '%s'!" % pkg
err(failed_msg)
borkd_pkgs[pkg] = bad_deps
failed = failed + 1
continue
elif not current and preserve_existing:
# don't replace the installer
succeeded = succeeded + 1
continue
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1
changes.append('*{0} --> {1}*'.format(pkg, version))
installers.append(pkg)
except UnknownLicense as ul:
err("{0}%: Unknown License '{1}'.".format(percent, str(ul)))
bad_installers.append(pkg)
failed = failed + 1
except KeyError:
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(pkg)
failed = failed + 1
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
info("------ {0} ------\n".format(results))
if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))
return installers, borkd_pkgs, changes
def regenerate_pkg(overlay, pkg, rosdistro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(rosdistro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s' available packages"
" in selected distro: %s" %
(pkg, get_package_names(rosdistro)))
try:
version = get_pkg_version(rosdistro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
rosdistro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe which was removed by clean_ros_recipe_dirs
prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format(
yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe)
existing = overlay.repo.git.status('--porcelain', '--', prefix)
if existing:
# The git status --porcelain output will look like this:
# D meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb
# we want just the path with filename
if len(existing.split('\n')) > 1:
warn('More than 1 recipe was output by "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
if existing.split()[0] != 'D':
err('Unexpected output from "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing.split()[1]
else:
# If it isn't shown in git status, it could still exist as normal
# unchanged file when --only option is being used
import glob
existing = glob.glob('{0}/{1}'.format(repo_dir, prefix))
if existing:
if len(existing) > 1:
err('More than 1 recipe was output by "git status '
'--porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.
format(yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing[0]
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(rosdistro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
def regenerate_pkg(overlay, pkg, distro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
)
existing = glob.glob('{}_*.bb'.format(prefix))
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
existing = existing[0]
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(distro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'ros{1}-{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes['ros{0}-{1}'.format(
yoctoRecipe._get_ros_version(distro.name),
recipe)] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe