def _expand_keywords(distro_name, keywords):
names = set([])
if ARG_ALL_PACKAGES in keywords:
wet_distro = get_wet_distro(distro_name)
released_package_names, _ = get_package_names(wet_distro)
names.update(released_package_names)
if distro_name == 'groovy':
dry_distro = get_dry_distro(distro_name)
released_stack_names, _ = get_stack_names(dry_distro)
names.update(released_stack_names)
if ARG_CURRENT_ENVIRONMENT in keywords:
names.update(_get_packages_in_environment())
return names
def _expand_keywords(distro_name, keywords):
names = set([])
if ARG_ALL_PACKAGES in keywords:
wet_distro = get_wet_distro(distro_name)
released_package_names, _ = get_package_names(wet_distro)
names.update(released_package_names)
if distro_name == 'groovy':
dry_distro = get_dry_distro(distro_name)
released_stack_names, _ = get_stack_names(dry_distro)
names.update(released_stack_names)
if ARG_CURRENT_ENVIRONMENT in keywords:
names.update(_get_packages_in_environment())
return names
def generate_rosinstall(distro_name, names,
deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
excludes=None,
flat=False,
tar=False):
# classify package/stack names
names, keywords = _split_special_keywords(names)
names, unknown_names = _classify_names(distro_name, names)
if unknown_names:
logger.warn('The following not released packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn('The following not released packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
names.update(expanded_names)
if not names.wet_package_names and not names.dry_stack_names:
raise RuntimeError('No packages/stacks left after ignoring not released')
logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
# classify deps-up-to
deps_up_to_names, keywords = _split_special_keywords(deps_up_to or [])
deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names)
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
deps_up_to_names.update(expanded_names)
if deps_up_to:
logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))
# classify excludes
exclude_names, keywords = _split_special_keywords(excludes or [])
exclude_names, unknown_names = _classify_names(distro_name, exclude_names)
if unknown_names:
logger.warn("The following not released '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
exclude_names.update(expanded_names)
if excludes:
logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))
result = copy.deepcopy(names)
# clear wet packages if not requested
if dry_only:
result.wet_package_names.clear()
# clear dry packages if not requested and no dependencies
if wet_only and not deps and not deps_up_to:
result.dry_stack_names.clear()
# remove excluded names from the list of wet and dry names
result.wet_package_names -= exclude_names.wet_package_names
result.dry_stack_names -= exclude_names.dry_stack_names
if not result.wet_package_names and not result.dry_stack_names:
raise RuntimeError('No packages/stacks left after applying the exclusions')
if result.wet_package_names:
logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
if result.dry_stack_names:
logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
# extend the names with recursive dependencies
if deps or deps_up_to:
# add dry dependencies
if result.dry_stack_names:
dry_distro = get_dry_distro(distro_name)
_, unreleased_stack_names = get_stack_names(dry_distro)
excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
result.dry_stack_names |= dry_dependencies
if not dry_only:
# add wet dependencies of dry stuff
logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
for depend in wet_dependencies:
if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
continue
wet_distro = get_wet_distro(distro_name)
assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
result.wet_package_names.add(depend)
# add wet dependencies
if result.wet_package_names:
wet_distro = get_wet_distro(distro_name)
_, unreleased_package_names = get_package_names(wet_distro)
excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes, limit_depth=deps_depth)
logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect result with recursive dependencies on
if deps_up_to:
# intersect with wet dependencies on
if deps_up_to_names.wet_package_names:
wet_distro = get_wet_distro(distro_name)
# wet depends on do not include the names since they are excluded to stop recursion asap
wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names, limit=result.wet_package_names)
# keep all names which are already in the result set
wet_package_names |= result.wet_package_names & names.wet_package_names
result.wet_package_names = wet_package_names
else:
result.wet_package_names.clear()
logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect with dry dependencies on
dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
if dry_dependency_names and not wet_only:
dry_distro = get_dry_distro(distro_name)
# dry depends on do not include the names since they are excluded to stop recursion asap
dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
# keep all names which are already in the result set
dry_stack_names |= result.dry_stack_names & names.dry_stack_names
result.dry_stack_names = dry_stack_names
else:
result.dry_stack_names.clear()
logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))
# exclude passed in names
if deps_only:
result.wet_package_names -= set(names.wet_package_names)
result.dry_stack_names -= set(names.dry_stack_names)
# exclude wet packages based on build type
if catkin_only or non_catkin_only:
wet_distro = get_wet_distro(distro_name)
for pkg_name in list(result.wet_package_names):
pkg_xml = wet_distro.get_release_package_xml(pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
result.wet_package_names.remove(pkg_name)
continue
build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
if catkin_only ^ (build_type == 'catkin'):
result.wet_package_names.remove(pkg_name)
# get wet and/or dry rosinstall data
rosinstall_data = []
if not dry_only and result.wet_package_names:
logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
wet_distro = get_wet_distro(distro_name)
wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
rosinstall_data += wet_rosinstall_data
if not wet_only and result.dry_stack_names:
logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
dry_distro = get_dry_distro(distro_name)
dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
rosinstall_data += dry_rosinstall_data
return rosinstall_data
def generate_rosinstall(distro_name, names,
from_paths=None, repo_names=None,
deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
excludes=None, exclude_paths=None,
flat=False,
tar=False,
upstream_version_tag=False, upstream_source_version=False):
# classify package/stack names
names, pkg_keywords = _split_special_keywords(names)
# find packages recursively in include paths
if from_paths:
include_names_from_path = set([])
[include_names_from_path.update(_get_package_names(from_path)) for from_path in from_paths]
logger.debug("The following wet packages found in '--from-path' will be considered: %s" % ', '.join(sorted(include_names_from_path)))
names.update(include_names_from_path)
# Allow special keywords in repos
repo_names, repo_keywords = _split_special_keywords(repo_names or [])
if set(repo_keywords).difference(set([ARG_ALL_PACKAGES])):
raise RuntimeError('The only keyword supported by repos is %r' % (ARG_ALL_PACKAGES))
if ARG_ALL_PACKAGES in repo_keywords:
wet_distro = get_wet_distro(distro_name)
repo_names = wet_distro.repositories.keys()
# expand repository names into package names
repo_names, unknown_repo_names = _classify_repo_names(distro_name, repo_names)
if unknown_repo_names:
logger.warn('The following unknown repositories will be ignored: %s' % (', '.join(sorted(unknown_repo_names))))
wet_package_names, unreleased_repo_names = _get_packages_for_repos(distro_name, repo_names, source=upstream_source_version)
names.update(wet_package_names)
if unreleased_repo_names and not upstream_version_tag and not upstream_source_version:
logger.warn('The following unreleased repositories will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
if unreleased_repo_names and (deps or deps_up_to) and (upstream_version_tag or upstream_source_version):
logger.warn('The dependencies of the following unreleased repositories are unknown and will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
has_repos = ((repo_names - unreleased_repo_names) and (upstream_version_tag or upstream_source_version)) or (unreleased_repo_names and upstream_source_version)
names, unknown_names = _classify_names(distro_name, names, source=upstream_source_version)
if unknown_names:
logger.warn('The following unreleased packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
if pkg_keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, pkg_keywords), source=upstream_source_version)
if unknown_names:
logger.warn('The following unreleased packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
names.update(expanded_names)
if not names.wet_package_names and not names.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after ignoring unreleased')
if names.wet_package_names or names.dry_stack_names:
logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
if unreleased_repo_names:
logger.debug('Unreleased repositories: %s' % ', '.join(sorted(unreleased_repo_names)))
# classify deps-up-to
deps_up_to_names, deps_keywords = _split_special_keywords(deps_up_to or [])
deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names, source=upstream_source_version)
if unknown_names:
logger.warn("The following unreleased '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if deps_keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, deps_keywords), source=upstream_source_version)
if unknown_names:
logger.warn("The following unreleased '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
deps_up_to_names.update(expanded_names)
if deps_up_to:
logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))
# classify excludes
exclude_names, excludes_keywords = _split_special_keywords(excludes or [])
if exclude_paths:
exclude_names_from_path = set([])
[exclude_names_from_path.update(_get_package_names(exclude_path)) for exclude_path in exclude_paths]
logger.debug("The following wet packages found in '--exclude-path' will be excluded: %s" % ', '.join(sorted(exclude_names_from_path)))
exclude_names.update(exclude_names_from_path)
exclude_names, unknown_names = _classify_names(distro_name, exclude_names, source=upstream_source_version)
if unknown_names:
logger.warn("The following unreleased '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if excludes_keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, excludes_keywords), source=upstream_source_version)
exclude_names.update(expanded_names)
if excludes:
logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))
result = copy.deepcopy(names)
# clear wet packages if not requested
if dry_only:
result.wet_package_names.clear()
# clear dry packages if not requested and no dependencies
if wet_only and not deps and not deps_up_to:
result.dry_stack_names.clear()
# remove excluded names from the list of wet and dry names
result.wet_package_names -= exclude_names.wet_package_names
result.dry_stack_names -= exclude_names.dry_stack_names
if not result.wet_package_names and not result.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after applying the exclusions')
if result.wet_package_names:
logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
if result.dry_stack_names:
logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
# extend the names with recursive dependencies
if deps or deps_up_to:
# add dry dependencies
if result.dry_stack_names:
dry_distro = get_dry_distro(distro_name)
_, unreleased_stack_names = get_stack_names(dry_distro)
excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
result.dry_stack_names |= dry_dependencies
if not dry_only:
# add wet dependencies of dry stuff
logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
for depend in wet_dependencies:
if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
continue
wet_distro = get_wet_distro(distro_name)
assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
result.wet_package_names.add(depend)
# add wet dependencies
if result.wet_package_names:
wet_distro = get_wet_distro(distro_name)
_, unreleased_package_names = get_package_names(wet_distro)
excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes,
limit_depth=deps_depth, source=upstream_source_version)
logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect result with recursive dependencies on
if deps_up_to:
# intersect with wet dependencies on
if deps_up_to_names.wet_package_names:
wet_distro = get_wet_distro(distro_name)
# wet depends on do not include the names since they are excluded to stop recursion asap
wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names,
limit=result.wet_package_names, source=upstream_source_version)
# keep all names which are already in the result set
wet_package_names |= result.wet_package_names & names.wet_package_names
result.wet_package_names = wet_package_names
else:
result.wet_package_names.clear()
logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect with dry dependencies on
dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
if dry_dependency_names and not wet_only:
dry_distro = get_dry_distro(distro_name)
# dry depends on do not include the names since they are excluded to stop recursion asap
dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
# keep all names which are already in the result set
dry_stack_names |= result.dry_stack_names & names.dry_stack_names
result.dry_stack_names = dry_stack_names
else:
result.dry_stack_names.clear()
logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))
# exclude passed in names
if deps_only:
result.wet_package_names -= set(names.wet_package_names)
result.dry_stack_names -= set(names.dry_stack_names)
# exclude wet packages based on build type
if catkin_only or non_catkin_only:
wet_distro = get_wet_distro(distro_name)
for pkg_name in list(result.wet_package_names):
pkg_xml = wet_distro.get_release_package_xml(pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
result.wet_package_names.remove(pkg_name)
continue
build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
if catkin_only ^ (build_type == 'catkin'):
result.wet_package_names.remove(pkg_name)
# get wet and/or dry rosinstall data
rosinstall_data = []
if not dry_only and (result.wet_package_names or has_repos):
wet_distro = get_wet_distro(distro_name)
if upstream_version_tag or upstream_source_version:
# determine repositories based on package names and passed in repository names
repos = {}
for pkg_name in result.wet_package_names:
if upstream_source_version and wet_distro.source_packages:
pkg = wet_distro.source_packages[pkg_name]
repos[pkg.repository_name] = wet_distro.repositories[pkg.repository_name]
else:
pkg = wet_distro.release_packages[pkg_name]
if pkg.repository_name not in repos:
repo = wet_distro.repositories[pkg.repository_name]
release_repo = repo.release_repository
assert not upstream_version_tag or release_repo.version is not None, "Package '%s' in repository '%s' does not have a release version" % (pkg_name, pkg.repository_name)
repos[pkg.repository_name] = repo
# If asked to get upstream development then the release state doesn't matter
if upstream_source_version:
repo_names = repo_names.union(unreleased_repo_names)
for repo_name in repo_names:
if repo_name not in repos:
repos[repo_name] = wet_distro.repositories[repo_name]
# ignore repos which lack information
repos_without_source = [repo_name for repo_name, repo in repos.items() if not repo.source_repository]
if repos_without_source:
logger.warn('The following repositories with an unknown upstream will be ignored: %s' % ', '.join(sorted(repos_without_source)))
[repos.pop(repo_name) for repo_name in repos_without_source]
if upstream_version_tag:
repos_without_release = [repo_name for repo_name, repo in repos.items() if not repo.release_repository or not repo.release_repository.version]
if repos_without_release:
logger.warn('The following repositories without a release will be ignored: %s' % ', '.join(sorted(repos_without_release)))
[repos.pop(repo_name) for repo_name in repos_without_release]
logger.debug('Generate rosinstall entries for wet repositories: %s' % ', '.join(sorted(repos.keys())))
wet_rosinstall_data = generate_rosinstall_for_repos(repos, version_tag=upstream_version_tag, tar=tar)
rosinstall_data += wet_rosinstall_data
else:
logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
rosinstall_data += wet_rosinstall_data
if not wet_only and result.dry_stack_names:
logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
dry_distro = get_dry_distro(distro_name)
dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
rosinstall_data += dry_rosinstall_data
return rosinstall_data
def generate_rosinstall(distro_name, names,
from_paths=None, repo_names=None,
deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
excludes=None, exclude_paths=None,
flat=False,
tar=False,
upstream_version_tag=False, upstream_source_version=False):
# classify package/stack names
names, keywords = _split_special_keywords(names)
# find packages recursively in include paths
if from_paths:
include_names_from_path = set([])
[include_names_from_path.update(_get_package_names(from_path)) for from_path in from_paths]
logger.debug("The following wet packages found in '--from-path' will be considered: %s" % ', '.join(sorted(include_names_from_path)))
names.update(include_names_from_path)
# expand repository names into package names
repo_names, unknown_repo_names = _classify_repo_names(distro_name, repo_names)
if unknown_repo_names:
logger.warn('The following unknown repositories will be ignored: %s' % (', '.join(sorted(unknown_repo_names))))
wet_package_names, unreleased_repo_names = _get_packages_for_repos(distro_name, repo_names)
names.update(wet_package_names)
if unreleased_repo_names and not upstream_version_tag and not upstream_source_version:
logger.warn('The following unreleased repositories will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
if unreleased_repo_names and (deps or deps_up_to) and (upstream_version_tag or upstream_source_version):
logger.warn('The dependencies of the following unreleased repositories are unknown and will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
has_repos = ((repo_names - unreleased_repo_names) and (upstream_version_tag or upstream_source_version)) or (unreleased_repo_names and upstream_source_version)
names, unknown_names = _classify_names(distro_name, names)
if unknown_names:
logger.warn('The following not released packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn('The following not released packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
names.update(expanded_names)
if not names.wet_package_names and not names.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after ignoring not released')
if names.wet_package_names or names.dry_stack_names:
logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
if unreleased_repo_names:
logger.debug('Unreleased repositories: %s' % ', '.join(sorted(unreleased_repo_names)))
# classify deps-up-to
deps_up_to_names, keywords = _split_special_keywords(deps_up_to or [])
deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names)
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
if unknown_names:
logger.warn("The following not released '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
deps_up_to_names.update(expanded_names)
if deps_up_to:
logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))
# classify excludes
exclude_names, keywords = _split_special_keywords(excludes or [])
if exclude_paths:
exclude_names_from_path = set([])
[exclude_names_from_path.update(_get_package_names(exclude_path)) for exclude_path in exclude_paths]
logger.debug("The following wet packages found in '--exclude-path' will be excluded: %s" % ', '.join(sorted(exclude_names_from_path)))
exclude_names.update(exclude_names_from_path)
exclude_names, unknown_names = _classify_names(distro_name, exclude_names)
if unknown_names:
logger.warn("The following not released '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
if keywords:
expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
exclude_names.update(expanded_names)
if excludes:
logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))
result = copy.deepcopy(names)
# clear wet packages if not requested
if dry_only:
result.wet_package_names.clear()
# clear dry packages if not requested and no dependencies
if wet_only and not deps and not deps_up_to:
result.dry_stack_names.clear()
# remove excluded names from the list of wet and dry names
result.wet_package_names -= exclude_names.wet_package_names
result.dry_stack_names -= exclude_names.dry_stack_names
if not result.wet_package_names and not result.dry_stack_names and not has_repos:
raise RuntimeError('No packages/stacks left after applying the exclusions')
if result.wet_package_names:
logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
if result.dry_stack_names:
logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
# extend the names with recursive dependencies
if deps or deps_up_to:
# add dry dependencies
if result.dry_stack_names:
dry_distro = get_dry_distro(distro_name)
_, unreleased_stack_names = get_stack_names(dry_distro)
excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
result.dry_stack_names |= dry_dependencies
if not dry_only:
# add wet dependencies of dry stuff
logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
for depend in wet_dependencies:
if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
continue
wet_distro = get_wet_distro(distro_name)
assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
result.wet_package_names.add(depend)
# add wet dependencies
if result.wet_package_names:
wet_distro = get_wet_distro(distro_name)
_, unreleased_package_names = get_package_names(wet_distro)
excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes, limit_depth=deps_depth)
logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect result with recursive dependencies on
if deps_up_to:
# intersect with wet dependencies on
if deps_up_to_names.wet_package_names:
wet_distro = get_wet_distro(distro_name)
# wet depends on do not include the names since they are excluded to stop recursion asap
wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names, limit=result.wet_package_names)
# keep all names which are already in the result set
wet_package_names |= result.wet_package_names & names.wet_package_names
result.wet_package_names = wet_package_names
else:
result.wet_package_names.clear()
logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))
# intersect with dry dependencies on
dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
if dry_dependency_names and not wet_only:
dry_distro = get_dry_distro(distro_name)
# dry depends on do not include the names since they are excluded to stop recursion asap
dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
# keep all names which are already in the result set
dry_stack_names |= result.dry_stack_names & names.dry_stack_names
result.dry_stack_names = dry_stack_names
else:
result.dry_stack_names.clear()
logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))
# exclude passed in names
if deps_only:
result.wet_package_names -= set(names.wet_package_names)
result.dry_stack_names -= set(names.dry_stack_names)
# exclude wet packages based on build type
if catkin_only or non_catkin_only:
wet_distro = get_wet_distro(distro_name)
for pkg_name in list(result.wet_package_names):
pkg_xml = wet_distro.get_release_package_xml(pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
result.wet_package_names.remove(pkg_name)
continue
build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
if catkin_only ^ (build_type == 'catkin'):
result.wet_package_names.remove(pkg_name)
# get wet and/or dry rosinstall data
rosinstall_data = []
if not dry_only and (result.wet_package_names or has_repos):
wet_distro = get_wet_distro(distro_name)
if upstream_version_tag or upstream_source_version:
# determine repositories based on package names and passed in repository names
repos = {}
for pkg_name in result.wet_package_names:
pkg = wet_distro.release_packages[pkg_name]
if pkg.repository_name not in repos:
repo = wet_distro.repositories[pkg.repository_name]
release_repo = repo.release_repository
assert not upstream_version_tag or release_repo.version is not None, "Package '%s' in repository '%s' does not have a release version" % (pkg_name, pkg.repository_name)
repos[pkg.repository_name] = repo
for repo_name in repo_names:
if repo_name not in repos:
repos[repo_name] = wet_distro.repositories[repo_name]
# ignore repos which lack information
repos_without_source = [repo_name for repo_name, repo in repos.items() if not repo.source_repository]
if repos_without_source:
logger.warn('The following repositories with an unknown upstream will be ignored: %s' % ', '.join(sorted(repos_without_source)))
[repos.pop(repo_name) for repo_name in repos_without_source]
if upstream_version_tag:
repos_without_release = [repo_name for repo_name, repo in repos.items() if not repo.release_repository or not repo.release_repository.version]
if repos_without_release:
logger.warn('The following repositories without a release will be ignored: %s' % ', '.join(sorted(repos_without_release)))
[repos.pop(repo_name) for repo_name in repos_without_release]
logger.debug('Generate rosinstall entries for wet repositories: %s' % ', '.join(sorted(repos.keys())))
wet_rosinstall_data = generate_rosinstall_for_repos(repos, version_tag=upstream_version_tag, tar=tar)
rosinstall_data += wet_rosinstall_data
else:
logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
rosinstall_data += wet_rosinstall_data
if not wet_only and result.dry_stack_names:
logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
dry_distro = get_dry_distro(distro_name)
dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
rosinstall_data += dry_rosinstall_data
return rosinstall_data