def _expand_keywords(distro_name, keywords):
    names = set([])
    if ARG_ALL_PACKAGES in keywords:
        wet_distro = get_wet_distro(distro_name)
        released_package_names, _ = get_package_names(wet_distro)
        names.update(released_package_names)
        if distro_name == 'groovy':
            dry_distro = get_dry_distro(distro_name)
            released_stack_names, _ = get_stack_names(dry_distro)
            names.update(released_stack_names)
    if ARG_CURRENT_ENVIRONMENT in keywords:
        names.update(_get_packages_in_environment())
    return names
Ejemplo n.º 2
0
def _expand_keywords(distro_name, keywords):
    names = set([])
    if ARG_ALL_PACKAGES in keywords:
        wet_distro = get_wet_distro(distro_name)
        released_package_names, _ = get_package_names(wet_distro)
        names.update(released_package_names)
        if distro_name == 'groovy':
            dry_distro = get_dry_distro(distro_name)
            released_stack_names, _ = get_stack_names(dry_distro)
            names.update(released_stack_names)
    if ARG_CURRENT_ENVIRONMENT in keywords:
        names.update(_get_packages_in_environment())
    return names
def generate_rosinstall(distro_name, names,
    deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
    wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
    excludes=None,
    flat=False,
    tar=False):
    # classify package/stack names
    names, keywords = _split_special_keywords(names)
    names, unknown_names = _classify_names(distro_name, names)
    if unknown_names:
        logger.warn('The following not released packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
    if keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
        if unknown_names:
            logger.warn('The following not released packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
        names.update(expanded_names)
    if not names.wet_package_names and not names.dry_stack_names:
        raise RuntimeError('No packages/stacks left after ignoring not released')
    logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))

    # classify deps-up-to
    deps_up_to_names, keywords = _split_special_keywords(deps_up_to or [])
    deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names)
    if unknown_names:
        logger.warn("The following not released '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
    if keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
        if unknown_names:
            logger.warn("The following not released '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
        deps_up_to_names.update(expanded_names)
    if deps_up_to:
        logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))

    # classify excludes
    exclude_names, keywords = _split_special_keywords(excludes or [])
    exclude_names, unknown_names = _classify_names(distro_name, exclude_names)
    if unknown_names:
        logger.warn("The following not released '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
    if keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
        exclude_names.update(expanded_names)
    if excludes:
        logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))

    result = copy.deepcopy(names)
    # clear wet packages if not requested
    if dry_only:
        result.wet_package_names.clear()
    # clear dry packages if not requested and no dependencies
    if wet_only and not deps and not deps_up_to:
        result.dry_stack_names.clear()

    # remove excluded names from the list of wet and dry names
    result.wet_package_names -= exclude_names.wet_package_names
    result.dry_stack_names -= exclude_names.dry_stack_names
    if not result.wet_package_names and not result.dry_stack_names:
        raise RuntimeError('No packages/stacks left after applying the exclusions')

    if result.wet_package_names:
        logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
    if result.dry_stack_names:
        logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))

    # extend the names with recursive dependencies
    if deps or deps_up_to:
        # add dry dependencies
        if result.dry_stack_names:
            dry_distro = get_dry_distro(distro_name)
            _, unreleased_stack_names = get_stack_names(dry_distro)
            excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
            dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
            logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
            result.dry_stack_names |= dry_dependencies

            if not dry_only:
                # add wet dependencies of dry stuff
                logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
                for depend in wet_dependencies:
                    if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
                        continue
                    wet_distro = get_wet_distro(distro_name)
                    assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
                    result.wet_package_names.add(depend)
        # add wet dependencies
        if result.wet_package_names:
            wet_distro = get_wet_distro(distro_name)
            _, unreleased_package_names = get_package_names(wet_distro)
            excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
            result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes, limit_depth=deps_depth)
            logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))

    # intersect result with recursive dependencies on
    if deps_up_to:
        # intersect with wet dependencies on
        if deps_up_to_names.wet_package_names:
            wet_distro = get_wet_distro(distro_name)
            # wet depends on do not include the names since they are excluded to stop recursion asap
            wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names, limit=result.wet_package_names)
            # keep all names which are already in the result set
            wet_package_names |= result.wet_package_names & names.wet_package_names
            result.wet_package_names = wet_package_names
        else:
            result.wet_package_names.clear()
        logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))

        # intersect with dry dependencies on
        dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
        if dry_dependency_names and not wet_only:
            dry_distro = get_dry_distro(distro_name)
            # dry depends on do not include the names since they are excluded to stop recursion asap
            dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
            # keep all names which are already in the result set
            dry_stack_names |= result.dry_stack_names & names.dry_stack_names
            result.dry_stack_names = dry_stack_names
        else:
            result.dry_stack_names.clear()
        logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))

    # exclude passed in names
    if deps_only:
        result.wet_package_names -= set(names.wet_package_names)
        result.dry_stack_names -= set(names.dry_stack_names)

    # exclude wet packages based on build type
    if catkin_only or non_catkin_only:
        wet_distro = get_wet_distro(distro_name)
        for pkg_name in list(result.wet_package_names):
            pkg_xml = wet_distro.get_release_package_xml(pkg_name)
            try:
                pkg = parse_package_string(pkg_xml)
            except InvalidPackage as e:
                logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
                result.wet_package_names.remove(pkg_name)
                continue
            build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
            if catkin_only ^ (build_type == 'catkin'):
                result.wet_package_names.remove(pkg_name)

    # get wet and/or dry rosinstall data
    rosinstall_data = []
    if not dry_only and result.wet_package_names:
        logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
        wet_distro = get_wet_distro(distro_name)
        wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
        rosinstall_data += wet_rosinstall_data
    if not wet_only and result.dry_stack_names:
        logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
        dry_distro = get_dry_distro(distro_name)
        dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
        rosinstall_data += dry_rosinstall_data
    return rosinstall_data
Ejemplo n.º 4
0
def generate_rosinstall(distro_name, names,
    from_paths=None, repo_names=None,
    deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
    wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
    excludes=None, exclude_paths=None,
    flat=False,
    tar=False,
    upstream_version_tag=False, upstream_source_version=False):

    # classify package/stack names
    names, pkg_keywords = _split_special_keywords(names)

    # find packages recursively in include paths
    if from_paths:
        include_names_from_path = set([])
        [include_names_from_path.update(_get_package_names(from_path)) for from_path in from_paths]
        logger.debug("The following wet packages found in '--from-path' will be considered: %s" % ', '.join(sorted(include_names_from_path)))
        names.update(include_names_from_path)

    # Allow special keywords in repos
    repo_names, repo_keywords = _split_special_keywords(repo_names or [])
    if set(repo_keywords).difference(set([ARG_ALL_PACKAGES])):
        raise RuntimeError('The only keyword supported by repos is %r' % (ARG_ALL_PACKAGES))

    if ARG_ALL_PACKAGES in repo_keywords:
        wet_distro = get_wet_distro(distro_name)
        repo_names = wet_distro.repositories.keys()

    # expand repository names into package names
    repo_names, unknown_repo_names = _classify_repo_names(distro_name, repo_names)
    if unknown_repo_names:
        logger.warn('The following unknown repositories will be ignored: %s' % (', '.join(sorted(unknown_repo_names))))
    wet_package_names, unreleased_repo_names = _get_packages_for_repos(distro_name, repo_names, source=upstream_source_version)
    names.update(wet_package_names)
    if unreleased_repo_names and not upstream_version_tag and not upstream_source_version:
        logger.warn('The following unreleased repositories will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
    if unreleased_repo_names and (deps or deps_up_to) and (upstream_version_tag or upstream_source_version):
        logger.warn('The dependencies of the following unreleased repositories are unknown and will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
    has_repos = ((repo_names - unreleased_repo_names) and (upstream_version_tag or upstream_source_version)) or (unreleased_repo_names and upstream_source_version)

    names, unknown_names = _classify_names(distro_name, names, source=upstream_source_version)
    if unknown_names:
        logger.warn('The following unreleased packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
    if pkg_keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, pkg_keywords), source=upstream_source_version)
        if unknown_names:
            logger.warn('The following unreleased packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
        names.update(expanded_names)
    if not names.wet_package_names and not names.dry_stack_names and not has_repos:
        raise RuntimeError('No packages/stacks left after ignoring unreleased')
    if names.wet_package_names or names.dry_stack_names:
        logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
    if unreleased_repo_names:
        logger.debug('Unreleased repositories: %s' % ', '.join(sorted(unreleased_repo_names)))

    # classify deps-up-to
    deps_up_to_names, deps_keywords = _split_special_keywords(deps_up_to or [])
    deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names, source=upstream_source_version)
    if unknown_names:
        logger.warn("The following unreleased '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
    if deps_keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, deps_keywords), source=upstream_source_version)
        if unknown_names:
            logger.warn("The following unreleased '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
        deps_up_to_names.update(expanded_names)
    if deps_up_to:
        logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))

    # classify excludes
    exclude_names, excludes_keywords = _split_special_keywords(excludes or [])
    if exclude_paths:
        exclude_names_from_path = set([])
        [exclude_names_from_path.update(_get_package_names(exclude_path)) for exclude_path in exclude_paths]
        logger.debug("The following wet packages found in '--exclude-path' will be excluded: %s" % ', '.join(sorted(exclude_names_from_path)))
        exclude_names.update(exclude_names_from_path)
    exclude_names, unknown_names = _classify_names(distro_name, exclude_names, source=upstream_source_version)
    if unknown_names:
        logger.warn("The following unreleased '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
    if excludes_keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, excludes_keywords), source=upstream_source_version)
        exclude_names.update(expanded_names)
    if excludes:
        logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))

    result = copy.deepcopy(names)
    # clear wet packages if not requested
    if dry_only:
        result.wet_package_names.clear()
    # clear dry packages if not requested and no dependencies
    if wet_only and not deps and not deps_up_to:
        result.dry_stack_names.clear()

    # remove excluded names from the list of wet and dry names
    result.wet_package_names -= exclude_names.wet_package_names
    result.dry_stack_names -= exclude_names.dry_stack_names
    if not result.wet_package_names and not result.dry_stack_names and not has_repos:
        raise RuntimeError('No packages/stacks left after applying the exclusions')

    if result.wet_package_names:
        logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
    if result.dry_stack_names:
        logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))

    # extend the names with recursive dependencies
    if deps or deps_up_to:
        # add dry dependencies
        if result.dry_stack_names:
            dry_distro = get_dry_distro(distro_name)
            _, unreleased_stack_names = get_stack_names(dry_distro)
            excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
            dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
            logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
            result.dry_stack_names |= dry_dependencies

            if not dry_only:
                # add wet dependencies of dry stuff
                logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
                for depend in wet_dependencies:
                    if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
                        continue
                    wet_distro = get_wet_distro(distro_name)
                    assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
                    result.wet_package_names.add(depend)
        # add wet dependencies
        if result.wet_package_names:
            wet_distro = get_wet_distro(distro_name)
            _, unreleased_package_names = get_package_names(wet_distro)
            excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
            result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes,
                    limit_depth=deps_depth, source=upstream_source_version)
            logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))

    # intersect result with recursive dependencies on
    if deps_up_to:
        # intersect with wet dependencies on
        if deps_up_to_names.wet_package_names:
            wet_distro = get_wet_distro(distro_name)
            # wet depends on do not include the names since they are excluded to stop recursion asap
            wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names,
                    limit=result.wet_package_names, source=upstream_source_version)
            # keep all names which are already in the result set
            wet_package_names |= result.wet_package_names & names.wet_package_names
            result.wet_package_names = wet_package_names
        else:
            result.wet_package_names.clear()
        logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))

        # intersect with dry dependencies on
        dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
        if dry_dependency_names and not wet_only:
            dry_distro = get_dry_distro(distro_name)
            # dry depends on do not include the names since they are excluded to stop recursion asap
            dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
            # keep all names which are already in the result set
            dry_stack_names |= result.dry_stack_names & names.dry_stack_names
            result.dry_stack_names = dry_stack_names
        else:
            result.dry_stack_names.clear()
        logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))

    # exclude passed in names
    if deps_only:
        result.wet_package_names -= set(names.wet_package_names)
        result.dry_stack_names -= set(names.dry_stack_names)

    # exclude wet packages based on build type
    if catkin_only or non_catkin_only:
        wet_distro = get_wet_distro(distro_name)
        for pkg_name in list(result.wet_package_names):
            pkg_xml = wet_distro.get_release_package_xml(pkg_name)
            try:
                pkg = parse_package_string(pkg_xml)
            except InvalidPackage as e:
                logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
                result.wet_package_names.remove(pkg_name)
                continue
            build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
            if catkin_only ^ (build_type == 'catkin'):
                result.wet_package_names.remove(pkg_name)

    # get wet and/or dry rosinstall data
    rosinstall_data = []
    if not dry_only and (result.wet_package_names or has_repos):
        wet_distro = get_wet_distro(distro_name)
        if upstream_version_tag or upstream_source_version:
            # determine repositories based on package names and passed in repository names
            repos = {}
            for pkg_name in result.wet_package_names:
                if upstream_source_version and wet_distro.source_packages:
                    pkg = wet_distro.source_packages[pkg_name]
                    repos[pkg.repository_name] = wet_distro.repositories[pkg.repository_name]
                else:
                    pkg = wet_distro.release_packages[pkg_name]
                    if pkg.repository_name not in repos:
                        repo = wet_distro.repositories[pkg.repository_name]
                        release_repo = repo.release_repository
                        assert not upstream_version_tag or release_repo.version is not None, "Package '%s' in repository '%s' does not have a release version" % (pkg_name, pkg.repository_name)
                        repos[pkg.repository_name] = repo
            # If asked to get upstream development then the release state doesn't matter
            if upstream_source_version:
                repo_names = repo_names.union(unreleased_repo_names)
            for repo_name in repo_names:
                if repo_name not in repos:
                    repos[repo_name] = wet_distro.repositories[repo_name]
            # ignore repos which lack information
            repos_without_source = [repo_name for repo_name, repo in repos.items() if not repo.source_repository]
            if repos_without_source:
                logger.warn('The following repositories with an unknown upstream will be ignored: %s' % ', '.join(sorted(repos_without_source)))
                [repos.pop(repo_name) for repo_name in repos_without_source]
            if upstream_version_tag:
                repos_without_release = [repo_name for repo_name, repo in repos.items() if not repo.release_repository or not repo.release_repository.version]
                if repos_without_release:
                    logger.warn('The following repositories without a release will be ignored: %s' % ', '.join(sorted(repos_without_release)))
                    [repos.pop(repo_name) for repo_name in repos_without_release]
            logger.debug('Generate rosinstall entries for wet repositories: %s' % ', '.join(sorted(repos.keys())))
            wet_rosinstall_data = generate_rosinstall_for_repos(repos, version_tag=upstream_version_tag, tar=tar)
            rosinstall_data += wet_rosinstall_data
        else:
            logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
            wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
            rosinstall_data += wet_rosinstall_data
    if not wet_only and result.dry_stack_names:
        logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
        dry_distro = get_dry_distro(distro_name)
        dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
        rosinstall_data += dry_rosinstall_data
    return rosinstall_data
Ejemplo n.º 5
0
def generate_rosinstall(distro_name, names,
    from_paths=None, repo_names=None,
    deps=False, deps_up_to=None, deps_depth=None, deps_only=False,
    wet_only=False, dry_only=False, catkin_only=False, non_catkin_only=False,
    excludes=None, exclude_paths=None,
    flat=False,
    tar=False,
    upstream_version_tag=False, upstream_source_version=False):

    # classify package/stack names
    names, keywords = _split_special_keywords(names)

    # find packages recursively in include paths
    if from_paths:
        include_names_from_path = set([])
        [include_names_from_path.update(_get_package_names(from_path)) for from_path in from_paths]
        logger.debug("The following wet packages found in '--from-path' will be considered: %s" % ', '.join(sorted(include_names_from_path)))
        names.update(include_names_from_path)

    # expand repository names into package names
    repo_names, unknown_repo_names = _classify_repo_names(distro_name, repo_names)
    if unknown_repo_names:
        logger.warn('The following unknown repositories will be ignored: %s' % (', '.join(sorted(unknown_repo_names))))
    wet_package_names, unreleased_repo_names = _get_packages_for_repos(distro_name, repo_names)
    names.update(wet_package_names)
    if unreleased_repo_names and not upstream_version_tag and not upstream_source_version:
        logger.warn('The following unreleased repositories will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
    if unreleased_repo_names and (deps or deps_up_to) and (upstream_version_tag or upstream_source_version):
        logger.warn('The dependencies of the following unreleased repositories are unknown and will be ignored: %s' % ', '.join(sorted(unreleased_repo_names)))
    has_repos = ((repo_names - unreleased_repo_names) and (upstream_version_tag or upstream_source_version)) or (unreleased_repo_names and upstream_source_version)

    names, unknown_names = _classify_names(distro_name, names)
    if unknown_names:
        logger.warn('The following not released packages/stacks will be ignored: %s' % (', '.join(sorted(unknown_names))))
    if keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
        if unknown_names:
            logger.warn('The following not released packages/stacks from the %s will be ignored: %s' % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
        names.update(expanded_names)
    if not names.wet_package_names and not names.dry_stack_names and not has_repos:
        raise RuntimeError('No packages/stacks left after ignoring not released')
    if names.wet_package_names or names.dry_stack_names:
        logger.debug('Packages/stacks: %s' % ', '.join(sorted(names.wet_package_names | names.dry_stack_names)))
    if unreleased_repo_names:
        logger.debug('Unreleased repositories: %s' % ', '.join(sorted(unreleased_repo_names)))

    # classify deps-up-to
    deps_up_to_names, keywords = _split_special_keywords(deps_up_to or [])
    deps_up_to_names, unknown_names = _classify_names(distro_name, deps_up_to_names)
    if unknown_names:
        logger.warn("The following not released '--deps-up-to' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
    if keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
        if unknown_names:
            logger.warn("The following not released '--deps-up-to' packages/stacks from the %s will be ignored: %s" % (ROS_PACKAGE_PATH, ', '.join(sorted(unknown_names))))
        deps_up_to_names.update(expanded_names)
    if deps_up_to:
        logger.debug('Dependencies up to: %s' % ', '.join(sorted(deps_up_to_names.wet_package_names | deps_up_to_names.dry_stack_names)))

    # classify excludes
    exclude_names, keywords = _split_special_keywords(excludes or [])
    if exclude_paths:
        exclude_names_from_path = set([])
        [exclude_names_from_path.update(_get_package_names(exclude_path)) for exclude_path in exclude_paths]
        logger.debug("The following wet packages found in '--exclude-path' will be excluded: %s" % ', '.join(sorted(exclude_names_from_path)))
        exclude_names.update(exclude_names_from_path)
    exclude_names, unknown_names = _classify_names(distro_name, exclude_names)
    if unknown_names:
        logger.warn("The following not released '--exclude' packages/stacks will be ignored: %s" % (', '.join(sorted(unknown_names))))
    if keywords:
        expanded_names, unknown_names = _classify_names(distro_name, _expand_keywords(distro_name, keywords))
        exclude_names.update(expanded_names)
    if excludes:
        logger.debug('Excluded packages/stacks: %s' % ', '.join(sorted(exclude_names.wet_package_names | exclude_names.dry_stack_names)))

    result = copy.deepcopy(names)
    # clear wet packages if not requested
    if dry_only:
        result.wet_package_names.clear()
    # clear dry packages if not requested and no dependencies
    if wet_only and not deps and not deps_up_to:
        result.dry_stack_names.clear()

    # remove excluded names from the list of wet and dry names
    result.wet_package_names -= exclude_names.wet_package_names
    result.dry_stack_names -= exclude_names.dry_stack_names
    if not result.wet_package_names and not result.dry_stack_names and not has_repos:
        raise RuntimeError('No packages/stacks left after applying the exclusions')

    if result.wet_package_names:
        logger.debug('Wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
    if result.dry_stack_names:
        logger.debug('Dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))

    # extend the names with recursive dependencies
    if deps or deps_up_to:
        # add dry dependencies
        if result.dry_stack_names:
            dry_distro = get_dry_distro(distro_name)
            _, unreleased_stack_names = get_stack_names(dry_distro)
            excludes = exclude_names.dry_stack_names | deps_up_to_names.dry_stack_names | set(unreleased_stack_names)
            dry_dependencies, wet_dependencies = get_recursive_dependencies_of_dry(dry_distro, result.dry_stack_names, excludes=excludes)
            logger.debug('Dry stacks including dependencies: %s' % ', '.join(sorted(dry_dependencies)))
            result.dry_stack_names |= dry_dependencies

            if not dry_only:
                # add wet dependencies of dry stuff
                logger.debug('Wet dependencies of dry stacks: %s' % ', '.join(sorted(wet_dependencies)))
                for depend in wet_dependencies:
                    if depend in exclude_names.wet_package_names or depend in deps_up_to_names.wet_package_names:
                        continue
                    wet_distro = get_wet_distro(distro_name)
                    assert depend in wet_distro.release_packages, "Package '%s' does not have a version" % depend
                    result.wet_package_names.add(depend)
        # add wet dependencies
        if result.wet_package_names:
            wet_distro = get_wet_distro(distro_name)
            _, unreleased_package_names = get_package_names(wet_distro)
            excludes = exclude_names.wet_package_names | deps_up_to_names.wet_package_names | set(unreleased_package_names)
            result.wet_package_names |= get_recursive_dependencies_of_wet(wet_distro, result.wet_package_names, excludes=excludes, limit_depth=deps_depth)
            logger.debug('Wet packages including dependencies: %s' % ', '.join(sorted(result.wet_package_names)))

    # intersect result with recursive dependencies on
    if deps_up_to:
        # intersect with wet dependencies on
        if deps_up_to_names.wet_package_names:
            wet_distro = get_wet_distro(distro_name)
            # wet depends on do not include the names since they are excluded to stop recursion asap
            wet_package_names = get_recursive_dependencies_on_of_wet(wet_distro, deps_up_to_names.wet_package_names, excludes=names.wet_package_names, limit=result.wet_package_names)
            # keep all names which are already in the result set
            wet_package_names |= result.wet_package_names & names.wet_package_names
            result.wet_package_names = wet_package_names
        else:
            result.wet_package_names.clear()
        logger.debug('Wet packages after intersection: %s' % ', '.join(sorted(result.wet_package_names)))

        # intersect with dry dependencies on
        dry_dependency_names = result.wet_package_names | deps_up_to_names.dry_stack_names
        if dry_dependency_names and not wet_only:
            dry_distro = get_dry_distro(distro_name)
            # dry depends on do not include the names since they are excluded to stop recursion asap
            dry_stack_names = get_recursive_dependencies_on_of_dry(dry_distro, dry_dependency_names, excludes=names.dry_stack_names, limit=result.dry_stack_names)
            # keep all names which are already in the result set
            dry_stack_names |= result.dry_stack_names & names.dry_stack_names
            result.dry_stack_names = dry_stack_names
        else:
            result.dry_stack_names.clear()
        logger.debug('Dry stacks after intersection: %s' % ', '.join(sorted(result.dry_stack_names)))

    # exclude passed in names
    if deps_only:
        result.wet_package_names -= set(names.wet_package_names)
        result.dry_stack_names -= set(names.dry_stack_names)

    # exclude wet packages based on build type
    if catkin_only or non_catkin_only:
        wet_distro = get_wet_distro(distro_name)
        for pkg_name in list(result.wet_package_names):
            pkg_xml = wet_distro.get_release_package_xml(pkg_name)
            try:
                pkg = parse_package_string(pkg_xml)
            except InvalidPackage as e:
                logger.warn("The package '%s' has an invalid manifest and will be ignored: %s" % (pkg_name, e))
                result.wet_package_names.remove(pkg_name)
                continue
            build_type = ([e.content for e in pkg.exports if e.tagname == 'build_type'][0]) if 'build_type' in [e.tagname for e in pkg.exports] else 'catkin'
            if catkin_only ^ (build_type == 'catkin'):
                result.wet_package_names.remove(pkg_name)

    # get wet and/or dry rosinstall data
    rosinstall_data = []
    if not dry_only and (result.wet_package_names or has_repos):
        wet_distro = get_wet_distro(distro_name)
        if upstream_version_tag or upstream_source_version:
            # determine repositories based on package names and passed in repository names
            repos = {}
            for pkg_name in result.wet_package_names:
                pkg = wet_distro.release_packages[pkg_name]
                if pkg.repository_name not in repos:
                    repo = wet_distro.repositories[pkg.repository_name]
                    release_repo = repo.release_repository
                    assert not upstream_version_tag or release_repo.version is not None, "Package '%s' in repository '%s' does not have a release version" % (pkg_name, pkg.repository_name)
                    repos[pkg.repository_name] = repo
            for repo_name in repo_names:
                if repo_name not in repos:
                    repos[repo_name] = wet_distro.repositories[repo_name]
            # ignore repos which lack information
            repos_without_source = [repo_name for repo_name, repo in repos.items() if not repo.source_repository]
            if repos_without_source:
                logger.warn('The following repositories with an unknown upstream will be ignored: %s' % ', '.join(sorted(repos_without_source)))
                [repos.pop(repo_name) for repo_name in repos_without_source]
            if upstream_version_tag:
                repos_without_release = [repo_name for repo_name, repo in repos.items() if not repo.release_repository or not repo.release_repository.version]
                if repos_without_release:
                    logger.warn('The following repositories without a release will be ignored: %s' % ', '.join(sorted(repos_without_release)))
                    [repos.pop(repo_name) for repo_name in repos_without_release]
            logger.debug('Generate rosinstall entries for wet repositories: %s' % ', '.join(sorted(repos.keys())))
            wet_rosinstall_data = generate_rosinstall_for_repos(repos, version_tag=upstream_version_tag, tar=tar)
            rosinstall_data += wet_rosinstall_data
        else:
            logger.debug('Generate rosinstall entries for wet packages: %s' % ', '.join(sorted(result.wet_package_names)))
            wet_rosinstall_data = generate_wet_rosinstall(wet_distro, result.wet_package_names, flat=flat, tar=tar)
            rosinstall_data += wet_rosinstall_data
    if not wet_only and result.dry_stack_names:
        logger.debug('Generate rosinstall entries for dry stacks: %s' % ', '.join(sorted(result.dry_stack_names)))
        dry_distro = get_dry_distro(distro_name)
        dry_rosinstall_data = generate_dry_rosinstall(dry_distro, result.dry_stack_names)
        rosinstall_data += dry_rosinstall_data
    return rosinstall_data