Пример #1
0
def get_recursive_dependencies_on(distro,
                                  package_names,
                                  excludes=None,
                                  limit=None):
    excludes = set(excludes or [])
    limit = set(limit or [])

    # to improve performance limit search space if possible
    if limit:
        released_names, _ = get_package_names(distro)
        excludes.update(set(released_names) - limit - set(package_names))

    dependencies = set([])
    walker = DependencyWalker(distro)
    # redirect all stderr output to logger
    stderr = sys.stderr
    sys.stderr = CustomLogger()
    try:
        for pkg_name in package_names:
            dependencies |= walker.get_recursive_depends_on(
                pkg_name, ['buildtool', 'build', 'run', 'test'],
                ignore_pkgs=dependencies | excludes)
    finally:
        sys.stderr = stderr
    dependencies -= set(package_names)
    return dependencies
Пример #2
0
def get_recursive_dependencies_on(distro, package_names, excludes=None, limit=None):
    excludes = set(excludes or [])
    limit = set(limit or [])

    # to improve performance limit search space if possible
    if limit:
        released_names, _ = get_package_names(distro)
        excludes.update(set(released_names) - limit - set(package_names))

    dependencies = set([])
    walker = DependencyWalker(distro)
    # redirect all stderr output to logger
    stderr = sys.stderr
    sys.stderr = CustomLogger()
    try:
        for pkg_name in package_names:
            dependencies |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build', 'run', 'test'], ignore_pkgs=dependencies | excludes)
    finally:
        sys.stderr = stderr
    dependencies -= set(package_names)
    return dependencies
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on,
                       repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
                       sudo=False, no_chroot=False):
    from catkin_pkg.package import InvalidPackage, parse_package_string
    from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
    from rosdistro.dependency_walker import DependencyWalker
    from rosdistro.manifest_provider import get_release_tag

    index = get_index(get_index_url())
    print "Parsing rosdistro file for %s" % ros_distro
    release = get_cached_release(index, ros_distro)
    print "Parsing devel file for %s" % ros_distro
    source_file = get_source_file(index, ros_distro)

    # Create rosdep object
    print "Create rosdep object"
    rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)

    # download the repo_list from source
    print "Creating rosinstall file for repo list"
    rosinstall = ""
    for repo_name, version in zip(repo_list, version_list):
        if version == 'devel':
            if repo_name not in source_file.repositories:
                raise BuildException("Repository %s does not exist in Devel Distro" % repo_name)
            print "Using devel distro file to download repositories"
            rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name])
        else:
            if repo_name not in release.repositories:
                raise BuildException("Repository %s does not exist in Ros Distro" % repo_name)
            repo = release.repositories[repo_name]
            if version not in ['latest', 'master']:
                assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
            assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
            for pkg_name in repo.package_names:
                release_tag = get_release_tag(repo, pkg_name)
                if version in ['latest', 'master']:
                    release_tag = '/'.join(release_tag.split('/')[:-1])
                print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name)
                rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag)
    print "rosinstall file for all repositories: \n %s" % rosinstall
    with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
        f.write(rosinstall)
    print "Install repo list from source"
    os.makedirs(repo_sourcespace)
    call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))

    # get the repositories build dependencies
    print "Get build dependencies of repo list"
    repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)
    # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
    if 'catkin' not in repo_build_dependencies:
        repo_build_dependencies.append('catkin')
    print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
    apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file for repositories that use catkin
    print "Removing the CMakeLists.txt file generated by rosinstall"
    os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
    print "Create a new CMakeLists.txt file using catkin"

    # get environment
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)

    # check if source workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)

    # make build folder and change into it
    os.makedirs(repo_buildspace)
    os.chdir(repo_buildspace)

    # make test results dir
    test_results_dir = os.path.join(workspace, 'test_results')
    if os.path.exists(test_results_dir):
        shutil.rmtree(test_results_dir)
    os.makedirs(test_results_dir)

    if not non_catkin_pkgs:
        print "Build catkin workspace"
        call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
        repos_test_results_dir = os.path.join(test_results_dir, 'repos')
        call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
        #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories and tests
        print "Build repo list"
        call("make", ros_env)
        call("make tests", ros_env)

        # get the repositories test and run dependencies
        print "Get test and run dependencies of repo list"
        repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True)
        print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies))
        apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)

        # run tests
        print "Test repo list"
        call("make run_tests", ros_env)

    else:
        print "Build workspace with non-catkin packages in isolation"
        # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
        os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
        call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env)
        setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh')
        ros_env = get_ros_env(setup_file)

    # don't do depends-on on things not in release
    not_in_release = set(repo_list) - set(release.repositories.keys())
    if not_in_release:
        print "Removed [%s] repositories which are not in the " %\
            ', '.join(sorted(not_in_release)), \
            "release file for depends-on testing"
        repo_list = list(set(repo_list) - not_in_release)

    # see if we need to do more work or not
    if not test_depends_on:
        print "We're not testing the depends-on repositories"
        ensure_test_results(test_results_dir)
        return

    # get repo_list depends-on list
    print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list)
    walker = DependencyWalker(release)
    depends_on = set([])
    try:
        for repo_name in repo_list:
            print('repo_name', repo_name)
            repo = release.repositories[repo_name]
            for pkg_name in repo.package_names:
                print('pkg_name', pkg_name)
                depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on)
                print('depends_on', depends_on)
    except RuntimeError:
        print "Exception %s: If you are not in the rosdistro and only in the devel", \
            " builds there will be no depends on"
        depends_on = set([])

    print "Build depends_on list of pkg list: %s" % (', '.join(depends_on))
    if len(depends_on) == 0:
        print "No wet packages depend on our repo list. Test finished here"
        ensure_test_results(test_results_dir)
        return

    # install depends_on packages from source from release repositories
    rosinstall = ''
    non_catkin_pkgs = []
    for pkg_name in depends_on:
        repo = release.repositories[release.packages[pkg_name].repository_name]
        if repo.version is None:
            continue
        pkg_xml = release.get_package_xml(pkg_name)
        if pkg_xml is None:
            raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name)
        try:
            pkg = parse_package_string(pkg_xml)
        except InvalidPackage as e:
            raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e))
        if _is_non_catkin_package(pkg):
            non_catkin_pkgs.append(pkg.name)
        rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name)

    if non_catkin_pkgs:
        print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
        create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
        return

    print "Rosinstall for depends_on:\n %s" % rosinstall
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print "Created rosinstall file for depends on"

    # install all repository and system dependencies of the depends_on list
    print "Install all depends_on from source: %s" % (', '.join(depends_on))
    os.makedirs(dependson_sourcespace)
    call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))

    # check if depends_on workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace)
    if non_catkin_pkgs:
        print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
        create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
        return

    # get build and test dependencies of depends_on list
    dependson_build_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False):
        print "  Checking dependency %s" % d
        if d in dependson_build_dependencies:
            print "    Already in dependson_build_dependencies"
        if d in depends_on:
            print "    Is a direct dependency of the repo list, and is installed from source"
        if d in repo_list:
            print "    Is one of the repositories tested"
        if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
            dependson_build_dependencies.append(d)
    print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))
    dependson_test_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True):
        if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
            dependson_test_dependencies.append(d)
    print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))

    # install build dependencies
    print "Install all build dependencies of the depends_on list"
    apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file again
    print "Removing the CMakeLists.txt file generated by rosinstall"
    os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
    os.makedirs(dependson_buildspace)
    os.chdir(dependson_buildspace)
    print "Create a new CMakeLists.txt file using catkin"
    call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
    depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
    call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
    #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))

    # build repositories
    print "Build depends-on packages"
    call("make", ros_env)

    # install test dependencies
    print "Install all test dependencies of the depends_on list"
    apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)

    # test repositories
    print "Test depends-on packages"
    call("make run_tests", ros_env)
    ensure_test_results(test_results_dir)
Пример #4
0
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on,
                       repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
                       sudo=False, no_chroot=False):
    from catkin_pkg.package import InvalidPackage, parse_package_string
    from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
    from rosdistro.dependency_walker import DependencyWalker
    from rosdistro.manifest_provider import get_release_tag

    index = get_index(get_index_url())
    print "Parsing rosdistro file for %s" % ros_distro
    release = get_cached_release(index, ros_distro)
    print "Parsing devel file for %s" % ros_distro
    source_file = get_source_file(index, ros_distro)

    # Create rosdep object
    print "Create rosdep object"
    rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)

    # download the repo_list from source
    print "Creating rosinstall file for repo list"
    rosinstall = ""
    for repo_name, version in zip(repo_list, version_list):
        if version == 'devel':
            if repo_name not in source_file.repositories:
                raise BuildException("Repository %s does not exist in Devel Distro" % repo_name)
            print "Using devel distro file to download repositories"
            rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name])
        else:
            if repo_name not in release.repositories:
                raise BuildException("Repository %s does not exist in Ros Distro" % repo_name)
            repo = release.repositories[repo_name]
            if version not in ['latest', 'master']:
                assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
            assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
            for pkg_name in repo.package_names:
                release_tag = get_release_tag(repo, pkg_name)
                if version in ['latest', 'master']:
                    release_tag = '/'.join(release_tag.split('/')[:-1])
                print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name)
                rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag)
    print "rosinstall file for all repositories: \n %s" % rosinstall
    with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
        f.write(rosinstall)
    print "Install repo list from source"
    os.makedirs(repo_sourcespace)
    call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))

    # get the repositories build dependencies
    print "Get build dependencies of repo list"
    repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)
    # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
    if 'catkin' not in repo_build_dependencies:
        repo_build_dependencies.append('catkin')
    print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
    apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file for repositories that use catkin
    print "Removing the CMakeLists.txt file generated by rosinstall"
    os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
    print "Create a new CMakeLists.txt file using catkin"

    # get environment
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)

    # check if source workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)

    # make build folder and change into it
    os.makedirs(repo_buildspace)
    os.chdir(repo_buildspace)

    # make test results dir
    test_results_dir = os.path.join(workspace, 'test_results')
    if os.path.exists(test_results_dir):
        shutil.rmtree(test_results_dir)
    os.makedirs(test_results_dir)

    if not non_catkin_pkgs:
        print "Build catkin workspace"
        call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
        repos_test_results_dir = os.path.join(test_results_dir, 'repos')
        call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
        #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories and tests
        print "Build repo list"
        call("make", ros_env)
        call("make tests", ros_env)

        # get the repositories test and run dependencies
        print "Get test and run dependencies of repo list"
        repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True)
        print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies))
        apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)

        # run tests
        print "Test repo list"
        call("make run_tests", ros_env)

    else:
        print "Build workspace with non-catkin packages in isolation"
        # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
        os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
        call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env)
        setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh')
        ros_env = get_ros_env(setup_file)

    # see if we need to do more work or not
    if not test_depends_on:
        print "We're not testing the depends-on repositories"
        ensure_test_results(test_results_dir)
        return

    # get repo_list depends-on list
    print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list)
    walker = DependencyWalker(release)
    depends_on = set([])
    try:
        for repo_name in repo_list:
            print('repo_name', repo_name)
            repo = release.repositories[repo_name]
            for pkg_name in repo.package_names:
                print('pkg_name', pkg_name)
                depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on)
                print('depends_on', depends_on)
    except RuntimeError:
        print "Exception %s: If you are not in the rosdistro and only in the devel", \
            " builds there will be no depends on"
        depends_on = set([])

    print "Build depends_on list of pkg list: %s" % (', '.join(depends_on))
    if len(depends_on) == 0:
        print "No wet packages depend on our repo list. Test finished here"
        ensure_test_results(test_results_dir)
        return

    # install depends_on packages from source from release repositories
    rosinstall = ''
    non_catkin_pkgs = []
    for pkg_name in depends_on:
        repo = release.repositories[release.packages[pkg_name].repository_name]
        if repo.version is None:
            continue
        pkg_xml = release.get_package_xml(pkg_name)
        if pkg_xml is None:
            raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name)
        try:
            pkg = parse_package_string(pkg_xml)
        except InvalidPackage as e:
            raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e))
        if _is_non_catkin_package(pkg):
            non_catkin_pkgs.append(pkg.name)
        rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name)

    if non_catkin_pkgs:
        print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
        create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
        return

    print "Rosinstall for depends_on:\n %s" % rosinstall
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print "Created rosinstall file for depends on"

    # install all repository and system dependencies of the depends_on list
    print "Install all depends_on from source: %s" % (', '.join(depends_on))
    os.makedirs(dependson_sourcespace)
    call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))

    # check if depends_on workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace)
    if non_catkin_pkgs:
        print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
        create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
        return

    # get build and test dependencies of depends_on list
    dependson_build_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False):
        print "  Checking dependency %s" % d
        if d in dependson_build_dependencies:
            print "    Already in dependson_build_dependencies"
        if d in depends_on:
            print "    Is a direct dependency of the repo list, and is installed from source"
        if d in repo_list:
            print "    Is one of the repositories tested"
        if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
            dependson_build_dependencies.append(d)
    print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))
    dependson_test_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True):
        if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
            dependson_test_dependencies.append(d)
    print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))

    # install build dependencies
    print "Install all build dependencies of the depends_on list"
    apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file again
    print "Removing the CMakeLists.txt file generated by rosinstall"
    os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
    os.makedirs(dependson_buildspace)
    os.chdir(dependson_buildspace)
    print "Create a new CMakeLists.txt file using catkin"
    call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
    depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
    call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
    #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))

    # build repositories
    print "Build depends-on packages"
    call("make", ros_env)

    # install test dependencies
    print "Install all test dependencies of the depends_on list"
    apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)

    # test repositories
    print "Test depends-on packages"
    call("make run_tests", ros_env)
    ensure_test_results(test_results_dir)