def build_repo_messages(catkin_packages, docspace, ros_distro):
    #we'll replace cmake files with our own since we only want to do message generation
    replace_catkin_cmake_files(catkin_packages)
    build_errors = []
    #For groovy, this isn't too bad, we just set up a workspace
    old_dir = os.getcwd()
    repo_devel = os.path.join(docspace, 'build_repo')
    if not os.path.exists(repo_devel):
        os.makedirs(repo_devel)
    os.chdir(repo_devel)
    print "Removing the CMakeLists.txt file generated by rosinstall"
    os.remove(os.path.join(docspace, 'CMakeLists.txt'))
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' %ros_distro)
    print "Calling cmake..."
    call("catkin_init_workspace %s"%docspace, ros_env)
    try:
        call("cmake ..", ros_env)
        ros_env = get_ros_env(os.path.join(repo_devel, 'devel/setup.bash'))
        generate_messages_catkin(ros_env)
        source = 'source %s' % (os.path.abspath(os.path.join(repo_devel, 'devel/setup.bash')))
    except BuildException as e:
        print "FAILED TO CALL CMAKE ON CATKIN REPOS"
        print "There will be no messages in documentation and some python docs may fail"
        print "Exception: %s" % e
        source = ''
        build_errors.append("catkin_workspace for repository")
    os.chdir(old_dir)
    return (source, build_errors)
def build_repo_messages(catkin_packages, docspace, ros_distro):
    #we'll replace cmake files with our own since we only want to do message generation
    replace_catkin_cmake_files(catkin_packages)
    build_errors = []
    #For groovy, this isn't too bad, we just set up a workspace
    old_dir = os.getcwd()
    repo_devel = os.path.join(docspace, 'build_repo')
    if not os.path.exists(repo_devel):
        os.makedirs(repo_devel)
    os.chdir(repo_devel)
    print "Removing the CMakeLists.txt file generated by rosinstall"
    os.remove(os.path.join(docspace, 'CMakeLists.txt'))
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    print "Calling cmake..."
    call("catkin_init_workspace %s" % docspace, ros_env)
    try:
        call("cmake ..", ros_env)
        ros_env = get_ros_env(os.path.join(repo_devel, 'devel/setup.bash'))
        generate_messages_catkin(ros_env)
        source = 'source %s' % (os.path.abspath(
            os.path.join(repo_devel, 'devel/setup.bash')))
    except BuildException as e:
        print "FAILED TO CALL CMAKE ON CATKIN REPOS"
        print "There will be no messages in documentation and some python docs may fail"
        print "Exception: %s" % e
        source = ''
        build_errors.append("catkin_workspace for repository")
    os.chdir(old_dir)
    return (source, build_errors)
예제 #3
0
def build_repo_messages(catkin_packages, docspace, ros_distro,
                        local_install_path):
    #we'll replace cmake files with our own since we only want to do message generation
    has_plain_cmake = replace_catkin_cmake_files(catkin_packages)
    build_errors = []
    if has_plain_cmake:
        print('')
        print('Skip building packages since they contain plain-cmake packages')
        print('There will be no messages in documentation')
        print('')
        return '', build_errors
    #For groovy, this isn't too bad, we just set up a workspace
    old_dir = os.getcwd()
    repo_devel = os.path.join(docspace, 'build_repo')
    if not os.path.exists(repo_devel):
        os.makedirs(repo_devel)
    os.chdir(repo_devel)
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(docspace, 'CMakeLists.txt'))
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    print("Calling cmake...")
    call("catkin_init_workspace %s" % docspace, ros_env)
    try:
        call("cmake -DCMAKE_INSTALL_PREFIX:PATH=%s .." % local_install_path,
             ros_env)
        ros_env = get_ros_env(os.path.join(repo_devel, 'devel/setup.bash'))
        generate_messages_catkin(ros_env)
        try:
            call("make install", ros_env)
        except BuildException as e:
            print("FAILED TO CALL MAKE INSTALL ON CATKIN REPOS")
            print("Exception: %s" % e)
            sys.exit(1)
        source = 'source %s' % (os.path.abspath(
            os.path.join(repo_devel, 'devel/setup.bash')))
    except BuildException as e:
        print("FAILED TO CALL CMAKE ON CATKIN REPOS")
        print(
            "There will be no messages in documentation and some python docs may fail"
        )
        print("Exception: %s" % e)
        source = ''
        build_errors.append("catkin_workspace for repository")
    os.chdir(old_dir)
    return (source, build_errors)
def build_repo_messages(catkin_packages, docspace, ros_distro, local_install_path):
    #we'll replace cmake files with our own since we only want to do message generation
    has_plain_cmake = replace_catkin_cmake_files(catkin_packages)
    build_errors = []
    if has_plain_cmake:
        print('')
        print('Skip building packages since they contain plain-cmake packages')
        print('There will be no messages in documentation')
        print('')
        return '', build_errors
    #For groovy, this isn't too bad, we just set up a workspace
    old_dir = os.getcwd()
    repo_devel = os.path.join(docspace, 'build_repo')
    if not os.path.exists(repo_devel):
        os.makedirs(repo_devel)
    os.chdir(repo_devel)
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(docspace, 'CMakeLists.txt'))
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    print("Calling cmake...")
    call("catkin_init_workspace %s" % docspace, ros_env)
    try:
        call("cmake -DCMAKE_INSTALL_PREFIX:PATH=%s .." % local_install_path, ros_env)
        ros_env = get_ros_env(os.path.join(repo_devel, 'devel/setup.bash'))
        generate_messages_catkin(ros_env)
        try:
            call("make install", ros_env)
        except BuildException as e:
            print("FAILED TO CALL MAKE INSTALL ON CATKIN REPOS")
            print("Exception: %s" % e)
            sys.exit(1)
        source = 'source %s' % (os.path.abspath(os.path.join(repo_devel, 'devel/setup.bash')))
    except BuildException as e:
        print("FAILED TO CALL CMAKE ON CATKIN REPOS")
        print("There will be no messages in documentation and some python docs may fail")
        print("Exception: %s" % e)
        source = ''
        build_errors.append("catkin_workspace for repository")
    os.chdir(old_dir)
    return (source, build_errors)
예제 #5
0
def build_repo_messages_catkin_stacks(stacks, ros_distro, local_install_path):
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)

    #Make sure to create the local install path if it doesn't exist
    if os.path.exists(local_install_path):
        shutil.rmtree(local_install_path)
    os.makedirs(local_install_path)
    os.makedirs(os.path.join(local_install_path,
                             'lib/python2.7/dist-packages'))
    os.makedirs(os.path.join(local_install_path,
                             'lib/python2.6/dist-packages'))
    os.makedirs(os.path.join(local_install_path, 'share'))
    os.makedirs(os.path.join(local_install_path, 'bin'))
    build_errors = []

    for stack, path in stacks.iteritems():
        #check to see if the stack is catkin or not
        cmake_file = os.path.join(path, 'CMakeLists.txt')

        catkin = False
        #If a CMakeLists.txt file doesn't exist, we assume the stack is not catkinized
        if os.path.isfile(cmake_file):
            with open(cmake_file, 'r') as f:
                read_file = f.read()
                if 'catkin_stack' in read_file:
                    catkin = True

        #if the stack is a catkin stack, we want to build and install it locally
        if catkin:
            old_dir = os.getcwd()
            os.chdir(path)
            if not os.path.exists('build'):
                os.makedirs('build')
            os.chdir('build')
            try:
                call(
                    "cmake -DCMAKE_INSTALL_PREFIX:PATH=%s .." %
                    local_install_path, ros_env)
                call("make", ros_env)
                call("make install", ros_env)
            except BuildException as e:
                print "FAILED TO BUILD %s, messages for this package cannot be generated." % (
                    stack)
                print "Failure on %s, with env path %s" % (stack, ros_env)
                print "Exception: %s" % e
                build_errors.append(stack)
            os.chdir(old_dir)

    #We'll throw the appropriate stuff on our python path
    export = "export PYTHONPATH=%s/lib/python2.7/dist-packages:$PYTHONPATH" % local_install_path
    return (export, build_errors)
def build_repo_messages_catkin_stacks(stacks, ros_distro, local_install_path):
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' %ros_distro)

    #Make sure to create the local install path if it doesn't exist
    if os.path.exists(local_install_path):
        shutil.rmtree(local_install_path)
    os.makedirs(local_install_path)
    os.makedirs(os.path.join(local_install_path, 'lib/python2.7/dist-packages'))
    os.makedirs(os.path.join(local_install_path, 'lib/python2.6/dist-packages'))
    os.makedirs(os.path.join(local_install_path, 'share'))
    os.makedirs(os.path.join(local_install_path, 'bin'))
    build_errors = []

    for stack, path in stacks.iteritems():
        #check to see if the stack is catkin or not
        cmake_file = os.path.join(path, 'CMakeLists.txt')

        catkin = False
        #If a CMakeLists.txt file doesn't exist, we assume the stack is not catkinized
        if os.path.isfile(cmake_file):
            with open(cmake_file, 'r') as f:
                read_file = f.read()
                if 'catkin_stack' in read_file:
                    catkin = True

        #if the stack is a catkin stack, we want to build and install it locally
        if catkin:
            old_dir = os.getcwd()
            os.chdir(path)
            if not os.path.exists('build'):
                os.makedirs('build')
            os.chdir('build')
            try:
                call("cmake -DCMAKE_INSTALL_PREFIX:PATH=%s .." % local_install_path, ros_env)
                call("make", ros_env)
                call("make install", ros_env)
            except BuildException as e:
                print "FAILED TO BUILD %s, messages for this package cannot be generated." % (stack)
                print "Failure on %s, with env path %s" % (stack, ros_env)
                print "Exception: %s" % e
                build_errors.append(stack)
            os.chdir(old_dir)

    #We'll throw the appropriate stuff on our python path
    export = "export PYTHONPATH=%s/lib/python2.7/dist-packages:$PYTHONPATH" % local_install_path
    return (export, build_errors)
def build_repo_messages_manifest(manifest_packages, build_order, ros_distro):
    #Now, we go through all of our manifest packages and try to generate
    #messages, or add them to the pythonpath if they turn out to be catkin
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' %ros_distro)
    path_string = ''
    build_errors = []

    #Make sure to build with our special cmake file to only do message generation
    replace_manifest_cmake_files(manifest_packages)

    #Make sure to build in dependency order
    for name in build_order:
        if not name in manifest_packages or name in ['rosdoc_lite', 'catkin']:
            continue

        path = manifest_packages[name]

        cmake_file = os.path.join(path, 'CMakeLists.txt')
        if os.path.isfile(cmake_file):
            catkin = False
            messages = False
            services = False
            #Check to see whether the package is catkin or not
            #Also check whether we actually need to build messages
            #and services since rosbuild creates the build targets
            #no matter what
            with open(cmake_file, 'r') as f:
                read_file = f.read()
                if 'catkin_project' in read_file:
                    catkin = True
                if 'rosbuild_genmsg' in read_file:
                    messages = True
                if 'rosbuild_gensrv' in read_file:
                    services = True

            #If it is catkin, then we'll do our best to put the right things on the python path
            #TODO: Note that this will not generate messages, we can try to put this in later
            #but fuerte catkin makes it kind of hard to do correctly
            if catkin:
                print "Not doing anything for catkin package"
                #print "Creating an export line that guesses the appropriate python paths for each package"
                #print "WARNING: This will not properly generate message files within this repo for python documentation."
                #if os.path.isdir(os.path.join(path, 'src')):
                #    path_string = "%s:%s" %(os.path.join(path, 'src'), path_string)

            #If it's not catkin, then we'll generate python messages
            else:
                old_dir = os.getcwd()
                os.chdir(path)
                if not os.path.exists('build'):
                    os.makedirs('build')
                os.chdir('build')
                ros_env['ROS_PACKAGE_PATH'] = '%s:%s' % (path, ros_env['ROS_PACKAGE_PATH'])
                try:
                    call("cmake ..", ros_env)
                    generate_messages_dry(ros_env, name, messages, services)
                except BuildException as e:
                    print "FAILED TO CALL CMAKE ON %s, messages for this package cannot be generated." % (name)
                    print "Are you sure that the package specifies its dependencies correctly?"
                    print "Failure on %s, with env path %s" % (name, ros_env)
                    print "Exception: %s" % e
                    build_errors.append(name)
                os.chdir(old_dir)
        else:
            #If the package does not have a CmakeLists.txt file, we still want
            #to add it to our package path because other packages may depend on it
            ros_env['ROS_PACKAGE_PATH'] = '%s:%s' % (path, ros_env['ROS_PACKAGE_PATH'])

    if path_string:
        return ("export PYTHONPATH=%s:$PYTHONPATH" % path_string, build_errors)

    return ("export PYTHONPATH=$PYTHONPATH", build_errors)
예제 #8
0
def build_repo_messages_manifest(manifest_packages, build_order, ros_distro):
    #Now, we go through all of our manifest packages and try to generate
    #messages, or add them to the pythonpath if they turn out to be catkin
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    path_string = ''
    build_errors = []

    #Make sure to build with our special cmake file to only do message generation
    replace_manifest_cmake_files(manifest_packages)

    #Make sure to build in dependency order
    for name in build_order:
        if not name in manifest_packages or name in ['rosdoc_lite', 'catkin']:
            continue

        path = manifest_packages[name]

        cmake_file = os.path.join(path, 'CMakeLists.txt')
        if os.path.isfile(cmake_file):
            catkin = False
            messages = False
            services = False
            #Check to see whether the package is catkin or not
            #Also check whether we actually need to build messages
            #and services since rosbuild creates the build targets
            #no matter what
            with open(cmake_file, 'r') as f:
                read_file = f.read()
                if 'catkin_project' in read_file:
                    catkin = True
                if 'rosbuild_genmsg' in read_file:
                    messages = True
                if 'rosbuild_gensrv' in read_file:
                    services = True

            #If it is catkin, then we'll do our best to put the right things on the python path
            #TODO: Note that this will not generate messages, we can try to put this in later
            #but fuerte catkin makes it kind of hard to do correctly
            if catkin:
                print("Not doing anything for catkin package")
                #print "Creating an export line that guesses the appropriate python paths for each package"
                #print "WARNING: This will not properly generate message files within this repo for python documentation."
                #if os.path.isdir(os.path.join(path, 'src')):
                #    path_string = "%s:%s" %(os.path.join(path, 'src'), path_string)

            #If it's not catkin, then we'll generate python messages
            else:
                old_dir = os.getcwd()
                os.chdir(path)
                if not os.path.exists('build'):
                    os.makedirs('build')
                os.chdir('build')
                ros_env['ROS_PACKAGE_PATH'] = '%s:%s' % (
                    path, ros_env['ROS_PACKAGE_PATH'])
                try:
                    call("cmake ..", ros_env)
                    generate_messages_dry(ros_env, name, messages, services)
                except BuildException as e:
                    print(
                        "FAILED TO CALL CMAKE ON %s, messages for this package cannot be generated."
                        % (name))
                    print(
                        "Are you sure that the package specifies its dependencies correctly?"
                    )
                    print("Failure on %s, with env path %s" % (name, ros_env))
                    print("Exception: %s" % e)
                    build_errors.append(name)
                os.chdir(old_dir)
        else:
            #If the package does not have a CmakeLists.txt file, we still want
            #to add it to our package path because other packages may depend on it
            ros_env['ROS_PACKAGE_PATH'] = '%s:%s' % (
                path, ros_env['ROS_PACKAGE_PATH'])

    if path_string:
        return ("export PYTHONPATH=%s:$PYTHONPATH" % path_string, build_errors)

    return ("export PYTHONPATH=$PYTHONPATH", build_errors)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on,
                       repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
                       sudo=False, no_chroot=False):
    append_pymodules_if_needed()
    from catkin_pkg.package import InvalidPackage, parse_package_string
    from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
    from rosdistro.dependency_walker import DependencyWalker
    from rosdistro.manifest_provider import get_release_tag

    index = get_index(get_index_url())
    print("Parsing rosdistro file for %s" % ros_distro)
    release = get_cached_release(index, ros_distro)
    print("Parsing devel file for %s" % ros_distro)
    source_file = get_source_file(index, ros_distro)

    # Create rosdep object
    print("Create rosdep object")
    strands_rosdep = "https://raw.githubusercontent.com/strands-project/rosdistro/strands-devel/rosdep/sources.list.d/50-strands.list"
    strands_rosdep_list_file = "/etc/ros/rosdep/sources.list.d/50-strands.list"

    additional_rosdeps={strands_rosdep_list_file: strands_rosdep}

    rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot, additional_rosdeps=additional_rosdeps)

    if repo_list:
        # download the repo_list from source
        print("Creating rosinstall file for repo list")
        rosinstall = ""
        for repo_name, version in zip(repo_list, version_list):
            if version == 'devel':
                if repo_name not in source_file.repositories:
                    raise BuildException("Repository %s does not exist in Devel Distro" % repo_name)
                print("Using devel distro file to download repositories")
                rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name])
            else:
                if repo_name not in release.repositories:
                    raise BuildException("Repository %s does not exist in Ros Distro" % repo_name)
                repo = release.repositories[repo_name]
                if version not in ['latest', 'master']:
                    assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
                assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
                for pkg_name in repo.package_names:
                    release_tag = get_release_tag(repo, pkg_name)
                    if version in ['latest', 'master']:
                        release_tag = '/'.join(release_tag.split('/')[:-1])
                    print('Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name))
                    rosinstall += _generate_rosinstall_for_pkg_version(release.repositories[repo_name], pkg_name, release_tag)
        print("rosinstall file for all repositories: \n %s" % rosinstall)
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print("Install repo list from source")
        os.makedirs(repo_sourcespace)
        call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))

    extract_notification_recipients(repo_sourcespace)

    # get the repositories build dependencies
    print("Get build dependencies of repo list")
    repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, run_depends=True)
    # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
    if 'catkin' not in repo_build_dependencies:
        repo_build_dependencies.append('catkin')
    print("Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)))
    apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file for repositories that use catkin
    root_cmakelists = os.path.join(repo_sourcespace, 'CMakeLists.txt')
    if os.path.exists(root_cmakelists):
        print("Removing the CMakeLists.txt file generated by rosinstall")
        os.remove(root_cmakelists)
    print("Create a new CMakeLists.txt file using catkin")

    # get environment
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)

    # check if source workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)

    # make build folder and change into it
    os.makedirs(repo_buildspace)
    os.chdir(repo_buildspace)

    # make test results dir
    test_results_dir = os.path.join(workspace, 'test_results')
    if os.path.exists(test_results_dir):
        shutil.rmtree(test_results_dir)
    os.makedirs(test_results_dir)

    if not non_catkin_pkgs:
        print("Build catkin workspace")
        call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
        repos_test_results_dir = os.path.join(test_results_dir, 'repos')
        # set env variable to hot fix https://github.com/strands-project/strands_ci/issues/13
        ros_env['ROS_TEST_RESULTS_DIR'] = repos_test_results_dir
        call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
        #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories and tests
        print("Build repo list")
        call("make", ros_env)
        call("make install", ros_env)
        call("make tests", ros_env)

        # get the repositories test and run dependencies
        print("Get run dependencies of repo list")
        repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, run_depends=True)
        print("Install run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)))
        apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)

        # get environment after installing test and run dependencies
        ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
        # set env variable to hot fix https://github.com/strands-project/strands_ci/issues/13
        ros_env['ROS_TEST_RESULTS_DIR'] = repos_test_results_dir

        # run tests
        print("Test repo list")
        call("vglrun -d :0 make run_tests", ros_env)

        # anything after this should build on this env
        ros_env = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
        # set env variable to hot fix https://github.com/strands-project/strands_ci/issues/13
        ros_env['ROS_TEST_RESULTS_DIR'] = repos_test_results_dir

    else:
        print("Build workspace with non-catkin packages in isolation")
        # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
        os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
        call('catkin_make_isolated -C %s --source %s --install' % (repo_buildspace, repo_sourcespace), ros_env)
        setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh')
        # anything after this should build on this env
        ros_env = get_ros_env(setup_file)

    # don't do depends-on on things not in release
    not_in_release = set(repo_list) - set(release.repositories.keys())
    if not_in_release:
        print("Removed [%s] repositories which are not in the " %
              ', '.join(sorted(not_in_release)),
              "release file for depends-on testing")
        repo_list = list(set(repo_list) - not_in_release)

    # see if we need to do more work or not
    if not test_depends_on:
        print("We're not testing the depends-on repositories")
        ensure_test_results(test_results_dir)
        return

    # get repo_list depends-on list
    print("Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list))
    walker = DependencyWalker(release)
    depends_on = set([])
    try:
        for repo_name in repo_list:
            print('repo_name', repo_name)
            repo = release.repositories[repo_name]
            for pkg_name in repo.package_names:
                print('pkg_name', pkg_name)
                depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build', 'test'], ignore_pkgs=depends_on)
                print('depends_on', depends_on)
        # remove all packages which are already in the workspace
        from catkin_pkg.packages import find_packages
        pkgs = find_packages(repo_sourcespace)
        depends_on -= set([pkg.name for pkg in pkgs.values()])
    except RuntimeError:
        print("Exception %s: If you are not in the rosdistro and only in the devel",
              " builds there will be no depends on")
        depends_on = set([])

    print("Build depends_on list of pkg list: %s" % (', '.join(depends_on)))
    if len(depends_on) == 0:
        print("No wet packages depend on our repo list. Test finished here")
        ensure_test_results(test_results_dir)
        return

    # install depends_on packages from source from release repositories
    rosinstall = ''
    non_catkin_pkgs = []
    for pkg_name in depends_on:
        repo = release.repositories[release.packages[pkg_name].repository_name]
        if repo.version is None:
            continue
        pkg_xml = release.get_package_xml(pkg_name)
        if pkg_xml is None:
            raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name)
        try:
            pkg = parse_package_string(pkg_xml)
        except InvalidPackage as e:
            raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e))
        if _is_non_catkin_package(pkg):
            non_catkin_pkgs.append(pkg.name)
        rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name)

    if non_catkin_pkgs:
        print('Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)))
        create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
        return

    print("Rosinstall for depends_on:\n %s" % rosinstall)
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print("Created rosinstall file for depends on")

    # install all repository and system dependencies of the depends_on list
    print("Install all depends_on from source: %s" % (', '.join(depends_on)))
    os.makedirs(dependson_sourcespace)
    call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))

    # check if depends_on workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace)
    if non_catkin_pkgs:
        print('Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)))
        create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
        return

    # get build and run dependencies of depends_on list
    dependson_build_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=True, run_depends=False):
        print("  Checking dependency %s" % d)
        if d in dependson_build_dependencies:
            print("    Already in dependson_build_dependencies")
        if d in depends_on:
            print("    Is a direct dependency of the repo list, and is installed from source")
        if d in repo_list:
            print("    Is one of the repositories tested")
        if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
            dependson_build_dependencies.append(d)
    print("Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)))
    dependson_test_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=False, run_depends=True):
        if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
            dependson_test_dependencies.append(d)
    print("Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)))

    # install build dependencies
    print("Install all build dependencies of the depends_on list")
    apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file again
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
    os.makedirs(dependson_buildspace)
    os.chdir(dependson_buildspace)
    print("Create a new CMakeLists.txt file using catkin")
    call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
    depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
    # set env variable to hot fix https://github.com/strands-project/strands_ci/issues/13
    ros_env['ROS_TEST_RESULTS_DIR'] = depends_on_test_results_dir
    call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
    #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))

    # build repositories
    print("Build depends-on packages")
    call("make", ros_env)

    # install test dependencies
    print("Install all test dependencies of the depends_on list")
    apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)

    # test repositories
    print("Test depends-on packages")
    call("make run_tests", ros_env)
    ensure_test_results(test_results_dir)
def _test_repositories_fuerte(ros_distro, repo_list, version_list, workspace, test_depends_on,
                              repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
                              sudo=False, no_chroot=False):
    import rosdistro

    # parse the rosdistro file
    print("Parsing rosdistro file for %s" % ros_distro)
    distro = rosdistro.RosDistro(ros_distro)
    print("Parsing devel file for %s" % ros_distro)
    devel = rosdistro.DevelDistro(ros_distro)

    # Create rosdep object
    print("Create rosdep object")
    rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)

    # download the repo_list from source
    print("Creating rosinstall file for repo list")
    rosinstall = ""
    for repo, version in zip(repo_list, version_list):
        if version == 'devel':
            if repo not in devel.repositories:
                raise BuildException("Repository %s does not exist in Devel Distro" % repo)
            print("Using devel distro file to download repositories")
            rosinstall += devel.repositories[repo].get_rosinstall()
        else:
            if repo not in distro.get_repositories():
                raise BuildException("Repository %s does not exist in Ros Distro" % repo)
            if version in ['latest', 'master']:
                print("Using latest release distro file to download repositories")
                rosinstall += distro.get_rosinstall(repo, version='master')
            else:
                print("Using version %s of release distro file to download repositories" % version)
                rosinstall += distro.get_rosinstall(repo, version)
    print("rosinstall file for all repositories: \n %s" % rosinstall)
    with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
        f.write(rosinstall)
    print("Install repo list from source")
    os.makedirs(repo_sourcespace)
    call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))

    # get the repositories build dependencies
    print("Get build dependencies of repo list")
    repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, run_depends=False)
    print("Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)))
    apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file for repositories that use catkin
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
    print("Create a new CMakeLists.txt file using catkin")
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
    test_results_dir = os.path.join(workspace, 'test_results')
    repos_test_results_dir = os.path.join(test_results_dir, 'repos')

    # set env variable to hot fix https://github.com/strands-project/strands_ci/issues/13
    ros_env['ROS_TEST_RESULTS_DIR'] = repos_test_results_dir


    os.makedirs(repo_buildspace)
    os.chdir(repo_buildspace)
    call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
    #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

    # build repositories and tests
    print("Build repo list")
    call("make", ros_env)
    call("make tests", ros_env)

    # get the repositories run dependencies
    print("Get test and run dependencies of repo list")
    repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, run_depends=True)
    print("Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)))
    apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)

    # run tests
    print("Test repo list")
    call("make run_tests", ros_env)

    # see if we need to do more work or not
    if not test_depends_on:
        print("We're not testing the depends-on repositories")
        ensure_test_results(test_results_dir)
        return

    # get repo_list depends-on list
    print("Get list of wet repositories that build-depend on repo list %s" % ', '.join(repo_list))
    depends_on = []
    try:
        for d in distro.get_depends_on(repo_list)['build'] + distro.get_depends_on(repo_list)['buildtool']:
            if not d in depends_on and not d in repo_list:
                depends_on.append(d)
    except RuntimeError:
        print("Exception %s: If you are not in the rosdistro and only in the devel",
              " builds there will be no depends on")
        depends_on = []

    print("Build depends_on list of repo list: %s" % (', '.join(depends_on)))
    if len(depends_on) == 0:
        print("No wet repositories depend on our repo list. Test finished here")
        ensure_test_results(test_results_dir)
        return

    # install depends_on repositories from source
    rosinstall = distro.get_rosinstall(depends_on)
    print("Rosinstall for depends_on:\n %s" % rosinstall)
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print("Created rosinstall file for depends on")

    # install all repository and system dependencies of the depends_on list
    print("Install all depends_on from source: %s" % (', '.join(depends_on)))
    os.makedirs(dependson_sourcespace)
    call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))

    # get build and run dependencies of depends_on list
    dependson_build_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=True, run_depends=False):
        print("  Checking dependency %s" % d)
        if d in dependson_build_dependencies:
            print("    Already in dependson_build_dependencies")
        if d in depends_on:
            print("    Is a direct dependency of the repo list, and is installed from source")
        if d in repo_list:
            print("    Is on of the repositories tested")
        if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
            dependson_build_dependencies.append(d)
    print("Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)))
    dependson_test_dependencies = []
    for d in get_dependencies(dependson_sourcespace, build_depends=False, run_depends=True):
        if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
            dependson_test_dependencies.append(d)
    print("Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)))

    # install build dependencies
    print("Install all build dependencies of the depends_on list")
    apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file again
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
    os.makedirs(dependson_buildspace)
    os.chdir(dependson_buildspace)
    print("Create a new CMakeLists.txt file using catkin")
    call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
    depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')


    call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
    #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))

    # build repositories
    print("Build depends-on repositories")
    call("make", ros_env)

    # install test dependencies
    print("Install all test dependencies of the depends_on list")
    apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)

    # test repositories
    print("Test depends-on repositories")
    call("make run_tests", ros_env)
    ensure_test_results(test_results_dir)
예제 #11
0
def _test_repositories(ros_distro,
                       repo_list,
                       version_list,
                       workspace,
                       test_depends_on,
                       repo_sourcespace,
                       dependson_sourcespace,
                       repo_buildspace,
                       dependson_buildspace,
                       sudo=False,
                       no_chroot=False):
    append_pymodules_if_needed()
    from catkin_pkg.package import InvalidPackage, parse_package_string
    from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
    from rosdistro.dependency_walker import DependencyWalker
    from rosdistro.manifest_provider import get_release_tag

    index = get_index(get_index_url())
    print("Parsing rosdistro file for %s" % ros_distro)
    release = get_cached_release(index, ros_distro)
    print("Parsing devel file for %s" % ros_distro)
    source_file = get_source_file(index, ros_distro)

    # Create rosdep object
    print("Create rosdep object")
    rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)

    if repo_list:
        # download the repo_list from source
        print("Creating rosinstall file for repo list")
        rosinstall = ""
        for repo_name, version in zip(repo_list, version_list):
            if version == 'devel':
                if repo_name not in source_file.repositories:
                    raise BuildException(
                        "Repository %s does not exist in Devel Distro" %
                        repo_name)
                print("Using devel distro file to download repositories")
                rosinstall += _generate_rosinstall_for_repo(
                    source_file.repositories[repo_name])
            else:
                if repo_name not in release.repositories:
                    raise BuildException(
                        "Repository %s does not exist in Ros Distro" %
                        repo_name)
                repo = release.repositories[repo_name]
                if version not in ['latest', 'master']:
                    assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
                assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
                for pkg_name in repo.package_names:
                    release_tag = get_release_tag(repo, pkg_name)
                    if version in ['latest', 'master']:
                        release_tag = '/'.join(release_tag.split('/')[:-1])
                    print(
                        'Using tag "%s" of release distro file to download package "%s from repo "%s'
                        % (version, pkg_name, repo_name))
                    rosinstall += _generate_rosinstall_for_pkg_version(
                        release.repositories[repo_name], pkg_name, release_tag)
        print("rosinstall file for all repositories: \n %s" % rosinstall)
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print("Install repo list from source")
        os.makedirs(repo_sourcespace)
        call("rosinstall %s %s/repo.rosinstall --catkin" %
             (repo_sourcespace, workspace))

    extract_notification_recipients(repo_sourcespace)

    # get the repositories build dependencies
    print("Get build dependencies of repo list")
    repo_build_dependencies = get_dependencies(repo_sourcespace,
                                               build_depends=True,
                                               run_depends=True)
    # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
    if 'catkin' not in repo_build_dependencies:
        repo_build_dependencies.append('catkin')
    print("Install build dependencies of repo list: %s" %
          (', '.join(repo_build_dependencies)))
    apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file for repositories that use catkin
    root_cmakelists = os.path.join(repo_sourcespace, 'CMakeLists.txt')
    if os.path.exists(root_cmakelists):
        print("Removing the CMakeLists.txt file generated by rosinstall")
        os.remove(root_cmakelists)
    print("Create a new CMakeLists.txt file using catkin")

    # get environment
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)

    # check if source workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)

    # make build folder and change into it
    os.makedirs(repo_buildspace)
    os.chdir(repo_buildspace)

    # make test results dir
    test_results_dir = os.path.join(workspace, 'test_results')
    if os.path.exists(test_results_dir):
        shutil.rmtree(test_results_dir)
    os.makedirs(test_results_dir)

    if not non_catkin_pkgs:
        print("Build catkin workspace")
        call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
        repos_test_results_dir = os.path.join(test_results_dir, 'repos')
        call(
            "cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" %
            (repo_sourcespace, repos_test_results_dir), ros_env)
        #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories and tests
        print("Build repo list")
        call("make", ros_env)
        call("make tests", ros_env)

        # get the repositories test and run dependencies
        print("Get run dependencies of repo list")
        repo_test_dependencies = get_dependencies(repo_sourcespace,
                                                  build_depends=False,
                                                  run_depends=True)
        print("Install run dependencies of repo list: %s" %
              (', '.join(repo_test_dependencies)))
        apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)

        # get environment after installing test and run dependencies
        ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)

        # run tests
        print("Test repo list")
        call("make run_tests", ros_env)

        # anything after this should build on this env
        ros_env = get_ros_env(os.path.join(repo_buildspace,
                                           'devel/setup.bash'))

    else:
        print("Build workspace with non-catkin packages in isolation")
        # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
        os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
        call(
            'catkin_make_isolated -C %s --source %s --install' %
            (repo_buildspace, repo_sourcespace), ros_env)
        setup_file = os.path.join(repo_buildspace, 'install_isolated',
                                  'setup.sh')
        # anything after this should build on this env
        ros_env = get_ros_env(setup_file)

    # don't do depends-on on things not in release
    not_in_release = set(repo_list) - set(release.repositories.keys())
    if not_in_release:
        print(
            "Removed [%s] repositories which are not in the " %
            ', '.join(sorted(not_in_release)),
            "release file for depends-on testing")
        repo_list = list(set(repo_list) - not_in_release)

    # see if we need to do more work or not
    if not test_depends_on:
        print("We're not testing the depends-on repositories")
        ensure_test_results(test_results_dir)
        return

    # get repo_list depends-on list
    print("Get list of wet repositories that build-depend on repo list: %s" %
          ', '.join(repo_list))
    walker = DependencyWalker(release)
    depends_on = set([])
    try:
        for repo_name in repo_list:
            print('repo_name', repo_name)
            repo = release.repositories[repo_name]
            for pkg_name in repo.package_names:
                print('pkg_name', pkg_name)
                depends_on |= walker.get_recursive_depends_on(
                    pkg_name, ['buildtool', 'build', 'test'],
                    ignore_pkgs=depends_on)
                print('depends_on', depends_on)
        # remove all packages which are already in the workspace
        from catkin_pkg.packages import find_packages
        pkgs = find_packages(repo_sourcespace)
        depends_on -= set([pkg.name for pkg in pkgs.values()])
    except RuntimeError:
        print(
            "Exception %s: If you are not in the rosdistro and only in the devel",
            " builds there will be no depends on")
        depends_on = set([])

    print("Build depends_on list of pkg list: %s" % (', '.join(depends_on)))
    if len(depends_on) == 0:
        print("No wet packages depend on our repo list. Test finished here")
        ensure_test_results(test_results_dir)
        return

    # install depends_on packages from source from release repositories
    rosinstall = ''
    non_catkin_pkgs = []
    for pkg_name in depends_on:
        repo = release.repositories[release.packages[pkg_name].repository_name]
        if repo.version is None:
            continue
        pkg_xml = release.get_package_xml(pkg_name)
        if pkg_xml is None:
            raise BuildException(
                'Could not retrieve package.xml for package "%s" from rosdistro cache'
                % pkg_name)
        try:
            pkg = parse_package_string(pkg_xml)
        except InvalidPackage as e:
            raise BuildException(
                'package.xml for package "%s" from rosdistro cache is invalid: %s'
                % (pkg_name, e))
        if _is_non_catkin_package(pkg):
            non_catkin_pkgs.append(pkg.name)
        rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name)

    if non_catkin_pkgs:
        print(
            'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here'
            % ', '.join(sorted(non_catkin_pkgs)))
        create_test_result(
            test_results_dir,
            failure=
            'Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.'
            % ', '.join(sorted(non_catkin_pkgs)))
        return

    print("Rosinstall for depends_on:\n %s" % rosinstall)
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print("Created rosinstall file for depends on")

    # install all repository and system dependencies of the depends_on list
    print("Install all depends_on from source: %s" % (', '.join(depends_on)))
    os.makedirs(dependson_sourcespace)
    call("rosinstall --catkin %s %s/depends_on.rosinstall" %
         (dependson_sourcespace, workspace))

    # check if depends_on workspace contains only package built with catkin
    non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace)
    if non_catkin_pkgs:
        print(
            'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here'
            % ', '.join(sorted(non_catkin_pkgs)))
        create_test_result(
            test_results_dir,
            failure=
            'Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.'
            % ', '.join(sorted(non_catkin_pkgs)))
        return

    # get build and run dependencies of depends_on list
    dependson_build_dependencies = []
    for d in get_dependencies(dependson_sourcespace,
                              build_depends=True,
                              run_depends=False):
        print("  Checking dependency %s" % d)
        if d in dependson_build_dependencies:
            print("    Already in dependson_build_dependencies")
        if d in depends_on:
            print(
                "    Is a direct dependency of the repo list, and is installed from source"
            )
        if d in repo_list:
            print("    Is one of the repositories tested")
        if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
            dependson_build_dependencies.append(d)
    print("Build dependencies of depends_on list are %s" %
          (', '.join(dependson_build_dependencies)))
    dependson_test_dependencies = []
    for d in get_dependencies(dependson_sourcespace,
                              build_depends=False,
                              run_depends=True):
        if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
            dependson_test_dependencies.append(d)
    print("Test dependencies of depends_on list are %s" %
          (', '.join(dependson_test_dependencies)))

    # install build dependencies
    print("Install all build dependencies of the depends_on list")
    apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file again
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
    os.makedirs(dependson_buildspace)
    os.chdir(dependson_buildspace)
    print("Create a new CMakeLists.txt file using catkin")
    call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
    depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
    call(
        "cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" %
        (dependson_sourcespace, depends_on_test_results_dir), ros_env)
    #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))

    # build repositories
    print("Build depends-on packages")
    call("make", ros_env)

    # install test dependencies
    print("Install all test dependencies of the depends_on list")
    apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)

    # test repositories
    print("Test depends-on packages")
    call("make run_tests", ros_env)
    ensure_test_results(test_results_dir)
예제 #12
0
def _test_repositories_fuerte(ros_distro,
                              repo_list,
                              version_list,
                              workspace,
                              test_depends_on,
                              repo_sourcespace,
                              dependson_sourcespace,
                              repo_buildspace,
                              dependson_buildspace,
                              sudo=False,
                              no_chroot=False):
    import rosdistro

    # parse the rosdistro file
    print("Parsing rosdistro file for %s" % ros_distro)
    distro = rosdistro.RosDistro(ros_distro)
    print("Parsing devel file for %s" % ros_distro)
    devel = rosdistro.DevelDistro(ros_distro)

    # Create rosdep object
    print("Create rosdep object")
    rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)

    # download the repo_list from source
    print("Creating rosinstall file for repo list")
    rosinstall = ""
    for repo, version in zip(repo_list, version_list):
        if version == 'devel':
            if repo not in devel.repositories:
                raise BuildException(
                    "Repository %s does not exist in Devel Distro" % repo)
            print("Using devel distro file to download repositories")
            rosinstall += devel.repositories[repo].get_rosinstall()
        else:
            if repo not in distro.get_repositories():
                raise BuildException(
                    "Repository %s does not exist in Ros Distro" % repo)
            if version in ['latest', 'master']:
                print(
                    "Using latest release distro file to download repositories"
                )
                rosinstall += distro.get_rosinstall(repo, version='master')
            else:
                print(
                    "Using version %s of release distro file to download repositories"
                    % version)
                rosinstall += distro.get_rosinstall(repo, version)
    print("rosinstall file for all repositories: \n %s" % rosinstall)
    with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
        f.write(rosinstall)
    print("Install repo list from source")
    os.makedirs(repo_sourcespace)
    call("rosinstall %s %s/repo.rosinstall --catkin" %
         (repo_sourcespace, workspace))

    # get the repositories build dependencies
    print("Get build dependencies of repo list")
    repo_build_dependencies = get_dependencies(repo_sourcespace,
                                               build_depends=True,
                                               run_depends=False)
    print("Install build dependencies of repo list: %s" %
          (', '.join(repo_build_dependencies)))
    apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file for repositories that use catkin
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
    print("Create a new CMakeLists.txt file using catkin")
    ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
    test_results_dir = os.path.join(workspace, 'test_results')
    repos_test_results_dir = os.path.join(test_results_dir, 'repos')
    os.makedirs(repo_buildspace)
    os.chdir(repo_buildspace)
    call(
        "cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" %
        (repo_sourcespace, repos_test_results_dir), ros_env)
    #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

    # build repositories and tests
    print("Build repo list")
    call("make", ros_env)
    call("make tests", ros_env)

    # get the repositories run dependencies
    print("Get test and run dependencies of repo list")
    repo_test_dependencies = get_dependencies(repo_sourcespace,
                                              build_depends=False,
                                              run_depends=True)
    print("Install test and run dependencies of repo list: %s" %
          (', '.join(repo_test_dependencies)))
    apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)

    # run tests
    print("Test repo list")
    call("make run_tests", ros_env)

    # see if we need to do more work or not
    if not test_depends_on:
        print("We're not testing the depends-on repositories")
        ensure_test_results(test_results_dir)
        return

    # get repo_list depends-on list
    print("Get list of wet repositories that build-depend on repo list %s" %
          ', '.join(repo_list))
    depends_on = []
    try:
        for d in distro.get_depends_on(repo_list)[
                'build'] + distro.get_depends_on(repo_list)['buildtool']:
            if not d in depends_on and not d in repo_list:
                depends_on.append(d)
    except RuntimeError:
        print(
            "Exception %s: If you are not in the rosdistro and only in the devel",
            " builds there will be no depends on")
        depends_on = []

    print("Build depends_on list of repo list: %s" % (', '.join(depends_on)))
    if len(depends_on) == 0:
        print(
            "No wet repositories depend on our repo list. Test finished here")
        ensure_test_results(test_results_dir)
        return

    # install depends_on repositories from source
    rosinstall = distro.get_rosinstall(depends_on)
    print("Rosinstall for depends_on:\n %s" % rosinstall)
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print("Created rosinstall file for depends on")

    # install all repository and system dependencies of the depends_on list
    print("Install all depends_on from source: %s" % (', '.join(depends_on)))
    os.makedirs(dependson_sourcespace)
    call("rosinstall --catkin %s %s/depends_on.rosinstall" %
         (dependson_sourcespace, workspace))

    # get build and run dependencies of depends_on list
    dependson_build_dependencies = []
    for d in get_dependencies(dependson_sourcespace,
                              build_depends=True,
                              run_depends=False):
        print("  Checking dependency %s" % d)
        if d in dependson_build_dependencies:
            print("    Already in dependson_build_dependencies")
        if d in depends_on:
            print(
                "    Is a direct dependency of the repo list, and is installed from source"
            )
        if d in repo_list:
            print("    Is on of the repositories tested")
        if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
            dependson_build_dependencies.append(d)
    print("Build dependencies of depends_on list are %s" %
          (', '.join(dependson_build_dependencies)))
    dependson_test_dependencies = []
    for d in get_dependencies(dependson_sourcespace,
                              build_depends=False,
                              run_depends=True):
        if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
            dependson_test_dependencies.append(d)
    print("Test dependencies of depends_on list are %s" %
          (', '.join(dependson_test_dependencies)))

    # install build dependencies
    print("Install all build dependencies of the depends_on list")
    apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)

    # replace the CMakeLists.txt file again
    print("Removing the CMakeLists.txt file generated by rosinstall")
    os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
    os.makedirs(dependson_buildspace)
    os.chdir(dependson_buildspace)
    print("Create a new CMakeLists.txt file using catkin")
    call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
    depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
    call(
        "cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" %
        (dependson_sourcespace, depends_on_test_results_dir), ros_env)
    #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))

    # build repositories
    print("Build depends-on repositories")
    call("make", ros_env)

    # install test dependencies
    print("Install all test dependencies of the depends_on list")
    apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)

    # test repositories
    print("Test depends-on repositories")
    call("make run_tests", ros_env)
    ensure_test_results(test_results_dir)