def get_recursive_dependencies_on(distro, package_names, excludes=None, limit=None): excludes = set(excludes or []) limit = set(limit or []) # to improve performance limit search space if possible if limit: released_names, _ = get_package_names(distro) excludes.update(set(released_names) - limit - set(package_names)) dependencies = set([]) walker = DependencyWalker(distro) # redirect all stderr output to logger stderr = sys.stderr sys.stderr = CustomLogger() try: for pkg_name in package_names: dependencies |= walker.get_recursive_depends_on( pkg_name, ['buildtool', 'build', 'run', 'test'], ignore_pkgs=dependencies | excludes) finally: sys.stderr = stderr dependencies -= set(package_names) return dependencies
def get_recursive_dependencies_on(distro, package_names, excludes=None, limit=None): excludes = set(excludes or []) limit = set(limit or []) # to improve performance limit search space if possible if limit: released_names, _ = get_package_names(distro) excludes.update(set(released_names) - limit - set(package_names)) dependencies = set([]) walker = DependencyWalker(distro) # redirect all stderr output to logger stderr = sys.stderr sys.stderr = CustomLogger() try: for pkg_name in package_names: dependencies |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build', 'run', 'test'], ignore_pkgs=dependencies | excludes) finally: sys.stderr = stderr dependencies -= set(package_names) return dependencies
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env) #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories and tests print "Build repo list" call("make", ros_env) call("make tests", ros_env) # get the repositories test and run dependencies print "Get test and run dependencies of repo list" repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True) print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)) apt_get_install(repo_test_dependencies, rosdep_resolver, sudo) # run tests print "Test repo list" call("make run_tests", ros_env) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) # don't do depends-on on things not in release not_in_release = set(repo_list) - set(release.repositories.keys()) if not_in_release: print "Removed [%s] repositories which are not in the " %\ ', '.join(sorted(not_in_release)), \ "release file for depends-on testing" repo_list = list(set(repo_list) - not_in_release) # see if we need to do more work or not if not test_depends_on: print "We're not testing the depends-on repositories" ensure_test_results(test_results_dir) return # get repo_list depends-on list print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list) walker = DependencyWalker(release) depends_on = set([]) try: for repo_name in repo_list: print('repo_name', repo_name) repo = release.repositories[repo_name] for pkg_name in repo.package_names: print('pkg_name', pkg_name) depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on) print('depends_on', depends_on) except RuntimeError: print "Exception %s: If you are not in the rosdistro and only in the devel", \ " builds there will be no depends on" depends_on = set([]) print "Build depends_on list of pkg list: %s" % (', '.join(depends_on)) if len(depends_on) == 0: print "No wet packages depend on our repo list. Test finished here" ensure_test_results(test_results_dir) return # install depends_on packages from source from release repositories rosinstall = '' non_catkin_pkgs = [] for pkg_name in depends_on: repo = release.repositories[release.packages[pkg_name].repository_name] if repo.version is None: continue pkg_xml = release.get_package_xml(pkg_name) if pkg_xml is None: raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e)) if _is_non_catkin_package(pkg): non_catkin_pkgs.append(pkg.name) rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return print "Rosinstall for depends_on:\n %s" % rosinstall with open(workspace + "/depends_on.rosinstall", 'w') as f: f.write(rosinstall) print "Created rosinstall file for depends on" # install all repository and system dependencies of the depends_on list print "Install all depends_on from source: %s" % (', '.join(depends_on)) os.makedirs(dependson_sourcespace) call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace)) # check if depends_on workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return # get build and test dependencies of depends_on list dependson_build_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False): print " Checking dependency %s" % d if d in dependson_build_dependencies: print " Already in dependson_build_dependencies" if d in depends_on: print " Is a direct dependency of the repo list, and is installed from source" if d in repo_list: print " Is one of the repositories tested" if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list: dependson_build_dependencies.append(d) print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)) dependson_test_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True): if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list: dependson_test_dependencies.append(d) print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)) # install build dependencies print "Install all build dependencies of the depends_on list" apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file again print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt')) os.makedirs(dependson_buildspace) os.chdir(dependson_buildspace) print "Create a new CMakeLists.txt file using catkin" call("catkin_init_workspace %s" % dependson_sourcespace, ros_env) depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env) #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash')) # build repositories print "Build depends-on packages" call("make", ros_env) # install test dependencies print "Install all test dependencies of the depends_on list" apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo) # test repositories print "Test depends-on packages" call("make run_tests", ros_env) ensure_test_results(test_results_dir)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env) #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories and tests print "Build repo list" call("make", ros_env) call("make tests", ros_env) # get the repositories test and run dependencies print "Get test and run dependencies of repo list" repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True) print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)) apt_get_install(repo_test_dependencies, rosdep_resolver, sudo) # run tests print "Test repo list" call("make run_tests", ros_env) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) # see if we need to do more work or not if not test_depends_on: print "We're not testing the depends-on repositories" ensure_test_results(test_results_dir) return # get repo_list depends-on list print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list) walker = DependencyWalker(release) depends_on = set([]) try: for repo_name in repo_list: print('repo_name', repo_name) repo = release.repositories[repo_name] for pkg_name in repo.package_names: print('pkg_name', pkg_name) depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on) print('depends_on', depends_on) except RuntimeError: print "Exception %s: If you are not in the rosdistro and only in the devel", \ " builds there will be no depends on" depends_on = set([]) print "Build depends_on list of pkg list: %s" % (', '.join(depends_on)) if len(depends_on) == 0: print "No wet packages depend on our repo list. Test finished here" ensure_test_results(test_results_dir) return # install depends_on packages from source from release repositories rosinstall = '' non_catkin_pkgs = [] for pkg_name in depends_on: repo = release.repositories[release.packages[pkg_name].repository_name] if repo.version is None: continue pkg_xml = release.get_package_xml(pkg_name) if pkg_xml is None: raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e)) if _is_non_catkin_package(pkg): non_catkin_pkgs.append(pkg.name) rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return print "Rosinstall for depends_on:\n %s" % rosinstall with open(workspace + "/depends_on.rosinstall", 'w') as f: f.write(rosinstall) print "Created rosinstall file for depends on" # install all repository and system dependencies of the depends_on list print "Install all depends_on from source: %s" % (', '.join(depends_on)) os.makedirs(dependson_sourcespace) call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace)) # check if depends_on workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return # get build and test dependencies of depends_on list dependson_build_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False): print " Checking dependency %s" % d if d in dependson_build_dependencies: print " Already in dependson_build_dependencies" if d in depends_on: print " Is a direct dependency of the repo list, and is installed from source" if d in repo_list: print " Is one of the repositories tested" if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list: dependson_build_dependencies.append(d) print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)) dependson_test_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True): if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list: dependson_test_dependencies.append(d) print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)) # install build dependencies print "Install all build dependencies of the depends_on list" apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file again print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt')) os.makedirs(dependson_buildspace) os.chdir(dependson_buildspace) print "Create a new CMakeLists.txt file using catkin" call("catkin_init_workspace %s" % dependson_sourcespace, ros_env) depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env) #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash')) # build repositories print "Build depends-on packages" call("make", ros_env) # install test dependencies print "Install all test dependencies of the depends_on list" apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo) # test repositories print "Test depends-on packages" call("make run_tests", ros_env) ensure_test_results(test_results_dir)