def github_manifest_provider(_dist_name, repo, pkg_name): assert repo.version if 'github.com' not in repo.url: logger.debug('Skip non-github url "%s"' % repo.url) raise RuntimeError('can not handle non github urls') release_tag = get_release_tag(repo, pkg_name) if not check_remote_tag_exists(repo.url, release_tag): raise RuntimeError('specified tag "%s" is not a git tag' % release_tag) url = repo.url if url.endswith('.git'): url = url[:-4] url += '/%s/package.xml' % release_tag if url.startswith('git://'): url = 'https://' + url[6:] if url.startswith('https://'): url = 'https://raw.' + url[8:] try: logger.debug('Load package.xml file from url "%s"' % url) package_xml = urlopen(url).read() return package_xml except URLError as e: logger.debug('- failed (%s), trying "%s"' % (e, url)) raise RuntimeError()
def _generate_rosinstall_for_package(distro, pkg_name, flat=False, tar=False): pkg = distro.release_packages[pkg_name] repo = distro.repositories[pkg.repository_name].release_repository assert repo is not None and repo.version is not None, 'Package "%s" does not have a version"' % pkg_name local_name = pkg_name if not flat and repo.package_names != [pkg_name]: local_name = "%s/%s" % (repo.name, local_name) url = repo.url release_tag = get_release_tag(repo, pkg_name) if tar: # the repository name might be different than repo.name coming from rosdistro repo_name = os.path.basename(url[:-4]) suffix = ".git" if not url.endswith(suffix): raise RuntimeError("The repository '%s' must end with '%s': %s" % (repo_name, suffix, url)) url = url[: -len(suffix)] + "/archive/{0}.tar.gz".format(release_tag) data = [ { "tar": { "local-name": local_name, "uri": url, "version": "{0}-{1}".format(repo_name, release_tag.replace("/", "-")), } } ] else: data = [{"git": {"local-name": local_name, "uri": url, "version": release_tag}}] return data
def _generate_rosinstall_for_pkg(repo, pkg_name): from rosdistro.manifest_provider import get_release_tag repo_data = { 'local-name': pkg_name, 'uri': repo.url, 'version': get_release_tag(repo, pkg_name) } return yaml.safe_dump([{repo.type: repo_data}], default_style=False)
def git_manifest_provider(_dist_name, repo, pkg_name): assert repo.version try: release_tag = get_release_tag(repo, pkg_name) package_xml = _get_package_xml(repo.url, release_tag) return package_xml except Exception as e: raise RuntimeError('Unable to fetch package.xml: %s' % e)
def get_repository_specification_for_released_package(dist_file, pkg_name): repo_name = dist_file.release_packages[pkg_name].repository_name release_repo = dist_file.repositories[repo_name].release_repository version_tag = get_release_tag(release_repo, pkg_name) return RepositorySpecification(pkg_name, { 'type': 'git', 'url': release_repo.url, 'version': version_tag, })
def get_repository_specification_for_released_package(dist_file, pkg_name): repo_name = dist_file.release_packages[pkg_name].repository_name release_repo = dist_file.repositories[repo_name].release_repository version_tag = get_release_tag(release_repo, pkg_name) return RepositorySpecification( pkg_name, { 'type': 'git', 'url': release_repo.url, 'version': version_tag, })
def _generate_rosinstall_for_package(distro, pkg_name, flat=False, tar=False): pkg = distro.release_packages[pkg_name] repo = distro.repositories[pkg.repository_name].release_repository assert repo is not None and repo.version is not None, 'Package "%s" does not have a version"' % pkg_name local_name = pkg_name if not flat and repo.package_names != [pkg_name]: local_name = '%s/%s' % (repo.name, local_name) release_tag = get_release_tag(repo, pkg_name) return _generate_rosinstall(local_name, repo.url, release_tag, tar=tar)
def __init__(self, distro, pkg_name, tar_dir, has_patches=False): pkg = distro.release_packages[pkg_name] repo = distro.repositories[pkg.repository_name].release_repository ros_pkg = RosPackage(pkg_name, repo) pkg_rosinstall = _generate_rosinstall(pkg_name, repo.url, get_release_tag(repo, pkg_name), True) self.recipe = _gen_recipe_for_package(distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall, tar_dir)
def __init__(self, rosdistro, pkg_name, srcrev_cache, skip_keys): pkg = rosdistro.release_packages[pkg_name] repo = rosdistro.repositories[pkg.repository_name].release_repository ros_pkg = RosPackage(pkg_name, repo) pkg_rosinstall = _generate_rosinstall(pkg_name, repo.url, get_release_tag(repo, pkg_name), True) self.recipe = _gen_recipe_for_package(rosdistro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall, srcrev_cache, skip_keys)
def __init__(self, distro, pkg_name, has_patches=False): pkg = distro.release_packages[pkg_name] repo = distro.repositories[pkg.repository_name].release_repository ros_pkg = RosPackage(pkg_name, repo) pkg_rosinstall =\ _generate_rosinstall(pkg_name, repo.url, get_release_tag(repo, pkg_name), True) self.pkgbuild =\ _gen_pkgbuild_for_package(distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall) self.pkgbuild.has_patches = has_patches if pkg_name in no_python3: self.pkgbuild.python_3 = False
def github_manifest_provider(_dist_name, repo, pkg_name): assert repo.version if 'github.com' not in repo.url: logger.debug('Skip non-github url "%s"' % repo.url) raise RuntimeError('can not handle non github urls') release_tag = get_release_tag(repo, pkg_name) url = repo.url.replace('.git', '/%s/package.xml' % release_tag) url = url.replace('git://', 'https://') url = url.replace('https://', 'https://raw.') try: logger.debug('Load package.xml file from url "%s"' % url) package_xml = urllib2.urlopen(url).read() return package_xml except urllib2.URLError as e: logger.debug('- failed (%s), trying "%s"' % (e, url)) raise RuntimeError()
def get_info(self): res = {} if not self._distribution_file: return res for name in self._distribution_file.repositories.keys(): repo = self._distribution_file.repositories[name] if repo.release_repository or repo.source_repository: res[name] = {'distro': self.distro + "_wet", 'version': [], 'url': [], 'branch': []} # first add devel for name in self._distribution_file.repositories.keys(): r = self._distribution_file.repositories[name].source_repository if not r: continue if not r.version: res[name]['branch'].append("") else: res[name]['branch'].append(r.version) res[name]['version'].append('devel') res[name]['url'].append(r.url) # then add release for name in self._distribution_file.repositories.keys(): r = self._distribution_file.repositories[name].release_repository if not r: continue if 'release' not in r.tags: continue release_tag = get_release_tag(r, '{package}') if r.version is not None: release_tag_without_version = release_tag.replace('/%s' % r.version, '').replace('/%s' % r.version.split('-')[0], '') else: release_tag_without_version = release_tag.replace('/{version}', '').replace('/{upstream_version}', '') res[name]['version'].append('latest') res[name]['url'].append(r.url) res[name]['branch'].append(release_tag_without_version) if r.version: res[name]['version'].append(r.version.split('-')[0]) res[name]['url'].append(r.url) res[name]['branch'].append(release_tag) #res[name]['branch'].append("release/pkg_name/" + r.version.split('-')[0]) return res
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env) #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories and tests print "Build repo list" call("make", ros_env) call("make tests", ros_env) # get the repositories test and run dependencies print "Get test and run dependencies of repo list" repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True) print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)) apt_get_install(repo_test_dependencies, rosdep_resolver, sudo) # run tests print "Test repo list" call("make run_tests", ros_env) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) # don't do depends-on on things not in release not_in_release = set(repo_list) - set(release.repositories.keys()) if not_in_release: print "Removed [%s] repositories which are not in the " %\ ', '.join(sorted(not_in_release)), \ "release file for depends-on testing" repo_list = list(set(repo_list) - not_in_release) # see if we need to do more work or not if not test_depends_on: print "We're not testing the depends-on repositories" ensure_test_results(test_results_dir) return # get repo_list depends-on list print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list) walker = DependencyWalker(release) depends_on = set([]) try: for repo_name in repo_list: print('repo_name', repo_name) repo = release.repositories[repo_name] for pkg_name in repo.package_names: print('pkg_name', pkg_name) depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on) print('depends_on', depends_on) except RuntimeError: print "Exception %s: If you are not in the rosdistro and only in the devel", \ " builds there will be no depends on" depends_on = set([]) print "Build depends_on list of pkg list: %s" % (', '.join(depends_on)) if len(depends_on) == 0: print "No wet packages depend on our repo list. Test finished here" ensure_test_results(test_results_dir) return # install depends_on packages from source from release repositories rosinstall = '' non_catkin_pkgs = [] for pkg_name in depends_on: repo = release.repositories[release.packages[pkg_name].repository_name] if repo.version is None: continue pkg_xml = release.get_package_xml(pkg_name) if pkg_xml is None: raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e)) if _is_non_catkin_package(pkg): non_catkin_pkgs.append(pkg.name) rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return print "Rosinstall for depends_on:\n %s" % rosinstall with open(workspace + "/depends_on.rosinstall", 'w') as f: f.write(rosinstall) print "Created rosinstall file for depends on" # install all repository and system dependencies of the depends_on list print "Install all depends_on from source: %s" % (', '.join(depends_on)) os.makedirs(dependson_sourcespace) call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace)) # check if depends_on workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return # get build and test dependencies of depends_on list dependson_build_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False): print " Checking dependency %s" % d if d in dependson_build_dependencies: print " Already in dependson_build_dependencies" if d in depends_on: print " Is a direct dependency of the repo list, and is installed from source" if d in repo_list: print " Is one of the repositories tested" if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list: dependson_build_dependencies.append(d) print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)) dependson_test_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True): if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list: dependson_test_dependencies.append(d) print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)) # install build dependencies print "Install all build dependencies of the depends_on list" apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file again print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt')) os.makedirs(dependson_buildspace) os.chdir(dependson_buildspace) print "Create a new CMakeLists.txt file using catkin" call("catkin_init_workspace %s" % dependson_sourcespace, ros_env) depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env) #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash')) # build repositories print "Build depends-on packages" call("make", ros_env) # install test dependencies print "Install all test dependencies of the depends_on list" apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo) # test repositories print "Test depends-on packages" call("make run_tests", ros_env) ensure_test_results(test_results_dir)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env) #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories and tests print "Build repo list" call("make", ros_env) call("make tests", ros_env) # get the repositories test and run dependencies print "Get test and run dependencies of repo list" repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True) print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)) apt_get_install(repo_test_dependencies, rosdep_resolver, sudo) # run tests print "Test repo list" call("make run_tests", ros_env) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) # see if we need to do more work or not if not test_depends_on: print "We're not testing the depends-on repositories" ensure_test_results(test_results_dir) return # get repo_list depends-on list print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list) walker = DependencyWalker(release) depends_on = set([]) try: for repo_name in repo_list: print('repo_name', repo_name) repo = release.repositories[repo_name] for pkg_name in repo.package_names: print('pkg_name', pkg_name) depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on) print('depends_on', depends_on) except RuntimeError: print "Exception %s: If you are not in the rosdistro and only in the devel", \ " builds there will be no depends on" depends_on = set([]) print "Build depends_on list of pkg list: %s" % (', '.join(depends_on)) if len(depends_on) == 0: print "No wet packages depend on our repo list. Test finished here" ensure_test_results(test_results_dir) return # install depends_on packages from source from release repositories rosinstall = '' non_catkin_pkgs = [] for pkg_name in depends_on: repo = release.repositories[release.packages[pkg_name].repository_name] if repo.version is None: continue pkg_xml = release.get_package_xml(pkg_name) if pkg_xml is None: raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e)) if _is_non_catkin_package(pkg): non_catkin_pkgs.append(pkg.name) rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return print "Rosinstall for depends_on:\n %s" % rosinstall with open(workspace + "/depends_on.rosinstall", 'w') as f: f.write(rosinstall) print "Created rosinstall file for depends on" # install all repository and system dependencies of the depends_on list print "Install all depends_on from source: %s" % (', '.join(depends_on)) os.makedirs(dependson_sourcespace) call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace)) # check if depends_on workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return # get build and test dependencies of depends_on list dependson_build_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False): print " Checking dependency %s" % d if d in dependson_build_dependencies: print " Already in dependson_build_dependencies" if d in depends_on: print " Is a direct dependency of the repo list, and is installed from source" if d in repo_list: print " Is one of the repositories tested" if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list: dependson_build_dependencies.append(d) print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)) dependson_test_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True): if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list: dependson_test_dependencies.append(d) print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)) # install build dependencies print "Install all build dependencies of the depends_on list" apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file again print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt')) os.makedirs(dependson_buildspace) os.chdir(dependson_buildspace) print "Create a new CMakeLists.txt file using catkin" call("catkin_init_workspace %s" % dependson_sourcespace, ros_env) depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env) #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash')) # build repositories print "Build depends-on packages" call("make", ros_env) # install test dependencies print "Install all test dependencies of the depends_on list" apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo) # test repositories print "Test depends-on packages" call("make run_tests", ros_env) ensure_test_results(test_results_dir)
def _generate_rosinstall_for_pkg(repo, pkg_name): from rosdistro.manifest_provider import get_release_tag return _generate_rosinstall_for_pkg_version(repo, pkg_name, get_release_tag(repo, pkg_name))
def get_released_repo(self, pkg_name): pkg = self._distro.release_packages[pkg_name] repo = self._distro.repositories[ pkg.repository_name].release_repository release_tag = get_release_tag(repo, pkg_name) return repo.url, release_tag
def get_wet_rosinstall(release_file, name, prefix=None): repo = _find_repo(release_file, name) if repo is None: # wait, what? return None return build_rosinstall(name, repo.url, 'git', get_release_tag(repo, name), prefix)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, run_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') helper = subprocess.Popen(('cmake %s -DCMAKE_TOOLCHAIN_FILE=/opt/ros/groovy/share/ros/core/rosbuild/rostoolchain.cmake -DCATKIN_TEST_RESULTS_DIR=%s'%(repo_sourcespace,repos_test_results_dir)).split(' '), env=ros_env) helper.communicate() res = 0 if helper.returncode != 0: res = helper.returncode ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories print "Build repo list" print "CMAKE_PREFIX_PATH: %s"%ros_env['CMAKE_PREFIX_PATH'] call("make", ros_env) # Concatenate filelists print '----------------- Concatenate filelists ----------------- ' filelist = '%s'%repo_buildspace + '/filelist.lst' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/concatenate_filelists.py --dir %s --filelist %s'%(workspace,repo_buildspace, filelist)).split(' '), env=os.environ) helper.communicate() print '////////////////// cma analysis done ////////////////// \n\n' # Run CMA print '----------------- Run CMA analysis ----------------- ' cmaf = repo_sourcespace#repo_buildspace helper = subprocess.Popen(('pal QACPP -cmaf %s -list %s'%(cmaf, filelist)).split(' '), env=os.environ) helper.communicate() print '////////////////// cma analysis done ////////////////// \n\n' # Export metrics to yaml and csv files # get uri infos #uri= distro.get_repositories()[repo_list[0]].url repo_name = repo_list[0] repo_data = release.get_data()['repositories'][repo_name] print "repo_data", repo_data uri = repo_data['url'] uri_info = 'master' #repo_data['version'] vcs_type = 'git' # repo_data['type'] print '----------------- Export metrics to yaml and csv files ----------------- ' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/export_metrics_to_yaml_wet.py --path %s --path_src %s --doc metrics --csv csv --config %s/jenkins_scripts/code_quality/export_config.yaml --distro %s --stack %s --uri %s --uri_info %s --vcs_type %s'%(workspace, repo_buildspace, repo_sourcespace, workspace, ros_distro, repo_name, uri, uri_info, vcs_type)).split(' '), env=os.environ) helper.communicate() print '////////////////// export metrics to yaml and csv files done ////////////////// \n\n' # Push results to server print '----------------- Push results to server ----------------- ' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/push_results_to_server_wet.py --path %s --doc metrics --path_src %s --meta_package %s'%(workspace, repo_buildspace, repo_sourcespace, repo_list)).split(' '), env=os.environ) helper.communicate() print '////////////////// push results to server done ////////////////// \n\n' # Upload results to QAVerify print ' ----------------- upload results to QAVerify ----------------- ' shutil.rmtree(os.path.join(workspace, 'snapshots_path'), ignore_errors=True) os.makedirs(os.path.join(workspace, 'snapshots_path')) snapshots_path = '%s/snapshots_path'%workspace project_name = repo_list[0] + '-' + ros_distro helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/upload_to_QAVerify_wet.py --path %s --snapshot %s --project %s --stack_name %s'%(workspace, repo_buildspace, snapshots_path, project_name, repo_list[0])).split(' '), env=os.environ) helper.communicate() print '////////////////// upload results to QAVerify done ////////////////// \n\n' if os.path.exists(snapshots_path): shutil.rmtree(snapshots_path) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) if res != 0: print "helper_return_code is: %s"%(helper.returncode) assert 'analysis_wet.py failed' raise Exception("analysis_wet.py failed. Check out the console output above for details.") # create dummy test results env = dict() env['INSTALL_DIR'] = os.getenv('INSTALL_DIR', '') test_results_path = workspace + '/test_results' if os.path.exists(test_results_path): shutil.rmtree(test_results_path) os.makedirs(test_results_path) test_file= test_results_path + '/test_file.xml' f = open(test_file, 'w') f.write('<?xml version="1.0" encoding="UTF-8"?>\n') f.write('<testsuite tests="1" failures="0" time="1" errors="0" name="dummy test">\n') f.write(' <testcase name="dummy rapport" classname="Results" /> \n') f.write('</testsuite> \n') f.close()
def _generate_rosinstall_for_pkg(repo, pkg_name): from rosdistro.manifest_provider import get_release_tag return _generate_rosinstall_for_pkg_version( repo, pkg_name, get_release_tag(repo, pkg_name))