def main(repo_type, rosdistro_name): index = get_index(get_index_url()) if repo_type == 'doc': try: distro_file = get_doc_file(index, rosdistro_name) except RuntimeError as e: print("Could not load doc file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr) return False if repo_type == 'source': try: distro_file = get_source_file(index, rosdistro_name) except RuntimeError as e: print("Could not load source file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr) return False for repo_name in sorted(distro_file.repositories.keys()): sys.stdout.write('.') sys.stdout.flush() repo = distro_file.repositories[repo_name] try: if (repo.type == 'git'): check_git_repo(repo.url, repo.version) elif (repo.type == 'hg'): check_hg_repo(repo.url, repo.version) elif (repo.type == 'svn'): check_svn_repo(repo.url, repo.version) else: print() print("Unknown type '%s' for repository '%s'" % (repo.type, repo.name), file=sys.stderr) except RuntimeError as e: print() print("Could not fetch repository '%s': %s (%s) [%s]" % (repo.name, repo.url, repo.version, e), file=sys.stderr) print() return True
def get_manifest_from_rosdistro(package_name, distro_name): """ Get the rosdistro repository data and package information. @param package_name: name of package or repository to get manifest information for. It gives package symbols precedence over repository names. @type package_name: str @param distro_name: name of ROS distribution @type distro_name: str @return: (manifest data, 'package'|'repository'). @rtype: ({str: str}, str, str) @raise IOError: if data cannot be loaded """ data = {} type_ = None index = get_index(get_index_url()) try: release_cache = get_cached_release(index, distro_name) except RuntimeError as runerr: if (runerr.message.startswith("Unknown release")): return None raise if package_name in release_cache.packages: pkg = release_cache.packages[package_name] #print('pkg', pkg.name) pkg_xml = release_cache.get_package_xml(package_name) pkg_manifest = parse_package_string(pkg_xml) data['description'] = pkg_manifest.description website_url = [u.url for u in pkg_manifest.urls if u.type == 'website'] if website_url: data['url'] = website_url[0] repo_name = pkg.repository_name meta_export = [exp for exp in pkg_manifest.exports if exp.tagname == 'metapackage'] if meta_export: type_ = 'metapackage' else: type_ = 'package' else: repo_name = package_name type_ = 'repository' data['repo_name'] = repo_name if repo_name in release_cache.repositories: repo = release_cache.repositories[repo_name] data['packages'] = repo.package_names source_file = get_source_file(index, distro_name) if repo_name in source_file.repositories: repo = source_file.repositories[repo_name] data['vcs'] = repo.type data['vcs_uri'] = repo.url data['vcs_version'] = repo.version else: return None return (data, type_, None)
def test_get_source_file(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) src_file = get_source_file(i, 'foo') _validate_src_file(src_file)
def document_repo(workspace, docspace, ros_distro, repo, platform, arch, homepage, no_chroot, skip_garbage, doc_conf, depends_conf, tags_db): doc_job = "doc-%s-%s" % (ros_distro, repo) #Get the list of repositories that should have documentation run on them #These are all of the repos that are not in the depends rosinsall file repos_to_doc = get_repositories_from_rosinstall(doc_conf) repo_path = os.path.realpath("%s" % (docspace)) print("Repo path %s" % repo_path) #Walk through the installed repositories and find old-style packages, new-stye packages, and stacks stacks, manifest_packages, catkin_packages, repo_map = build_repo_structure(repo_path, doc_conf, depends_conf) if ros_distro == 'indigo': if stacks or manifest_packages: print("Ignoring dry packages and stacks in '%s'" % ros_distro) stacks = {} manifest_packages = {} if not catkin_packages: raise BuildException('No catkin packages found') print("Running documentation generation on\npackages: %s" % (manifest_packages.keys() + catkin_packages.keys())) #print "Catkin packages: %s" % catkin_packages #print "Manifest packages: %s" % manifest_packages #print "Stacks: %s" % stacks #Get any non local apt dependencies ros_dep = rosdep.RosDepResolver(ros_distro, no_chroot=no_chroot) import rosdistro if ros_distro == 'electric': apt = rosdistro.AptDistro(platform, arch, shadow=False) else: apt = rosdistro.AptDistro(platform, arch, shadow=True) apt_deps = get_apt_deps(apt, ros_dep, ros_distro, catkin_packages, stacks, manifest_packages) print("Apt dependencies: %s" % apt_deps) #Get rosdistro release file if there are catkin packages to get status if catkin_packages and ros_distro not in ['electric', 'fuerte']: print("Fetch rosdistro files for: %s" % ros_distro) index = rosdistro.get_index(rosdistro.get_index_url()) rosdistro_release_file = rosdistro.get_release_file(index, ros_distro) rosdistro_source_file = rosdistro.get_source_file(index, ros_distro) else: rosdistro_release_file = None rosdistro_source_file = None #Build a local dependency graph to be used for build order local_dep_graph = build_local_dependency_graph(catkin_packages, manifest_packages) doc_path = os.path.realpath("%s/doc/%s" % (docspace, ros_distro)) if os.path.exists(doc_path): shutil.rmtree(doc_path) #Write stack manifest files for all stacks, we can just do this off the #stack.xml files write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db, doc_job, homepage) #Need to make sure to re-order packages to be run in dependency order build_order = get_dependency_build_order(local_dep_graph) print("Build order that honors deps:\n%s" % build_order) #We'll need the full list of apt_deps to get tag files full_apt_deps = get_full_apt_deps(apt_deps, apt) if not no_chroot: print("Installing all dependencies for %s" % repo) # XXX this is a really ugly hack to make the hydro doc job for ros_comm pass # otherwise roslisp pulls in the rosgraph_msgs package as a Debian dependency # which then break catkin_basic since it include the msgs CMake multiple files # resulting in duplicate target names (https://github.com/ros/ros_comm/issues/471) if repo == 'ros_comm' and 'ros-hydro-roslisp' in apt_deps: apt_deps.remove('ros-hydro-roslisp') if apt_deps: call("apt-get install %s --yes" % (' '.join(apt_deps))) print("Done installing dependencies") #Set up the list of things that need to be sourced to run rosdoc_lite #TODO: Hack for electric if ros_distro == 'electric': #lucid doesn't have /usr/local on the path by default... weird sources = ['export PATH=/usr/local/sbin:/usr/local/bin:$PATH'] sources.append('source /opt/ros/fuerte/setup.bash') sources.append('export ROS_PACKAGE_PATH=/opt/ros/electric/stacks:$ROS_PACKAGE_PATH') else: sources = ['source /opt/ros/%s/setup.bash' % ros_distro] #We assume that there will be no build errors to start build_errors = [] #Everything that is after fuerte supports catkin workspaces, so everything #that has packages with package.xml files local_install_path = os.path.join(docspace, 'local_installs') if os.path.exists(local_install_path): shutil.rmtree(local_install_path) #Make sure to create some subfolders under the local install path def makedirs(path): if not os.path.exists(path): os.makedirs(path) makedirs(os.path.join(local_install_path, 'bin')) makedirs(os.path.join(local_install_path, 'lib/python2.7/dist-packages')) makedirs(os.path.join(local_install_path, 'share')) if catkin_packages \ and not 'rosdoc_lite' in catkin_packages.keys() and not 'catkin' in catkin_packages.keys(): source, errs = build_repo_messages(catkin_packages, docspace, ros_distro, local_install_path) build_errors.extend(errs) if source: sources.append(source) #For fuerte catkin, we need to check if we should build catkin stacks source, errs = build_repo_messages_catkin_stacks(stacks, ros_distro, local_install_path) build_errors.extend(errs) sources.append(source) #For all our manifest packages (dry or fuerte catkin) we want to build #messages. Note, for fuerte catkin, we have to build all the code and #install locally to get message generation source, errs = build_repo_messages_manifest(manifest_packages, build_order, ros_distro) build_errors.extend(errs) sources.append(source) #We want to pull all the tagfiles available once from the server tags_location = os.path.join(workspace, ros_distro) if os.path.exists(tags_location): shutil.rmtree(tags_location) command = ['bash', '-c', 'rsync -e "ssh -o StrictHostKeyChecking=no" -qrz [email protected]:/home/rosbot/docs/%s/tags %s' % (ros_distro, tags_location)] call_with_list(command) repo_tags = document_packages(manifest_packages, catkin_packages, build_order, repos_to_doc, sources, tags_db, full_apt_deps, ros_dep, repo_map, repo_path, docspace, ros_distro, homepage, doc_job, tags_location, doc_path, rosdistro_release_file, rosdistro_source_file) #Copy the files to the appropriate place folders = sorted(set(stacks.keys() + manifest_packages.keys() + catkin_packages.keys())) if folders: dsts = ['%s/api/%s' % (doc_path, f) for f in folders] for dst in dsts: with open(os.path.join(dst, 'stamp'), 'w'): pass command = ['bash', '-c', 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr --delete %s [email protected]:/home/rosbot/docs/%s/api' % (' '.join(dsts), ros_distro)] call_with_list(command) folders = ['%s/changelogs' % doc_path, '%s/tags' % doc_path] folders = [f for f in folders if os.path.exists(f)] if folders: command = ['bash', '-c', 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr %s [email protected]:/home/rosbot/docs/%s' % (' '.join(folders), ros_distro)] call_with_list(command) if not skip_garbage: #Remove the autogenerated doc files since they take up a lot of space if left on the server shutil.rmtree(tags_location) shutil.rmtree(doc_path) #Write the new tags to the database if there are any to write for name, tags in repo_tags.iteritems(): #Get the apt name of the current stack/repo if ros_dep.has_ros(name): deb_name = ros_dep.to_apt(name)[0] else: deb_name = "ros-%s-%s" % (ros_distro, name.replace('_', '-')) #We only want to write tags for packages that have a valid deb name #For others, the only way to get cross referencing is to document everything #together with a rosinstall file if apt.has_package(deb_name): tags_db.set_tags(deb_name, tags) #Make sure to write changes to tag files and deps #We don't want to write hashes on an unsuccessful build excludes = ['rosinstall_hashes'] if build_errors else [] tags_db.commit_db(excludes) tags_db.delete_tag_index_repo() #Tell jenkins that we've succeeded print("Preparing xml test results") try: os.makedirs(os.path.join(workspace, 'test_results')) print("Created test results directory") except Exception: pass if build_errors: import yaml copy_test_results(workspace, docspace, """Failed to generate messages by calling cmake for %s. Look in the console for cmake failures, search for "CMake Error" Also, are you sure that the rosinstall files are pulling from the right branch for %s? Check the repos below, you can update information the %s.rosinstall and %s-depends.rosinstall files by submitting a pull request at https://github.com/ros/rosdistro/%s Documentation rosinstall:\n%s Depends rosinstall:\n%s""" % (build_errors, ros_distro, repo, repo, ros_distro, yaml.safe_dump(doc_conf, default_flow_style=False), yaml.safe_dump(depends_conf, default_flow_style=False)), "message_generation_failure") else: copy_test_results(workspace, docspace)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, run_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') helper = subprocess.Popen(('cmake %s -DCMAKE_TOOLCHAIN_FILE=/opt/ros/groovy/share/ros/core/rosbuild/rostoolchain.cmake -DCATKIN_TEST_RESULTS_DIR=%s'%(repo_sourcespace,repos_test_results_dir)).split(' '), env=ros_env) helper.communicate() res = 0 if helper.returncode != 0: res = helper.returncode ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories print "Build repo list" print "CMAKE_PREFIX_PATH: %s"%ros_env['CMAKE_PREFIX_PATH'] call("make", ros_env) # Concatenate filelists print '----------------- Concatenate filelists ----------------- ' filelist = '%s'%repo_buildspace + '/filelist.lst' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/concatenate_filelists.py --dir %s --filelist %s'%(workspace,repo_buildspace, filelist)).split(' '), env=os.environ) helper.communicate() print '////////////////// cma analysis done ////////////////// \n\n' # Run CMA print '----------------- Run CMA analysis ----------------- ' cmaf = repo_sourcespace#repo_buildspace helper = subprocess.Popen(('pal QACPP -cmaf %s -list %s'%(cmaf, filelist)).split(' '), env=os.environ) helper.communicate() print '////////////////// cma analysis done ////////////////// \n\n' # Export metrics to yaml and csv files # get uri infos #uri= distro.get_repositories()[repo_list[0]].url repo_name = repo_list[0] repo_data = release.get_data()['repositories'][repo_name] print "repo_data", repo_data uri = repo_data['url'] uri_info = 'master' #repo_data['version'] vcs_type = 'git' # repo_data['type'] print '----------------- Export metrics to yaml and csv files ----------------- ' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/export_metrics_to_yaml_wet.py --path %s --path_src %s --doc metrics --csv csv --config %s/jenkins_scripts/code_quality/export_config.yaml --distro %s --stack %s --uri %s --uri_info %s --vcs_type %s'%(workspace, repo_buildspace, repo_sourcespace, workspace, ros_distro, repo_name, uri, uri_info, vcs_type)).split(' '), env=os.environ) helper.communicate() print '////////////////// export metrics to yaml and csv files done ////////////////// \n\n' # Push results to server print '----------------- Push results to server ----------------- ' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/push_results_to_server_wet.py --path %s --doc metrics --path_src %s --meta_package %s'%(workspace, repo_buildspace, repo_sourcespace, repo_list)).split(' '), env=os.environ) helper.communicate() print '////////////////// push results to server done ////////////////// \n\n' # Upload results to QAVerify print ' ----------------- upload results to QAVerify ----------------- ' shutil.rmtree(os.path.join(workspace, 'snapshots_path'), ignore_errors=True) os.makedirs(os.path.join(workspace, 'snapshots_path')) snapshots_path = '%s/snapshots_path'%workspace project_name = repo_list[0] + '-' + ros_distro helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/upload_to_QAVerify_wet.py --path %s --snapshot %s --project %s --stack_name %s'%(workspace, repo_buildspace, snapshots_path, project_name, repo_list[0])).split(' '), env=os.environ) helper.communicate() print '////////////////// upload results to QAVerify done ////////////////// \n\n' if os.path.exists(snapshots_path): shutil.rmtree(snapshots_path) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) if res != 0: print "helper_return_code is: %s"%(helper.returncode) assert 'analysis_wet.py failed' raise Exception("analysis_wet.py failed. Check out the console output above for details.") # create dummy test results env = dict() env['INSTALL_DIR'] = os.getenv('INSTALL_DIR', '') test_results_path = workspace + '/test_results' if os.path.exists(test_results_path): shutil.rmtree(test_results_path) os.makedirs(test_results_path) test_file= test_results_path + '/test_file.xml' f = open(test_file, 'w') f.write('<?xml version="1.0" encoding="UTF-8"?>\n') f.write('<testsuite tests="1" failures="0" time="1" errors="0" name="dummy test">\n') f.write(' <testcase name="dummy rapport" classname="Results" /> \n') f.write('</testsuite> \n') f.close()
def document_repo( workspace, docspace, ros_distro, repo, platform, arch, homepage, no_chroot, skip_garbage, doc_conf, depends_conf, tags_db, ): doc_job = "doc-%s-%s" % (ros_distro, repo) # Get the list of repositories that should have documentation run on them # These are all of the repos that are not in the depends rosinsall file repos_to_doc = get_repositories_from_rosinstall(doc_conf) repo_path = os.path.realpath("%s" % (docspace)) print("Repo path %s" % repo_path) # Walk through the installed repositories and find old-style packages, new-stye packages, and stacks stacks, manifest_packages, catkin_packages, repo_map = build_repo_structure(repo_path, doc_conf, depends_conf) if ros_distro == "indigo": if stacks or manifest_packages: print("Ignoring dry packages and stacks in '%s'" % ros_distro) stacks = {} manifest_packages = {} if not catkin_packages: raise BuildException("No catkin packages found") print("Running documentation generation on\npackages: %s" % (manifest_packages.keys() + catkin_packages.keys())) # print "Catkin packages: %s" % catkin_packages # print "Manifest packages: %s" % manifest_packages # print "Stacks: %s" % stacks # Get any non local apt dependencies ros_dep = rosdep.RosDepResolver(ros_distro, no_chroot=no_chroot) import rosdistro if ros_distro == "electric": apt = rosdistro.AptDistro(platform, arch, shadow=False) else: apt = rosdistro.AptDistro(platform, arch, shadow=True) apt_deps = get_apt_deps(apt, ros_dep, ros_distro, catkin_packages, stacks, manifest_packages) print("Apt dependencies: %s" % apt_deps) # Get rosdistro release file if there are catkin packages to get status if catkin_packages and ros_distro not in ["electric", "fuerte"]: print("Fetch rosdistro files for: %s" % ros_distro) index = rosdistro.get_index(rosdistro.get_index_url()) rosdistro_release_file = rosdistro.get_release_file(index, ros_distro) rosdistro_source_file = rosdistro.get_source_file(index, ros_distro) else: rosdistro_release_file = None rosdistro_source_file = None # Build a local dependency graph to be used for build order local_dep_graph = build_local_dependency_graph(catkin_packages, manifest_packages) doc_path = os.path.realpath("%s/doc/%s" % (docspace, ros_distro)) if os.path.exists(doc_path): shutil.rmtree(doc_path) # Write stack manifest files for all stacks, we can just do this off the # stack.xml files write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db, doc_job, homepage) # Need to make sure to re-order packages to be run in dependency order build_order = get_dependency_build_order(local_dep_graph) print("Build order that honors deps:\n%s" % build_order) # We'll need the full list of apt_deps to get tag files full_apt_deps = get_full_apt_deps(apt_deps, apt) if not no_chroot: print("Installing all dependencies for %s" % repo) # XXX this is a really ugly hack to make the hydro doc job for ros_comm pass # otherwise roslisp pulls in the rosgraph_msgs package as a Debian dependency # which then break catkin_basic since it include the msgs CMake multiple files # resulting in duplicate target names (https://github.com/ros/ros_comm/issues/471) if repo == "ros_comm" and "ros-hydro-roslisp" in apt_deps: apt_deps.remove("ros-hydro-roslisp") if apt_deps: call("apt-get install %s --yes" % (" ".join(apt_deps))) print("Done installing dependencies") # Set up the list of things that need to be sourced to run rosdoc_lite # TODO: Hack for electric if ros_distro == "electric": # lucid doesn't have /usr/local on the path by default... weird sources = ["export PATH=/usr/local/sbin:/usr/local/bin:$PATH"] sources.append("source /opt/ros/fuerte/setup.bash") sources.append("export ROS_PACKAGE_PATH=/opt/ros/electric/stacks:$ROS_PACKAGE_PATH") else: sources = ["source /opt/ros/%s/setup.bash" % ros_distro] # We assume that there will be no build errors to start build_errors = [] # Everything that is after fuerte supports catkin workspaces, so everything # that has packages with package.xml files local_install_path = os.path.join(docspace, "local_installs") if os.path.exists(local_install_path): shutil.rmtree(local_install_path) # Make sure to create some subfolders under the local install path def makedirs(path): if not os.path.exists(path): os.makedirs(path) makedirs(os.path.join(local_install_path, "bin")) makedirs(os.path.join(local_install_path, "lib/python2.7/dist-packages")) makedirs(os.path.join(local_install_path, "share")) if catkin_packages and not "rosdoc_lite" in catkin_packages.keys() and not "catkin" in catkin_packages.keys(): source, errs = build_repo_messages(catkin_packages, docspace, ros_distro, local_install_path) build_errors.extend(errs) if source: sources.append(source) # For fuerte catkin, we need to check if we should build catkin stacks source, errs = build_repo_messages_catkin_stacks(stacks, ros_distro, local_install_path) build_errors.extend(errs) sources.append(source) # For all our manifest packages (dry or fuerte catkin) we want to build # messages. Note, for fuerte catkin, we have to build all the code and # install locally to get message generation source, errs = build_repo_messages_manifest(manifest_packages, build_order, ros_distro) build_errors.extend(errs) sources.append(source) # We want to pull all the tagfiles available once from the server tags_location = os.path.join(workspace, ros_distro) if os.path.exists(tags_location): shutil.rmtree(tags_location) command = [ "bash", "-c", 'rsync -e "ssh -o StrictHostKeyChecking=no" -qrz [email protected]:/home/rosbot/docs/%s/tags %s' % (ros_distro, tags_location), ] call_with_list(command) repo_tags = document_packages( manifest_packages, catkin_packages, build_order, repos_to_doc, sources, tags_db, full_apt_deps, ros_dep, repo_map, repo_path, docspace, ros_distro, homepage, doc_job, tags_location, doc_path, rosdistro_release_file, rosdistro_source_file, ) # Copy the files to the appropriate place folders = sorted(set(stacks.keys() + manifest_packages.keys() + catkin_packages.keys())) if folders: dsts = ["%s/api/%s" % (doc_path, f) for f in folders] for dst in dsts: with open(os.path.join(dst, "stamp"), "w"): pass command = [ "bash", "-c", 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr --delete %s [email protected]:/home/rosbot/docs/%s/api' % (" ".join(dsts), ros_distro), ] call_with_list(command) folders = ["%s/changelogs" % doc_path, "%s/tags" % doc_path] folders = [f for f in folders if os.path.exists(f)] if folders: command = [ "bash", "-c", 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr %s [email protected]:/home/rosbot/docs/%s' % (" ".join(folders), ros_distro), ] call_with_list(command) if not skip_garbage: # Remove the autogenerated doc files since they take up a lot of space if left on the server shutil.rmtree(tags_location) shutil.rmtree(doc_path) # Write the new tags to the database if there are any to write for name, tags in repo_tags.iteritems(): # Get the apt name of the current stack/repo if ros_dep.has_ros(name): deb_name = ros_dep.to_apt(name)[0] else: deb_name = "ros-%s-%s" % (ros_distro, name.replace("_", "-")) # We only want to write tags for packages that have a valid deb name # For others, the only way to get cross referencing is to document everything # together with a rosinstall file if apt.has_package(deb_name): tags_db.set_tags(deb_name, tags) # Make sure to write changes to tag files and deps # We don't want to write hashes on an unsuccessful build excludes = ["rosinstall_hashes"] if build_errors else [] tags_db.commit_db(excludes) tags_db.delete_tag_index_repo() # Tell jenkins that we've succeeded print("Preparing xml test results") try: os.makedirs(os.path.join(workspace, "test_results")) print("Created test results directory") except Exception: pass if build_errors: import yaml copy_test_results( workspace, docspace, """Failed to generate messages by calling cmake for %s. Look in the console for cmake failures, search for "CMake Error" Also, are you sure that the rosinstall files are pulling from the right branch for %s? Check the repos below, you can update information the %s.rosinstall and %s-depends.rosinstall files by submitting a pull request at https://github.com/ros/rosdistro/%s Documentation rosinstall:\n%s Depends rosinstall:\n%s""" % ( build_errors, ros_distro, repo, repo, ros_distro, yaml.safe_dump(doc_conf, default_flow_style=False), yaml.safe_dump(depends_conf, default_flow_style=False), ), "message_generation_failure", ) else: copy_test_results(workspace, docspace)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env) #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories and tests print "Build repo list" call("make", ros_env) call("make tests", ros_env) # get the repositories test and run dependencies print "Get test and run dependencies of repo list" repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True) print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)) apt_get_install(repo_test_dependencies, rosdep_resolver, sudo) # run tests print "Test repo list" call("make run_tests", ros_env) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) # don't do depends-on on things not in release not_in_release = set(repo_list) - set(release.repositories.keys()) if not_in_release: print "Removed [%s] repositories which are not in the " %\ ', '.join(sorted(not_in_release)), \ "release file for depends-on testing" repo_list = list(set(repo_list) - not_in_release) # see if we need to do more work or not if not test_depends_on: print "We're not testing the depends-on repositories" ensure_test_results(test_results_dir) return # get repo_list depends-on list print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list) walker = DependencyWalker(release) depends_on = set([]) try: for repo_name in repo_list: print('repo_name', repo_name) repo = release.repositories[repo_name] for pkg_name in repo.package_names: print('pkg_name', pkg_name) depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on) print('depends_on', depends_on) except RuntimeError: print "Exception %s: If you are not in the rosdistro and only in the devel", \ " builds there will be no depends on" depends_on = set([]) print "Build depends_on list of pkg list: %s" % (', '.join(depends_on)) if len(depends_on) == 0: print "No wet packages depend on our repo list. Test finished here" ensure_test_results(test_results_dir) return # install depends_on packages from source from release repositories rosinstall = '' non_catkin_pkgs = [] for pkg_name in depends_on: repo = release.repositories[release.packages[pkg_name].repository_name] if repo.version is None: continue pkg_xml = release.get_package_xml(pkg_name) if pkg_xml is None: raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e)) if _is_non_catkin_package(pkg): non_catkin_pkgs.append(pkg.name) rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return print "Rosinstall for depends_on:\n %s" % rosinstall with open(workspace + "/depends_on.rosinstall", 'w') as f: f.write(rosinstall) print "Created rosinstall file for depends on" # install all repository and system dependencies of the depends_on list print "Install all depends_on from source: %s" % (', '.join(depends_on)) os.makedirs(dependson_sourcespace) call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace)) # check if depends_on workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return # get build and test dependencies of depends_on list dependson_build_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False): print " Checking dependency %s" % d if d in dependson_build_dependencies: print " Already in dependson_build_dependencies" if d in depends_on: print " Is a direct dependency of the repo list, and is installed from source" if d in repo_list: print " Is one of the repositories tested" if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list: dependson_build_dependencies.append(d) print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)) dependson_test_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True): if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list: dependson_test_dependencies.append(d) print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)) # install build dependencies print "Install all build dependencies of the depends_on list" apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file again print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt')) os.makedirs(dependson_buildspace) os.chdir(dependson_buildspace) print "Create a new CMakeLists.txt file using catkin" call("catkin_init_workspace %s" % dependson_sourcespace, ros_env) depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env) #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash')) # build repositories print "Build depends-on packages" call("make", ros_env) # install test dependencies print "Install all test dependencies of the depends_on list" apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo) # test repositories print "Test depends-on packages" call("make run_tests", ros_env) ensure_test_results(test_results_dir)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on, repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace, sudo=False, no_chroot=False): from catkin_pkg.package import InvalidPackage, parse_package_string from rosdistro import get_cached_release, get_index, get_index_url, get_source_file from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag index = get_index(get_index_url()) print "Parsing rosdistro file for %s" % ros_distro release = get_cached_release(index, ros_distro) print "Parsing devel file for %s" % ros_distro source_file = get_source_file(index, ros_distro) # Create rosdep object print "Create rosdep object" rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot) # download the repo_list from source print "Creating rosinstall file for repo list" rosinstall = "" for repo_name, version in zip(repo_list, version_list): if version == 'devel': if repo_name not in source_file.repositories: raise BuildException("Repository %s does not exist in Devel Distro" % repo_name) print "Using devel distro file to download repositories" rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name]) else: if repo_name not in release.repositories: raise BuildException("Repository %s does not exist in Ros Distro" % repo_name) repo = release.repositories[repo_name] if version not in ['latest', 'master']: assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name for pkg_name in repo.package_names: release_tag = get_release_tag(repo, pkg_name) if version in ['latest', 'master']: release_tag = '/'.join(release_tag.split('/')[:-1]) print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name) rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag) print "rosinstall file for all repositories: \n %s" % rosinstall with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f: f.write(rosinstall) print "Install repo list from source" os.makedirs(repo_sourcespace) call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace)) # get the repositories build dependencies print "Get build dependencies of repo list" repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False) # ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available if 'catkin' not in repo_build_dependencies: repo_build_dependencies.append('catkin') print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies)) apt_get_install(repo_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file for repositories that use catkin print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt')) print "Create a new CMakeLists.txt file using catkin" # get environment ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # check if source workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace) # make build folder and change into it os.makedirs(repo_buildspace) os.chdir(repo_buildspace) # make test results dir test_results_dir = os.path.join(workspace, 'test_results') if os.path.exists(test_results_dir): shutil.rmtree(test_results_dir) os.makedirs(test_results_dir) if not non_catkin_pkgs: print "Build catkin workspace" call("catkin_init_workspace %s" % repo_sourcespace, ros_env) repos_test_results_dir = os.path.join(test_results_dir, 'repos') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env) #ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash')) # build repositories and tests print "Build repo list" call("make", ros_env) call("make tests", ros_env) # get the repositories test and run dependencies print "Get test and run dependencies of repo list" repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True) print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies)) apt_get_install(repo_test_dependencies, rosdep_resolver, sudo) # run tests print "Test repo list" call("make run_tests", ros_env) else: print "Build workspace with non-catkin packages in isolation" # work around catkin_make_isolated issue (at least with version 0.5.65 of catkin) os.makedirs(os.path.join(repo_buildspace, 'devel_isolated')) call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env) setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh') ros_env = get_ros_env(setup_file) # see if we need to do more work or not if not test_depends_on: print "We're not testing the depends-on repositories" ensure_test_results(test_results_dir) return # get repo_list depends-on list print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list) walker = DependencyWalker(release) depends_on = set([]) try: for repo_name in repo_list: print('repo_name', repo_name) repo = release.repositories[repo_name] for pkg_name in repo.package_names: print('pkg_name', pkg_name) depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on) print('depends_on', depends_on) except RuntimeError: print "Exception %s: If you are not in the rosdistro and only in the devel", \ " builds there will be no depends on" depends_on = set([]) print "Build depends_on list of pkg list: %s" % (', '.join(depends_on)) if len(depends_on) == 0: print "No wet packages depend on our repo list. Test finished here" ensure_test_results(test_results_dir) return # install depends_on packages from source from release repositories rosinstall = '' non_catkin_pkgs = [] for pkg_name in depends_on: repo = release.repositories[release.packages[pkg_name].repository_name] if repo.version is None: continue pkg_xml = release.get_package_xml(pkg_name) if pkg_xml is None: raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name) try: pkg = parse_package_string(pkg_xml) except InvalidPackage as e: raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e)) if _is_non_catkin_package(pkg): non_catkin_pkgs.append(pkg.name) rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return print "Rosinstall for depends_on:\n %s" % rosinstall with open(workspace + "/depends_on.rosinstall", 'w') as f: f.write(rosinstall) print "Created rosinstall file for depends on" # install all repository and system dependencies of the depends_on list print "Install all depends_on from source: %s" % (', '.join(depends_on)) os.makedirs(dependson_sourcespace) call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace)) # check if depends_on workspace contains only package built with catkin non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace) if non_catkin_pkgs: print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs)) create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs))) return # get build and test dependencies of depends_on list dependson_build_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False): print " Checking dependency %s" % d if d in dependson_build_dependencies: print " Already in dependson_build_dependencies" if d in depends_on: print " Is a direct dependency of the repo list, and is installed from source" if d in repo_list: print " Is one of the repositories tested" if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list: dependson_build_dependencies.append(d) print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies)) dependson_test_dependencies = [] for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True): if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list: dependson_test_dependencies.append(d) print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies)) # install build dependencies print "Install all build dependencies of the depends_on list" apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo) # replace the CMakeLists.txt file again print "Removing the CMakeLists.txt file generated by rosinstall" os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt')) os.makedirs(dependson_buildspace) os.chdir(dependson_buildspace) print "Create a new CMakeLists.txt file using catkin" call("catkin_init_workspace %s" % dependson_sourcespace, ros_env) depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on') call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env) #ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash')) # build repositories print "Build depends-on packages" call("make", ros_env) # install test dependencies print "Install all test dependencies of the depends_on list" apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo) # test repositories print "Test depends-on packages" call("make run_tests", ros_env) ensure_test_results(test_results_dir)