def test_verify_files_parsable():
url = 'file://' + FILES_DIR + '/index.yaml'
index = get_index(url)
release_file = get_release_file(index, 'foo')
data = yaml_from_release_file(release_file)
with open(os.path.join(FILES_DIR, 'foo', 'release.yaml'), 'r') as f:
expected = f.read()
assert data == expected, get_diff(expected, data)
def handle_arguments(self, args):
self.interactive = args.interactive
self.debian_inc = args.debian_inc
self.os_name = args.os_name
self.distros = args.distros
if self.distros in [None, []]:
index = rosdistro.get_index(rosdistro.get_index_url())
release_file = rosdistro.get_release_file(index, self.rosdistro)
if self.os_name not in release_file.platforms:
error(
"No platforms defined for os '{0}' in release file for the '{1}' distro."
.format(self.os_name, self.rosdistro),
exit=True)
self.distros = release_file.platforms[self.os_name]
self.install_prefix = args.install_prefix
if args.install_prefix is None:
self.install_prefix = self.default_install_prefix
self.prefix = args.prefix
self.branches = match_branches_with_prefix(self.prefix, get_branches)
if len(self.branches) == 0:
error("No packages found, check your --prefix or --src arguments.",
exit=True)
self.packages = {}
self.tag_names = {}
self.names = []
self.branch_args = []
self.debian_branches = []
for branch in self.branches:
package = get_package_from_branch(branch)
if package is None:
# This is an ignored package
continue
self.packages[package.name] = package
self.names.append(package.name)
args = self.generate_branching_arguments(package, branch)
# First branch is debian/[<rosdistro>/]<package>
self.debian_branches.append(args[0][0])
self.branch_args.extend(args)
def handle_arguments(self, args):
self.interactive = args.interactive
self.debian_inc = args.debian_inc
self.os_name = args.os_name
self.distros = args.distros
if self.distros in [None, []]:
index = rosdistro.get_index(rosdistro.get_index_url())
release_file = rosdistro.get_release_file(index, self.rosdistro)
if self.os_name not in release_file.platforms:
error("No platforms defined for os '{0}' in release file for the '{1}' distro."
.format(self.os_name, self.rosdistro), exit=True)
self.distros = release_file.platforms[self.os_name]
self.install_prefix = args.install_prefix
if args.install_prefix is None:
self.install_prefix = self.default_install_prefix
self.prefix = args.prefix
self.branches = match_branches_with_prefix(self.prefix, get_branches)
if len(self.branches) == 0:
error(
"No packages found, check your --prefix or --src arguments.",
exit=True
)
self.packages = {}
self.tag_names = {}
self.names = []
self.branch_args = []
self.debian_branches = []
for branch in self.branches:
package = get_package_from_branch(branch)
if package is None:
# This is an ignored package
continue
self.packages[package.name] = package
self.names.append(package.name)
args = self.generate_branching_arguments(package, branch)
# First branch is debian/[<rosdistro>/]<package>
self.debian_branches.append(args[0][0])
self.branch_args.extend(args)
def get_release_file(distro):
_check_cache()
if distro not in _RDCache.release_files:
_RDCache.release_files[distro] = rosdistro.get_release_file(get_index(), distro)
return _RDCache.release_files[distro]
def get_target_distros(rosdistro):
print("Fetching targets")
index = get_index(get_index_url())
rel_file = get_release_file(index, rosdistro)
return rel_file.platforms['fedora']
def test_get_release_file():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
rel_file = get_release_file(i, 'foo')
_validate_rel_file(rel_file)
def document_repo(workspace, docspace, ros_distro, repo,
platform, arch, homepage, no_chroot, skip_garbage,
doc_conf, depends_conf, tags_db):
doc_job = "doc-%s-%s" % (ros_distro, repo)
#Get the list of repositories that should have documentation run on them
#These are all of the repos that are not in the depends rosinsall file
repos_to_doc = get_repositories_from_rosinstall(doc_conf)
repo_path = os.path.realpath("%s" % (docspace))
print("Repo path %s" % repo_path)
#Walk through the installed repositories and find old-style packages, new-stye packages, and stacks
stacks, manifest_packages, catkin_packages, repo_map = build_repo_structure(repo_path, doc_conf, depends_conf)
if ros_distro == 'indigo':
if stacks or manifest_packages:
print("Ignoring dry packages and stacks in '%s'" % ros_distro)
stacks = {}
manifest_packages = {}
if not catkin_packages:
raise BuildException('No catkin packages found')
print("Running documentation generation on\npackages: %s" % (manifest_packages.keys() + catkin_packages.keys()))
#print "Catkin packages: %s" % catkin_packages
#print "Manifest packages: %s" % manifest_packages
#print "Stacks: %s" % stacks
#Get any non local apt dependencies
ros_dep = rosdep.RosDepResolver(ros_distro, no_chroot=no_chroot)
import rosdistro
if ros_distro == 'electric':
apt = rosdistro.AptDistro(platform, arch, shadow=False)
else:
apt = rosdistro.AptDistro(platform, arch, shadow=True)
apt_deps = get_apt_deps(apt, ros_dep, ros_distro, catkin_packages, stacks, manifest_packages)
print("Apt dependencies: %s" % apt_deps)
#Get rosdistro release file if there are catkin packages to get status
if catkin_packages and ros_distro not in ['electric', 'fuerte']:
print("Fetch rosdistro files for: %s" % ros_distro)
index = rosdistro.get_index(rosdistro.get_index_url())
rosdistro_release_file = rosdistro.get_release_file(index, ros_distro)
rosdistro_source_file = rosdistro.get_source_file(index, ros_distro)
else:
rosdistro_release_file = None
rosdistro_source_file = None
#Build a local dependency graph to be used for build order
local_dep_graph = build_local_dependency_graph(catkin_packages, manifest_packages)
doc_path = os.path.realpath("%s/doc/%s" % (docspace, ros_distro))
if os.path.exists(doc_path):
shutil.rmtree(doc_path)
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db, doc_job, homepage)
#Need to make sure to re-order packages to be run in dependency order
build_order = get_dependency_build_order(local_dep_graph)
print("Build order that honors deps:\n%s" % build_order)
#We'll need the full list of apt_deps to get tag files
full_apt_deps = get_full_apt_deps(apt_deps, apt)
if not no_chroot:
print("Installing all dependencies for %s" % repo)
# XXX this is a really ugly hack to make the hydro doc job for ros_comm pass
# otherwise roslisp pulls in the rosgraph_msgs package as a Debian dependency
# which then break catkin_basic since it include the msgs CMake multiple files
# resulting in duplicate target names (https://github.com/ros/ros_comm/issues/471)
if repo == 'ros_comm' and 'ros-hydro-roslisp' in apt_deps:
apt_deps.remove('ros-hydro-roslisp')
if apt_deps:
call("apt-get install %s --yes" % (' '.join(apt_deps)))
print("Done installing dependencies")
#Set up the list of things that need to be sourced to run rosdoc_lite
#TODO: Hack for electric
if ros_distro == 'electric':
#lucid doesn't have /usr/local on the path by default... weird
sources = ['export PATH=/usr/local/sbin:/usr/local/bin:$PATH']
sources.append('source /opt/ros/fuerte/setup.bash')
sources.append('export ROS_PACKAGE_PATH=/opt/ros/electric/stacks:$ROS_PACKAGE_PATH')
else:
sources = ['source /opt/ros/%s/setup.bash' % ros_distro]
#We assume that there will be no build errors to start
build_errors = []
#Everything that is after fuerte supports catkin workspaces, so everything
#that has packages with package.xml files
local_install_path = os.path.join(docspace, 'local_installs')
if os.path.exists(local_install_path):
shutil.rmtree(local_install_path)
#Make sure to create some subfolders under the local install path
def makedirs(path):
if not os.path.exists(path):
os.makedirs(path)
makedirs(os.path.join(local_install_path, 'bin'))
makedirs(os.path.join(local_install_path, 'lib/python2.7/dist-packages'))
makedirs(os.path.join(local_install_path, 'share'))
if catkin_packages \
and not 'rosdoc_lite' in catkin_packages.keys() and not 'catkin' in catkin_packages.keys():
source, errs = build_repo_messages(catkin_packages, docspace, ros_distro, local_install_path)
build_errors.extend(errs)
if source:
sources.append(source)
#For fuerte catkin, we need to check if we should build catkin stacks
source, errs = build_repo_messages_catkin_stacks(stacks, ros_distro, local_install_path)
build_errors.extend(errs)
sources.append(source)
#For all our manifest packages (dry or fuerte catkin) we want to build
#messages. Note, for fuerte catkin, we have to build all the code and
#install locally to get message generation
source, errs = build_repo_messages_manifest(manifest_packages, build_order, ros_distro)
build_errors.extend(errs)
sources.append(source)
#We want to pull all the tagfiles available once from the server
tags_location = os.path.join(workspace, ros_distro)
if os.path.exists(tags_location):
shutil.rmtree(tags_location)
command = ['bash', '-c',
'rsync -e "ssh -o StrictHostKeyChecking=no" -qrz [email protected]:/home/rosbot/docs/%s/tags %s' % (ros_distro, tags_location)]
call_with_list(command)
repo_tags = document_packages(manifest_packages, catkin_packages, build_order,
repos_to_doc, sources, tags_db, full_apt_deps,
ros_dep, repo_map, repo_path, docspace, ros_distro,
homepage, doc_job, tags_location, doc_path,
rosdistro_release_file, rosdistro_source_file)
#Copy the files to the appropriate place
folders = sorted(set(stacks.keys() + manifest_packages.keys() + catkin_packages.keys()))
if folders:
dsts = ['%s/api/%s' % (doc_path, f) for f in folders]
for dst in dsts:
with open(os.path.join(dst, 'stamp'), 'w'):
pass
command = ['bash', '-c', 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr --delete %s [email protected]:/home/rosbot/docs/%s/api' % (' '.join(dsts), ros_distro)]
call_with_list(command)
folders = ['%s/changelogs' % doc_path, '%s/tags' % doc_path]
folders = [f for f in folders if os.path.exists(f)]
if folders:
command = ['bash', '-c', 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr %s [email protected]:/home/rosbot/docs/%s' % (' '.join(folders), ros_distro)]
call_with_list(command)
if not skip_garbage:
#Remove the autogenerated doc files since they take up a lot of space if left on the server
shutil.rmtree(tags_location)
shutil.rmtree(doc_path)
#Write the new tags to the database if there are any to write
for name, tags in repo_tags.iteritems():
#Get the apt name of the current stack/repo
if ros_dep.has_ros(name):
deb_name = ros_dep.to_apt(name)[0]
else:
deb_name = "ros-%s-%s" % (ros_distro, name.replace('_', '-'))
#We only want to write tags for packages that have a valid deb name
#For others, the only way to get cross referencing is to document everything
#together with a rosinstall file
if apt.has_package(deb_name):
tags_db.set_tags(deb_name, tags)
#Make sure to write changes to tag files and deps
#We don't want to write hashes on an unsuccessful build
excludes = ['rosinstall_hashes'] if build_errors else []
tags_db.commit_db(excludes)
tags_db.delete_tag_index_repo()
#Tell jenkins that we've succeeded
print("Preparing xml test results")
try:
os.makedirs(os.path.join(workspace, 'test_results'))
print("Created test results directory")
except Exception:
pass
if build_errors:
import yaml
copy_test_results(workspace, docspace,
"""Failed to generate messages by calling cmake for %s.
Look in the console for cmake failures, search for "CMake Error"
Also, are you sure that the rosinstall files are pulling from the right branch for %s? Check the repos below,
you can update information the %s.rosinstall and %s-depends.rosinstall files by submitting a pull request at
https://github.com/ros/rosdistro/%s
Documentation rosinstall:\n%s
Depends rosinstall:\n%s""" % (build_errors,
ros_distro,
repo,
repo,
ros_distro,
yaml.safe_dump(doc_conf, default_flow_style=False),
yaml.safe_dump(depends_conf, default_flow_style=False)),
"message_generation_failure")
else:
copy_test_results(workspace, docspace)
def test_get_release_file():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
rel_file = get_release_file(i, 'foo')
_validate_rel_file(rel_file)
def get_release_file(distro):
global _rosdistro_release_files
if distro not in _rosdistro_release_files:
_rosdistro_release_files[distro] = rosdistro.get_release_file(get_index(), distro)
return _rosdistro_release_files[distro]
def _get_rosdistro_release(distro):
index = rosdistro.get_index(rosdistro.get_index_url())
return rosdistro.get_release_file(index, distro)
def document_repo(
workspace,
docspace,
ros_distro,
repo,
platform,
arch,
homepage,
no_chroot,
skip_garbage,
doc_conf,
depends_conf,
tags_db,
):
doc_job = "doc-%s-%s" % (ros_distro, repo)
# Get the list of repositories that should have documentation run on them
# These are all of the repos that are not in the depends rosinsall file
repos_to_doc = get_repositories_from_rosinstall(doc_conf)
repo_path = os.path.realpath("%s" % (docspace))
print("Repo path %s" % repo_path)
# Walk through the installed repositories and find old-style packages, new-stye packages, and stacks
stacks, manifest_packages, catkin_packages, repo_map = build_repo_structure(repo_path, doc_conf, depends_conf)
if ros_distro == "indigo":
if stacks or manifest_packages:
print("Ignoring dry packages and stacks in '%s'" % ros_distro)
stacks = {}
manifest_packages = {}
if not catkin_packages:
raise BuildException("No catkin packages found")
print("Running documentation generation on\npackages: %s" % (manifest_packages.keys() + catkin_packages.keys()))
# print "Catkin packages: %s" % catkin_packages
# print "Manifest packages: %s" % manifest_packages
# print "Stacks: %s" % stacks
# Get any non local apt dependencies
ros_dep = rosdep.RosDepResolver(ros_distro, no_chroot=no_chroot)
import rosdistro
if ros_distro == "electric":
apt = rosdistro.AptDistro(platform, arch, shadow=False)
else:
apt = rosdistro.AptDistro(platform, arch, shadow=True)
apt_deps = get_apt_deps(apt, ros_dep, ros_distro, catkin_packages, stacks, manifest_packages)
print("Apt dependencies: %s" % apt_deps)
# Get rosdistro release file if there are catkin packages to get status
if catkin_packages and ros_distro not in ["electric", "fuerte"]:
print("Fetch rosdistro files for: %s" % ros_distro)
index = rosdistro.get_index(rosdistro.get_index_url())
rosdistro_release_file = rosdistro.get_release_file(index, ros_distro)
rosdistro_source_file = rosdistro.get_source_file(index, ros_distro)
else:
rosdistro_release_file = None
rosdistro_source_file = None
# Build a local dependency graph to be used for build order
local_dep_graph = build_local_dependency_graph(catkin_packages, manifest_packages)
doc_path = os.path.realpath("%s/doc/%s" % (docspace, ros_distro))
if os.path.exists(doc_path):
shutil.rmtree(doc_path)
# Write stack manifest files for all stacks, we can just do this off the
# stack.xml files
write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db, doc_job, homepage)
# Need to make sure to re-order packages to be run in dependency order
build_order = get_dependency_build_order(local_dep_graph)
print("Build order that honors deps:\n%s" % build_order)
# We'll need the full list of apt_deps to get tag files
full_apt_deps = get_full_apt_deps(apt_deps, apt)
if not no_chroot:
print("Installing all dependencies for %s" % repo)
# XXX this is a really ugly hack to make the hydro doc job for ros_comm pass
# otherwise roslisp pulls in the rosgraph_msgs package as a Debian dependency
# which then break catkin_basic since it include the msgs CMake multiple files
# resulting in duplicate target names (https://github.com/ros/ros_comm/issues/471)
if repo == "ros_comm" and "ros-hydro-roslisp" in apt_deps:
apt_deps.remove("ros-hydro-roslisp")
if apt_deps:
call("apt-get install %s --yes" % (" ".join(apt_deps)))
print("Done installing dependencies")
# Set up the list of things that need to be sourced to run rosdoc_lite
# TODO: Hack for electric
if ros_distro == "electric":
# lucid doesn't have /usr/local on the path by default... weird
sources = ["export PATH=/usr/local/sbin:/usr/local/bin:$PATH"]
sources.append("source /opt/ros/fuerte/setup.bash")
sources.append("export ROS_PACKAGE_PATH=/opt/ros/electric/stacks:$ROS_PACKAGE_PATH")
else:
sources = ["source /opt/ros/%s/setup.bash" % ros_distro]
# We assume that there will be no build errors to start
build_errors = []
# Everything that is after fuerte supports catkin workspaces, so everything
# that has packages with package.xml files
local_install_path = os.path.join(docspace, "local_installs")
if os.path.exists(local_install_path):
shutil.rmtree(local_install_path)
# Make sure to create some subfolders under the local install path
def makedirs(path):
if not os.path.exists(path):
os.makedirs(path)
makedirs(os.path.join(local_install_path, "bin"))
makedirs(os.path.join(local_install_path, "lib/python2.7/dist-packages"))
makedirs(os.path.join(local_install_path, "share"))
if catkin_packages and not "rosdoc_lite" in catkin_packages.keys() and not "catkin" in catkin_packages.keys():
source, errs = build_repo_messages(catkin_packages, docspace, ros_distro, local_install_path)
build_errors.extend(errs)
if source:
sources.append(source)
# For fuerte catkin, we need to check if we should build catkin stacks
source, errs = build_repo_messages_catkin_stacks(stacks, ros_distro, local_install_path)
build_errors.extend(errs)
sources.append(source)
# For all our manifest packages (dry or fuerte catkin) we want to build
# messages. Note, for fuerte catkin, we have to build all the code and
# install locally to get message generation
source, errs = build_repo_messages_manifest(manifest_packages, build_order, ros_distro)
build_errors.extend(errs)
sources.append(source)
# We want to pull all the tagfiles available once from the server
tags_location = os.path.join(workspace, ros_distro)
if os.path.exists(tags_location):
shutil.rmtree(tags_location)
command = [
"bash",
"-c",
'rsync -e "ssh -o StrictHostKeyChecking=no" -qrz [email protected]:/home/rosbot/docs/%s/tags %s'
% (ros_distro, tags_location),
]
call_with_list(command)
repo_tags = document_packages(
manifest_packages,
catkin_packages,
build_order,
repos_to_doc,
sources,
tags_db,
full_apt_deps,
ros_dep,
repo_map,
repo_path,
docspace,
ros_distro,
homepage,
doc_job,
tags_location,
doc_path,
rosdistro_release_file,
rosdistro_source_file,
)
# Copy the files to the appropriate place
folders = sorted(set(stacks.keys() + manifest_packages.keys() + catkin_packages.keys()))
if folders:
dsts = ["%s/api/%s" % (doc_path, f) for f in folders]
for dst in dsts:
with open(os.path.join(dst, "stamp"), "w"):
pass
command = [
"bash",
"-c",
'rsync -e "ssh -o StrictHostKeyChecking=no" -qr --delete %s [email protected]:/home/rosbot/docs/%s/api'
% (" ".join(dsts), ros_distro),
]
call_with_list(command)
folders = ["%s/changelogs" % doc_path, "%s/tags" % doc_path]
folders = [f for f in folders if os.path.exists(f)]
if folders:
command = [
"bash",
"-c",
'rsync -e "ssh -o StrictHostKeyChecking=no" -qr %s [email protected]:/home/rosbot/docs/%s'
% (" ".join(folders), ros_distro),
]
call_with_list(command)
if not skip_garbage:
# Remove the autogenerated doc files since they take up a lot of space if left on the server
shutil.rmtree(tags_location)
shutil.rmtree(doc_path)
# Write the new tags to the database if there are any to write
for name, tags in repo_tags.iteritems():
# Get the apt name of the current stack/repo
if ros_dep.has_ros(name):
deb_name = ros_dep.to_apt(name)[0]
else:
deb_name = "ros-%s-%s" % (ros_distro, name.replace("_", "-"))
# We only want to write tags for packages that have a valid deb name
# For others, the only way to get cross referencing is to document everything
# together with a rosinstall file
if apt.has_package(deb_name):
tags_db.set_tags(deb_name, tags)
# Make sure to write changes to tag files and deps
# We don't want to write hashes on an unsuccessful build
excludes = ["rosinstall_hashes"] if build_errors else []
tags_db.commit_db(excludes)
tags_db.delete_tag_index_repo()
# Tell jenkins that we've succeeded
print("Preparing xml test results")
try:
os.makedirs(os.path.join(workspace, "test_results"))
print("Created test results directory")
except Exception:
pass
if build_errors:
import yaml
copy_test_results(
workspace,
docspace,
"""Failed to generate messages by calling cmake for %s.
Look in the console for cmake failures, search for "CMake Error"
Also, are you sure that the rosinstall files are pulling from the right branch for %s? Check the repos below,
you can update information the %s.rosinstall and %s-depends.rosinstall files by submitting a pull request at
https://github.com/ros/rosdistro/%s
Documentation rosinstall:\n%s
Depends rosinstall:\n%s"""
% (
build_errors,
ros_distro,
repo,
repo,
ros_distro,
yaml.safe_dump(doc_conf, default_flow_style=False),
yaml.safe_dump(depends_conf, default_flow_style=False),
),
"message_generation_failure",
)
else:
copy_test_results(workspace, docspace)
def _get_rosdistro_release(distro):
index = rosdistro.get_index(rosdistro.get_index_url())
return rosdistro.get_release_file(index, distro)
def get_release_file(distro):
global _rosdistro_release_files
if distro not in _rosdistro_release_files:
_rosdistro_release_files[distro] = rosdistro.get_release_file(
get_index(), distro)
return _rosdistro_release_files[distro]
