def get_nonlocal_dependencies(catkin_packages, stacks, manifest_packages):
append_pymodules_if_needed()
from catkin_pkg import packages
import rospkg
depends = []
#First, we build the catkin deps
for name, path in catkin_packages.iteritems():
pkg_info = packages.parse_package(path)
depends.extend([d.name \
for d in pkg_info.build_depends + pkg_info.test_depends + pkg_info.run_depends \
if not d.name in catkin_packages and not d.name in depends])
#Next, we build the manifest deps for stacks
for name, path in stacks.iteritems():
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
depends.extend([d.name \
for d in stack_manifest.depends + stack_manifest.rosdeps \
if not d.name in catkin_packages \
and not d.name in stacks \
and not d.name in depends])
#Next, we build manifest deps for packages
for name, path in manifest_packages.iteritems():
pkg_manifest = rospkg.parse_manifest_file(path, rospkg.MANIFEST_FILE)
depends.extend([d.name \
for d in pkg_manifest.depends + pkg_manifest.rosdeps \
if not d.name in catkin_packages \
and not d.name in stacks \
and not d.name in manifest_packages \
and not d.name in depends])
return depends
def get_nonlocal_dependencies(catkin_packages, stacks, manifest_packages):
append_pymodules_if_needed()
from catkin_pkg import packages
import rospkg
depends = []
#First, we build the catkin deps
for name, path in catkin_packages.iteritems():
pkg_info = packages.parse_package(path)
depends.extend([d.name \
for d in pkg_info.build_depends + pkg_info.test_depends + pkg_info.run_depends \
if not d.name in catkin_packages and not d.name in depends])
#Next, we build the manifest deps for stacks
for name, path in stacks.iteritems():
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
depends.extend([d.name \
for d in stack_manifest.depends + stack_manifest.rosdeps \
if not d.name in catkin_packages \
and not d.name in stacks \
and not d.name in depends])
#Next, we build manifest deps for packages
for name, path in manifest_packages.iteritems():
pkg_manifest = rospkg.parse_manifest_file(path, rospkg.MANIFEST_FILE)
depends.extend([d.name \
for d in pkg_manifest.depends + pkg_manifest.rosdeps \
if not d.name in catkin_packages \
and not d.name in stacks \
and not d.name in manifest_packages \
and not d.name in depends])
return depends
def get_nonlocal_dependencies(catkin_packages,
stacks,
manifest_packages,
build_depends=True,
test_depends=True):
append_pymodules_if_needed()
from catkin_pkg import packages
import rospkg
depends = []
#First, we build the catkin deps
for name, path in catkin_packages.iteritems():
pkg_info = packages.parse_package(path)
if build_depends:
depends.extend([
d.name
for d in pkg_info.buildtool_depends + pkg_info.build_depends
if not d.name in catkin_packages and not d.name in depends
])
if test_depends:
depends.extend([
d.name for d in pkg_info.test_depends + pkg_info.run_depends
if not d.name in catkin_packages and not d.name in depends
])
#Next, we build the manifest deps for stacks
for name, path in stacks.iteritems():
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
if stack_manifest.is_catkin:
depends.extend(
get_catkin_stack_deps(os.path.join(path, 'stack.xml')))
else:
depends.extend([
d.name for d in stack_manifest.depends + stack_manifest.rosdeps
if not d.name in catkin_packages and not d.name in stacks
and not d.name in depends
])
#Next, we build manifest deps for packages
for name, path in manifest_packages.iteritems():
pkg_manifest = rospkg.parse_manifest_file(path, rospkg.MANIFEST_FILE)
depends.extend([
d.name for d in pkg_manifest.depends + pkg_manifest.rosdeps
if not d.name in catkin_packages and not d.name in stacks
and not d.name in manifest_packages and not d.name in depends
])
return depends
def convert_stack_to_debian_data(self, stack):
data = {}
# Name, Version, Description
data['Name'] = stack.name
data['Version'] = stack.version
data['Description'] = debianize_string(stack.description)
# Website
data['Homepage'] = stack.url
# Build rule template
data['BuildType'] = 'cmake'
# Copyright
data['Copyright'] = stack.copyright
# Debian Increment Number
data['DebianInc'] = self.debian_inc
# Package name
data['Package'] = self.get_stackage_name(stack)
# Installation prefix
data['InstallationPrefix'] = self.install_prefix
# Dependencies
data['Depends'] = set([d.name for d in stack.depends])
data['BuildDepends'] = set([d.name for d in stack.build_depends])
# Maintainers
maintainers = []
for m in stack.maintainers:
maintainer = m.name
if m.email:
maintainer += ' <%s>' % m.email
maintainers.append(maintainer)
data['Maintainer'] = ', '.join(maintainers)
### Augment Description with Package list
# Go over the different subfolders and find all the packages
package_descriptions = {}
# search for manifest in current folder and direct subfolders
cwd = os.getcwd()
for dir_name in [cwd] + os.listdir(cwd):
if not os.path.isdir(dir_name):
continue
dir_path = os.path.join('.', dir_name)
for file_name in os.listdir(dir_path):
if file_name == 'manifest.xml':
# Can cold import,
# because bloom.util already checked for rospkg
import rospkg
# parse the manifest, in case it is not valid
manifest = rospkg.parse_manifest_file(dir_path, file_name)
# remove markups
if manifest.description is None:
manifest.description = ''
description = debianize_string(manifest.description)
if dir_name == '.':
dir_name = stack.name
package_descriptions[dir_name] = description
# Enhance the description with the list of packages in the stack
if package_descriptions:
if data['Description']:
data['Description'] += '\n .\n'
data['Description'] += ' This stack contains the packages:'
for name, description in package_descriptions.items():
data['Description'] += '\n * %s: %s' % (name, description)
return data
def plugin_help_request(self, plugin_descriptor):
package_name = plugin_descriptor.attributes()['package_name']
package_path = get_package_path(package_name)
try:
manifest = parse_manifest_file(package_path, MANIFEST_FILE)
except (InvalidManifest, IOError):
return
webbrowser.open(manifest.url)
def plugin_help_request(self, plugin_descriptor):
package_name = plugin_descriptor.attributes()['package_name']
package_path = get_package_path(package_name)
try:
manifest = parse_manifest_file(package_path, MANIFEST_FILE)
except (InvalidManifest, IOError):
return
webbrowser.open(manifest.url)
def write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db, doc_job, homepage):
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
for stack, path in stacks.iteritems():
import rospkg
#Get the dependencies of a dry stack from the stack.xml
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
stack_packages = get_repo_manifests(path, manifest='package').keys()
stack_relative_doc_path = "%s/doc/%s/api/%s" % (docspace, ros_distro, stack)
stack_doc_path = os.path.abspath(stack_relative_doc_path)
write_stack_manifest(stack_doc_path, stack, stack_manifest, repo_map[stack]['type'], repo_map[stack]['url'], "%s/%s/api/%s/html" % (homepage, ros_distro, stack), stack_packages, tags_db, repo_map[stack]['name'], doc_job, repo_map[stack]['version'])
def get_nonlocal_dependencies(catkin_packages, stacks, manifest_packages, build_depends=True, test_depends=True):
append_pymodules_if_needed()
from catkin_pkg import packages
import rospkg
depends = []
#First, we build the catkin deps
for name, path in catkin_packages.iteritems():
pkg_info = packages.parse_package(path)
if build_depends:
depends.extend([d.name
for d in pkg_info.buildtool_depends + pkg_info.build_depends
if not d.name in catkin_packages and not d.name in depends])
if test_depends:
depends.extend([d.name
for d in pkg_info.test_depends + pkg_info.run_depends
if not d.name in catkin_packages and not d.name in depends])
#Next, we build the manifest deps for stacks
for name, path in stacks.iteritems():
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
if stack_manifest.is_catkin:
depends.extend(get_catkin_stack_deps(os.path.join(path, 'stack.xml')))
else:
depends.extend([d.name
for d in stack_manifest.depends + stack_manifest.rosdeps
if not d.name in catkin_packages
and not d.name in stacks
and not d.name in depends])
#Next, we build manifest deps for packages
for name, path in manifest_packages.iteritems():
pkg_manifest = rospkg.parse_manifest_file(path, rospkg.MANIFEST_FILE)
depends.extend([d.name
for d in pkg_manifest.depends + pkg_manifest.rosdeps
if not d.name in catkin_packages
and not d.name in stacks
and not d.name in manifest_packages
and not d.name in depends])
return depends
def write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db, doc_job, homepage):
# ensure folder exists to not fail the subsequent commands in doc_stack
distro_path = os.path.join(docspace, 'doc', ros_distro)
if not os.path.exists(distro_path):
os.makedirs(distro_path)
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
for stack, path in stacks.iteritems():
import rospkg
#Get the dependencies of a dry stack from the stack.xml
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
stack_packages = get_repo_manifests(path, manifest='package').keys()
stack_relative_doc_path = "%s/api/%s" % (distro_path, stack)
stack_doc_path = os.path.abspath(stack_relative_doc_path)
write_stack_manifest(stack_doc_path, stack, stack_manifest, repo_map[stack]['type'], repo_map[stack]['url'], "%s/%s/api/%s/html" % (homepage, ros_distro, stack), stack_packages, tags_db, repo_map[stack]['name'], doc_job, repo_map[stack]['version'])
def __init__(self):
self.rospack = rospkg.RosPack()
PACKAGE_NAME = rospkg.get_package_name(inspect.getfile(inspect.currentframe()))
PACKAGE_PATH = self.rospack.get_path(PACKAGE_NAME)
DOCUMENTATION_FOLDER = "pkg_documentation"
self.DOCUMENTATION_PATH = os.path.join(PACKAGE_PATH, DOCUMENTATION_FOLDER)
self.EXTENSION = '.rst'
self.DOC = "_doc"
self.CONFIG_FILENAME = "conf.py"
self.MAIN_RST_FILE = os.path.join(self.DOCUMENTATION_PATH, 'pkg' + self.DOC + self.EXTENSION)
self.CONFIG_FILE = os.path.join(PACKAGE_PATH, 'src', self.CONFIG_FILENAME)
self.MAIN_CONF_FILE = os.path.join(PACKAGE_PATH , self.CONFIG_FILENAME )
self.packages = parse_manifest_file(PACKAGE_PATH, "manifest.xml").depends
self.PACKAGES_MAP = {}
def write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db,
doc_job, homepage):
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
for stack, path in stacks.iteritems():
import rospkg
#Get the dependencies of a dry stack from the stack.xml
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
stack_packages = get_repo_manifests(path, manifest='package').keys()
stack_relative_doc_path = "%s/doc/%s/api/%s" % (docspace, ros_distro,
stack)
stack_doc_path = os.path.abspath(stack_relative_doc_path)
write_stack_manifest(
stack_doc_path, stack, stack_manifest, repo_map[stack]['type'],
repo_map[stack]['url'],
"%s/%s/api/%s/html" % (homepage, ros_distro, stack),
stack_packages, tags_db, repo_map[stack]['name'], doc_job,
repo_map[stack]['version'])
def write_stack_manifests(stacks, docspace, ros_distro, repo_map, tags_db,
doc_job, homepage):
# ensure folder exists to not fail the subsequent commands in doc_stack
distro_path = os.path.join(docspace, 'doc', ros_distro)
if not os.path.exists(distro_path):
os.makedirs(distro_path)
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
for stack, path in stacks.iteritems():
import rospkg
#Get the dependencies of a dry stack from the stack.xml
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
stack_packages = get_repo_manifests(path, manifest='package').keys()
stack_relative_doc_path = "%s/api/%s" % (distro_path, stack)
stack_doc_path = os.path.abspath(stack_relative_doc_path)
write_stack_manifest(
stack_doc_path, stack, stack_manifest, repo_map[stack]['type'],
repo_map[stack]['url'],
"%s/%s/api/%s/html" % (homepage, ros_distro, stack),
stack_packages, tags_db, repo_map[stack]['name'], doc_job,
repo_map[stack]['version'])
def build_local_dependency_graph(catkin_packages, manifest_packages):
append_pymodules_if_needed()
from catkin_pkg import packages
import rospkg
depends = {}
#First, we build the catkin dep tree
for name, path in catkin_packages.iteritems():
depends[name] = []
pkg_info = packages.parse_package(path)
for d in pkg_info.buildtool_depends + pkg_info.build_depends + pkg_info.test_depends + pkg_info.run_depends:
if d.name in catkin_packages and d.name != name:
depends[name].append(d.name)
#Next, we build the manifest dep tree
for name, path in manifest_packages.iteritems():
manifest = rospkg.parse_manifest_file(path, rospkg.MANIFEST_FILE)
depends[name] = []
for d in manifest.depends + manifest.rosdeps:
if (d.name in catkin_packages or d.name in manifest_packages) and d.name != name:
depends[name].append(str(d.name))
return depends
def build_local_dependency_graph(catkin_packages, manifest_packages):
append_pymodules_if_needed()
from catkin_pkg import packages
import rospkg
depends = {}
#First, we build the catkin dep tree
for name, path in catkin_packages.iteritems():
depends[name] = []
pkg_info = packages.parse_package(path)
for dep in pkg_info.buildtool_depends + pkg_info.build_depends + pkg_info.test_depends + pkg_info.run_depends:
if dep.name in catkin_packages and dep.name != name:
depends[name].append(dep.name)
#Next, we build the manifest dep tree
for name, path in manifest_packages.iteritems():
manifest = rospkg.parse_manifest_file(path, rospkg.MANIFEST_FILE)
depends[name] = []
for dep in manifest.depends + manifest.rosdeps:
if (dep.name in catkin_packages or dep.name in manifest_packages) and dep.name != name:
depends[name].append(str(dep.name))
return depends
def process_package_xml(args, directory=None):
cwd = directory if directory else '.'
xml_path = os.path.join(cwd, 'package.xml')
if not os.path.exists(xml_path):
bailout("No package.xml file found at: {0}".format(xml_path))
try:
from catkin_pkg.package import parse_package
except ImportError:
error("catkin_pkg was not detected, please install it.",
file=sys.stderr)
sys.exit(1)
package = parse_package(xml_path)
data = {}
data['Name'] = package.name
data['Version'] = package.version
data['Description'] = debianize_string(package.description)
websites = [str(url) for url in package.urls if url.type == 'website']
homepage = websites[0] if websites else ''
if homepage == '':
warning("No homepage set, defaulting to ''")
data['Homepage'] = homepage
data['Catkin-ChangelogType'] = ''
data['Catkin-DebRulesType'] = 'cmake'
data['Catkin-DebRulesFile'] = ''
# data['Catkin-CopyrightType'] = package.copyright
# data['copyright'] = package.copyright
data['DebianInc'] = args.debian_revision
if args.rosdistro == 'backports':
data['Package'] = sanitize_package_name("%s" % (package.name))
else:
data['Package'] = \
sanitize_package_name("ros-%s-%s" % (args.rosdistro, package.name))
data['ROS_DISTRO'] = args.rosdistro
# allow override of these values
if args.rosdistro == 'backports':
data['INSTALL_PREFIX'] = \
args.install_prefix if args.install_prefix != None else '/usr'
else:
data['INSTALL_PREFIX'] = \
args.install_prefix if args.install_prefix != None \
else '/opt/ros/%s' % args.rosdistro
data['Depends'] = set([d.name for d in package.run_depends])
build_deps = (package.build_depends + package.buildtool_depends)
data['BuildDepends'] = set([d.name for d in build_deps])
maintainers = []
for m in package.maintainers:
maintainers.append(str(m))
data['Maintainer'] = ', '.join(maintainers)
# Go over the different subfolders and find all the packages
package_descriptions = {}
# search for manifest in current folder and direct subfolders
for dir_name in [cwd] + os.listdir(cwd):
if not os.path.isdir(dir_name):
continue
dir_path = os.path.join('.', dir_name)
for file_name in os.listdir(dir_path):
if file_name == 'manifest.xml':
# parse the manifest, in case it is not valid
manifest = rospkg.parse_manifest_file(dir_path, file_name)
# remove markups
if manifest.description is None:
manifest.description = ''
description = debianize_string(manifest.description)
if dir_name == '.':
dir_name = package.name
package_descriptions[dir_name] = description
# Enhance the description with the list of packages in the stack
if package_descriptions:
if data['Description']:
data['Description'] += '\n .\n'
data['Description'] += ' This stack contains the packages:'
for name, description in package_descriptions.items():
data['Description'] += '\n * %s: %s' % (name, description)
return data
def process_stack_xml(args, cwd=None):
"""Reads in a stack.xml file and does some post processing on it"""
cwd = cwd if cwd else '.'
xml_path = os.path.join(cwd, 'stack.xml')
if not os.path.exists(xml_path):
bailout("No stack.xml file found at: {0}".format(xml_path))
stack = rospkg.stack.parse_stack_file(xml_path)
data = {}
data['Name'] = stack.name
data['Version'] = stack.version
data['Description'] = debianize_string(stack.description)
data['Homepage'] = stack.url
data['Catkin-ChangelogType'] = ''
data['Catkin-DebRulesType'] = stack.build_type
data['Catkin-DebRulesFile'] = stack.build_type_file
data['Catkin-CopyrightType'] = stack.copyright
data['copyright'] = stack.copyright
data['DebianInc'] = args.debian_revision
if args.rosdistro == 'backports':
data['Package'] = sanitize_package_name("%s" % (stack.name))
else:
data['Package'] = \
sanitize_package_name("ros-%s-%s" % (args.rosdistro, stack.name))
data['ROS_DISTRO'] = args.rosdistro
# allow override of these values
if args.rosdistro == 'backports':
data['INSTALL_PREFIX'] = \
args.install_prefix if args.install_prefix != None else '/usr'
else:
data['INSTALL_PREFIX'] = \
args.install_prefix if args.install_prefix != None \
else '/opt/ros/%s' % args.rosdistro
data['Depends'] = set([d.name for d in stack.depends])
data['BuildDepends'] = set([d.name for d in stack.build_depends])
maintainers = []
for m in stack.maintainers:
maintainer = m.name
if m.email:
maintainer += ' <%s>' % m.email
maintainers.append(maintainer)
data['Maintainer'] = ', '.join(maintainers)
# Go over the different subfolders and find all the packages
package_descriptions = {}
# search for manifest in current folder and direct subfolders
for dir_name in [cwd] + os.listdir(cwd):
if not os.path.isdir(dir_name):
continue
dir_path = os.path.join('.', dir_name)
for file_name in os.listdir(dir_path):
if file_name == 'manifest.xml':
# parse the manifest, in case it is not valid
manifest = rospkg.parse_manifest_file(dir_path, file_name)
# remove markups
if manifest.description is None:
manifest.description = ''
description = debianize_string(manifest.description)
if dir_name == '.':
dir_name = stack.name
package_descriptions[dir_name] = description
# Enhance the description with the list of packages in the stack
if package_descriptions:
if data['Description']:
data['Description'] += '\n .\n'
data['Description'] += ' This stack contains the packages:'
for name, description in package_descriptions.items():
data['Description'] += '\n * %s: %s' % (name, description)
return data
def document_repo(workspace, docspace, ros_distro, repo, platform, arch):
append_pymodules_if_needed()
print "Working on distro %s and repo %s" % (ros_distro, repo)
try:
repo_url = 'https://raw.github.com/ros/rosdistro/master/doc/%s/%s.rosinstall'%(ros_distro, repo)
f = urllib2.urlopen(repo_url)
if f.code != 200:
raise BuildException("Could not find a valid rosinstall file for %s at %s" % (repo, repo_url))
conf = yaml.load(f.read())
except (urllib2.URLError, urllib2.HTTPError) as e:
raise BuildException("Could not find a valid rosinstall file for %s at %s" % (repo, repo_url))
depends_conf = []
try:
depends_repo_url = 'https://raw.github.com/ros/rosdistro/master/doc/%s/%s_depends.rosinstall'%(ros_distro, repo)
f = urllib2.urlopen(depends_repo_url)
if f.code == 200:
print "Found a depends rosinstall file for %s" % repo
depends_conf = yaml.load(f.read())
except (urllib2.URLError, urllib2.HTTPError) as e:
print "Did not find a depends rosinstall file for %s" % repo
#Get the list of repositories that should have documentation run on them
#These are all of the repos that are not in the depends rosinsall file
repos_to_doc = get_repositories_from_rosinstall(conf)
#TODO: Change this or parameterize or whatever
homepage = 'http://ros.org/rosdoclite'
#Select the appropriate rosinstall file
rosinstall = yaml.dump(conf + depends_conf, default_style=False)
print "Rosinstall for repo %s:\n%s"%(repo, rosinstall)
with open(workspace+"/repo.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for repo %s, installing repo..."%repo
#TODO Figure out why rosinstall insists on having ROS available when called with nobuild, but not catkin
call("rosinstall %s %s/repo.rosinstall --nobuild --catkin" % (docspace, workspace))
repo_path = os.path.abspath("%s" % (docspace))
print "Repo path %s" % repo_path
stacks = {}
manifest_packages = {}
catkin_packages = {}
repo_map = {}
local_info = []
for install_item in conf + depends_conf:
key = install_item.keys()[0]
local_info.append({'type': key, 'name': install_item[key]['local-name'], 'url': install_item[key]['uri']})
#Get any stacks, manifest packages, or catkin packages (package.xml) in each repo
for item in local_info:
local_name = item['name']
local_path = os.path.join(repo_path, local_name)
print "Looking for the following packages in %s" % local_path
local_stacks = get_repo_manifests(local_path, manifest='stack')
local_manifest_packages = get_repo_manifests(local_path, manifest='package')
local_catkin_packages = get_repo_packages(local_path)
#Since rospkg is kind of screwed up and always finds package.xml files, we
#need to filter out packages that are catkin_packages but still listed in
#manifest or stack packages
for name in local_catkin_packages.iterkeys():
if name in local_stacks:
del local_stacks[name]
if name in local_manifest_packages:
del local_manifest_packages[name]
#Now, we need to update our repo map
for name in local_stacks.keys() + local_manifest_packages.keys() + local_catkin_packages.keys():
repo_map[name] = item
#Finally, we'll merge these dictionaries into our global dicts
stacks.update(local_stacks)
manifest_packages.update(local_manifest_packages)
catkin_packages.update(local_catkin_packages)
print "Running documentation generation on\npackages: %s" % (manifest_packages.keys() + catkin_packages.keys())
print "Catkin packages: %s" % catkin_packages
print "Manifest packages: %s" % manifest_packages
print "Stacks: %s" % stacks
#Load information about existing tags
tags_db = TagsDb(ros_distro, workspace)
#Get any non local dependencies and install them
apt_deps = []
ros_dep = RosDepResolver(ros_distro)
apt = AptDepends(platform, arch)
deps = get_nonlocal_dependencies(catkin_packages, stacks, manifest_packages)
print "Dependencies: %s" % deps
for dep in deps:
if ros_dep.has_ros(dep):
apt_dep = ros_dep.to_apt(dep)
apt_deps.extend(apt_dep)
else:
apt_dep = "ros-%s-%s" % (ros_distro, dep.replace('_', '-'))
if apt.has_package(apt_dep):
apt_deps.append(apt_dep)
else:
print "WARNING, could not find dependency %s, not adding to list" % dep
print "Apt dependencies: %s" % apt_deps
#Build a local dependency graph to be used for build order
local_dep_graph = build_local_dependency_graph(catkin_packages, manifest_packages)
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
for stack, path in stacks.iteritems():
import rospkg
#Get the dependencies of a dry stack from the stack.xml
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
stack_packages = get_repo_manifests(path, manifest='package').keys()
deps = [d.name for d in stack_manifest.depends]
stack_relative_doc_path = "%s/doc/%s/api/%s" % (docspace, ros_distro, stack)
stack_doc_path = os.path.abspath(stack_relative_doc_path)
write_stack_manifest(stack_doc_path, stack, stack_manifest, repo_map[stack]['type'], repo_map[stack]['url'], "%s/%s/api/%s/html" %(homepage, ros_distro, stack), stack_packages, tags_db)
#Need to make sure to re-order packages to be run in dependency order
build_order = get_dependency_build_order(local_dep_graph)
print "Build order that honors deps:\n%s" % build_order
#We'll need the full list of apt_deps to get tag files
full_apt_deps = copy.deepcopy(apt_deps)
for dep in apt_deps:
print "Getting dependencies for %s" % dep
full_apt_deps.extend(apt.depends(dep))
#Make sure that we don't have any duplicates
full_apt_deps = list(set(full_apt_deps))
print "Installing all dependencies for %s" % repo
if apt_deps:
call("apt-get install %s --yes" % (' '.join(apt_deps)))
print "Done installing dependencies"
#Set up the list of things that need to be sourced to run rosdoc_lite
sources = ['source /opt/ros/%s/setup.bash' % ros_distro]
#Everything that is after fuerte supports catkin workspaces, so everything
#that has packages with package.xml files
if catkin_packages and not 'rosdoc_lite' in catkin_packages.keys():
sources.append(build_repo_messages(docspace, ros_distro))
#For all our manifest packages (dry or fuerte catkin) we want to build
#messages. Note, for fuerte catkin the messages arent' generated, TODO
#to come back and fix this if necessary
sources.append(build_repo_messages_manifest(manifest_packages, build_order, ros_distro))
repo_tags = {}
for package in build_order:
#don't document packages that we're supposed to build but not supposed to document
if not repo_map[package]['name'] in repos_to_doc:
print "Package: %s, in repo: %s, is not supposed to be documented. Skipping." % (package, repo_map[package]['name'])
continue
#Pull the package from the correct place
if package in catkin_packages:
package_path = catkin_packages[package]
else:
package_path = manifest_packages[package]
#Build a tagfile list from dependencies for use by rosdoc
build_tagfile(full_apt_deps, tags_db, 'rosdoc_tags.yaml', package, build_order, docspace, ros_distro)
relative_doc_path = "%s/doc/%s/api/%s" % (docspace, ros_distro, package)
pkg_doc_path = os.path.abspath(relative_doc_path)
relative_tags_path = "%s/api/%s/tags/%s.tag" % (ros_distro, package, package)
tags_path = os.path.abspath("%s/doc/%s" % (docspace, relative_tags_path))
print "Documenting %s [%s]..." % (package, package_path)
#Generate the command we'll use to document the stack
command = ['bash', '-c', '%s \
&& export ROS_PACKAGE_PATH=%s:$ROS_PACKAGE_PATH \
&& rosdoc_lite %s -o %s -g %s -t rosdoc_tags.yaml' \
%(' && '.join(sources), repo_path, package_path, pkg_doc_path, tags_path) ]
#proc = subprocess.Popen(command, stdout=subprocess.PIPE)
proc = subprocess.Popen(command)
proc.communicate()
#Some doc runs won't generate tag files, so we need to check if they
#exist before adding them to the list
if(os.path.exists(tags_path)):
package_tags = {'location':'%s/%s'%(homepage, relative_tags_path),
'docs_url':'../../../api/%s/html'%(package),
'package':'%s'%package}
#If the package has a deb name, then we'll store the tags for it
#alongside that name
if ros_dep.has_ros(package):
pkg_deb_name = ros_dep.to_apt(package)[0]
tags_db.set_tags(pkg_deb_name, [package_tags])
#Otherwise, we'll store tags for it alongside it's repo, which we
#assume can be made into a deb name
else:
repo_tags.setdefault(repo_map[package]['name'], []).append(package_tags)
#We also need to add information to each package manifest that we only
#have availalbe in this script like vcs location and type
write_distro_specific_manifest(os.path.join(pkg_doc_path, 'manifest.yaml'),
package, repo_map[package]['type'], repo_map[package]['url'], "%s/%s/api/%s/html" %(homepage, ros_distro, package),
tags_db)
print "Done"
doc_path = os.path.abspath("%s/doc/%s" % (docspace, ros_distro))
#Copy the files to the appropriate place
#call("rsync -e \"ssh -o StrictHostKeyChecking=no\" -qr %s rosbuild@wgs32:/var/www/www.ros.org/html/rosdoclite" % (doc_path))
command = ['bash', '-c', 'rsync -e "ssh -o StrictHostKeyChecking=no" -qr %s rosbuild@wgs32:/var/www/www.ros.org/html/rosdoclite' % doc_path]
call_with_list(command)
#Write the new tags to the database if there are any to write
for name, tags in repo_tags.iteritems():
#Get the apt name of the current stack/repo
if ros_dep.has_ros(name):
deb_name = ros_dep.to_apt(name)[0]
else:
deb_name = "ros-%s-%s" % (ros_distro, name.replace('_', '-'))
#We only want to write tags for packages that have a valid deb name
#For others, the only way to get cross referencing is to document everything
#together with a rosinstall file
if apt.has_package(deb_name):
tags_db.set_tags(deb_name, tags)
#Make sure to write changes to tag files and deps
tags_db.commit_db()
#Tell jenkins that we've succeeded
print "Preparing xml test results"
try:
os.makedirs(os.path.join(workspace, 'test_results'))
print "Created test results directory"
except:
pass
call("cp %s %s"%(os.path.join(workspace, 'buildfarm/templates/junit_dummy_ouput_template.xml'),
os.path.join(workspace, 'test_results/')))
def document_repo(workspace, docspace, ros_distro, repo, platform, arch):
append_pymodules_if_needed()
print "Working on distro %s and repo %s" % (ros_distro, repo)
try:
repo_url = 'https://raw.github.com/ros/rosdistro/master/doc/%s/%s.rosinstall' % (
ros_distro, repo)
f = urllib2.urlopen(repo_url)
if f.code != 200:
raise BuildException(
"Could not find a valid rosinstall file for %s at %s" %
(repo, repo_url))
conf = yaml.load(f.read())
except (urllib2.URLError, urllib2.HTTPError) as e:
raise BuildException(
"Could not find a valid rosinstall file for %s at %s" %
(repo, repo_url))
depends_conf = []
try:
depends_repo_url = 'https://raw.github.com/ros/rosdistro/master/doc/%s/%s_depends.rosinstall' % (
ros_distro, repo)
f = urllib2.urlopen(depends_repo_url)
if f.code == 200:
print "Found a depends rosinstall file for %s" % repo
depends_conf = yaml.load(f.read())
except (urllib2.URLError, urllib2.HTTPError) as e:
print "Did not find a depends rosinstall file for %s" % repo
#Get the list of repositories that should have documentation run on them
#These are all of the repos that are not in the depends rosinsall file
repos_to_doc = get_repositories_from_rosinstall(conf)
#TODO: Change this or parameterize or whatever
homepage = 'http://ros.org/rosdoclite'
#Select the appropriate rosinstall file
rosinstall = yaml.dump(conf + depends_conf, default_style=False)
print "Rosinstall for repo %s:\n%s" % (repo, rosinstall)
with open(workspace + "/repo.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for repo %s, installing repo..." % repo
#TODO Figure out why rosinstall insists on having ROS available when called with nobuild, but not catkin
call("rosinstall %s %s/repo.rosinstall --nobuild --catkin" %
(docspace, workspace))
repo_path = os.path.abspath("%s" % (docspace))
print "Repo path %s" % repo_path
stacks = {}
manifest_packages = {}
catkin_packages = {}
repo_map = {}
local_info = []
for install_item in conf + depends_conf:
key = install_item.keys()[0]
local_info.append({
'type': key,
'name': install_item[key]['local-name'],
'url': install_item[key]['uri']
})
#Get any stacks, manifest packages, or catkin packages (package.xml) in each repo
for item in local_info:
local_name = item['name']
local_path = os.path.join(repo_path, local_name)
print "Looking for the following packages in %s" % local_path
local_stacks = get_repo_manifests(local_path, manifest='stack')
local_manifest_packages = get_repo_manifests(local_path,
manifest='package')
local_catkin_packages = get_repo_packages(local_path)
#Since rospkg is kind of screwed up and always finds package.xml files, we
#need to filter out packages that are catkin_packages but still listed in
#manifest or stack packages
for name in local_catkin_packages.iterkeys():
if name in local_stacks:
del local_stacks[name]
if name in local_manifest_packages:
del local_manifest_packages[name]
#Now, we need to update our repo map
for name in local_stacks.keys() + local_manifest_packages.keys(
) + local_catkin_packages.keys():
repo_map[name] = item
#Finally, we'll merge these dictionaries into our global dicts
stacks.update(local_stacks)
manifest_packages.update(local_manifest_packages)
catkin_packages.update(local_catkin_packages)
print "Running documentation generation on\npackages: %s" % (
manifest_packages.keys() + catkin_packages.keys())
print "Catkin packages: %s" % catkin_packages
print "Manifest packages: %s" % manifest_packages
print "Stacks: %s" % stacks
#Load information about existing tags
tags_db = TagsDb(ros_distro, workspace)
#Get any non local dependencies and install them
apt_deps = []
ros_dep = RosDepResolver(ros_distro)
apt = AptDepends(platform, arch)
deps = get_nonlocal_dependencies(catkin_packages, stacks,
manifest_packages)
print "Dependencies: %s" % deps
for dep in deps:
if ros_dep.has_ros(dep):
apt_dep = ros_dep.to_apt(dep)
apt_deps.extend(apt_dep)
else:
apt_dep = "ros-%s-%s" % (ros_distro, dep.replace('_', '-'))
if apt.has_package(apt_dep):
apt_deps.append(apt_dep)
else:
print "WARNING, could not find dependency %s, not adding to list" % dep
print "Apt dependencies: %s" % apt_deps
#Build a local dependency graph to be used for build order
local_dep_graph = build_local_dependency_graph(catkin_packages,
manifest_packages)
#Write stack manifest files for all stacks, we can just do this off the
#stack.xml files
for stack, path in stacks.iteritems():
import rospkg
#Get the dependencies of a dry stack from the stack.xml
stack_manifest = rospkg.parse_manifest_file(path, rospkg.STACK_FILE)
stack_packages = get_repo_manifests(path, manifest='package').keys()
deps = [d.name for d in stack_manifest.depends]
stack_relative_doc_path = "%s/doc/%s/api/%s" % (docspace, ros_distro,
stack)
stack_doc_path = os.path.abspath(stack_relative_doc_path)
write_stack_manifest(
stack_doc_path, stack, stack_manifest, repo_map[stack]['type'],
repo_map[stack]['url'],
"%s/%s/api/%s/html" % (homepage, ros_distro, stack),
stack_packages, tags_db)
#Need to make sure to re-order packages to be run in dependency order
build_order = get_dependency_build_order(local_dep_graph)
print "Build order that honors deps:\n%s" % build_order
#We'll need the full list of apt_deps to get tag files
full_apt_deps = copy.deepcopy(apt_deps)
for dep in apt_deps:
print "Getting dependencies for %s" % dep
full_apt_deps.extend(apt.depends(dep))
#Make sure that we don't have any duplicates
full_apt_deps = list(set(full_apt_deps))
print "Installing all dependencies for %s" % repo
if apt_deps:
call("apt-get install %s --yes" % (' '.join(apt_deps)))
print "Done installing dependencies"
#Set up the list of things that need to be sourced to run rosdoc_lite
sources = ['source /opt/ros/%s/setup.bash' % ros_distro]
#Everything that is after fuerte supports catkin workspaces, so everything
#that has packages with package.xml files
if catkin_packages and not 'rosdoc_lite' in catkin_packages.keys():
sources.append(build_repo_messages(docspace, ros_distro))
#For all our manifest packages (dry or fuerte catkin) we want to build
#messages. Note, for fuerte catkin the messages arent' generated, TODO
#to come back and fix this if necessary
sources.append(
build_repo_messages_manifest(manifest_packages, build_order,
ros_distro))
repo_tags = {}
for package in build_order:
#don't document packages that we're supposed to build but not supposed to document
if not repo_map[package]['name'] in repos_to_doc:
print "Package: %s, in repo: %s, is not supposed to be documented. Skipping." % (
package, repo_map[package]['name'])
continue
#Pull the package from the correct place
if package in catkin_packages:
package_path = catkin_packages[package]
else:
package_path = manifest_packages[package]
#Build a tagfile list from dependencies for use by rosdoc
build_tagfile(full_apt_deps, tags_db, 'rosdoc_tags.yaml', package,
build_order, docspace, ros_distro)
relative_doc_path = "%s/doc/%s/api/%s" % (docspace, ros_distro,
package)
pkg_doc_path = os.path.abspath(relative_doc_path)
relative_tags_path = "%s/api/%s/tags/%s.tag" % (ros_distro, package,
package)
tags_path = os.path.abspath("%s/doc/%s" %
(docspace, relative_tags_path))
print "Documenting %s [%s]..." % (package, package_path)
#Generate the command we'll use to document the stack
command = ['bash', '-c', '%s \
&& export ROS_PACKAGE_PATH=%s:$ROS_PACKAGE_PATH \
&& rosdoc_lite %s -o %s -g %s -t rosdoc_tags.yaml' \
%(' && '.join(sources), repo_path, package_path, pkg_doc_path, tags_path) ]
#proc = subprocess.Popen(command, stdout=subprocess.PIPE)
proc = subprocess.Popen(command)
proc.communicate()
#Some doc runs won't generate tag files, so we need to check if they
#exist before adding them to the list
if (os.path.exists(tags_path)):
package_tags = {
'location': '%s/%s' % (homepage, relative_tags_path),
'docs_url': '../../../api/%s/html' % (package),
'package': '%s' % package
}
#If the package has a deb name, then we'll store the tags for it
#alongside that name
if ros_dep.has_ros(package):
pkg_deb_name = ros_dep.to_apt(package)[0]
tags_db.set_tags(pkg_deb_name, [package_tags])
#Otherwise, we'll store tags for it alongside it's repo, which we
#assume can be made into a deb name
else:
repo_tags.setdefault(repo_map[package]['name'],
[]).append(package_tags)
#We also need to add information to each package manifest that we only
#have availalbe in this script like vcs location and type
write_distro_specific_manifest(
os.path.join(pkg_doc_path, 'manifest.yaml'), package,
repo_map[package]['type'], repo_map[package]['url'],
"%s/%s/api/%s/html" % (homepage, ros_distro, package), tags_db)
print "Done"
doc_path = os.path.abspath("%s/doc/%s" % (docspace, ros_distro))
#Copy the files to the appropriate place
#call("rsync -e \"ssh -o StrictHostKeyChecking=no\" -qr %s rosbuild@wgs32:/var/www/www.ros.org/html/rosdoclite" % (doc_path))
command = [
'bash', '-c',
'rsync -e "ssh -o StrictHostKeyChecking=no" -qr %s rosbuild@wgs32:/var/www/www.ros.org/html/rosdoclite'
% doc_path
]
call_with_list(command)
#Write the new tags to the database if there are any to write
for name, tags in repo_tags.iteritems():
#Get the apt name of the current stack/repo
if ros_dep.has_ros(name):
deb_name = ros_dep.to_apt(name)[0]
else:
deb_name = "ros-%s-%s" % (ros_distro, name.replace('_', '-'))
#We only want to write tags for packages that have a valid deb name
#For others, the only way to get cross referencing is to document everything
#together with a rosinstall file
if apt.has_package(deb_name):
tags_db.set_tags(deb_name, tags)
#Make sure to write changes to tag files and deps
tags_db.commit_db()
#Tell jenkins that we've succeeded
print "Preparing xml test results"
try:
os.makedirs(os.path.join(workspace, 'test_results'))
print "Created test results directory"
except:
pass
call("cp %s %s" % (os.path.join(
workspace, 'buildfarm/templates/junit_dummy_ouput_template.xml'),
os.path.join(workspace, 'test_results/')))