def call_abi_checker(workspace_root, ros_version, env):
import rosdistro
condition_context = {}
condition_context['ROS_DISTRO'] = env['ROS_DISTRO']
condition_context['ROS_VERSION'] = ros_version
condition_context['ROS_PYTHON_VERSION'] = \
(env or os.environ).get('ROS_PYTHON_VERSION')
pkgs = get_packages_in_workspaces(workspace_root, condition_context)
pkg_names = [pkg.name for pkg in pkgs.values()]
assert pkg_names, 'No packages found in the workspace'
# Filter packages in source space that has been released
index = rosdistro.get_index(rosdistro.get_index_url())
dist_file = rosdistro.get_distribution_file(index, env['ROS_DISTRO'])
pkg_names_released = [
pkg_name for pkg_name in pkg_names
if pkg_name in dist_file.release_packages
]
assert len(
workspace_root
) == 1, 'auto-abi tool needs the implementation of multiple local-dir'
# ROS_DISTRO is set in the env object
cmd = [
'auto-abi.py ' + '--orig-type ros-pkg --orig ' +
",".join(pkg_names_released) + ' ' + '--new-type ros-ws --new ' +
os.path.join(workspace_root[0], 'install_isolated') + ' ' +
'--report-dir ' + workspace_root[0] + ' ' + '--no-fail-if-empty ' +
'--display-exec-time'
]
print("Invoking '%s'" % (cmd))
return subprocess.call(cmd, shell=True, stderr=subprocess.STDOUT, env=env)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for building the binarydeb")
add_argument_rosdistro_name(parser)
add_argument_package_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_distribution_repository_urls(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_binarydeb_dir(parser)
add_argument_dockerfile_dir(parser)
args = parser.parse_args(argv)
debian_package_name = get_debian_package_name(
args.rosdistro_name, args.package_name)
# get expected package version from rosdistro
index = get_index(args.rosdistro_index_url)
dist_file = get_distribution_file(index, args.rosdistro_name)
assert args.package_name in dist_file.release_packages
pkg = dist_file.release_packages[args.package_name]
repo = dist_file.repositories[pkg.repository_name]
package_version = repo.release_repository.version
debian_package_version = package_version
# find PKGBUILD dependencies
pkgbuild_proc = subprocess.Popen(["/bin/bash","-c","source PKGBUILD ; echo $(printf \"'%s' \" \"${makedepends[@]}\") $(printf \"'%s' \" \"${depends[@]}\")"], stdout=subprocess.PIPE)
pkgbuild_out,_ = pkgbuild_proc.communicate()
archlinux_pkg_names = pkgbuild_proc.decode('ascii').split(" ")
# generate Dockerfile
data = {
'os_name': args.os_name,
'os_code_name': args.os_code_name,
'arch': args.arch,
'uid': get_user_id(),
'distribution_repository_urls': args.distribution_repository_urls,
'distribution_repository_keys': get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'dependencies': archlinux_pkg_names,
'rosdistro_name': args.rosdistro_name,
'package_name': args.package_name,
'binarydeb_dir': args.binarydeb_dir,
}
create_dockerfile(
'release/binary_archlinux_task.Dockerfile.em', data, args.dockerfile_dir)
# output hints about necessary volumes to mount
ros_buildfarm_basepath = os.path.normpath(
os.path.join(os.path.dirname(__file__), '..', '..'))
print('Mount the following volumes when running the container:')
print(' -v %s:/tmp/ros_buildfarm:ro' % ros_buildfarm_basepath)
print(' -v %s:/tmp/binary_archlinux' % args.binarydeb_dir)
def main(index_url, rosdistro_name):
index = get_index(index_url)
try:
distribution_file = get_distribution_file(index, rosdistro_name)
except RuntimeError as e:
print("Could not load distribution file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr)
return False
success = True
for repo_name in sorted(distribution_file.repositories.keys()):
sys.stdout.write('.')
sys.stdout.flush()
repo = distribution_file.repositories[repo_name]
repos = [repo.release_repository, repo.source_repository, repo.doc_repository]
for repo in [r for r in repos if r]:
if repo.url.startswith('file://'):
print()
print("Repository '%s' with url '%s' must not be a local 'file://' url" % (repo_name, repo.url), file=sys.stderr)
success = False
if repo.type == 'git':
prefixes = ['http://github.com/', '[email protected]:']
for prefix in prefixes:
if repo.url.startswith(prefix):
print()
print("Repository '%s' with url '%s' must use 'https://github.com/%s' instead" % (repo_name, repo.url, repo.url[len(prefix):]), file=sys.stderr)
success = False
print()
return success
def test_build_caches():
with Fold():
print(
"""Checking if the 'package.xml' files for all packages are fetchable.
If this fails you can run 'rosdistro_build_cache index.yaml' to perform the same check locally.
""")
index = 'file://' + os.path.abspath(INDEX_YAML)
index = get_index(index)
dist_names = sorted(index.distributions.keys())
dist_names = [n for n in dist_names if n not in eol_distro_names]
errors = []
caches = OrderedDict()
for dist_name in dist_names:
with Fold():
try:
cache = generate_distribution_cache(index, dist_name)
except RuntimeError as e:
errors.append(str(e))
caches[dist_name] = cache
# also check topological order to prevent circular dependencies
for dist_name, cache in caches.items():
pkgs = {}
for pkg_name, pkg_xml in cache.release_package_xmls.items():
pkgs[pkg_name] = parse_package_string(pkg_xml)
try:
topological_order_packages(pkgs)
except RuntimeError as e:
errors.append('%s: %s' % (dist_name, e))
if errors:
raise RuntimeError('\n'.join(errors))
def test_get_doc_build_files():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
files = get_doc_build_files(i, 'foo')
assert len(files) == 1
build_file = files[0]
assert build_file.jenkins_job_timeout == 23
class RepoForm(FlaskForm):
architectures = ['amd64', 'arm64', 'armhf', 'i386', 'source']
selected_arch = MultiCheckboxField(
'Architectures',
choices=[(arch, arch) for arch in architectures],
validators=[DataRequired('Select at least one valid architecture')],
)
from rosdistro import get_distribution_file, get_index, get_index_url
index = get_index(get_index_url())
ros_distributions = index.distributions.keys()
distributions_combo_list = list()
list_index = 0
for item in ros_distributions:
distribution_file = get_distribution_file(index, item)
for ubuntu in distribution_file.release_platforms['ubuntu']:
distributions_combo_list.append(
(list_index, dict({
'ros': item,
'ubuntu': ubuntu
})))
list_index += 1
distributions_combo_list = sorted(distributions_combo_list,
key=lambda v: v[1]['ros'])
selected_distros = MultiCheckboxField(
'Distributions',
choices=[(str(dist[0]), dist[1]['ros'] + ' - ' + dist[1]['ubuntu'])
for dist in distributions_combo_list],
validators=[DataRequired('Select at least one valid distribution')],
)
submit = SubmitField('Next')
def __init__(self, name, ros_distro):
SrcAptBase.__init__(self, name)
self.ros_distro = self.detect_ros_distribution(ros_distro)
self.rosdistro_index = rosdistro.get_index(rosdistro.get_index_url())
self.cache = rosdistro.get_distribution_cache(self.rosdistro_index,
self.ros_distro)
self.distro_file = self.cache.distribution_file
# More logic could be needed with new ros distributions
# ROS1 - https://www.ros.org/reps/rep-0003.html
# ROS2 - http://design.ros2.org/articles/changes.html
if self.ros_distro == 'melodic':
self.compilation_flags.append('--std=c++14')
else:
self.compilation_flags.append('--std=c++17')
# needed for gazebo_ros_pkgs
self.compilation_flags.append('-DBOOST_HAS_PTHREADS=1')
# gtest-vendor is ROS2
self.compilation_flags.append('-I' +
join('/opt/ros/', self.ros_distro, 'src', 'gtest_vendor', 'include'))
# flag to avoid problems in rcutils
# https://github.com/osrf/auto-abi-checker/issues/17
self.compilation_flags.append('-DRCUTILS__STDATOMIC_HELPER_H_')
# flags for rmw_connext packages
self.compilation_flags.append('-DRTI_UNIX')
for rti_path in glob.glob('/opt/rti.com/rti_connext_dds-*'):
self.compilation_flags.append('-I' + rti_path + '/include/')
self.compilation_flags.append('-I' + rti_path + '/include/ndds')
# Needs to add /opt/ros includes to compile ROS software
self.compilation_flags.append('-I' +
join('/opt/ros/', self.ros_distro, 'include'))
def test_get_release_builds():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
d = get_release(i, 'foo')
builds = get_release_builds(i, d)
assert len(builds) == 1
build = builds[0]
assert build.jenkins_sourcedeb_job_timeout == 5
assert build.jenkins_binarydeb_job_timeout == 42
os_names = build.get_target_os_names()
assert set(os_names) == set(['ubuntu'])
os_code_names = build.get_target_os_code_names('ubuntu')
assert set(os_code_names) == set(['precise', 'quantal', 'raring'])
arches = build.get_target_arches('ubuntu', 'precise')
assert set(arches) == set(['amd64', 'i386'])
c = build.get_target_configuration()
assert len(c.keys()) == 2
assert set(c.keys()) == set(['apt_target_repository', 'foo'])
assert c['apt_target_repository'] == 'http://repo.example.com/'
assert c['foo'] == 'bar'
c = build.get_target_configuration('ubuntu', 'precise')
assert 'foo' in c.keys()
assert c['foo'] == 'bar'
assert 'ping' in c.keys()
assert c['ping'] == 'pong'
c = build.get_target_configuration('ubuntu', 'precise', 'amd64')
assert 'foo' in c.keys()
assert c['foo'] == 'baz'
def _check_platform_helper() -> Tuple[str, dict, dict]:
"""
Check ROS_DISTRO environment variables and distribution installed.
:return: string of distro name, dict of distribution info, dict of release platforms info
"""
distro_name = os.environ.get('ROS_DISTRO')
if not distro_name:
doctor_error('ROS_DISTRO is not set.')
return
distro_name = distro_name.lower()
u = rosdistro.get_index_url()
if not u:
doctor_error(
'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL. '
'Check network setting to make sure machine is connected to internet.'
)
return
i = rosdistro.get_index(u)
distro_info = i.distributions.get(distro_name)
if not distro_info:
doctor_warn(f'Distribution name {distro_name} is not found')
return
try:
distro_data = rosdistro.get_distribution(i, distro_name).get_data()
except AttributeError:
distro_data = ''
return distro_name, distro_info, distro_data
def main(index_url, rosdistro_name):
index = get_index(index_url)
try:
distribution_file = get_distribution_file(index, rosdistro_name)
except RuntimeError as e:
print("Could not load distribution file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr)
return False
success = True
for repo_name in sorted(distribution_file.repositories.keys()):
sys.stdout.write('.')
sys.stdout.flush()
repo = distribution_file.repositories[repo_name]
repos = [repo.release_repository, repo.source_repository, repo.doc_repository]
for repo in [r for r in repos if r]:
if repo.url.startswith('file://'):
print()
print("Repository '%s' with url '%s' must not be a local 'file://' url" % (repo_name, repo.url), file=sys.stderr)
success = False
if repo.type == 'git':
prefixes = ['http://github.com/', '[email protected]:']
for prefix in prefixes:
if repo.url.startswith(prefix):
print()
print("Repository '%s' with url '%s' must use 'https://github.com/%s' instead" % (repo_name, repo.url, repo.url[len(prefix):]), file=sys.stderr)
success = False
for prefix in prefixes + ['https://github.com/']:
if repo.url.startswith(prefix) and not repo.url.endswith('.git'):
print()
print("Repository '%s' with url '%s' should end with `.git` but does not." % (repo_name, repo.url))
success = False
print()
return success
def get_rosdistro(quiet):
global _rosdistro_cache
dist = None
if "ROS_DISTRO" in os.environ:
distro_id = os.environ["ROS_DISTRO"]
if distro_id not in _rosdistro_cache:
try:
from rosdistro import get_index, get_index_url, get_cached_distribution
url = get_index_url()
if not quiet:
sys.stderr.write(
"catkin_lint: downloading %s package index from %s\n" %
(distro_id, url))
index = get_index(url)
dist = get_cached_distribution(index,
distro_id,
allow_lazy_load=True)
except Exception as err:
if not quiet:
sys.stderr.write(
"catkin_lint: cannot initialize rosdistro: %s\n" %
str(err))
_rosdistro_cache[distro_id] = dist
dist = _rosdistro_cache[distro_id]
return Rosdistro(dist=dist, quiet=quiet)
def main(repo_type, rosdistro_name):
index = get_index(get_index_url())
if repo_type == 'doc':
try:
distro_file = get_doc_file(index, rosdistro_name)
except RuntimeError as e:
print("Could not load doc file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr)
return False
if repo_type == 'source':
try:
distro_file = get_source_file(index, rosdistro_name)
except RuntimeError as e:
print("Could not load source file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr)
return False
for repo_name in sorted(distro_file.repositories.keys()):
sys.stdout.write('.')
sys.stdout.flush()
repo = distro_file.repositories[repo_name]
try:
if (repo.type == 'git'):
check_git_repo(repo.url, repo.version)
elif (repo.type == 'hg'):
check_hg_repo(repo.url, repo.version)
elif (repo.type == 'svn'):
check_svn_repo(repo.url, repo.version)
else:
print()
print("Unknown type '%s' for repository '%s'" % (repo.type, repo.name), file=sys.stderr)
except RuntimeError as e:
print()
print("Could not fetch repository '%s': %s (%s) [%s]" % (repo.name, repo.url, repo.version, e), file=sys.stderr)
print()
return True
def test_get_index_from_http_with_query_parameters():
import subprocess
import sys
import time
url = 'http://localhost:9876/index_v3.yaml?raw&at=master'
# start a http server and wait
if sys.version_info < (3, 0, 0):
proc = subprocess.Popen(
[sys.executable, '-m', 'SimpleHTTPServer', '9876'], cwd=FILES_DIR)
else:
proc = subprocess.Popen([sys.executable, '-m', 'http.server', '9876'],
cwd=FILES_DIR)
time.sleep(0.5)
try:
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
# test if every url has the same queries
for dist_urls in i.distributions['foo'].values():
if not isinstance(dist_urls, list):
dist_urls = [dist_urls]
for dist_url in dist_urls:
assert dist_url.endswith('?raw&at=master')
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
get_distribution_file(i, 'foo')
finally:
proc.terminate()
def get_sourcerpm(rosdistro_index_url,
rosdistro_name,
package_name,
sourcepkg_dir,
skip_download_sourcepkg=False):
# ensure that no source subfolder exists
rpm_package_name = get_os_package_name(rosdistro_name, package_name)
if not skip_download_sourcepkg:
# get expected package version from rosdistro
from rosdistro import get_distribution_cache
from rosdistro import get_index
index = get_index(rosdistro_index_url)
dist_cache = get_distribution_cache(index, rosdistro_name)
dist_file = dist_cache.distribution_file
assert package_name in dist_file.release_packages
pkg = dist_file.release_packages[package_name]
repo = dist_file.repositories[pkg.repository_name]
package_version = repo.release_repository.version
cmd = [
'mock', '--resultdir',
'%s' % sourcepkg_dir, '--no-cleanup-after', '--verbose', '--root',
'ros_buildfarm', '--dnf-cmd', '--', 'download', '--source',
'--disablerepo', '*', '--enablerepo',
'ros-buildfarm-target-source',
'%s-%s.*' % (rpm_package_name, package_version)
]
print("Invoking '%s'" % ' '.join(cmd))
subprocess.check_call(cmd, cwd=sourcepkg_dir)
def test_get_release_builds():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
d = get_release(i, 'foo')
builds = get_release_builds(i, d)
assert len(builds) == 1
build = builds[0]
assert build.jenkins_sourcedeb_job_timeout == 5
assert build.jenkins_binarydeb_job_timeout == 42
os_names = build.get_target_os_names()
assert set(os_names) == set(['ubuntu'])
os_code_names = build.get_target_os_code_names('ubuntu')
assert set(os_code_names) == set(['precise', 'quantal', 'raring'])
arches = build.get_target_arches('ubuntu', 'precise')
assert set(arches) == set(['amd64', 'i386'])
c = build.get_target_configuration()
assert len(c.keys()) == 2
assert set(c.keys()) == set(['apt_target_repository', 'foo'])
assert c['apt_target_repository'] == 'http://repo.example.com/'
assert c['foo'] == 'bar'
c = build.get_target_configuration('ubuntu', 'precise')
assert 'foo' in c.keys()
assert c['foo'] == 'bar'
assert 'ping' in c.keys()
assert c['ping'] == 'pong'
c = build.get_target_configuration('ubuntu', 'precise', 'amd64')
assert 'foo' in c.keys()
assert c['foo'] == 'baz'
def test_get_index_from_http_with_query_parameters():
import subprocess
import sys
import time
url = 'http://localhost:9876/index_v3.yaml?raw&at=master'
# start a http server and wait
if sys.version_info < (3, 0, 0):
proc = subprocess.Popen([sys.executable, '-m', 'SimpleHTTPServer', '9876'],
cwd=FILES_DIR)
else:
proc = subprocess.Popen([sys.executable, '-m', 'http.server', '9876'],
cwd=FILES_DIR)
time.sleep(0.5)
try:
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
# test if every url has the same queries
for key, dist_urls in i.distributions['foo'].items():
if key in ('distribution_status', 'distribution_type'):
continue
if not isinstance(dist_urls, list):
dist_urls = [dist_urls]
for dist_url in dist_urls:
assert dist_url.endswith('?raw&at=master')
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
get_distribution_file(i, 'foo')
finally:
proc.terminate()
def test_get_doc_build_files():
url = 'file://' + FILES_DIR + '/index.yaml'
i = get_index(url)
files = get_doc_build_files(i, 'foo')
assert len(files) == 1
build_file = files[0]
assert build_file.jenkins_job_timeout == 23
def test_build_caches():
with Fold():
print("""Checking if the 'package.xml' files for all packages are fetchable.
If this fails you can run 'rosdistro_build_cache index.yaml' to perform the same check locally.
""")
index = 'file://' + os.path.abspath(INDEX_YAML)
index = get_index(index)
dist_names = sorted(index.distributions.keys())
dist_names = [n for n in dist_names if n not in eol_distro_names]
errors = []
caches = OrderedDict()
for dist_name in dist_names:
with Fold():
try:
cache = generate_distribution_cache(index, dist_name)
except RuntimeError as e:
errors.append(str(e))
caches[dist_name] = cache
# also check topological order to prevent circular dependencies
for dist_name, cache in caches.items():
pkgs = {}
print("Parsing manifest files for '%s'" % dist_name)
for pkg_name, pkg_xml in cache.release_package_xmls.items():
pkgs[pkg_name] = parse_package_string(pkg_xml)
print("Order all packages in '%s' topologically" % dist_name)
try:
topological_order_packages(pkgs)
except RuntimeError as e:
errors.append('%s: %s' % (dist_name, e))
if errors:
raise RuntimeError('\n'.join(errors))
def main(repo_type, rosdistro_name):
index = get_index(get_index_url())
try:
distribution_file = get_distribution_file(index, rosdistro_name)
except RuntimeError as e:
print("Could not load distribution file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr)
return False
for repo_name in sorted(distribution_file.repositories.keys()):
sys.stdout.write('.')
sys.stdout.flush()
repo = distribution_file.repositories[repo_name]
if repo_type == 'doc':
repo = repo.doc_repository
if repo_type == 'source':
repo = repo.source_repository
if not repo:
continue
try:
if (repo.type == 'git'):
check_git_repo(repo.url, repo.version)
elif (repo.type == 'hg'):
check_hg_repo(repo.url, repo.version)
elif (repo.type == 'svn'):
check_svn_repo(repo.url, repo.version)
else:
print()
print("Unknown type '%s' for repository '%s'" % (repo.type, repo.name), file=sys.stderr)
except RuntimeError as e:
print()
print("Could not fetch repository '%s': %s (%s) [%s]" % (repo.name, repo.url, repo.version, e), file=sys.stderr)
print()
return True
def get_sources(rosdistro_index_url, rosdistro_name, pkg_name, os_name, os_code_name, sources_dir):
from rosdistro import get_distribution_file
from rosdistro import get_index
index = get_index(rosdistro_index_url)
dist_file = get_distribution_file(index, rosdistro_name)
if pkg_name not in dist_file.release_packages:
return "Not a released package name: %s" % pkg_name
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
if not repo.release_repository.version:
return "Repository '%s' has no release version" % repo_name
pkg_version = repo.release_repository.version
tag = _get_source_tag(rosdistro_name, pkg_name, pkg_version, os_name, os_code_name)
cmd = [
"git",
"clone",
"--branch",
tag,
# fetch all branches and tags but no history
"--depth",
"1",
"--no-single-branch",
repo.release_repository.url,
sources_dir,
]
print("Invoking '%s'" % " ".join(cmd))
subprocess.check_call(cmd)
# ensure that the package version is correct
source_version = dpkg_parsechangelog(sources_dir, ["Version"])[0]
if not source_version.startswith(pkg_version) or (
len(source_version) > len(pkg_version) and source_version[len(pkg_version)] in "0123456789"
):
raise RuntimeError(
(
"The cloned package version from the GBP (%s) does not match "
+ "the expected package version from the distribution file (%s)"
)
% (source_version, pkg_version)
)
# output package version for job description
print("Package '%s' version: %s" % (pkg_name, source_version))
# output package maintainers for job notification
from catkin_pkg.package import parse_package
pkg = parse_package(sources_dir)
maintainer_emails = set([])
for m in pkg.maintainers:
maintainer_emails.add(m.email)
if maintainer_emails:
print("Package maintainer emails: %s" % " ".join(sorted(maintainer_emails)))
def configure_release_jobs(
config_url, rosdistro_name, release_build_name,
append_timestamp=False):
config = get_config_index(config_url)
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
index = get_index(config.rosdistro_index_url)
dist_cache = None
if build_file.notify_maintainers or build_file.abi_incompatibility_assumed:
dist_cache = get_distribution_cache(index, rosdistro_name)
# get targets
targets = []
for os_name in build_file.targets.keys():
for os_code_name in build_file.targets[os_name].keys():
targets.append((os_name, os_code_name))
print('The build file contains the following targets:')
for os_name, os_code_name in targets:
print(' - %s %s: %s' % (os_name, os_code_name, ', '.join(
build_file.targets[os_name][os_code_name])))
dist_file = get_distribution_file(index, rosdistro_name)
jenkins = connect(config.jenkins_url)
configure_import_package_job(
config_url, rosdistro_name, release_build_name,
config=config, build_file=build_file, jenkins=jenkins)
view_name = get_release_view_name(rosdistro_name, release_build_name)
view = configure_release_view(jenkins, view_name)
pkg_names = dist_file.release_packages.keys()
pkg_names = build_file.filter_packages(pkg_names)
for pkg_name in sorted(pkg_names):
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
if not repo.release_repository:
print(("Skipping package '%s' in repository '%s': no release " +
"section") % (pkg_name, repo_name))
continue
if not repo.release_repository.version:
print(("Skipping package '%s' in repository '%s': no release " +
"version") % (pkg_name, repo_name))
continue
for os_name, os_code_name in targets:
configure_release_job(
config_url, rosdistro_name, release_build_name,
pkg_name, os_name, os_code_name,
append_timestamp=append_timestamp,
config=config, build_file=build_file,
index=index, dist_file=dist_file, dist_cache=dist_cache,
jenkins=jenkins, view=view,
generate_import_package_job=False)
def get_manifest_from_rosdistro(package_name, distro_name):
"""
Get the rosdistro repository data and package information.
@param package_name: name of package or repository to get manifest information for.
It gives package symbols precedence over repository names.
@type package_name: str
@param distro_name: name of ROS distribution
@type distro_name: str
@return: (manifest data, 'package'|'repository').
@rtype: ({str: str}, str, str)
@raise IOError: if data cannot be loaded
"""
data = {}
type_ = None
index = get_index(get_index_url())
try:
distribution_cache = get_cached_distribution(index, distro_name)
except RuntimeError as runerr:
if (runerr.message.startswith("Unknown release")):
return None
raise
if package_name in distribution_cache.release_packages:
pkg = distribution_cache.release_packages[package_name]
#print('pkg', pkg.name)
pkg_xml = distribution_cache.get_release_package_xml(package_name)
pkg_manifest = parse_package_string(pkg_xml)
data['description'] = pkg_manifest.description
website_url = [u.url for u in pkg_manifest.urls if u.type == 'website']
if website_url:
data['url'] = website_url[0]
repo_name = pkg.repository_name
meta_export = [exp for exp in pkg_manifest.exports if exp.tagname == 'metapackage']
if meta_export:
type_ = 'metapackage'
else:
type_ = 'package'
else:
repo_name = package_name
type_ = 'repository'
data['repo_name'] = repo_name
if repo_name in distribution_cache.repositories:
repo = distribution_cache.repositories[repo_name].release_repository
if repo:
data['packages'] = repo.package_names
if repo_name in distribution_cache.repositories:
repo = distribution_cache.repositories[repo_name].source_repository
if not repo:
return None
data['vcs'] = repo.type
data['vcs_uri'] = repo.url
data['vcs_version'] = repo.version
else:
return None
return (data, type_, None)
def main(repo_type, rosdistro_name, check_for_wet_packages=False):
index = get_index(get_index_url())
try:
distribution_file = get_distribution_file(index, rosdistro_name)
except RuntimeError as e:
print("Could not load distribution file for distro '%s': %s" % (rosdistro_name, e), file=sys.stderr)
return False
for repo_name in sorted(distribution_file.repositories.keys()):
sys.stdout.write('.')
sys.stdout.flush()
repo = distribution_file.repositories[repo_name]
if repo_type == 'doc':
repo = repo.doc_repository
if repo_type == 'source':
repo = repo.source_repository
if not repo:
continue
try:
if (repo.type == 'git'):
check_git_repo(repo.url, repo.version)
elif (repo.type == 'hg'):
check_hg_repo(repo.url, repo.version)
elif (repo.type == 'svn'):
check_svn_repo(repo.url, repo.version)
else:
print()
print("Unknown type '%s' for repository '%s'" % (repo.type, repo.name), file=sys.stderr)
continue
except RuntimeError as e:
print()
print("Could not fetch repository '%s': %s (%s) [%s]" % (repo.name, repo.url, repo.version, e), file=sys.stderr)
continue
if check_for_wet_packages:
path = tempfile.mkdtemp()
try:
if repo.type == 'git':
clone_git_repo(repo.url, repo.version, path)
elif repo.type == 'hg':
clone_hg_repo(repo.url, repo.version, path)
elif repo.type == 'svn':
checkout_svn_repo(repo.url, repo.version, path)
except RuntimeError as e:
print()
print("Could not clone repository '%s': %s (%s) [%s]" % (repo.name, repo.url, repo.version, e), file=sys.stderr)
continue
else:
package_paths = find_package_paths(path)
if not package_paths:
print()
print("Repository '%s' (%s [%s]) does not contain any wet packages" % (repo.name, repo.url, repo.version), file=sys.stderr)
continue
finally:
shutil.rmtree(path)
print()
return True
def test_verify_files_parsable():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
index = get_index(url)
distribution_file = get_distribution_file(index, 'foo')
data = yaml_from_distribution_file(distribution_file)
with open(os.path.join(FILES_DIR, 'foo', 'distribution.yaml'), 'r') as f:
expected = f.read()
assert data == expected, get_diff(expected, data)
def test_get_index_v2():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
assert 'distribution_status' not in i.distributions['foo']
assert 'distribution_type' not in i.distributions['foo']
def get_all_distribution_files(url=None):
if not url:
url = rosdistro.get_index_url()
distribution_files = []
i = rosdistro.get_index(url)
for d in i.distributions:
distribution_files.append(rosdistro.get_distribution_file(i, d))
return distribution_files
def test_verify_files_parsable():
url = 'file://' + FILES_DIR + '/index.yaml'
index = get_index(url)
release_file = get_release_file(index, 'foo')
data = yaml_from_release_file(release_file)
with open(os.path.join(FILES_DIR, 'foo', 'release.yaml'), 'r') as f:
expected = f.read()
assert data == expected, get_diff(expected, data)
def execute(self, rosdistro_path, distro):
self.rosdistro_path = rosdistro_path
internal_index_path = (rosdistro_path / 'rosdistro' /
'index.yaml').resolve().as_uri()
self.internal_index = get_index(internal_index_path)
self.internal_distro = get_distribution(self.internal_index, distro)
self.internal_distro_file = self.internal_index.distributions[distro][
'distribution'][-1]
def test_get_distribution_file():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
dist_file = get_distribution_file(i, 'foo')
_validate_dist_file(dist_file)
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 1
def test_get_index_v2():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
assert 'distribution_status' not in i.distributions['foo']
assert 'distribution_type' not in i.distributions['foo']
def get_all_distribution_files(url=None):
if not url:
url = rosdistro.get_index_url()
distribution_files = []
i = rosdistro.get_index(url)
for d in i.distributions:
distribution_files.append(rosdistro.get_distribution_file(i, d))
return distribution_files
def test_verify_files_parsable():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
index = get_index(url)
distribution_file = get_distribution_file(index, 'foo')
data = yaml_from_distribution_file(distribution_file)
with open(os.path.join(FILES_DIR, 'foo', 'distribution.yaml'), 'r') as f:
expected = f.read()
assert data == expected, get_diff(expected, data)
def test_get_distribution_file():
url = 'file://' + FILES_DIR + '/index_v2.yaml'
i = get_index(url)
dist_file = get_distribution_file(i, 'foo')
_validate_dist_file(dist_file)
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 1
def partition_packages(config_url,
rosdistro_name,
release_build_name,
target,
cache_dir,
deduplicate_dependencies=False,
dist_cache=None):
"""Check all packages in the rosdistro and compare to the debian packages repository.
Return the set of all packages and the set of missing ones.
"""
# fetch debian package list
config = get_config_index(config_url)
index = get_index(config.rosdistro_index_url)
dist_file = rosdistro_get_distribution_file(index, rosdistro_name)
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
# Check that apt repos status
repo_index = get_package_repo_data(build_file.target_repository, [target],
cache_dir)[target]
# for each release package which matches the release build file
# check if a binary package exists
binary_packages = set()
all_pkg_names = dist_file.release_packages.keys()
# Remove packages without versions declared.
def get_package_version(dist_file, pkg_name):
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
return repo.release_repository.version
all_pkg_names = [
p for p in all_pkg_names if get_package_version(dist_file, p)
]
distribution = get_cached_distribution(index,
rosdistro_name,
cache=dist_cache)
pkg_names = filter_buildfile_packages_recursively(all_pkg_names,
build_file, distribution)
for pkg_name in sorted(pkg_names):
debian_pkg_name = get_os_package_name(rosdistro_name, pkg_name)
if debian_pkg_name in repo_index:
binary_packages.add(pkg_name)
# check that all elements from whitelist are present
missing_binary_packages = set(pkg_names) - binary_packages
if deduplicate_dependencies:
# Do not list missing packages that are dependencies of other missing ones
cached_pkgs = get_package_manifests(distribution)
missing_binary_packages = filter_blocked_dependent_package_names(
cached_pkgs, missing_binary_packages)
return binary_packages, missing_binary_packages
def test_get_index_v3():
url = 'file://' + FILES_DIR + '/index_v3.yaml'
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
get_distribution_file(i, 'foo')
def check_sync_criteria(
config_url, rosdistro_name, release_build_name, os_code_name, arch,
cache_dir):
# fetch debian package list
config = get_config_index(config_url)
index = get_index(config.rosdistro_index_url)
dist_file = get_distribution_file(index, rosdistro_name)
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
Target = namedtuple('Target', 'os_name os_code_name arch')
target = Target('ubuntu', os_code_name, arch)
repo_index = get_debian_repo_index(
build_file.target_repository, target, cache_dir)
# for each release package which matches the release build file
# check if a binary package exists
binary_packages = {}
all_pkg_names = dist_file.release_packages.keys()
pkg_names = build_file.filter_packages(all_pkg_names)
for pkg_name in sorted(pkg_names):
debian_pkg_name = get_debian_package_name(rosdistro_name, pkg_name)
binary_packages[pkg_name] = debian_pkg_name in repo_index
# check that all elements from whitelist are present
if build_file.sync_packages:
missing_binary_packages = len([
pkg_name
for pkg_name, has_binary_package in binary_packages.items()
if has_binary_package])
if missing_binary_packages:
print('The following binary packages are missing to sync:',
file=sys.stderr)
for pkg_name in sorted(missing_binary_packages):
print('-', pkg_name, file=sys.stderr)
return False
print('All required binary packages are available:')
for pkg_name in sorted(build_file.sync_packages):
print('-', pkg_name)
# check that count is satisfied
if build_file.sync_package_count is not None:
binary_package_count = len([
pkg_name
for pkg_name, has_binary_package in binary_packages.items()
if has_binary_package])
if binary_package_count < build_file.sync_package_count:
print('Only %d binary packages available ' % binary_package_count +
'(at least %d are required to sync)' %
build_file.sync_package_count, file=sys.stderr)
return False
print('%d binary packages available ' % binary_package_count +
'(more or equal then the configured sync limit of %d)' %
build_file.sync_package_count)
return True
def __init__(self, distro):
self.distro = distro
try:
index = get_index(get_index_url())
self._distribution_file = get_distribution_cache(index, distro).distribution_file
except:
logger.error("Could not load rosdistro distribution cache")
self._distribution_file = None
def main():
rosdistro_index = rosdistro.get_index(ROSDISTRO_URL)
cache = generate_distribution_cache(rosdistro_index, 'indigo')
cached_distro = rosdistro.get_cached_distribution(rosdistro_index,
'indigo',
cache=cache)
root_packages = {'roscpp'}
package_names = root_packages.union(
get_recursive_dependencies(cached_distro, root_packages))
print(f'Found {len(package_names)} packages.')
rosinstall_data = generate_rosinstall(cached_distro,
package_names,
flat=True,
tar=True)
remote_files = []
for rosinstall_pkg in rosinstall_data:
name = rosinstall_pkg['tar']['local-name']
url = rosinstall_pkg['tar']['uri'].replace('.tar.gz', '.zip')
print(name, url)
# Fetch tarball to get its sha1sum
r = requests.get(url)
r.raise_for_status()
sha1sum = hashlib.sha1(r.content).hexdigest()
remote_files.append({
'name': name,
'url': url,
'sha1': sha1sum,
})
sh.mkdir('-p', 'ros/rosdistro')
# Save BUCK file with remote_file rules
with open('ros/rosdistro/BUCK', 'w') as out_f:
for rf in remote_files:
s = f"""remote_file(
name = '{rf['name']}.zip',
url = '{rf['url']}',
sha1 = '{rf['sha1']}',
visibility = ['PUBLIC'],
)
"""
out_f.write(s)
# Save DEFS file with the list of tarballs
with open('ros/rosdistro/DEFS', 'w') as out_f:
out_f.write("rosdistro_tarballs = [\n{}\n]".format('\n'.join([
f" '//ros/rosdistro:{rf['name']}.zip'," for rf in remote_files
])))
def test_get_index_v3():
url = 'file://' + FILES_DIR + '/index_v3.yaml'
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
get_distribution_file(i, 'foo')
def check_sync_criteria(config_url, rosdistro_name, release_build_name,
os_code_name, arch, cache_dir):
# fetch debian package list
config = get_config_index(config_url)
index = get_index(config.rosdistro_index_url)
dist_file = get_distribution_file(index, rosdistro_name)
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
target = Target('ubuntu', os_code_name, arch)
repo_index = get_debian_repo_index(build_file.target_repository, target,
cache_dir)
# for each release package which matches the release build file
# check if a binary package exists
binary_packages = {}
all_pkg_names = dist_file.release_packages.keys()
pkg_names = build_file.filter_packages(all_pkg_names)
for pkg_name in sorted(pkg_names):
debian_pkg_name = get_debian_package_name(rosdistro_name, pkg_name)
binary_packages[pkg_name] = debian_pkg_name in repo_index
# check that all elements from whitelist are present
if build_file.sync_packages:
missing_binary_packages = [
pkg_name for pkg_name in build_file.sync_packages
if pkg_name not in binary_packages or not binary_packages[pkg_name]
]
if missing_binary_packages:
print('The following binary packages are missing to sync:',
file=sys.stderr)
for pkg_name in sorted(missing_binary_packages):
print('-', pkg_name, file=sys.stderr)
return False
print('All required binary packages are available:')
for pkg_name in sorted(build_file.sync_packages):
print('-', pkg_name)
# check that count is satisfied
if build_file.sync_package_count is not None:
binary_package_count = len([
pkg_name
for pkg_name, has_binary_package in binary_packages.items()
if has_binary_package
])
if binary_package_count < build_file.sync_package_count:
print('Only %d binary packages available ' % binary_package_count +
'(at least %d are required to sync)' %
build_file.sync_package_count,
file=sys.stderr)
return False
print('%d binary packages available ' % binary_package_count +
'(more or equal then the configured sync limit of %d)' %
build_file.sync_package_count)
return True
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Create a workspace from vcs repos files.')
add_argument_rosdistro_name(parser)
add_argument_repos_file_urls(parser)
add_argument_repository_names(parser, optional=True)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
help='The path of the desired workspace',
required=True)
args = parser.parse_args(argv)
assert args.repos_file_urls or args.repository_names
ensure_workspace_exists(args.workspace_root)
repos_files = []
if args.repository_names:
with Scope('SUBSECTION', 'get repository information from rosdistro'):
index = get_index(get_index_url())
dist = get_distribution(index, args.rosdistro_name)
data = {}
for repo_name in args.repository_names:
repo = dist.repositories[repo_name]
src_repo = repo.source_repository
repo_data = {
'type': src_repo.type,
'url': src_repo.url,
}
if src_repo.version is not None:
repo_data['version'] = src_repo.version
data[repo_name] = repo_data
repos_file = os.path.join(args.workspace_root,
'repositories-from-rosdistro.repos')
with open(repos_file, 'w') as h:
h.write(
yaml.safe_dump({'repositories': data},
default_flow_style=False))
repos_files.append(repos_file)
with Scope('SUBSECTION', 'fetch repos files(s)'):
for repos_file_url in args.repos_file_urls:
repos_file = os.path.join(args.workspace_root,
os.path.basename(repos_file_url))
print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file))
urlretrieve(repos_file_url, repos_file)
repos_files += [repos_file]
with Scope('SUBSECTION', 'import repositories'):
source_space = os.path.join(args.workspace_root, 'src')
for repos_file in repos_files:
print('Importing repositories from \'%s\'' % (repos_file))
import_repositories(source_space, repos_file, args.test_branch)
with Scope('SUBSECTION', 'vcs export --exact'):
# if a repo has been rebased against the default branch vcs can't detect the remote
export_repositories(args.workspace_root, check=not args.test_branch)
def get_sources(rosdistro_index_url, rosdistro_name, pkg_name, os_name,
os_code_name, sources_dir):
from rosdistro import get_cached_distribution
from rosdistro import get_index
index = get_index(rosdistro_index_url)
dist_file = get_cached_distribution(index, rosdistro_name)
if pkg_name not in dist_file.release_packages:
return 'Not a released package name: %s' % pkg_name
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
if not repo.release_repository.version:
return "Repository '%s' has no release version" % repo_name
pkg_version = repo.release_repository.version
tag = _get_source_tag(rosdistro_name, pkg_name, pkg_version, os_name,
os_code_name)
cmd = [
'git',
'clone',
'--branch',
tag,
# fetch all branches and tags but no history
'--depth',
'1',
'--no-single-branch',
repo.release_repository.url,
sources_dir
]
print("Invoking '%s'" % ' '.join(cmd))
subprocess.check_call(cmd)
# ensure that the package version is correct
source_version = dpkg_parsechangelog(sources_dir, ['Version'])[0]
if not source_version.startswith(pkg_version) or \
(len(source_version) > len(pkg_version) and
source_version[len(pkg_version)] in '0123456789'):
raise RuntimeError(
('The cloned package version from the GBP (%s) does not match ' +
'the expected package version from the distribution file (%s)') %
(source_version, pkg_version))
# output package version for job description
print("Package '%s' version: %s" % (pkg_name, source_version))
# output package maintainers for job notification
from catkin_pkg.package import parse_package
pkg = parse_package(sources_dir)
maintainer_emails = set([])
for m in pkg.maintainers:
maintainer_emails.add(m.email)
if maintainer_emails:
print('Package maintainer emails: %s' %
' '.join(sorted(maintainer_emails)))
def get_all_distribution_filenames(url=None):
if not url:
url = rosdistro.get_index_url()
distribution_filenames = []
i = rosdistro.get_index(url)
for d in i.distributions.values():
dpath = os.path.abspath(urlparse(d['distribution']).path)
distribution_filenames.append(dpath)
return distribution_filenames
def init_environment():
global os_name, os_version, rdistro, ctx, os_installers, default_os_installer, dist_data, rindex, rcache, rview
ctx = create_default_installer_context()
os_installers = ctx.get_os_installer_keys(os_name)
default_os_installer = ctx.get_default_os_installer_key(os_name)
rindex = get_index(get_index_url())
dist_data = _get_dist_file_data(rindex, rdistro, 'distribution')
rcache = get_distribution(rindex, rdistro)
rview = get_catkin_view(rdistro, os_name, os_version, False)
def get_eol_distribution_filenames(url=None):
if not url:
url = rosdistro.get_index_url()
distribution_filenames = []
i = rosdistro.get_index(url)
for d_name, d in i.distributions.items():
if d_name in EOL_DISTROS:
dpath = os.path.abspath(urlparse(d['distribution']).path)
distribution_filenames.append(dpath)
return distribution_filenames
def test_build_caches():
print("""
Checking if the 'package.xml' files for all packages are fetchable.
If this fails you can run 'rosdistro_build_cache index.yaml' to perform the same check locally.
""")
index = 'file://' + os.path.abspath(INDEX_YAML)
index = get_index(index)
dist_names = sorted(index.distributions.keys())
dist_names = [n for n in dist_names if n not in eol_distro_names]
generate_distribution_caches(INDEX_YAML, dist_names=dist_names)
def test_get_index_v3_invalid():
url = 'file://' + FILES_DIR + '/index_v3_invalid.yaml'
i = get_index(url)
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
try:
get_distribution_file(i, 'foo')
assert False
except AssertionError:
pass
def test_rosdistro_urls():
index_url = 'file://' + FILES_DIR + '/index.yaml'
index = get_index(index_url)
success = True
for distro_name in index.distributions.keys():
print("""
Checking if distribution.yaml contains valid urls for known hosting services.
If this fails you can run 'scripts/check_rosdistro_urls.py file://`pwd`/%s %s' to perform the same check locally.
""" % ('index.yaml', distro_name))
success &= check_rosdistro_urls(index_url, distro_name)
assert success
def test_get_index_v3_invalid():
url = 'file://' + FILES_DIR + '/index_v3_invalid.yaml'
i = get_index(url)
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
try:
get_distribution_file(i, 'foo')
assert False
except AssertionError:
pass
def get_index():
global _rosdistro_index
if _rosdistro_index is None:
_rosdistro_index = rosdistro.get_index(rosdistro.get_index_url())
if _rosdistro_index.version == 1:
error("This version of bloom does not support rosdistro version "
"'{0}', please use an older version of bloom."
.format(_rosdistro_index.version), exit=True)
if _rosdistro_index.version > 2:
error("This version of bloom does not support rosdistro version "
"'{0}', please update bloom.".format(_rosdistro_index.version), exit=True)
return _rosdistro_index
def test_rosdistro_urls():
index_url = 'file://' + FILES_DIR + '/index.yaml'
index = get_index(index_url)
failed_distros = []
for distro_name in index.distributions.keys():
print("""
Checking if distribution.yaml contains valid urls for known hosting services.
If this fails you can run 'scripts/check_rosdistro_urls.py file://`pwd`/%s %s' to perform the same check locally.
""" % ('index.yaml', distro_name))
if not check_rosdistro_urls(index_url, distro_name):
failed_distros.append(distro_name)
assert not failed_distros, "There were problems with urls in the 'distribution.yaml' file for these distros: %s" % failed_distros
def test_get_index_v4():
url = 'file://' + FILES_DIR + '/index_v4.yaml'
i = get_index(url)
assert len(i.distributions.keys()) == 1
assert 'foo' in i.distributions.keys()
assert i.distributions['foo']['distribution_status'] == 'active'
assert i.distributions['foo']['distribution_type'] == 'ros1'
dist_files = get_distribution_files(i, 'foo')
assert len(dist_files) == 2
get_distribution_file(i, 'foo')
def get_distro(self, distro):
if self.__distribution is None:
try:
index = get_index(get_index_url())
self.__distribution = get_distribution_file(
index,
distro
)
except:
print "failed to get data about repo %s in distribution %s" % (self.repo_name, self.distro_name)
raise
return self.__distribution
def scrape_for_release_message_packages(track):
url = rosdistro.get_index_url()
index = rosdistro.get_index(url)
cache = rosdistro.get_release_cache(index, 'hydro')
packages = []
for package_name, package_string in cache.package_xmls.iteritems():
package = catkin_pkg.package.parse_package_string(package_string)
#print(" Name: %s" % package_name)
#print(" Buildtool Depends %s" % package.build)
if has_build_depend_on_message_generation(package):
packages.append({'name': package_name, 'version': package.version})
return packages
def generate_deb_status_table(package, rosdistro_from, rosdistro_to):
DISTROS = collections.OrderedDict()
rosdistro_index = get_index(get_index_url())
for distro in sorted(rosdistro_index.distributions.keys()):
distribution_files = get_distribution_files(rosdistro_index, distro)
if len(distribution_files) > 1:
sys.stderr.write('distribution_files has multiple entories {}\n'.format(distribution_files))
sys.exit(1)
platform = distribution_files[0].release_platforms['ubuntu']
DISTROS[distro] = platform
#print('DISTROS[{}] = {}'.format(distro, platform))
table = []
for bit, arch in zip(['v8', 'hf', '32', '64'],
['arm64', 'armhf', 'i386', 'amd64']):
if not table: # first row
headers = ['Package']
row = ['{} ({})'.format(package, arch)]
for distro, os_list in DISTROS.items():
if not (ord(rosdistro_from) <= ord(distro[0]) <=
ord(rosdistro_to)):
continue
for os in os_list:
if arch.startswith('arm'):
if os == 'xenial':
os_arch = 'ux{bit}_u'.format(bit=bit)
else:
os_arch = 'arm_u'
else:
os_arch = 'u'
if not table: # first row
headers.append(
'{} ({})'.format(distro.capitalize(), os.capitalize()))
url = 'http://build.ros.org/job/{prefix_ros}bin_{os_arch}{prefix_os}{bit}__{package}__ubuntu_{os}_{arch}__binary' # NOQA
url = url.format(
bit=bit,
arch=arch,
os_arch=os_arch,
prefix_os=os[0].upper(),
prefix_ros=distro[0].upper(),
package=package,
os=os,
)
template_md = '[]({url})'
row.append(template_md.format(url=url))
table.append(row)
print(tabulate.tabulate(table, headers=headers, tablefmt='pipe'))
def test_build_caches():
with Fold():
print('Checking that the index.yaml and index-v4.yaml files contain '
'the same information expect additional metadata in the v4.')
index_v3 = get_index('file://' + os.path.abspath(INDEX_V3_YAML))
index_v4 = get_index('file://' + os.path.abspath(INDEX_V4_YAML))
dist_names_v3 = list(sorted(index_v3.distributions.keys()))
dist_names_v4 = list(sorted(index_v4.distributions.keys()))
assert dist_names_v3 == dist_names_v4, \
'Different set of distribution names'
for dist_name in dist_names_v3:
dist_v3_data = index_v3.distributions[dist_name]
dist_v4_data = index_v4.distributions[dist_name]
for key, value in dist_v3_data.items():
assert key in dist_v4_data, \
"For distribution '%s' index.yaml contains the key '%s' " \
"but v4 doesn't contain it" % (dist_name, key)
assert dist_v4_data[key] == value, \
"For distribution '%s' both yaml files contains the key " \
"'%s' but with different values" % (dist_name, key)