def github_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
if 'github.com' not in repo.url:
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
release_tag = get_release_tag(repo, pkg_name)
if not check_remote_tag_exists(repo.url, release_tag):
raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
url = repo.url
if url.endswith('.git'):
url = url[:-4]
url += '/%s/package.xml' % release_tag
if url.startswith('git://'):
url = 'https://' + url[6:]
if url.startswith('https://'):
url = 'https://raw.' + url[8:]
try:
logger.debug('Load package.xml file from url "%s"' % url)
package_xml = urlopen(url).read()
return package_xml
except URLError as e:
logger.debug('- failed (%s), trying "%s"' % (e, url))
raise RuntimeError()
def _generate_rosinstall_for_package(distro, pkg_name, flat=False, tar=False):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
assert repo is not None and repo.version is not None, 'Package "%s" does not have a version"' % pkg_name
local_name = pkg_name
if not flat and repo.package_names != [pkg_name]:
local_name = "%s/%s" % (repo.name, local_name)
url = repo.url
release_tag = get_release_tag(repo, pkg_name)
if tar:
# the repository name might be different than repo.name coming from rosdistro
repo_name = os.path.basename(url[:-4])
suffix = ".git"
if not url.endswith(suffix):
raise RuntimeError("The repository '%s' must end with '%s': %s" % (repo_name, suffix, url))
url = url[: -len(suffix)] + "/archive/{0}.tar.gz".format(release_tag)
data = [
{
"tar": {
"local-name": local_name,
"uri": url,
"version": "{0}-{1}".format(repo_name, release_tag.replace("/", "-")),
}
}
]
else:
data = [{"git": {"local-name": local_name, "uri": url, "version": release_tag}}]
return data
def _generate_rosinstall_for_pkg(repo, pkg_name):
from rosdistro.manifest_provider import get_release_tag
repo_data = {
'local-name': pkg_name,
'uri': repo.url,
'version': get_release_tag(repo, pkg_name)
}
return yaml.safe_dump([{repo.type: repo_data}], default_style=False)
def git_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
try:
release_tag = get_release_tag(repo, pkg_name)
package_xml = _get_package_xml(repo.url, release_tag)
return package_xml
except Exception as e:
raise RuntimeError('Unable to fetch package.xml: %s' % e)
def git_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
try:
release_tag = get_release_tag(repo, pkg_name)
package_xml = _get_package_xml(repo.url, release_tag)
return package_xml
except Exception as e:
raise RuntimeError('Unable to fetch package.xml: %s' % e)
def get_repository_specification_for_released_package(dist_file, pkg_name):
repo_name = dist_file.release_packages[pkg_name].repository_name
release_repo = dist_file.repositories[repo_name].release_repository
version_tag = get_release_tag(release_repo, pkg_name)
return RepositorySpecification(pkg_name, {
'type': 'git',
'url': release_repo.url,
'version': version_tag,
})
def get_repository_specification_for_released_package(dist_file, pkg_name):
repo_name = dist_file.release_packages[pkg_name].repository_name
release_repo = dist_file.repositories[repo_name].release_repository
version_tag = get_release_tag(release_repo, pkg_name)
return RepositorySpecification(
pkg_name, {
'type': 'git',
'url': release_repo.url,
'version': version_tag,
})
def _generate_rosinstall_for_package(distro, pkg_name, flat=False, tar=False):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
assert repo is not None and repo.version is not None, 'Package "%s" does not have a version"' % pkg_name
local_name = pkg_name
if not flat and repo.package_names != [pkg_name]:
local_name = '%s/%s' % (repo.name, local_name)
release_tag = get_release_tag(repo, pkg_name)
return _generate_rosinstall(local_name, repo.url, release_tag, tar=tar)
def _generate_rosinstall_for_package(distro, pkg_name, flat=False, tar=False):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
assert repo is not None and repo.version is not None, 'Package "%s" does not have a version"' % pkg_name
local_name = pkg_name
if not flat and repo.package_names != [pkg_name]:
local_name = '%s/%s' % (repo.name, local_name)
release_tag = get_release_tag(repo, pkg_name)
return _generate_rosinstall(local_name, repo.url, release_tag, tar=tar)
def __init__(self, distro, pkg_name, tar_dir, has_patches=False):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
ros_pkg = RosPackage(pkg_name, repo)
pkg_rosinstall = _generate_rosinstall(pkg_name, repo.url,
get_release_tag(repo, pkg_name),
True)
self.recipe = _gen_recipe_for_package(distro, pkg_name, pkg, repo,
ros_pkg, pkg_rosinstall, tar_dir)
def __init__(self, rosdistro, pkg_name, srcrev_cache, skip_keys):
pkg = rosdistro.release_packages[pkg_name]
repo = rosdistro.repositories[pkg.repository_name].release_repository
ros_pkg = RosPackage(pkg_name, repo)
pkg_rosinstall = _generate_rosinstall(pkg_name, repo.url,
get_release_tag(repo, pkg_name),
True)
self.recipe = _gen_recipe_for_package(rosdistro, pkg_name, pkg, repo,
ros_pkg, pkg_rosinstall,
srcrev_cache, skip_keys)
def __init__(self, distro, pkg_name, has_patches=False):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
ros_pkg = RosPackage(pkg_name, repo)
pkg_rosinstall =\
_generate_rosinstall(pkg_name, repo.url,
get_release_tag(repo, pkg_name), True)
self.pkgbuild =\
_gen_pkgbuild_for_package(distro, pkg_name,
pkg, repo, ros_pkg, pkg_rosinstall)
self.pkgbuild.has_patches = has_patches
if pkg_name in no_python3:
self.pkgbuild.python_3 = False
def github_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
if 'github.com' not in repo.url:
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
release_tag = get_release_tag(repo, pkg_name)
url = repo.url.replace('.git', '/%s/package.xml' % release_tag)
url = url.replace('git://', 'https://')
url = url.replace('https://', 'https://raw.')
try:
logger.debug('Load package.xml file from url "%s"' % url)
package_xml = urllib2.urlopen(url).read()
return package_xml
except urllib2.URLError as e:
logger.debug('- failed (%s), trying "%s"' % (e, url))
raise RuntimeError()
def get_info(self):
res = {}
if not self._distribution_file:
return res
for name in self._distribution_file.repositories.keys():
repo = self._distribution_file.repositories[name]
if repo.release_repository or repo.source_repository:
res[name] = {'distro': self.distro + "_wet", 'version': [], 'url': [], 'branch': []}
# first add devel
for name in self._distribution_file.repositories.keys():
r = self._distribution_file.repositories[name].source_repository
if not r:
continue
if not r.version:
res[name]['branch'].append("")
else:
res[name]['branch'].append(r.version)
res[name]['version'].append('devel')
res[name]['url'].append(r.url)
# then add release
for name in self._distribution_file.repositories.keys():
r = self._distribution_file.repositories[name].release_repository
if not r:
continue
if 'release' not in r.tags:
continue
release_tag = get_release_tag(r, '{package}')
if r.version is not None:
release_tag_without_version = release_tag.replace('/%s' % r.version, '').replace('/%s' % r.version.split('-')[0], '')
else:
release_tag_without_version = release_tag.replace('/{version}', '').replace('/{upstream_version}', '')
res[name]['version'].append('latest')
res[name]['url'].append(r.url)
res[name]['branch'].append(release_tag_without_version)
if r.version:
res[name]['version'].append(r.version.split('-')[0])
res[name]['url'].append(r.url)
res[name]['branch'].append(release_tag)
#res[name]['branch'].append("release/pkg_name/" + r.version.split('-')[0])
return res
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on,
repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
sudo=False, no_chroot=False):
from catkin_pkg.package import InvalidPackage, parse_package_string
from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
from rosdistro.dependency_walker import DependencyWalker
from rosdistro.manifest_provider import get_release_tag
index = get_index(get_index_url())
print "Parsing rosdistro file for %s" % ros_distro
release = get_cached_release(index, ros_distro)
print "Parsing devel file for %s" % ros_distro
source_file = get_source_file(index, ros_distro)
# Create rosdep object
print "Create rosdep object"
rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)
# download the repo_list from source
print "Creating rosinstall file for repo list"
rosinstall = ""
for repo_name, version in zip(repo_list, version_list):
if version == 'devel':
if repo_name not in source_file.repositories:
raise BuildException("Repository %s does not exist in Devel Distro" % repo_name)
print "Using devel distro file to download repositories"
rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name])
else:
if repo_name not in release.repositories:
raise BuildException("Repository %s does not exist in Ros Distro" % repo_name)
repo = release.repositories[repo_name]
if version not in ['latest', 'master']:
assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
for pkg_name in repo.package_names:
release_tag = get_release_tag(repo, pkg_name)
if version in ['latest', 'master']:
release_tag = '/'.join(release_tag.split('/')[:-1])
print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name)
rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag)
print "rosinstall file for all repositories: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install repo list from source"
os.makedirs(repo_sourcespace)
call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))
# get the repositories build dependencies
print "Get build dependencies of repo list"
repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)
# ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
if 'catkin' not in repo_build_dependencies:
repo_build_dependencies.append('catkin')
print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file for repositories that use catkin
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
print "Create a new CMakeLists.txt file using catkin"
# get environment
ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
# check if source workspace contains only package built with catkin
non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)
# make build folder and change into it
os.makedirs(repo_buildspace)
os.chdir(repo_buildspace)
# make test results dir
test_results_dir = os.path.join(workspace, 'test_results')
if os.path.exists(test_results_dir):
shutil.rmtree(test_results_dir)
os.makedirs(test_results_dir)
if not non_catkin_pkgs:
print "Build catkin workspace"
call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
repos_test_results_dir = os.path.join(test_results_dir, 'repos')
call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
#ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories and tests
print "Build repo list"
call("make", ros_env)
call("make tests", ros_env)
# get the repositories test and run dependencies
print "Get test and run dependencies of repo list"
repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True)
print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies))
apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)
# run tests
print "Test repo list"
call("make run_tests", ros_env)
else:
print "Build workspace with non-catkin packages in isolation"
# work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env)
setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh')
ros_env = get_ros_env(setup_file)
# don't do depends-on on things not in release
not_in_release = set(repo_list) - set(release.repositories.keys())
if not_in_release:
print "Removed [%s] repositories which are not in the " %\
', '.join(sorted(not_in_release)), \
"release file for depends-on testing"
repo_list = list(set(repo_list) - not_in_release)
# see if we need to do more work or not
if not test_depends_on:
print "We're not testing the depends-on repositories"
ensure_test_results(test_results_dir)
return
# get repo_list depends-on list
print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list)
walker = DependencyWalker(release)
depends_on = set([])
try:
for repo_name in repo_list:
print('repo_name', repo_name)
repo = release.repositories[repo_name]
for pkg_name in repo.package_names:
print('pkg_name', pkg_name)
depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on)
print('depends_on', depends_on)
except RuntimeError:
print "Exception %s: If you are not in the rosdistro and only in the devel", \
" builds there will be no depends on"
depends_on = set([])
print "Build depends_on list of pkg list: %s" % (', '.join(depends_on))
if len(depends_on) == 0:
print "No wet packages depend on our repo list. Test finished here"
ensure_test_results(test_results_dir)
return
# install depends_on packages from source from release repositories
rosinstall = ''
non_catkin_pkgs = []
for pkg_name in depends_on:
repo = release.repositories[release.packages[pkg_name].repository_name]
if repo.version is None:
continue
pkg_xml = release.get_package_xml(pkg_name)
if pkg_xml is None:
raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e))
if _is_non_catkin_package(pkg):
non_catkin_pkgs.append(pkg.name)
rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name)
if non_catkin_pkgs:
print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
return
print "Rosinstall for depends_on:\n %s" % rosinstall
with open(workspace + "/depends_on.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for depends on"
# install all repository and system dependencies of the depends_on list
print "Install all depends_on from source: %s" % (', '.join(depends_on))
os.makedirs(dependson_sourcespace)
call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))
# check if depends_on workspace contains only package built with catkin
non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace)
if non_catkin_pkgs:
print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
return
# get build and test dependencies of depends_on list
dependson_build_dependencies = []
for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False):
print " Checking dependency %s" % d
if d in dependson_build_dependencies:
print " Already in dependson_build_dependencies"
if d in depends_on:
print " Is a direct dependency of the repo list, and is installed from source"
if d in repo_list:
print " Is one of the repositories tested"
if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
dependson_build_dependencies.append(d)
print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))
dependson_test_dependencies = []
for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True):
if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
dependson_test_dependencies.append(d)
print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))
# install build dependencies
print "Install all build dependencies of the depends_on list"
apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file again
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
os.makedirs(dependson_buildspace)
os.chdir(dependson_buildspace)
print "Create a new CMakeLists.txt file using catkin"
call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
#ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))
# build repositories
print "Build depends-on packages"
call("make", ros_env)
# install test dependencies
print "Install all test dependencies of the depends_on list"
apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)
# test repositories
print "Test depends-on packages"
call("make run_tests", ros_env)
ensure_test_results(test_results_dir)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on,
repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
sudo=False, no_chroot=False):
from catkin_pkg.package import InvalidPackage, parse_package_string
from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
from rosdistro.dependency_walker import DependencyWalker
from rosdistro.manifest_provider import get_release_tag
index = get_index(get_index_url())
print "Parsing rosdistro file for %s" % ros_distro
release = get_cached_release(index, ros_distro)
print "Parsing devel file for %s" % ros_distro
source_file = get_source_file(index, ros_distro)
# Create rosdep object
print "Create rosdep object"
rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)
# download the repo_list from source
print "Creating rosinstall file for repo list"
rosinstall = ""
for repo_name, version in zip(repo_list, version_list):
if version == 'devel':
if repo_name not in source_file.repositories:
raise BuildException("Repository %s does not exist in Devel Distro" % repo_name)
print "Using devel distro file to download repositories"
rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name])
else:
if repo_name not in release.repositories:
raise BuildException("Repository %s does not exist in Ros Distro" % repo_name)
repo = release.repositories[repo_name]
if version not in ['latest', 'master']:
assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
for pkg_name in repo.package_names:
release_tag = get_release_tag(repo, pkg_name)
if version in ['latest', 'master']:
release_tag = '/'.join(release_tag.split('/')[:-1])
print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name)
rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag)
print "rosinstall file for all repositories: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install repo list from source"
os.makedirs(repo_sourcespace)
call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))
# get the repositories build dependencies
print "Get build dependencies of repo list"
repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)
# ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
if 'catkin' not in repo_build_dependencies:
repo_build_dependencies.append('catkin')
print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file for repositories that use catkin
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
print "Create a new CMakeLists.txt file using catkin"
# get environment
ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
# check if source workspace contains only package built with catkin
non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)
# make build folder and change into it
os.makedirs(repo_buildspace)
os.chdir(repo_buildspace)
# make test results dir
test_results_dir = os.path.join(workspace, 'test_results')
if os.path.exists(test_results_dir):
shutil.rmtree(test_results_dir)
os.makedirs(test_results_dir)
if not non_catkin_pkgs:
print "Build catkin workspace"
call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
repos_test_results_dir = os.path.join(test_results_dir, 'repos')
call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
#ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories and tests
print "Build repo list"
call("make", ros_env)
call("make tests", ros_env)
# get the repositories test and run dependencies
print "Get test and run dependencies of repo list"
repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True)
print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies))
apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)
# run tests
print "Test repo list"
call("make run_tests", ros_env)
else:
print "Build workspace with non-catkin packages in isolation"
# work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env)
setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh')
ros_env = get_ros_env(setup_file)
# see if we need to do more work or not
if not test_depends_on:
print "We're not testing the depends-on repositories"
ensure_test_results(test_results_dir)
return
# get repo_list depends-on list
print "Get list of wet repositories that build-depend on repo list: %s" % ', '.join(repo_list)
walker = DependencyWalker(release)
depends_on = set([])
try:
for repo_name in repo_list:
print('repo_name', repo_name)
repo = release.repositories[repo_name]
for pkg_name in repo.package_names:
print('pkg_name', pkg_name)
depends_on |= walker.get_recursive_depends_on(pkg_name, ['buildtool', 'build'], ignore_pkgs=depends_on)
print('depends_on', depends_on)
except RuntimeError:
print "Exception %s: If you are not in the rosdistro and only in the devel", \
" builds there will be no depends on"
depends_on = set([])
print "Build depends_on list of pkg list: %s" % (', '.join(depends_on))
if len(depends_on) == 0:
print "No wet packages depend on our repo list. Test finished here"
ensure_test_results(test_results_dir)
return
# install depends_on packages from source from release repositories
rosinstall = ''
non_catkin_pkgs = []
for pkg_name in depends_on:
repo = release.repositories[release.packages[pkg_name].repository_name]
if repo.version is None:
continue
pkg_xml = release.get_package_xml(pkg_name)
if pkg_xml is None:
raise BuildException('Could not retrieve package.xml for package "%s" from rosdistro cache' % pkg_name)
try:
pkg = parse_package_string(pkg_xml)
except InvalidPackage as e:
raise BuildException('package.xml for package "%s" from rosdistro cache is invalid: %s' % (pkg_name, e))
if _is_non_catkin_package(pkg):
non_catkin_pkgs.append(pkg.name)
rosinstall += _generate_rosinstall_for_pkg(repo, pkg_name)
if non_catkin_pkgs:
print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
return
print "Rosinstall for depends_on:\n %s" % rosinstall
with open(workspace + "/depends_on.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for depends on"
# install all repository and system dependencies of the depends_on list
print "Install all depends_on from source: %s" % (', '.join(depends_on))
os.makedirs(dependson_sourcespace)
call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))
# check if depends_on workspace contains only package built with catkin
non_catkin_pkgs = _get_non_catkin_packages(dependson_sourcespace)
if non_catkin_pkgs:
print 'Non-catkin packages depend on our repo list (%s). Skipping depends_on packages here' % ', '.join(sorted(non_catkin_pkgs))
create_test_result(test_results_dir, failure='Non-catkin packages depend on the repos (%s). Skip building and testing depends_on packages.' % ', '.join(sorted(non_catkin_pkgs)))
return
# get build and test dependencies of depends_on list
dependson_build_dependencies = []
for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False):
print " Checking dependency %s" % d
if d in dependson_build_dependencies:
print " Already in dependson_build_dependencies"
if d in depends_on:
print " Is a direct dependency of the repo list, and is installed from source"
if d in repo_list:
print " Is one of the repositories tested"
if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
dependson_build_dependencies.append(d)
print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))
dependson_test_dependencies = []
for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True):
if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
dependson_test_dependencies.append(d)
print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))
# install build dependencies
print "Install all build dependencies of the depends_on list"
apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file again
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
os.makedirs(dependson_buildspace)
os.chdir(dependson_buildspace)
print "Create a new CMakeLists.txt file using catkin"
call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
#ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))
# build repositories
print "Build depends-on packages"
call("make", ros_env)
# install test dependencies
print "Install all test dependencies of the depends_on list"
apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)
# test repositories
print "Test depends-on packages"
call("make run_tests", ros_env)
ensure_test_results(test_results_dir)
def _generate_rosinstall_for_pkg(repo, pkg_name):
from rosdistro.manifest_provider import get_release_tag
return _generate_rosinstall_for_pkg_version(repo, pkg_name, get_release_tag(repo, pkg_name))
def get_released_repo(self, pkg_name):
pkg = self._distro.release_packages[pkg_name]
repo = self._distro.repositories[
pkg.repository_name].release_repository
release_tag = get_release_tag(repo, pkg_name)
return repo.url, release_tag
def get_wet_rosinstall(release_file, name, prefix=None):
repo = _find_repo(release_file, name)
if repo is None: # wait, what?
return None
return build_rosinstall(name, repo.url, 'git', get_release_tag(repo, name), prefix)
def _test_repositories(ros_distro, repo_list, version_list, workspace, test_depends_on,
repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
sudo=False, no_chroot=False):
from catkin_pkg.package import InvalidPackage, parse_package_string
from rosdistro import get_cached_release, get_index, get_index_url, get_source_file
from rosdistro.dependency_walker import DependencyWalker
from rosdistro.manifest_provider import get_release_tag
index = get_index(get_index_url())
print "Parsing rosdistro file for %s" % ros_distro
release = get_cached_release(index, ros_distro)
print "Parsing devel file for %s" % ros_distro
source_file = get_source_file(index, ros_distro)
# Create rosdep object
print "Create rosdep object"
rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)
# download the repo_list from source
print "Creating rosinstall file for repo list"
rosinstall = ""
for repo_name, version in zip(repo_list, version_list):
if version == 'devel':
if repo_name not in source_file.repositories:
raise BuildException("Repository %s does not exist in Devel Distro" % repo_name)
print "Using devel distro file to download repositories"
rosinstall += _generate_rosinstall_for_repo(source_file.repositories[repo_name])
else:
if repo_name not in release.repositories:
raise BuildException("Repository %s does not exist in Ros Distro" % repo_name)
repo = release.repositories[repo_name]
if version not in ['latest', 'master']:
assert repo.version is not None, 'Repository "%s" does not have a version set' % repo_name
assert 'release' in repo.tags, 'Repository "%s" does not have a "release" tag set' % repo_name
for pkg_name in repo.package_names:
release_tag = get_release_tag(repo, pkg_name)
if version in ['latest', 'master']:
release_tag = '/'.join(release_tag.split('/')[:-1])
print 'Using tag "%s" of release distro file to download package "%s from repo "%s' % (version, pkg_name, repo_name)
rosinstall += _generate_rosinstall_for_repo(release.repositories[repo_name], version=release_tag)
print "rosinstall file for all repositories: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install repo list from source"
os.makedirs(repo_sourcespace)
call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))
# get the repositories build dependencies
print "Get build dependencies of repo list"
repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, run_depends=False)
# ensure that catkin gets installed, for non-catkin packages so that catkin_make_isolated is available
if 'catkin' not in repo_build_dependencies:
repo_build_dependencies.append('catkin')
print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file for repositories that use catkin
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
print "Create a new CMakeLists.txt file using catkin"
# get environment
ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
# check if source workspace contains only package built with catkin
non_catkin_pkgs = _get_non_catkin_packages(repo_sourcespace)
# make build folder and change into it
os.makedirs(repo_buildspace)
os.chdir(repo_buildspace)
# make test results dir
test_results_dir = os.path.join(workspace, 'test_results')
if os.path.exists(test_results_dir):
shutil.rmtree(test_results_dir)
os.makedirs(test_results_dir)
if not non_catkin_pkgs:
print "Build catkin workspace"
call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
repos_test_results_dir = os.path.join(test_results_dir, 'repos')
helper = subprocess.Popen(('cmake %s -DCMAKE_TOOLCHAIN_FILE=/opt/ros/groovy/share/ros/core/rosbuild/rostoolchain.cmake -DCATKIN_TEST_RESULTS_DIR=%s'%(repo_sourcespace,repos_test_results_dir)).split(' '), env=ros_env)
helper.communicate()
res = 0
if helper.returncode != 0:
res = helper.returncode
ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories
print "Build repo list"
print "CMAKE_PREFIX_PATH: %s"%ros_env['CMAKE_PREFIX_PATH']
call("make", ros_env)
# Concatenate filelists
print '----------------- Concatenate filelists ----------------- '
filelist = '%s'%repo_buildspace + '/filelist.lst'
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/concatenate_filelists.py --dir %s --filelist %s'%(workspace,repo_buildspace, filelist)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// cma analysis done ////////////////// \n\n'
# Run CMA
print '----------------- Run CMA analysis ----------------- '
cmaf = repo_sourcespace#repo_buildspace
helper = subprocess.Popen(('pal QACPP -cmaf %s -list %s'%(cmaf, filelist)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// cma analysis done ////////////////// \n\n'
# Export metrics to yaml and csv files
# get uri infos
#uri= distro.get_repositories()[repo_list[0]].url
repo_name = repo_list[0]
repo_data = release.get_data()['repositories'][repo_name]
print "repo_data", repo_data
uri = repo_data['url']
uri_info = 'master' #repo_data['version']
vcs_type = 'git' # repo_data['type']
print '----------------- Export metrics to yaml and csv files ----------------- '
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/export_metrics_to_yaml_wet.py --path %s --path_src %s --doc metrics --csv csv --config %s/jenkins_scripts/code_quality/export_config.yaml --distro %s --stack %s --uri %s --uri_info %s --vcs_type %s'%(workspace, repo_buildspace, repo_sourcespace, workspace, ros_distro, repo_name, uri, uri_info, vcs_type)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// export metrics to yaml and csv files done ////////////////// \n\n'
# Push results to server
print '----------------- Push results to server ----------------- '
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/push_results_to_server_wet.py --path %s --doc metrics --path_src %s --meta_package %s'%(workspace, repo_buildspace, repo_sourcespace, repo_list)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// push results to server done ////////////////// \n\n'
# Upload results to QAVerify
print ' ----------------- upload results to QAVerify ----------------- '
shutil.rmtree(os.path.join(workspace, 'snapshots_path'), ignore_errors=True)
os.makedirs(os.path.join(workspace, 'snapshots_path'))
snapshots_path = '%s/snapshots_path'%workspace
project_name = repo_list[0] + '-' + ros_distro
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/upload_to_QAVerify_wet.py --path %s --snapshot %s --project %s --stack_name %s'%(workspace, repo_buildspace, snapshots_path, project_name, repo_list[0])).split(' '), env=os.environ)
helper.communicate()
print '////////////////// upload results to QAVerify done ////////////////// \n\n'
if os.path.exists(snapshots_path):
shutil.rmtree(snapshots_path)
else:
print "Build workspace with non-catkin packages in isolation"
# work around catkin_make_isolated issue (at least with version 0.5.65 of catkin)
os.makedirs(os.path.join(repo_buildspace, 'devel_isolated'))
call('catkin_make_isolated --source %s --install-space install_isolated --install' % repo_sourcespace, ros_env)
setup_file = os.path.join(repo_buildspace, 'install_isolated', 'setup.sh')
ros_env = get_ros_env(setup_file)
if res != 0:
print "helper_return_code is: %s"%(helper.returncode)
assert 'analysis_wet.py failed'
raise Exception("analysis_wet.py failed. Check out the console output above for details.")
# create dummy test results
env = dict()
env['INSTALL_DIR'] = os.getenv('INSTALL_DIR', '')
test_results_path = workspace + '/test_results'
if os.path.exists(test_results_path):
shutil.rmtree(test_results_path)
os.makedirs(test_results_path)
test_file= test_results_path + '/test_file.xml'
f = open(test_file, 'w')
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
f.write('<testsuite tests="1" failures="0" time="1" errors="0" name="dummy test">\n')
f.write(' <testcase name="dummy rapport" classname="Results" /> \n')
f.write('</testsuite> \n')
f.close()
def get_wet_rosinstall(release_file, name, prefix=None):
repo = _find_repo(release_file, name)
if repo is None: # wait, what?
return None
return build_rosinstall(name, repo.url, 'git', get_release_tag(repo, name),
prefix)
def _generate_rosinstall_for_pkg(repo, pkg_name):
from rosdistro.manifest_provider import get_release_tag
return _generate_rosinstall_for_pkg_version(
repo, pkg_name, get_release_tag(repo, pkg_name))
