def main():
# parse command line options
#(options, args) = get_options(['rosdistro', 'buildsystem'], ['repeat', 'source-only'])
parser = optparse.OptionParser()
(options, args) = parser.parse_args()
if not options:
return -1
platform = args[0] # e.g. precise
arch = args[1] # e.g. amd64
email = args[2] # e.g. [email protected]
ros_distro = args[3] # e.g. groovy
# parse the rosdistro file
print "Parsing rosdistro file for %s"%ros_distro
distro = rosdistro.RosDistro(ros_distro)
print "Parsing devel file for %s"%ros_distro
devel = rosdistro.DevelDistro(ros_distro)
env = get_environment2()
try:
for stack in distro.get_repositories():
print 'Analyzing stack %s'%stack
h = subprocess.Popen(('run_chroot_jenkins_now %s %s %s metrics %s %s wet'%(platform, arch, email, ros_distro,stack)).split(' '),env=env)
h.communicate()
except Exception, ex:
print "%s. Check the console output for test failure details."%ex
traceback.print_exc(file=sys.stdout)
raise ex
def build_downstream_post_fuerte(ros_distro, build_repo, workspace, server):
ros_package_path = os.environ['ROS_PACKAGE_PATH']
b_r_short = build_repo.split('__')[0]
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository')
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')
repo_buildspace = os.path.join(tmpdir, 'build_repository')
dependson_sourcespace = os.path.join(tmpdir, 'src_depends_on')
dependson_sourcespace_wet = os.path.join(tmpdir, 'src_depends_on', 'wet')
dependson_sourcespace_dry = os.path.join(tmpdir, 'src_depends_on', 'dry')
dependson_buildspace = os.path.join(tmpdir, 'build_depend_on')
dry_test_results_dir = os.path.join(dependson_sourcespace_dry,
'test_results')
# TODO clean up test result dirs
# get depends on
print "Get list of released repositories that (build-)depend on repository %s" % b_r_short
ros_depends_on = []
distro_ros = rosdistro.RosDistro(ros_distro, 'http://%s/~jenkins' % server)
for d in distro_ros.get_depends_on([
b_r_short
])['build'] + distro_ros.get_depends_on([b_r_short])['buildtool']:
if not d in ros_depends_on and d != b_r_short:
ros_depends_on.append(d)
# TODO get_depends_on of cob_distro release (only for intern repos necessary)
cob_depends_on = []
#distro_cob = cob_distro.CobDistro(ros_distro)
#for d in distro_cob.get_depends_on([b_r_short])['build'] + distro_cob.get_depends_on([b_r_short])['buildtool']:
# if not d in cob_depends_on and d != b_r_short:
# cob_depends_on.append(d)
if len(ros_depends_on + cob_depends_on) == 0:
print "No repository depends on repository %s. Test finished" % b_r_short
return
print "Build depends_on list of repository %s:\n - %s" % (
'\n - '.join(ros_depends_on + cob_depends_on))
# install depends_on repository from source
rosinstall = distro_ros.get_rosinstall(ros_depends_on)
#rosinstall += distro_cob.get_rosinstall(cob_depends_on) TODO
print "Rosinstall for depends_on:\n %s" % rosinstall
with open(workspace + "/depends_on.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for depends on"
# install all repository and system dependencies of the depends_on list
print "Install all depends_on from source: %s" % (
', '.join(ros_depends_on))
os.makedirs(dependson_sourcespace)
common.call("rosinstall %s %s/depends_on.rosinstall /opt/ros/%s" %
(dependson_sourcespace, workspace, ros_distro))
# all packages in dependson_sourcespace
(catkin_packages, stacks,
manifest_packages) = common.get_all_packages(dependson_sourcespace)
print catkin_packages
print stacks
print manifest_packages
# get build and test dependencies of depends_on list
dependson_build_dependencies = []
for d in common.get_nonlocal_dependencies(catkin_packages,
stacks, {},
build_depends=True,
test_depends=False):
print " Checking dependency %s" % d
if d in dependson_build_dependencies:
print " Already in dependson_build_dependencies"
elif d in ros_depends_on or d in cob_depends_on:
print " Is a direct dependency of the repository, and is installed from source"
elif d == b_r_short:
print " Is the tested repository"
else:
dependson_build_dependencies.append(d)
print "Build dependencies of depends_on list are %s" % (
', '.join(dependson_build_dependencies))
dependson_test_dependencies = []
for d in common.get_nonlocal_dependencies(catkin_packages,
stacks, {},
build_depends=False,
test_depends=True):
if d not in dependson_test_dependencies + ros_depends_on + cob_depends_on and d != b_r_short:
dependson_test_dependencies.append(d)
print "Test dependencies of depends_on list are %s" % (
', '.join(dependson_test_dependencies))
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
os.makedirs(dependson_sourcespace_wet)
os.makedirs(dependson_sourcespace_dry)
# get all folders in dependson_sourcespace
sourcespace_dirs = [
name for name in os.listdir(dependson_sourcespace)
if os.path.isdir(os.path.join(dependson_sourcespace, name))
]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys():
shutil.move(os.path.join(dependson_sourcespace, dir),
os.path.join(dependson_sourcespace_wet, dir))
if dir in stacks.keys():
shutil.move(os.path.join(dependson_sourcespace, dir),
os.path.join(dependson_sourcespace_dry, dir))
# Create rosdep object
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except:
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
# install build dependencies
print "Install all build dependencies of the depends_on list: %s" % (
', '.join(dependson_build_dependencies))
common.apt_get_install_also_nonrosdep(dependson_build_dependencies,
rosdep_resolver)
# env
print "Setting up environment"
ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
common.call("env", ros_env)
ros_env_dependson = common.get_ros_env(
os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join(
[dependson_buildspace, repo_buildspace, ros_package_path])
common.call("env", ros_env_dependson)
### catkin repositories
if catkin_packages != {}:
os.makedirs(dependson_buildspace)
os.chdir(dependson_buildspace)
print "Create a new CMakeLists.txt for catkin packages"
if ros_distro == 'fuerte':
common.call(
"ln -s %s %s" %
(os.path.join(dependson_sourcespace_wet, 'catkin', 'cmake',
'toplevel.cmake'),
os.path.join(dependson_sourcespace_wet, 'CMakeLists.txt')))
else:
common.call("catkin_init_workspace %s" % dependson_sourcespace_wet,
ros_env_dependson)
try:
common.call("cmake %s" % dependson_sourcespace_wet,
ros_env_dependson)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to cmake wet repositories")
#ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories
print "Build wet depends_on list"
try:
common.call("make", ros_env_dependson)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to make wet packages")
if dependson_test_dependencies != []:
# install test dependencies
print "Install all test dependencies of the depends_on list: %s" % (
', '.join(dependson_test_dependencies))
common.apt_get_install_also_nonrosdep(dependson_test_dependencies,
rosdep_resolver)
# test repositories
try:
common.call("make run_tests", ros_env_dependson)
except common.BuildException as ex:
print ex.msg
# copy test results
common.copy_test_results(workspace, dependson_buildspace)
### rosbuild repositories
if stacks != {}:
# env
print "Setting up environment"
ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([
dependson_buildspace, repo_buildspace, dependson_sourcespace_dry,
repo_sourcespace_dry, ros_package_path
])
common.call("env", ros_env_dependson)
#print "Make rosdep"
#common.call("rosmake rosdep", ros_env)
#for stack in stacks.keys():
# common.call("rosdep install -y %s" % stack, ros_env_repo)
dependson_sourcespace_dry_dirs = [
name for name in os.listdir(dependson_sourcespace_dry)
if os.path.isdir(os.path.join(dependson_sourcespace_dry, name))
]
print "Build dry depends_on repositories:\n - %s" % '\n - '.join(
dependson_sourcespace_dry_dirs)
os.mkdir(dry_test_results_dir)
for dry_dependson in dependson_sourcespace_dry_dirs:
try:
common.call(
"rosmake -rV --profile --pjobs=8 --output=%s %s" %
(dry_test_results_dir, b_r_short), ros_env_dependson)
except:
raise common.BuildException("Failed to rosmake %s" % b_r_short)
try:
common.call(
"rosmake -rV --profile --pjobs=8 --test-only --output=%s %s"
% (dry_test_results_dir, b_r_short), ros_env_dependson)
# TODO output dir ??
except:
print "Failed to test %s" % dry_dependson
# copy test results
common.call("rosrun rosunit clean_junit_xml.py", ros_env_dependson)
common.copy_test_results(workspace, dependson_sourcespace_dry)
def _test_repositories_fuerte(ros_distro, repo_list, version_list, workspace, test_depends_on,
repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
sudo=False, no_chroot=False):
import rosdistro
# parse the rosdistro file
print "Parsing rosdistro file for %s" % ros_distro
distro = rosdistro.RosDistro(ros_distro)
print "Parsing devel file for %s" % ros_distro
devel = rosdistro.DevelDistro(ros_distro)
# Create rosdep object
print "Create rosdep object"
rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)
# download the repo_list from source
print "Creating rosinstall file for repo list"
rosinstall = ""
for repo, version in zip(repo_list, version_list):
if version == 'devel':
if repo not in devel.repositories:
raise BuildException("Repository %s does not exist in Devel Distro" % repo)
print "Using devel distro file to download repositories"
rosinstall += devel.repositories[repo].get_rosinstall()
else:
if repo not in distro.get_repositories():
raise BuildException("Repository %s does not exist in Ros Distro" % repo)
if version in ['latest', 'master']:
print "Using latest release distro file to download repositories"
rosinstall += distro.get_rosinstall(repo, version='master')
else:
print "Using version %s of release distro file to download repositories" % version
rosinstall += distro.get_rosinstall(repo, version)
print "rosinstall file for all repositories: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install repo list from source"
os.makedirs(repo_sourcespace)
call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))
# get the repositories build dependencies
print "Get build dependencies of repo list"
repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, run_depends=False)
print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file for repositories that use catkin
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
print "Create a new CMakeLists.txt file using catkin"
ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
test_results_dir = os.path.join(workspace, 'test_results')
repos_test_results_dir = os.path.join(test_results_dir, 'repos')
os.makedirs(repo_buildspace)
os.chdir(repo_buildspace)
helper = subprocess.Popen(('cmake %s -DCMAKE_TOOLCHAIN_FILE=/opt/ros/groovy/share/ros/core/rosbuild/rostoolchain.cmake -DCATKIN_TEST_RESULTS_DIR=%s'%(repo_sourcespace,repos_test_results_dir)).split(' '), env=ros_env)
helper.communicate()
res = 0
if helper.returncode != 0:
res = helper.returncode
ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories
print "Build repo list"
print "CMAKE_PREFIX_PATH: %s"%ros_env['CMAKE_PREFIX_PATH']
call("make", ros_env)
# Concatenate filelists
print '----------------- Concatenate filelists ----------------- '
filelist = '%s'%repo_buildspace + '/filelist.lst'
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/concatenate_filelists.py --dir %s --filelist %s'%(workspace,repo_buildspace, filelist)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// cma analysis done ////////////////// \n\n'
# Run CMA
print '----------------- Run CMA analysis ----------------- '
cmaf = repo_sourcespace#repo_buildspace
helper = subprocess.Popen(('pal QACPP -cmaf %s -list %s'%(cmaf, filelist)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// cma analysis done ////////////////// \n\n'
# Export metrics to yaml and csv files
# get uri infos
uri= distro.get_repositories()[repo_list[0]].url
uri_info= 'master'
vcs_type= 'git'
print '----------------- Export metrics to yaml and csv files ----------------- '
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/export_metrics_to_yaml_wet.py --path %s --path_src %s --doc metrics --csv csv --config %s/jenkins_scripts/code_quality/export_config.yaml --distro %s --stack %s --uri %s --uri_info %s --vcs_type %s'%(workspace, repo_buildspace, repo_sourcespace, workspace, ros_distro, repo_list, uri, uri_info, vcs_type)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// export metrics to yaml and csv files done ////////////////// \n\n'
# Push results to server
print '----------------- Push results to server ----------------- '
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/push_results_to_server_wet.py --path %s --doc metrics --path_src %s --meta_package %s'%(workspace, repo_buildspace, repo_sourcespace, repo_list)).split(' '), env=os.environ)
helper.communicate()
print '////////////////// push results to server done ////////////////// \n\n'
# Upload results to QAVerify
print ' ----------------- upload results to QAVerify ----------------- '
shutil.rmtree(os.path.join(workspace, 'snapshots_path'), ignore_errors=True)
os.makedirs(os.path.join(workspace, 'snapshots_path'))
snapshots_path = '%s/snapshots_path'%workspace
project_name = repo_list[0] + '-' + ros_distro
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/wet/upload_to_QAVerify_wet.py --path %s --snapshot %s --project %s --stack_name %s'%(workspace, repo_buildspace, snapshots_path, project_name, repo_list[0])).split(' '), env=os.environ)
helper.communicate()
print '////////////////// upload results to QAVerify done ////////////////// \n\n'
if os.path.exists(snapshots_path):
shutil.rmtree(snapshots_path)
if res != 0:
print "helper_return_code is: %s"%(helper.returncode)
assert 'analysis_wet.py failed'
raise Exception("analysis_wet.py failed. Check out the console output above for details.")
# create dummy test results
env = dict()
env['INSTALL_DIR'] = os.getenv('INSTALL_DIR', '')
test_results_path = workspace + '/test_results'
if os.path.exists(test_results_path):
shutil.rmtree(test_results_path)
os.makedirs(test_results_path)
test_file= test_results_path + '/test_file.xml'
f = open(test_file, 'w')
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
f.write('<testsuite tests="1" failures="0" time="1" errors="0" name="dummy test">\n')
f.write(' <testcase name="dummy rapport" classname="Results" /> \n')
f.write('</testsuite> \n')
f.close()
def _test_repositories_fuerte(ros_distro, repo_list, version_list, workspace, test_depends_on,
repo_sourcespace, dependson_sourcespace, repo_buildspace, dependson_buildspace,
sudo=False, no_chroot=False):
import rosdistro
# parse the rosdistro file
print "Parsing rosdistro file for %s" % ros_distro
distro = rosdistro.RosDistro(ros_distro)
print "Parsing devel file for %s" % ros_distro
devel = rosdistro.DevelDistro(ros_distro)
# Create rosdep object
print "Create rosdep object"
rosdep_resolver = rosdep.RosDepResolver(ros_distro, sudo, no_chroot)
# download the repo_list from source
print "Creating rosinstall file for repo list"
rosinstall = ""
for repo, version in zip(repo_list, version_list):
if version == 'devel':
if repo not in devel.repositories:
raise BuildException("Repository %s does not exist in Devel Distro" % repo)
print "Using devel distro file to download repositories"
rosinstall += devel.repositories[repo].get_rosinstall()
else:
if repo not in distro.get_repositories():
raise BuildException("Repository %s does not exist in Ros Distro" % repo)
if version in ['latest', 'master']:
print "Using latest release distro file to download repositories"
rosinstall += distro.get_rosinstall(repo, version='master')
else:
print "Using version %s of release distro file to download repositories" % version
rosinstall += distro.get_rosinstall(repo, version)
print "rosinstall file for all repositories: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install repo list from source"
os.makedirs(repo_sourcespace)
call("rosinstall %s %s/repo.rosinstall --catkin" % (repo_sourcespace, workspace))
# get the repositories build dependencies
print "Get build dependencies of repo list"
repo_build_dependencies = get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)
print "Install build dependencies of repo list: %s" % (', '.join(repo_build_dependencies))
apt_get_install(repo_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file for repositories that use catkin
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(repo_sourcespace, 'CMakeLists.txt'))
print "Create a new CMakeLists.txt file using catkin"
ros_env = get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
call("catkin_init_workspace %s" % repo_sourcespace, ros_env)
test_results_dir = os.path.join(workspace, 'test_results')
repos_test_results_dir = os.path.join(test_results_dir, 'repos')
os.makedirs(repo_buildspace)
os.chdir(repo_buildspace)
call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (repo_sourcespace, repos_test_results_dir), ros_env)
#ros_env_repo = get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories and tests
print "Build repo list"
call("make", ros_env)
call("make tests", ros_env)
# get the repositories test and run dependencies
print "Get test and run dependencies of repo list"
repo_test_dependencies = get_dependencies(repo_sourcespace, build_depends=False, test_depends=True)
print "Install test and run dependencies of repo list: %s" % (', '.join(repo_test_dependencies))
apt_get_install(repo_test_dependencies, rosdep_resolver, sudo)
# run tests
print "Test repo list"
call("make run_tests", ros_env)
# see if we need to do more work or not
if not test_depends_on:
print "We're not testing the depends-on repositories"
ensure_test_results(test_results_dir)
return
# get repo_list depends-on list
print "Get list of wet repositories that build-depend on repo list %s" % ', '.join(repo_list)
depends_on = []
try:
for d in distro.get_depends_on(repo_list)['build'] + distro.get_depends_on(repo_list)['buildtool']:
if not d in depends_on and not d in repo_list:
depends_on.append(d)
except RuntimeError:
print "Exception %s: If you are not in the rosdistro and only in the devel", \
" builds there will be no depends on"
depends_on = []
print "Build depends_on list of repo list: %s" % (', '.join(depends_on))
if len(depends_on) == 0:
print "No wet repositories depend on our repo list. Test finished here"
ensure_test_results(test_results_dir)
return
# install depends_on repositories from source
rosinstall = distro.get_rosinstall(depends_on)
print "Rosinstall for depends_on:\n %s" % rosinstall
with open(workspace + "/depends_on.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for depends on"
# install all repository and system dependencies of the depends_on list
print "Install all depends_on from source: %s" % (', '.join(depends_on))
os.makedirs(dependson_sourcespace)
call("rosinstall --catkin %s %s/depends_on.rosinstall" % (dependson_sourcespace, workspace))
# get build and test dependencies of depends_on list
dependson_build_dependencies = []
for d in get_dependencies(dependson_sourcespace, build_depends=True, test_depends=False):
print " Checking dependency %s" % d
if d in dependson_build_dependencies:
print " Already in dependson_build_dependencies"
if d in depends_on:
print " Is a direct dependency of the repo list, and is installed from source"
if d in repo_list:
print " Is on of the repositories tested"
if not d in dependson_build_dependencies and not d in depends_on and not d in repo_list:
dependson_build_dependencies.append(d)
print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))
dependson_test_dependencies = []
for d in get_dependencies(dependson_sourcespace, build_depends=False, test_depends=True):
if not d in dependson_test_dependencies and not d in depends_on and not d in repo_list:
dependson_test_dependencies.append(d)
print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))
# install build dependencies
print "Install all build dependencies of the depends_on list"
apt_get_install(dependson_build_dependencies, rosdep_resolver, sudo)
# replace the CMakeLists.txt file again
print "Removing the CMakeLists.txt file generated by rosinstall"
os.remove(os.path.join(dependson_sourcespace, 'CMakeLists.txt'))
os.makedirs(dependson_buildspace)
os.chdir(dependson_buildspace)
print "Create a new CMakeLists.txt file using catkin"
call("catkin_init_workspace %s" % dependson_sourcespace, ros_env)
depends_on_test_results_dir = os.path.join(test_results_dir, 'depends_on')
call("cmake %s -DCATKIN_TEST_RESULTS_DIR=%s" % (dependson_sourcespace, depends_on_test_results_dir), ros_env)
#ros_env_depends_on = get_ros_env(os.path.join(dependson_buildspace, 'devel/setup.bash'))
# build repositories
print "Build depends-on repositories"
call("make", ros_env)
# install test dependencies
print "Install all test dependencies of the depends_on list"
apt_get_install(dependson_test_dependencies, rosdep_resolver, sudo)
# test repositories
print "Test depends-on repositories"
call("make run_tests", ros_env)
ensure_test_results(test_results_dir)