def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[
0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_file(pipeline_repos_owner,
server_name,
user_name,
file_location=os.environ["WORKSPACE"])
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# for hardware build: only support hydro
if ros_distro == "groovy":
sys.exit(False)
# for hardware build: remove old build artifacts
limit = 3 # limit amount of old builds
print workspace
while len(os.listdir(workspace + "/..")
) > limit + 1: # we will not count the "current" sym link
list = os.listdir(workspace + "/..")
shutil.rmtree(workspace + "/../" + sorted(list)[0])
# set up directories variables
tmpdir = workspace
repo_checkoutspace = os.path.join(
tmpdir, 'checkout'
) # location to store repositories in (will be separated in wet and dry later on)
os.makedirs(repo_checkoutspace)
repo_sourcespace = os.path.join(tmpdir,
'src') # location to build repositories in
os.makedirs(repo_sourcespace)
repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet',
'src') # wet (catkin) repositories
os.makedirs(repo_sourcespace_wet)
repo_sourcespace_dry = os.path.join(repo_sourcespace,
'dry') # dry (rosbuild) repositories
os.makedirs(repo_sourcespace_dry)
repo_build_logs = os.path.join(tmpdir,
'build_logs') # location for build logs
os.makedirs(repo_build_logs)
# init catkin workspace
ros_env_repo = common.get_ros_env(
'/opt/ros/%s/setup.bash' %
ros_distro) # source ros_distro (needed to do a catkin_init_workspace)
# for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
if os.path.isdir("/u/robot/git/care-o-bot/devel"):
ros_env_repo = common.get_ros_env(
'/u/robot/git/care-o-bot/devel/setup.bash')
common.call("catkin_init_workspace %s" % repo_sourcespace_wet,
ros_env_repo,
verbose=False) # init wet workspace
common.call("catkin_make --directory %s/wet" % repo_sourcespace,
ros_env_repo,
verbose=False) # build wet workspace to generate a setup.bash
################
### checkout ###
################
time_checkout = datetime.datetime.now()
print "=====> entering checkout step at", time_checkout
# download build_repo from source #
print "Creating rosinstall file for repository %s" % build_repo
rosinstall = ""
if build_identifier in pipe_repos: # check if triggering identifier is really present in pipeline config
rosinstall += pipe_repos[build_identifier].get_rosinstall()
else:
err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
raise common.BuildException(err_msg)
# write rosinstall file
print "Rosinstall file for repository: \n %s" % rosinstall
with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
f.write(rosinstall)
print "Install repository from source:"
# rosinstall repos
common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s" %
(repo_checkoutspace, workspace, ros_distro))
# get the repositories build dependencies
print "Get build dependencies of repo"
# get all packages in checkoutspace
(catkin_packages, stacks,
manifest_packages) = common.get_all_packages(repo_checkoutspace)
# check if build_repo is wet or dry and get all corresponding deps
build_repo_type = ''
if build_repo in catkin_packages: # wet repo with metapackage
print "repo %s is wet" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(
catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif (build_repo
not in catkin_packages) and (build_repo not in stacks) and (
catkin_packages != []): # wet repo without metapackage
print "repo %s is wet without metapackage" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(
catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif build_repo in stacks: # dry repo with stack
print "repo %s is dry" % build_repo
build_repo_type = 'dry'
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks,
{})
#TODO elif : # dry repo without stack
else: # build_repo is neither wet nor dry
raise common.BuildException(
"Repository %s to build not found in checkoutspace" % build_repo)
# install user-defined/customized dependencies of build_repo from source
rosinstall = ''
fulfilled_deps = []
for dep in repo_build_dependencies:
if dep in pipe_repos[build_identifier].dependencies:
print "Install user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[
dep].get_rosinstall()
fulfilled_deps.append(dep)
# install additional, indirect user-defined dependencies
for dep in pipe_repos[build_identifier].dependencies:
if dep not in fulfilled_deps:
print "Install additional user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[
dep].get_rosinstall()
fulfilled_deps.append(dep)
# check if all user-defined/customized dependencies are satisfied
if sorted(fulfilled_deps) != sorted(
pipe_repos[build_identifier].dependencies):
print "Not all user-defined build dependencies are fulfilled"
print "User-defined build dependencies:\n - %s" % '\n - '.join(
pipe_repos[build_identifier].dependencies)
print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
raise common.BuildException(
"Not all user-defined build dependencies are fulfilled")
if rosinstall != '':
# write .rosinstall file
print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install user-defined build dependencies from source"
# rosinstall depends
common.call(
"rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s" %
(repo_checkoutspace, workspace, ros_distro))
# get also deps of just installed user-defined/customized dependencies
(catkin_packages, stacks,
manifest_packages) = common.get_all_packages(repo_checkoutspace)
if build_repo_type == 'wet':
if stacks != {}:
raise common.BuildException(
"Catkin (wet) package %s depends on (dry) stack(s):\n%s" %
(build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_build_dependencies = common.get_nonlocal_dependencies(
catkin_packages, {}, {},
build_depends=True,
test_depends=False)
else: # dry build repo
# take all packages
repo_build_dependencies = common.get_nonlocal_dependencies(
catkin_packages,
stacks, {},
build_depends=True,
test_depends=False)
repo_build_dependencies = [
dep for dep in repo_build_dependencies if dep not in fulfilled_deps
]
print ""
print "Found the following packages"
print " wet packages: ", catkin_packages.keys()
print " dry stacks: ", stacks.keys()
print " dry packages: ", manifest_packages.keys()
print ""
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
# get all folders in repo_checkoutspace
checkoutspace_dirs = [
name for name in os.listdir(repo_checkoutspace)
if os.path.isdir(os.path.join(repo_checkoutspace, name))
]
for dir in checkoutspace_dirs:
if dir in catkin_packages.keys(): # wet repo with metapackage
shutil.move(os.path.join(repo_checkoutspace, dir),
os.path.join(repo_sourcespace_wet, dir))
elif build_repo_type == 'wet' and dir == build_repo: # wet repo without metapackage
shutil.move(os.path.join(repo_checkoutspace, dir),
os.path.join(repo_sourcespace_wet, dir))
elif dir in stacks.keys(): # dry repo with stack
shutil.move(os.path.join(repo_checkoutspace, dir),
os.path.join(repo_sourcespace_dry, dir))
#TODO elif: # dry repo without stack
#else:
# raise common.BuildException("Could not separate %s into wet or dry sourcespace." %dir)
# remove checkout dir
common.call("rm -rf %s" % repo_checkoutspace)
# setup ros workspace
print "Set up ros workspace and setup environment variables"
ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
common.call("rosws init %s /opt/ros/%s" % (repo_sourcespace, ros_distro),
verbose=False) # init workspace pointing to ros_distro
# for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
if os.path.isdir("/u/robot/git/care-o-bot"):
common.call("rosws merge -t %s /u/robot/git/care-o-bot" %
repo_sourcespace,
verbose=False) # merge robot account workspace
common.call("rosws merge -t %s %s/wet/devel" %
(repo_sourcespace, repo_sourcespace),
verbose=False) # merge wet workspace
common.call("rosws merge -t %s %s/dry" %
(repo_sourcespace, repo_sourcespace),
verbose=False) # merge dry workspace
ros_env_repo = common.get_ros_env(
repo_sourcespace + '/setup.bash') # source wet and dry workspace
############################
### install dependencies ###
############################
time_install = datetime.datetime.now()
print "=====> entering dependency installation step at", time_install
# Create rosdep object
rosdep_resolver = None
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except: # when init fails the first time
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
print "Install build dependencies: %s" % (
', '.join(repo_build_dependencies))
missing_packages = common.apt_get_check_also_nonrosdep(
repo_build_dependencies, ros_distro, rosdep_resolver)
if len(missing_packages) > 0:
raise common.BuildException(
"Some dependencies are missing. Please ask your administrator to install the following packages: %s"
% missing_packages)
print "All denendencies already installed."
#############
### build ###
#############
time_build = datetime.datetime.now()
print "=====> entering build step at", time_build
# get amount of cores available on host system
cores = multiprocessing.cpu_count()
### catkin repositories
if catkin_packages != {}:
print "Build wet packages: ", catkin_packages.keys()
try:
common.call("catkin_make --directory %s/wet" % repo_sourcespace,
ros_env_repo)
except common.BuildException as ex:
print ex.msg
raise common.BuildException(
"Failed to catkin_make wet repositories")
### rosbuild repositories
if build_repo_type == 'dry':
# build dry repositories
print "Build dry stacks: ", stacks.keys()
print "Build dry packages: ", manifest_packages.keys()
packages_to_build = " ".join(stacks.keys()) + " ".join(
manifest_packages.keys())
try:
common.call(
"rosmake -rV --skip-blacklist --profile --pjobs=%s --output=%s %s"
% (cores + 1, repo_build_logs, packages_to_build),
ros_env_repo)
except common.BuildException as ex:
try:
shutil.move(repo_build_logs,
os.path.join(workspace, "build_logs"))
finally:
print ex.msg
raise common.BuildException(
"Failed to rosmake dry repositories")
###########
### end ###
###########
# for hardware builds: create sym link to new build
common.call("rm -f %s/../current" % workspace)
common.call("ln -sf %s %s/../current" % (workspace, workspace))
# steps: parsing, checkout, install, analysis, build, finish
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_checkout - time_parsing)
print "checkout in ", (time_install - time_checkout)
print "install dependencies in ", (time_build - time_install)
print "build in ", (time_finish - time_build)
print "total ", (time_finish - time_parsing)
print ""
print "For manual testing, please run the following line in your testing terminal"
print "source " + repo_sourcespace + "/setup.bash"
print ""
def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository') # location to store repositories in
repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src') # wet (catkin) repositories
os.makedirs(repo_sourcespace_wet)
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry') # dry (rosbuild) repositories
os.makedirs(repo_sourcespace_dry)
repo_static_analysis_results = os.path.join(tmpdir, 'src_repository', 'static_analysis_results') # location for static code test results
os.makedirs(repo_static_analysis_results)
dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs') # location for build logs
os.makedirs(dry_build_logs)
# setup ros workspace and setup environment variables
print "Set up ros workspace and setup environment variables"
ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env_repo)
os.chdir(repo_sourcespace_wet + "/..")
common.call("catkin_make", envir = ros_env_repo, verbose = False) # this means running "make" on an empty catkin workspace to generate a setup.bash
ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace_wet, '../devel/setup.bash')) # source catkin workspace
ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace_dry, os.environ['ROS_PACKAGE_PATH']]) # add dry path to ROS_PACKAGE_PATH
################
### checkout ###
################
time_checkout = datetime.datetime.now()
print "=====> entering checkout step at", time_checkout
# download build_repo from source #
print "Creating rosinstall file for repository %s" % build_repo
rosinstall = ""
if build_identifier in pipe_repos: # check if triggering identifier is really present in pipeline config
rosinstall += pipe_repos[build_identifier].get_rosinstall()
else:
err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
raise common.BuildException(err_msg)
# write rosinstall file
print "Rosinstall file for repository: \n %s" % rosinstall
with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
f.write(rosinstall)
print "Install repository from source:"
# rosinstall repos
common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get the repositories build dependencies
print "Get build dependencies of repo"
# get all packages in sourcespace
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
# (debug) output
if options.verbose:
print "Packages in %s:" % repo_sourcespace
print "Catkin: ", catkin_packages
print "Rosbuild:\n Stacks: ", stacks
print " Packages: ", manifest_packages
# get deps directly for catkin (like in willow code)
try:
print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
except:
pass
# deps catkin
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# deps stacks
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# check if build_repo is wet or dry and get all corresponding deps
build_repo_type = ''
if build_repo in catkin_packages: # wet repo with metapackage
print "repo %s is wet" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif (build_repo not in catkin_packages) and (build_repo not in stacks) and (catkin_packages != []): # wet repo without metapackage
print "repo %s is wet without metapackage" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif build_repo in stacks: # dry repo with stack
print "repo %s is dry" % build_repo
build_repo_type = 'dry'
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
#TODO elif : # dry repo without stack
else: # build_repo is neither wet nor dry
raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)
# install user-defined/customized dependencies of build_repo from source
rosinstall = ''
fulfilled_deps = []
for dep in repo_build_dependencies:
if dep in pipe_repos[build_identifier].dependencies:
print "Install user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# install additional, indirect user-defined dependencies
for dep in pipe_repos[build_identifier].dependencies:
if dep not in fulfilled_deps:
print "Install additional user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# check if all user-defined/customized dependencies are satisfied
if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
print "Not all user-defined build dependencies are fulfilled"
print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
raise common.BuildException("Not all user-defined build dependencies are fulfilled")
if rosinstall != '':
# write .rosinstall file
print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install user-defined build dependencies from source"
# rosinstall depends
common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get also deps of just installed user-defined/customized dependencies
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if build_repo_type == 'wet':
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
else: # dry build repo
# take all packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
# get all folders in repo_sourcespace
sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys(): # wet repo with metapackage
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
elif build_repo_type == 'wet' and dir == build_repo: # wet repo without metapackage
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
elif dir in stacks.keys(): # dry repo with stack
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))
#TODO elif: # dry repo without stack
#else:
# raise common.BuildException("Could not separate %s into wet or dry sourcespace." %dir)
############################
### install dependencies ###
############################
time_install = datetime.datetime.now()
print "=====> entering dependency installation step at", time_install
# Create rosdep object
rosdep_resolver = None
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except: # when init fails the first time
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
#print "Install wet rosdep dependencies" # TODO: this throws an error if the dependency is not a system dependency or a released package. How can we handle wet dependencies which are in the same local workspace?
#for dir in catkin_packages.keys():
# common.call("rosdep install -y %s" % dir, ros_env_repo)
print "Install dry rosdep dependencies"
for dir in stacks.keys():
common.call("rosdep install -y %s" % dir, ros_env_repo)
#######################
### static analysis ###
#######################
time_analysis = datetime.datetime.now()
print "=====> entering static analysis step at", time_analysis
#TODO
# Cpplint
# Counting lines of code/comments
# cppcheck
cppcheck.run([repo_sourcespace_wet, repo_sourcespace_dry], repo_static_analysis_results)
# Coverage
# create tests results directory in workspace
os.mkdir(workspace + "/static_analysis_results")
# copy test results
common.copy_static_analysis_results(repo_static_analysis_results, workspace + "/static_analysis_results")
#############
### build ###
#############
time_build = datetime.datetime.now()
print "=====> entering build step at", time_build
### catkin repositories
if catkin_packages != {}:
os.chdir(repo_sourcespace_wet + "/..")
try:
common.call("catkin_make", ros_env_repo)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to catkin_make wet repositories")
### rosbuild repositories
if build_repo_type == 'dry':
# build dry repositories
print "Build repository %s" % build_repo
try:
common.call("rosmake -rV --skip-blacklist --profile --pjobs=8 --output=%s %s" %
(dry_build_logs, build_repo), ros_env_repo)
except common.BuildException as ex:
try:
shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
finally:
print ex.msg
raise common.BuildException("Failed to rosmake %s" % build_repo)
# the end (steps: parsing, checkout, install, analysis, build, finish)
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_checkout - time_parsing)
print "checkout in ", (time_install - time_checkout)
print "install dependencies in ", (time_analysis - time_install)
print "static code analysis in ", (time_build - time_analysis)
print "build in ", (time_finish - time_build)
print "total ", (time_finish - time_parsing)
print ""
def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[
0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_file(pipeline_repos_owner,
server_name,
user_name,
file_location=os.environ["WORKSPACE"])
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = '/tmp'
repo_sourcespace = os.path.join(tmpdir,
'src') # location to build repositories in
repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet',
'src') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(repo_sourcespace,
'dry') # dry (rosbuild) repositories
repo_test_results = os.path.join(
tmpdir, 'test_results') # location for test results
if not os.path.exists(repo_test_results):
os.makedirs(repo_test_results)
repo_build_logs = os.path.join(tmpdir,
'build_logs') # location for build logs
# source wet and dry workspace
ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')
ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results
############
### test ###
############
time_test = datetime.datetime.now()
print "=====> entering testing step at", time_test
# get amount of cores available on host system
cores = multiprocessing.cpu_count()
# get current repository name
user, repo_name = cp_instance.split_github_url(
pipe_repos[build_identifier].data['url'])
### catkin repositories
(catkin_packages, stacks,
manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
if (len(catkin_packages) > 0) and (repo_name in catkin_packages):
os.environ['ROS_TEST_RESULTS_DIR'] = os.environ[
'CATKIN_TEST_RESULTS_DIR']
# get list of dependencies to test
test_repos_list_wet = []
# add all packages in main repository
(catkin_test_packages, stacks, manifest_packages
) = common.get_all_packages(catkin_packages[repo_name] + '/..')
for pkg_name, pkg_dir in catkin_test_packages.items():
test_repos_list_wet.append(pkg_name)
# add all packages in dep repositories (if "test" is set to True)
for dep in pipe_repos[build_identifier].dependencies.keys():
if pipe_repos[build_identifier].dependencies[dep].test:
(catkin_test_dep_packages, stacks, manifest_packages
) = common.get_all_packages(catkin_packages[dep] + '/..')
for pkg_name, pkg_dir in catkin_test_dep_packages.items():
test_repos_list_wet.append(pkg_name)
# test wet repositories
print "Testing the following wet repositories %s" % test_repos_list_wet
test_list = ' '.join(test_repos_list_wet)
try:
common.call(
"/opt/VirtualGL/bin/vglrun catkin_make --directory %s/wet test --pkg %s"
% (repo_sourcespace, test_list), ros_env_repo)
finally:
# clean and copy test xml files
common.clean_and_copy_test_results(
repo_sourcespace + "/wet/build/test_results",
workspace + "/test_results"
) # FIXME: is there a way to let catkin write test results to repo_test_results
### rosbuild repositories
(catkin_packages, stacks,
manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
if (len(stacks) > 0) and (repo_name in stacks):
# get list of dependencies to test
test_repos_list_dry = [build_repo]
for dep, depObj in pipe_repos[build_identifier].dependencies.items():
if depObj.test and dep in stacks:
test_repos_list_dry.append(dep)
# test dry repositories
packages_to_test_list = test_repos_list_dry
for repo in test_repos_list_dry:
(catkin_packages, stacks, manifest_packages
) = common.get_all_packages(repo_sourcespace_dry + "/" + repo)
packages_to_test_list = packages_to_test_list + manifest_packages.keys(
)
print "Test dry packages: ", packages_to_test_list
packages_to_test = " ".join(test_repos_list_dry) + " " + " ".join(
packages_to_test_list)
common.call(
"/opt/VirtualGL/bin/vglrun rosmake -rV --skip-blacklist --profile --pjobs=%s --test-only --output=%s %s"
% (cores, repo_build_logs, packages_to_test), ros_env_repo)
# clean and copy test xml files
common.clean_and_copy_test_results(repo_test_results,
workspace + "/test_results")
# in case we have no tests executed (neither wet nor dry), we'll generate some dummy test result
common.clean_and_copy_test_results(repo_test_results,
workspace + "/test_results")
###########
### end ###
###########
# steps: parsing, test
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_test - time_parsing)
print "test in ", (time_finish - time_test)
print "total ", (time_finish - time_parsing)
print ""
def build_downstream_post_fuerte(ros_distro, build_repo, workspace, server):
ros_package_path = os.environ['ROS_PACKAGE_PATH']
b_r_short = build_repo.split('__')[0]
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository')
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')
repo_buildspace = os.path.join(tmpdir, 'build_repository')
dependson_sourcespace = os.path.join(tmpdir, 'src_depends_on')
dependson_sourcespace_wet = os.path.join(tmpdir, 'src_depends_on', 'wet')
dependson_sourcespace_dry = os.path.join(tmpdir, 'src_depends_on', 'dry')
dependson_buildspace = os.path.join(tmpdir, 'build_depend_on')
dry_test_results_dir = os.path.join(dependson_sourcespace_dry,
'test_results')
# TODO clean up test result dirs
# get depends on
print "Get list of released repositories that (build-)depend on repository %s" % b_r_short
ros_depends_on = []
distro_ros = rosdistro.RosDistro(ros_distro, 'http://%s/~jenkins' % server)
for d in distro_ros.get_depends_on([
b_r_short
])['build'] + distro_ros.get_depends_on([b_r_short])['buildtool']:
if not d in ros_depends_on and d != b_r_short:
ros_depends_on.append(d)
# TODO get_depends_on of cob_distro release (only for intern repos necessary)
cob_depends_on = []
#distro_cob = cob_distro.CobDistro(ros_distro)
#for d in distro_cob.get_depends_on([b_r_short])['build'] + distro_cob.get_depends_on([b_r_short])['buildtool']:
# if not d in cob_depends_on and d != b_r_short:
# cob_depends_on.append(d)
if len(ros_depends_on + cob_depends_on) == 0:
print "No repository depends on repository %s. Test finished" % b_r_short
return
print "Build depends_on list of repository %s:\n - %s" % (
'\n - '.join(ros_depends_on + cob_depends_on))
# install depends_on repository from source
rosinstall = distro_ros.get_rosinstall(ros_depends_on)
#rosinstall += distro_cob.get_rosinstall(cob_depends_on) TODO
print "Rosinstall for depends_on:\n %s" % rosinstall
with open(workspace + "/depends_on.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for depends on"
# install all repository and system dependencies of the depends_on list
print "Install all depends_on from source: %s" % (
', '.join(ros_depends_on))
os.makedirs(dependson_sourcespace)
common.call("rosinstall %s %s/depends_on.rosinstall /opt/ros/%s" %
(dependson_sourcespace, workspace, ros_distro))
# all packages in dependson_sourcespace
(catkin_packages, stacks,
manifest_packages) = common.get_all_packages(dependson_sourcespace)
print catkin_packages
print stacks
print manifest_packages
# get build and test dependencies of depends_on list
dependson_build_dependencies = []
for d in common.get_nonlocal_dependencies(catkin_packages,
stacks, {},
build_depends=True,
test_depends=False):
print " Checking dependency %s" % d
if d in dependson_build_dependencies:
print " Already in dependson_build_dependencies"
elif d in ros_depends_on or d in cob_depends_on:
print " Is a direct dependency of the repository, and is installed from source"
elif d == b_r_short:
print " Is the tested repository"
else:
dependson_build_dependencies.append(d)
print "Build dependencies of depends_on list are %s" % (
', '.join(dependson_build_dependencies))
dependson_test_dependencies = []
for d in common.get_nonlocal_dependencies(catkin_packages,
stacks, {},
build_depends=False,
test_depends=True):
if d not in dependson_test_dependencies + ros_depends_on + cob_depends_on and d != b_r_short:
dependson_test_dependencies.append(d)
print "Test dependencies of depends_on list are %s" % (
', '.join(dependson_test_dependencies))
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
os.makedirs(dependson_sourcespace_wet)
os.makedirs(dependson_sourcespace_dry)
# get all folders in dependson_sourcespace
sourcespace_dirs = [
name for name in os.listdir(dependson_sourcespace)
if os.path.isdir(os.path.join(dependson_sourcespace, name))
]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys():
shutil.move(os.path.join(dependson_sourcespace, dir),
os.path.join(dependson_sourcespace_wet, dir))
if dir in stacks.keys():
shutil.move(os.path.join(dependson_sourcespace, dir),
os.path.join(dependson_sourcespace_dry, dir))
# Create rosdep object
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except:
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
# install build dependencies
print "Install all build dependencies of the depends_on list: %s" % (
', '.join(dependson_build_dependencies))
common.apt_get_install_also_nonrosdep(dependson_build_dependencies,
rosdep_resolver)
# env
print "Setting up environment"
ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
common.call("env", ros_env)
ros_env_dependson = common.get_ros_env(
os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join(
[dependson_buildspace, repo_buildspace, ros_package_path])
common.call("env", ros_env_dependson)
### catkin repositories
if catkin_packages != {}:
os.makedirs(dependson_buildspace)
os.chdir(dependson_buildspace)
print "Create a new CMakeLists.txt for catkin packages"
if ros_distro == 'fuerte':
common.call(
"ln -s %s %s" %
(os.path.join(dependson_sourcespace_wet, 'catkin', 'cmake',
'toplevel.cmake'),
os.path.join(dependson_sourcespace_wet, 'CMakeLists.txt')))
else:
common.call("catkin_init_workspace %s" % dependson_sourcespace_wet,
ros_env_dependson)
try:
common.call("cmake %s" % dependson_sourcespace_wet,
ros_env_dependson)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to cmake wet repositories")
#ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories
print "Build wet depends_on list"
try:
common.call("make", ros_env_dependson)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to make wet packages")
if dependson_test_dependencies != []:
# install test dependencies
print "Install all test dependencies of the depends_on list: %s" % (
', '.join(dependson_test_dependencies))
common.apt_get_install_also_nonrosdep(dependson_test_dependencies,
rosdep_resolver)
# test repositories
try:
common.call("make run_tests", ros_env_dependson)
except common.BuildException as ex:
print ex.msg
# copy test results
common.copy_test_results(workspace, dependson_buildspace)
### rosbuild repositories
if stacks != {}:
# env
print "Setting up environment"
ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([
dependson_buildspace, repo_buildspace, dependson_sourcespace_dry,
repo_sourcespace_dry, ros_package_path
])
common.call("env", ros_env_dependson)
#print "Make rosdep"
#common.call("rosmake rosdep", ros_env)
#for stack in stacks.keys():
# common.call("rosdep install -y %s" % stack, ros_env_repo)
dependson_sourcespace_dry_dirs = [
name for name in os.listdir(dependson_sourcespace_dry)
if os.path.isdir(os.path.join(dependson_sourcespace_dry, name))
]
print "Build dry depends_on repositories:\n - %s" % '\n - '.join(
dependson_sourcespace_dry_dirs)
os.mkdir(dry_test_results_dir)
for dry_dependson in dependson_sourcespace_dry_dirs:
try:
common.call(
"rosmake -rV --profile --pjobs=8 --output=%s %s" %
(dry_test_results_dir, b_r_short), ros_env_dependson)
except:
raise common.BuildException("Failed to rosmake %s" % b_r_short)
try:
common.call(
"rosmake -rV --profile --pjobs=8 --test-only --output=%s %s"
% (dry_test_results_dir, b_r_short), ros_env_dependson)
# TODO output dir ??
except:
print "Failed to test %s" % dry_dependson
# copy test results
common.call("rosrun rosunit clean_junit_xml.py", ros_env_dependson)
common.copy_test_results(workspace, dependson_sourcespace_dry)
def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
ros_package_path = os.environ['ROS_PACKAGE_PATH']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print datetime.datetime.now()
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
print "Graphic Test: True"
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository') # location to store repositories in
repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry') # dry (rosbuild) repositories
repo_test_results_dry = os.path.join(tmpdir, 'src_repository', 'test_results') # location for dry test results
repo_test_results_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'build', 'test_results') # location for wet test results
#repo_buildspace = os.path.join(tmpdir, 'build_repository') # location for build output
dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs') # location for build logs
# get environment variables
ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
if options.verbose:
common.call("env", ros_env_repo)
############
### test ###
############
time_test = datetime.datetime.now()
print "=====> entering testing step at", time_test
### catkin repositories
print "test catkin repositories"
ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results_wet
if os.listdir(repo_sourcespace_wet):
os.chdir(repo_sourcespace_wet + "/..")
# get wet repositories test and run dependencies
#print "Get test and run dependencies of repo list"
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
# get list of dependencies to test
test_repos_list_wet = []
for dep, depObj in pipe_repos[build_identifier].dependencies.items():
if depObj.test and dep in catkin_packages:
test_repos_list_wet.append(dep)
print "Test the following wet repositories %s" % test_repos_list_wet
#test_error_msg = None
try:
test_list = ' '.join( test_repos_list_wet )
if test_list:
common.call( "/opt/VirtualGL/bin/vglrun catkin_make test --pkg %s" % test_list, ros_env_repo)
except common.BuildException as ex:
print ex.msg
#print traceback.format_exc()
# test_error_msg = ex.msg
# clean test xml files
common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
for file in os.listdir(os.path.join(repo_test_results_wet)):
file_path = os.path.join(repo_test_results_wet, file)
try:
if not file.startswith("_hudson"):
shutil.rmtree(file_path)
except Exception as e:
print e
# copy wet test results
common.copy_test_results(repo_test_results_wet, workspace + "/test_results")
### rosbuild repositories
print "test rosbuild repositories"
if not os.path.isdir(repo_test_results_dry):
os.mkdir(repo_test_results_dry)
ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results_dry
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
if build_repo in stacks:
# get list of dependencies to test
test_repos_list_dry = []
for dep, depObj in pipe_repos[build_identifier].dependencies.items():
if depObj.test and dep in stacks:
test_repos_list_dry.append(dep)
# test dry repositories
print "Test the following dry repositories %s" % test_repos_list_dry
try:
build_list = " ".join(test_repos_list_dry + [build_repo])
common.call("/opt/VirtualGL/bin/vglrun rosmake -rV --profile --test-only --output=%s %s" %
( dry_build_logs, build_list ), ros_env_repo)
except common.BuildException as ex:
print ex.msg
# clean test xml files
common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
for file in os.listdir(os.path.join(repo_test_results_dry)):
file_path = os.path.join(repo_test_results_dry, file)
try:
if not file.startswith("_hudson"):
shutil.rmtree(file_path)
except Exception as e:
print e
# copy dry test results
common.copy_test_results(repo_test_results_dry, workspace + "/test_results")
# the end (steps: parsing, test)
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_test - time_parsing)
print "test in ", (time_finish - time_test)
print "total ", (time_finish - time_parsing)
print ""
def main():
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 6:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo graphic_test" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
graphic_test = True if args[5] == "true" else False # TODO optional
build_repo_only = True if args[6] == "true" else False
# environment variables
workspace = os.environ['WORKSPACE']
ros_package_path = os.environ['ROS_PACKAGE_PATH']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print datetime.datetime.now()
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
print "Graphic Test: %s" % graphic_test
print "Build Repo Only: %s" % build_repo_only
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository') # location to store repositories in
repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry') # dry (rosbuild) repositories
repo_buildspace = os.path.join(tmpdir, 'build_repository') # location for build output
dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs') # location for build logs
if ros_distro != 'electric':
# Create rosdep object
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except: # when init fails the first time
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
# env
print "Set up ros environment variables"
ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
if options.verbose:
common.call("env", ros_env)
### catkin repositories
print datetime.datetime.now()
if os.listdir(repo_sourcespace_wet):
# set up catkin workspace
#if ros_distro == 'fuerte':
#if 'catkin' not in catkin_packages.keys():
## add catkin package to rosinstall
#rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
#print "Install catkin"
## rosinstall catkin
#common.call("rosinstall --verbose %s %s/repo.rosinstall /opt/ros/%s"
#% (repo_sourcespace_wet, workspace, ros_distro))
#print "Create a CMakeLists.txt for catkin packages"
#common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
#os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
#else:
#common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env)
if not os.path.isdir(repo_buildspace):
os.mkdir(repo_buildspace)
os.chdir(repo_buildspace)
# test repositories
print "Test wet repository list"
test_error_msg = None
try:
common.call("make tests", ros_env)
except common.BuildException as ex:
print ex.msg
test_error_msg = ex.msg
# get wet repositories test and run dependencies
print "Get test and run dependencies of repo list"
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_test_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=False, test_depends=True)
if repo_test_dependencies != [] and test_error_msg is None:
print "Install test and run dependencies of repository list: %s" % (', '.join(repo_test_dependencies))
common.apt_get_install_also_nonrosdep(repo_test_dependencies, ros_distro, rosdep_resolver)
# run tests
print "Test repository list"
try:
common.call("%smake run_tests" % ("/opt/VirtualGL/bin/vglrun " if graphic_test else ""), ros_env) # TODO check how to test a list of repos
except common.BuildException as ex:
print ex.msg
test_error_msg = ex.msg
# copy test results
common.copy_test_results(workspace, repo_buildspace, test_error_msg)
### rosbuild repositories
print datetime.datetime.now()
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
if build_repo in stacks:
# get list of dependencies to test
test_repos_list = []
for dep in pipe_repos[build_identifier].dependencies:
if dep in stacks: # TODO option to select deps to build
test_repos_list.append(dep)
ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
if options.verbose:
common.call("env", ros_env_repo)
# test dry repositories
print "Test repository %s" % build_repo
try:
build_list = " ".join(test_repos_list + [build_repo])
if build_repo_only:
build_list = build_repo
common.call("%srosmake -rV --profile %s--test-only --output=%s %s" %
("/opt/VirtualGL/bin/vglrun " if graphic_test else "",
"--pjobs=8 " if not graphic_test else "",
dry_build_logs, build_list), ros_env_repo)
except common.BuildException as ex:
print ex.msg
# copy test results
common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
for file in os.listdir(os.path.join(repo_sourcespace, "test_results")):
file_path = os.path.join(repo_sourcespace, "test_results", file)
print file_path
try:
if not file.startswith("_hudson"):
shutil.rmtree(file_path)
except Exception as e:
print e
common.copy_test_results(workspace, repo_sourcespace)
print datetime.datetime.now()
try:
shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
except IOError as ex:
print "No build logs found: %s" % ex
def main():
#########################
### parsing arguments ###
#########################
print "=====> WORKAROUND INSTALLATION ..."
common.call("sudo apt-get install -y python-numpy python-scipy python-matplotlib python-pygraphviz python-tk ros-hydro-controller-manage* ros-hydro-joint-* ros-hydro-desktop-full ros-hydro-gazebo-ros-control ros-hydro-ros-control")
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = '/tmp'
repo_sourcespace = os.path.join(tmpdir, 'src') # location to build repositories in
repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet', 'src') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(repo_sourcespace, 'dry') # dry (rosbuild) repositories
repo_test_results = os.path.join(tmpdir, 'test_results') # location for test results
if not os.path.exists(repo_test_results):
os.makedirs(repo_test_results)
repo_build_logs = os.path.join(tmpdir, 'build_logs') # location for build logs
# source wet and dry workspace
ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')
ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results
############
### test ###
############
time_test = datetime.datetime.now()
print "=====> entering testing step at", time_test
# get amount of cores available on host system
cores = multiprocessing.cpu_count()
# get current repository name
user, repo_name = cp_instance.split_github_url(pipe_repos[build_identifier].data['url'])
### catkin repositories
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
if (len(catkin_packages) > 0) and (repo_name in catkin_packages):
# get list of dependencies to test
test_repos_list_wet = []
# add all packages in main repository
(catkin_test_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[repo_name] + '/..')
for pkg_name, pkg_dir in catkin_test_packages.items():
test_repos_list_wet.append(pkg_name)
# add all packages in dep repositories (if "test" is set to True)
for dep in pipe_repos[build_identifier].dependencies.keys():
if pipe_repos[build_identifier].dependencies[dep].test:
(catkin_test_dep_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[dep] + '/..')
for pkg_name, pkg_dir in catkin_test_dep_packages.items():
test_repos_list_wet.append(pkg_name)
print "Testing the following wet repositorie %s" % build_repo
try:
test_list = ' '.join( test_repos_list_wet )
print "test_list: %s" % test_list
common.call("catkin_make --directory %s/wet run_tests_%s" % (repo_sourcespace, build_repo))
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to catkin_make test wet repositories")
# clean and copy test xml files
common.clean_and_copy_test_results(repo_sourcespace + "/wet/build/test_results", workspace + "/test_results") # FIXME: is there a way to let catkin write test results to repo_test_results
### rosbuild repositories
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
if (len(stacks) > 0) and (repo_name in stacks):
# get list of dependencies to test
test_repos_list_dry = [build_repo]
for dep, depObj in pipe_repos[build_identifier].dependencies.items():
if depObj.test and dep in stacks:
test_repos_list_dry.append(dep)
# test dry repositories
print "Test the following dry repositories %s" % test_repos_list_dry
try:
build_list = " ".join(test_repos_list_dry)
common.call("rosmake -rV --skip-blacklist --profile --pjobs=%s --test-only --output=%s %s" %
( cores, repo_build_logs, build_list ), ros_env_repo)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to rosmake test dry repositories")
# clean and copy test xml files
common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")
# in case we have no tests executed (neither wet nor dry), we'll generate some dummy test result
common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")
################
### Analysis ###
################
time_ana = datetime.datetime.now()
print "=====> entering analysis step at", time_ana
path_video = "$WORKSPACE/videoFiles"
com_folder = "mkdir -p " + path_video
common.call(com_folder)
print "Files in $BAG_PATH:"
print common.call(os.path.expandvars("ls -al $BAG_PATH"))
analyse()
print "=====> making images"
com_pix = "rosrun navigation_test_analysis simple_viewer.py AUTO"
common.call(com_pix)
###########
### end ###
###########
# steps: parsing, test, analysis
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_test - time_parsing)
print "test in ", (time_ana - time_test)
print "analysis in ", (time_finish - time_ana)
print "total ", (time_finish - time_parsing)
print ""
# Trying to fix: Deleting project workspace... Cannot delete workspace: java.io.IOException: Unable to delete
common.call("sudo rm -rf $WORKSPACE/test_results/*")
def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = '/tmp'
repo_sourcespace = os.path.join(tmpdir, 'src') # location to build repositories in
repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet', 'src') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(repo_sourcespace, 'dry') # dry (rosbuild) repositories
repo_test_results = os.path.join(tmpdir, 'test_results') # location for test results
if not os.path.exists(repo_test_results):
os.makedirs(repo_test_results)
repo_build_logs = os.path.join(tmpdir, 'build_logs') # location for build logs
# source wet and dry workspace
ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')
ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results
############
### test ###
############
time_test = datetime.datetime.now()
print "=====> entering testing step at", time_test
# get amount of cores available on host system
cores = multiprocessing.cpu_count()
# get current repository name
user, repo_name = cp_instance.split_github_url(pipe_repos[build_identifier].data['url'])
### catkin repositories
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
if (len(catkin_packages) > 0) and (repo_name in catkin_packages):
# get list of dependencies to test
test_repos_list_wet = []
# add all packages in main repository
(catkin_test_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[repo_name] + '/..')
for pkg_name, pkg_dir in catkin_test_packages.items():
test_repos_list_wet.append(pkg_name)
# add all packages in dep repositories (if "test" is set to True)
for dep in pipe_repos[build_identifier].dependencies.keys():
if pipe_repos[build_identifier].dependencies[dep].test:
(catkin_test_dep_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[dep] + '/..')
for pkg_name, pkg_dir in catkin_test_dep_packages.items():
test_repos_list_wet.append(pkg_name)
# test wet repositories
print "Testing the following wet repositories %s" % test_repos_list_wet
test_list = ' '.join( test_repos_list_wet )
common.call( "catkin_make --directory %s/wet test --pkg %s" % (repo_sourcespace, test_list), ros_env_repo)
# clean and copy test xml files
common.clean_and_copy_test_results(repo_sourcespace + "/wet/build/test_results", workspace + "/test_results") # FIXME: is there a way to let catkin write test results to repo_test_results
### rosbuild repositories
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
if (len(stacks) > 0) and (repo_name in stacks):
# get list of dependencies to test
test_repos_list_dry = [build_repo]
test_repos_list_dry.append(' ') # Solves the bug: [ rosmake ] WARNING: Skipped command line arguments: ['cob_navigation_testsnavigation_test_skeleton'] because they could not be resolved to a stack name or a package name.
for dep, depObj in pipe_repos[build_identifier].dependencies.items():
if depObj.test and dep in stacks:
test_repos_list_dry.append(dep)
# test dry repositories
packages_to_test_list = test_repos_list_dry
for repo in test_repos_list_dry:
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry + "/" + repo)
packages_to_test_list = packages_to_test_list + manifest_packages.keys()
print "Test dry packages: ", packages_to_test_list
packages_to_test = " ".join(test_repos_list_dry) + " " + " ".join(packages_to_test_list)
common.call("rosmake -rV --skip-blacklist --profile --pjobs=%s --test-only --output=%s %s" %
( cores, repo_build_logs, packages_to_test ), ros_env_repo)
# clean and copy test xml files
common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")
# in case we have no tests executed (neither wet nor dry), we'll generate some dummy test result
common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")
###########
### end ###
###########
# steps: parsing, test
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_test - time_parsing)
print "test in ", (time_finish - time_test)
print "total ", (time_finish - time_parsing)
print ""
def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
ros_package_path = os.environ['ROS_PACKAGE_PATH']
# cob_pipe object
print "LIST:" + str(cob_pipe.CobPipe()) +"REPOOWN:" + str(pipeline_repos_owner)+ "SERVER:" + str(server_name) + "USER:"******"Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
#remove old and create new folder
#while len(os.listdir(workspace)) >= 7:
# list = os.listdir(workspace)
# shutil.rmtree(workspace + "/" + sorted(list)[0]) #with common.call rm -rf later
#workspace = workspace + '/' + str(datetime.datetime.now())
#print str(workspace)
# set up directories variables
tmpdir = os.path.join(workspace, 'test_repositories') #TODO check for old versions (delete oldest) and create new directory with timestamp
common.call("rm -rf " + tmpdir)
repo_sourcespace = os.path.join(tmpdir, 'src_repository') # location to store repositories in
repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry') # dry (rosbuild) repositories
repo_static_analysis_results = os.path.join(tmpdir, 'src_repository', 'static_analysis_results') # location for static code test results
repo_buildspace = os.path.join(tmpdir, 'build_repository') # location for build output
dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs') # location for build logs
################
### checkout ###
################
time_checkout = datetime.datetime.now()
print "=====> entering checkout step at", time_checkout
# download build_repo from source #
print "Creating rosinstall file for repository %s" % build_repo
rosinstall = ""
if build_identifier in pipe_repos: # check if triggering identifier is really present in pipeline config
rosinstall += pipe_repos[build_identifier].get_rosinstall()
rosinstall += "- other: {local-name: /u/robot}" #TODO: not hardcoding (with multiple directories?)
else:
err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
raise common.BuildException(err_msg)
# write rosinstall file
print "Rosinstall file for repository: \n %s" % rosinstall
with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
f.write(rosinstall)
print "Install repository from source:"
# create repo sourcespace directory 'src_repository'
os.makedirs(repo_sourcespace)
# rosinstall repos
common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get the repositories build dependencies
print "Get build dependencies of repo"
# get all packages in sourcespace
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if ros_distro == 'electric' and catkin_packages != {}:
raise common.BuildException("Found wet packages while building in ros electric")
# (debug) output
if options.verbose:
print "Packages in %s:" % repo_sourcespace
print "Catkin: ", catkin_packages
print "Rosbuild:\n Stacks: ", stacks
print " Packages: ", manifest_packages
# get deps directly for catkin (like in willow code)
try:
print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
except:
pass
# deps catkin
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# deps stacks
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# check if build_repo is wet or dry and get all corresponding deps
build_repo_type = ''
if build_repo in catkin_packages:
print "repo %s is wet" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif build_repo in stacks:
print "repo %s is dry" % build_repo
build_repo_type = 'dry'
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
else:
# build_repo is neither wet nor dry
raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)
# install user-defined/customized dependencies of build_repo from source
rosinstall = ''
fulfilled_deps = []
for dep in repo_build_dependencies:
if dep in pipe_repos[build_identifier].dependencies:
print "Install user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# install additional, indirect user-defined dependencies
for dep in pipe_repos[build_identifier].dependencies:
if dep not in fulfilled_deps:
print "Install additional user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# check if all user-defined/customized dependencies are satisfied
if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
print "Not all user-defined build dependencies are fulfilled"
print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
raise common.BuildException("Not all user-defined build dependencies are fulfilled")
if rosinstall != '':
# write .rosinstall file
print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
rosinstall += "- other: {local-name: /u/robot}"#TODO: not hardcoding (with multiple directories?)
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install user-defined build dependencies from source"
# rosinstall depends
common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get also deps of just installed user-defined/customized dependencies
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if build_repo_type == 'wet':
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
else: # dry build repo
# take all packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
os.makedirs(repo_sourcespace_wet)
os.makedirs(repo_sourcespace_dry)
# get all folders in repo_sourcespace
sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys():
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
if dir in stacks.keys():
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))
############################
### install dependencies ###
############################
time_install = datetime.datetime.now()
print "=====> entering dependency installation step at", time_install
# Create rosdep object
rosdep_resolver = None
print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
#common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
print "ASK YOUR ADMIN TO INSTALL:" + str(repo_build_dependencies)
print "ASK YOUR ADMIN TO INSTALL THE PACKAGES ABOVE"
# check which packages are installed
# FAIL WHEN SOMETHING MISSING
#############
### build ###
#############
time_build = datetime.datetime.now()
print "=====> entering build step at", time_build
# env
print "Set up ros environment variables"
ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
if options.verbose:
common.call("env", ros_env_repo)
### catkin repositories
if catkin_packages != {}:
# set up catkin workspace
if ros_distro == 'fuerte':
if 'catkin' not in catkin_packages.keys():
# add catkin package to rosinstall
rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
print "Install catkin"
# rosinstall catkin
common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace_wet, workspace, ros_distro))
print "Create a CMakeLists.txt for catkin packages"
common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
else:
common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env_repo)
#os.mkdir(repo_buildspace)
os.chdir(repo_sourcespace_wet + "/..")
try:
common.call("catkin_make", ros_env_repo)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to catkin_make wet repositories")
# setup ros environment to source wet packages before building dry ones
ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace_wet, '../devel/setup.bash'))
### rosbuild repositories
if build_repo_type == 'dry':
ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
if options.verbose:
common.call("env", ros_env_repo)
if ros_distro == 'electric':
print "Rosdep"
common.call("rosmake rosdep", ros_env_repo)
for stack in stacks.keys():
common.call("rosdep install -y %s" % stack, ros_env_repo)
# build dry repositories
print "Build repository %s" % build_repo
try:
common.call("rosmake -i -rV --skip-blacklist --profile --pjobs=8 --output=%s %s" %
(dry_build_logs, build_repo), ros_env_repo)
except common.BuildException as ex:
try:
shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
finally:
print ex.msg
raise common.BuildException("Failed to rosmake %s" % build_repo)
# the end (steps: parsing, checkout, install, analysis, build, finish)
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_checkout - time_parsing)
print "checkout in ", (time_install - time_checkout)
print "install dependencies in ", (time_build - time_install)
print "build in ", (time_finish - time_build)
print ""
print "For testing, please run the following line in your testing terminal"
print "source " + repo_sourcespace + "/setup.bash"
print ""
def build_downstream_post_fuerte(ros_distro, build_repo, workspace, server):
ros_package_path = os.environ['ROS_PACKAGE_PATH']
b_r_short = build_repo.split('__')[0]
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository')
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')
repo_buildspace = os.path.join(tmpdir, 'build_repository')
dependson_sourcespace = os.path.join(tmpdir, 'src_depends_on')
dependson_sourcespace_wet = os.path.join(tmpdir, 'src_depends_on', 'wet')
dependson_sourcespace_dry = os.path.join(tmpdir, 'src_depends_on', 'dry')
dependson_buildspace = os.path.join(tmpdir, 'build_depend_on')
dry_test_results_dir = os.path.join(dependson_sourcespace_dry, 'test_results')
# TODO clean up test result dirs
# get depends on
print "Get list of released repositories that (build-)depend on repository %s" % b_r_short
ros_depends_on = []
distro_ros = rosdistro.RosDistro(ros_distro, 'http://%s/~jenkins' % server)
for d in distro_ros.get_depends_on([b_r_short])['build'] + distro_ros.get_depends_on([b_r_short])['buildtool']:
if not d in ros_depends_on and d != b_r_short:
ros_depends_on.append(d)
# TODO get_depends_on of cob_distro release (only for intern repos necessary)
cob_depends_on = []
#distro_cob = cob_distro.CobDistro(ros_distro)
#for d in distro_cob.get_depends_on([b_r_short])['build'] + distro_cob.get_depends_on([b_r_short])['buildtool']:
# if not d in cob_depends_on and d != b_r_short:
# cob_depends_on.append(d)
if len(ros_depends_on + cob_depends_on) == 0:
print "No repository depends on repository %s. Test finished" % b_r_short
return
print "Build depends_on list of repository %s:\n - %s" % ('\n - '.join(ros_depends_on + cob_depends_on))
# install depends_on repository from source
rosinstall = distro_ros.get_rosinstall(ros_depends_on)
#rosinstall += distro_cob.get_rosinstall(cob_depends_on) TODO
print "Rosinstall for depends_on:\n %s" % rosinstall
with open(workspace + "/depends_on.rosinstall", 'w') as f:
f.write(rosinstall)
print "Created rosinstall file for depends on"
# install all repository and system dependencies of the depends_on list
print "Install all depends_on from source: %s" % (', '.join(ros_depends_on))
os.makedirs(dependson_sourcespace)
common.call("rosinstall -j 8 %s %s/depends_on.rosinstall /opt/ros/%s" % (dependson_sourcespace, workspace, ros_distro))
# all packages in dependson_sourcespace
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(dependson_sourcespace)
print catkin_packages
print stacks
print manifest_packages
# get build and test dependencies of depends_on list
dependson_build_dependencies = []
for d in common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False):
print " Checking dependency %s" % d
if d in dependson_build_dependencies:
print " Already in dependson_build_dependencies"
elif d in ros_depends_on or d in cob_depends_on:
print " Is a direct dependency of the repository, and is installed from source"
elif d == b_r_short:
print " Is the tested repository"
else:
dependson_build_dependencies.append(d)
print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))
dependson_test_dependencies = []
for d in common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=False, test_depends=True):
if d not in dependson_test_dependencies + ros_depends_on + cob_depends_on and d != b_r_short:
dependson_test_dependencies.append(d)
print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
os.makedirs(dependson_sourcespace_wet)
os.makedirs(dependson_sourcespace_dry)
# get all folders in dependson_sourcespace
sourcespace_dirs = [name for name in os.listdir(dependson_sourcespace) if os.path.isdir(os.path.join(dependson_sourcespace, name))]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys():
shutil.move(os.path.join(dependson_sourcespace, dir), os.path.join(dependson_sourcespace_wet, dir))
if dir in stacks.keys():
shutil.move(os.path.join(dependson_sourcespace, dir), os.path.join(dependson_sourcespace_dry, dir))
# Create rosdep object
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except:
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
# install build dependencies
print "Install all build dependencies of the depends_on list: %s" % (', '.join(dependson_build_dependencies))
common.apt_get_install_also_nonrosdep(dependson_build_dependencies, rosdep_resolver)
# env
print "Setting up environment"
ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
common.call("env", ros_env)
ros_env_dependson = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([dependson_buildspace, repo_buildspace, ros_package_path])
common.call("env", ros_env_dependson)
### catkin repositories
if catkin_packages != {}:
os.makedirs(dependson_buildspace)
os.chdir(dependson_buildspace)
print "Create a new CMakeLists.txt for catkin packages"
if ros_distro == 'fuerte':
common.call("ln -s %s %s" % (os.path.join(dependson_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
os.path.join(dependson_sourcespace_wet, 'CMakeLists.txt')))
else:
common.call("catkin_init_workspace %s" % dependson_sourcespace_wet, ros_env_dependson)
try:
common.call("cmake %s" % dependson_sourcespace_wet, ros_env_dependson)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to cmake wet repositories")
#ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories
print "Build wet depends_on list"
try:
common.call("make", ros_env_dependson)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to make wet packages")
if dependson_test_dependencies != []:
# install test dependencies
print "Install all test dependencies of the depends_on list: %s" % (', '.join(dependson_test_dependencies))
common.apt_get_install_also_nonrosdep(dependson_test_dependencies, rosdep_resolver)
### rosbuild repositories
if stacks != {}:
# env
print "Setting up environment"
ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([dependson_buildspace, repo_buildspace,
dependson_sourcespace_dry, repo_sourcespace_dry,
ros_package_path])
common.call("env", ros_env_dependson)
#print "Make rosdep"
#common.call("rosmake rosdep", ros_env)
#for stack in stacks.keys():
# common.call("rosdep install -y %s" % stack, ros_env_repo)
dependson_sourcespace_dry_dirs = [name for name in os.listdir(dependson_sourcespace_dry) if os.path.isdir(os.path.join(dependson_sourcespace_dry, name))]
print "Build dry depends_on repositories:\n - %s" % '\n - '.join(dependson_sourcespace_dry_dirs)
os.mkdir(dry_test_results_dir)
for dry_dependson in dependson_sourcespace_dry_dirs:
try:
common.call("rosmake -rV --profile --pjobs=8 --output=%s %s" % (dry_test_results_dir, b_r_short), ros_env_dependson)
except:
raise common.BuildException("Failed to rosmake %s" % b_r_short)
def main():
#########################
### parsing arguments ###
#########################
time_parsing = datetime.datetime.now()
print "=====> entering argument parsing step at", time_parsing
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# for hardware build: only support hydro
if ros_distro == "groovy":
sys.exit(False)
# for hardware build: remove old build artifacts
limit = 3 # limit amount of old builds
print workspace
while len(os.listdir(workspace + "/..")) > limit + 1: # we will not count the "current" sym link
list = os.listdir(workspace + "/..")
shutil.rmtree(workspace + "/../" + sorted(list)[0])
# set up directories variables
tmpdir = workspace
repo_checkoutspace = os.path.join(tmpdir, 'checkout') # location to store repositories in (will be separated in wet and dry later on)
os.makedirs(repo_checkoutspace)
repo_sourcespace = os.path.join(tmpdir, 'src') # location to build repositories in
os.makedirs(repo_sourcespace)
repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet', 'src') # wet (catkin) repositories
os.makedirs(repo_sourcespace_wet)
repo_sourcespace_dry = os.path.join(repo_sourcespace, 'dry') # dry (rosbuild) repositories
os.makedirs(repo_sourcespace_dry)
repo_build_logs = os.path.join(tmpdir, 'build_logs') # location for build logs
os.makedirs(repo_build_logs)
# init catkin workspace
ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro) # source ros_distro (needed to do a catkin_init_workspace)
# for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
if os.path.isdir("/u/robot/git/care-o-bot/devel"):
ros_env_repo = common.get_ros_env('/u/robot/git/care-o-bot/devel/setup.bash')
common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env_repo, verbose=False) # init wet workspace
common.call("catkin_make --directory %s/wet" % repo_sourcespace, ros_env_repo, verbose=False) # build wet workspace to generate a setup.bash
################
### checkout ###
################
time_checkout = datetime.datetime.now()
print "=====> entering checkout step at", time_checkout
# download build_repo from source #
print "Creating rosinstall file for repository %s" % build_repo
rosinstall = ""
if build_identifier in pipe_repos: # check if triggering identifier is really present in pipeline config
rosinstall += pipe_repos[build_identifier].get_rosinstall()
else:
err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
raise common.BuildException(err_msg)
# write rosinstall file
print "Rosinstall file for repository: \n %s" % rosinstall
with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
f.write(rosinstall)
print "Install repository from source:"
# rosinstall repos
common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
% (repo_checkoutspace, workspace, ros_distro))
# get the repositories build dependencies
print "Get build dependencies of repo"
# get all packages in checkoutspace
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_checkoutspace)
# check if build_repo is wet or dry and get all corresponding deps
build_repo_type = ''
if build_repo in catkin_packages: # wet repo with metapackage
print "repo %s is wet" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif (build_repo not in catkin_packages) and (build_repo not in stacks) and (catkin_packages != []): # wet repo without metapackage
print "repo %s is wet without metapackage" % build_repo
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif build_repo in stacks: # dry repo with stack
print "repo %s is dry" % build_repo
build_repo_type = 'dry'
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
#TODO elif : # dry repo without stack
else: # build_repo is neither wet nor dry
raise common.BuildException("Repository %s to build not found in checkoutspace" % build_repo)
# install user-defined/customized dependencies of build_repo from source
rosinstall = ''
fulfilled_deps = []
for dep in repo_build_dependencies:
if dep in pipe_repos[build_identifier].dependencies:
print "Install user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# install additional, indirect user-defined dependencies
for dep in pipe_repos[build_identifier].dependencies:
if dep not in fulfilled_deps:
print "Install additional user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# check if all user-defined/customized dependencies are satisfied
if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
print "Not all user-defined build dependencies are fulfilled"
print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
raise common.BuildException("Not all user-defined build dependencies are fulfilled")
if rosinstall != '':
# write .rosinstall file
print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install user-defined build dependencies from source"
# rosinstall depends
common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
% (repo_checkoutspace, workspace, ros_distro))
# get also deps of just installed user-defined/customized dependencies
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_checkoutspace)
if build_repo_type == 'wet':
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
else: # dry build repo
# take all packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]
print ""
print "Found the following packages"
print " wet packages: ", catkin_packages.keys()
print " dry stacks: ", stacks.keys()
print " dry packages: ", manifest_packages.keys()
print ""
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
# get all folders in repo_checkoutspace
checkoutspace_dirs = [name for name in os.listdir(repo_checkoutspace) if os.path.isdir(os.path.join(repo_checkoutspace, name))]
for dir in checkoutspace_dirs:
if dir in catkin_packages.keys(): # wet repo with metapackage
shutil.move(os.path.join(repo_checkoutspace, dir), os.path.join(repo_sourcespace_wet, dir))
elif build_repo_type == 'wet' and dir == build_repo: # wet repo without metapackage
shutil.move(os.path.join(repo_checkoutspace, dir), os.path.join(repo_sourcespace_wet, dir))
elif dir in stacks.keys(): # dry repo with stack
shutil.move(os.path.join(repo_checkoutspace, dir), os.path.join(repo_sourcespace_dry, dir))
#TODO elif: # dry repo without stack
#else:
# raise common.BuildException("Could not separate %s into wet or dry sourcespace." %dir)
# remove checkout dir
common.call("rm -rf %s" % repo_checkoutspace)
# setup ros workspace
print "Set up ros workspace and setup environment variables"
ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
common.call("rosws init %s /opt/ros/%s" %(repo_sourcespace, ros_distro), verbose=False) # init workspace pointing to ros_distro
# for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
if os.path.isdir("/u/robot/git/care-o-bot"):
common.call("rosws merge -t %s /u/robot/git/care-o-bot" %repo_sourcespace, verbose=False) # merge robot account workspace
common.call("rosws merge -t %s %s/wet/devel" % (repo_sourcespace, repo_sourcespace), verbose=False) # merge wet workspace
common.call("rosws merge -t %s %s/dry" % (repo_sourcespace, repo_sourcespace), verbose=False) # merge dry workspace
ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash') # source wet and dry workspace
############################
### install dependencies ###
############################
time_install = datetime.datetime.now()
print "=====> entering dependency installation step at", time_install
# Create rosdep object
rosdep_resolver = None
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except: # when init fails the first time
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
missing_packages = common.apt_get_check_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
if len(missing_packages) > 0:
raise common.BuildException("Some dependencies are missing. Please ask your administrator to install the following packages: %s" % missing_packages)
print "All denendencies already installed."
#############
### build ###
#############
time_build = datetime.datetime.now()
print "=====> entering build step at", time_build
# get amount of cores available on host system
cores = multiprocessing.cpu_count()
### catkin repositories
if catkin_packages != {}:
print "Build wet packages: ", catkin_packages.keys()
try:
common.call("catkin_make --directory %s/wet" % repo_sourcespace, ros_env_repo)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to catkin_make wet repositories")
### rosbuild repositories
if build_repo_type == 'dry':
# build dry repositories
print "Build dry stacks: ", stacks.keys()
print "Build dry packages: ", manifest_packages.keys()
packages_to_build = " ".join(stacks.keys()) + " ".join(manifest_packages.keys())
try:
common.call("rosmake -rV --skip-blacklist --profile --pjobs=%s --output=%s %s" %
(cores + 1, repo_build_logs, packages_to_build), ros_env_repo)
except common.BuildException as ex:
try:
shutil.move(repo_build_logs, os.path.join(workspace, "build_logs"))
finally:
print ex.msg
raise common.BuildException("Failed to rosmake dry repositories")
###########
### end ###
###########
# for hardware builds: create sym link to new build
common.call("rm -f %s/../current" % workspace)
common.call("ln -sf %s %s/../current" %(workspace, workspace))
# steps: parsing, checkout, install, analysis, build, finish
time_finish = datetime.datetime.now()
print "=====> finished script at", time_finish
print "durations:"
print "parsing arguments in ", (time_checkout - time_parsing)
print "checkout in ", (time_install - time_checkout)
print "install dependencies in ", (time_build - time_install)
print "build in ", (time_finish - time_build)
print "total ", (time_finish - time_parsing)
print ""
print "For manual testing, please run the following line in your testing terminal"
print "source " + repo_sourcespace + "/setup.bash"
print ""
def main():
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
ros_package_path = os.environ['ROS_PACKAGE_PATH']
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print datetime.datetime.now()
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository') # location to store repositories in
repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry') # dry (rosbuild) repositories
repo_buildspace = os.path.join(tmpdir, 'build_repository') # location for build output
dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs') # location for build logs
# download build_repo from source
print "Creating rosinstall file for repository %s" % build_repo
rosinstall = ""
if build_identifier in pipe_repos: # check if triggering identifier is really present in pipeline config
rosinstall += pipe_repos[build_identifier].get_rosinstall()
else:
err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
raise common.BuildException(err_msg)
# write rosinstall file
print "Rosinstall file for repository: \n %s" % rosinstall
with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
f.write(rosinstall)
print "Install repository from source:"
# create repo sourcespace directory 'src_repository'
os.makedirs(repo_sourcespace)
# rosinstall repos
common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get the repositories build dependencies
print "Get build dependencies of repo"
# get all packages in sourcespace
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if ros_distro == 'electric' and catkin_packages != {}:
raise common.BuildException("Found wet packages while building in ros electric")
# (debug) output
if options.verbose:
print "Packages in %s:" % repo_sourcespace
print "Catkin: ", catkin_packages
print "Rosbuild:\n Stacks: ", stacks
print " Packages: ", manifest_packages
# get deps directly for catkin (like in willow code)
try:
print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
except:
pass
# deps catkin
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# deps stacks
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# check if build_repo is wet or dry and get all corresponding deps
build_repo_type = ''
if build_repo in catkin_packages:
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif build_repo in stacks:
build_repo_type = 'dry'
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
else:
# build_repo is neither wet nor dry
raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)
# install user-defined/customized dependencies of build_repo from source
rosinstall = ''
fulfilled_deps = []
for dep in repo_build_dependencies:
if dep in pipe_repos[build_identifier].dependencies:
print "Install user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# install additional, indirect user-defined dependencies
for dep in pipe_repos[build_identifier].dependencies:
if dep not in fulfilled_deps:
print "Install additional user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# check if all user-defined/customized dependencies are satisfied
if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
print "Not all user-defined build dependencies are fulfilled"
print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
raise common.BuildException("Not all user-defined build dependencies are fulfilled")
if rosinstall != '':
# write .rosinstall file
print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install user-defined build dependencies from source"
# rosinstall depends
common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get also deps of just installed user-defined/customized dependencies
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if build_repo_type == 'wet':
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
else: # dry build repo
# take all packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]
rosdep_resolver = None
if ros_distro != 'electric':
# Create rosdep object
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except: # when init fails the first time
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
print datetime.datetime.now()
print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
print datetime.datetime.now()
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
os.makedirs(repo_sourcespace_wet)
os.makedirs(repo_sourcespace_dry)
# get all folders in repo_sourcespace
sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys():
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
if dir in stacks.keys():
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))
# env
print "Set up ros environment variables"
ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
if options.verbose:
common.call("env", ros_env)
### catkin repositories
print datetime.datetime.now()
if catkin_packages != {}:
# set up catkin workspace
if ros_distro == 'fuerte':
if 'catkin' not in catkin_packages.keys():
# add catkin package to rosinstall
rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
print "Install catkin"
# rosinstall catkin
common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace_wet, workspace, ros_distro))
print "Create a CMakeLists.txt for catkin packages"
common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
else:
common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env)
os.mkdir(repo_buildspace)
os.chdir(repo_buildspace)
try:
common.call("cmake %s" % repo_sourcespace_wet + '/', ros_env)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to cmake wet repositories")
#ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories
print "Build wet repository list"
try:
common.call("make", ros_env)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to make wet packages")
### rosbuild repositories
print datetime.datetime.now()
if build_repo_type == 'dry':
ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
if options.verbose:
common.call("env", ros_env_repo)
if ros_distro == 'electric':
print "Rosdep"
common.call("rosmake rosdep", ros_env)
for stack in stacks.keys():
common.call("rosdep install -y %s" % stack, ros_env_repo)
# build dry repositories
print "Build repository %s" % build_repo
try:
common.call("rosmake -rV --profile --pjobs=8 --output=%s %s" %
(dry_build_logs, build_repo), ros_env_repo)
except common.BuildException as ex:
try:
shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
finally:
print ex.msg
raise common.BuildException("Failed to rosmake %s" % build_repo)
print datetime.datetime.now()
def main():
# parse parameter values
parser = optparse.OptionParser()
parser.add_option('-v', '--verbose', action='store_true', default=False)
(options, args) = parser.parse_args()
if len(args) < 5:
print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
raise common.BuildException("Wrong arguments for build script")
# get arguments
pipeline_repos_owner = args[0]
server_name = args[1]
user_name = args[2]
ros_distro = args[3]
build_identifier = args[4] # repository + suffix
build_repo = build_identifier.split('__')[0] # only repository to build
# environment variables
workspace = os.environ['WORKSPACE']
ros_package_path = os.environ['ROS_PACKAGE_PATH']
# (debug) output
print "\n", 50 * 'X'
print datetime.datetime.now()
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "\n", 50 * 'X'
# update sourcelist and upgrade installed basic packages
#common.output("Updating chroot enviroment")
#common.call("apt-get update")
#common.call("apt-get dist-upgrade -y")
#common.output("Updating rosinstall") # TODO run install frequently in chroot_tarball_updater an remove here
#common.call("pip install -U rosinstall")
# install packages needed for execution (depending on ros_distro)
#common.output("Installing necessary packages:", decoration='')
#if ros_distro == 'electric':
#print " catkin-pkg and rospkg"
#common.call("pip install -U catkin-pkg rospkg")
#elif ros_distro == 'fuerte':
#print " rospkg and rosdep"
#common.call("pip install -U rospkg rosdep")
#elif ros_distro == 'groovy':
#print " python-catkin-pkg and python-rosdistro"
#common.call("apt-get install python-catkin-pkg python-rosdistro --yes")
# cob_pipe object
cp_instance = cob_pipe.CobPipe()
cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
pipe_repos = cp_instance.repositories
common.output("Pipeline configuration successfully loaded", blankline='b')
# (debug) output
print "\n", 50 * 'X'
print datetime.datetime.now()
print "\nTesting on ros distro: %s" % ros_distro
print "Testing repository: %s" % build_repo
if build_repo != build_identifier:
print " with suffix: %s" % build_identifier.split('__')[1]
print "Using source: %s" % pipe_repos[build_identifier].url
print "Testing branch/version: %s" % pipe_repos[build_identifier].version
print "\n", 50 * 'X'
# set up directories variables
tmpdir = os.path.join('/tmp', 'test_repositories')
repo_sourcespace = os.path.join(tmpdir, 'src_repository') # location to store repositories in
repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet') # wet (catkin) repositories
repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry') # dry (rosbuild) repositories
repo_buildspace = os.path.join(tmpdir, 'build_repository') # location for build output
dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs') # location for build logs
# download build_repo from source
print "Creating rosinstall file for repository %s" % build_repo
rosinstall = ""
if build_identifier in pipe_repos: # check if triggering identifier is really present in pipeline config
rosinstall += pipe_repos[build_identifier].get_rosinstall()
else:
err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
raise common.BuildException(err_msg)
# write rosinstall file
print "Rosinstall file for repository: \n %s" % rosinstall
with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
f.write(rosinstall)
print "Install repository from source:"
# create repo sourcespace directory 'src_repository'
os.makedirs(repo_sourcespace)
# rosinstall repos
common.call("rosinstall --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get the repositories build dependencies
print "Get build dependencies of repo"
# get all packages in sourcespace
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if ros_distro == 'electric' and catkin_packages != {}:
raise common.BuildException("Found wet packages while building in ros electric")
# (debug) output
if options.verbose:
print "Packages in %s:" % repo_sourcespace
print "Catkin: ", catkin_packages
print "Rosbuild:\n Stacks: ", stacks
print " Packages: ", manifest_packages
# get deps directly for catkin (like in willow code)
try:
print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
except:
pass
# deps catkin
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# deps stacks
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
# check if build_repo is wet or dry and get all corresponding deps
build_repo_type = ''
if build_repo in catkin_packages:
build_repo_type = 'wet'
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
elif build_repo in stacks:
build_repo_type = 'dry'
repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
else:
# build_repo is neither wet nor dry
raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)
# install user-defined/customized dependencies of build_repo from source
rosinstall = ''
fulfilled_deps = []
for dep in repo_build_dependencies:
if dep in pipe_repos[build_identifier].dependencies:
print "Install user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# install additional, indirect user-defined dependencies
for dep in pipe_repos[build_identifier].dependencies:
if dep not in fulfilled_deps:
print "Install additional user-defined build dependency %s from source" % dep
rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
fulfilled_deps.append(dep)
# check if all user-defined/customized dependencies are satisfied
if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
print "Not all user-defined build dependencies are fulfilled"
print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
raise common.BuildException("Not all user-defined build dependencies are fulfilled")
if rosinstall != '':
# write .rosinstall file
print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
f.write(rosinstall)
print "Install user-defined build dependencies from source"
# rosinstall depends
common.call("rosinstall --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace, workspace, ros_distro))
# get also deps of just installed user-defined/customized dependencies
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
if build_repo_type == 'wet':
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
else: # dry build repo
# take all packages
repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]
rosdep_resolver = None
if ros_distro != 'electric':
# Create rosdep object
print "Create rosdep object"
try:
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
except: # when init fails the first time
from time import sleep
sleep(10)
rosdep_resolver = rosdep.RosDepResolver(ros_distro)
print datetime.datetime.now()
print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
print datetime.datetime.now()
# separate installed repos in wet and dry
print "Separate installed repositories in wet and dry"
os.makedirs(repo_sourcespace_wet)
os.makedirs(repo_sourcespace_dry)
# get all folders in repo_sourcespace
sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
for dir in sourcespace_dirs:
if dir in catkin_packages.keys():
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
if dir in stacks.keys():
shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))
# env
print "Set up ros environment variables"
ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
if options.verbose:
common.call("env", ros_env)
### catkin repositories
print datetime.datetime.now()
if catkin_packages != {}:
# set up catkin workspace
if ros_distro == 'fuerte':
if 'catkin' not in catkin_packages.keys():
# add catkin package to rosinstall
rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
print "Install catkin"
# rosinstall catkin
common.call("rosinstall --verbose %s %s/repo.rosinstall /opt/ros/%s"
% (repo_sourcespace_wet, workspace, ros_distro))
print "Create a CMakeLists.txt for catkin packages"
common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
else:
common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env)
os.mkdir(repo_buildspace)
os.chdir(repo_buildspace)
try:
common.call("cmake %s" % repo_sourcespace_wet + '/', ros_env)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to cmake wet repositories")
#ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))
# build repositories and tests
print "Build wet repository list"
try:
common.call("make", ros_env)
except common.BuildException as ex:
print ex.msg
raise common.BuildException("Failed to make wet packages")
test_error_msg = None
try:
common.call("make tests", ros_env)
except common.BuildException as ex:
print ex.msg
test_error_msg = ex.msg
# get wet repositories test and run dependencies
print "Get test and run dependencies of repo list"
(catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
if stacks != {}:
raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
% (build_repo, '- ' + '\n- '.join(stacks)))
# take only wet packages
repo_test_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=False, test_depends=True)
if repo_test_dependencies != [] and test_error_msg is None:
print "Install test and run dependencies of repository list: %s" % (', '.join(repo_test_dependencies))
common.apt_get_install_also_nonrosdep(repo_test_dependencies, ros_distro, rosdep_resolver)
# run tests
print "Test repository list"
try:
common.call("make run_tests", ros_env)
except common.BuildException as ex:
print ex.msg
test_error_msg = ex.msg
# copy test results
common.copy_test_results(workspace, repo_buildspace, test_error_msg)
### rosbuild repositories
print datetime.datetime.now()
if build_repo_type == 'dry':
# get list of dependencies to test
test_repos_list = []
for dep in pipe_repos[build_identifier].dependencies:
if dep in stacks:
test_repos_list.append(dep)
ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
if options.verbose:
common.call("env", ros_env_repo)
if ros_distro == 'electric':
print "Rosdep"
common.call("rosmake rosdep", ros_env)
for stack in stacks.keys():
common.call("rosdep install -y %s" % stack, ros_env_repo)
# build dry repositories and tests
print "Build repository %s" % build_repo
try:
common.call("rosmake -rV --profile --pjobs=8 --output=%s %s" %
(dry_build_logs, build_repo), ros_env_repo)
except common.BuildException as ex:
try:
shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
finally:
print ex.msg
raise common.BuildException("Failed to rosmake %s" % build_repo)
try:
common.call("rosmake -rV --profile --pjobs=8 --test-only --output=%s %s" %
(dry_build_logs, " ".join(test_repos_list + [build_repo])), ros_env_repo)
except common.BuildException as ex:
print ex.msg
# copy test results
common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
for file in os.listdir(os.path.join(repo_sourcespace, "test_results")):
file_path = os.path.join(repo_sourcespace, "test_results", file)
print file_path
try:
if not file.startswith("_hudson"):
shutil.rmtree(file_path)
except Exception as e:
print e
common.copy_test_results(workspace, repo_sourcespace)
print datetime.datetime.now()
try:
shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
except IOError as ex:
print "No build logs found: %s" % ex