def main():
(options, args) = get_options(['rosdistro'],
['delete', 'wait', 'stack', 'os'])
if not options:
return -1
# generate hudson config files
distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
if options.stack:
stack_list = options.stack
for s in stack_list:
if not s in distro_obj.released_stacks:
if s in distro_obj.stacks:
print "Stack %s is in rosdistro file, but it is not yet released (version is set to 'null'" % s
else:
print "Stack %s is not in rosdistro file" % s
return
else:
stack_list = distro_obj.released_stacks
devel_configs = {}
for stack_name in stack_list:
devel_configs.update(
create_devel_configs(options.os, distro_obj.release_name,
distro_obj.stacks[stack_name]))
# schedule jobs
schedule_jobs(devel_configs, options.wait, options.delete)
def main():
(options, args) = get_options(['rosdistro', 'overlay', 'variant'], ['database'])
if not options:
return -1
# Parse distro file
distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
# generate rosinstall file for variant
if options.overlay == 'yes':
rosinstall = rosdistro.variant_to_rosinstall(options.variant, distro_obj, 'release-tar')
else:
rosinstall = rosdistro.extended_variant_to_rosinstall(options.variant, distro_obj, 'release-tar')
print rosinstall
# write to database
if options.database:
try:
connection = sqlite.connect(options.database)
cursor = connection.cursor()
try:
cursor.execute('CREATE TABLE rosinstall (stamp TIMESTAMP, rosdistro TEXT, variant TEXT, overlay TEXT)')
except sqlite.OperationalError:
pass
cursor.execute('INSERT INTO rosinstall VALUES (?, ?, ?, ?)', (datetime.datetime.now(), options.rosdistro, options.variant, options.overlay))
connection.commit()
except:
pass
def main():
# parse command line options
(options, args) = get_options(['rosdistro'], [])
if not options:
return -1
# set environment
env = get_environment()
env['PATH'] = '/opt/ros/%s/ros/bin:%s'%(options.rosdistro, os.environ['PATH'])
env['ROS_PACKAGE_PATH'] = '%s:/opt/ros/%s/stacks'%(env['WORKSPACE'], options.rosdistro)
env['ROS_ROOT'] = '/opt/ros/%s/ros'%options.rosdistro
env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src'
# Parse distro file
rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
print 'Operating on ROS distro %s'%rosdistro_obj.release_name
# Install Debian packages of ALL stacks in distro
call('sudo apt-get update', env)
print 'Installing all stacks of ros distro %s: %s'%(options.rosdistro, str(rosdistro_obj.stacks.keys()))
for stack in rosdistro_obj.stacks:
call('sudo apt-get install %s --yes'%(stack_to_deb(stack, options.rosdistro)), env, ignore_fail=True)
# install system dependencies of all packages
res = call('rospack list', env, 'Getting list of all packages')
packages = [p.split(' ')[0] for p in res.split('\n') if p != '']
for pkg in packages:
if not pkg in rosdistro_obj.stacks:
res = call('rospack find %s'%pkg, env, 'Check if package is blacklisted')
if not os.path.isfile(res[0:len(res)-1]+'/ROS_BUILD_BLACKLIST'):
call('rosdep install -y %s'%pkg, env, 'Installing system dependencies of package %s'%pkg)
# Run hudson helper
print 'Running Hudson Helper'
helper = subprocess.Popen(('./hudson_helper --dir-test %s build'%env['WORKSPACE']).split(' '), env=env)
helper.communicate()
return helper.returncode
def analyze(ros_distro, stack_name, workspace, test_depends_on):
print "Testing on distro %s"%ros_distro
print "Testing stack %s"%stack_name
# global try
try:
# Declare variables
STACK_DIR = 'stack_overlay'
DEPENDS_DIR = 'depends_overlay'
DEPENDS_ON_DIR = 'depends_on_overlay'
# set environment
print "Setting up environment"
env = get_environment()
env['INSTALL_DIR'] = os.getcwd()
env['STACK_BUILD_DIR'] = env['INSTALL_DIR'] + '/build/' + stack_name
env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks'%(env['INSTALL_DIR']+'/'+STACK_DIR + '/' + stack_name,
env['INSTALL_DIR']+'/'+DEPENDS_DIR,
env['INSTALL_DIR']+'/'+DEPENDS_ON_DIR,
ros_distro)
print "ROS_PACKAGE_PATH = %s"%(env['ROS_PACKAGE_PATH'])
if 'ros' == stack_name:
env['ROS_ROOT'] = env['INSTALL_DIR']+'/'+STACK_DIR+'/ros'
print "We're building ROS, so setting the ROS_ROOT to %s"%(env['ROS_ROOT'])
else:
env['ROS_ROOT'] = '/opt/ros/%s/ros'%ros_distro
env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src'
env['PATH'] = '%s:%s:/opt/ros/%s/ros/bin:%s'%(env['QACPPBIN'],env['HTMLVIEWBIN'],ros_distro, os.environ['PATH']) #%s:%s:%s
#print 'PATH %s'%( env['PATH'])
print "Environment set to %s"%str(env)
#distro = rosdistro.Distro(get_rosdistro_file(ros_distro))
# Parse distro file
rosdistro_obj = rosdistro.Distro(get_rosdistro_file(ros_distro))
print 'Operating on ROS distro %s'%rosdistro_obj.release_name
def main():
(options, args) = get_options(['rosdistro'], ['delete', 'wait', 'stack'])
if not options:
return -1
# Parse distro file
distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
print 'Operating on ROS distro %s' % distro_obj.release_name
# generate hudson config files
post_release_configs = {}
if options.stack:
stack_list = options.stack
else:
stack_list = distro_obj.released_stacks
for stack_name in stack_list:
post_release_configs.update(
create_post_release_configs(
distro_obj.release_name,
distro_obj.released_stacks[stack_name]))
# schedule jobs
schedule_jobs(post_release_configs, options.wait, options.delete)
def main():
# parse command line options
(options, args) = get_options(['rosdistro'], ['repeat', 'source-only'])
if not options:
return -1
# Parse distro file
rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
print 'Operating on ROS distro %s' % rosdistro_obj.release_name
env = get_environment()
try:
for stack in rosdistro_obj.released_stacks:
print 'Analyzing stack %s' % stack
#h = subprocess.Popen(('./analyze_stack.py --rosdistro %s --stack %s'%(options.rosdistro,stack)).split(' '), env=env)
h = subprocess.Popen((
'run_chroot_jenkins_now lucid amd64 [email protected] metrics %s %s dry'
% (options.rosdistro, stack)).split(' '),
env=env)
h.communicate()
except Exception, ex:
print "%s. Check the console output for test failure details." % ex
traceback.print_exc(file=sys.stdout)
raise ex
def get_options(required, optional):
parser = optparse.OptionParser()
ops = required + optional
if 'os' in ops:
parser.add_option('--os',
dest='os',
default='ubuntu',
action='store',
help='OS name')
if 'rosdistro' in ops:
parser.add_option('--rosdistro',
dest='rosdistro',
default=None,
action='store',
help='Ros distro name')
if 'stack' in ops:
parser.add_option('--stack',
dest='stack',
default=None,
action='append',
help='Stack name')
if 'email' in ops:
parser.add_option('--email',
dest='email',
default=None,
action='store',
help='Email address to send results to')
if 'arch' in ops:
parser.add_option('--arch',
dest='arch',
default=None,
action='append',
help='Architecture to test')
if 'ubuntu' in ops:
parser.add_option('--ubuntu',
dest='ubuntu',
default=None,
action='append',
help='Ubuntu distribution to test')
if 'repeat' in ops:
parser.add_option('--repeat',
dest='repeat',
default=0,
action='store',
help='How many times to repeat the test')
if 'source-only' in ops:
parser.add_option(
'--source-only',
dest='source_only',
default=False,
action='store_true',
help="Build everything from source, don't use Debian packages")
if 'delete' in ops:
parser.add_option('--delete',
dest='delete',
default=False,
action='store_true',
help='Delete jobs from Hudson')
if 'wait' in ops:
parser.add_option(
'--wait',
dest='wait',
default=False,
action='store_true',
help='Wait for running jobs to finish to reconfigure them')
if 'rosinstall' in ops:
parser.add_option(
'--rosinstall',
dest='rosinstall',
default=None,
action='store',
help="Specify the rosinstall file that refers to unreleased code.")
if 'overlay' in ops:
parser.add_option('--overlay',
dest='overlay',
default=None,
action='store',
help='Create overlay file')
if 'variant' in ops:
parser.add_option(
'--variant',
dest='variant',
default=None,
action='store',
help="Specify the variant to create a rosinstall for")
if 'database' in ops:
parser.add_option('--database',
dest='database',
default=None,
action='store',
help="Specify database file")
(options, args) = parser.parse_args()
# make repeat an int
if 'repeat' in ops:
options.repeat = int(options.repeat)
# check if required arguments are there
for r in required:
if not eval('options.%s' % r):
print 'You need to specify "--%s"' % r
return (None, args)
# postprocessing
if 'email' in ops and options.email and not '@' in options.email:
options.email = options.email + '@willowgarage.com'
# check if rosdistro exists
if 'rosdistro' in ops and (not options.rosdistro or not options.rosdistro
in UBUNTU_DISTRO_MAP.keys()):
print 'You provided an invalid "--rosdistro %s" argument. Options are %s' % (
options.rosdistro, UBUNTU_DISTRO_MAP.keys())
return (None, args)
# check if stacks exist
if 'stack' in ops and options.stack:
distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
for s in options.stack:
if not s in distro_obj.stacks:
print 'Stack "%s" does not exist in the %s disro file.' % (
s, options.rosdistro)
print 'You need to add this stack to the rosdistro file'
return (None, args)
# check if variant exists
if 'variant' in ops and options.variant:
distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
if not options.variant in distro_obj.variants:
print 'Variant "%s" does not exist in the %s disro file.' % (
options.variant, options.rosdistro)
return (None, args)
return (options, args)
def main():
# parse command line options
(options, args) = get_options(['rosdistro', 'stack'], [])
if not options:
return -1
if len(options.stack) > 1:
print "You can only provide one stack at a time"
return -1
options.stack = options.stack[0]
# set environment
env = get_environment()
env['ROS_PACKAGE_PATH'] = '%s:%s:/opt/ros/%s/stacks' % (
env['WORKSPACE'] + '/' + options.stack,
env['INSTALL_DIR'] + '/' + DEPENDS_ON_DIR, options.rosdistro)
if options.stack == 'ros':
env['ROS_ROOT'] = env['WORKSPACE'] + '/' + options.stack
print "We're building ROS, so setting the ROS_ROOT to %s" % (
env['ROS_ROOT'])
else:
env['ROS_ROOT'] = '/opt/ros/%s/ros' % options.rosdistro
env['PYTHONPATH'] = env['ROS_ROOT'] + '/core/roslib/src'
env['PATH'] = '/opt/ros/%s/ros/bin:%s' % (options.rosdistro,
os.environ['PATH'])
# Parse distro file
rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
print 'Operating on ROS distro %s' % rosdistro_obj.release_name
# Install Debian packages of stack dependencies
print 'Installing debian packages of stack dependencies'
call('sudo apt-get update', env)
with open('%s/%s/stack.xml' %
(os.environ['WORKSPACE'], options.stack)) as stack_file:
depends = stack_manifest.parse(stack_file.read()).depends
if len(depends) != 0:
call(
'sudo apt-get install %s --yes' %
(stacks_to_debs(depends, options.rosdistro)), env,
'Installing dependencies of stack "%s": %s' %
(options.stack, str(depends)))
# Install system dependencies
print 'Installing system dependencies'
call('rosmake rosdep', env)
call('rosdep install -y %s' % options.stack, env,
'Installing system dependencies of stack %s' % options.stack)
# Run hudson helper for stacks only
print 'Running Hudson Helper'
env['ROS_TEST_RESULTS_DIR'] = os.environ[
'ROS_TEST_RESULTS_DIR'] + '/' + options.stack
helper = subprocess.Popen(('./hudson_helper --dir-test %s/%s build' %
(env['WORKSPACE'], options.stack)).split(' '),
env=env)
helper.communicate()
if helper.returncode != 0:
return helper.returncode
# Install Debian packages of ALL stacks in distro
print 'Installing all released stacks of ros distro %s: %s' % (
options.rosdistro, str(rosdistro_obj.released_stacks.keys()))
for stack in rosdistro_obj.released_stacks:
call('sudo apt-get install %s --yes' %
(stack_to_deb(stack, options.rosdistro)),
env,
ignore_fail=True)
# Install all stacks that depend on this stack
print 'Installing all stacks that depend on these stacks from source'
res = call('rosstack depends-on %s' % options.stack, env,
'Getting list of stacks that depends on %s' % options.stack)
print 'These stacks depend on the stacks we are testing: "%s"' % str(res)
if res == '':
print 'No stack depends on %s, finishing test.' % options.stack
return 0
rosinstall = stacks_to_rosinstall(res.split('\n'),
rosdistro_obj.released_stacks,
'release-tar')
print 'Running rosinstall on "%s"' % rosinstall
rosinstall_file = '%s.rosinstall' % DEPENDS_ON_DIR
with open(rosinstall_file, 'w') as f:
f.write(rosinstall)
call(
'rosinstall --rosdep-yes %s /opt/ros/%s %s/%s %s' %
(DEPENDS_ON_DIR, options.rosdistro, os.environ['WORKSPACE'],
options.stack, rosinstall_file), env,
'Install of stacks that depend on %s from source' % options.stack)
# Remove stacks that depend on this stack from Debians
print 'Removing all stack from Debian that depend on this stack'
call(
'sudo apt-get remove %s --yes' %
stack_to_deb(options.stack, options.rosdistro), env)
# Run hudson helper for all stacks
print 'Running Hudson Helper'
env['ROS_TEST_RESULTS_DIR'] = os.environ[
'ROS_TEST_RESULTS_DIR'] + '/' + DEPENDS_ON_DIR
helper = subprocess.Popen(
('./hudson_helper --dir-test %s build' % DEPENDS_ON_DIR).split(' '),
env=env)
helper.communicate()
return helper.returncode
import roslib
roslib.load_manifest("rosdistro")
import rosdistro
import yaml
release_name = 'unstable'
uri = "https://code.ros.org/svn/release/trunk/distros/%s.rosdistro" % release_name
distro = rosdistro.Distro(uri)
print 'ros', distro.stacks['ros'].repo
print 'nxt', distro.stacks['nxt'].repo
print 'pr2_doors', distro.stacks['pr2_doors'].repo
print 'geometry', distro.stacks['geometry'].repo
for s in distro.stack_names:
print s, distro.stacks[s].repo
print "geometry"
print yaml.dump(
rosdistro.stack_to_rosinstall(distro.stacks['geometry'], "distro"))
print "nxt distro"
print yaml.dump(rosdistro.stack_to_rosinstall(distro.stacks['nxt'], "distro"))
print "nxt release"
print yaml.dump(rosdistro.stack_to_rosinstall(distro.stacks['nxt'], "release"))
print "pr2all"
print yaml.dump(rosdistro.variant_to_rosinstall("pr2all", distro, "distro"))
print "distro"
print yaml.dump(rosdistro.distro_to_rosinstall(distro, "distro"))
def main():
# global try
try:
print "Parsing arguments"
# parse command line options
(options, args) = get_options(['stack', 'rosdistro'],
['repeat', 'source-only'])
print "options", options, "args", args
if not options:
return -1
# set environment
print "Setting up environment"
env = get_environment()
env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks' % (
env['INSTALL_DIR'] + '/' + STACK_DIR,
env['INSTALL_DIR'] + '/' + DEPENDS_DIR,
env['INSTALL_DIR'] + '/' + DEPENDS_ON_DIR, options.rosdistro)
if 'ros' in options.stack:
env['ROS_ROOT'] = env['INSTALL_DIR'] + '/' + STACK_DIR + '/ros'
print "We're building ROS, so setting the ROS_ROOT to %s" % (
env['ROS_ROOT'])
ros_tested_ignore_return = True
else:
env['ROS_ROOT'] = '/opt/ros/%s/ros' % options.rosdistro
ros_tested_ignore_return = False
env['PYTHONPATH'] = env['ROS_ROOT'] + '/core/roslib/src'
env['PATH'] = '/opt/ros/%s/ros/bin:%s' % (options.rosdistro,
os.environ['PATH'])
print "Environment set to %s" % str(env)
# Parse distro file
rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro))
print 'Operating on ROS distro %s' % rosdistro_obj.release_name
# Install the stacks to test from source
print 'Installing the stacks to test from source'
rosinstall = ''
for stack in options.stack:
rosinstall += stack_to_rosinstall(rosdistro_obj.stacks[stack],
'devel')
rosinstall_file = '%s.rosinstall' % STACK_DIR
print 'Generating rosinstall file [%s]' % (rosinstall_file)
print 'Contents:\n\n' + rosinstall + '\n\n'
with open(rosinstall_file, 'w') as f:
f.write(rosinstall)
print 'rosinstall file [%s] generated' % (rosinstall_file)
call('rosinstall --rosdep-yes %s /opt/ros/%s %s' %
(STACK_DIR, options.rosdistro, rosinstall_file),
env,
'Install the stacks to test from source.',
ignore_fail=ros_tested_ignore_return)
# get all stack dependencies of stacks we're testing
print "Computing dependencies of stacks we're testing"
depends_all = []
for stack in options.stack:
stack_xml = '%s/%s/stack.xml' % (STACK_DIR, stack)
call('ls %s' % stack_xml, env,
'Checking if stack %s contains "stack.xml" file' % stack)
with open(stack_xml) as stack_file:
depends_one = [
str(d)
for d in stack_manifest.parse(stack_file.read()).depends
] # convert to list
print 'Dependencies of stack %s: %s' % (stack,
str(depends_one))
for d in depends_one:
if not d in options.stack and not d in depends_all:
print 'Adding dependencies of stack %s' % d
get_depends_all(rosdistro_obj, d, depends_all)
print 'Resulting total dependencies of all stacks that get tested: %s' % str(
depends_all)
if len(depends_all) > 0:
if options.source_only:
# Install dependencies from source
print 'Installing stack dependencies from source'
rosinstall = stacks_to_rosinstall(
depends_all, rosdistro_obj.released_stacks, 'release-tar')
rosinstall_file = '%s.rosinstall' % DEPENDS_DIR
print 'Generating rosinstall file [%s]' % (rosinstall_file)
print 'Contents:\n\n' + rosinstall + '\n\n'
with open(rosinstall_file, 'w') as f:
f.write(rosinstall)
print 'rosinstall file [%s] generated' % (rosinstall_file)
call('rosinstall --rosdep-yes %s /opt/ros/%s %s' %
(DEPENDS_DIR, options.rosdistro, rosinstall_file),
env,
'Install the stack dependencies from source.',
ignore_fail=ros_tested_ignore_return)
else:
# Install Debian packages of stack dependencies
print 'Installing debian packages of "%s" dependencies: %s' % (
stack, str(depends_all))
call('sudo apt-get update', env)
call(
'sudo apt-get install %s --yes' %
(stacks_to_debs(depends_all, options.rosdistro)), env)
else:
print 'Stack(s) %s do(es) not have any dependencies, not installing anything now' % str(
options.stack)
# Install system dependencies of stacks re're testing
print "Installing system dependencies of stacks we're testing"
call('rosmake rosdep', env)
for stack in options.stack:
call('rosdep install -y %s' % stack, env,
'Install system dependencies of stack %s' % stack)
# Run hudson helper for stacks only
print "Running Hudson Helper for stacks we're testing"
res = 0
for r in range(0, int(options.repeat) + 1):
env['ROS_TEST_RESULTS_DIR'] = env[
'ROS_TEST_RESULTS_DIR'] + '/' + STACK_DIR + '_run_' + str(r)
helper = subprocess.Popen(
('./hudson_helper --dir-test %s build' % STACK_DIR).split(' '),
env=env)
helper.communicate()
if helper.returncode != 0:
res = helper.returncode
if res != 0:
return res
# parse debian repository configuration file to get stack dependencies
(arch, ubuntudistro) = get_sys_info()
print "Parsing apt repository configuration file to get stack dependencies, for %s machine running %s" % (
arch, ubuntudistro)
apt_deps = parse_apt(ubuntudistro, arch, options.rosdistro)
if not apt_deps.has_debian_package(options.stack):
print "Stack does not yet have a Debian package. No need to test dependenies"
return 0
# all stacks that depends on the tested stacks, excluding the tested stacks.
depends_on_all = apt_deps.depends_on_all(options.stack)
remove(depends_on_all, options.stack)
# if tested stacks are all in a variant, then only test stacks that are also in a variant
variant_stacks = []
for name, v in rosdistro_obj.variants.iteritems():
variant_stacks = variant_stacks + v.stack_names
all_in_variant = True
for s in options.stack:
if not s in variant_stacks:
all_in_variant = False
if all_in_variant:
print "Limiting test to stacks that are in a variant"
for s in depends_on_all:
if not s in variant_stacks:
depends_on_all.remove(s)
# all stack dependencies of above stack list, except for the test stack dependencies
depends_all_depends_on_all = apt_deps.depends_all(depends_on_all)
remove(depends_all_depends_on_all, options.stack)
remove(depends_all_depends_on_all, depends_all)
# Install dependencies of depends_on_all stacks, excluding dependencies of test stacks.
if len(depends_all_depends_on_all) > 0:
print "Install dependencies of depends_on_all stacks, excluding dependencies of test stacks."
if not options.source_only:
# Install Debian packages of 'depends_all_depends_on_all' list
print 'Installing Debian package of %s' % str(
depends_all_depends_on_all)
call(
'sudo apt-get install %s --yes' % (stacks_to_debs(
depends_all_depends_on_all, options.rosdistro)), env)
else:
# Install source of 'depends_all_depends_on_all' list
print 'Installing source of %s' % str(
depends_all_depends_on_all)
rosinstall = stacks_to_rosinstall(
depends_all_depends_on_all, rosdistro_obj.released_stacks,
'release-tar')
rosinstall_file = '%s_depends_all_depends_on_all.rosinstall' % DEPENDS_ON_DIR
print 'Generating rosinstall file [%s]' % (rosinstall_file)
print 'Contents:\n\n' + rosinstall + '\n\n'
with open(rosinstall_file, 'w') as f:
f.write(rosinstall)
print 'rosinstall file [%s] generated' % (rosinstall_file)
call(
'rosinstall --rosdep-yes %s /opt/ros/%s %s %s' %
(DEPENDS_ON_DIR, options.rosdistro, STACK_DIR,
rosinstall_file),
env,
'Install dependencies of depends_on_all stacks, excluding dependencies of test stacks.',
ignore_fail=ros_tested_ignore_return)
else:
print "No dependencies of depends_on_all stacks"
# Install all stacks that depend on this stack from source
if len(depends_on_all) > 0:
print 'Installing depends_on_all stacks from source: %s' % str(
depends_on_all)
rosinstall = stacks_to_rosinstall(depends_on_all,
rosdistro_obj.released_stacks,
'release-tar')
rosinstall_file = '%s.rosinstall' % DEPENDS_ON_DIR
print 'Generating rosinstall file [%s]' % (rosinstall_file)
print 'Contents:\n\n' + rosinstall + '\n\n'
with open(rosinstall_file, 'w') as f:
f.write(rosinstall)
print 'rosinstall file [%s] generated' % (rosinstall_file)
call(
'rosinstall --rosdep-yes %s /opt/ros/%s %s %s' %
(DEPENDS_ON_DIR, options.rosdistro, STACK_DIR,
rosinstall_file),
env,
'Install the stacks that depend on the stacks that are getting tested from source.',
ignore_fail=ros_tested_ignore_return)
# Run hudson helper for all stacks
print 'Running Hudson Helper'
env['ROS_TEST_RESULTS_DIR'] = env[
'ROS_TEST_RESULTS_DIR'] + '/' + DEPENDS_ON_DIR
helper = subprocess.Popen(('./hudson_helper --dir-test %s build' %
DEPENDS_ON_DIR).split(' '),
env=env)
helper.communicate()
return helper.returncode
else:
print "No stacks depends on this stack. Tests finished"
# global except
except Exception, ex:
print "Global exception caught. Generating email. Exception:", ex
generate_email(
"%s. Check the console output for test failure details." % ex, env)
traceback.print_exc(file=sys.stdout)
raise ex
def analyze(ros_distro, stack_name, workspace, test_depends_on):
print "Testing on distro %s"%ros_distro
print "Testing stack %s"%stack_name
# global try
try:
# Declare variables
STACK_DIR = 'stack_overlay'
DEPENDS_DIR = 'depends_overlay'
DEPENDS_ON_DIR = 'depends_on_overlay'
# set environment
print "Setting up environment"
env = get_environment()
env['INSTALL_DIR'] = os.getcwd()
env['STACK_BUILD_DIR'] = env['INSTALL_DIR'] + '/build/' + stack_name
env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks'%(env['INSTALL_DIR']+'/'+STACK_DIR + '/' + stack_name,
env['INSTALL_DIR']+'/'+DEPENDS_DIR,
env['INSTALL_DIR']+'/'+DEPENDS_ON_DIR,
ros_distro)
print "ROS_PACKAGE_PATH = %s"%(env['ROS_PACKAGE_PATH'])
if 'ros' == stack_name:
env['ROS_ROOT'] = env['INSTALL_DIR']+'/'+STACK_DIR+'/ros'
print "We're building ROS, so setting the ROS_ROOT to %s"%(env['ROS_ROOT'])
else:
env['ROS_ROOT'] = '/opt/ros/%s/ros'%ros_distro
env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src'
env['PATH'] = '%s:%s:/opt/ros/%s/ros/bin:%s'%(env['QACPPBIN'],env['HTMLVIEWBIN'],ros_distro, os.environ['PATH']) #%s:%s:%s
#print 'PATH %s'%( env['PATH'])
print "Environment set to %s"%str(env)
# Create_new/remove_old STACK_DIR,build,doc,cvs folder
stack_path = env['INSTALL_DIR']+'/'+STACK_DIR + '/'
if os.path.exists(stack_path):
shutil.rmtree(stack_path)
os.makedirs(stack_path)
build_path = env['INSTALL_DIR'] + '/build/'
if os.path.exists(build_path):
shutil.rmtree(build_path)
os.makedirs(build_path)
doc_path = env['INSTALL_DIR'] + '/doc/'
if os.path.exists(doc_path):
shutil.rmtree(doc_path)
os.makedirs(doc_path)
csv_path = env['INSTALL_DIR'] + '/csv/'
if os.path.exists(csv_path):
shutil.rmtree(csv_path)
os.makedirs(csv_path)
snapshots_path = env['INSTALL_DIR'] + '/snapshots/'
if os.path.exists(snapshots_path):
shutil.rmtree(snapshots_path)
# create dummy test results
test_results_path = env['INSTALL_DIR'] + '/test_results'
if os.path.exists(test_results_path):
shutil.rmtree(test_results_path)
os.makedirs(test_results_path)
test_file= test_results_path + '/test_file.xml'
f = open(test_file, 'w')
f.write('<?xml version="1.0" encoding="UTF-8"?>\n')
f.write('<testsuite tests="1" failures="0" time="1" errors="0" name="dummy test">\n')
f.write(' <testcase name="dummy rapport" classname="Results" /> \n')
f.write('</testsuite> \n')
f.close()
# Parse distro file
rosdistro_obj = rosdistro.Distro(get_rosdistro_file(ros_distro))
print 'Operating on ROS distro %s'%rosdistro_obj.release_name
# Install the stacks to test from source
call('echo -e "\033[33;33m Color Text"', env,
'Set output-color for installing to yellow')
print 'Installing the stacks to test from source'
rosinstall_file = '%s.rosinstall'%STACK_DIR
if os.path.exists(rosinstall_file):
os.remove(rosinstall_file)
if os.path.exists('%s/.rosinstall'%STACK_DIR):
os.remove('%s/.rosinstall'%STACK_DIR)
rosinstall = ''
#for stack in options.stack:
print 'stack: %s'%(stack_name)
rosinstall += stack_to_rosinstall(rosdistro_obj.stacks[stack_name], 'devel')
print 'Generating rosinstall file [%s]'%(rosinstall_file)
print 'Contents:\n\n'+rosinstall+'\n\n'
with open(rosinstall_file, 'w') as f:
f.write(rosinstall)
print 'rosinstall file [%s] generated'%(rosinstall_file)
call('rosinstall --rosdep-yes %s /opt/ros/%s %s'%(STACK_DIR, ros_distro, rosinstall_file), env,
'Install the stacks to test from source.')
# get all stack dependencies of stacks we're testing
print "Computing dependencies of stacks we're testing"
depends_all = []
#for stack in stack_name:
stack_xml = '%s/%s/stack.xml'%(STACK_DIR, stack_name)
call('ls %s'%stack_xml, env, 'Checking if stack %s contains "stack.xml" file'%stack_name)
with open(stack_xml) as stack_file:
depends_one = [str(d) for d in stack_manifest.parse(stack_file.read()).depends] # convert to list
print 'Dependencies of stack %s: %s'%(stack_name, str(depends_one))
for d in depends_one:
#if not d in stack_name and not d in depends_all:
if d != stack_name and not d in depends_all:
print 'Adding dependencies of stack %s'%d
get_depends_all(rosdistro_obj, d, depends_all)
print 'Resulting total dependencies of all stacks that get tested: %s'%str(depends_all)
if len(depends_all) > 0:
# Install Debian packages of stack dependencies
print 'Installing debian packages of %s dependencies: %s'%(stack_name, str(depends_all))
call('sudo apt-get update', env)
call('sudo apt-get install %s --yes'%(stacks_to_debs(depends_all, ros_distro)), env)
else:
print 'Stack(s) %s do(es) not have any dependencies, not installing anything now'%str(stack_name)
# Install system dependencies of stacks we're testing
print "Installing system dependencies of stacks we're testing"
call('rosmake rosdep', env)
#for stack in stack_name:
call('rosdep install -y %s'%stack_name, env,
'Install system dependencies of stack %s'%stack_name)
# Get uri data
vcs = rosdistro_obj.stacks[stack_name].vcs_config
uri_data = {}
if vcs.type == 'svn':
uri_data['vcs_type'] = 'svn'
uri_data['uri'] = vcs.anon_dev
uri_data['uri_info'] = 'empty'
elif vcs.type == 'git':
uri_data['vcs_type'] = 'git'
uri_data['uri'] = vcs.anon_repo_uri
# Get branch
p = subprocess.Popen(["git", "branch"],cwd=r'%s/%s/%s/'%(workspace,STACK_DIR,stack_name), env=env,stdout=subprocess.PIPE)
out = p.communicate()[0]
branch = out[2:]
uri_data['uri_info'] = branch
print "branch: %s"%branch
elif vcs.type == 'hg':
uri_data['vcs_type'] = 'hg'
uri_data['uri'] = vcs.anon_repo_uri
# Get revision number
p = subprocess.Popen(["hg", "log", "-l", "1", "--template", "{node}"],cwd=r'%s/%s/%s/'%(workspace,STACK_DIR,stack_name), env=env,stdout=subprocess.PIPE)
out = p.communicate()[0]
revision_number = out[:12] #first 12 numbers represents the revision number
uri_data['uri_info'] = revision_number
print "revision_number: %s"%revision_number
uri = uri_data['uri']
uri_info = uri_data['uri_info']
vcs_type = uri_data['vcs_type']
# Run hudson helper for stacks only
call('echo -e "\033[33;34m Color Text"', env,
'Set color from build-output to blue')
print "Running Hudson Helper for stacks we're testing"
res = 0
#for r in range(0, int(options.repeat)+1):
for r in range(0, int(0)+1):
env['ROS_TEST_RESULTS_DIR'] = env['ROS_TEST_RESULTS_DIR'] + '/' + STACK_DIR + '_run_' + str(r)
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/build_helper.py --dir %s build'%(workspace,STACK_DIR + '/' + stack_name)).split(' '), env=env)
helper.communicate()
print "helper_return_code is: %s"%(helper.returncode)
if helper.returncode != 0:
res = helper.returncode
# Concatenate filelists
call('echo -e "\033[33;0m Color Text"', env, 'Set color to white')
print '----------------- Concatenate filelists ----------------- '
stack_dir = STACK_DIR + '/' + str(stack_name)
filelist = stack_dir + '/filelist.lst'
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/concatenate_filelists.py --dir %s --filelist %s'%(workspace,stack_dir, filelist)).split(' '), env=env)
helper.communicate()
print '////////////////// concatenate filelists done ////////////////// \n\n'
# Run CMA
print '----------------- Run CMA analysis ----------------- '
cmaf = stack_dir + '/' + str(stack_name)
helper = subprocess.Popen(('pal QACPP -cmaf %s -list %s'%(cmaf, filelist)).split(' '), env=env)
helper.communicate()
print '////////////////// cma analysis done ////////////////// \n\n'
# Export metrics to yaml and csv files
print '----------------- Export metrics to yaml and csv files ----------------- '
print 'stack_dir: %s '%str(stack_dir)
print 'stack_name: %s '%str(stack_name)
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/export_metrics_to_yaml.py --path %s --doc metrics --csv csv --config %s/jenkins_scripts/code_quality/export_config.yaml --distro %s --stack %s --uri %s --uri_info %s --vcs_type %s'%(workspace,stack_dir,workspace, ros_distro, stack_name, uri, uri_info, vcs_type)).split(' '), env=env)
helper.communicate()
print '////////////////// export metrics to yaml and csv files done ////////////////// \n\n'
# Push results to server
print '----------------- Push results to server ----------------- '
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/push_results_to_server.py --path %s --doc metrics'%(workspace,stack_dir)).split(' '), env=env)
helper.communicate()
print '////////////////// push results to server done ////////////////// \n\n'
# Upload results to QAVerify
print ' ----------------- upload results to QAVerify ----------------- '
project_name = stack_name + '-' + ros_distro
helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/upload_to_QAVerify.py --path %s --snapshot %s --project %s --stack_name %s'%(workspace, workspace, snapshots_path, project_name, stack_name)).split(' '), env=env)
helper.communicate()
print '////////////////// upload results to QAVerify done ////////////////// \n\n'
print ' ----------------- Remove directories ----------------- '
# Remove STACK_DIR, build, doc, cvs-folder's
if os.path.exists(stack_path):
shutil.rmtree(stack_path)
if os.path.exists(build_path):
shutil.rmtree(build_path)
if os.path.exists(doc_path):
shutil.rmtree(doc_path)
if os.path.exists(csv_path):
shutil.rmtree(csv_path)
if os.path.exists(snapshots_path):
shutil.rmtree(snapshots_path)
print '////////////////// Remove directories ////////////////// \n\n'
print 'ANALYSIS PROCESS OF STACK %s DONE\n\n'%str(stack_name)
if res != 0:
print "helper_return_code is: %s"%(helper.returncode)
raise Exception("build_helper.py failed. Often an analysis mistake. Check out the console output above for details.")
return res
# global except
except Exception, ex:
print "Global exception caught."
print "%s. Check the console output for test failure details."%ex
traceback.print_exc(file=sys.stdout)
raise ex
