def main(): (options, args) = get_options(['rosdistro'], ['delete', 'wait', 'stack', 'os']) if not options: return -1 # generate hudson config files distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) if options.stack: stack_list = options.stack for s in stack_list: if not s in distro_obj.released_stacks: if s in distro_obj.stacks: print "Stack %s is in rosdistro file, but it is not yet released (version is set to 'null'" % s else: print "Stack %s is not in rosdistro file" % s return else: stack_list = distro_obj.released_stacks devel_configs = {} for stack_name in stack_list: devel_configs.update( create_devel_configs(options.os, distro_obj.release_name, distro_obj.stacks[stack_name])) # schedule jobs schedule_jobs(devel_configs, options.wait, options.delete)
def main(): (options, args) = get_options(['rosdistro', 'overlay', 'variant'], ['database']) if not options: return -1 # Parse distro file distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) # generate rosinstall file for variant if options.overlay == 'yes': rosinstall = rosdistro.variant_to_rosinstall(options.variant, distro_obj, 'release-tar') else: rosinstall = rosdistro.extended_variant_to_rosinstall(options.variant, distro_obj, 'release-tar') print rosinstall # write to database if options.database: try: connection = sqlite.connect(options.database) cursor = connection.cursor() try: cursor.execute('CREATE TABLE rosinstall (stamp TIMESTAMP, rosdistro TEXT, variant TEXT, overlay TEXT)') except sqlite.OperationalError: pass cursor.execute('INSERT INTO rosinstall VALUES (?, ?, ?, ?)', (datetime.datetime.now(), options.rosdistro, options.variant, options.overlay)) connection.commit() except: pass
def main(): # parse command line options (options, args) = get_options(['rosdistro'], []) if not options: return -1 # set environment env = get_environment() env['PATH'] = '/opt/ros/%s/ros/bin:%s'%(options.rosdistro, os.environ['PATH']) env['ROS_PACKAGE_PATH'] = '%s:/opt/ros/%s/stacks'%(env['WORKSPACE'], options.rosdistro) env['ROS_ROOT'] = '/opt/ros/%s/ros'%options.rosdistro env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src' # Parse distro file rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) print 'Operating on ROS distro %s'%rosdistro_obj.release_name # Install Debian packages of ALL stacks in distro call('sudo apt-get update', env) print 'Installing all stacks of ros distro %s: %s'%(options.rosdistro, str(rosdistro_obj.stacks.keys())) for stack in rosdistro_obj.stacks: call('sudo apt-get install %s --yes'%(stack_to_deb(stack, options.rosdistro)), env, ignore_fail=True) # install system dependencies of all packages res = call('rospack list', env, 'Getting list of all packages') packages = [p.split(' ')[0] for p in res.split('\n') if p != ''] for pkg in packages: if not pkg in rosdistro_obj.stacks: res = call('rospack find %s'%pkg, env, 'Check if package is blacklisted') if not os.path.isfile(res[0:len(res)-1]+'/ROS_BUILD_BLACKLIST'): call('rosdep install -y %s'%pkg, env, 'Installing system dependencies of package %s'%pkg) # Run hudson helper print 'Running Hudson Helper' helper = subprocess.Popen(('./hudson_helper --dir-test %s build'%env['WORKSPACE']).split(' '), env=env) helper.communicate() return helper.returncode
def analyze(ros_distro, stack_name, workspace, test_depends_on): print "Testing on distro %s"%ros_distro print "Testing stack %s"%stack_name # global try try: # Declare variables STACK_DIR = 'stack_overlay' DEPENDS_DIR = 'depends_overlay' DEPENDS_ON_DIR = 'depends_on_overlay' # set environment print "Setting up environment" env = get_environment() env['INSTALL_DIR'] = os.getcwd() env['STACK_BUILD_DIR'] = env['INSTALL_DIR'] + '/build/' + stack_name env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks'%(env['INSTALL_DIR']+'/'+STACK_DIR + '/' + stack_name, env['INSTALL_DIR']+'/'+DEPENDS_DIR, env['INSTALL_DIR']+'/'+DEPENDS_ON_DIR, ros_distro) print "ROS_PACKAGE_PATH = %s"%(env['ROS_PACKAGE_PATH']) if 'ros' == stack_name: env['ROS_ROOT'] = env['INSTALL_DIR']+'/'+STACK_DIR+'/ros' print "We're building ROS, so setting the ROS_ROOT to %s"%(env['ROS_ROOT']) else: env['ROS_ROOT'] = '/opt/ros/%s/ros'%ros_distro env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src' env['PATH'] = '%s:%s:/opt/ros/%s/ros/bin:%s'%(env['QACPPBIN'],env['HTMLVIEWBIN'],ros_distro, os.environ['PATH']) #%s:%s:%s #print 'PATH %s'%( env['PATH']) print "Environment set to %s"%str(env) #distro = rosdistro.Distro(get_rosdistro_file(ros_distro)) # Parse distro file rosdistro_obj = rosdistro.Distro(get_rosdistro_file(ros_distro)) print 'Operating on ROS distro %s'%rosdistro_obj.release_name
def main(): (options, args) = get_options(['rosdistro'], ['delete', 'wait', 'stack']) if not options: return -1 # Parse distro file distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) print 'Operating on ROS distro %s' % distro_obj.release_name # generate hudson config files post_release_configs = {} if options.stack: stack_list = options.stack else: stack_list = distro_obj.released_stacks for stack_name in stack_list: post_release_configs.update( create_post_release_configs( distro_obj.release_name, distro_obj.released_stacks[stack_name])) # schedule jobs schedule_jobs(post_release_configs, options.wait, options.delete)
def main(): # parse command line options (options, args) = get_options(['rosdistro'], ['repeat', 'source-only']) if not options: return -1 # Parse distro file rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) print 'Operating on ROS distro %s' % rosdistro_obj.release_name env = get_environment() try: for stack in rosdistro_obj.released_stacks: print 'Analyzing stack %s' % stack #h = subprocess.Popen(('./analyze_stack.py --rosdistro %s --stack %s'%(options.rosdistro,stack)).split(' '), env=env) h = subprocess.Popen(( 'run_chroot_jenkins_now lucid amd64 [email protected] metrics %s %s dry' % (options.rosdistro, stack)).split(' '), env=env) h.communicate() except Exception, ex: print "%s. Check the console output for test failure details." % ex traceback.print_exc(file=sys.stdout) raise ex
def get_options(required, optional): parser = optparse.OptionParser() ops = required + optional if 'os' in ops: parser.add_option('--os', dest='os', default='ubuntu', action='store', help='OS name') if 'rosdistro' in ops: parser.add_option('--rosdistro', dest='rosdistro', default=None, action='store', help='Ros distro name') if 'stack' in ops: parser.add_option('--stack', dest='stack', default=None, action='append', help='Stack name') if 'email' in ops: parser.add_option('--email', dest='email', default=None, action='store', help='Email address to send results to') if 'arch' in ops: parser.add_option('--arch', dest='arch', default=None, action='append', help='Architecture to test') if 'ubuntu' in ops: parser.add_option('--ubuntu', dest='ubuntu', default=None, action='append', help='Ubuntu distribution to test') if 'repeat' in ops: parser.add_option('--repeat', dest='repeat', default=0, action='store', help='How many times to repeat the test') if 'source-only' in ops: parser.add_option( '--source-only', dest='source_only', default=False, action='store_true', help="Build everything from source, don't use Debian packages") if 'delete' in ops: parser.add_option('--delete', dest='delete', default=False, action='store_true', help='Delete jobs from Hudson') if 'wait' in ops: parser.add_option( '--wait', dest='wait', default=False, action='store_true', help='Wait for running jobs to finish to reconfigure them') if 'rosinstall' in ops: parser.add_option( '--rosinstall', dest='rosinstall', default=None, action='store', help="Specify the rosinstall file that refers to unreleased code.") if 'overlay' in ops: parser.add_option('--overlay', dest='overlay', default=None, action='store', help='Create overlay file') if 'variant' in ops: parser.add_option( '--variant', dest='variant', default=None, action='store', help="Specify the variant to create a rosinstall for") if 'database' in ops: parser.add_option('--database', dest='database', default=None, action='store', help="Specify database file") (options, args) = parser.parse_args() # make repeat an int if 'repeat' in ops: options.repeat = int(options.repeat) # check if required arguments are there for r in required: if not eval('options.%s' % r): print 'You need to specify "--%s"' % r return (None, args) # postprocessing if 'email' in ops and options.email and not '@' in options.email: options.email = options.email + '@willowgarage.com' # check if rosdistro exists if 'rosdistro' in ops and (not options.rosdistro or not options.rosdistro in UBUNTU_DISTRO_MAP.keys()): print 'You provided an invalid "--rosdistro %s" argument. Options are %s' % ( options.rosdistro, UBUNTU_DISTRO_MAP.keys()) return (None, args) # check if stacks exist if 'stack' in ops and options.stack: distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) for s in options.stack: if not s in distro_obj.stacks: print 'Stack "%s" does not exist in the %s disro file.' % ( s, options.rosdistro) print 'You need to add this stack to the rosdistro file' return (None, args) # check if variant exists if 'variant' in ops and options.variant: distro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) if not options.variant in distro_obj.variants: print 'Variant "%s" does not exist in the %s disro file.' % ( options.variant, options.rosdistro) return (None, args) return (options, args)
def main(): # parse command line options (options, args) = get_options(['rosdistro', 'stack'], []) if not options: return -1 if len(options.stack) > 1: print "You can only provide one stack at a time" return -1 options.stack = options.stack[0] # set environment env = get_environment() env['ROS_PACKAGE_PATH'] = '%s:%s:/opt/ros/%s/stacks' % ( env['WORKSPACE'] + '/' + options.stack, env['INSTALL_DIR'] + '/' + DEPENDS_ON_DIR, options.rosdistro) if options.stack == 'ros': env['ROS_ROOT'] = env['WORKSPACE'] + '/' + options.stack print "We're building ROS, so setting the ROS_ROOT to %s" % ( env['ROS_ROOT']) else: env['ROS_ROOT'] = '/opt/ros/%s/ros' % options.rosdistro env['PYTHONPATH'] = env['ROS_ROOT'] + '/core/roslib/src' env['PATH'] = '/opt/ros/%s/ros/bin:%s' % (options.rosdistro, os.environ['PATH']) # Parse distro file rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) print 'Operating on ROS distro %s' % rosdistro_obj.release_name # Install Debian packages of stack dependencies print 'Installing debian packages of stack dependencies' call('sudo apt-get update', env) with open('%s/%s/stack.xml' % (os.environ['WORKSPACE'], options.stack)) as stack_file: depends = stack_manifest.parse(stack_file.read()).depends if len(depends) != 0: call( 'sudo apt-get install %s --yes' % (stacks_to_debs(depends, options.rosdistro)), env, 'Installing dependencies of stack "%s": %s' % (options.stack, str(depends))) # Install system dependencies print 'Installing system dependencies' call('rosmake rosdep', env) call('rosdep install -y %s' % options.stack, env, 'Installing system dependencies of stack %s' % options.stack) # Run hudson helper for stacks only print 'Running Hudson Helper' env['ROS_TEST_RESULTS_DIR'] = os.environ[ 'ROS_TEST_RESULTS_DIR'] + '/' + options.stack helper = subprocess.Popen(('./hudson_helper --dir-test %s/%s build' % (env['WORKSPACE'], options.stack)).split(' '), env=env) helper.communicate() if helper.returncode != 0: return helper.returncode # Install Debian packages of ALL stacks in distro print 'Installing all released stacks of ros distro %s: %s' % ( options.rosdistro, str(rosdistro_obj.released_stacks.keys())) for stack in rosdistro_obj.released_stacks: call('sudo apt-get install %s --yes' % (stack_to_deb(stack, options.rosdistro)), env, ignore_fail=True) # Install all stacks that depend on this stack print 'Installing all stacks that depend on these stacks from source' res = call('rosstack depends-on %s' % options.stack, env, 'Getting list of stacks that depends on %s' % options.stack) print 'These stacks depend on the stacks we are testing: "%s"' % str(res) if res == '': print 'No stack depends on %s, finishing test.' % options.stack return 0 rosinstall = stacks_to_rosinstall(res.split('\n'), rosdistro_obj.released_stacks, 'release-tar') print 'Running rosinstall on "%s"' % rosinstall rosinstall_file = '%s.rosinstall' % DEPENDS_ON_DIR with open(rosinstall_file, 'w') as f: f.write(rosinstall) call( 'rosinstall --rosdep-yes %s /opt/ros/%s %s/%s %s' % (DEPENDS_ON_DIR, options.rosdistro, os.environ['WORKSPACE'], options.stack, rosinstall_file), env, 'Install of stacks that depend on %s from source' % options.stack) # Remove stacks that depend on this stack from Debians print 'Removing all stack from Debian that depend on this stack' call( 'sudo apt-get remove %s --yes' % stack_to_deb(options.stack, options.rosdistro), env) # Run hudson helper for all stacks print 'Running Hudson Helper' env['ROS_TEST_RESULTS_DIR'] = os.environ[ 'ROS_TEST_RESULTS_DIR'] + '/' + DEPENDS_ON_DIR helper = subprocess.Popen( ('./hudson_helper --dir-test %s build' % DEPENDS_ON_DIR).split(' '), env=env) helper.communicate() return helper.returncode
import roslib roslib.load_manifest("rosdistro") import rosdistro import yaml release_name = 'unstable' uri = "https://code.ros.org/svn/release/trunk/distros/%s.rosdistro" % release_name distro = rosdistro.Distro(uri) print 'ros', distro.stacks['ros'].repo print 'nxt', distro.stacks['nxt'].repo print 'pr2_doors', distro.stacks['pr2_doors'].repo print 'geometry', distro.stacks['geometry'].repo for s in distro.stack_names: print s, distro.stacks[s].repo print "geometry" print yaml.dump( rosdistro.stack_to_rosinstall(distro.stacks['geometry'], "distro")) print "nxt distro" print yaml.dump(rosdistro.stack_to_rosinstall(distro.stacks['nxt'], "distro")) print "nxt release" print yaml.dump(rosdistro.stack_to_rosinstall(distro.stacks['nxt'], "release")) print "pr2all" print yaml.dump(rosdistro.variant_to_rosinstall("pr2all", distro, "distro")) print "distro" print yaml.dump(rosdistro.distro_to_rosinstall(distro, "distro"))
def main(): # global try try: print "Parsing arguments" # parse command line options (options, args) = get_options(['stack', 'rosdistro'], ['repeat', 'source-only']) print "options", options, "args", args if not options: return -1 # set environment print "Setting up environment" env = get_environment() env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks' % ( env['INSTALL_DIR'] + '/' + STACK_DIR, env['INSTALL_DIR'] + '/' + DEPENDS_DIR, env['INSTALL_DIR'] + '/' + DEPENDS_ON_DIR, options.rosdistro) if 'ros' in options.stack: env['ROS_ROOT'] = env['INSTALL_DIR'] + '/' + STACK_DIR + '/ros' print "We're building ROS, so setting the ROS_ROOT to %s" % ( env['ROS_ROOT']) ros_tested_ignore_return = True else: env['ROS_ROOT'] = '/opt/ros/%s/ros' % options.rosdistro ros_tested_ignore_return = False env['PYTHONPATH'] = env['ROS_ROOT'] + '/core/roslib/src' env['PATH'] = '/opt/ros/%s/ros/bin:%s' % (options.rosdistro, os.environ['PATH']) print "Environment set to %s" % str(env) # Parse distro file rosdistro_obj = rosdistro.Distro(get_rosdistro_file(options.rosdistro)) print 'Operating on ROS distro %s' % rosdistro_obj.release_name # Install the stacks to test from source print 'Installing the stacks to test from source' rosinstall = '' for stack in options.stack: rosinstall += stack_to_rosinstall(rosdistro_obj.stacks[stack], 'devel') rosinstall_file = '%s.rosinstall' % STACK_DIR print 'Generating rosinstall file [%s]' % (rosinstall_file) print 'Contents:\n\n' + rosinstall + '\n\n' with open(rosinstall_file, 'w') as f: f.write(rosinstall) print 'rosinstall file [%s] generated' % (rosinstall_file) call('rosinstall --rosdep-yes %s /opt/ros/%s %s' % (STACK_DIR, options.rosdistro, rosinstall_file), env, 'Install the stacks to test from source.', ignore_fail=ros_tested_ignore_return) # get all stack dependencies of stacks we're testing print "Computing dependencies of stacks we're testing" depends_all = [] for stack in options.stack: stack_xml = '%s/%s/stack.xml' % (STACK_DIR, stack) call('ls %s' % stack_xml, env, 'Checking if stack %s contains "stack.xml" file' % stack) with open(stack_xml) as stack_file: depends_one = [ str(d) for d in stack_manifest.parse(stack_file.read()).depends ] # convert to list print 'Dependencies of stack %s: %s' % (stack, str(depends_one)) for d in depends_one: if not d in options.stack and not d in depends_all: print 'Adding dependencies of stack %s' % d get_depends_all(rosdistro_obj, d, depends_all) print 'Resulting total dependencies of all stacks that get tested: %s' % str( depends_all) if len(depends_all) > 0: if options.source_only: # Install dependencies from source print 'Installing stack dependencies from source' rosinstall = stacks_to_rosinstall( depends_all, rosdistro_obj.released_stacks, 'release-tar') rosinstall_file = '%s.rosinstall' % DEPENDS_DIR print 'Generating rosinstall file [%s]' % (rosinstall_file) print 'Contents:\n\n' + rosinstall + '\n\n' with open(rosinstall_file, 'w') as f: f.write(rosinstall) print 'rosinstall file [%s] generated' % (rosinstall_file) call('rosinstall --rosdep-yes %s /opt/ros/%s %s' % (DEPENDS_DIR, options.rosdistro, rosinstall_file), env, 'Install the stack dependencies from source.', ignore_fail=ros_tested_ignore_return) else: # Install Debian packages of stack dependencies print 'Installing debian packages of "%s" dependencies: %s' % ( stack, str(depends_all)) call('sudo apt-get update', env) call( 'sudo apt-get install %s --yes' % (stacks_to_debs(depends_all, options.rosdistro)), env) else: print 'Stack(s) %s do(es) not have any dependencies, not installing anything now' % str( options.stack) # Install system dependencies of stacks re're testing print "Installing system dependencies of stacks we're testing" call('rosmake rosdep', env) for stack in options.stack: call('rosdep install -y %s' % stack, env, 'Install system dependencies of stack %s' % stack) # Run hudson helper for stacks only print "Running Hudson Helper for stacks we're testing" res = 0 for r in range(0, int(options.repeat) + 1): env['ROS_TEST_RESULTS_DIR'] = env[ 'ROS_TEST_RESULTS_DIR'] + '/' + STACK_DIR + '_run_' + str(r) helper = subprocess.Popen( ('./hudson_helper --dir-test %s build' % STACK_DIR).split(' '), env=env) helper.communicate() if helper.returncode != 0: res = helper.returncode if res != 0: return res # parse debian repository configuration file to get stack dependencies (arch, ubuntudistro) = get_sys_info() print "Parsing apt repository configuration file to get stack dependencies, for %s machine running %s" % ( arch, ubuntudistro) apt_deps = parse_apt(ubuntudistro, arch, options.rosdistro) if not apt_deps.has_debian_package(options.stack): print "Stack does not yet have a Debian package. No need to test dependenies" return 0 # all stacks that depends on the tested stacks, excluding the tested stacks. depends_on_all = apt_deps.depends_on_all(options.stack) remove(depends_on_all, options.stack) # if tested stacks are all in a variant, then only test stacks that are also in a variant variant_stacks = [] for name, v in rosdistro_obj.variants.iteritems(): variant_stacks = variant_stacks + v.stack_names all_in_variant = True for s in options.stack: if not s in variant_stacks: all_in_variant = False if all_in_variant: print "Limiting test to stacks that are in a variant" for s in depends_on_all: if not s in variant_stacks: depends_on_all.remove(s) # all stack dependencies of above stack list, except for the test stack dependencies depends_all_depends_on_all = apt_deps.depends_all(depends_on_all) remove(depends_all_depends_on_all, options.stack) remove(depends_all_depends_on_all, depends_all) # Install dependencies of depends_on_all stacks, excluding dependencies of test stacks. if len(depends_all_depends_on_all) > 0: print "Install dependencies of depends_on_all stacks, excluding dependencies of test stacks." if not options.source_only: # Install Debian packages of 'depends_all_depends_on_all' list print 'Installing Debian package of %s' % str( depends_all_depends_on_all) call( 'sudo apt-get install %s --yes' % (stacks_to_debs( depends_all_depends_on_all, options.rosdistro)), env) else: # Install source of 'depends_all_depends_on_all' list print 'Installing source of %s' % str( depends_all_depends_on_all) rosinstall = stacks_to_rosinstall( depends_all_depends_on_all, rosdistro_obj.released_stacks, 'release-tar') rosinstall_file = '%s_depends_all_depends_on_all.rosinstall' % DEPENDS_ON_DIR print 'Generating rosinstall file [%s]' % (rosinstall_file) print 'Contents:\n\n' + rosinstall + '\n\n' with open(rosinstall_file, 'w') as f: f.write(rosinstall) print 'rosinstall file [%s] generated' % (rosinstall_file) call( 'rosinstall --rosdep-yes %s /opt/ros/%s %s %s' % (DEPENDS_ON_DIR, options.rosdistro, STACK_DIR, rosinstall_file), env, 'Install dependencies of depends_on_all stacks, excluding dependencies of test stacks.', ignore_fail=ros_tested_ignore_return) else: print "No dependencies of depends_on_all stacks" # Install all stacks that depend on this stack from source if len(depends_on_all) > 0: print 'Installing depends_on_all stacks from source: %s' % str( depends_on_all) rosinstall = stacks_to_rosinstall(depends_on_all, rosdistro_obj.released_stacks, 'release-tar') rosinstall_file = '%s.rosinstall' % DEPENDS_ON_DIR print 'Generating rosinstall file [%s]' % (rosinstall_file) print 'Contents:\n\n' + rosinstall + '\n\n' with open(rosinstall_file, 'w') as f: f.write(rosinstall) print 'rosinstall file [%s] generated' % (rosinstall_file) call( 'rosinstall --rosdep-yes %s /opt/ros/%s %s %s' % (DEPENDS_ON_DIR, options.rosdistro, STACK_DIR, rosinstall_file), env, 'Install the stacks that depend on the stacks that are getting tested from source.', ignore_fail=ros_tested_ignore_return) # Run hudson helper for all stacks print 'Running Hudson Helper' env['ROS_TEST_RESULTS_DIR'] = env[ 'ROS_TEST_RESULTS_DIR'] + '/' + DEPENDS_ON_DIR helper = subprocess.Popen(('./hudson_helper --dir-test %s build' % DEPENDS_ON_DIR).split(' '), env=env) helper.communicate() return helper.returncode else: print "No stacks depends on this stack. Tests finished" # global except except Exception, ex: print "Global exception caught. Generating email. Exception:", ex generate_email( "%s. Check the console output for test failure details." % ex, env) traceback.print_exc(file=sys.stdout) raise ex
def analyze(ros_distro, stack_name, workspace, test_depends_on): print "Testing on distro %s"%ros_distro print "Testing stack %s"%stack_name # global try try: # Declare variables STACK_DIR = 'stack_overlay' DEPENDS_DIR = 'depends_overlay' DEPENDS_ON_DIR = 'depends_on_overlay' # set environment print "Setting up environment" env = get_environment() env['INSTALL_DIR'] = os.getcwd() env['STACK_BUILD_DIR'] = env['INSTALL_DIR'] + '/build/' + stack_name env['ROS_PACKAGE_PATH'] = '%s:%s:%s:/opt/ros/%s/stacks'%(env['INSTALL_DIR']+'/'+STACK_DIR + '/' + stack_name, env['INSTALL_DIR']+'/'+DEPENDS_DIR, env['INSTALL_DIR']+'/'+DEPENDS_ON_DIR, ros_distro) print "ROS_PACKAGE_PATH = %s"%(env['ROS_PACKAGE_PATH']) if 'ros' == stack_name: env['ROS_ROOT'] = env['INSTALL_DIR']+'/'+STACK_DIR+'/ros' print "We're building ROS, so setting the ROS_ROOT to %s"%(env['ROS_ROOT']) else: env['ROS_ROOT'] = '/opt/ros/%s/ros'%ros_distro env['PYTHONPATH'] = env['ROS_ROOT']+'/core/roslib/src' env['PATH'] = '%s:%s:/opt/ros/%s/ros/bin:%s'%(env['QACPPBIN'],env['HTMLVIEWBIN'],ros_distro, os.environ['PATH']) #%s:%s:%s #print 'PATH %s'%( env['PATH']) print "Environment set to %s"%str(env) # Create_new/remove_old STACK_DIR,build,doc,cvs folder stack_path = env['INSTALL_DIR']+'/'+STACK_DIR + '/' if os.path.exists(stack_path): shutil.rmtree(stack_path) os.makedirs(stack_path) build_path = env['INSTALL_DIR'] + '/build/' if os.path.exists(build_path): shutil.rmtree(build_path) os.makedirs(build_path) doc_path = env['INSTALL_DIR'] + '/doc/' if os.path.exists(doc_path): shutil.rmtree(doc_path) os.makedirs(doc_path) csv_path = env['INSTALL_DIR'] + '/csv/' if os.path.exists(csv_path): shutil.rmtree(csv_path) os.makedirs(csv_path) snapshots_path = env['INSTALL_DIR'] + '/snapshots/' if os.path.exists(snapshots_path): shutil.rmtree(snapshots_path) # create dummy test results test_results_path = env['INSTALL_DIR'] + '/test_results' if os.path.exists(test_results_path): shutil.rmtree(test_results_path) os.makedirs(test_results_path) test_file= test_results_path + '/test_file.xml' f = open(test_file, 'w') f.write('<?xml version="1.0" encoding="UTF-8"?>\n') f.write('<testsuite tests="1" failures="0" time="1" errors="0" name="dummy test">\n') f.write(' <testcase name="dummy rapport" classname="Results" /> \n') f.write('</testsuite> \n') f.close() # Parse distro file rosdistro_obj = rosdistro.Distro(get_rosdistro_file(ros_distro)) print 'Operating on ROS distro %s'%rosdistro_obj.release_name # Install the stacks to test from source call('echo -e "\033[33;33m Color Text"', env, 'Set output-color for installing to yellow') print 'Installing the stacks to test from source' rosinstall_file = '%s.rosinstall'%STACK_DIR if os.path.exists(rosinstall_file): os.remove(rosinstall_file) if os.path.exists('%s/.rosinstall'%STACK_DIR): os.remove('%s/.rosinstall'%STACK_DIR) rosinstall = '' #for stack in options.stack: print 'stack: %s'%(stack_name) rosinstall += stack_to_rosinstall(rosdistro_obj.stacks[stack_name], 'devel') print 'Generating rosinstall file [%s]'%(rosinstall_file) print 'Contents:\n\n'+rosinstall+'\n\n' with open(rosinstall_file, 'w') as f: f.write(rosinstall) print 'rosinstall file [%s] generated'%(rosinstall_file) call('rosinstall --rosdep-yes %s /opt/ros/%s %s'%(STACK_DIR, ros_distro, rosinstall_file), env, 'Install the stacks to test from source.') # get all stack dependencies of stacks we're testing print "Computing dependencies of stacks we're testing" depends_all = [] #for stack in stack_name: stack_xml = '%s/%s/stack.xml'%(STACK_DIR, stack_name) call('ls %s'%stack_xml, env, 'Checking if stack %s contains "stack.xml" file'%stack_name) with open(stack_xml) as stack_file: depends_one = [str(d) for d in stack_manifest.parse(stack_file.read()).depends] # convert to list print 'Dependencies of stack %s: %s'%(stack_name, str(depends_one)) for d in depends_one: #if not d in stack_name and not d in depends_all: if d != stack_name and not d in depends_all: print 'Adding dependencies of stack %s'%d get_depends_all(rosdistro_obj, d, depends_all) print 'Resulting total dependencies of all stacks that get tested: %s'%str(depends_all) if len(depends_all) > 0: # Install Debian packages of stack dependencies print 'Installing debian packages of %s dependencies: %s'%(stack_name, str(depends_all)) call('sudo apt-get update', env) call('sudo apt-get install %s --yes'%(stacks_to_debs(depends_all, ros_distro)), env) else: print 'Stack(s) %s do(es) not have any dependencies, not installing anything now'%str(stack_name) # Install system dependencies of stacks we're testing print "Installing system dependencies of stacks we're testing" call('rosmake rosdep', env) #for stack in stack_name: call('rosdep install -y %s'%stack_name, env, 'Install system dependencies of stack %s'%stack_name) # Get uri data vcs = rosdistro_obj.stacks[stack_name].vcs_config uri_data = {} if vcs.type == 'svn': uri_data['vcs_type'] = 'svn' uri_data['uri'] = vcs.anon_dev uri_data['uri_info'] = 'empty' elif vcs.type == 'git': uri_data['vcs_type'] = 'git' uri_data['uri'] = vcs.anon_repo_uri # Get branch p = subprocess.Popen(["git", "branch"],cwd=r'%s/%s/%s/'%(workspace,STACK_DIR,stack_name), env=env,stdout=subprocess.PIPE) out = p.communicate()[0] branch = out[2:] uri_data['uri_info'] = branch print "branch: %s"%branch elif vcs.type == 'hg': uri_data['vcs_type'] = 'hg' uri_data['uri'] = vcs.anon_repo_uri # Get revision number p = subprocess.Popen(["hg", "log", "-l", "1", "--template", "{node}"],cwd=r'%s/%s/%s/'%(workspace,STACK_DIR,stack_name), env=env,stdout=subprocess.PIPE) out = p.communicate()[0] revision_number = out[:12] #first 12 numbers represents the revision number uri_data['uri_info'] = revision_number print "revision_number: %s"%revision_number uri = uri_data['uri'] uri_info = uri_data['uri_info'] vcs_type = uri_data['vcs_type'] # Run hudson helper for stacks only call('echo -e "\033[33;34m Color Text"', env, 'Set color from build-output to blue') print "Running Hudson Helper for stacks we're testing" res = 0 #for r in range(0, int(options.repeat)+1): for r in range(0, int(0)+1): env['ROS_TEST_RESULTS_DIR'] = env['ROS_TEST_RESULTS_DIR'] + '/' + STACK_DIR + '_run_' + str(r) helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/build_helper.py --dir %s build'%(workspace,STACK_DIR + '/' + stack_name)).split(' '), env=env) helper.communicate() print "helper_return_code is: %s"%(helper.returncode) if helper.returncode != 0: res = helper.returncode # Concatenate filelists call('echo -e "\033[33;0m Color Text"', env, 'Set color to white') print '----------------- Concatenate filelists ----------------- ' stack_dir = STACK_DIR + '/' + str(stack_name) filelist = stack_dir + '/filelist.lst' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/concatenate_filelists.py --dir %s --filelist %s'%(workspace,stack_dir, filelist)).split(' '), env=env) helper.communicate() print '////////////////// concatenate filelists done ////////////////// \n\n' # Run CMA print '----------------- Run CMA analysis ----------------- ' cmaf = stack_dir + '/' + str(stack_name) helper = subprocess.Popen(('pal QACPP -cmaf %s -list %s'%(cmaf, filelist)).split(' '), env=env) helper.communicate() print '////////////////// cma analysis done ////////////////// \n\n' # Export metrics to yaml and csv files print '----------------- Export metrics to yaml and csv files ----------------- ' print 'stack_dir: %s '%str(stack_dir) print 'stack_name: %s '%str(stack_name) helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/export_metrics_to_yaml.py --path %s --doc metrics --csv csv --config %s/jenkins_scripts/code_quality/export_config.yaml --distro %s --stack %s --uri %s --uri_info %s --vcs_type %s'%(workspace,stack_dir,workspace, ros_distro, stack_name, uri, uri_info, vcs_type)).split(' '), env=env) helper.communicate() print '////////////////// export metrics to yaml and csv files done ////////////////// \n\n' # Push results to server print '----------------- Push results to server ----------------- ' helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/push_results_to_server.py --path %s --doc metrics'%(workspace,stack_dir)).split(' '), env=env) helper.communicate() print '////////////////// push results to server done ////////////////// \n\n' # Upload results to QAVerify print ' ----------------- upload results to QAVerify ----------------- ' project_name = stack_name + '-' + ros_distro helper = subprocess.Popen(('%s/jenkins_scripts/code_quality/upload_to_QAVerify.py --path %s --snapshot %s --project %s --stack_name %s'%(workspace, workspace, snapshots_path, project_name, stack_name)).split(' '), env=env) helper.communicate() print '////////////////// upload results to QAVerify done ////////////////// \n\n' print ' ----------------- Remove directories ----------------- ' # Remove STACK_DIR, build, doc, cvs-folder's if os.path.exists(stack_path): shutil.rmtree(stack_path) if os.path.exists(build_path): shutil.rmtree(build_path) if os.path.exists(doc_path): shutil.rmtree(doc_path) if os.path.exists(csv_path): shutil.rmtree(csv_path) if os.path.exists(snapshots_path): shutil.rmtree(snapshots_path) print '////////////////// Remove directories ////////////////// \n\n' print 'ANALYSIS PROCESS OF STACK %s DONE\n\n'%str(stack_name) if res != 0: print "helper_return_code is: %s"%(helper.returncode) raise Exception("build_helper.py failed. Often an analysis mistake. Check out the console output above for details.") return res # global except except Exception, ex: print "Global exception caught." print "%s. Check the console output for test failure details."%ex traceback.print_exc(file=sys.stdout) raise ex