Exemplo n.º 1
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[
            0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]  # repository + suffix
    build_repo = build_identifier.split('__')[0]  # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner,
                                      server_name,
                                      user_name,
                                      file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro: %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # for hardware build: only support hydro
    if ros_distro == "groovy":
        sys.exit(False)

    # for hardware build: remove old build artifacts
    limit = 3  # limit amount of old builds
    print workspace
    while len(os.listdir(workspace + "/..")
              ) > limit + 1:  # we will not count the "current" sym link
        list = os.listdir(workspace + "/..")
        shutil.rmtree(workspace + "/../" + sorted(list)[0])

    # set up directories variables
    tmpdir = workspace
    repo_checkoutspace = os.path.join(
        tmpdir, 'checkout'
    )  # location to store repositories in (will be separated in wet and dry later on)
    os.makedirs(repo_checkoutspace)
    repo_sourcespace = os.path.join(tmpdir,
                                    'src')  # location to build repositories in
    os.makedirs(repo_sourcespace)
    repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet',
                                        'src')  # wet (catkin) repositories
    os.makedirs(repo_sourcespace_wet)
    repo_sourcespace_dry = os.path.join(repo_sourcespace,
                                        'dry')  # dry (rosbuild) repositories
    os.makedirs(repo_sourcespace_dry)
    repo_build_logs = os.path.join(tmpdir,
                                   'build_logs')  # location for build logs
    os.makedirs(repo_build_logs)

    # init catkin workspace
    ros_env_repo = common.get_ros_env(
        '/opt/ros/%s/setup.bash' %
        ros_distro)  # source ros_distro (needed to do a catkin_init_workspace)
    # for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
    if os.path.isdir("/u/robot/git/care-o-bot/devel"):
        ros_env_repo = common.get_ros_env(
            '/u/robot/git/care-o-bot/devel/setup.bash')
    common.call("catkin_init_workspace %s" % repo_sourcespace_wet,
                ros_env_repo,
                verbose=False)  # init wet workspace
    common.call("catkin_make --directory %s/wet" % repo_sourcespace,
                ros_env_repo,
                verbose=False)  # build wet workspace to generate a setup.bash

    ################
    ### checkout ###
    ################
    time_checkout = datetime.datetime.now()
    print "=====> entering checkout step at", time_checkout

    # download build_repo from source #
    print "Creating rosinstall file for repository %s" % build_repo
    rosinstall = ""
    if build_identifier in pipe_repos:  # check if triggering identifier is really present in pipeline config
        rosinstall += pipe_repos[build_identifier].get_rosinstall()
    else:
        err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
        raise common.BuildException(err_msg)

    # write rosinstall file
    print "Rosinstall file for repository: \n %s" % rosinstall
    with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
        f.write(rosinstall)
    print "Install repository from source:"
    # rosinstall repos
    common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s" %
                (repo_checkoutspace, workspace, ros_distro))

    # get the repositories build dependencies
    print "Get build dependencies of repo"

    # get all packages in checkoutspace
    (catkin_packages, stacks,
     manifest_packages) = common.get_all_packages(repo_checkoutspace)

    # check if build_repo is wet or dry and get all corresponding deps
    build_repo_type = ''
    if build_repo in catkin_packages:  # wet repo with metapackage
        print "repo %s is wet" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(
            catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif (build_repo
          not in catkin_packages) and (build_repo not in stacks) and (
              catkin_packages != []):  # wet repo without metapackage
        print "repo %s is wet without metapackage" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(
            catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif build_repo in stacks:  # dry repo with stack
        print "repo %s is dry" % build_repo
        build_repo_type = 'dry'
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks,
                                                                   {})
    #TODO elif : # dry repo without stack
    else:  # build_repo is neither wet nor dry
        raise common.BuildException(
            "Repository %s to build not found in checkoutspace" % build_repo)

    # install user-defined/customized dependencies of build_repo from source
    rosinstall = ''
    fulfilled_deps = []
    for dep in repo_build_dependencies:
        if dep in pipe_repos[build_identifier].dependencies:
            print "Install user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[
                dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # install additional, indirect user-defined dependencies
    for dep in pipe_repos[build_identifier].dependencies:
        if dep not in fulfilled_deps:
            print "Install additional user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[
                dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # check if all user-defined/customized dependencies are satisfied
    if sorted(fulfilled_deps) != sorted(
            pipe_repos[build_identifier].dependencies):
        print "Not all user-defined build dependencies are fulfilled"
        print "User-defined build dependencies:\n - %s" % '\n - '.join(
            pipe_repos[build_identifier].dependencies)
        print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
        raise common.BuildException(
            "Not all user-defined build dependencies are fulfilled")

    if rosinstall != '':
        # write .rosinstall file
        print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print "Install user-defined build dependencies from source"
        # rosinstall depends
        common.call(
            "rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s" %
            (repo_checkoutspace, workspace, ros_distro))

        # get also deps of just installed user-defined/customized dependencies
        (catkin_packages, stacks,
         manifest_packages) = common.get_all_packages(repo_checkoutspace)

        if build_repo_type == 'wet':
            if stacks != {}:
                raise common.BuildException(
                    "Catkin (wet) package %s depends on (dry) stack(s):\n%s" %
                    (build_repo, '- ' + '\n- '.join(stacks)))
            # take only wet packages
            repo_build_dependencies = common.get_nonlocal_dependencies(
                catkin_packages, {}, {},
                build_depends=True,
                test_depends=False)
        else:  # dry build repo
            # take all packages
            repo_build_dependencies = common.get_nonlocal_dependencies(
                catkin_packages,
                stacks, {},
                build_depends=True,
                test_depends=False)
        repo_build_dependencies = [
            dep for dep in repo_build_dependencies if dep not in fulfilled_deps
        ]

    print ""
    print "Found the following packages"
    print "  wet packages:     ", catkin_packages.keys()
    print "  dry stacks:       ", stacks.keys()
    print "  dry packages:     ", manifest_packages.keys()
    print ""

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    # get all folders in repo_checkoutspace
    checkoutspace_dirs = [
        name for name in os.listdir(repo_checkoutspace)
        if os.path.isdir(os.path.join(repo_checkoutspace, name))
    ]
    for dir in checkoutspace_dirs:
        if dir in catkin_packages.keys():  # wet repo with metapackage
            shutil.move(os.path.join(repo_checkoutspace, dir),
                        os.path.join(repo_sourcespace_wet, dir))
        elif build_repo_type == 'wet' and dir == build_repo:  # wet repo without metapackage
            shutil.move(os.path.join(repo_checkoutspace, dir),
                        os.path.join(repo_sourcespace_wet, dir))
        elif dir in stacks.keys():  # dry repo with stack
            shutil.move(os.path.join(repo_checkoutspace, dir),
                        os.path.join(repo_sourcespace_dry, dir))
        #TODO elif: # dry repo without stack
        #else:
        #    raise common.BuildException("Could not separate %s into wet or dry sourcespace." %dir)
    # remove checkout dir
    common.call("rm -rf %s" % repo_checkoutspace)

    # setup ros workspace
    print "Set up ros workspace and setup environment variables"
    ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    common.call("rosws init %s /opt/ros/%s" % (repo_sourcespace, ros_distro),
                verbose=False)  # init workspace pointing to ros_distro

    # for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
    if os.path.isdir("/u/robot/git/care-o-bot"):
        common.call("rosws merge -t %s /u/robot/git/care-o-bot" %
                    repo_sourcespace,
                    verbose=False)  # merge robot account workspace

    common.call("rosws merge -t %s %s/wet/devel" %
                (repo_sourcespace, repo_sourcespace),
                verbose=False)  # merge wet workspace
    common.call("rosws merge -t %s %s/dry" %
                (repo_sourcespace, repo_sourcespace),
                verbose=False)  # merge dry workspace
    ros_env_repo = common.get_ros_env(
        repo_sourcespace + '/setup.bash')  # source wet and dry workspace

    ############################
    ### install dependencies ###
    ############################
    time_install = datetime.datetime.now()
    print "=====> entering dependency installation step at", time_install

    # Create rosdep object
    rosdep_resolver = None
    print "Create rosdep object"
    try:
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)
    except:  # when init fails the first time
        from time import sleep
        sleep(10)
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    print "Install build dependencies: %s" % (
        ', '.join(repo_build_dependencies))
    missing_packages = common.apt_get_check_also_nonrosdep(
        repo_build_dependencies, ros_distro, rosdep_resolver)
    if len(missing_packages) > 0:
        raise common.BuildException(
            "Some dependencies are missing. Please ask your administrator to install the following packages: %s"
            % missing_packages)
    print "All denendencies already installed."

    #############
    ### build ###
    #############
    time_build = datetime.datetime.now()
    print "=====> entering build step at", time_build

    # get amount of cores available on host system
    cores = multiprocessing.cpu_count()

    ### catkin repositories
    if catkin_packages != {}:
        print "Build wet packages: ", catkin_packages.keys()
        try:
            common.call("catkin_make --directory %s/wet" % repo_sourcespace,
                        ros_env_repo)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException(
                "Failed to catkin_make wet repositories")

    ### rosbuild repositories
    if build_repo_type == 'dry':
        # build dry repositories
        print "Build dry stacks:   ", stacks.keys()
        print "Build dry packages: ", manifest_packages.keys()
        packages_to_build = " ".join(stacks.keys()) + " ".join(
            manifest_packages.keys())
        try:
            common.call(
                "rosmake -rV --skip-blacklist --profile --pjobs=%s --output=%s %s"
                % (cores + 1, repo_build_logs, packages_to_build),
                ros_env_repo)
        except common.BuildException as ex:
            try:
                shutil.move(repo_build_logs,
                            os.path.join(workspace, "build_logs"))
            finally:
                print ex.msg
                raise common.BuildException(
                    "Failed to rosmake dry repositories")

    ###########
    ### end ###
    ###########
    # for hardware builds: create sym link to new build
    common.call("rm -f %s/../current" % workspace)
    common.call("ln -sf %s %s/../current" % (workspace, workspace))

    # steps: parsing, checkout, install, analysis, build, finish
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_checkout - time_parsing)
    print "checkout in                ", (time_install - time_checkout)
    print "install dependencies in    ", (time_build - time_install)
    print "build in                   ", (time_finish - time_build)
    print "total                      ", (time_finish - time_parsing)
    print ""
    print "For manual testing, please run the following line in your testing terminal"
    print "source " + repo_sourcespace + "/setup.bash"
    print ""
Exemplo n.º 2
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')                      # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src')    # wet (catkin) repositories
    os.makedirs(repo_sourcespace_wet)
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')           # dry (rosbuild) repositories
    os.makedirs(repo_sourcespace_dry)
    repo_static_analysis_results = os.path.join(tmpdir, 'src_repository', 'static_analysis_results') # location for static code test results
    os.makedirs(repo_static_analysis_results)
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')              # location for build logs
    os.makedirs(dry_build_logs)

    # setup ros workspace and setup environment variables
    print "Set up ros workspace and setup environment variables"
    ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env_repo)
    os.chdir(repo_sourcespace_wet + "/..")
    common.call("catkin_make", envir = ros_env_repo, verbose = False) # this means running "make" on an empty catkin workspace to generate a setup.bash
    ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace_wet, '../devel/setup.bash')) # source catkin workspace
    ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace_dry, os.environ['ROS_PACKAGE_PATH']]) # add dry path to ROS_PACKAGE_PATH

    ################
    ### checkout ###
    ################
    time_checkout = datetime.datetime.now()
    print "=====> entering checkout step at", time_checkout

    # download build_repo from source #
    print "Creating rosinstall file for repository %s" % build_repo
    rosinstall = ""
    if build_identifier in pipe_repos:  # check if triggering identifier is really present in pipeline config
        rosinstall += pipe_repos[build_identifier].get_rosinstall()
    else:
        err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
        raise common.BuildException(err_msg)

    # write rosinstall file
    print "Rosinstall file for repository: \n %s" % rosinstall
    with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
        f.write(rosinstall)
    print "Install repository from source:"
    # rosinstall repos
    common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                % (repo_sourcespace, workspace, ros_distro))

    # get the repositories build dependencies
    print "Get build dependencies of repo"

    # get all packages in sourcespace
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)

    # (debug) output
    if options.verbose:
        print "Packages in %s:" % repo_sourcespace
        print "Catkin: ", catkin_packages
        print "Rosbuild:\n  Stacks: ", stacks
        print "  Packages: ", manifest_packages

        # get deps directly for catkin (like in willow code)
        try:
            print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
        except:
            pass
        # deps catkin
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
        # deps stacks
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
        print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))

    # check if build_repo is wet or dry and get all corresponding deps
    build_repo_type = ''
    if build_repo in catkin_packages: # wet repo with metapackage
        print "repo %s is wet" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif (build_repo not in catkin_packages) and (build_repo not in stacks) and (catkin_packages != []): # wet repo without metapackage
        print "repo %s is wet without metapackage" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif build_repo in stacks: # dry repo with stack
        print "repo %s is dry" % build_repo
        build_repo_type = 'dry'
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
    #TODO elif : # dry repo without stack
    else: # build_repo is neither wet nor dry
        raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)

    # install user-defined/customized dependencies of build_repo from source
    rosinstall = ''
    fulfilled_deps = []
    for dep in repo_build_dependencies:
        if dep in pipe_repos[build_identifier].dependencies:
            print "Install user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # install additional, indirect user-defined dependencies
    for dep in pipe_repos[build_identifier].dependencies:
        if dep not in fulfilled_deps:
            print "Install additional user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # check if all user-defined/customized dependencies are satisfied
    if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
        print "Not all user-defined build dependencies are fulfilled"
        print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
        print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
        raise common.BuildException("Not all user-defined build dependencies are fulfilled")

    if rosinstall != '':
        # write .rosinstall file
        print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print "Install user-defined build dependencies from source"
        # rosinstall depends
        common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                    % (repo_sourcespace, workspace, ros_distro))

        # get also deps of just installed user-defined/customized dependencies
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
        if build_repo_type == 'wet':
            if stacks != {}:
                raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                            % (build_repo, '- ' + '\n- '.join(stacks)))
            # take only wet packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        else:  # dry build repo
            # take all packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
        repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    # get all folders in repo_sourcespace
    sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
    for dir in sourcespace_dirs:
        if dir in catkin_packages.keys(): # wet repo with metapackage
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
        elif build_repo_type == 'wet' and dir == build_repo: # wet repo without metapackage
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
        elif dir in stacks.keys(): # dry repo with stack
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))
        #TODO elif: # dry repo without stack
        #else:
        #    raise common.BuildException("Could not separate %s into wet or dry sourcespace." %dir) 

    ############################
    ### install dependencies ###
    ############################
    time_install = datetime.datetime.now()
    print "=====> entering dependency installation step at", time_install

    # Create rosdep object
    rosdep_resolver = None
    print "Create rosdep object"
    try:
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)
    except:  # when init fails the first time
        from time import sleep
        sleep(10)
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
    common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)

    #print "Install wet rosdep dependencies" # TODO: this throws an error if the dependency is not a system dependency or a released package. How can we handle wet dependencies which are in the same local workspace?
    #for dir in catkin_packages.keys():
    #    common.call("rosdep install -y %s" % dir, ros_env_repo)

    print "Install dry rosdep dependencies"
    for dir in stacks.keys():
        common.call("rosdep install -y %s" % dir, ros_env_repo)

    #######################
    ### static analysis ###
    #######################
    time_analysis = datetime.datetime.now()
    print "=====> entering static analysis step at", time_analysis

    #TODO
    # Cpplint
    # Counting lines of code/comments
    # cppcheck
    cppcheck.run([repo_sourcespace_wet, repo_sourcespace_dry], repo_static_analysis_results)
    # Coverage

    # create tests results directory in workspace
    os.mkdir(workspace + "/static_analysis_results")
    
    # copy test results
    common.copy_static_analysis_results(repo_static_analysis_results, workspace + "/static_analysis_results")

    #############
    ### build ###
    #############
    time_build = datetime.datetime.now()
    print "=====> entering build step at", time_build

    ### catkin repositories
    if catkin_packages != {}:
        os.chdir(repo_sourcespace_wet + "/..")
        try:
            common.call("catkin_make", ros_env_repo)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to catkin_make wet repositories")

    ### rosbuild repositories
    if build_repo_type == 'dry':
        # build dry repositories
        print "Build repository %s" % build_repo
        try:
            common.call("rosmake -rV --skip-blacklist --profile --pjobs=8 --output=%s %s" %
                        (dry_build_logs, build_repo), ros_env_repo)
        except common.BuildException as ex:
            try:
                shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
            finally:
                print ex.msg
                raise common.BuildException("Failed to rosmake %s" % build_repo)

    # the end (steps: parsing, checkout, install, analysis, build, finish)
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_checkout - time_parsing)
    print "checkout in                ", (time_install - time_checkout)
    print "install dependencies in    ", (time_analysis - time_install)
    print "static code analysis in    ", (time_build - time_analysis)
    print "build in                   ", (time_finish - time_build)
    print "total                      ", (time_finish - time_parsing)
    print ""
Exemplo n.º 3
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[
            0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]  # repository + suffix
    build_repo = build_identifier.split('__')[0]  # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner,
                                      server_name,
                                      user_name,
                                      file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro: %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = '/tmp'
    repo_sourcespace = os.path.join(tmpdir,
                                    'src')  # location to build repositories in
    repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet',
                                        'src')  # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(repo_sourcespace,
                                        'dry')  # dry (rosbuild) repositories
    repo_test_results = os.path.join(
        tmpdir, 'test_results')  # location for test results
    if not os.path.exists(repo_test_results):
        os.makedirs(repo_test_results)
    repo_build_logs = os.path.join(tmpdir,
                                   'build_logs')  # location for build logs

    # source wet and dry workspace
    ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')
    ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results

    ############
    ### test ###
    ############
    time_test = datetime.datetime.now()
    print "=====> entering testing step at", time_test

    # get amount of cores available on host system
    cores = multiprocessing.cpu_count()

    # get current repository name
    user, repo_name = cp_instance.split_github_url(
        pipe_repos[build_identifier].data['url'])

    ### catkin repositories
    (catkin_packages, stacks,
     manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
    if (len(catkin_packages) > 0) and (repo_name in catkin_packages):
        os.environ['ROS_TEST_RESULTS_DIR'] = os.environ[
            'CATKIN_TEST_RESULTS_DIR']

        # get list of dependencies to test
        test_repos_list_wet = []

        # add all packages in main repository
        (catkin_test_packages, stacks, manifest_packages
         ) = common.get_all_packages(catkin_packages[repo_name] + '/..')
        for pkg_name, pkg_dir in catkin_test_packages.items():
            test_repos_list_wet.append(pkg_name)

        # add all packages in dep repositories (if "test" is set to True)
        for dep in pipe_repos[build_identifier].dependencies.keys():
            if pipe_repos[build_identifier].dependencies[dep].test:
                (catkin_test_dep_packages, stacks, manifest_packages
                 ) = common.get_all_packages(catkin_packages[dep] + '/..')
                for pkg_name, pkg_dir in catkin_test_dep_packages.items():
                    test_repos_list_wet.append(pkg_name)

        # test wet repositories
        print "Testing the following wet repositories %s" % test_repos_list_wet
        test_list = ' '.join(test_repos_list_wet)
        try:
            common.call(
                "/opt/VirtualGL/bin/vglrun catkin_make --directory %s/wet test --pkg %s"
                % (repo_sourcespace, test_list), ros_env_repo)
        finally:
            # clean and copy test xml files
            common.clean_and_copy_test_results(
                repo_sourcespace + "/wet/build/test_results",
                workspace + "/test_results"
            )  # FIXME: is there a way to let catkin write test results to repo_test_results

    ### rosbuild repositories
    (catkin_packages, stacks,
     manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
    if (len(stacks) > 0) and (repo_name in stacks):
        # get list of dependencies to test
        test_repos_list_dry = [build_repo]
        for dep, depObj in pipe_repos[build_identifier].dependencies.items():
            if depObj.test and dep in stacks:
                test_repos_list_dry.append(dep)

        # test dry repositories
        packages_to_test_list = test_repos_list_dry
        for repo in test_repos_list_dry:
            (catkin_packages, stacks, manifest_packages
             ) = common.get_all_packages(repo_sourcespace_dry + "/" + repo)
            packages_to_test_list = packages_to_test_list + manifest_packages.keys(
            )
        print "Test dry packages: ", packages_to_test_list
        packages_to_test = " ".join(test_repos_list_dry) + " " + " ".join(
            packages_to_test_list)
        common.call(
            "/opt/VirtualGL/bin/vglrun rosmake -rV --skip-blacklist --profile --pjobs=%s --test-only --output=%s %s"
            % (cores, repo_build_logs, packages_to_test), ros_env_repo)

        # clean and copy test xml files
        common.clean_and_copy_test_results(repo_test_results,
                                           workspace + "/test_results")

    # in case we have no tests executed (neither wet nor dry), we'll generate some dummy test result
    common.clean_and_copy_test_results(repo_test_results,
                                       workspace + "/test_results")

    ###########
    ### end ###
    ###########
    # steps: parsing, test
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_test - time_parsing)
    print "test in                    ", (time_finish - time_test)
    print "total                      ", (time_finish - time_parsing)
    print ""
Exemplo n.º 4
0
def build_downstream_post_fuerte(ros_distro, build_repo, workspace, server):
    ros_package_path = os.environ['ROS_PACKAGE_PATH']
    b_r_short = build_repo.split('__')[0]

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')
    repo_buildspace = os.path.join(tmpdir, 'build_repository')
    dependson_sourcespace = os.path.join(tmpdir, 'src_depends_on')
    dependson_sourcespace_wet = os.path.join(tmpdir, 'src_depends_on', 'wet')
    dependson_sourcespace_dry = os.path.join(tmpdir, 'src_depends_on', 'dry')
    dependson_buildspace = os.path.join(tmpdir, 'build_depend_on')
    dry_test_results_dir = os.path.join(dependson_sourcespace_dry,
                                        'test_results')

    # TODO clean up test result dirs

    # get depends on
    print "Get list of released repositories that (build-)depend on repository %s" % b_r_short
    ros_depends_on = []
    distro_ros = rosdistro.RosDistro(ros_distro, 'http://%s/~jenkins' % server)
    for d in distro_ros.get_depends_on([
            b_r_short
    ])['build'] + distro_ros.get_depends_on([b_r_short])['buildtool']:
        if not d in ros_depends_on and d != b_r_short:
            ros_depends_on.append(d)
    # TODO get_depends_on of cob_distro release (only for intern repos necessary)
    cob_depends_on = []
    #distro_cob = cob_distro.CobDistro(ros_distro)
    #for d in distro_cob.get_depends_on([b_r_short])['build'] + distro_cob.get_depends_on([b_r_short])['buildtool']:
    #    if not d in cob_depends_on and d != b_r_short:
    #        cob_depends_on.append(d)
    if len(ros_depends_on + cob_depends_on) == 0:
        print "No repository depends on repository %s. Test finished" % b_r_short
        return

    print "Build depends_on list of repository %s:\n - %s" % (
        '\n - '.join(ros_depends_on + cob_depends_on))

    # install depends_on repository from source
    rosinstall = distro_ros.get_rosinstall(ros_depends_on)
    #rosinstall += distro_cob.get_rosinstall(cob_depends_on) TODO
    print "Rosinstall for depends_on:\n %s" % rosinstall
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print "Created rosinstall file for depends on"

    # install all repository and system dependencies of the depends_on list
    print "Install all depends_on from source: %s" % (
        ', '.join(ros_depends_on))
    os.makedirs(dependson_sourcespace)
    common.call("rosinstall %s %s/depends_on.rosinstall /opt/ros/%s" %
                (dependson_sourcespace, workspace, ros_distro))

    # all packages in dependson_sourcespace
    (catkin_packages, stacks,
     manifest_packages) = common.get_all_packages(dependson_sourcespace)
    print catkin_packages
    print stacks
    print manifest_packages

    # get build and test dependencies of depends_on list
    dependson_build_dependencies = []
    for d in common.get_nonlocal_dependencies(catkin_packages,
                                              stacks, {},
                                              build_depends=True,
                                              test_depends=False):
        print "  Checking dependency %s" % d
        if d in dependson_build_dependencies:
            print "    Already in dependson_build_dependencies"
        elif d in ros_depends_on or d in cob_depends_on:
            print "    Is a direct dependency of the repository, and is installed from source"
        elif d == b_r_short:
            print "    Is the tested repository"
        else:
            dependson_build_dependencies.append(d)
    print "Build dependencies of depends_on list are %s" % (
        ', '.join(dependson_build_dependencies))

    dependson_test_dependencies = []
    for d in common.get_nonlocal_dependencies(catkin_packages,
                                              stacks, {},
                                              build_depends=False,
                                              test_depends=True):
        if d not in dependson_test_dependencies + ros_depends_on + cob_depends_on and d != b_r_short:
            dependson_test_dependencies.append(d)
    print "Test dependencies of depends_on list are %s" % (
        ', '.join(dependson_test_dependencies))

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    os.makedirs(dependson_sourcespace_wet)
    os.makedirs(dependson_sourcespace_dry)
    # get all folders in dependson_sourcespace
    sourcespace_dirs = [
        name for name in os.listdir(dependson_sourcespace)
        if os.path.isdir(os.path.join(dependson_sourcespace, name))
    ]
    for dir in sourcespace_dirs:
        if dir in catkin_packages.keys():
            shutil.move(os.path.join(dependson_sourcespace, dir),
                        os.path.join(dependson_sourcespace_wet, dir))
        if dir in stacks.keys():
            shutil.move(os.path.join(dependson_sourcespace, dir),
                        os.path.join(dependson_sourcespace_dry, dir))

    # Create rosdep object
    print "Create rosdep object"
    try:
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)
    except:
        from time import sleep
        sleep(10)
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    # install build dependencies
    print "Install all build dependencies of the depends_on list: %s" % (
        ', '.join(dependson_build_dependencies))
    common.apt_get_install_also_nonrosdep(dependson_build_dependencies,
                                          rosdep_resolver)

    # env
    print "Setting up environment"
    ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    common.call("env", ros_env)
    ros_env_dependson = common.get_ros_env(
        os.path.join(repo_sourcespace, 'setup.bash'))
    ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join(
        [dependson_buildspace, repo_buildspace, ros_package_path])
    common.call("env", ros_env_dependson)

    ### catkin repositories
    if catkin_packages != {}:
        os.makedirs(dependson_buildspace)
        os.chdir(dependson_buildspace)
        print "Create a new CMakeLists.txt for catkin packages"
        if ros_distro == 'fuerte':
            common.call(
                "ln -s %s %s" %
                (os.path.join(dependson_sourcespace_wet, 'catkin', 'cmake',
                              'toplevel.cmake'),
                 os.path.join(dependson_sourcespace_wet, 'CMakeLists.txt')))
        else:
            common.call("catkin_init_workspace %s" % dependson_sourcespace_wet,
                        ros_env_dependson)

        try:
            common.call("cmake %s" % dependson_sourcespace_wet,
                        ros_env_dependson)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to cmake wet repositories")
        #ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories
        print "Build wet depends_on list"
        try:
            common.call("make", ros_env_dependson)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to make wet packages")

        if dependson_test_dependencies != []:
            # install test dependencies
            print "Install all test dependencies of the depends_on list: %s" % (
                ', '.join(dependson_test_dependencies))
            common.apt_get_install_also_nonrosdep(dependson_test_dependencies,
                                                  rosdep_resolver)

            # test repositories
            try:
                common.call("make run_tests", ros_env_dependson)
            except common.BuildException as ex:
                print ex.msg

        # copy test results
        common.copy_test_results(workspace, dependson_buildspace)

    ### rosbuild repositories
    if stacks != {}:
        # env
        print "Setting up environment"
        ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([
            dependson_buildspace, repo_buildspace, dependson_sourcespace_dry,
            repo_sourcespace_dry, ros_package_path
        ])
        common.call("env", ros_env_dependson)

        #print "Make rosdep"
        #common.call("rosmake rosdep", ros_env)
        #for stack in stacks.keys():
        #    common.call("rosdep install -y %s" % stack, ros_env_repo)

        dependson_sourcespace_dry_dirs = [
            name for name in os.listdir(dependson_sourcespace_dry)
            if os.path.isdir(os.path.join(dependson_sourcespace_dry, name))
        ]
        print "Build dry depends_on repositories:\n - %s" % '\n - '.join(
            dependson_sourcespace_dry_dirs)
        os.mkdir(dry_test_results_dir)
        for dry_dependson in dependson_sourcespace_dry_dirs:
            try:
                common.call(
                    "rosmake -rV --profile --pjobs=8 --output=%s %s" %
                    (dry_test_results_dir, b_r_short), ros_env_dependson)
            except:
                raise common.BuildException("Failed to rosmake %s" % b_r_short)
            try:
                common.call(
                    "rosmake -rV --profile --pjobs=8 --test-only --output=%s %s"
                    % (dry_test_results_dir, b_r_short), ros_env_dependson)
                # TODO output dir ??
            except:
                print "Failed to test %s" % dry_dependson
        # copy test results
        common.call("rosrun rosunit clean_junit_xml.py", ros_env_dependson)
        common.copy_test_results(workspace, dependson_sourcespace_dry)
Exemplo n.º 5
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[
            0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]  # repository + suffix
    build_repo = build_identifier.split('__')[0]  # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner,
                                      server_name,
                                      user_name,
                                      file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(
        tmpdir, 'src_repository')  # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet',
                                        'src')  # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository',
                                        'dry')  # dry (rosbuild) repositories
    repo_test_results_dry = os.path.join(
        tmpdir, 'src_repository',
        'test_results')  # location for dry test results
    repo_test_results_wet = os.path.join(
        tmpdir, 'src_repository', 'wet', 'build',
        'test_results')  # location for wet test results
    #repo_buildspace = os.path.join(tmpdir, 'build_repository')                                        # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry,
                                  'build_logs')  # location for build logs

    # get environment variables
    ros_env_repo = common.get_ros_env(
        os.path.join(repo_sourcespace, 'wet', 'devel', 'setup.bash'))
    ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join(
        [repo_sourcespace, ros_package_path])

    ############
    ### test ###
    ############
    time_test = datetime.datetime.now()
    print "=====> entering testing step at", time_test

    # FIXME: HACK BEGIN

    common.call("ls", envir=ros_env_repo, verbose=False)
    common.call("roslaunch fiad_scenario_system system.launch",
                envir=ros_env_repo,
                verbose=False)

    # FIXME: HACK END

    # the end (steps: parsing, test)
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_test - time_parsing)
    print "test in                    ", (time_finish - time_test)
    print "total                      ", (time_finish - time_parsing)
    print ""
Exemplo n.º 6
0
def main():
    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 6:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo graphic_test" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    graphic_test = True if args[5] == "true" else False  # TODO optional
    build_repo_only = True if args[6] == "true" else False
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print datetime.datetime.now()
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    print "Graphic Test: %s" % graphic_test
    print "Build Repo Only: %s" % build_repo_only
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')               # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet')    # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')    # dry (rosbuild) repositories
    repo_buildspace = os.path.join(tmpdir, 'build_repository')              # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')       # location for build logs

    if ros_distro != 'electric':
        # Create rosdep object
        print "Create rosdep object"
        try:
            rosdep_resolver = rosdep.RosDepResolver(ros_distro)
        except:  # when init fails the first time
            from time import sleep
            sleep(10)
            rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    # env
    print "Set up ros environment variables"
    ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    if options.verbose:
        common.call("env", ros_env)

    ### catkin repositories
    print datetime.datetime.now()

    if os.listdir(repo_sourcespace_wet):
        # set up catkin workspace
        #if ros_distro == 'fuerte':
            #if 'catkin' not in catkin_packages.keys():
                ## add catkin package to rosinstall
                #rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
                #print "Install catkin"
                ## rosinstall catkin
                #common.call("rosinstall --verbose %s %s/repo.rosinstall /opt/ros/%s"
                            #% (repo_sourcespace_wet, workspace, ros_distro))

            #print "Create a CMakeLists.txt for catkin packages"
            #common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
                                         #os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
        #else:
            #common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env)

        if not os.path.isdir(repo_buildspace):
            os.mkdir(repo_buildspace)
        os.chdir(repo_buildspace)

        # test repositories
        print "Test wet repository list"
        test_error_msg = None
        try:
            common.call("make tests", ros_env)
        except common.BuildException as ex:
            print ex.msg
            test_error_msg = ex.msg

        # get wet repositories test and run dependencies
        print "Get test and run dependencies of repo list"
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
        if stacks != {}:
            raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                        % (build_repo, '- ' + '\n- '.join(stacks)))
        # take only wet packages
        repo_test_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=False, test_depends=True)
        if repo_test_dependencies != [] and test_error_msg is None:
            print "Install test and run dependencies of repository list: %s" % (', '.join(repo_test_dependencies))
            common.apt_get_install_also_nonrosdep(repo_test_dependencies, ros_distro, rosdep_resolver)

            # run tests
            print "Test repository list"
            try:
                common.call("%smake run_tests" % ("/opt/VirtualGL/bin/vglrun " if graphic_test else ""), ros_env)  # TODO check how to test a list of repos
            except common.BuildException as ex:
                print ex.msg
                test_error_msg = ex.msg

        # copy test results
        common.copy_test_results(workspace, repo_buildspace, test_error_msg)

    ### rosbuild repositories
    print datetime.datetime.now()
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
    if build_repo in stacks:
        # get list of dependencies to test
        test_repos_list = []
        for dep in pipe_repos[build_identifier].dependencies:
            if dep in stacks:  # TODO option to select deps to build
                test_repos_list.append(dep)

        ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
        ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
        if options.verbose:
            common.call("env", ros_env_repo)

        # test dry repositories
        print "Test repository %s" % build_repo
        try:
            build_list = " ".join(test_repos_list + [build_repo])
            if build_repo_only:
                build_list = build_repo
            common.call("%srosmake -rV --profile %s--test-only --output=%s %s" %
                        ("/opt/VirtualGL/bin/vglrun " if graphic_test else "",
                         "--pjobs=8 " if not graphic_test else "",
                         dry_build_logs, build_list), ros_env_repo)
        except common.BuildException as ex:
            print ex.msg

        # copy test results
        common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
        for file in os.listdir(os.path.join(repo_sourcespace, "test_results")):
            file_path = os.path.join(repo_sourcespace, "test_results", file)
            print file_path
            try:
                if not file.startswith("_hudson"):
                    shutil.rmtree(file_path)
            except Exception as e:
                print e

        common.copy_test_results(workspace, repo_sourcespace)
        print datetime.datetime.now()
        try:
            shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
        except IOError as ex:
            print "No build logs found: %s" % ex
Exemplo n.º 7
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro: %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = '/tmp'
    repo_sourcespace = os.path.join(tmpdir, 'src')                                 # location to build repositories in
    repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet', 'src')            # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(repo_sourcespace, 'dry')                   # dry (rosbuild) repositories
    repo_test_results = os.path.join(tmpdir, 'test_results')                       # location for test results
    if not os.path.exists(repo_test_results):
        os.makedirs(repo_test_results)
    repo_build_logs = os.path.join(tmpdir, 'build_logs')                           # location for build logs

    # source wet and dry workspace
    ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')
    ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results

    ############
    ### test ###
    ############
    time_test = datetime.datetime.now()
    print "=====> entering testing step at", time_test

    # get amount of cores available on host system
    cores = multiprocessing.cpu_count()

    # get current repository name    
    user, repo_name = cp_instance.split_github_url(pipe_repos[build_identifier].data['url'])

    ### catkin repositories
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
    if (len(catkin_packages) > 0) and (repo_name in catkin_packages):
        # get list of dependencies to test
        test_repos_list_wet = []

        # add all packages in main repository
        (catkin_test_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[repo_name] + '/..')
        for pkg_name, pkg_dir in catkin_test_packages.items():
            test_repos_list_wet.append(pkg_name)

        # add all packages in dep repositories (if "test" is set to True)
        for dep in pipe_repos[build_identifier].dependencies.keys():
            if pipe_repos[build_identifier].dependencies[dep].test:
                (catkin_test_dep_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[dep] + '/..')
                for pkg_name, pkg_dir in catkin_test_dep_packages.items():
                    test_repos_list_wet.append(pkg_name)

        # test wet repositories
        print "Testing the following wet repositories %s" % test_repos_list_wet
        test_list = ' '.join( test_repos_list_wet )
        common.call( "catkin_make --directory %s/wet test --pkg %s" % (repo_sourcespace, test_list), ros_env_repo)

        # clean and copy test xml files
        common.clean_and_copy_test_results(repo_sourcespace + "/wet/build/test_results", workspace + "/test_results") # FIXME: is there a way to let catkin write test results to repo_test_results

    ### rosbuild repositories
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
    if (len(stacks) > 0) and (repo_name in stacks):
        # get list of dependencies to test
        test_repos_list_dry = [build_repo]
        test_repos_list_dry.append(' ') # Solves the bug: [ rosmake ] WARNING: Skipped command line arguments: ['cob_navigation_testsnavigation_test_skeleton'] because they could not be resolved to a stack name or a package name.
        for dep, depObj in pipe_repos[build_identifier].dependencies.items():
            if depObj.test and dep in stacks:
                test_repos_list_dry.append(dep)

        # test dry repositories
        packages_to_test_list = test_repos_list_dry
        for repo in test_repos_list_dry:
            (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry + "/" + repo)
            packages_to_test_list = packages_to_test_list + manifest_packages.keys()
        print "Test dry packages: ", packages_to_test_list
        packages_to_test = " ".join(test_repos_list_dry) + " " + " ".join(packages_to_test_list)
        common.call("rosmake -rV --skip-blacklist --profile --pjobs=%s --test-only --output=%s %s" %
                    ( cores, repo_build_logs, packages_to_test ), ros_env_repo)

        # clean and copy test xml files
        common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")
        
    # in case we have no tests executed (neither wet nor dry), we'll generate some dummy test result
    common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")

    ###########
    ### end ###
    ###########
    # steps: parsing, test
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_test - time_parsing)
    print "test in                    ", (time_finish - time_test)
    print "total                      ", (time_finish - time_parsing)
    print ""
Exemplo n.º 8
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print datetime.datetime.now()
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    print "Graphic Test: True"
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')                                         # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src')                       # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')                              # dry (rosbuild) repositories
    repo_test_results_dry = os.path.join(tmpdir, 'src_repository', 'test_results')                    # location for dry test results
    repo_test_results_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'build', 'test_results')    # location for wet test results
    #repo_buildspace = os.path.join(tmpdir, 'build_repository')                                        # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')                                 # location for build logs

    # get environment variables    
    ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
    ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
    if options.verbose:
        common.call("env", ros_env_repo)

    ############
    ### test ###
    ############
    time_test = datetime.datetime.now()
    print "=====> entering testing step at", time_test

    ### catkin repositories
    print "test catkin repositories"
    ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results_wet
    if os.listdir(repo_sourcespace_wet):
        os.chdir(repo_sourcespace_wet + "/..")

        # get wet repositories test and run dependencies
        #print "Get test and run dependencies of repo list"
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
        # get list of dependencies to test
        test_repos_list_wet = []
        for dep, depObj in pipe_repos[build_identifier].dependencies.items():
            if depObj.test and dep in catkin_packages:
                test_repos_list_wet.append(dep)

        print "Test the following wet repositories %s" % test_repos_list_wet

        #test_error_msg = None
        try:
            test_list = ' '.join( test_repos_list_wet )
            if test_list:
                common.call( "/opt/VirtualGL/bin/vglrun catkin_make test --pkg %s" % test_list, ros_env_repo)

        except common.BuildException as ex:
            print ex.msg
            #print traceback.format_exc()
        #    test_error_msg = ex.msg

        # clean test xml files
        common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
        for file in os.listdir(os.path.join(repo_test_results_wet)):
            file_path = os.path.join(repo_test_results_wet, file)
            try:
                if not file.startswith("_hudson"):
                    shutil.rmtree(file_path)
            except Exception as e:
                print e

        # copy wet test results
        common.copy_test_results(repo_test_results_wet, workspace + "/test_results")

    ### rosbuild repositories
    print "test rosbuild repositories"
    if not os.path.isdir(repo_test_results_dry):
        os.mkdir(repo_test_results_dry)
    ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results_dry
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
    if build_repo in stacks:
        # get list of dependencies to test
        test_repos_list_dry = []
        for dep, depObj in pipe_repos[build_identifier].dependencies.items():
            if depObj.test and dep in stacks:
                test_repos_list_dry.append(dep)

        # test dry repositories
        print "Test the following dry repositories %s" % test_repos_list_dry
        try:
            build_list = " ".join(test_repos_list_dry + [build_repo])
            common.call("/opt/VirtualGL/bin/vglrun rosmake -rV --profile --test-only --output=%s %s" %
                        ( dry_build_logs, build_list ), ros_env_repo)
        except common.BuildException as ex:
            print ex.msg

        # clean test xml files
        common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
        for file in os.listdir(os.path.join(repo_test_results_dry)):
            file_path = os.path.join(repo_test_results_dry, file)
            try:
                if not file.startswith("_hudson"):
                    shutil.rmtree(file_path)
            except Exception as e:
                print e

        # copy dry test results
        common.copy_test_results(repo_test_results_dry, workspace + "/test_results")
    
    # the end (steps: parsing, test)
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_test - time_parsing)
    print "test in                    ", (time_finish - time_test)
    print "total                      ", (time_finish - time_parsing)
    print ""
Exemplo n.º 9
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    print "=====> WORKAROUND INSTALLATION ..."
    common.call("sudo apt-get install -y python-numpy python-scipy python-matplotlib python-pygraphviz python-tk ros-hydro-controller-manage* ros-hydro-joint-* ros-hydro-desktop-full ros-hydro-gazebo-ros-control ros-hydro-ros-control")
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro: %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = '/tmp'
    repo_sourcespace = os.path.join(tmpdir, 'src')                                 # location to build repositories in
    repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet', 'src')            # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(repo_sourcespace, 'dry')                   # dry (rosbuild) repositories
    repo_test_results = os.path.join(tmpdir, 'test_results')                       # location for test results
    if not os.path.exists(repo_test_results):
        os.makedirs(repo_test_results)
    repo_build_logs = os.path.join(tmpdir, 'build_logs')                           # location for build logs

    # source wet and dry workspace
    ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')
    ros_env_repo['ROS_TEST_RESULTS_DIR'] = repo_test_results

    ############
    ### test ###
    ############
    time_test = datetime.datetime.now()
    print "=====> entering testing step at", time_test

    # get amount of cores available on host system
    cores = multiprocessing.cpu_count()

    # get current repository name    
    user, repo_name = cp_instance.split_github_url(pipe_repos[build_identifier].data['url'])

    ### catkin repositories
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
    if (len(catkin_packages) > 0) and (repo_name in catkin_packages):
        # get list of dependencies to test
        test_repos_list_wet = []

        # add all packages in main repository
        (catkin_test_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[repo_name] + '/..')
        for pkg_name, pkg_dir in catkin_test_packages.items():
            test_repos_list_wet.append(pkg_name)

        # add all packages in dep repositories (if "test" is set to True)
        for dep in pipe_repos[build_identifier].dependencies.keys():
            if pipe_repos[build_identifier].dependencies[dep].test:
                (catkin_test_dep_packages, stacks, manifest_packages) = common.get_all_packages(catkin_packages[dep] + '/..')
                for pkg_name, pkg_dir in catkin_test_dep_packages.items():
                    test_repos_list_wet.append(pkg_name)

        print "Testing the following wet repositorie %s" % build_repo
        try:
            test_list = ' '.join( test_repos_list_wet )
            print "test_list: %s" % test_list
            common.call("catkin_make --directory %s/wet run_tests_%s" % (repo_sourcespace, build_repo))
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to catkin_make test wet repositories")

        # clean and copy test xml files
        common.clean_and_copy_test_results(repo_sourcespace + "/wet/build/test_results", workspace + "/test_results") # FIXME: is there a way to let catkin write test results to repo_test_results

    ### rosbuild repositories
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_dry)
    if (len(stacks) > 0) and (repo_name in stacks):
        # get list of dependencies to test
        test_repos_list_dry = [build_repo]
        for dep, depObj in pipe_repos[build_identifier].dependencies.items():
            if depObj.test and dep in stacks:
                test_repos_list_dry.append(dep)

        # test dry repositories
        print "Test the following dry repositories %s" % test_repos_list_dry
        try:
            build_list = " ".join(test_repos_list_dry)
            common.call("rosmake -rV --skip-blacklist --profile --pjobs=%s --test-only --output=%s %s" %
                        ( cores, repo_build_logs, build_list ), ros_env_repo)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to rosmake test dry repositories")

        # clean and copy test xml files
        common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")
        
    # in case we have no tests executed (neither wet nor dry), we'll generate some dummy test result
    common.clean_and_copy_test_results(repo_test_results, workspace + "/test_results")

    ################
    ### Analysis ###
    ################
    time_ana = datetime.datetime.now()
    print "=====> entering analysis step at", time_ana

    path_video = "$WORKSPACE/videoFiles"

    com_folder = "mkdir -p " + path_video
    common.call(com_folder)
   
    print "Files in $BAG_PATH:"
    print common.call(os.path.expandvars("ls -al $BAG_PATH"))
   
    analyse()

    print "=====> making images"
    com_pix = "rosrun navigation_test_analysis simple_viewer.py AUTO"
    common.call(com_pix)

    ###########
    ### end ###
    ###########
    # steps: parsing, test, analysis
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_test - time_parsing)
    print "test in                    ", (time_ana - time_test)
    print "analysis in                ", (time_finish - time_ana)
    print "total                      ", (time_finish - time_parsing)
    print ""
 
    # Trying to fix: Deleting project workspace... Cannot delete workspace: java.io.IOException: Unable to delete 
    common.call("sudo rm -rf $WORKSPACE/test_results/*")
Exemplo n.º 10
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')                                         # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src')                       # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')                              # dry (rosbuild) repositories
    repo_test_results_dry = os.path.join(tmpdir, 'src_repository', 'test_results')                    # location for dry test results
    repo_test_results_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'build', 'test_results')    # location for wet test results
    #repo_buildspace = os.path.join(tmpdir, 'build_repository')                                        # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')                                 # location for build logs

    # get environment variables    
    ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'wet', 'devel', 'setup.bash'))
    ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])

    ############
    ### test ###
    ############
    time_test = datetime.datetime.now()
    print "=====> entering testing step at", time_test
    
    # FIXME: HACK BEGIN
    
    common.call("ls", envir=ros_env_repo, verbose=False)
    common.call("roslaunch fiad_scenario_system system.launch", envir=ros_env_repo, verbose=False)
    
    # FIXME: HACK END
    
    # the end (steps: parsing, test)
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_test - time_parsing)
    print "test in                    ", (time_finish - time_test)
    print "total                      ", (time_finish - time_parsing)
    print ""
Exemplo n.º 11
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # cob_pipe object
    print "LIST:" + str(cob_pipe.CobPipe()) +"REPOOWN:" + str(pipeline_repos_owner)+ "SERVER:" + str(server_name) + "USER:"******"Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    #remove old and create new folder
    #while len(os.listdir(workspace)) >= 7:
    #    list = os.listdir(workspace)
    #    shutil.rmtree(workspace + "/" + sorted(list)[0]) #with common.call rm -rf later        
    #workspace = workspace + '/' + str(datetime.datetime.now())
    #print str(workspace)


    # set up directories variables
    tmpdir = os.path.join(workspace, 'test_repositories') #TODO check for old versions (delete oldest) and create new directory with timestamp
    common.call("rm -rf " + tmpdir)
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')                      # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet', 'src')    # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')           # dry (rosbuild) repositories
    repo_static_analysis_results = os.path.join(tmpdir, 'src_repository', 'static_analysis_results') # location for static code test results
    repo_buildspace = os.path.join(tmpdir, 'build_repository')                     # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')              # location for build logs

    ################
    ### checkout ###
    ################
    time_checkout = datetime.datetime.now()
    print "=====> entering checkout step at", time_checkout

    # download build_repo from source #
    print "Creating rosinstall file for repository %s" % build_repo
    rosinstall = ""
    if build_identifier in pipe_repos:  # check if triggering identifier is really present in pipeline config
        rosinstall += pipe_repos[build_identifier].get_rosinstall()
        rosinstall += "- other: {local-name: /u/robot}" #TODO: not hardcoding (with multiple directories?)
    else:
        err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
        raise common.BuildException(err_msg)

    # write rosinstall file
    print "Rosinstall file for repository: \n %s" % rosinstall
    with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
        f.write(rosinstall)
    print "Install repository from source:"
    # create repo sourcespace directory 'src_repository'
    os.makedirs(repo_sourcespace)
    # rosinstall repos
    common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                % (repo_sourcespace, workspace, ros_distro))

    # get the repositories build dependencies
    print "Get build dependencies of repo"

    # get all packages in sourcespace
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
    if ros_distro == 'electric' and catkin_packages != {}:
        raise common.BuildException("Found wet packages while building in ros electric")

    # (debug) output
    if options.verbose:
        print "Packages in %s:" % repo_sourcespace
        print "Catkin: ", catkin_packages
        print "Rosbuild:\n  Stacks: ", stacks
        print "  Packages: ", manifest_packages

        # get deps directly for catkin (like in willow code)
        try:
            print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
        except:
            pass
        # deps catkin
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
        # deps stacks
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
        print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))

    # check if build_repo is wet or dry and get all corresponding deps
    build_repo_type = ''
    if build_repo in catkin_packages:
        print "repo %s is wet" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif build_repo in stacks:
        print "repo %s is dry" % build_repo
        build_repo_type = 'dry'
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
    else:
        # build_repo is neither wet nor dry
        raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)

    # install user-defined/customized dependencies of build_repo from source
    rosinstall = ''
    fulfilled_deps = []
    for dep in repo_build_dependencies:
        if dep in pipe_repos[build_identifier].dependencies:
            print "Install user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # install additional, indirect user-defined dependencies
    for dep in pipe_repos[build_identifier].dependencies:
        if dep not in fulfilled_deps:
            print "Install additional user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # check if all user-defined/customized dependencies are satisfied
    if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
        print "Not all user-defined build dependencies are fulfilled"
        print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
        print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
        raise common.BuildException("Not all user-defined build dependencies are fulfilled")
    if rosinstall != '':
        # write .rosinstall file
        print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
        rosinstall += "- other: {local-name: /u/robot}"#TODO: not hardcoding (with multiple directories?)
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print "Install user-defined build dependencies from source"
        # rosinstall depends
        common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                    % (repo_sourcespace, workspace, ros_distro))

        # get also deps of just installed user-defined/customized dependencies
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
        if build_repo_type == 'wet':
            if stacks != {}:
                raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                            % (build_repo, '- ' + '\n- '.join(stacks)))
            # take only wet packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        else:  # dry build repo
            # take all packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
        repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    os.makedirs(repo_sourcespace_wet)
    os.makedirs(repo_sourcespace_dry)
    # get all folders in repo_sourcespace
    sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
    for dir in sourcespace_dirs:
        if dir in catkin_packages.keys():
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
        if dir in stacks.keys():
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))
    

    ############################
    ### install dependencies ###
    ############################
    time_install = datetime.datetime.now()
    print "=====> entering dependency installation step at", time_install

    # Create rosdep object
    rosdep_resolver = None

    print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
    #common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
    print "ASK YOUR ADMIN TO INSTALL:" + str(repo_build_dependencies)
    
    print "ASK YOUR ADMIN TO INSTALL THE PACKAGES ABOVE"
    # check which packages are installed
    # FAIL WHEN SOMETHING MISSING
    #############
    ### build ###
    #############
    time_build = datetime.datetime.now()
    print "=====> entering build step at", time_build

    # env
    print "Set up ros environment variables"
    ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    if options.verbose:
        common.call("env", ros_env_repo)

    ### catkin repositories
    if catkin_packages != {}:
        # set up catkin workspace
        if ros_distro == 'fuerte':
            if 'catkin' not in catkin_packages.keys():
                # add catkin package to rosinstall
                rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
                print "Install catkin"
                # rosinstall catkin
                common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
                            % (repo_sourcespace_wet, workspace, ros_distro))

            print "Create a CMakeLists.txt for catkin packages"
            common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
                                         os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
        else:
            common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env_repo)

        #os.mkdir(repo_buildspace)
        os.chdir(repo_sourcespace_wet + "/..")
        try:
            common.call("catkin_make", ros_env_repo)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to catkin_make wet repositories")
        
        # setup ros environment to source wet packages before building dry ones
        ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace_wet, '../devel/setup.bash'))

    ### rosbuild repositories
    if build_repo_type == 'dry':
        ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
        if options.verbose:
            common.call("env", ros_env_repo)

        if ros_distro == 'electric':
            print "Rosdep"
            common.call("rosmake rosdep", ros_env_repo)
        for stack in stacks.keys():
            common.call("rosdep install -y %s" % stack, ros_env_repo)

        # build dry repositories
        print "Build repository %s" % build_repo
        try:
            common.call("rosmake -i -rV --skip-blacklist --profile --pjobs=8 --output=%s %s" %
                        (dry_build_logs, build_repo), ros_env_repo)
        except common.BuildException as ex:
            try:
                shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
            finally:
                print ex.msg
                raise common.BuildException("Failed to rosmake %s" % build_repo)

    # the end (steps: parsing, checkout, install, analysis, build, finish)
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_checkout - time_parsing)
    print "checkout in                ", (time_install - time_checkout)
    print "install dependencies in    ", (time_build - time_install)
    print "build in                   ", (time_finish - time_build)
    print ""
    print "For testing, please run the following line in your testing terminal"
    print "source " + repo_sourcespace + "/setup.bash"
    print ""
Exemplo n.º 12
0
def build_downstream_post_fuerte(ros_distro, build_repo, workspace, server):
    ros_package_path = os.environ['ROS_PACKAGE_PATH']
    b_r_short = build_repo.split('__')[0]

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')
    repo_buildspace = os.path.join(tmpdir, 'build_repository')
    dependson_sourcespace = os.path.join(tmpdir, 'src_depends_on')
    dependson_sourcespace_wet = os.path.join(tmpdir, 'src_depends_on', 'wet')
    dependson_sourcespace_dry = os.path.join(tmpdir, 'src_depends_on', 'dry')
    dependson_buildspace = os.path.join(tmpdir, 'build_depend_on')
    dry_test_results_dir = os.path.join(dependson_sourcespace_dry, 'test_results')

    # TODO clean up test result dirs

    # get depends on
    print "Get list of released repositories that (build-)depend on repository %s" % b_r_short
    ros_depends_on = []
    distro_ros = rosdistro.RosDistro(ros_distro, 'http://%s/~jenkins' % server)
    for d in distro_ros.get_depends_on([b_r_short])['build'] + distro_ros.get_depends_on([b_r_short])['buildtool']:
        if not d in ros_depends_on and d != b_r_short:
            ros_depends_on.append(d)
    # TODO get_depends_on of cob_distro release (only for intern repos necessary)
    cob_depends_on = []
    #distro_cob = cob_distro.CobDistro(ros_distro)
    #for d in distro_cob.get_depends_on([b_r_short])['build'] + distro_cob.get_depends_on([b_r_short])['buildtool']:
    #    if not d in cob_depends_on and d != b_r_short:
    #        cob_depends_on.append(d)
    if len(ros_depends_on + cob_depends_on) == 0:
        print "No repository depends on repository %s. Test finished" % b_r_short
        return

    print "Build depends_on list of repository %s:\n - %s" % ('\n - '.join(ros_depends_on + cob_depends_on))

    # install depends_on repository from source
    rosinstall = distro_ros.get_rosinstall(ros_depends_on)
    #rosinstall += distro_cob.get_rosinstall(cob_depends_on) TODO
    print "Rosinstall for depends_on:\n %s" % rosinstall
    with open(workspace + "/depends_on.rosinstall", 'w') as f:
        f.write(rosinstall)
    print "Created rosinstall file for depends on"

    # install all repository and system dependencies of the depends_on list
    print "Install all depends_on from source: %s" % (', '.join(ros_depends_on))
    os.makedirs(dependson_sourcespace)
    common.call("rosinstall -j 8 %s %s/depends_on.rosinstall /opt/ros/%s" % (dependson_sourcespace, workspace, ros_distro))

    # all packages in dependson_sourcespace
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(dependson_sourcespace)
    print catkin_packages
    print stacks
    print manifest_packages

    # get build and test dependencies of depends_on list
    dependson_build_dependencies = []
    for d in common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False):
        print "  Checking dependency %s" % d
        if d in dependson_build_dependencies:
            print "    Already in dependson_build_dependencies"
        elif d in ros_depends_on or d in cob_depends_on:
            print "    Is a direct dependency of the repository, and is installed from source"
        elif d == b_r_short:
            print "    Is the tested repository"
        else:
            dependson_build_dependencies.append(d)
    print "Build dependencies of depends_on list are %s" % (', '.join(dependson_build_dependencies))

    dependson_test_dependencies = []
    for d in common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=False, test_depends=True):
        if d not in dependson_test_dependencies + ros_depends_on + cob_depends_on and d != b_r_short:
            dependson_test_dependencies.append(d)
    print "Test dependencies of depends_on list are %s" % (', '.join(dependson_test_dependencies))

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    os.makedirs(dependson_sourcespace_wet)
    os.makedirs(dependson_sourcespace_dry)
    # get all folders in dependson_sourcespace
    sourcespace_dirs = [name for name in os.listdir(dependson_sourcespace) if os.path.isdir(os.path.join(dependson_sourcespace, name))]
    for dir in sourcespace_dirs:
        if dir in catkin_packages.keys():
            shutil.move(os.path.join(dependson_sourcespace, dir), os.path.join(dependson_sourcespace_wet, dir))
        if dir in stacks.keys():
            shutil.move(os.path.join(dependson_sourcespace, dir), os.path.join(dependson_sourcespace_dry, dir))

    # Create rosdep object
    print "Create rosdep object"
    try:
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)
    except:
        from time import sleep
        sleep(10)
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    # install build dependencies
    print "Install all build dependencies of the depends_on list: %s" % (', '.join(dependson_build_dependencies))
    common.apt_get_install_also_nonrosdep(dependson_build_dependencies, rosdep_resolver)

    # env
    print "Setting up environment"
    ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    common.call("env", ros_env)
    ros_env_dependson = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
    ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([dependson_buildspace, repo_buildspace, ros_package_path])
    common.call("env", ros_env_dependson)

    ### catkin repositories
    if catkin_packages != {}:
        os.makedirs(dependson_buildspace)
        os.chdir(dependson_buildspace)
        print "Create a new CMakeLists.txt for catkin packages"
        if ros_distro == 'fuerte':
            common.call("ln -s %s %s" % (os.path.join(dependson_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
                                         os.path.join(dependson_sourcespace_wet, 'CMakeLists.txt')))
        else:
            common.call("catkin_init_workspace %s" % dependson_sourcespace_wet, ros_env_dependson)

        try:
            common.call("cmake %s" % dependson_sourcespace_wet, ros_env_dependson)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to cmake wet repositories")
        #ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories
        print "Build wet depends_on list"
        try:
            common.call("make", ros_env_dependson)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to make wet packages")

        if dependson_test_dependencies != []:
            # install test dependencies
            print "Install all test dependencies of the depends_on list: %s" % (', '.join(dependson_test_dependencies))
            common.apt_get_install_also_nonrosdep(dependson_test_dependencies, rosdep_resolver)

    ### rosbuild repositories
    if stacks != {}:
        # env
        print "Setting up environment"
        ros_env_dependson['ROS_PACKAGE_PATH'] = ':'.join([dependson_buildspace, repo_buildspace,
                                                          dependson_sourcespace_dry, repo_sourcespace_dry,
                                                          ros_package_path])
        common.call("env", ros_env_dependson)

        #print "Make rosdep"
        #common.call("rosmake rosdep", ros_env)
        #for stack in stacks.keys():
        #    common.call("rosdep install -y %s" % stack, ros_env_repo)

        dependson_sourcespace_dry_dirs = [name for name in os.listdir(dependson_sourcespace_dry) if os.path.isdir(os.path.join(dependson_sourcespace_dry, name))]
        print "Build dry depends_on repositories:\n - %s" % '\n - '.join(dependson_sourcespace_dry_dirs)
        os.mkdir(dry_test_results_dir)
        for dry_dependson in dependson_sourcespace_dry_dirs:
            try:
                common.call("rosmake -rV --profile --pjobs=8 --output=%s %s" % (dry_test_results_dir, b_r_short), ros_env_dependson)
            except:
                raise common.BuildException("Failed to rosmake %s" % b_r_short)
Exemplo n.º 13
0
def main():
    #########################
    ### parsing arguments ###
    #########################
    time_parsing = datetime.datetime.now()
    print "=====> entering argument parsing step at", time_parsing

    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_file(pipeline_repos_owner, server_name, user_name, file_location=os.environ["WORKSPACE"])
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print "\nTesting on ros distro: %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # for hardware build: only support hydro
    if ros_distro == "groovy":
        sys.exit(False)

    # for hardware build: remove old build artifacts 
    limit = 3 # limit amount of old builds
    print workspace
    while len(os.listdir(workspace + "/..")) > limit + 1: # we will not count the "current" sym link
        list = os.listdir(workspace + "/..")
        shutil.rmtree(workspace + "/../" + sorted(list)[0])

    # set up directories variables
    tmpdir = workspace
    repo_checkoutspace = os.path.join(tmpdir, 'checkout')                          # location to store repositories in (will be separated in wet and dry later on)
    os.makedirs(repo_checkoutspace)
    repo_sourcespace = os.path.join(tmpdir, 'src')                                 # location to build repositories in
    os.makedirs(repo_sourcespace)
    repo_sourcespace_wet = os.path.join(repo_sourcespace, 'wet', 'src')            # wet (catkin) repositories
    os.makedirs(repo_sourcespace_wet)
    repo_sourcespace_dry = os.path.join(repo_sourcespace, 'dry')                   # dry (rosbuild) repositories
    os.makedirs(repo_sourcespace_dry)
    repo_build_logs = os.path.join(tmpdir, 'build_logs')                           # location for build logs
    os.makedirs(repo_build_logs)

    # init catkin workspace
    ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)                            # source ros_distro (needed to do a catkin_init_workspace)
    # for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
    if os.path.isdir("/u/robot/git/care-o-bot/devel"):
        ros_env_repo = common.get_ros_env('/u/robot/git/care-o-bot/devel/setup.bash')
    common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env_repo, verbose=False)         # init wet workspace
    common.call("catkin_make --directory %s/wet" % repo_sourcespace, ros_env_repo, verbose=False)       # build wet workspace to generate a setup.bash

    ################
    ### checkout ###
    ################
    time_checkout = datetime.datetime.now()
    print "=====> entering checkout step at", time_checkout

    # download build_repo from source #
    print "Creating rosinstall file for repository %s" % build_repo
    rosinstall = ""
    if build_identifier in pipe_repos:  # check if triggering identifier is really present in pipeline config
        rosinstall += pipe_repos[build_identifier].get_rosinstall()
    else:
        err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
        raise common.BuildException(err_msg)

    # write rosinstall file
    print "Rosinstall file for repository: \n %s" % rosinstall
    with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
        f.write(rosinstall)
    print "Install repository from source:"
    # rosinstall repos
    common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
                % (repo_checkoutspace, workspace, ros_distro))

    # get the repositories build dependencies
    print "Get build dependencies of repo"

    # get all packages in checkoutspace
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_checkoutspace)

    # check if build_repo is wet or dry and get all corresponding deps
    build_repo_type = ''
    if build_repo in catkin_packages: # wet repo with metapackage
        print "repo %s is wet" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif (build_repo not in catkin_packages) and (build_repo not in stacks) and (catkin_packages != []): # wet repo without metapackage
        print "repo %s is wet without metapackage" % build_repo
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif build_repo in stacks: # dry repo with stack
        print "repo %s is dry" % build_repo
        build_repo_type = 'dry'
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
    #TODO elif : # dry repo without stack
    else: # build_repo is neither wet nor dry
        raise common.BuildException("Repository %s to build not found in checkoutspace" % build_repo)

    # install user-defined/customized dependencies of build_repo from source
    rosinstall = ''
    fulfilled_deps = []
    for dep in repo_build_dependencies:
        if dep in pipe_repos[build_identifier].dependencies:
            print "Install user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # install additional, indirect user-defined dependencies
    for dep in pipe_repos[build_identifier].dependencies:
        if dep not in fulfilled_deps:
            print "Install additional user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # check if all user-defined/customized dependencies are satisfied
    if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
        print "Not all user-defined build dependencies are fulfilled"
        print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
        print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
        raise common.BuildException("Not all user-defined build dependencies are fulfilled")

    if rosinstall != '':
        # write .rosinstall file
        print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print "Install user-defined build dependencies from source"
        # rosinstall depends
        common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
                    % (repo_checkoutspace, workspace, ros_distro))

        # get also deps of just installed user-defined/customized dependencies
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_checkoutspace)

        if build_repo_type == 'wet':
            if stacks != {}:
                raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                            % (build_repo, '- ' + '\n- '.join(stacks)))
            # take only wet packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        else:  # dry build repo
            # take all packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
        repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]

    print ""
    print "Found the following packages"
    print "  wet packages:     ", catkin_packages.keys()
    print "  dry stacks:       ", stacks.keys()
    print "  dry packages:     ", manifest_packages.keys()
    print ""

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    # get all folders in repo_checkoutspace
    checkoutspace_dirs = [name for name in os.listdir(repo_checkoutspace) if os.path.isdir(os.path.join(repo_checkoutspace, name))]
    for dir in checkoutspace_dirs:
        if dir in catkin_packages.keys(): # wet repo with metapackage
            shutil.move(os.path.join(repo_checkoutspace, dir), os.path.join(repo_sourcespace_wet, dir))
        elif build_repo_type == 'wet' and dir == build_repo: # wet repo without metapackage
            shutil.move(os.path.join(repo_checkoutspace, dir), os.path.join(repo_sourcespace_wet, dir))
        elif dir in stacks.keys(): # dry repo with stack
            shutil.move(os.path.join(repo_checkoutspace, dir), os.path.join(repo_sourcespace_dry, dir))
        #TODO elif: # dry repo without stack
        #else:
        #    raise common.BuildException("Could not separate %s into wet or dry sourcespace." %dir) 
    # remove checkout dir
    common.call("rm -rf %s" % repo_checkoutspace)

    # setup ros workspace
    print "Set up ros workspace and setup environment variables"
    ros_env_repo = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    common.call("rosws init %s /opt/ros/%s" %(repo_sourcespace, ros_distro), verbose=False)             # init workspace pointing to ros_distro

    # for hardware build: merge workspace with robot account #FIXME: this should be parameterisable in plugin (select a path to a setup.bash file in the admin config and mark a checkbox for the user config)
    if os.path.isdir("/u/robot/git/care-o-bot"):
        common.call("rosws merge -t %s /u/robot/git/care-o-bot" %repo_sourcespace, verbose=False)      # merge robot account workspace

    common.call("rosws merge -t %s %s/wet/devel" % (repo_sourcespace, repo_sourcespace), verbose=False) # merge wet workspace
    common.call("rosws merge -t %s %s/dry" % (repo_sourcespace, repo_sourcespace), verbose=False)       # merge dry workspace
    ros_env_repo = common.get_ros_env(repo_sourcespace + '/setup.bash')                                 # source wet and dry workspace

    ############################
    ### install dependencies ###
    ############################
    time_install = datetime.datetime.now()
    print "=====> entering dependency installation step at", time_install

    # Create rosdep object
    rosdep_resolver = None
    print "Create rosdep object"
    try:
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)
    except:  # when init fails the first time
        from time import sleep
        sleep(10)
        rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
    missing_packages = common.apt_get_check_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
    if len(missing_packages) > 0:
        raise common.BuildException("Some dependencies are missing. Please ask your administrator to install the following packages: %s" % missing_packages)
    print "All denendencies already installed."

    #############
    ### build ###
    #############
    time_build = datetime.datetime.now()
    print "=====> entering build step at", time_build

    # get amount of cores available on host system
    cores = multiprocessing.cpu_count()

    ### catkin repositories
    if catkin_packages != {}:
        print "Build wet packages: ", catkin_packages.keys()
        try:
            common.call("catkin_make --directory %s/wet" % repo_sourcespace, ros_env_repo)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to catkin_make wet repositories")

    ### rosbuild repositories
    if build_repo_type == 'dry':
        # build dry repositories
        print "Build dry stacks:   ", stacks.keys()
        print "Build dry packages: ", manifest_packages.keys()
        packages_to_build = " ".join(stacks.keys()) + " ".join(manifest_packages.keys())
        try:
            common.call("rosmake -rV --skip-blacklist --profile --pjobs=%s --output=%s %s" %
                        (cores + 1, repo_build_logs, packages_to_build), ros_env_repo)
        except common.BuildException as ex:
            try:
                shutil.move(repo_build_logs, os.path.join(workspace, "build_logs"))
            finally:
                print ex.msg
                raise common.BuildException("Failed to rosmake dry repositories")

    ###########
    ### end ###
    ###########
    # for hardware builds: create sym link to new build
    common.call("rm -f %s/../current" % workspace)
    common.call("ln -sf %s %s/../current" %(workspace, workspace))
    
    # steps: parsing, checkout, install, analysis, build, finish
    time_finish = datetime.datetime.now()
    print "=====> finished script at", time_finish
    print "durations:"
    print "parsing arguments in       ", (time_checkout - time_parsing)
    print "checkout in                ", (time_install - time_checkout)
    print "install dependencies in    ", (time_build - time_install)
    print "build in                   ", (time_finish - time_build)
    print "total                      ", (time_finish - time_parsing)
    print ""
    print "For manual testing, please run the following line in your testing terminal"
    print "source " + repo_sourcespace + "/setup.bash"
    print ""
Exemplo n.º 14
0
def main():
    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print datetime.datetime.now()
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')               # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet')    # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')    # dry (rosbuild) repositories
    repo_buildspace = os.path.join(tmpdir, 'build_repository')              # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')       # location for build logs

    # download build_repo from source
    print "Creating rosinstall file for repository %s" % build_repo
    rosinstall = ""
    if build_identifier in pipe_repos:  # check if triggering identifier is really present in pipeline config
        rosinstall += pipe_repos[build_identifier].get_rosinstall()
    else:
        err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
        raise common.BuildException(err_msg)

    # write rosinstall file
    print "Rosinstall file for repository: \n %s" % rosinstall
    with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
        f.write(rosinstall)
    print "Install repository from source:"
    # create repo sourcespace directory 'src_repository'
    os.makedirs(repo_sourcespace)
    # rosinstall repos
    common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                % (repo_sourcespace, workspace, ros_distro))

    # get the repositories build dependencies
    print "Get build dependencies of repo"

    # get all packages in sourcespace
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
    if ros_distro == 'electric' and catkin_packages != {}:
        raise common.BuildException("Found wet packages while building in ros electric")

    # (debug) output
    if options.verbose:
        print "Packages in %s:" % repo_sourcespace
        print "Catkin: ", catkin_packages
        print "Rosbuild:\n  Stacks: ", stacks
        print "  Packages: ", manifest_packages

        # get deps directly for catkin (like in willow code)
        try:
            print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
        except:
            pass
        # deps catkin
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
        # deps stacks
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
        print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))

    # check if build_repo is wet or dry and get all corresponding deps
    build_repo_type = ''
    if build_repo in catkin_packages:
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif build_repo in stacks:
        build_repo_type = 'dry'
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
    else:
        # build_repo is neither wet nor dry
        raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)

    # install user-defined/customized dependencies of build_repo from source
    rosinstall = ''
    fulfilled_deps = []
    for dep in repo_build_dependencies:
        if dep in pipe_repos[build_identifier].dependencies:
            print "Install user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # install additional, indirect user-defined dependencies
    for dep in pipe_repos[build_identifier].dependencies:
        if dep not in fulfilled_deps:
            print "Install additional user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # check if all user-defined/customized dependencies are satisfied
    if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
        print "Not all user-defined build dependencies are fulfilled"
        print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
        print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
        raise common.BuildException("Not all user-defined build dependencies are fulfilled")

    if rosinstall != '':
        # write .rosinstall file
        print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print "Install user-defined build dependencies from source"
        # rosinstall depends
        common.call("rosinstall -j 8 --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                    % (repo_sourcespace, workspace, ros_distro))

        # get also deps of just installed user-defined/customized dependencies
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
        if build_repo_type == 'wet':
            if stacks != {}:
                raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                            % (build_repo, '- ' + '\n- '.join(stacks)))
            # take only wet packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        else:  # dry build repo
            # take all packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
        repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]

    rosdep_resolver = None
    if ros_distro != 'electric':
        # Create rosdep object
        print "Create rosdep object"
        try:
            rosdep_resolver = rosdep.RosDepResolver(ros_distro)
        except:  # when init fails the first time
            from time import sleep
            sleep(10)
            rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    print datetime.datetime.now()
    print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
    common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
    print datetime.datetime.now()

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    os.makedirs(repo_sourcespace_wet)
    os.makedirs(repo_sourcespace_dry)
    # get all folders in repo_sourcespace
    sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
    for dir in sourcespace_dirs:
        if dir in catkin_packages.keys():
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
        if dir in stacks.keys():
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))

    # env
    print "Set up ros environment variables"
    ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    if options.verbose:
        common.call("env", ros_env)

    ### catkin repositories
    print datetime.datetime.now()
    if catkin_packages != {}:
        # set up catkin workspace
        if ros_distro == 'fuerte':
            if 'catkin' not in catkin_packages.keys():
                # add catkin package to rosinstall
                rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
                print "Install catkin"
                # rosinstall catkin
                common.call("rosinstall -j 8 --verbose %s %s/repo.rosinstall /opt/ros/%s"
                            % (repo_sourcespace_wet, workspace, ros_distro))

            print "Create a CMakeLists.txt for catkin packages"
            common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
                                         os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
        else:
            common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env)

        os.mkdir(repo_buildspace)
        os.chdir(repo_buildspace)
        try:
            common.call("cmake %s" % repo_sourcespace_wet + '/', ros_env)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to cmake wet repositories")
        #ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories
        print "Build wet repository list"
        try:
            common.call("make", ros_env)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to make wet packages")

    ### rosbuild repositories
    print datetime.datetime.now()
    if build_repo_type == 'dry':
        ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
        ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
        if options.verbose:
            common.call("env", ros_env_repo)

        if ros_distro == 'electric':
            print "Rosdep"
            common.call("rosmake rosdep", ros_env)
        for stack in stacks.keys():
            common.call("rosdep install -y %s" % stack, ros_env_repo)

        # build dry repositories
        print "Build repository %s" % build_repo
        try:
            common.call("rosmake -rV --profile --pjobs=8 --output=%s %s" %
                        (dry_build_logs, build_repo), ros_env_repo)
        except common.BuildException as ex:
            try:
                shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
            finally:
                print ex.msg
                raise common.BuildException("Failed to rosmake %s" % build_repo)

        print datetime.datetime.now()
Exemplo n.º 15
0
def main():
    # parse parameter values
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose', action='store_true', default=False)
    (options, args) = parser.parse_args()

    if len(args) < 5:
        print "Usage: %s pipeline_repos_owner server_name user_name ros_distro build_repo" % sys.argv[0]
        raise common.BuildException("Wrong arguments for build script")

    # get arguments
    pipeline_repos_owner = args[0]
    server_name = args[1]
    user_name = args[2]
    ros_distro = args[3]
    build_identifier = args[4]                      # repository + suffix
    build_repo = build_identifier.split('__')[0]    # only repository to build
    # environment variables
    workspace = os.environ['WORKSPACE']
    ros_package_path = os.environ['ROS_PACKAGE_PATH']

    # (debug) output
    print "\n", 50 * 'X'
    print datetime.datetime.now()
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "\n", 50 * 'X'

    # update sourcelist and upgrade installed basic packages
    #common.output("Updating chroot enviroment")
    #common.call("apt-get update")
    #common.call("apt-get dist-upgrade -y")

    #common.output("Updating rosinstall")  # TODO run install frequently in chroot_tarball_updater an remove here
    #common.call("pip install -U rosinstall")

    # install packages needed for execution (depending on ros_distro)
    #common.output("Installing necessary packages:", decoration='')
    #if ros_distro == 'electric':
        #print "  catkin-pkg and rospkg"
        #common.call("pip install -U catkin-pkg rospkg")
    #elif ros_distro == 'fuerte':
        #print "  rospkg and rosdep"
        #common.call("pip install -U rospkg rosdep")
    #elif ros_distro == 'groovy':
        #print "  python-catkin-pkg and python-rosdistro"
        #common.call("apt-get install python-catkin-pkg python-rosdistro --yes")

    # cob_pipe object
    cp_instance = cob_pipe.CobPipe()
    cp_instance.load_config_from_url(pipeline_repos_owner, server_name, user_name)
    pipe_repos = cp_instance.repositories
    common.output("Pipeline configuration successfully loaded", blankline='b')

    # (debug) output
    print "\n", 50 * 'X'
    print datetime.datetime.now()
    print "\nTesting on ros distro:  %s" % ros_distro
    print "Testing repository: %s" % build_repo
    if build_repo != build_identifier:
        print "       with suffix: %s" % build_identifier.split('__')[1]
    print "Using source: %s" % pipe_repos[build_identifier].url
    print "Testing branch/version: %s" % pipe_repos[build_identifier].version
    print "\n", 50 * 'X'

    # set up directories variables
    tmpdir = os.path.join('/tmp', 'test_repositories')
    repo_sourcespace = os.path.join(tmpdir, 'src_repository')               # location to store repositories in
    repo_sourcespace_wet = os.path.join(tmpdir, 'src_repository', 'wet')    # wet (catkin) repositories
    repo_sourcespace_dry = os.path.join(tmpdir, 'src_repository', 'dry')    # dry (rosbuild) repositories
    repo_buildspace = os.path.join(tmpdir, 'build_repository')              # location for build output
    dry_build_logs = os.path.join(repo_sourcespace_dry, 'build_logs')       # location for build logs

    # download build_repo from source
    print "Creating rosinstall file for repository %s" % build_repo
    rosinstall = ""
    if build_identifier in pipe_repos:  # check if triggering identifier is really present in pipeline config
        rosinstall += pipe_repos[build_identifier].get_rosinstall()
    else:
        err_msg = "Pipeline was triggered by repository %s which is not in pipeline config!" % build_identifier
        raise common.BuildException(err_msg)

    # write rosinstall file
    print "Rosinstall file for repository: \n %s" % rosinstall
    with open(os.path.join(workspace, 'repo.rosinstall'), 'w') as f:
        f.write(rosinstall)
    print "Install repository from source:"
    # create repo sourcespace directory 'src_repository'
    os.makedirs(repo_sourcespace)
    # rosinstall repos
    common.call("rosinstall --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                % (repo_sourcespace, workspace, ros_distro))

    # get the repositories build dependencies
    print "Get build dependencies of repo"

    # get all packages in sourcespace
    (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
    if ros_distro == 'electric' and catkin_packages != {}:
        raise common.BuildException("Found wet packages while building in ros electric")

    # (debug) output
    if options.verbose:
        print "Packages in %s:" % repo_sourcespace
        print "Catkin: ", catkin_packages
        print "Rosbuild:\n  Stacks: ", stacks
        print "  Packages: ", manifest_packages

        # get deps directly for catkin (like in willow code)
        try:
            print "Found wet build dependencies:\n%s" % '- ' + '\n- '.join(sorted(common.get_dependencies(repo_sourcespace, build_depends=True, test_depends=False)))
        except:
            pass
        # deps catkin
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        print "Found wet dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))
        # deps stacks
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
        print "Found dry dependencies:\n%s" % '- ' + '\n- '.join(sorted(repo_build_dependencies))

    # check if build_repo is wet or dry and get all corresponding deps
    build_repo_type = ''
    if build_repo in catkin_packages:
        build_repo_type = 'wet'
        repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
    elif build_repo in stacks:
        build_repo_type = 'dry'
        repo_build_dependencies = common.get_nonlocal_dependencies({}, stacks, {})
    else:
        # build_repo is neither wet nor dry
        raise common.BuildException("Repository %s to build not found in sourcespace" % build_repo)

    # install user-defined/customized dependencies of build_repo from source
    rosinstall = ''
    fulfilled_deps = []
    for dep in repo_build_dependencies:
        if dep in pipe_repos[build_identifier].dependencies:
            print "Install user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # install additional, indirect user-defined dependencies
    for dep in pipe_repos[build_identifier].dependencies:
        if dep not in fulfilled_deps:
            print "Install additional user-defined build dependency %s from source" % dep
            rosinstall += pipe_repos[build_identifier].dependencies[dep].get_rosinstall()
            fulfilled_deps.append(dep)

    # check if all user-defined/customized dependencies are satisfied
    if sorted(fulfilled_deps) != sorted(pipe_repos[build_identifier].dependencies):
        print "Not all user-defined build dependencies are fulfilled"
        print "User-defined build dependencies:\n - %s" % '\n - '.join(pipe_repos[build_identifier].dependencies)
        print "Fulfilled dependencies:\n - %s" % '\n - '.join(fulfilled_deps)
        raise common.BuildException("Not all user-defined build dependencies are fulfilled")

    if rosinstall != '':
        # write .rosinstall file
        print "Rosinstall file for user-defined build dependencies: \n %s" % rosinstall
        with open(os.path.join(workspace, "repo.rosinstall"), 'w') as f:
            f.write(rosinstall)
        print "Install user-defined build dependencies from source"
        # rosinstall depends
        common.call("rosinstall --verbose --continue-on-error %s %s/repo.rosinstall /opt/ros/%s"
                    % (repo_sourcespace, workspace, ros_distro))

        # get also deps of just installed user-defined/customized dependencies
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace)
        if build_repo_type == 'wet':
            if stacks != {}:
                raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                            % (build_repo, '- ' + '\n- '.join(stacks)))
            # take only wet packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=True, test_depends=False)
        else:  # dry build repo
            # take all packages
            repo_build_dependencies = common.get_nonlocal_dependencies(catkin_packages, stacks, {}, build_depends=True, test_depends=False)
        repo_build_dependencies = [dep for dep in repo_build_dependencies if dep not in fulfilled_deps]

    rosdep_resolver = None
    if ros_distro != 'electric':
        # Create rosdep object
        print "Create rosdep object"
        try:
            rosdep_resolver = rosdep.RosDepResolver(ros_distro)
        except:  # when init fails the first time
            from time import sleep
            sleep(10)
            rosdep_resolver = rosdep.RosDepResolver(ros_distro)

    print datetime.datetime.now()
    print "Install build dependencies: %s" % (', '.join(repo_build_dependencies))
    common.apt_get_install_also_nonrosdep(repo_build_dependencies, ros_distro, rosdep_resolver)
    print datetime.datetime.now()

    # separate installed repos in wet and dry
    print "Separate installed repositories in wet and dry"
    os.makedirs(repo_sourcespace_wet)
    os.makedirs(repo_sourcespace_dry)
    # get all folders in repo_sourcespace
    sourcespace_dirs = [name for name in os.listdir(repo_sourcespace) if os.path.isdir(os.path.join(repo_sourcespace, name))]
    for dir in sourcespace_dirs:
        if dir in catkin_packages.keys():
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_wet, dir))
        if dir in stacks.keys():
            shutil.move(os.path.join(repo_sourcespace, dir), os.path.join(repo_sourcespace_dry, dir))

    # env
    print "Set up ros environment variables"
    ros_env = common.get_ros_env('/opt/ros/%s/setup.bash' % ros_distro)
    if options.verbose:
        common.call("env", ros_env)

    ### catkin repositories
    print datetime.datetime.now()
    if catkin_packages != {}:
        # set up catkin workspace
        if ros_distro == 'fuerte':
            if 'catkin' not in catkin_packages.keys():
                # add catkin package to rosinstall
                rosinstall = "\n- git: {local-name: catkin, uri: 'git://github.com/ros/catkin.git', version: fuerte-devel}"
                print "Install catkin"
                # rosinstall catkin
                common.call("rosinstall --verbose %s %s/repo.rosinstall /opt/ros/%s"
                            % (repo_sourcespace_wet, workspace, ros_distro))

            print "Create a CMakeLists.txt for catkin packages"
            common.call("ln -s %s %s" % (os.path.join(repo_sourcespace_wet, 'catkin', 'cmake', 'toplevel.cmake'),
                                         os.path.join(repo_sourcespace_wet, 'CMakeLists.txt')))
        else:
            common.call("catkin_init_workspace %s" % repo_sourcespace_wet, ros_env)

        os.mkdir(repo_buildspace)
        os.chdir(repo_buildspace)
        try:
            common.call("cmake %s" % repo_sourcespace_wet + '/', ros_env)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to cmake wet repositories")
        #ros_env_repo = common.get_ros_env(os.path.join(repo_buildspace, 'devel/setup.bash'))

        # build repositories and tests
        print "Build wet repository list"
        try:
            common.call("make", ros_env)
        except common.BuildException as ex:
            print ex.msg
            raise common.BuildException("Failed to make wet packages")

        test_error_msg = None
        try:
            common.call("make tests", ros_env)
        except common.BuildException as ex:
            print ex.msg
            test_error_msg = ex.msg

        # get wet repositories test and run dependencies
        print "Get test and run dependencies of repo list"
        (catkin_packages, stacks, manifest_packages) = common.get_all_packages(repo_sourcespace_wet)
        if stacks != {}:
            raise common.BuildException("Catkin (wet) package %s depends on (dry) stack(s):\n%s"
                                        % (build_repo, '- ' + '\n- '.join(stacks)))
        # take only wet packages
        repo_test_dependencies = common.get_nonlocal_dependencies(catkin_packages, {}, {}, build_depends=False, test_depends=True)
        if repo_test_dependencies != [] and test_error_msg is None:
            print "Install test and run dependencies of repository list: %s" % (', '.join(repo_test_dependencies))
            common.apt_get_install_also_nonrosdep(repo_test_dependencies, ros_distro, rosdep_resolver)

            # run tests
            print "Test repository list"
            try:
                common.call("make run_tests", ros_env)
            except common.BuildException as ex:
                print ex.msg
                test_error_msg = ex.msg

        # copy test results
        common.copy_test_results(workspace, repo_buildspace, test_error_msg)

    ### rosbuild repositories
    print datetime.datetime.now()
    if build_repo_type == 'dry':
        # get list of dependencies to test
        test_repos_list = []
        for dep in pipe_repos[build_identifier].dependencies:
            if dep in stacks:
                test_repos_list.append(dep)

        ros_env_repo = common.get_ros_env(os.path.join(repo_sourcespace, 'setup.bash'))
        ros_env_repo['ROS_PACKAGE_PATH'] = ':'.join([repo_sourcespace, ros_package_path])
        if options.verbose:
            common.call("env", ros_env_repo)

        if ros_distro == 'electric':
            print "Rosdep"
            common.call("rosmake rosdep", ros_env)
        for stack in stacks.keys():
            common.call("rosdep install -y %s" % stack, ros_env_repo)

        # build dry repositories and tests
        print "Build repository %s" % build_repo
        try:
            common.call("rosmake -rV --profile --pjobs=8 --output=%s %s" %
                        (dry_build_logs, build_repo), ros_env_repo)
        except common.BuildException as ex:
            try:
                shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
            finally:
                print ex.msg
                raise common.BuildException("Failed to rosmake %s" % build_repo)
        try:
            common.call("rosmake -rV --profile --pjobs=8 --test-only --output=%s %s" %
                        (dry_build_logs, " ".join(test_repos_list + [build_repo])), ros_env_repo)
        except common.BuildException as ex:
            print ex.msg

        # copy test results
        common.call("rosrun rosunit clean_junit_xml.py", ros_env_repo)
        for file in os.listdir(os.path.join(repo_sourcespace, "test_results")):
            file_path = os.path.join(repo_sourcespace, "test_results", file)
            print file_path
            try:
                if not file.startswith("_hudson"):
                    shutil.rmtree(file_path)
            except Exception as e:
                print e

        common.copy_test_results(workspace, repo_sourcespace)
        print datetime.datetime.now()
        try:
            shutil.move(dry_build_logs, os.path.join(workspace, "build_logs"))
        except IOError as ex:
            print "No build logs found: %s" % ex