Example #1
0
def _package_args_handler(command, parser, options, args):
    # package or stack names as args.  have to convert stack names to packages.
    # - overrides to enable testing
    rospack = rospkg.RosPack()
    rosstack = rospkg.RosStack()
    lookup = _get_default_RosdepLookup(options)
    loader = lookup.get_loader()
    
    if options.rosdep_all:
        if args:
            parser.error("cannot specify additional arguments with -a")
        else:
            # let the loader filter the -a. This will take out some
            # packages that are catkinized (for now).
            args = loader.get_loadable_resources()
            not_found = []
    elif not args:
        parser.error("no packages or stacks specified")

    val = rospkg.expand_to_packages(args, rospack, rosstack)
    packages = val[0]
    not_found = val[1]
    if not_found:
        raise rospkg.ResourceNotFound(not_found[0], rospack.get_ros_paths())

    if 0 and not packages: # disable, let individual handlers specify behavior
        # possible with empty stacks
        print("No packages in arguments, aborting")
        return

    return command_handlers[command](lookup, packages, options)
Example #2
0
def test_expand_to_packages():
    from rospkg import expand_to_packages, RosPack, RosStack
    path = os.path.join(get_stack_test_path(), 's1')
    rospack = RosPack(ros_paths=[path])
    rosstack = RosStack(ros_paths=[path])

    try:
        expand_to_packages('foo', rospack, rosstack)
        assert False, "should have raised ValueError"
    except ValueError:
        pass

    valid, invalid = expand_to_packages(['foo'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2'])
    assert not invalid

    valid, invalid = expand_to_packages(['foo_pkg'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg'])
    assert not invalid

    valid, invalid = expand_to_packages(['foo', 'foo_pkg'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2'])
    assert not invalid

    valid, invalid = expand_to_packages(['foo', 'bar'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert not invalid

    valid, invalid = expand_to_packages(['foo', 'bar_pkg'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert not invalid

    valid, invalid = expand_to_packages(['foo', 'bar_pkg', 'bar'], rospack,
                                        rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert not invalid

    valid, invalid = expand_to_packages(
        ['foo', 'fake1', 'bar_pkg', 'bar', 'fake2'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert set(invalid) == set(['fake1', 'fake2'])
Example #3
0
def test_expand_to_packages():
    from rospkg import expand_to_packages, RosPack, RosStack
    path = os.path.join(get_stack_test_path(), 's1')
    rospack = RosPack(ros_paths=[path])
    rosstack = RosStack(ros_paths=[path])

    try:
        expand_to_packages('foo', rospack, rosstack)
        assert False, "should have raised ValueError"
    except ValueError:
        pass
    
    valid, invalid = expand_to_packages(['foo'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2'])
    assert not invalid
    
    valid, invalid = expand_to_packages(['foo_pkg'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg'])
    assert not invalid
    
    valid, invalid = expand_to_packages(['foo', 'foo_pkg'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2'])
    assert not invalid
    
    valid, invalid = expand_to_packages(['foo', 'bar'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert not invalid
                                
    valid, invalid = expand_to_packages(['foo', 'bar_pkg'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert not invalid
                                
    valid, invalid = expand_to_packages(['foo', 'bar_pkg', 'bar'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert not invalid

    valid, invalid = expand_to_packages(['foo', 'fake1', 'bar_pkg', 'bar', 'fake2'], rospack, rosstack)
    assert set(valid) == set(['foo_pkg', 'foo_pkg_2', 'bar_pkg'])
    assert set(invalid) == set(['fake1', 'fake2'])
Example #4
0
def expand_to_packages(names, env=None):
    """
    Expand names into a list of packages. Names can either be of packages or stacks.

    @param names: names of stacks or packages
    @type  names: [str]
    @return: ([packages], [not_found]). expand_packages() returns two
    lists. The first is of packages names. The second is a list of
    names for which no matching stack or package was found. Lists may have duplicates.
    @rtype: ([str], [str])
    """
    if env is None:
        env = os.environ
    ros_paths = rospkg.get_ros_paths(env)
    rospack = rospkg.RosPack(ros_paths)
    rosstack = rospkg.RosStack(ros_paths)
    return rospkg.expand_to_packages(names, rospack, rosstack)
Example #5
0
def expand_to_packages(names, env=None):
    """
    Expand names into a list of packages. Names can either be of packages or stacks.

    @param names: names of stacks or packages
    @type  names: [str]
    @return: ([packages], [not_found]). expand_packages() returns two
    lists. The first is of packages names. The second is a list of
    names for which no matching stack or package was found. Lists may have duplicates.
    @rtype: ([str], [str])
    """
    if env is None:
        env = os.environ
    ros_paths = rospkg.get_ros_paths(env)
    rospack = rospkg.RosPack(ros_paths)
    rosstack = rospkg.RosStack(ros_paths)
    return rospkg.expand_to_packages(names, rospack, rosstack)
Example #6
0
def _package_args_handler(command, parser, options, args):
    if options.rosdep_all:
        if args:
            parser.error('cannot specify additional arguments with -a')
        else:
            # let the loader filter the -a. This will take out some
            # packages that are catkinized (for now).
            lookup = _get_default_RosdepLookup(options)
            loader = lookup.get_loader()
            args = loader.get_loadable_resources()
            not_found = []
    elif not args:
        parser.error('no packages or stacks specified')

    # package or stack names as args.  have to convert stack names to packages.
    # - overrides to enable testing
    packages = []
    not_found = []
    if options.from_paths:
        for path in args:
            if options.verbose:
                print("Using argument '{0}' as a path to search.".format(path))
            if not os.path.exists(path):
                print("given path '{0}' does not exist".format(path))
                return 1
            path = os.path.abspath(path)
            if 'ROS_PACKAGE_PATH' not in os.environ:
                os.environ['ROS_PACKAGE_PATH'] = '{0}'.format(path)
            else:
                os.environ['ROS_PACKAGE_PATH'] = '{0}{1}{2}'.format(
                    path, os.pathsep, os.environ['ROS_PACKAGE_PATH'])
            pkgs = find_catkin_packages_in(path, options.verbose)
            packages.extend(pkgs)
        # Make packages list unique
        packages = list(set(packages))
    else:
        rospack = rospkg.RosPack()
        rosstack = rospkg.RosStack()
        val = rospkg.expand_to_packages(args, rospack, rosstack)
        packages = val[0]
        not_found = val[1]
    if not_found:
        raise rospkg.ResourceNotFound(not_found[0], rospack.get_ros_paths())

    # Handle the --ignore-src option
    if command in ['install', 'check'] and options.ignore_src:
        if options.verbose:
            print('Searching ROS_PACKAGE_PATH for '
                  'sources: ' + str(os.environ['ROS_PACKAGE_PATH'].split(':')))
        ws_pkgs = get_workspace_packages()
        for path in os.environ['ROS_PACKAGE_PATH'].split(':'):
            path = os.path.abspath(path.strip())
            if os.path.exists(path):
                pkgs = find_catkin_packages_in(path, options.verbose)
                ws_pkgs.extend(pkgs)
            elif options.verbose:
                print('Skipping non-existent path ' + path)
        set_workspace_packages(ws_pkgs)

    lookup = _get_default_RosdepLookup(options)

    # Handle the --skip-keys option by pretending that they are packages in the catkin workspace
    if command in ['install', 'check'] and options.skip_keys:
        if options.verbose:
            print('Skipping the specified rosdep keys:\n- ' +
                  '\n- '.join(options.skip_keys))
        lookup.skipped_keys = options.skip_keys

    if 0 and not packages:  # disable, let individual handlers specify behavior
        # possible with empty stacks
        print('No packages in arguments, aborting')
        return

    return command_handlers[command](lookup, packages, options)
Example #7
0
    def main(self):
        """
        main command-line entrypoint
        """
        parser = OptionParser(
            usage="usage: %prog [options] [PACKAGE]...",
            description=
            "rosmake recursively builds all dependencies before building a package",
            prog='rosmake')
        parser.add_option("--test-only",
                          dest="test_only",
                          default=False,
                          action="store_true",
                          help="only run tests")
        parser.add_option("-t",
                          dest="test",
                          default=False,
                          action="store_true",
                          help="build and test packages")
        parser.add_option("-a",
                          "--all",
                          dest="build_all",
                          default=False,
                          action="store_true",
                          help="select all packages")
        parser.add_option(
            "-i",
            "--mark-installed",
            dest="mark_installed",
            default=False,
            action="store_true",
            help=
            "On successful build, mark specified packages as installed with ROS_NOBUILD"
        )
        parser.add_option(
            "-u",
            "--unmark-installed",
            dest="unmark_installed",
            default=False,
            action="store_true",
            help=
            "Remove ROS_NOBUILD from the specified packages.  This will not build anything."
        )
        parser.add_option("-v",
                          dest="verbose",
                          default=False,
                          action="store_true",
                          help="display errored builds")
        parser.add_option("-r",
                          "-k",
                          "--robust",
                          dest="best_effort",
                          default=False,
                          action="store_true",
                          help="do not stop build on error")
        parser.add_option("--build-everything",
                          dest="robust",
                          default=False,
                          action="store_true",
                          help="build all packages regardless of errors")
        parser.add_option("-V",
                          dest="full_verbose",
                          default=False,
                          action="store_true",
                          help="display all builds")
        parser.add_option(
            "-s",
            "--specified-only",
            dest="specified_only",
            default=False,
            action="store_true",
            help="only build packages specified on the command line")
        parser.add_option("--buildtest",
                          dest="buildtest",
                          action="append",
                          help="package to buildtest")
        parser.add_option("--buildtest1",
                          dest="buildtest1",
                          action="append",
                          help="package to buildtest1")
        parser.add_option("--output",
                          dest="output_dir",
                          action="store",
                          help="where to output results")
        parser.add_option("--pre-clean",
                          dest="pre_clean",
                          action="store_true",
                          help="run make clean first")
        parser.add_option("--bootstrap",
                          dest="bootstrap",
                          default=False,
                          action="store_true",
                          help="DEPRECATED, UNUSED")
        parser.add_option("--disable-logging",
                          dest="logging_enabled",
                          default=True,
                          action="store_false",
                          help="turn off all logs")
        parser.add_option("--target",
                          dest="target",
                          action="store",
                          help="run make with this target")
        parser.add_option(
            "--pjobs",
            dest="ros_parallel_jobs",
            type="int",
            action="store",
            help=
            "Override ROS_PARALLEL_JOBS environment variable with this number of jobs."
        )
        parser.add_option("--threads",
                          dest="threads",
                          type="int",
                          default=os.environ.get("ROSMAKE_THREADS",
                                                 parallel_build.num_cpus()),
                          action="store",
                          help="Build up to N packages in parallel")
        parser.add_option("--profile",
                          dest="print_profile",
                          default=False,
                          action="store_true",
                          help="print time profile after build")
        parser.add_option(
            "--skip-blacklist",
            dest="skip_blacklist",
            default=False,
            action="store_true",
            help=
            "skip packages containing a file called ROS_BUILD_BLACKLIST (Default behavior will ignore the presence of ROS_BUILD_BLACKLIST)"
        )
        parser.add_option(
            "--skip-blacklist-osx",
            dest="skip_blacklist_osx",
            default=False,
            action="store_true",
            help=
            "deprecated option. it will do nothing, please use platform declarations and --require-platform instead"
        )

        parser.add_option(
            "--status-rate",
            dest="status_update_rate",
            action="store",
            help="How fast to update the status bar in Hz.  Default: 5Hz")

        options, args = parser.parse_args()
        self.printer.print_all('rosmake starting...')

        rospack = self.rospack
        rosstack = self.rosstack

        testing = False
        building = True
        if options.test_only:
            testing = True
            building = False
        elif options.test:
            testing = True

        if options.ros_parallel_jobs:
            self.ros_parallel_jobs = options.ros_parallel_jobs

        self.robust_build = options.robust
        self.best_effort = options.best_effort
        self.threads = options.threads
        self.skip_blacklist = options.skip_blacklist
        if options.skip_blacklist_osx:
            self.printer.print_all(
                "Option --skip-blacklist-osx is deprecated. It will do nothing, please use platform declarations and --require-platform instead"
            )
        self.logging_enabled = options.logging_enabled

        # pass through verbosity options
        self.printer.full_verbose = options.full_verbose
        self.printer.verbose = options.verbose
        if options.status_update_rate:
            if float(options.status_update_rate) > 0:
                self.printer.duration = 1.0 / float(options.status_update_rate)
            else:
                self.printer.duration = 0

        packages = []
        #load packages from arguments
        if options.build_all:
            packages = [
                x for x in rospack.list()
                if not self.rospack.get_manifest(x).is_catkin
            ]
            self.printer.print_all("Building all packages")
        else:  # no need to extend if all already selected
            if options.buildtest:
                for p in options.buildtest:
                    packages.extend(self.rospack.get_depends_on(p))
                    self.printer.print_all(
                        "buildtest requested for package %s adding it and all dependent packages: "
                        % p)

            if options.buildtest1:
                for p in options.buildtest1:
                    packages.extend(
                        self.rospack.get_depends_on(p, implicit=False))
                    self.printer.print_all(
                        "buildtest1 requested for package %s adding it and all depends-on1 packages: "
                        % p)

        if len(packages) == 0 and len(args) == 0:
            p = os.path.basename(os.path.abspath('.'))
            try:
                if os.path.samefile(rospack.get_path(p), '.'):
                    packages = [p]
                    self.printer.print_all(
                        "No package specified.  Building %s" % packages)
                else:
                    self.printer.print_all(
                        "No package selected and the current directory is not the correct path for package '%s'."
                        % p)

            except rospkg.ResourceNotFound as ex:
                try:
                    stack_dir = rosstack.get_path(p)
                    if os.path.samefile(stack_dir, '.'):
                        packages = [p]
                        self.printer.print_all(
                            "No package specified.  Building stack %s" %
                            packages)
                    else:
                        self.printer.print_all(
                            "No package or stack arguments and the current directory is not the correct path for stack '%s'. Stack directory is: %s."
                            % (p, rosstack.get_path(p)))
                except:
                    self.printer.print_all(
                        "No package or stack specified.  And current directory '%s' is not a package name or stack name."
                        % p)
        else:
            packages.extend(args)

        self.printer.print_all("Packages requested are: %s" % packages)

        # Setup logging
        if self.logging_enabled:
            date_time_stamp = "rosmake_output-" + time.strftime(
                "%Y%m%d-%H%M%S")
            if options.output_dir:
                #self.log_dir = os.path.join(os.getcwd(), options.output_dir, date_time_stamp);
                self.log_dir = os.path.abspath(options.output_dir)
            else:
                self.log_dir = os.path.join(rospkg.get_ros_home(), "rosmake",
                                            date_time_stamp)

            self.printer.print_all("Logging to directory %s" % self.log_dir)
            if os.path.exists(
                    self.log_dir) and not os.path.isdir(self.log_dir):
                self.printer.print_all(
                    "Log destination %s is a file; please remove it or choose a new destination"
                    % self.log_dir)
                sys.exit(1)
            if not os.path.exists(self.log_dir):
                self.printer.print_verbose("%s doesn't exist: creating" %
                                           self.log_dir)
                makedirs_with_parent_perms(self.log_dir)

            self.printer.print_verbose("Finished setting up logging")

        stacks_arguments = [s for s in packages if s in rosstack.list()]
        (self.specified_packages,
         self.rejected_packages) = rospkg.expand_to_packages(
             packages, rospack, rosstack)

        self.printer.print_all("Expanded args %s to:\n%s" %
                               (packages, self.specified_packages))
        if self.rejected_packages:
            self.printer.print_all(
                "WARNING: The following args could not be parsed as stacks or packages: %s"
                % self.rejected_packages)
        if len(self.specified_packages) + len(stacks_arguments) == 0:
            self.printer.print_all(
                "ERROR: No arguments could be parsed into valid package or stack names."
            )
            self.printer.running = False
            return False

        if options.unmark_installed:
            for p in self.specified_packages:
                if self.flag_tracker.remove_nobuild(p):
                    self.printer.print_all("Removed ROS_NOBUILD from %s" % p)
            self.printer.running = False
            return True

        required_packages = self.specified_packages[:]

        # catch packages of dependent stacks when specified stack is zero-sized #3528
        # add them to required list but not the specified list.
        for s in stacks_arguments:
            if not rosstack.packages_of(s):
                for d in rosstack.get_depends(s, implicit=False):
                    try:
                        required_packages.extend(rosstack.packages_of(d))
                    except ResourceNotFound:
                        self.printer.print_all(
                            'WARNING: The stack "%s" was not found. We will assume it is using the new buildsystem and try to continue...'
                            % d)

        # deduplicate required_packages
        required_packages = list(set(required_packages))

        # make sure all dependencies are satisfied and if not warn
        buildable_packages = []
        for p in required_packages:
            (buildable, error,
             str) = self.flag_tracker.can_build(p, self.skip_blacklist, [],
                                                False)
            if buildable:
                buildable_packages.append(p)

        #generate the list of packages necessary to build(in order of dependencies)
        counter = 0
        for p in required_packages:

            counter = counter + 1
            self.printer.print_verbose(
                "Processing %s and all dependencies(%d of %d requested)" %
                (p, counter, len(packages)))
            self.build_or_recurse(p)

        # remove extra packages if specified-only flag is set
        if options.specified_only:
            new_list = []
            for pkg in self.build_list:
                if pkg in self.specified_packages:
                    new_list.append(pkg)
                    self.dependency_tracker = parallel_build.DependencyTracker(
                        self.specified_packages, rospack=self.rospack
                    )  # this will make the tracker only respond to packages in the list

            self.printer.print_all(
                "specified-only option was used, only building packages %s" %
                new_list)
            self.build_list = new_list

        if options.pre_clean:
            build_queue = parallel_build.BuildQueue(
                self.build_list,
                parallel_build.DependencyTracker([], rospack=self.rospack),
                robust_build=True)
            self.parallel_build_pkgs(build_queue,
                                     "clean",
                                     threads=options.threads)

        build_passed = True

        if building:
            self.printer.print_verbose("Building packages %s" %
                                       self.build_list)
            build_queue = parallel_build.BuildQueue(self.build_list,
                                                    self.dependency_tracker,
                                                    robust_build=options.robust
                                                    or options.best_effort)
            if None not in self.result.keys():
                self.result[None] = {}

            build_passed = self.parallel_build_pkgs(build_queue,
                                                    options.target,
                                                    threads=options.threads)

        tests_passed = True
        if build_passed and testing:
            self.printer.print_verbose("Testing packages %s" % packages)
            build_queue = parallel_build.BuildQueue(
                self.specified_packages,
                parallel_build.DependencyTracker(self.specified_packages,
                                                 rospack=self.rospack),
                robust_build=True)
            tests_passed = self.parallel_build_pkgs(build_queue,
                                                    "test",
                                                    threads=1)

        if options.mark_installed:
            if build_passed and tests_passed:
                for p in self.specified_packages:
                    if self.flag_tracker.add_nobuild(p):
                        self.printer.print_all(
                            "Marking %s as installed with a ROS_NOBUILD file" %
                            p)
            else:
                self.printer.print_all(
                    "All builds and tests did not pass cannot mark packages as installed. "
                )

        self.finish_time = time.time()  #note: before profiling
        self.generate_summary_output(self.log_dir)

        if options.print_profile:
            self.printer.print_all(self.get_profile_string())

        self.printer.running = False
        return build_passed and tests_passed
Example #8
0
def _package_args_handler(command, parser, options, args):
    if options.rosdep_all:
        if args:
            parser.error("cannot specify additional arguments with -a")
        else:
            # let the loader filter the -a. This will take out some
            # packages that are catkinized (for now).
            lookup = _get_default_RosdepLookup(options)
            loader = lookup.get_loader()
            args = loader.get_loadable_resources()
            not_found = []
    elif not args:
        parser.error("no packages or stacks specified")

    # package or stack names as args.  have to convert stack names to packages.
    # - overrides to enable testing
    packages = []
    not_found = []
    if options.from_paths:
        for path in args:
            if options.verbose:
                print("Using argument '{0}' as a path to search.".format(path))
            if not os.path.exists(path):
                print("given path '{0}' does not exist".format(path))
                return 1
            path = os.path.abspath(path)
            if 'ROS_PACKAGE_PATH' not in os.environ:
                os.environ['ROS_PACKAGE_PATH'] = '{0}'.format(path)
            else:
                os.environ['ROS_PACKAGE_PATH'] += ':{0}'.format(path)
            pkgs = find_catkin_packages_in(path, options.verbose)
            packages.extend(pkgs)
        # Make packages list unique
        packages = list(set(packages))
    else:
        rospack = rospkg.RosPack()
        rosstack = rospkg.RosStack()
        val = rospkg.expand_to_packages(args, rospack, rosstack)
        packages = val[0]
        not_found = val[1]
    if not_found:
        raise rospkg.ResourceNotFound(not_found[0], rospack.get_ros_paths())

    # Handle the --ignore-src option
    if command in ['install', 'check'] and options.ignore_src:
        if options.verbose:
            print("Searching ROS_PACKAGE_PATH for "
                  "sources: " + str(os.environ['ROS_PACKAGE_PATH'].split(':')))
        ws_pkgs = get_workspace_packages()
        for path in os.environ['ROS_PACKAGE_PATH'].split(':'):
            path = os.path.abspath(path.strip())
            if os.path.exists(path):
                pkgs = find_catkin_packages_in(path, options.verbose)
                ws_pkgs.extend(pkgs)
            elif options.verbose:
                print("Skipping non-existent path " + path)
        set_workspace_packages(ws_pkgs)

    lookup = _get_default_RosdepLookup(options)

    if 0 and not packages:  # disable, let individual handlers specify behavior
        # possible with empty stacks
        print("No packages in arguments, aborting")
        return

    return command_handlers[command](lookup, packages, options)
Example #9
0
    def main(self):
        """
        main command-line entrypoint
        """
        parser = OptionParser(usage="usage: %prog [options] [PACKAGE]...",
                              description="rosmake recursively builds all dependencies before building a package", prog='rosmake')
        parser.add_option("--test-only", dest="test_only", default=False,
                          action="store_true", help="only run tests")
        parser.add_option("-t", dest="test", default=False,
                          action="store_true", help="build and test packages")
        parser.add_option("-a", "--all", dest="build_all", default=False,
                          action="store_true", help="select all packages")
        parser.add_option("-i", "--mark-installed", dest="mark_installed", default=False,
                          action="store_true", help="On successful build, mark specified packages as installed with ROS_NOBUILD")
        parser.add_option("-u", "--unmark-installed", dest="unmark_installed", default=False,
                          action="store_true", help="Remove ROS_NOBUILD from the specified packages.  This will not build anything.")
        parser.add_option("-v", dest="verbose", default=False,
                          action="store_true", help="display errored builds")
        parser.add_option("-r","-k", "--robust", dest="best_effort", default=False,
                           action="store_true", help="do not stop build on error")
        parser.add_option("--build-everything", dest="robust", default=False,
                           action="store_true", help="build all packages regardless of errors")
        parser.add_option("-V", dest="full_verbose", default=False,
                          action="store_true", help="display all builds")
        parser.add_option("-s", "--specified-only", dest="specified_only", default=False,
                          action="store_true", help="only build packages specified on the command line")
        parser.add_option("--buildtest", dest="buildtest",
                          action="append", help="package to buildtest")
        parser.add_option("--buildtest1", dest="buildtest1",
                          action="append", help="package to buildtest1")
        parser.add_option("--output", dest="output_dir",
                          action="store", help="where to output results")
        parser.add_option("--pre-clean", dest="pre_clean",
                          action="store_true", help="run make clean first")
        parser.add_option("--bootstrap", dest="bootstrap", default=False,
                          action="store_true", help="DEPRECATED, UNUSED")
        parser.add_option("--disable-logging", dest="logging_enabled", default=True,
                          action="store_false", help="turn off all logs")
        parser.add_option("--target", dest="target",
                          action="store", help="run make with this target")
        parser.add_option("--pjobs", dest="ros_parallel_jobs", type="int",
                          action="store", help="Override ROS_PARALLEL_JOBS environment variable with this number of jobs.")
        parser.add_option("--threads", dest="threads", type="int", default = os.environ.get("ROSMAKE_THREADS", parallel_build.num_cpus()),
                          action="store", help="Build up to N packages in parallel")
        parser.add_option("--profile", dest="print_profile", default=False,
                          action="store_true", help="print time profile after build")
        parser.add_option("--skip-blacklist", dest="skip_blacklist", 
                          default=False, action="store_true", 
                          help="skip packages containing a file called ROS_BUILD_BLACKLIST (Default behavior will ignore the presence of ROS_BUILD_BLACKLIST)")
        parser.add_option("--skip-blacklist-osx", dest="skip_blacklist_osx", 
                          default=False, action="store_true", 
                          help="deprecated option. it will do nothing, please use platform declarations and --require-platform instead")

        parser.add_option("--status-rate", dest="status_update_rate",
                          action="store", help="How fast to update the status bar in Hz.  Default: 5Hz")
        

        options, args = parser.parse_args()
        self.printer.print_all('rosmake starting...')

        rospack = self.rospack
        rosstack = self.rosstack

        testing = False
        building = True
        if options.test_only:
            testing = True
            building = False
        elif options.test:
            testing = True

        if options.ros_parallel_jobs:
            self.ros_parallel_jobs = options.ros_parallel_jobs

        self.robust_build = options.robust
        self.best_effort = options.best_effort
        self.threads = options.threads
        self.skip_blacklist = options.skip_blacklist
        if options.skip_blacklist_osx:
            self.printer.print_all("Option --skip-blacklist-osx is deprecated. It will do nothing, please use platform declarations and --require-platform instead");
        self.logging_enabled = options.logging_enabled

        # pass through verbosity options
        self.printer.full_verbose = options.full_verbose
        self.printer.verbose = options.verbose
        if options.status_update_rate:
            if float(options.status_update_rate)> 0:
                self.printer.duration = 1.0/float(options.status_update_rate)
            else:
                self.printer.duration = 0

        packages = []
        #load packages from arguments
        if options.build_all:
            packages = [x for x in rospack.list() if not self.rospack.get_manifest(x).is_catkin]
            self.printer.print_all( "Building all packages")
        else:      # no need to extend if all already selected   
            if options.buildtest:
              for p in options.buildtest:
                packages.extend(self.rospack.get_depends_on(p)) 
                self.printer.print_all( "buildtest requested for package %s adding it and all dependent packages: "%p)

            if options.buildtest1:
              for p in options.buildtest1:
                packages.extend(self.rospack.get_depends_on(p, implicit=False)) 
                self.printer.print_all( "buildtest1 requested for package %s adding it and all depends-on1 packages: "%p)

        if len(packages) == 0 and len(args) == 0:
            p = os.path.basename(os.path.abspath('.'))
            try:
              if os.path.samefile(rospack.get_path(p), '.'):
                packages = [p]
                self.printer.print_all( "No package specified.  Building %s"%packages)
              else:
                self.printer.print_all("No package selected and the current directory is not the correct path for package '%s'."%p)
                
            except rospkg.ResourceNotFound as ex:
                try:
                    stack_dir = rosstack.get_path(p)
                    if os.path.samefile(stack_dir, '.'):
                        packages = [p]
                        self.printer.print_all( "No package specified.  Building stack %s"%packages)
                    else:
                        self.printer.print_all("No package or stack arguments and the current directory is not the correct path for stack '%s'. Stack directory is: %s."%(p, rosstack.get_path(p)))
                except:
                    self.printer.print_all("No package or stack specified.  And current directory '%s' is not a package name or stack name."%p)
        else:
            packages.extend(args)

        self.printer.print_all( "Packages requested are: %s"%packages)
        
        # Setup logging
        if self.logging_enabled:
          date_time_stamp =  "rosmake_output-" + time.strftime("%Y%m%d-%H%M%S")
          if options.output_dir:
              #self.log_dir = os.path.join(os.getcwd(), options.output_dir, date_time_stamp);
              self.log_dir = os.path.abspath(options.output_dir)
          else:
              self.log_dir = os.path.join(rospkg.get_ros_home(), "rosmake", date_time_stamp);

          self.printer.print_all("Logging to directory %s"%self.log_dir)
          if os.path.exists (self.log_dir) and not os.path.isdir(self.log_dir):
              self.printer.print_all( "Log destination %s is a file; please remove it or choose a new destination"%self.log_dir)
              sys.exit(1)
          if not os.path.exists (self.log_dir):
              self.printer.print_verbose("%s doesn't exist: creating"%self.log_dir)
              makedirs_with_parent_perms(self.log_dir)

          self.printer.print_verbose("Finished setting up logging")

        stacks_arguments = [s for s in packages if s in rosstack.list()]
        (self.specified_packages, self.rejected_packages) = rospkg.expand_to_packages(packages, rospack, rosstack)

        self.printer.print_all("Expanded args %s to:\n%s"%(packages, self.specified_packages))
        if self.rejected_packages:
            self.printer.print_all("WARNING: The following args could not be parsed as stacks or packages: %s"%self.rejected_packages)
        if len(self.specified_packages) + len(stacks_arguments) == 0:
            self.printer.print_all("ERROR: No arguments could be parsed into valid package or stack names.")
            self.printer.running = False
            return False

        if options.unmark_installed:
            for p in self.specified_packages:
                if self.flag_tracker.remove_nobuild(p):
                    self.printer.print_all("Removed ROS_NOBUILD from %s"%p)
            self.printer.running = False
            return True
            
        required_packages = self.specified_packages[:]

        # catch dependent packages which are inside of zero sized stacks #3528 
        # add them to required list but not the specified list. 
        for s in stacks_arguments:
            for d in rosstack.get_depends(s, implicit=False):
                required_packages.extend(rosstack.packages_of(d))

        # deduplicate required_packages
        required_packages = list(set(required_packages))

        # make sure all dependencies are satisfied and if not warn
        buildable_packages = []
        for p in required_packages:
            (buildable, error, str) = self.flag_tracker.can_build(p, self.skip_blacklist, [], False)
            if buildable: 
                buildable_packages.append(p)

        #generate the list of packages necessary to build(in order of dependencies)
        counter = 0
        for p in required_packages:

            counter = counter + 1
            self.printer.print_verbose( "Processing %s and all dependencies(%d of %d requested)"%(p, counter, len(packages)))
            self.build_or_recurse(p)

        # remove extra packages if specified-only flag is set
        if options.specified_only:
          new_list = []
          for pkg in self.build_list:
            if pkg in self.specified_packages:
              new_list.append(pkg)
              self.dependency_tracker = parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack) # this will make the tracker only respond to packages in the list
        
          self.printer.print_all("specified-only option was used, only building packages %s"%new_list)
          self.build_list = new_list

        if options.pre_clean:
          build_queue = parallel_build.BuildQueue(self.build_list, parallel_build.DependencyTracker([], rospack=self.rospack), robust_build = True)
          self.parallel_build_pkgs(build_queue, "clean", threads = options.threads)

        build_passed = True

        if building:
          self.printer.print_verbose ("Building packages %s"% self.build_list)
          build_queue = parallel_build.BuildQueue(self.build_list, self.dependency_tracker, robust_build = options.robust or options.best_effort)
          if None not in self.result.keys():
                self.result[None] = {}

          build_passed = self.parallel_build_pkgs(build_queue, options.target, threads = options.threads)

        tests_passed = True
        if build_passed and testing:
            self.printer.print_verbose ("Testing packages %s"% packages)
            build_queue = parallel_build.BuildQueue(self.specified_packages, parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack), robust_build = True)
            tests_passed = self.parallel_build_pkgs(build_queue, "test", threads = 1)


        if  options.mark_installed:
            if build_passed and tests_passed: 
                for p in self.specified_packages:
                    if self.flag_tracker.add_nobuild(p):
                        self.printer.print_all("Marking %s as installed with a ROS_NOBUILD file"%p)
            else:
                self.printer.print_all("All builds and tests did not pass cannot mark packages as installed. ")


        self.finish_time = time.time() #note: before profiling
        self.generate_summary_output(self.log_dir)
        
        if options.print_profile:
            self.printer.print_all (self.get_profile_string())

        self.printer.running = False
        return build_passed and tests_passed