def __init__(self, num_jobs=None, max_load=None, max_mem=None): """ :param num_jobs: the maximum number of jobs available :param max_load: do not dispatch additional jobs if this system load value is exceeded :param max_mem: do not dispatch additional jobs if system physical memory usage exceeds this value (see _set_max_mem for additional documentation) """ assert(_MakeJobServer._singleton is None) if not num_jobs: try: num_jobs = cpu_count() except NotImplementedError: log('@{yf}WARNING: Failed to determine the cpu_count, falling back to 1 jobs as the default.@|') num_jobs = 1 else: num_jobs = int(num_jobs) self.num_jobs = num_jobs self.max_load = max_load self._set_max_mem(max_mem) self.job_pipe = os.pipe() # Initialize the pipe with num_jobs tokens for i in range(num_jobs): os.write(self.job_pipe[1], b'+')
def yes_no_loop(question): while True: resp = str(raw_input(question + " [yN]: ")) if resp.lower() in ['n', 'no'] or len(resp) == 0: return False elif resp.lower() in ['y', 'yes']: return True log(clr("[clean] Please answer either \"yes\" or \"no\"."))
def dry_run(context, packages, no_deps, start_with): # Print Summary log(context.summary()) # Find list of packages in the workspace packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built(packages, context) # Assert start_with package is in the workspace verify_start_with_option(start_with, packages, all_packages, packages_to_be_built + packages_to_be_built_deps) if not no_deps: # Extend packages to be built to include their deps packages_to_be_built.extend(packages_to_be_built_deps) # Also resort packages_to_be_built = topological_order_packages(dict(packages_to_be_built)) # Print packages log("Packages to be built:") max_name_len = str(max([len(pkg.name) for pth, pkg in packages_to_be_built])) prefix = clr('@{pf}' + ('------ ' if start_with else '- ') + '@|') for pkg_path, pkg in packages_to_be_built: build_type = get_build_type(pkg) if build_type == 'catkin' and 'metapackage' in [e.tagname for e in pkg.exports]: build_type = 'metapackage' if start_with and pkg.name == start_with: start_with = None log(clr("{prefix}@{cf}{name:<" + max_name_len + "}@| (@{yf}{build_type}@|)") .format(prefix=clr('@!@{kf}(skip)@| ') if start_with else prefix, name=pkg.name, build_type=build_type)) log("Total packages: " + str(len(packages_to_be_built)))
def safe_rmtree(path, workspace_root, force): """Safely remove a path outside of the workspace root.""" # Check if the path is inside the workspace path_in_workspace = path.find(workspace_root) == 0 yes = True if not path_in_workspace and not force: log(clr("[clean] Warning: `{}` is outside of the workspace root. (Use" " --force to skip this check)".format(path))) yes = yes_no_loop("Are you sure you want to entirely remove `{}`?".format(path)) if yes: shutil.rmtree(path) else: log("[clean] Not removing `{}`".format(path))
def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=False): """ Initialize the global GNU Make jobserver. :param max_jobs: the maximum number of jobs available :param max_load: do not dispatch additional jobs if this system load value is exceeded :param max_mem: do not dispatch additional jobs if system physical memory usage exceeds this value (see _set_max_mem for additional documentation) """ # Check initialization if JobServer._initialized is True: return # Check if the jobserver is supported if JobServer._gnu_make is None: JobServer._gnu_make = GnuMake() if not JobServer._gnu_make.is_supported(): log( clr('@!@{yf}WARNING:@| Make job server not supported. The number of Make ' 'jobs may exceed the number of CPU cores.@|')) # Set gnu make compatibilty enabled JobServer._gnu_make_enabled = gnu_make_enabled # Set the maximum number of jobs if max_jobs is None: try: max_jobs = cpu_count() except NotImplementedError: log('@{yf}WARNING: Failed to determine the cpu_count, falling back to 1 jobs as the default.@|' ) max_jobs = 1 else: max_jobs = int(max_jobs) JobServer._set_max_jobs(max_jobs) JobServer._max_load = max_load JobServer._set_max_mem(max_mem) JobServer._initialized = True
def determine_packages_to_be_built(packages, context): """Returns list of packages which should be built, and those package's deps. :param packages: list of packages to be built, if None all packages are built :type packages: list :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :returns: tuple of packages to be built and those package's deps :rtype: tuple """ start = time.time() workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True) # If there are no packages raise if not workspace_packages: sys.exit("No packages were found in the source space '{0}'".format(context.source_space_abs)) log("Found '{0}' packages in {1}." .format(len(workspace_packages), format_time_delta(time.time() - start))) # Order the packages by topology ordered_packages = topological_order_packages(workspace_packages) # Set the packages in the workspace for the context context.packages = ordered_packages # Determin the packages which should be built packages_to_be_built = [] packages_to_be_built_deps = [] if packages: # First assert all of the packages given are in the workspace workspace_package_names = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) for package in packages: if package not in workspace_package_names: sys.exit("Given package '{0}' is not in the workspace".format(package)) # If metapackage, include run depends which are in the workspace package_obj = workspace_package_names[package][1] if 'metapackage' in [e.tagname for e in package_obj.exports]: for rdep in package_obj.run_depends: if rdep.name in workspace_package_names: packages.append(rdep.name) # Limit the packages to be built to just the provided packages for pkg_path, package in ordered_packages: if package.name in packages: packages_to_be_built.append((pkg_path, package)) # Get the recursive dependencies for each of these packages pkg_deps = get_cached_recursive_build_depends_in_workspace(package, ordered_packages) packages_to_be_built_deps.extend(pkg_deps) else: packages_to_be_built = ordered_packages return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def create_package_job(context, package, package_path, deps): docs_space = os.path.join(context.build_space_abs, '..', 'docs', package.name) docs_build_space = os.path.join(context.build_space_abs, 'docs', package.name) package_path_abs = os.path.join(context.source_space_abs, package_path) # Load rosdoc config, if it exists. rosdoc_yaml_path = os.path.join(package_path_abs, 'rosdoc.yaml') if os.path.exists(rosdoc_yaml_path): with open(rosdoc_yaml_path) as f: rosdoc_conf = yaml.load(f) else: if os.path.exists(os.path.join(package_path_abs, 'src')) or \ os.path.exists(os.path.join(package_path_abs, 'include')): rosdoc_conf = [{'builder': 'doxygen'}] else: rosdoc_conf = [] stages = [] # Create package docs spaces. stages.append(FunctionStage('mkdir_docs_build_space', makedirs, path=docs_build_space)) # Generate msg/srv/action docs with package summary page. stages.append(FunctionStage('generate_messages', generate_messages, package=package, package_path=package_path, output_path=docs_build_space)) stages.append(FunctionStage('generate_services', generate_services, package=package, package_path=package_path, output_path=docs_build_space)) stages.append(FunctionStage('generate_package_summary', generate_package_summary, package=package, package_path=package_path_abs, rosdoc_conf=rosdoc_conf, output_path=docs_build_space)) # Add steps to run native doc generators, as appropriate. This has to happen after # the package summary generates, as we're going to override the subdirectory index.html # files generated by that sphinx run. for conf in rosdoc_conf: try: stages.extend(getattr(builders, conf['builder'])( conf, package, deps, docs_space, package_path_abs, docs_build_space)) except AttributeError: log(fmt("[document] @!@{yf}Warning:@| Skipping unrecognized rosdoc builder [%s] for package [%s]" % (conf['builder'], package.name))) return Job(jid=package.name, deps=deps, env={}, stages=stages)
def safe_rmtree(path, workspace_root, force): """Safely remove a path outside of the workspace root.""" # Check if the path is inside the workspace path_in_workspace = path.find(workspace_root) == 0 yes = True if not path_in_workspace and not force: log( clr("[clean] Warning: `{}` is outside of the workspace root. (Use" " --force to skip this check)").format(path)) yes = yes_no_loop( "Are you sure you want to entirely remove `{}`?".format(path)) if yes: shutil.rmtree(path) else: log("[clean] Not removing `{}`".format(path))
def initialize(max_jobs=None, max_load=None, max_mem=None, gnu_make_enabled=False): """ Initialize the global GNU Make jobserver. :param max_jobs: the maximum number of jobs available :param max_load: do not dispatch additional jobs if this system load value is exceeded :param max_mem: do not dispatch additional jobs if system physical memory usage exceeds this value (see _set_max_mem for additional documentation) """ # Check initialization if JobServer._initialized is True: return # Check if the jobserver is supported if JobServer._gnu_make is None: JobServer._gnu_make = GnuMake() if not JobServer._gnu_make.is_supported(): log(clr('@!@{yf}WARNING:@| Make job server not supported. The number of Make ' 'jobs may exceed the number of CPU cores.@|')) # Set gnu make compatibilty enabled JobServer._gnu_make_enabled = gnu_make_enabled # Set the maximum number of jobs if max_jobs is None: try: max_jobs = cpu_count() except NotImplementedError: log('@{yf}WARNING: Failed to determine the cpu_count, falling back to 1 jobs as the default.@|') max_jobs = 1 else: max_jobs = int(max_jobs) JobServer._set_max_jobs(max_jobs) JobServer._max_load = max_load JobServer._set_max_mem(max_mem) JobServer._initialized = True
def initialize_jobserver(*args, **kwargs): """ Initialize the global GNU Make jobserver. :param num_jobs: the maximum number of jobs available :param max_load: do not dispatch additional jobs if this system load value is exceeded :param max_mem: do not dispatch additional jobs if system physical memory usage exceeds this value """ assert(_MakeJobServer._singleton is None) # Check if the jobserver is supported supported = _test_support() if not supported: _MakeJobServer._singleton = None log('@{yf}WARNING: Make job server not supported. The number of Make ' 'jobs may exceed the number of CPU cores.@|') return # Create the jobserver singleton _MakeJobServer._singleton = _MakeJobServer(*args, **kwargs)
def dry_run(context, packages, no_deps, start_with): # Print Summary log(context.summary()) # Get all the packages in the context source space # Suppress warnings since this is a utility function workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) # Find list of packages in the workspace packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built( packages, context, workspace_packages) # Assert start_with package is in the workspace verify_start_with_option(start_with, packages, all_packages, packages_to_be_built + packages_to_be_built_deps) if not no_deps: # Extend packages to be built to include their deps packages_to_be_built.extend(packages_to_be_built_deps) # Also resort packages_to_be_built = topological_order_packages( dict(packages_to_be_built)) # Print packages log("Packages to be built:") max_name_len = str( max([len(pkg.name) for pth, pkg in packages_to_be_built])) prefix = clr('@{pf}' + ('------ ' if start_with else '- ') + '@|') for pkg_path, pkg in packages_to_be_built: build_type = get_build_type(pkg) if build_type == 'catkin' and 'metapackage' in [ e.tagname for e in pkg.exports ]: build_type = 'metapackage' if start_with and pkg.name == start_with: start_with = None log( clr("{prefix}@{cf}{name:<" + max_name_len + "}@| (@{yf}{build_type}@|)").format( prefix=clr('@!@{kf}(skip)@| ') if start_with else prefix, name=pkg.name, build_type=build_type)) log("Total packages: " + str(len(packages_to_be_built)))
def document_workspace( context, packages=None, start_with=None, no_deps=False, n_jobs=None, force_color=False, quiet=False, interleave_output=False, no_status=False, limit_status_rate=10.0, no_notify=False, continue_on_failure=False, summarize_build=None ): pre_start_time = time.time() # Get all the packages in the context source space # Suppress warnings since this is a utility function workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) # If no_deps is given, ensure packages to build are provided if no_deps and packages is None: log(fmt("[document] @!@{rf}Error:@| With no_deps, you must specify packages to build.")) return # Find list of packages in the workspace packages_to_be_documented, packages_to_be_documented_deps, all_packages = determine_packages_to_be_built( packages, context, workspace_packages) if not no_deps: # Extend packages to be documented to include their deps packages_to_be_documented.extend(packages_to_be_documented_deps) # Also re-sort try: packages_to_be_documented = topological_order_packages(dict(packages_to_be_documented)) except AttributeError: log(fmt("[document] @!@{rf}Error:@| The workspace packages have a circular " "dependency, and cannot be documented. Please run `catkin list " "--deps` to determine the problematic package(s).")) return # Check the number of packages to be documented if len(packages_to_be_documented) == 0: log(fmt('[document] No packages to be documented.')) # Assert start_with package is in the workspace verify_start_with_option( start_with, packages, all_packages, packages_to_be_documented + packages_to_be_documented_deps) # Remove packages before start_with if start_with is not None: for path, pkg in list(packages_to_be_documented): if pkg.name != start_with: wide_log(fmt("@!@{pf}Skipping@| @{gf}---@| @{cf}{}@|").format(pkg.name)) packages_to_be_documented.pop(0) else: break # Get the names of all packages to be built packages_to_be_documented_names = [p.name for _, p in packages_to_be_documented] packages_to_be_documeted_deps_names = [p.name for _, p in packages_to_be_documented_deps] jobs = [] # Construct jobs for pkg_path, pkg in all_packages: if pkg.name not in packages_to_be_documented_names: continue # Get actual execution deps deps = [ p.name for _, p in get_cached_recursive_build_depends_in_workspace(pkg, packages_to_be_documented) ] jobs.append(create_package_job(context, pkg, pkg_path, deps)) # Special job for post-job summary sphinx step. jobs.append(create_summary_job(context, package_names=packages_to_be_documented_names)) # Queue for communicating status event_queue = Queue() try: # Spin up status output thread status_thread = ConsoleStatusController( 'document', ['package', 'packages'], jobs, n_jobs, [pkg.name for _, pkg in context.packages], [p for p in context.whitelist], [p for p in context.blacklist], event_queue, show_notifications=not no_notify, show_active_status=not no_status, show_buffered_stdout=not quiet and not interleave_output, show_buffered_stderr=not interleave_output, show_live_stdout=interleave_output, show_live_stderr=interleave_output, show_stage_events=not quiet, show_full_summary=(summarize_build is True), pre_start_time=pre_start_time, active_status_rate=limit_status_rate) status_thread.start() # Block while running N jobs asynchronously try: all_succeeded = run_until_complete(execute_jobs( 'document', jobs, None, event_queue, context.log_space_abs, max_toplevel_jobs=n_jobs, continue_on_failure=continue_on_failure, continue_without_deps=False)) except Exception: status_thread.keep_running = False all_succeeded = False status_thread.join(1.0) wide_log(str(traceback.format_exc())) status_thread.join(1.0) return 0 if all_succeeded else 1 except KeyboardInterrupt: wide_log("[document] Interrupted by user!") event_queue.put(None) return 130 # EOWNERDEAD
def build_isolated_workspace( context, packages=None, start_with=None, no_deps=False, jobs=None, force_cmake=False, force_color=False, quiet=False, interleave_output=False, no_status=False, lock_install=False, no_notify=False ): """Builds a catkin workspace in isolation This function will find all of the packages in the source space, start some executors, feed them packages to build based on dependencies and topological ordering, and then monitor the output of the executors, handling loggings of the builds, starting builds, failing builds, and finishing builds of packages, and handling the shutdown of the executors when appropriate. :param context: context in which to build the catkin workspace :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :param packages: list of packages to build, by default their dependencies will also be built :type packages: list :param start_with: package to start with, skipping all packages which proceed it in the topological order :type start_with: str :param no_deps: If True, the dependencies of packages will not be built first :type no_deps: bool :param jobs: number of parallel package build jobs :type jobs: int :param force_cmake: forces invocation of CMake if True, default is False :type force_cmake: bool :param force_color: forces colored output even if terminal does not support it :type force_color: bool :param quiet: suppresses the output of commands unless there is an error :type quiet: bool :param interleave_output: prints the output of commands as they are received :type interleave_output: bool :param no_status: disables status bar :type no_status: bool :param lock_install: causes executors to synchronize on access of install commands :type lock_install: bool :param no_notify: suppresses system notifications :type no_notify: bool :raises: SystemExit if buildspace is a file or no packages were found in the source space or if the provided options are invalid """ # If no_deps is given, ensure packages to build are provided if no_deps and packages is None: sys.exit("With --no-deps, you must specify packages to build.") # Make sure there is a build folder and it is not a file if os.path.exists(context.build_space_abs): if os.path.isfile(context.build_space_abs): sys.exit(clr( "@{rf}Error:@| Build space '{0}' exists but is a file and not a folder." .format(context.build_space_abs))) # If it dosen't exist, create it else: log("Creating build space directory, '{0}'".format(context.build_space_abs)) os.makedirs(context.build_space_abs) # Check for catkin_make droppings if context.corrupted_by_catkin_make(): sys.exit( clr("@{rf}Error:@| Build space `{0}` exists but appears to have previously been " "created by the `catkin_make` or `catkin_make_isolated` tool. " "Please choose a different directory to use with `catkin build` " "or clean the build space.".format(context.build_space_abs))) # Declare a buildspace marker describing the build config for error checking buildspace_marker_data = { 'workspace': context.workspace, 'profile': context.profile, 'install': context.install, 'install_space': context.install_space_abs, 'devel_space': context.devel_space_abs, 'source_space': context.source_space_abs} # Check build config if os.path.exists(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)): with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)) as buildspace_marker_file: existing_buildspace_marker_data = yaml.load(buildspace_marker_file) misconfig_lines = '' for (k, v) in existing_buildspace_marker_data.items(): new_v = buildspace_marker_data.get(k, None) if new_v != v: misconfig_lines += ( '\n - %s: %s (stored) is not %s (commanded)' % (k, v, new_v)) if len(misconfig_lines) > 0: sys.exit(clr( "\n@{rf}Error:@| Attempting to build a catkin workspace using build space: " "\"%s\" but that build space's most recent configuration " "differs from the commanded one in ways which will cause " "problems. Fix the following options or use @{yf}`catkin " "clean -b`@| to remove the build space: %s" % (context.build_space_abs, misconfig_lines))) # Write the current build config for config error checking with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE), 'w') as buildspace_marker_file: buildspace_marker_file.write(yaml.dump(buildspace_marker_data, default_flow_style=False)) # Summarize the context summary_notes = [] if force_cmake: summary_notes += [clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.")] log(context.summary(summary_notes)) # Find list of packages in the workspace packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built(packages, context) completed_packages = [] if no_deps: # Consider deps as "completed" completed_packages.extend(packages_to_be_built_deps) else: # Extend packages to be built to include their deps packages_to_be_built.extend(packages_to_be_built_deps) # Also resort packages_to_be_built = topological_order_packages(dict(packages_to_be_built)) max_package_name_length = max([len(pkg.name) for pth, pkg in packages_to_be_built]) # Assert start_with package is in the workspace verify_start_with_option(start_with, packages, all_packages, packages_to_be_built + packages_to_be_built_deps) # Setup pool of executors executors = {} # The communication queue can have ExecutorEvent's or str's passed into it from the executors comm_queue = Queue() # The job queue has Jobs put into it job_queue = Queue() # Lock for install space install_lock = Lock() if lock_install else FakeLock() # Determine the number of executors try: if jobs: jobs = int(jobs) if jobs < 1: sys.exit("Specified number of jobs '{0}' is not positive.".format(jobs)) except ValueError: sys.exit("Specified number of jobs '{0}' is no integer.".format(jobs)) try: jobs = cpu_count() if jobs is None else jobs except NotImplementedError: log('Failed to determine the cpu_count, falling back to 1 jobs as the default.') jobs = 1 if jobs is None else jobs # If only one set of jobs, turn on interleaving to get more responsive feedback if jobs == 1: # TODO: make the system more intelligent so that it can automatically switch to streaming output # when only one job is building, even if multiple jobs could be building quiet = False interleave_output = True # Start the executors for x in range(jobs): e = Executor(x, context, comm_queue, job_queue, install_lock) executors[x] = e e.start() # Variables for tracking running jobs and built/building packages start = time.time() total_packages = len(packages_to_be_built) package_count = 0 running_jobs = {} log_dir = os.path.join(context.build_space_abs, 'build_logs') color = True if not force_color and not is_tty(sys.stdout): color = True out = OutputController(log_dir, quiet, interleave_output, color, max_package_name_length, prefix_output=(jobs > 1)) if no_status: disable_wide_log() # Prime the job_queue ready_packages = [] if start_with is None: ready_packages = get_ready_packages(packages_to_be_built, running_jobs, completed_packages) while start_with is not None: ready_packages.extend(get_ready_packages(packages_to_be_built, running_jobs, completed_packages)) while ready_packages: pth, pkg = ready_packages.pop(0) if pkg.name != start_with: completed_packages.append(pkg.name) package_count += 1 wide_log("[build] Skipping package '{0}'".format(pkg.name)) else: ready_packages.insert(0, (pth, pkg)) start_with = None break running_jobs = queue_ready_packages(ready_packages, running_jobs, job_queue, context, force_cmake) assert running_jobs error_state = False errors = [] def set_error_state(error_state): if error_state: return # Set the error state to prevent new jobs error_state = True # Empty the job queue while not job_queue.empty(): job_queue.get() # Kill the executors by sending a None to the job queue for each of them for x in range(jobs): job_queue.put(None) # While any executors are running, process executor events while executors: try: # Try to get an event from the communications queue try: event = comm_queue.get(True, 0.1) except Empty: # timeout occured, create null event to pass through checks event = ExecutorEvent(None, None, None, None) if event.event_type == 'job_started': package_count += 1 running_jobs[event.package]['package_number'] = package_count running_jobs[event.package]['start_time'] = time.time() out.job_started(event.package) if event.event_type == 'command_started': out.command_started(event.package, event.data['cmd'], event.data['location']) if event.event_type == 'command_log': out.command_log(event.package, event.data['message']) if event.event_type == 'command_failed': out.command_failed(event.package, event.data['cmd'], event.data['location'], event.data['retcode']) # Add to list of errors errors.append(event) # Remove the command from the running jobs del running_jobs[event.package] # If it hasn't already been done, stop the executors set_error_state(error_state) if event.event_type == 'command_finished': out.command_finished(event.package, event.data['cmd'], event.data['location'], event.data['retcode']) if event.event_type == 'job_finished': completed_packages.append(event.package) run_time = format_time_delta(time.time() - running_jobs[event.package]['start_time']) out.job_finished(event.package, run_time) del running_jobs[event.package] # If shutting down, do not add new packages if error_state: continue # Calculate new packages if not no_status: wide_log('[build] Calculating new jobs...', end='\r') sys.stdout.flush() ready_packages = get_ready_packages(packages_to_be_built, running_jobs, completed_packages) running_jobs = queue_ready_packages(ready_packages, running_jobs, job_queue, context, force_cmake) # Make sure there are jobs to be/being processed, otherwise kill the executors if not running_jobs: # Kill the executors by sending a None to the job queue for each of them for x in range(jobs): job_queue.put(None) # If an executor exit event, join it and remove it from the executors list if event.event_type == 'exit': # If an executor has an exception, set the error state if event.data['reason'] == 'exception': set_error_state(error_state) errors.append(event) # Join and remove it executors[event.executor_id].join() del executors[event.executor_id] if not no_status: # Update the status bar on the screen executing_jobs = [] for name, value in running_jobs.items(): number, start_time = value['package_number'], value['start_time'] if number is None or start_time is None: continue executing_jobs.append({ 'number': number, 'name': name, 'run_time': format_time_delta_short(time.time() - start_time) }) msg = clr("[build - {run_time}] ").format(run_time=format_time_delta_short(time.time() - start)) # If errors post those if errors: for error in errors: msg += clr("[!{package}] ").format(package=error.package) # Print them in order of started number for job_msg_args in sorted(executing_jobs, key=lambda args: args['number']): msg += clr("[{name} - {run_time}] ").format(**job_msg_args) msg_rhs = clr("[{0}/{1} Active | {2}/{3} Completed]").format( len(executing_jobs), len(executors), len(packages) if no_deps else len(completed_packages), total_packages ) # Update title bar sys.stdout.write("\x1b]2;[build] {0}/{1}\x07".format( len(packages) if no_deps else len(completed_packages), total_packages )) # Update status bar wide_log(msg, rhs=msg_rhs, end='\r') sys.stdout.flush() except KeyboardInterrupt: wide_log("[build] User interrupted, stopping.") set_error_state(error_state) # All executors have shutdown sys.stdout.write("\x1b]2;\x07") if not errors: if context.isolate_devel: if not context.install: _create_unmerged_devel_setup(context) else: _create_unmerged_devel_setup_for_install(context) wide_log("[build] Finished.") if not no_notify: notify("Build Finished", "{0} packages built".format(total_packages)) return 0 else: wide_log(clr("[build] There were @!@{rf}errors@|:")) if not no_notify: notify("Build Failed", "there were {0} errors".format(len(errors))) for error in errors: if error.event_type == 'exit': wide_log("""Executor '{exec_id}' had an unhandle exception while processing package '{package}': {data[exc]} """.format(exec_id=error.executor_id + 1, **error.__dict__)) else: wide_log(clr(""" @{rf}Failed@| to build package '@{cf}{package}@|' because the following command: @!@{kf}# Command run in directory: @|{location} {cmd.cmd_str} @{rf}Exited@| with return code: @!{retcode}@|""").format(package=error.package, **error.data)) sys.exit(1)
def main(opts): # Context-aware args if opts.build_this or opts.start_with_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) # Suppress warnings since this won't necessaraly find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package(search_start_path=getcwd(), ws_path=ws_path, warnings=[]) except (InvalidPackage, RuntimeError): this_package = None # Handle context-based package building if opts.build_this: if this_package: opts.packages += [this_package] else: sys.exit( "catkin build: --this was specified, but this directory is not in a catkin package." ) # If --start--with was used without any packages and --this was specified, start with this package if opts.start_with_this: if this_package: opts.start_with = this_package else: sys.exit( "catkin build: --this was specified, but this directory is not in a catkin package." ) if opts.no_deps and not opts.packages: sys.exit("With --no-deps, you must specify packages to build.") # Load the context ctx = Context.load(opts.workspace, opts.profile, opts, append=True) # Initialize the build configuration make_args, makeflags, cli_flags, jobserver = configure_make_args( ctx.make_args, ctx.use_internal_make_jobserver) # Set the jobserver memory limit if jobserver and opts.mem_limit: log( clr("@!@{pf}EXPERIMENTAL: limit memory to '%s'@|" % str(opts.mem_limit))) # At this point psuitl will be required, check for it and bail out if not set try: import psutil # noqa except ImportError as exc: log("Could not import psutil, but psutil is required when using --mem-limit." ) log("Please either install psutil or avoid using --mem-limit.") sys.exit("Exception: {0}".format(exc)) set_jobserver_max_mem(opts.mem_limit) ctx.make_args = make_args # Load the environment of the workspace to extend if ctx.extend_path is not None: try: load_resultspace_environment(ctx.extend_path) except IOError as exc: log( clr("@!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) return 1 # Display list and leave the file system untouched if opts.dry_run: dry_run(ctx, opts.packages, opts.no_deps, opts.start_with) return # Check if the context is valid before writing any metadata if not ctx.source_space_exists(): print("catkin build: error: Unable to find source space `%s`" % ctx.source_space_abs) return 1 # Always save the last context under the build verb update_metadata(ctx.workspace, ctx.profile, 'build', ctx.get_stored_dict()) build_metadata = get_metadata(ctx.workspace, ctx.profile, 'build') if build_metadata.get('needs_force', False): opts.force_cmake = True update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': False}) # Save the context as the configuration if opts.save_config: Context.save(ctx) start = time.time() try: return build_isolated_workspace( ctx, packages=opts.packages, start_with=opts.start_with, no_deps=opts.no_deps, jobs=opts.parallel_jobs, force_cmake=opts.force_cmake, force_color=opts.force_color, quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, limit_status_rate=opts.limit_status_rate, lock_install=not opts.no_install_lock, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize # Can be True, False, or None ) finally: log("[build] Runtime: {0}".format( format_time_delta(time.time() - start)))
def main(opts): # Context-aware args if opts.build_this or opts.start_with_this: # Determine the enclosing package try: this_package = find_enclosing_package() except InvalidPackage: pass # Handle context-based package building if opts.build_this: if this_package: opts.packages += [this_package] else: sys.exit( "catkin build: --this was specified, but this directory is not in a catkin package." ) # If --start--with was used without any packages and --this was specified, start with this package if opts.start_with_this: if this_package: opts.start_with = this_package else: sys.exit( "catkin build: --this was specified, but this directory is not in a catkin package." ) if opts.no_deps and not opts.packages: sys.exit("With --no-deps, you must specify packages to build.") if not opts.force_color and not is_tty(sys.stdout): set_color(False) # Load the context ctx = Context.Load(opts.workspace, opts.profile, opts) # Load the environment of the workspace to extend if ctx.extend_path is not None: try: load_resultspace_environment(ctx.extend_path) except IOError as exc: log( clr("@!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) return 1 # Display list and leave the filesystem untouched if opts.dry_run: dry_run(ctx, opts.packages, opts.no_deps, opts.start_with) return # Check if the context is valid before writing any metadata if not ctx.source_space_exists(): print("catkin build: error: Unable to find source space `%s`" % ctx.source_space_abs) return 1 # Always save the last context under the build verb update_metadata(ctx.workspace, ctx.profile, 'build', ctx.get_stored_dict()) build_metadata = get_metadata(ctx.workspace, ctx.profile, 'build') if build_metadata.get('needs_force', False): opts.force_cmake = True update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': False}) # Save the context as the configuration if opts.save_config: Context.Save(ctx) start = time.time() try: return build_isolated_workspace( ctx, packages=opts.packages, start_with=opts.start_with, no_deps=opts.no_deps, jobs=opts.parallel_jobs, force_cmake=opts.force_cmake, force_color=opts.force_color, quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, limit_status_rate=opts.limit_status_rate, lock_install=not opts.no_install_lock, no_notify=opts.no_notify) finally: log("[build] Runtime: {0}".format( format_time_delta(time.time() - start)))
def clean_profile(opts, profile): # Load the context ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False) if not ctx: if not opts.workspace: log( "[clean] Error: The current or desired workspace could not be " "determined. Please run `catkin clean` from within a catkin " "workspace or specify the workspace explicitly with the " "`--workspace` option.") else: log( "[clean] Error: Could not clean workspace \"%s\" because it " "either does not exist or it has no catkin_tools metadata." % opts.workspace) return False profile = ctx.profile # Check if the user wants to do something explicit actions = [ 'build', 'devel', 'install', 'logs', 'packages', 'orphans', 'deinit', 'setup_files'] logs_exists = os.path.exists(ctx.log_space_abs) build_exists = os.path.exists(ctx.build_space_abs) devel_exists = os.path.exists(ctx.devel_space_abs) install_path = ( os.path.join(ctx.destdir, ctx.install_space_abs.lstrip(os.sep)) if ctx.destdir else ctx.install_space_abs) install_exists = os.path.exists(install_path) # Default is to clean all products for this profile no_specific_action = not any([ v for (k, v) in vars(opts).items() if k in actions]) clean_all = opts.deinit or no_specific_action # Initialize action options if clean_all: opts.logs = opts.build = opts.devel = opts.install = True # Make sure the user intends to clena everything spaces_to_clean = (opts.logs or opts.build or opts.devel or opts.install) spaces_to_clean_msgs = [] if spaces_to_clean and not (opts.yes or opts.dry_run): if opts.logs and logs_exists: spaces_to_clean_msgs.append(clr("[clean] Log Space: @{yf}{}").format(ctx.log_space_abs)) if opts.build and build_exists: spaces_to_clean_msgs.append(clr("[clean] Build Space: @{yf}{}").format(ctx.build_space_abs)) if opts.devel and devel_exists: spaces_to_clean_msgs.append(clr("[clean] Devel Space: @{yf}{}").format(ctx.devel_space_abs)) if opts.install and install_exists: spaces_to_clean_msgs.append(clr("[clean] Install Space: @{yf}{}").format(install_path)) if len(spaces_to_clean_msgs) == 0 and not opts.deinit: log("[clean] Nothing to be cleaned for profile: `{}`".format(profile)) return True if len(spaces_to_clean_msgs) > 0: log("") log(clr("[clean] @!@{yf}Warning:@| This will completely remove the " "following directories. (Use `--yes` to skip this check)")) for msg in spaces_to_clean_msgs: log(msg) try: yes = yes_no_loop( "\n[clean] Are you sure you want to completely remove the directories listed above?") if not yes: log(clr("[clean] Not removing any workspace directories for" " this profile.")) return True except KeyboardInterrupt: log("\n[clean] No actions performed.") sys.exit(0) # Initialize flag to be used on the next invocation needs_force = False try: # Remove all installspace files if opts.install and install_exists: log("[clean] Removing installspace: %s" % install_path) if not opts.dry_run: safe_rmtree(install_path, ctx.workspace, opts.force) # Remove all develspace files if opts.devel: if devel_exists: log("[clean] Removing develspace: %s" % ctx.devel_space_abs) if not opts.dry_run: safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force) # Clear the cached metadata from the last build run _, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build') if os.path.exists(build_metadata_file): os.unlink(build_metadata_file) # Clear the cached packages data, if it exists packages_metadata_path = ctx.package_metadata_path() if os.path.exists(packages_metadata_path): safe_rmtree(packages_metadata_path, ctx.workspace, opts.force) # Remove all buildspace files if opts.build and build_exists: log("[clean] Removing buildspace: %s" % ctx.build_space_abs) if not opts.dry_run: safe_rmtree(ctx.build_space_abs, ctx.workspace, opts.force) # Setup file removal if opts.setup_files: if devel_exists: log("[clean] Removing setup files from develspace: %s" % ctx.devel_space_abs) opts.packages.append('catkin') opts.packages.append('catkin_tools_prebuild') else: log("[clean] No develspace exists, no setup files to clean.") # Clean log files if opts.logs and logs_exists: log("[clean] Removing log space: {}".format(ctx.log_space_abs)) if not opts.dry_run: safe_rmtree(ctx.log_space_abs, ctx.workspace, opts.force) # Find orphaned packages if ctx.link_devel and not any([opts.build, opts.devel]): if opts.orphans: if os.path.exists(ctx.build_space_abs): log("[clean] Determining orphaned packages...") # Get all existing packages in source space and the # Suppress warnings since this is looking for packages which no longer exist found_source_packages = [ pkg.name for (path, pkg) in find_packages(ctx.source_space_abs, warnings=[]).items()] built_packages = [ pkg.name for (path, pkg) in find_packages(ctx.package_metadata_path(), warnings=[]).items()] # Look for orphaned products in the build space orphans = [p for p in built_packages if (p not in found_source_packages and p != 'catkin_tools_prebuild')] if len(orphans) > 0: opts.packages.extend(list(orphans)) else: log("[clean] No orphans in the workspace.") else: log("[clean] No buildspace exists, no potential for orphans.") # Remove specific packages if len(opts.packages) > 0: try: # Clean the packages needs_force = clean_packages( ctx, opts.packages, opts.dependents, opts.verbose, opts.dry_run) except KeyboardInterrupt: wide_log("[build] User interrupted!") return False elif opts.orphans or len(opts.packages) > 0: log("[clean] Error: Individual packages can only be cleaned from " "workspaces with symbolically-linked develspaces (`catkin " "config --link-devel`).") except: log("[clean] Failed to clean profile `{}`".format(profile)) needs_force = True raise finally: if needs_force: log(clr( "[clean] @/@!Note:@| @/Parts of the workspace have been cleaned which will " "necessitate re-configuring CMake on the next build.@|")) update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': True}) return True
def main(opts): # Context-aware args if opts.build_this or opts.start_with_this: # Determine the enclosing package try: this_package = find_enclosing_package() except InvalidPackage: pass # Handle context-based package building if opts.build_this: if this_package: opts.packages += [this_package] else: sys.exit("catkin build: --this was specified, but this directory is not contained by a catkin package.") # If --start--with was used without any packages and --this was specified, start with this package if opts.start_with_this: if this_package: opts.start_with = this_package else: sys.exit("catkin build: --this was specified, but this directory is not contained by a catkin package.") if opts.no_deps and not opts.packages: sys.exit("With --no-deps, you must specify packages to build.") if not opts.force_color and not is_tty(sys.stdout): set_color(False) # Load the context ctx = Context.Load(opts.workspace, opts.profile, opts) # Load the environment of the workspace to extend if ctx.extend_path is not None: try: load_resultspace_environment(ctx.extend_path) except IOError as exc: log(clr("@!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) return 1 # Display list and leave the filesystem untouched if opts.dry_run: dry_run(ctx, opts.packages, opts.no_deps, opts.start_with) return # Check if the context is valid before writing any metadata if not ctx.source_space_exists(): print("catkin build: error: Unable to find source space `%s`" % ctx.source_space_abs) return 1 # Always save the last context under the build verb update_metadata(ctx.workspace, ctx.profile, 'build', ctx.get_stored_dict()) build_metadata = get_metadata(ctx.workspace, ctx.profile, 'build') if build_metadata.get('needs_force', False): opts.force_cmake = True update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': False}) # Save the context as the configuration if opts.save_config: Context.Save(ctx) start = time.time() try: return build_isolated_workspace( ctx, packages=opts.packages, start_with=opts.start_with, no_deps=opts.no_deps, jobs=opts.parallel_jobs, force_cmake=opts.force_cmake, force_color=opts.force_color, quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, lock_install=not opts.no_install_lock, no_notify=opts.no_notify ) finally: log("[build] Runtime: {0}".format(format_time_delta(time.time() - start)))
def build_isolated_workspace(context, packages=None, start_with=None, no_deps=False, jobs=None, force_cmake=False, force_color=False, quiet=False, interleave_output=False, no_status=False, lock_install=False, no_notify=False): """Builds a catkin workspace in isolation This function will find all of the packages in the source space, start some executors, feed them packages to build based on dependencies and topological ordering, and then monitor the output of the executors, handling loggings of the builds, starting builds, failing builds, and finishing builds of packages, and handling the shutdown of the executors when appropriate. :param context: context in which to build the catkin workspace :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :param packages: list of packages to build, by default their dependencies will also be built :type packages: list :param start_with: package to start with, skipping all packages which proceed it in the topological order :type start_with: str :param no_deps: If True, the dependencies of packages will not be built first :type no_deps: bool :param jobs: number of parallel package build jobs :type jobs: int :param force_cmake: forces invocation of CMake if True, default is False :type force_cmake: bool :param force_color: forces colored output even if terminal does not support it :type force_color: bool :param quiet: suppresses the output of commands unless there is an error :type quiet: bool :param interleave_output: prints the output of commands as they are received :type interleave_output: bool :param no_status: disables status bar :type no_status: bool :param lock_install: causes executors to synchronize on access of install commands :type lock_install: bool :param no_notify: suppresses system notifications :type no_notify: bool :raises: SystemExit if buildspace is a file or no packages were found in the source space or if the provided options are invalid """ # If no_deps is given, ensure packages to build are provided if no_deps and packages is None: sys.exit("With --no-deps, you must specify packages to build.") # Make sure there is a build folder and it is not a file if os.path.exists(context.build_space_abs): if os.path.isfile(context.build_space_abs): sys.exit( clr("@{rf}Error:@| Build space '{0}' exists but is a file and not a folder." .format(context.build_space_abs))) # If it dosen't exist, create it else: log("Creating build space directory, '{0}'".format( context.build_space_abs)) os.makedirs(context.build_space_abs) # Check for catkin_make droppings if context.corrupted_by_catkin_make(): sys.exit( clr("@{rf}Error:@| Build space `{0}` exists but appears to have previously been " "created by the `catkin_make` or `catkin_make_isolated` tool. " "Please choose a different directory to use with `catkin build` " "or clean the build space.".format(context.build_space_abs))) # Declare a buildspace marker describing the build config for error checking buildspace_marker_data = { 'workspace': context.workspace, 'profile': context.profile, 'install': context.install, 'install_space': context.install_space_abs, 'devel_space': context.devel_space_abs, 'source_space': context.source_space_abs } # Check build config if os.path.exists( os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)): with open(os.path.join( context.build_space_abs, BUILDSPACE_MARKER_FILE)) as buildspace_marker_file: existing_buildspace_marker_data = yaml.load(buildspace_marker_file) misconfig_lines = '' for (k, v) in existing_buildspace_marker_data.items(): new_v = buildspace_marker_data.get(k, None) if new_v != v: misconfig_lines += ( '\n - %s: %s (stored) is not %s (commanded)' % (k, v, new_v)) if len(misconfig_lines) > 0: sys.exit( clr("\n@{rf}Error:@| Attempting to build a catkin workspace using build space: " "\"%s\" but that build space's most recent configuration " "differs from the commanded one in ways which will cause " "problems. Fix the following options or use @{yf}`catkin " "clean -b`@| to remove the build space: %s" % (context.build_space_abs, misconfig_lines))) # Write the current build config for config error checking with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE), 'w') as buildspace_marker_file: buildspace_marker_file.write( yaml.dump(buildspace_marker_data, default_flow_style=False)) # Summarize the context summary_notes = [] if force_cmake: summary_notes += [ clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.") ] log(context.summary(summary_notes)) # Find list of packages in the workspace packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built( packages, context) completed_packages = [] if no_deps: # Consider deps as "completed" completed_packages.extend(packages_to_be_built_deps) else: # Extend packages to be built to include their deps packages_to_be_built.extend(packages_to_be_built_deps) # Also resort packages_to_be_built = topological_order_packages( dict(packages_to_be_built)) max_package_name_length = max( [len(pkg.name) for pth, pkg in packages_to_be_built]) # Assert start_with package is in the workspace verify_start_with_option(start_with, packages, all_packages, packages_to_be_built + packages_to_be_built_deps) # Setup pool of executors executors = {} # The communication queue can have ExecutorEvent's or str's passed into it from the executors comm_queue = Queue() # The job queue has Jobs put into it job_queue = Queue() # Lock for install space install_lock = Lock() if lock_install else FakeLock() # Determine the number of executors try: if jobs: jobs = int(jobs) if jobs < 1: sys.exit( "Specified number of jobs '{0}' is not positive.".format( jobs)) except ValueError: sys.exit("Specified number of jobs '{0}' is no integer.".format(jobs)) try: jobs = cpu_count() if jobs is None else jobs except NotImplementedError: log('Failed to determine the cpu_count, falling back to 1 jobs as the default.' ) jobs = 1 if jobs is None else jobs # If only one set of jobs, turn on interleaving to get more responsive feedback if jobs == 1: # TODO: make the system more intelligent so that it can automatically switch to streaming output # when only one job is building, even if multiple jobs could be building quiet = False interleave_output = True # Start the executors for x in range(jobs): e = Executor(x, context, comm_queue, job_queue, install_lock) executors[x] = e e.start() try: # Finally close out now running executors # Variables for tracking running jobs and built/building packages start = time.time() total_packages = len(packages_to_be_built) package_count = 0 running_jobs = {} log_dir = os.path.join(context.build_space_abs, 'build_logs') color = True if not force_color and not is_tty(sys.stdout): color = True out = OutputController(log_dir, quiet, interleave_output, color, max_package_name_length, prefix_output=(jobs > 1)) if no_status: disable_wide_log() # Prime the job_queue ready_packages = [] if start_with is None: ready_packages = get_ready_packages(packages_to_be_built, running_jobs, completed_packages) while start_with is not None: ready_packages.extend( get_ready_packages(packages_to_be_built, running_jobs, completed_packages)) while ready_packages: pth, pkg = ready_packages.pop(0) if pkg.name != start_with: completed_packages.append(pkg.name) package_count += 1 wide_log("[build] Skipping package '{0}'".format(pkg.name)) else: ready_packages.insert(0, (pth, pkg)) start_with = None break running_jobs = queue_ready_packages(ready_packages, running_jobs, job_queue, context, force_cmake) assert running_jobs error_state = False errors = [] def set_error_state(error_state): if error_state: return # Set the error state to prevent new jobs error_state = True # Empty the job queue while not job_queue.empty(): job_queue.get() # Kill the executors by sending a None to the job queue for each of them for x in range(jobs): job_queue.put(None) # While any executors are running, process executor events while executors: try: # Try to get an event from the communications queue try: event = comm_queue.get(True, 0.1) except Empty: # timeout occured, create null event to pass through checks event = ExecutorEvent(None, None, None, None) if event.event_type == 'job_started': package_count += 1 running_jobs[ event.package]['package_number'] = package_count running_jobs[event.package]['start_time'] = time.time() out.job_started(event.package) if event.event_type == 'command_started': out.command_started(event.package, event.data['cmd'], event.data['location']) if event.event_type == 'command_log': out.command_log(event.package, event.data['message']) if event.event_type == 'command_failed': out.command_failed(event.package, event.data['cmd'], event.data['location'], event.data['retcode']) # Add to list of errors errors.append(event) # Remove the command from the running jobs del running_jobs[event.package] # If it hasn't already been done, stop the executors set_error_state(error_state) if event.event_type == 'command_finished': out.command_finished(event.package, event.data['cmd'], event.data['location'], event.data['retcode']) if event.event_type == 'job_finished': completed_packages.append(event.package) run_time = format_time_delta( time.time() - running_jobs[event.package]['start_time']) out.job_finished(event.package, run_time) del running_jobs[event.package] # If shutting down, do not add new packages if error_state: continue # Calculate new packages if not no_status: wide_log('[build] Calculating new jobs...', end='\r') sys.stdout.flush() ready_packages = get_ready_packages( packages_to_be_built, running_jobs, completed_packages) running_jobs = queue_ready_packages( ready_packages, running_jobs, job_queue, context, force_cmake) # Make sure there are jobs to be/being processed, otherwise kill the executors if not running_jobs: # Kill the executors by sending a None to the job queue for each of them for x in range(jobs): job_queue.put(None) # If an executor exit event, join it and remove it from the executors list if event.event_type == 'exit': # If an executor has an exception, set the error state if event.data['reason'] == 'exception': set_error_state(error_state) errors.append(event) # Join and remove it executors[event.executor_id].join() del executors[event.executor_id] if not no_status: # Update the status bar on the screen executing_jobs = [] for name, value in running_jobs.items(): number, job, start_time = value[ 'package_number'], value['job'], value[ 'start_time'] if number is None or start_time is None: continue executing_jobs.append({ 'number': number, 'name': name, 'run_time': format_time_delta_short(time.time() - start_time) }) msg = clr("[build - {run_time}] ").format( run_time=format_time_delta_short(time.time() - start)) # If errors post those if errors: for error in errors: msg += clr("[!{package}] ").format( package=error.package) # Print them in order of started number for job_msg_args in sorted( executing_jobs, key=lambda args: args['number']): msg += clr("[{name} - {run_time}] ").format( **job_msg_args) msg_rhs = clr( "[{0}/{1} Active | {2}/{3} Completed]").format( len(executing_jobs), len(executors), len(packages) if no_deps else len(completed_packages), total_packages) # Update title bar sys.stdout.write("\x1b]2;[build] {0}/{1}\x07".format( len(packages) if no_deps else len(completed_packages), total_packages)) # Update status bar wide_log(msg, rhs=msg_rhs, end='\r') sys.stdout.flush() except KeyboardInterrupt: wide_log("[build] User interrupted, stopping.") set_error_state(error_state) # All executors have shutdown sys.stdout.write("\x1b]2;\x07") if not errors: if context.isolate_devel: if not context.install: _create_unmerged_devel_setup(context) else: _create_unmerged_devel_setup_for_install(context) wide_log("[build] Finished.") if not no_notify: notify("Build Finished", "{0} packages built".format(total_packages)) return 0 else: wide_log(clr("[build] There were @!@{rf}errors@|:")) if not no_notify: notify("Build Failed", "there were {0} errors".format(len(errors))) for error in errors: if error.event_type == 'exit': wide_log( """Executor '{exec_id}' had an unhandle exception while processing package '{package}': {data[exc]} """.format(exec_id=error.executor_id + 1, **error.__dict__)) else: wide_log( clr(""" @{rf}Failed@| to build package '@{cf}{package}@|' because the following command: @!@{kf}# Command run in directory: @|{location} {cmd.cmd_str} @{rf}Exited@| with return code: @!{retcode}@|""").format( package=error.package, **error.data)) sys.exit(1) finally: # Ensure executors go down for x in range(jobs): job_queue.put(None)
class JobServer(object): # Whether the job server has been initialized _initialized = False # Flag designating whether the `make` program supports the GNU Make # jobserver interface _gnu_make_supported = None # Initialize variables _load_ok = True _mem_ok = True _internal_jobs = [] _max_load = 0 _max_jobs = 0 _job_pipe = os.pipe() # Setting fd inheritance is required in Python > 3.4 # This is set by default in Python 2.7 # For more info see: https://docs.python.org/3.4/library/os.html#fd-inheritance if hasattr(os, 'set_inheritable'): for fd in _job_pipe: os.set_inheritable(fd, True) if not os.get_inheritable(fd): log( clr('@{yf}@!Warning: jobserver file descriptors are not inheritable.@|' )) @classmethod def _set_max_jobs(cls, max_jobs): """Set the maximum number of jobs to be used with the jobserver. This will wait for all active jobs to be completed, then re-initialize the job pipe. """ # Read all possible tokens from the pipe try: os.read(cls._job_pipe[0], cls._max_jobs) except OSError as e: if e.errno != errno.EINTR: raise # Update max jobs cls._max_jobs = max_jobs # Initialize the pipe with max_jobs tokens for i in range(cls._max_jobs): os.write(cls._job_pipe[1], b'+') @classmethod def _set_max_mem(cls, max_mem): """ Set the maximum memory to keep instantiating jobs. :param max_mem: String describing the maximum memory that can be used on the system. It can either describe memory percentage or absolute amount. Use 'P%' for percentage or 'N' for absolute value in bytes, 'Nk' for kilobytes, 'Nm' for megabytes, and 'Ng' for gigabytes. :type max_mem: str """ if max_mem is None: cls._max_mem = None return elif type(max_mem) is float or type(max_mem) is int: mem_percent = max_mem elif type(max_mem) is str: m_percent = re.search(r'([0-9]+)\%', max_mem) m_abs = re.search(r'([0-9]+)([kKmMgG]{0,1})', max_mem) if m_percent is None and m_abs is None: cls._max_mem = None return if m_percent: mem_percent = m_abs.group(1) elif m_abs: val = float(m_abs.group(1)) mag_symbol = m_abs.group(2) _, total_mem = memory_usage() if mag_symbol == '': mag = 1.0 elif mag_symbol.lower() == 'k': mag = 1024.0 elif mag_symbol.lower() == 'm': mag = pow(1024.0, 2) elif mag_symbol.lower() == 'g': mag = pow(1024.0, 3) mem_percent = 100.0 * val * mag / total_mem cls._max_mem = max(0.0, min(100.0, float(mem_percent))) @classmethod def _check_load(cls): if cls._max_load is not None: try: load = os.getloadavg() if load[1] < cls._max_load: cls._load_ok = True else: cls._load_ok = False except NotImplementedError: cls._load_ok = True return cls._load_ok @classmethod def _check_mem(cls): if cls._max_mem is not None: mem_used, mem_total = memory_usage() mem_percent_used = 100.0 * float(mem_used) / float(mem_total) if mem_percent_used > cls._max_mem: cls._mem_ok = False else: cls._mem_ok = True return cls._mem_ok @classmethod def _check_conditions(cls): return (cls._check_load() and cls._check_mem()) or cls._running_jobs() == 0 @classmethod def _acquire(cls): """ Obtain a job server token. Be sure to call _release() to avoid deadlocks. """ try: # read a token from the job pipe token = os.read(cls._job_pipe[0], 1) return token except OSError as e: if e.errno != errno.EINTR: raise return None @classmethod def _release(cls): """ Write a token to the job pipe. """ os.write(cls._job_pipe[1], b'+') @classmethod def _running_jobs(cls): try: buf = array.array('i', [0]) if fcntl.ioctl(cls._job_pipe[0], FIONREAD, buf) == 0: return cls._max_jobs - buf[0] except NotImplementedError: pass except OSError: pass return cls._max_jobs
def main(opts): # Check for develdebug mode if opts.develdebug is not None: os.environ['TROLLIUSDEBUG'] = opts.develdebug.lower() logging.basicConfig(level=opts.develdebug.upper()) # Set color options opts.force_color = os.environ.get('CATKIN_TOOLS_FORCE_COLOR', opts.force_color) if (opts.force_color or is_tty(sys.stdout)) and not opts.no_color: set_color(True) else: set_color(False) # Context-aware args if opts.build_this or opts.start_with_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) # Suppress warnings since this won't necessaraly find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package(search_start_path=getcwd(), ws_path=ws_path, warnings=[]) except (InvalidPackage, RuntimeError): this_package = None # Handle context-based package building if opts.build_this: if this_package: opts.packages += [this_package] else: sys.exit( "[build] Error: In order to use --this, the current directory must be part of a catkin package." ) # If --start--with was used without any packages and --this was specified, start with this package if opts.start_with_this: if this_package: opts.start_with = this_package else: sys.exit( "[build] Error: In order to use --this, the current directory must be part of a catkin package." ) if opts.no_deps and not opts.packages and not opts.unbuilt: sys.exit( clr("[build] @!@{rf}Error:@| With --no-deps, you must specify packages to build." )) # Load the context ctx = Context.load(opts.workspace, opts.profile, opts, append=True) # Initialize the build configuration make_args, makeflags, cli_flags, jobserver = configure_make_args( ctx.make_args, ctx.jobs_args, ctx.use_internal_make_jobserver) # Set the jobserver memory limit if jobserver and opts.mem_limit: log( clr("@!@{pf}EXPERIMENTAL: limit memory to '%s'@|" % str(opts.mem_limit))) # At this point psuitl will be required, check for it and bail out if not set try: import psutil # noqa except ImportError as exc: log("Could not import psutil, but psutil is required when using --mem-limit." ) log("Please either install psutil or avoid using --mem-limit.") sys.exit("Exception: {0}".format(exc)) job_server.set_max_mem(opts.mem_limit) ctx.make_args = make_args # Load the environment of the workspace to extend if ctx.extend_path is not None: try: load_resultspace_environment(ctx.extend_path) except IOError as exc: sys.exit( clr("[build] @!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) # Check if the context is valid before writing any metadata if not ctx.source_space_exists(): sys.exit( clr("[build] @!@{rf}Error:@| Unable to find source space `%s`") % ctx.source_space_abs) # ensure the build space was previously built by catkin_tools previous_tool = get_previous_tool_used_on_the_space(ctx.build_space_abs) if previous_tool is not None and previous_tool != 'catkin build': if opts.override_build_tool_check: log( clr("@{yf}Warning: build space at '%s' was previously built by '%s', " "but --override-build-tool-check was passed so continuing anyways." % (ctx.build_space_abs, previous_tool))) else: sys.exit( clr("@{rf}The build space at '%s' was previously built by '%s'. " "Please remove the build space or pick a different build space." % (ctx.build_space_abs, previous_tool))) # the build space will be marked as catkin build's if dry run doesn't return # ensure the devel space was previously built by catkin_tools previous_tool = get_previous_tool_used_on_the_space(ctx.devel_space_abs) if previous_tool is not None and previous_tool != 'catkin build': if opts.override_build_tool_check: log( clr("@{yf}Warning: devel space at '%s' was previously built by '%s', " "but --override-build-tool-check was passed so continuing anyways." % (ctx.devel_space_abs, previous_tool))) else: sys.exit( clr("@{rf}The devel space at '%s' was previously built by '%s'. " "Please remove the devel space or pick a different devel space." % (ctx.devel_space_abs, previous_tool))) # the devel space will be marked as catkin build's if dry run doesn't return # Display list and leave the file system untouched if opts.dry_run: # TODO: Add unbuilt dry_run(ctx, opts.packages, opts.no_deps, opts.start_with) return # Print the build environment for a given package and leave the filesystem untouched if opts.get_env: return print_build_env(ctx, opts.get_env[0]) # Now mark the build and devel spaces as catkin build's since dry run didn't return. mark_space_as_built_by(ctx.build_space_abs, 'catkin build') mark_space_as_built_by(ctx.devel_space_abs, 'catkin build') # Get the last build context build_metadata = get_metadata(ctx.workspace, ctx.profile, 'build') # Force cmake if the CMake arguments have changed if build_metadata.get('cmake_args') != ctx.cmake_args: opts.force_cmake = True # Check the devel layout compatibility last_devel_layout = build_metadata.get('devel_layout', ctx.devel_layout) if last_devel_layout != ctx.devel_layout: sys.exit( clr("@{rf}@!Error:@|@{rf} The current devel space layout, `{}`," "is incompatible with the configured layout, `{}`.@|").format( last_devel_layout, ctx.devel_layout)) # Check if some other verb has changed the workspace in such a way that it needs to be forced if build_metadata.get('needs_force', False): opts.force_cmake = True update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': False}) # Always save the last context under the build verb update_metadata(ctx.workspace, ctx.profile, 'build', ctx.get_stored_dict()) # Save the context as the configuration if opts.save_config: Context.save(ctx) # Get parallel toplevel jobs try: parallel_jobs = int(opts.parallel_jobs) except TypeError: parallel_jobs = None # Set VERBOSE environment variable if opts.verbose: os.environ['VERBOSE'] = '1' return build_isolated_workspace( ctx, packages=opts.packages, start_with=opts.start_with, no_deps=opts.no_deps, unbuilt=opts.unbuilt, n_jobs=parallel_jobs, force_cmake=opts.force_cmake, pre_clean=opts.pre_clean, force_color=opts.force_color, quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, limit_status_rate=opts.limit_status_rate, lock_install=not opts.no_install_lock, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize # Can be True, False, or None )
def main(opts): # Check for develdebug mode if opts.develdebug is not None: os.environ['TROLLIUSDEBUG'] = opts.develdebug.lower() logging.basicConfig(level=opts.develdebug.upper()) # Set color options if (opts.force_color or is_tty(sys.stdout)) and not opts.no_color: set_color(True) else: set_color(False) # Context-aware args if opts.build_this or opts.start_with_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) # Suppress warnings since this won't necessaraly find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( search_start_path=getcwd(), ws_path=ws_path, warnings=[]) except (InvalidPackage, RuntimeError): this_package = None # Handle context-based package building if opts.build_this: if this_package: opts.packages += [this_package] else: sys.exit( "[build] Error: In order to use --this, the current directory must be part of a catkin package.") # If --start--with was used without any packages and --this was specified, start with this package if opts.start_with_this: if this_package: opts.start_with = this_package else: sys.exit( "[build] Error: In order to use --this, the current directory must be part of a catkin package.") if opts.no_deps and not opts.packages and not opts.unbuilt: sys.exit(clr("[build] @!@{rf}Error:@| With --no-deps, you must specify packages to build.")) # Load the context ctx = Context.load(opts.workspace, opts.profile, opts, append=True) # Initialize the build configuration make_args, makeflags, cli_flags, jobserver = configure_make_args( ctx.make_args, ctx.jobs_args, ctx.use_internal_make_jobserver) # Set the jobserver memory limit if jobserver and opts.mem_limit: log(clr("@!@{pf}EXPERIMENTAL: limit memory to '%s'@|" % str(opts.mem_limit))) # At this point psuitl will be required, check for it and bail out if not set try: import psutil # noqa except ImportError as exc: log("Could not import psutil, but psutil is required when using --mem-limit.") log("Please either install psutil or avoid using --mem-limit.") sys.exit("Exception: {0}".format(exc)) job_server.set_max_mem(opts.mem_limit) ctx.make_args = make_args # Load the environment of the workspace to extend if ctx.extend_path is not None: try: load_resultspace_environment(ctx.extend_path) except IOError as exc: sys.exit(clr("[build] @!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) # Check if the context is valid before writing any metadata if not ctx.source_space_exists(): sys.exit(clr("[build] @!@{rf}Error:@| Unable to find source space `%s`") % ctx.source_space_abs) # ensure the build space was previously built by catkin_tools previous_tool = get_previous_tool_used_on_the_space(ctx.build_space_abs) if previous_tool is not None and previous_tool != 'catkin build': if opts.override_build_tool_check: log(clr( "@{yf}Warning: build space at '%s' was previously built by '%s', " "but --override-build-tool-check was passed so continuing anyways." % (ctx.build_space_abs, previous_tool))) else: sys.exit(clr( "@{rf}The build space at '%s' was previously built by '%s'. " "Please remove the build space or pick a different build space." % (ctx.build_space_abs, previous_tool))) # the build space will be marked as catkin build's if dry run doesn't return # ensure the devel space was previously built by catkin_tools previous_tool = get_previous_tool_used_on_the_space(ctx.devel_space_abs) if previous_tool is not None and previous_tool != 'catkin build': if opts.override_build_tool_check: log(clr( "@{yf}Warning: devel space at '%s' was previously built by '%s', " "but --override-build-tool-check was passed so continuing anyways." % (ctx.devel_space_abs, previous_tool))) else: sys.exit(clr( "@{rf}The devel space at '%s' was previously built by '%s'. " "Please remove the devel space or pick a different devel space." % (ctx.devel_space_abs, previous_tool))) # the devel space will be marked as catkin build's if dry run doesn't return # Display list and leave the file system untouched if opts.dry_run: # TODO: Add unbuilt dry_run(ctx, opts.packages, opts.no_deps, opts.start_with) return # Print the build environment for a given package and leave the filesystem untouched if opts.get_env: return print_build_env(ctx, opts.get_env[0]) # Now mark the build and devel spaces as catkin build's since dry run didn't return. mark_space_as_built_by(ctx.build_space_abs, 'catkin build') mark_space_as_built_by(ctx.devel_space_abs, 'catkin build') # Get the last build context build_metadata = get_metadata(ctx.workspace, ctx.profile, 'build') if build_metadata.get('cmake_args') != ctx.cmake_args or build_metadata.get('cmake_args') != opts.cmake_args: opts.force_cmake = True if build_metadata.get('needs_force', False): opts.force_cmake = True update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': False}) # Always save the last context under the build verb update_metadata(ctx.workspace, ctx.profile, 'build', ctx.get_stored_dict()) # Save the context as the configuration if opts.save_config: Context.save(ctx) # Get parallel toplevel jobs try: parallel_jobs = int(opts.parallel_jobs) except TypeError: parallel_jobs = None # Set VERBOSE environment variable if opts.verbose: os.environ['VERBOSE'] = '1' return build_isolated_workspace( ctx, packages=opts.packages, start_with=opts.start_with, no_deps=opts.no_deps, unbuilt=opts.unbuilt, n_jobs=parallel_jobs, force_cmake=opts.force_cmake, pre_clean=opts.pre_clean, force_color=opts.force_color, quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, limit_status_rate=opts.limit_status_rate, lock_install=not opts.no_install_lock, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize # Can be True, False, or None )
def clean_profile(opts, profile): # Load the context ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False) if not ctx: if not opts.workspace: log(clr("[clean] @!@{rf}Error:@| The current or desired workspace could not be " "determined. Please run `catkin clean` from within a catkin " "workspace or specify the workspace explicitly with the " "`--workspace` option.")) else: log(clr("[clean] @!@{rf}Error:@| Could not clean workspace \"%s\" because it " "either does not exist or it has no catkin_tools metadata." % opts.workspace)) return False profile = ctx.profile # Check if the user wants to do something explicit actions = ['spaces', 'packages', 'clean_this', 'orphans', 'deinit', 'setup_files'] paths = {} # noqa paths_exists = {} # noqa paths['install'] = ( os.path.join(ctx.destdir, ctx.install_space_abs.lstrip(os.sep)) if ctx.destdir else ctx.install_space_abs) paths_exists['install'] = os.path.exists(paths['install']) and os.path.isdir(paths['install']) for space in Context.SPACES.keys(): if space in paths: continue paths[space] = getattr(ctx, '{}_space_abs'.format(space)) paths_exists[space] = getattr(ctx, '{}_space_exists'.format(space))() # Default is to clean all products for this profile no_specific_action = not any([ v for (k, v) in vars(opts).items() if k in actions]) clean_all = opts.deinit or no_specific_action # Initialize action options if clean_all: opts.spaces = [k for k in Context.SPACES.keys() if k != 'source'] # Make sure the user intends to clean everything spaces_to_clean_msgs = [] if opts.spaces and not (opts.yes or opts.dry_run): for space in opts.spaces: if getattr(ctx, '{}_space_exists'.format(space))(): space_name = Context.SPACES[space]['space'] space_abs = getattr(ctx, '{}_space_abs'.format(space)) spaces_to_clean_msgs.append(clr("[clean] {:14} @{yf}{}").format(space_name + ':', space_abs)) if len(spaces_to_clean_msgs) == 0 and not opts.deinit: log("[clean] Nothing to be cleaned for profile: `{}`".format(profile)) return True if len(spaces_to_clean_msgs) > 0: log("") log(clr("[clean] @!@{yf}Warning:@| This will completely remove the " "following directories. (Use `--yes` to skip this check)")) for msg in spaces_to_clean_msgs: log(msg) try: yes = yes_no_loop( "\n[clean] Are you sure you want to completely remove the directories listed above?") if not yes: log(clr("[clean] Not removing any workspace directories for" " this profile.")) return True except KeyboardInterrupt: log("\n[clean] No actions performed.") sys.exit(0) # Initialize flag to be used on the next invocation needs_force = False try: for space in opts.spaces: if space == 'devel': # Remove all develspace files if paths_exists['devel']: log("[clean] Removing {}: {}".format(Context.SPACES['devel']['space'], ctx.devel_space_abs)) if not opts.dry_run: safe_rmtree(ctx.devel_space_abs, ctx.workspace, opts.force) # Clear the cached metadata from the last build run _, build_metadata_file = get_metadata_paths(ctx.workspace, profile, 'build') if os.path.exists(build_metadata_file): os.unlink(build_metadata_file) # Clear the cached packages data, if it exists packages_metadata_path = ctx.package_metadata_path() if os.path.exists(packages_metadata_path): safe_rmtree(packages_metadata_path, ctx.workspace, opts.force) else: if paths_exists[space]: space_name = Context.SPACES[space]['space'] space_path = paths[space] log("[clean] Removing {}: {}".format(space_name, space_path)) if not opts.dry_run: safe_rmtree(space_path, ctx.workspace, opts.force) # Setup file removal if opts.setup_files: if paths_exists['devel']: log("[clean] Removing setup files from {}: {}".format(Context.SPACES['devel']['space'], paths['devel'])) opts.packages.append('catkin') opts.packages.append('catkin_tools_prebuild') else: log("[clean] No {} exists, no setup files to clean.".format(Context.SPACES['devel']['space'])) # Find orphaned packages if ctx.link_devel or ctx.isolate_devel and not ('devel' in opts.spaces or 'build' in opts.spaces): if opts.orphans: if os.path.exists(ctx.build_space_abs): log("[clean] Determining orphaned packages...") # Get all existing packages in source space and the # Suppress warnings since this is looking for packages which no longer exist found_source_packages = [ pkg.name for (path, pkg) in find_packages(ctx.source_space_abs, warnings=[]).items()] built_packages = [ pkg.name for (path, pkg) in find_packages(ctx.package_metadata_path(), warnings=[]).items()] # Look for orphaned products in the build space orphans = [p for p in built_packages if (p not in found_source_packages and p != 'catkin_tools_prebuild')] if len(orphans) > 0: opts.packages.extend(list(orphans)) else: log("[clean] No orphans in the workspace.") else: log("[clean] No {} exists, no potential for orphans.".format(Context.SPACES['build']['space'])) # Remove specific packages if len(opts.packages) > 0 or opts.clean_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) # Suppress warnings since this won't necessarily find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( search_start_path=getcwd(), ws_path=ws_path, warnings=[]) except InvalidPackage as ex: sys.exit(clr("[clean] @!@{rf}Error:@| The file {} is an invalid package.xml file." " See below for details:\n\n{}").format(ex.package_path, ex.msg)) # Handle context-based package cleaning if opts.clean_this: if this_package: opts.packages += [this_package] else: sys.exit( clr("[clean] @!@{rf}Error:@| In order to use --this, the current directory" " must be part of a catkin package.")) try: # Clean the packages needs_force = clean_packages( ctx, opts.packages, opts.dependents, opts.verbose, opts.dry_run) except KeyboardInterrupt: wide_log("[clean] User interrupted!") return False elif opts.orphans or len(opts.packages) > 0 or opts.clean_this: log(clr("[clean] @!@{rf}Error:@| Individual packages cannot be cleaned from " "workspaces with merged develspaces, use a symbolically-linked " "or isolated develspace instead.")) except: # noqa: E722 # Silencing E722 here since we immediately re-raise the exception. log("[clean] Failed to clean profile `{}`".format(profile)) needs_force = True raise finally: if needs_force: log(clr( "[clean] @/@!Note:@| @/Parts of the workspace have been cleaned which will " "necessitate re-configuring CMake on the next build.@|")) update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': True}) return True
def main(opts): # Context-aware args if opts.build_this or opts.start_with_this: # Determine the enclosing package try: ws_path = find_enclosing_workspace(getcwd()) # Suppress warnings since this won't necessaraly find all packages # in the workspace (it stops when it finds one package), and # relying on it for warnings could mislead people. this_package = find_enclosing_package( search_start_path=getcwd(), ws_path=ws_path, warnings=[]) except (InvalidPackage, RuntimeError): this_package = None # Handle context-based package building if opts.build_this: if this_package: opts.packages += [this_package] else: sys.exit("catkin build: --this was specified, but this directory is not in a catkin package.") # If --start--with was used without any packages and --this was specified, start with this package if opts.start_with_this: if this_package: opts.start_with = this_package else: sys.exit("catkin build: --this was specified, but this directory is not in a catkin package.") if opts.no_deps and not opts.packages: sys.exit("With --no-deps, you must specify packages to build.") # Load the context ctx = Context.load(opts.workspace, opts.profile, opts, append=True) # Initialize the build configuration make_args, makeflags, cli_flags, jobserver = configure_make_args(ctx.make_args, ctx.use_internal_make_jobserver) # Set the jobserver memory limit if jobserver and opts.mem_limit: log(clr("@!@{pf}EXPERIMENTAL: limit memory to '%s'@|" % str(opts.mem_limit))) # At this point psuitl will be required, check for it and bail out if not set try: import psutil # noqa except ImportError as exc: log("Could not import psutil, but psutil is required when using --mem-limit.") log("Please either install psutil or avoid using --mem-limit.") sys.exit("Exception: {0}".format(exc)) set_jobserver_max_mem(opts.mem_limit) ctx.make_args = make_args # Load the environment of the workspace to extend if ctx.extend_path is not None: try: load_resultspace_environment(ctx.extend_path) except IOError as exc: log(clr("@!@{rf}Error:@| Unable to extend workspace from \"%s\": %s" % (ctx.extend_path, exc.message))) return 1 # Display list and leave the file system untouched if opts.dry_run: dry_run(ctx, opts.packages, opts.no_deps, opts.start_with) return # Check if the context is valid before writing any metadata if not ctx.source_space_exists(): print("catkin build: error: Unable to find source space `%s`" % ctx.source_space_abs) return 1 # Always save the last context under the build verb update_metadata(ctx.workspace, ctx.profile, 'build', ctx.get_stored_dict()) build_metadata = get_metadata(ctx.workspace, ctx.profile, 'build') if build_metadata.get('needs_force', False): opts.force_cmake = True update_metadata(ctx.workspace, ctx.profile, 'build', {'needs_force': False}) # Save the context as the configuration if opts.save_config: Context.save(ctx) start = time.time() try: return build_isolated_workspace( ctx, packages=opts.packages, start_with=opts.start_with, no_deps=opts.no_deps, jobs=opts.parallel_jobs, force_cmake=opts.force_cmake, force_color=opts.force_color, quiet=not opts.verbose, interleave_output=opts.interleave_output, no_status=opts.no_status, limit_status_rate=opts.limit_status_rate, lock_install=not opts.no_install_lock, no_notify=opts.no_notify, continue_on_failure=opts.continue_on_failure, summarize_build=opts.summarize # Can be True, False, or None ) finally: log("[build] Runtime: {0}".format(format_time_delta(time.time() - start)))
def test_workspace(context, packages=None, tests=None, list_tests=False, start_with=None, n_jobs=None, force_color=False, quiet=False, interleave_output=False, no_status=False, limit_status_rate=10.0, no_notify=False, summarize_build=None): pre_start_time = time.time() # Get our list of packages based on what's in the source space and our # command line switches. packages_to_test = get_packages_to_test(context, packages) if len(packages_to_test) == 0: log(fmt('[test] No tests in the available packages.')) # Get the full list of tests available in those packages, as configured. packages_tests = get_packages_tests(context, packages_to_test) print packages_tests if list_tests: # Don't build or run, just list available targets. log(fmt('[test] Tests available in workspace packages:')) for package, tests in sorted(packages_tests): log(fmt('[test] * %s' % package.name)) for test in sorted(tests): log(fmt('[test] - %s' % test)) return 0 else: jobs = [] # Construct jobs for running tests. for package, package_tests in packages_tests: jobs.append(create_package_job(context, package, package_tests)) package_names = [p[0].name for p in packages_tests] jobs.append(create_results_check_job(context, package_names)) # Queue for communicating status. event_queue = Queue() try: # Spin up status output thread. status_thread = ConsoleStatusController( 'test', ['package', 'packages'], jobs, n_jobs, [pkg.name for _, pkg in context.packages], [p for p in context.whitelist], [p for p in context.blacklist], event_queue, show_notifications=not no_notify, show_active_status=not no_status, show_buffered_stdout=not quiet and not interleave_output, show_buffered_stderr=not interleave_output, show_live_stdout=interleave_output, show_live_stderr=interleave_output, show_stage_events=not quiet, show_full_summary=(summarize_build is True), pre_start_time=pre_start_time, active_status_rate=limit_status_rate) status_thread.start() # Block while running N jobs asynchronously try: all_succeeded = run_until_complete( execute_jobs('test', jobs, None, event_queue, context.log_space_abs, max_toplevel_jobs=n_jobs)) except Exception: status_thread.keep_running = False all_succeeded = False status_thread.join(1.0) wide_log(str(traceback.format_exc())) status_thread.join(1.0) return 0 if all_succeeded else 1 except KeyboardInterrupt: wide_log("[test] Interrupted by user!") event_queue.put(None) return 130 # EOWNERDEAD
def build_isolated_workspace(context, packages=None, start_with=None, no_deps=False, unbuilt=False, n_jobs=None, force_cmake=False, pre_clean=False, force_color=False, quiet=False, interleave_output=False, no_status=False, limit_status_rate=10.0, lock_install=False, no_notify=False, continue_on_failure=False, summarize_build=None, relaxed_constraints=False, influx_url=None, influx_db=None): """Builds a catkin workspace in isolation This function will find all of the packages in the source space, start some executors, feed them packages to build based on dependencies and topological ordering, and then monitor the output of the executors, handling loggings of the builds, starting builds, failing builds, and finishing builds of packages, and handling the shutdown of the executors when appropriate. :param context: context in which to build the catkin workspace :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :param packages: list of packages to build, by default their dependencies will also be built :type packages: list :param start_with: package to start with, skipping all packages which proceed it in the topological order :type start_with: str :param no_deps: If True, the dependencies of packages will not be built first :type no_deps: bool :param n_jobs: number of parallel package build n_jobs :type n_jobs: int :param force_cmake: forces invocation of CMake if True, default is False :type force_cmake: bool :param force_color: forces colored output even if terminal does not support it :type force_color: bool :param quiet: suppresses the output of commands unless there is an error :type quiet: bool :param interleave_output: prints the output of commands as they are received :type interleave_output: bool :param no_status: disables status bar :type no_status: bool :param limit_status_rate: rate to which status updates are limited; the default 0, places no limit. :type limit_status_rate: float :param lock_install: causes executors to synchronize on access of install commands :type lock_install: bool :param no_notify: suppresses system notifications :type no_notify: bool :param continue_on_failure: do not stop building other jobs on error :type continue_on_failure: bool :param summarize_build: if True summarizes the build at the end, if None and continue_on_failure is True and the the build fails, then the build will be summarized, but if False it never will be summarized. :type summarize_build: bool :param relaxed_constraints If true, do not use exec_deps for topological ordering :type relaxed_constraints bool :param influx_url Url to access an InfluxDB instance :type influx_url string Url of the form user:password@host:port :param influx_db Database name in InfluxDB :type influx_db string :raises: SystemExit if buildspace is a file or no packages were found in the source space or if the provided options are invalid """ pre_start_time = time.time() # Assert that the limit_status_rate is valid if limit_status_rate < 0: sys.exit( "[build] @!@{rf}Error:@| The value of --status-rate must be greater than or equal to zero." ) # Declare a buildspace marker describing the build config for error checking buildspace_marker_data = { 'workspace': context.workspace, 'profile': context.profile, 'install': context.install, 'install_space': context.install_space_abs, 'devel_space': context.devel_space_abs, 'source_space': context.source_space_abs } # Check build config if os.path.exists( os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)): with open(os.path.join( context.build_space_abs, BUILDSPACE_MARKER_FILE)) as buildspace_marker_file: existing_buildspace_marker_data = yaml.safe_load( buildspace_marker_file) misconfig_lines = '' for (k, v) in existing_buildspace_marker_data.items(): new_v = buildspace_marker_data.get(k, None) if new_v != v: misconfig_lines += ( '\n - %s: %s (stored) is not %s (commanded)' % (k, v, new_v)) if len(misconfig_lines) > 0: sys.exit( clr("\n@{rf}Error:@| Attempting to build a catkin workspace using build space: " "\"%s\" but that build space's most recent configuration " "differs from the commanded one in ways which will cause " "problems. Fix the following options or use @{yf}`catkin " "clean -b`@| to remove the build space: %s" % (context.build_space_abs, misconfig_lines))) # Summarize the context summary_notes = [] if force_cmake: summary_notes += [ clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.") ] log(context.summary(summary_notes)) # Make sure there is a build folder and it is not a file if os.path.exists(context.build_space_abs): if os.path.isfile(context.build_space_abs): sys.exit( clr("[build] @{rf}Error:@| Build space '{0}' exists but is a file and not a folder." .format(context.build_space_abs))) # If it dosen't exist, create it else: log("[build] Creating build space: '{0}'".format( context.build_space_abs)) os.makedirs(context.build_space_abs) # Write the current build config for config error checking with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE), 'w') as buildspace_marker_file: buildspace_marker_file.write( yaml.dump(buildspace_marker_data, default_flow_style=False)) # Get all the packages in the context source space # Suppress warnings since this is a utility function workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) # Get packages which have not been built yet built_packages, unbuilt_pkgs = get_built_unbuilt_packages( context, workspace_packages) # Handle unbuilt packages if unbuilt: # Check if there are any unbuilt if len(unbuilt_pkgs) > 0: # Add the unbuilt packages packages.extend(list(unbuilt_pkgs)) else: log("[build] No unbuilt packages to be built.") return # If no_deps is given, ensure packages to build are provided if no_deps and packages is None: log( clr("[build] @!@{rf}Error:@| With no_deps, you must specify packages to build." )) return # Find list of packages in the workspace packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built( packages, context, workspace_packages) if not no_deps: # Extend packages to be built to include their deps packages_to_be_built.extend(packages_to_be_built_deps) # Also re-sort try: packages_to_be_built = topological_order_packages( dict(packages_to_be_built)) except AttributeError: log( clr("[build] @!@{rf}Error:@| The workspace packages have a circular " "dependency, and cannot be built. Please run `catkin list " "--deps` to determine the problematic package(s).")) return # Check the number of packages to be built if len(packages_to_be_built) == 0: log(clr('[build] No packages to be built.')) # Assert start_with package is in the workspace verify_start_with_option(start_with, packages, all_packages, packages_to_be_built) # Populate .catkin file if we're not installing # NOTE: This is done to avoid the Catkin CMake code from doing it, # which isn't parallel-safe. Catkin CMake only modifies this file if # it's package source path isn't found. if not context.install: dot_catkin_file_path = os.path.join(context.devel_space_abs, '.catkin') # If the file exists, get the current paths if os.path.exists(dot_catkin_file_path): dot_catkin_paths = open(dot_catkin_file_path, 'r').read().split(';') else: dot_catkin_paths = [] # Update the list with the new packages (in topological order) packages_to_be_built_paths = [ os.path.join(context.source_space_abs, path) for path, pkg in packages_to_be_built ] new_dot_catkin_paths = [ os.path.join(context.source_space_abs, path) for path in [ os.path.join(context.source_space_abs, path) for path, pkg in all_packages ] if path in dot_catkin_paths or path in packages_to_be_built_paths ] # Write the new file if it's different, otherwise, leave it alone if dot_catkin_paths == new_dot_catkin_paths: wide_log("[build] Package table is up to date.") else: wide_log("[build] Updating package table.") open(dot_catkin_file_path, 'w').write(';'.join(new_dot_catkin_paths)) # Remove packages before start_with if start_with is not None: for path, pkg in list(packages_to_be_built): if pkg.name != start_with: wide_log( clr("@!@{pf}Skipping@| @{gf}---@| @{cf}{}@|").format( pkg.name)) packages_to_be_built.pop(0) else: break # Get the names of all packages to be built packages_to_be_built_names = [p.name for _, p in packages_to_be_built] packages_to_be_built_deps_names = [ p.name for _, p in packages_to_be_built_deps ] # Generate prebuild and prebuild clean jobs, if necessary prebuild_jobs = {} setup_util_present = os.path.exists( os.path.join(context.devel_space_abs, '_setup_util.py')) catkin_present = 'catkin' in (packages_to_be_built_names + packages_to_be_built_deps_names) catkin_built = 'catkin' in built_packages prebuild_built = 'catkin_tools_prebuild' in built_packages # Handle the prebuild jobs if the develspace is linked prebuild_pkg_deps = [] if context.link_devel: prebuild_pkg = None # Construct a dictionary to lookup catkin package by name pkg_dict = dict([(pkg.name, (pth, pkg)) for pth, pkg in all_packages]) if setup_util_present: # Setup util is already there, determine if it needs to be # regenerated if catkin_built: if catkin_present: prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin'] elif prebuild_built: if catkin_present: # TODO: Clean prebuild package ct_prebuild_pkg_path = get_prebuild_package( context.build_space_abs, context.devel_space_abs, force_cmake) ct_prebuild_pkg = parse_package(ct_prebuild_pkg_path) prebuild_jobs[ 'caktin_tools_prebuild'] = create_catkin_clean_job( context, ct_prebuild_pkg, ct_prebuild_pkg_path, dependencies=[], dry_run=False, clean_build=True, clean_devel=True, clean_install=True) # TODO: Build catkin package prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin'] prebuild_pkg_deps.append('catkin_tools_prebuild') else: # How did these get here?? log("Warning: devel space setup files have an unknown origin.") else: # Setup util needs to be generated if catkin_built or prebuild_built: log("Warning: generated devel space setup files have been deleted." ) if catkin_present: # Build catkin package prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin'] else: # Generate and buildexplicit prebuild package prebuild_pkg_path = get_prebuild_package( context.build_space_abs, context.devel_space_abs, force_cmake) prebuild_pkg = parse_package(prebuild_pkg_path) if prebuild_pkg is not None: # Create the prebuild job prebuild_job = create_catkin_build_job( context, prebuild_pkg, prebuild_pkg_path, build_dependencies=prebuild_pkg_deps, run_dependencies=[], force_cmake=force_cmake, pre_clean=pre_clean, prebuild=True) # Add the prebuld job prebuild_jobs[prebuild_job.jid] = prebuild_job # Remove prebuild jobs from normal job list for prebuild_jid, prebuild_job in prebuild_jobs.items(): if prebuild_jid in packages_to_be_built_names: packages_to_be_built_names.remove(prebuild_jid) # Initial jobs list is just the prebuild jobs jobs = [] + list(prebuild_jobs.values()) # Get all build type plugins build_job_creators = { ep.name: ep.load()['create_build_job'] for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs') } # It's a problem if there aren't any build types available if len(build_job_creators) == 0: sys.exit( 'Error: No build types available. Please check your catkin_tools installation.' ) # Construct jobs for pkg_path, pkg in all_packages: if pkg.name not in packages_to_be_built_names: continue # Ignore metapackages if 'metapackage' in [e.tagname for e in pkg.exports]: continue # Get actual execution deps build_deps = [ p.name for _, p in get_cached_recursive_build_depends_in_workspace( pkg, packages_to_be_built) if p.name not in prebuild_jobs ] build_for_run_deps = [ p.name for _, p in get_cached_recursive_run_depends_in_workspace( pkg, packages_to_be_built) if p.name not in prebuild_jobs ] # All jobs depend on the prebuild jobs if they're defined if not no_deps: if relaxed_constraints: build_for_run_deps = [ p.name for _, p in get_recursive_build_depends_for_run_depends_in_workspace( [pkg], packages_to_be_built) if p.name not in prebuild_jobs ] else: # revert to interpreting all dependencies as build dependencies build_deps = list(set(build_deps + build_for_run_deps)) build_for_run_deps = [] for j in prebuild_jobs.values(): build_deps.append(j.jid) # Determine the job parameters build_job_kwargs = dict(context=context, package=pkg, package_path=pkg_path, build_dependencies=build_deps, run_dependencies=build_for_run_deps, force_cmake=force_cmake, pre_clean=pre_clean) # Create the job based on the build type build_type = get_build_type(pkg) if build_type in build_job_creators: jobs.append(build_job_creators[build_type](**build_job_kwargs)) else: wide_log( clr("[build] @!@{yf}Warning:@| Skipping package `{}` because it " "has an unsupported package build type: `{}`").format( pkg.name, build_type)) wide_log(clr("[build] Note: Available build types:")) for bt_name in build_job_creators.keys(): wide_log(clr("[build] - `{}`".format(bt_name))) # Queue for communicating status event_queue = Queue() status_queue = Queue() monitoring_queue = Queue() class ForwardingQueue(threading.Thread): def __init__(self, queues): super(ForwardingQueue, self).__init__() self.keep_running = True self.queues = queues def run(self): while self.keep_running: event = event_queue.get(True) for queue in self.queues: queue.put(event) if event is None: break queue_thread = ForwardingQueue([status_queue, monitoring_queue]) threads = [queue_thread] try: # Spin up status output thread status_thread = ConsoleStatusController( 'build', ['package', 'packages'], jobs, n_jobs, [pkg.name for _, pkg in context.packages], [p for p in context.whitelist], [p for p in context.blacklist], status_queue, show_notifications=not no_notify, show_active_status=not no_status, show_buffered_stdout=not quiet and not interleave_output, show_buffered_stderr=not interleave_output, show_live_stdout=interleave_output, show_live_stderr=interleave_output, show_stage_events=not quiet, show_full_summary=(summarize_build is True), pre_start_time=pre_start_time, active_status_rate=limit_status_rate) threads.append(status_thread) if influx_db is not None: if not have_influx_db: sys.exit( "[build] @!@{rf}Error:@| InfluxDB monitoring is not possible, cannot import influxdb" ) match = re.match('^(.+):(.+)@(.+):(.+)$', influx_url) if not match: sys.exit( "[build] @!@{rf}Error:@| The value of --influx has to be of the form username:password@host:port" ) username, password, host, port = match.groups() influxdb_thread = InfluxDBStatusController(monitoring_queue, influx_db, host, port, username, password) threads.append(influxdb_thread) for thread in threads: thread.start() # Initialize locks locks = { 'installspace': asyncio.Lock() if lock_install else FakeLock() } # Block while running N jobs asynchronously try: all_succeeded = run_until_complete( execute_jobs('build', jobs, locks, event_queue, context.log_space_abs, max_toplevel_jobs=n_jobs, continue_on_failure=continue_on_failure, continue_without_deps=False, relaxed_constraints=relaxed_constraints)) except Exception: all_succeeded = False for thread in threads: thread.keep_running = False for thread in threads: thread.join(1.0) wide_log(str(traceback.format_exc())) event_queue.put(None) for thread in threads: thread.join(1.0) # Warn user about new packages now_built_packages, now_unbuilt_pkgs = get_built_unbuilt_packages( context, workspace_packages) new_pkgs = [p for p in unbuilt_pkgs if p not in now_unbuilt_pkgs] if len(new_pkgs) > 0: log( clr("[build] @/@!Note:@| @/Workspace packages have changed, " "please re-source setup files to use them.@|")) if all_succeeded: # Create isolated devel setup if necessary if context.isolate_devel: if not context.install: _create_unmerged_devel_setup(context, now_unbuilt_pkgs) else: _create_unmerged_devel_setup_for_install(context) return 0 else: return 1 except KeyboardInterrupt: wide_log("[build] Interrupted by user!") event_queue.put(None) return 130 # EOWNERDEAD return code is not part of the errno module.
def determine_packages_to_be_built(packages, context, workspace_packages): """Returns list of packages which should be built, and those package's deps. :param packages: list of packages to be built, if None all packages are built :type packages: list :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :returns: tuple of packages to be built and those package's deps :rtype: tuple """ start = time.time() # If there are no packages raise if not workspace_packages: log("[build] No packages were found in the source space '{0}'".format( context.source_space_abs)) else: wide_log("[build] Found '{0}' packages in {1}.".format( len(workspace_packages), format_time_delta(time.time() - start))) # Order the packages by topology ordered_packages = topological_order_packages(workspace_packages) # Set the packages in the workspace for the context context.packages = ordered_packages # Determine the packages which should be built packages_to_be_built = [] packages_to_be_built_deps = [] # Check if topological_order_packages determined any circular dependencies, if so print an error and fail. # If this is the case, the last entry of ordered packages is a tuple that starts with nil. if ordered_packages and ordered_packages[-1][0] is None: guilty_packages = ", ".join(ordered_packages[-1][1:]) sys.exit( "[build] Circular dependency detected in the following packages: {}" .format(guilty_packages)) workspace_package_names = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) # Determine the packages to be built if packages: # First assert all of the packages given are in the workspace for package in packages: if package not in workspace_package_names: # Try whether package is a pattern and matches glob_packages = expand_glob_package(package, workspace_package_names) if len(glob_packages) > 0: packages.extend(glob_packages) continue else: sys.exit( "[build] Given package '{0}' is not in the workspace " "and pattern does not match any package".format( package)) # If metapackage, include run depends which are in the workspace package_obj = workspace_package_names[package][1] if 'metapackage' in [e.tagname for e in package_obj.exports]: for rdep in package_obj.run_depends: if rdep.name in workspace_package_names: packages.append(rdep.name) # Limit the packages to be built to just the provided packages for pkg_path, package in ordered_packages: if package.name in packages: packages_to_be_built.append((pkg_path, package)) # Get the recursive dependencies for each of these packages pkg_deps = get_cached_recursive_build_depends_in_workspace( package, ordered_packages) packages_to_be_built_deps.extend(pkg_deps) else: # Only use whitelist when no other packages are specified if len(context.whitelist) > 0: # Expand glob patterns in whitelist whitelist = [] for whitelisted_package in context.whitelist: whitelist.extend( expand_glob_package(whitelisted_package, workspace_package_names)) packages_to_be_built = [ p for p in ordered_packages if (p[1].name in whitelist) ] else: packages_to_be_built = ordered_packages # Filter packages with blacklist if len(context.blacklist) > 0: # Expand glob patterns in blacklist blacklist = [] for blacklisted_package in context.blacklist: blacklist.extend( expand_glob_package(blacklisted_package, workspace_package_names)) # Apply blacklist to packages and dependencies packages_to_be_built = [ (path, pkg) for path, pkg in packages_to_be_built if (pkg.name not in blacklist or pkg.name in packages) ] packages_to_be_built_deps = [ (path, pkg) for path, pkg in packages_to_be_built_deps if (pkg.name not in blacklist or pkg.name in packages) ] return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def determine_packages_to_be_built(packages, context, workspace_packages): """Returns list of packages which should be built, and those package's deps. :param packages: list of packages to be built, if None all packages are built :type packages: list :param context: Workspace context :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :returns: tuple of packages to be built and those package's deps :rtype: tuple """ start = time.time() # If there are no packages raise if not workspace_packages: log("[build] No packages were found in the source space '{0}'".format( context.source_space_abs)) else: wide_log("[build] Found '{0}' packages in {1}.".format( len(workspace_packages), format_time_delta(time.time() - start))) # Order the packages by topology ordered_packages = topological_order_packages(workspace_packages) # Set the packages in the workspace for the context context.packages = ordered_packages # Determin the packages which should be built packages_to_be_built = [] packages_to_be_built_deps = [] # Determine the packages to be built if packages: # First assert all of the packages given are in the workspace workspace_package_names = dict([(pkg.name, (path, pkg)) for path, pkg in ordered_packages]) for package in packages: if package not in workspace_package_names: sys.exit("[build] Given package '{0}' is not in the workspace". format(package)) # If metapackage, include run depends which are in the workspace package_obj = workspace_package_names[package][1] if 'metapackage' in [e.tagname for e in package_obj.exports]: for rdep in package_obj.run_depends: if rdep.name in workspace_package_names: packages.append(rdep.name) # Limit the packages to be built to just the provided packages for pkg_path, package in ordered_packages: if package.name in packages: packages_to_be_built.append((pkg_path, package)) # Get the recursive dependencies for each of these packages pkg_build_deps = get_cached_recursive_build_depends_in_workspace( package, ordered_packages) packages_to_be_built_deps.extend(pkg_build_deps) pkg_run_deps = get_cached_recursive_run_depends_in_workspace( package, ordered_packages) packages_to_be_built_deps.extend(pkg_run_deps) else: # Only use whitelist when no other packages are specified if len(context.whitelist) > 0: packages_to_be_built = [ p for p in ordered_packages if (p[1].name in context.whitelist) ] else: packages_to_be_built = ordered_packages # Filter packages with blacklist if len(context.blacklist) > 0: packages_to_be_built = [ (path, pkg) for path, pkg in packages_to_be_built if (pkg.name not in context.blacklist or pkg.name in packages) ] packages_to_be_built_deps = [ (path, pkg) for path, pkg in packages_to_be_built_deps if (pkg.name not in context.blacklist or pkg.name in packages) ] ordered_packages = ordered_packages return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def create_package_job(context, package, package_path, deps, doc_deps): docs_space = os.path.join(context.docs_space_abs, package.name) docs_build_space = os.path.join(context.build_space_abs, 'docs', package.name) package_path_abs = os.path.join(context.source_space_abs, package_path) package_meta_path = context.package_metadata_path(package) # Load rosdoc config, if it exists. rosdoc_yaml_path = os.path.join(package_path_abs, 'rosdoc.yaml') for export in package.exports: if export.tagname == "rosdoc": config = export.attributes.get('config', '') if config: rosdoc_yaml_path_temp = os.path.join(package_path_abs, config) if os.path.isfile(rosdoc_yaml_path_temp): # Stop if configuration is found which exists rosdoc_yaml_path = rosdoc_yaml_path_temp break if os.path.isfile(rosdoc_yaml_path): with open(rosdoc_yaml_path) as f: rosdoc_conf = yaml.full_load(f) else: if os.path.isdir(os.path.join(package_path_abs, 'src')) or \ os.path.isdir(os.path.join(package_path_abs, 'include')): rosdoc_conf = [{'builder': 'doxygen'}] else: rosdoc_conf = [] stages = [] # Create package docs spaces. stages.append( FunctionStage('mkdir_docs_build_space', makedirs, path=docs_build_space)) # Generate msg/srv/action docs with package summary page. stages.append( FunctionStage('generate_messages', generate_messages, package=package, package_path=package_path, output_path=docs_build_space)) stages.append( FunctionStage('generate_services', generate_services, package=package, package_path=package_path, output_path=docs_build_space)) stages.append( FunctionStage('generate_package_summary', generate_package_summary, package=package, package_path=package_path_abs, rosdoc_conf=rosdoc_conf, output_path=docs_build_space)) # Cache document config stages.append( FunctionStage('cache_rosdoc_config', yaml_dump_file, contents=rosdoc_conf, dest_path=os.path.join(package_meta_path, 'rosdoc.yaml'))) job_env = {} # Add steps to run native doc generators, as appropriate. This has to happen after # the package summary generates, as we're going to override the subdirectory index.html # files generated by that sphinx run. for conf in rosdoc_conf: try: builder = conf['builder'] if builder == 'doxygen': docs_space = os.path.realpath(docs_space) docs_build_space = os.path.realpath(docs_build_space) package_path_abs = os.path.realpath(package_path_abs) stages.extend( getattr(builders, builder)(conf, package, deps, doc_deps, docs_space, package_path_abs, docs_build_space, job_env)) except AttributeError: log( fmt("[document] @!@{yf}Warning:@| Skipping unrecognized rosdoc builder [%s] for package [%s]" % (conf['builder'], package.name))) return Job(jid=package.name, deps=deps, env=job_env, stages=stages)
def build_isolated_workspace( context, packages=None, start_with=None, no_deps=False, unbuilt=False, n_jobs=None, force_cmake=False, pre_clean=False, force_color=False, quiet=False, interleave_output=False, no_status=False, limit_status_rate=10.0, lock_install=False, no_notify=False, continue_on_failure=False, summarize_build=None, ): """Builds a catkin workspace in isolation This function will find all of the packages in the source space, start some executors, feed them packages to build based on dependencies and topological ordering, and then monitor the output of the executors, handling loggings of the builds, starting builds, failing builds, and finishing builds of packages, and handling the shutdown of the executors when appropriate. :param context: context in which to build the catkin workspace :type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context` :param packages: list of packages to build, by default their dependencies will also be built :type packages: list :param start_with: package to start with, skipping all packages which proceed it in the topological order :type start_with: str :param no_deps: If True, the dependencies of packages will not be built first :type no_deps: bool :param n_jobs: number of parallel package build n_jobs :type n_jobs: int :param force_cmake: forces invocation of CMake if True, default is False :type force_cmake: bool :param force_color: forces colored output even if terminal does not support it :type force_color: bool :param quiet: suppresses the output of commands unless there is an error :type quiet: bool :param interleave_output: prints the output of commands as they are received :type interleave_output: bool :param no_status: disables status bar :type no_status: bool :param limit_status_rate: rate to which status updates are limited; the default 0, places no limit. :type limit_status_rate: float :param lock_install: causes executors to synchronize on access of install commands :type lock_install: bool :param no_notify: suppresses system notifications :type no_notify: bool :param continue_on_failure: do not stop building other jobs on error :type continue_on_failure: bool :param summarize_build: if True summarizes the build at the end, if None and continue_on_failure is True and the the build fails, then the build will be summarized, but if False it never will be summarized. :type summarize_build: bool :raises: SystemExit if buildspace is a file or no packages were found in the source space or if the provided options are invalid """ pre_start_time = time.time() # Assert that the limit_status_rate is valid if limit_status_rate < 0: sys.exit("[build] @!@{rf}Error:@| The value of --status-rate must be greater than or equal to zero.") # Declare a buildspace marker describing the build config for error checking buildspace_marker_data = { 'workspace': context.workspace, 'profile': context.profile, 'install': context.install, 'install_space': context.install_space_abs, 'devel_space': context.devel_space_abs, 'source_space': context.source_space_abs} # Check build config if os.path.exists(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)): with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE)) as buildspace_marker_file: existing_buildspace_marker_data = yaml.load(buildspace_marker_file) misconfig_lines = '' for (k, v) in existing_buildspace_marker_data.items(): new_v = buildspace_marker_data.get(k, None) if new_v != v: misconfig_lines += ( '\n - %s: %s (stored) is not %s (commanded)' % (k, v, new_v)) if len(misconfig_lines) > 0: sys.exit(clr( "\n@{rf}Error:@| Attempting to build a catkin workspace using build space: " "\"%s\" but that build space's most recent configuration " "differs from the commanded one in ways which will cause " "problems. Fix the following options or use @{yf}`catkin " "clean -b`@| to remove the build space: %s" % (context.build_space_abs, misconfig_lines))) # Summarize the context summary_notes = [] if force_cmake: summary_notes += [clr("@!@{cf}NOTE:@| Forcing CMake to run for each package.")] log(context.summary(summary_notes)) # Make sure there is a build folder and it is not a file if os.path.exists(context.build_space_abs): if os.path.isfile(context.build_space_abs): sys.exit(clr( "[build] @{rf}Error:@| Build space '{0}' exists but is a file and not a folder." .format(context.build_space_abs))) # If it dosen't exist, create it else: log("[build] Creating build space: '{0}'".format(context.build_space_abs)) os.makedirs(context.build_space_abs) # Write the current build config for config error checking with open(os.path.join(context.build_space_abs, BUILDSPACE_MARKER_FILE), 'w') as buildspace_marker_file: buildspace_marker_file.write(yaml.dump(buildspace_marker_data, default_flow_style=False)) # Get all the packages in the context source space # Suppress warnings since this is a utility function workspace_packages = find_packages(context.source_space_abs, exclude_subspaces=True, warnings=[]) # Get packages which have not been built yet unbuilt_pkgs = get_unbuilt_packages(context, workspace_packages) # Handle unbuilt packages if unbuilt: # Check if there are any unbuilt if len(unbuilt_pkgs) > 0: # Add the unbuilt packages packages.extend(list(unbuilt_pkgs)) else: log("[build] No unbuilt packages to be built.") return # If no_deps is given, ensure packages to build are provided if no_deps and packages is None: log(clr("[build] @!@{rf}Error:@| With no_deps, you must specify packages to build.")) return # Find list of packages in the workspace packages_to_be_built, packages_to_be_built_deps, all_packages = determine_packages_to_be_built( packages, context, workspace_packages) if not no_deps: # Extend packages to be built to include their deps packages_to_be_built.extend(packages_to_be_built_deps) # Also re-sort try: packages_to_be_built = topological_order_packages(dict(packages_to_be_built)) except AttributeError: log(clr("[build] @!@{rf}Error:@| The workspace packages have a circular " "dependency, and cannot be built. Please run `catkin list " "--deps` to determine the problematic package(s).")) return # Check the number of packages to be built if len(packages_to_be_built) == 0: log(clr('[build] No packages to be built.')) return # Assert start_with package is in the workspace verify_start_with_option( start_with, packages, all_packages, packages_to_be_built + packages_to_be_built_deps) # Populate .catkin file if we're not installing # NOTE: This is done to avoid the Catkin CMake code from doing it, # which isn't parallel-safe. Catkin CMake only modifies this file if # it's package source path isn't found. if not context.install: dot_catkin_file_path = os.path.join(context.devel_space_abs, '.catkin') # If the file exists, get the current paths if os.path.exists(dot_catkin_file_path): dot_catkin_paths = open(dot_catkin_file_path, 'r').read().split(';') else: dot_catkin_paths = [] # Update the list with the new packages (in topological order) packages_to_be_built_paths = [ os.path.join(context.source_space_abs, path) for path, pkg in packages_to_be_built ] new_dot_catkin_paths = [ os.path.join(context.source_space_abs, path) for path in [os.path.join(context.source_space_abs, path) for path, pkg in all_packages] if path in dot_catkin_paths or path in packages_to_be_built_paths ] # Write the new file if it's different, otherwise, leave it alone if dot_catkin_paths == new_dot_catkin_paths: wide_log("[build] Package table is up to date.") else: wide_log("[build] Updating package table.") open(dot_catkin_file_path, 'w').write(';'.join(new_dot_catkin_paths)) # Remove packages before start_with if start_with is not None: for path, pkg in list(packages_to_be_built): if pkg.name != start_with: wide_log(clr("@!@{pf}Skipping@| @{gf}---@| @{cf}{}@|").format(pkg.name)) packages_to_be_built.pop(0) else: break # Get the names of all packages to be built packages_to_be_built_names = [p.name for _, p in packages_to_be_built] packages_to_be_built_deps_names = [p.name for _, p in packages_to_be_built_deps] # Generate prebuild jobs, if necessary prebuild_jobs = {} setup_util_exists = os.path.exists(os.path.join(context.devel_space_abs, '_setup_util.py')) if context.link_devel and (not setup_util_exists or (force_cmake and len(packages) == 0)): wide_log('[build] Preparing linked develspace...') pkg_dict = dict([(pkg.name, (pth, pkg)) for pth, pkg in all_packages]) if 'catkin' in packages_to_be_built_names + packages_to_be_built_deps_names: # Use catkin as the prebuild package prebuild_pkg_path, prebuild_pkg = pkg_dict['catkin'] else: # Generate explicit prebuild package prebuild_pkg_path = generate_prebuild_package(context.build_space_abs, context.devel_space_abs, force_cmake) prebuild_pkg = parse_package(prebuild_pkg_path) # Create the prebuild job prebuild_job = create_catkin_build_job( context, prebuild_pkg, prebuild_pkg_path, dependencies=[], force_cmake=force_cmake, pre_clean=pre_clean, prebuild=True) # Add the prebuld job prebuild_jobs[prebuild_job.jid] = prebuild_job # Remove prebuild jobs from normal job list for prebuild_jid, prebuild_job in prebuild_jobs.items(): if prebuild_jid in packages_to_be_built_names: packages_to_be_built_names.remove(prebuild_jid) # Initial jobs list is just the prebuild jobs jobs = [] + list(prebuild_jobs.values()) # Get all build type plugins build_job_creators = { ep.name: ep.load()['create_build_job'] for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs') } # It's a problem if there aren't any build types available if len(build_job_creators) == 0: sys.exit('Error: No build types availalbe. Please check your catkin_tools installation.') # Construct jobs for pkg_path, pkg in all_packages: if pkg.name not in packages_to_be_built_names: continue # Ignore metapackages if 'metapackage' in [e.tagname for e in pkg.exports]: continue # Get actual execution deps deps = [ p.name for _, p in get_cached_recursive_build_depends_in_workspace(pkg, packages_to_be_built) if p.name not in prebuild_jobs ] # All jobs depend on the prebuild job if it's defined for j in prebuild_jobs.values(): deps.append(j.jid) # Determine the job parameters build_job_kwargs = dict( context=context, package=pkg, package_path=pkg_path, dependencies=deps, force_cmake=force_cmake, pre_clean=pre_clean) # Create the job based on the build type build_type = get_build_type(pkg) if build_type in build_job_creators: jobs.append(build_job_creators[build_type](**build_job_kwargs)) else: wide_log(clr( "[build] @!@{yf}Warning:@| Skipping package `{}` because it " "has an unsupported package build type: `{}`" ).format(pkg.name, build_type)) wide_log(clr("[build] Note: Available build types:")) for bt_name in build_job_creators.keys(): wide_log(clr("[build] - `{}`".format(bt_name))) # Queue for communicating status event_queue = Queue() try: # Spin up status output thread status_thread = ConsoleStatusController( 'build', ['package', 'packages'], jobs, n_jobs, [pkg.name for _, pkg in context.packages], [p for p in context.whitelist], [p for p in context.blacklist], event_queue, show_notifications=not no_notify, show_active_status=not no_status, show_buffered_stdout=not quiet and not interleave_output, show_buffered_stderr=not interleave_output, show_live_stdout=interleave_output, show_live_stderr=interleave_output, show_stage_events=not quiet, show_full_summary=(summarize_build is True), pre_start_time=pre_start_time, active_status_rate=limit_status_rate) status_thread.start() # Initialize locks locks = { 'installspace': asyncio.Lock() if lock_install else FakeLock() } # Block while running N jobs asynchronously try: all_succeeded = run_until_complete(execute_jobs( 'build', jobs, locks, event_queue, os.path.join(context.build_space_abs, '_logs'), max_toplevel_jobs=n_jobs, continue_on_failure=continue_on_failure, continue_without_deps=False)) except Exception: status_thread.keep_running = False all_succeeded = False status_thread.join(1.0) wide_log(str(traceback.format_exc())) status_thread.join(1.0) # Warn user about new packages now_unbuilt_pkgs = get_unbuilt_packages(context, workspace_packages) new_pkgs = [p for p in unbuilt_pkgs if p not in now_unbuilt_pkgs] if len(new_pkgs) > 0: log(clr("[build] @/@!Note:@| @/Workspace packages have changed, " "please re-source setup files to use them.@|")) if all_succeeded: # Create isolated devel setup if necessary if context.isolate_devel: if not context.install: _create_unmerged_devel_setup(context, now_unbuilt_pkgs) else: _create_unmerged_devel_setup_for_install(context) return 0 else: return 1 except KeyboardInterrupt: wide_log("[build] Interrupted by user!") event_queue.put(None)
def main(opts): # Check for exclusivity full_options = opts.deinit space_options = opts.logs or opts.build or opts.devel or opts.install package_options = len(opts.packages) > 0 or opts.orphans advanced_options = opts.setup_files if full_options: if space_options or package_options or advanced_options: log("[clean] Error: Using `--deinit` will remove all spaces, so" " additional partial cleaning options will be ignored.") elif space_options: if package_options: log("[clean] Error: Package arguments are not allowed with space" " arguments (--build, --devel, --install, --logs). See usage.") elif advanced_options: log("[clean] Error: Advanced arguments are not allowed with space" " arguments (--build, --devel, --install, --logs). See usage.") # Check for all profiles option if opts.all_profiles: profiles = get_profile_names(opts.workspace or os.getcwd()) else: profiles = [opts.profile] # Initialize job server job_server.initialize( max_jobs=1, max_load=None, gnu_make_enabled=False) # Clean the requested profiles retcode = 0 for profile in profiles: if not clean_profile(opts, profile): retcode = 1 # Warn before nuking .catkin_tools if retcode == 0: if opts.deinit and not opts.yes: log("") log(clr("[clean] @!@{yf}Warning:@| If you deinitialize this workspace" " you will lose all profiles and all saved build" " configuration. (Use `--yes` to skip this check)")) try: opts.deinit = yes_no_loop("\n[clean] Are you sure you want to deinitialize this workspace?") if not opts.deinit: log(clr("[clean] Not deinitializing workspace.")) except KeyboardInterrupt: log("\n[clean] No actions performed.") sys.exit(0) # Nuke .catkin_tools if opts.deinit: ctx = Context.load(opts.workspace, profile, opts, strict=True, load_env=False) metadata_dir = os.path.join(ctx.workspace, METADATA_DIR_NAME) log("[clean] Deinitializing workspace by removing catkin_tools config: %s" % metadata_dir) if not opts.dry_run: safe_rmtree(metadata_dir, ctx.workspace, opts.force) return retcode