def test_terminal_colors(self): test = '@_This is underlined@|' rslt = '\033[4mThis is underlined\033[0m\033[0m' assert fmt(test) == rslt test = 'This has bad stuff @! @/ @_ @| OK!' test = sanitize(test) rslt = 'This has bad stuff @! @/ @_ @| OK!\033[0m' assert fmt(test) == rslt test = char(2018) test = sanitize(test) rslt = char(2018) assert test == rslt
def test_terminal_colors(self): test = '@_This is underlined@|' rslt = '\033[4mThis is underlined\033[0m\033[0m' assert fmt(test) == rslt test = 'This has bad stuff @! @/ @_ @| OK!' test = sanitize(test) rslt = 'This has bad stuff @! @/ @_ @| OK!\033[0m' assert fmt(test) == rslt test = u'\u2018@' test = sanitize(test) rslt = u'\u2018@@' assert test == rslt
def test_terminal_colors(self): # since other test might disable ansi colors # we need to ensure it is enabled enable_ANSI_colors() assert ansi('reset') != '' test = '@_This is underlined@|' rslt = '\033[4mThis is underlined\033[0m\033[0m' assert fmt(test) == rslt test = 'This has bad stuff @! @/ @_ @| OK!' test = sanitize(test) rslt = 'This has bad stuff @! @/ @_ @| OK!\033[0m' assert fmt(test) == rslt test = char(2018) test = sanitize(test) rslt = char(2018) assert test == rslt
def get_train_imgs( obj_name, data_dir=None, only_appropriate=True, with_mask=True, ): """Find train image paths from data/obj_name""" if data_dir is None: data_dir = get_data_dir() obj_dir = os.path.join(data_dir, obj_name) if not os.path.exists(obj_dir): print( terminal_color.fmt('@{yellow}[WARNING] not found object data: {0}' ).format(obj_name)) else: os.chdir(obj_dir) for imgfile in os.listdir('.'): if not imgfile.endswith('.jpg'): continue if only_appropriate: # N1_30.jpg -> N1_30 basename, _ = os.path.splitext(imgfile) # N1_30 -> N1, 30 camera_pos, rotation_deg = basename.split('_') rotation_deg = int(rotation_deg) with open(os.path.join(data_dir, 'appropriate_images.yml')) as f: # {'N1': ['0-30']} appropriate_data = yaml.load(f)[obj_name] if (not appropriate_data) or (camera_pos not in appropriate_data): continue skip = True for min_max in appropriate_data[camera_pos]: _min, _max = map(int, min_max.split('-')) if _min <= rotation_deg <= _max: skip = False break if skip: continue train_path = os.path.join(obj_dir, imgfile) train_img = cv2.imread(train_path) if with_mask: maskfile = os.path.splitext(imgfile)[0] + '_mask.pbm' mask_path = os.path.join(obj_dir, 'masks', maskfile) mask = cv2.imread(mask_path) train_img = cv2.add(mask, train_img) yield train_img os.chdir(data_dir)
def package_depends_impl(pkg, depends=None): # takes and returns Package object if depends is None: depends = [] global pkg_map if pkg_map is None: pkg_map = get_pkg_map() if not pkg in pkg_map: print(terminal_color.fmt( '@{yellow}[WARNING] %s is not found in workspace') % (pkg)) return depends ros_depends = filter(lambda x: x in pkg_map, get_depends(pkg)) tmp_depends = filter(lambda x: x not in depends, ros_depends) depends.extend(tmp_depends) for p in tmp_depends: depends = package_depends_impl(p, depends) return depends
def get_train_imgs( obj_name, data_dir=None, only_appropriate=True, with_mask=True, ): """Find train image paths from data/obj_name""" if data_dir is None: data_dir = get_data_dir() obj_dir = os.path.join(data_dir, obj_name) if not os.path.exists(obj_dir): print(terminal_color.fmt( '@{yellow}[WARNING] not found object data: {0}' ).format(obj_name)) else: os.chdir(obj_dir) for imgfile in os.listdir('.'): if not imgfile.endswith('.jpg'): continue if only_appropriate: # N1_30.jpg -> N1_30 basename, _ = os.path.splitext(imgfile) # N1_30 -> N1, 30 camera_pos, rotation_deg = basename.split('_') rotation_deg = int(rotation_deg) with open(os.path.join(data_dir, 'appropriate_images.yml')) as f: # {'N1': ['0-30']} appropriate_data = yaml.load(f)[obj_name] if (not appropriate_data) or (camera_pos not in appropriate_data): continue skip = True for min_max in appropriate_data[camera_pos]: _min, _max = map(int, min_max.split('-')) if _min <= rotation_deg <= _max: skip = False break if skip: continue train_path = os.path.join(obj_dir, imgfile) train_img = cv2.imread(train_path) if with_mask: maskfile = os.path.splitext(imgfile)[0] + '_mask.pbm' mask_path = os.path.join(obj_dir, 'masks', maskfile) mask = cv2.imread(mask_path) train_img = cv2.add(mask, train_img) yield train_img os.chdir(data_dir)
def package_depends(pkg): # pkg is string global pkg_map if pkg_map is None: pkg_map = get_pkg_map() depends = {} depends_impl = package_depends_impl(pkg) for d in depends_impl: try: pkg_obj = pkg_map[d] p_path = os.path.dirname(pkg_obj.filename) if (os.path.exists(os.path.join(p_path, "msg")) or os.path.exists(os.path.join(p_path, "srv"))): depends[d] = pkg_obj except Exception as e: print(terminal_color.fmt( '@{yellow}[WARNING] path to %s is not found') % (pkg)) print(e) return [p.name for n,p in topological_order.topological_order_packages(depends)]
def colorize_line(line): cline = sanitize(line) cline = cline.replace( '-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '-- @{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@|' ) if line.startswith('-- ~~'): # -- ~~ - cline = cline.replace('~~ ', '@{pf}~~ @|') cline = cline.replace(' - ', ' - @!@{bf}') cline = cline.replace('(', '@|(') cline = cline.replace('(metapackage)', '@|(@{cf}metapackage@|)') cline = cline.replace('(plain cmake)', '@|(@{rf}plain cmake@|)') cline = cline.replace('(unknown)', '@|(@{yf}unknown@|)') if line.startswith('-- +++'): # -- +++ add_subdirectory(package) cline = cline.replace('+++', '@!@{gf}+++@|') cline = cline.replace('kin package: \'', 'kin package: \'@!@{bf}') cline = cline.replace(')', '@|)') cline = cline.replace('metapackage: \'', 'metapackage: \'@!@{bf}') cline = cline.replace('\'\n', '@|\'\n') cline = cline.replace('cmake package: \'', 'cmake package: \'@!@{bf}') cline = cline.replace('\'\n', '@|\'\n') if line.startswith('-- ==>'): cline = cline.replace('-- ==>', '-- @!@{bf}==>@|') if line.lower().startswith('warning'): # WARNING cline = ansi('yf') + cline if line.startswith('CMake Warning'): # CMake Warning... cline = cline.replace('CMake Warning', '@{yf}@!CMake Warning@|') if line.startswith('ERROR:'): # ERROR: cline = cline.replace('ERROR:', '@!@{rf}ERROR:@|') if line.startswith('CMake Error'): # CMake Error... cline = cline.replace('CMake Error', '@{rf}@!CMake Error@|') if line.startswith('Call Stack (most recent call first):'): # CMake Call Stack cline = cline.replace('Call Stack (most recent call first):', '@{cf}@_Call Stack (most recent call first):@|') return fmt(cline)
def colorize_line(line): cline = sanitize(line) cline = cline.replace( '-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~', '-- @{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@|') if line.startswith('-- ~~'): # -- ~~ - cline = cline.replace('~~ ', '@{pf}~~ @|') cline = cline.replace(' - ', ' - @!@{bf}') cline = cline.replace('(', '@|(') cline = cline.replace('(plain cmake)', '@|(@{rf}plain cmake@|)') cline = cline.replace('(unknown)', '@|(@{yf}unknown@|)') if line.startswith('-- +++'): # -- +++ add_subdirectory(package) cline = cline.replace('+++', '@!@{gf}+++@|') cline = cline.replace('kin package: \'', 'kin package: \'@!@{bf}') cline = cline.replace(')', '@|)') cline = cline.replace('\'\n', '@|\'\n') cline = cline.replace('cmake package: \'', 'cmake package: \'@!@{bf}') cline = cline.replace('\'\n', '@|\'\n') if line.startswith('-- ==>'): cline = cline.replace('-- ==>', '-- @!@{bf}==>@|') if line.lower().startswith('warning'): # WARNING cline = ansi('yf') + cline if line.startswith('CMake Warning'): # CMake Warning... cline = cline.replace('CMake Warning', '@{yf}@!CMake Warning@|') if line.startswith('ERROR:'): # ERROR: cline = cline.replace('ERROR:', '@!@{rf}ERROR:@|') if line.startswith('[guoguo]'): # guoguo: cline = cline.replace('[guoguo]:', '@!@{rf}[guoguo]@|') if line.startswith('CMake Error'): # CMake Error... cline = cline.replace('CMake Error', '@{rf}@!CMake Error@|') if line.startswith('Call Stack (most recent call first):'): # CMake Call Stack cline = cline.replace('Call Stack (most recent call first):', '@{cf}@_Call Stack (most recent call first):@|') return fmt(cline)
def colorize_line(line): cline = sanitize(line) cline = cline.replace( "-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~", "-- @{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~@|", ) if line.startswith("-- ~~"): # -- ~~ - cline = cline.replace("~~ ", "@{pf}~~ @|") cline = cline.replace(" - ", " - @!@{bf}") cline = cline.replace("(", "@|(") cline = cline.replace("(plain cmake)", "@|(@{rf}plain cmake@|)") cline = cline.replace("(unknown)", "@|(@{yf}unknown@|)") if line.startswith("-- +++"): # -- +++ add_subdirectory(package) cline = cline.replace("+++", "@!@{gf}+++@|") cline = cline.replace("kin package: '", "kin package: '@!@{bf}") cline = cline.replace(")", "@|)") cline = cline.replace("'\n", "@|'\n") cline = cline.replace("cmake package: '", "cmake package: '@!@{bf}") cline = cline.replace("'\n", "@|'\n") if line.startswith("-- ==>"): cline = cline.replace("-- ==>", "-- @!@{bf}==>@|") if line.lower().startswith("warning"): # WARNING cline = ansi("yf") + cline if line.startswith("CMake Warning"): # CMake Warning... cline = cline.replace("CMake Warning", "@{yf}@!CMake Warning@|") if line.startswith("ERROR:"): # ERROR: cline = cline.replace("ERROR:", "@!@{rf}ERROR:@|") if line.startswith("CMake Error"): # CMake Error... cline = cline.replace("CMake Error", "@{rf}@!CMake Error@|") if line.startswith("Call Stack (most recent call first):"): # CMake Call Stack cline = cline.replace("Call Stack (most recent call first):", "@{cf}@_Call Stack (most recent call first):@|") return fmt(cline)
def cprint(msg, end=None): print(fmt(msg), end=end)
def build_workspace_isolated( workspace='.', sourcespace=None, buildspace=None, develspace=None, installspace=None, merge=False, install=False, force_cmake=False, colorize=True, build_packages=None, quiet=False, cmake_args=None, make_args=None, catkin_make_args=None, continue_from_pkg=False, only_pkg_with_deps=None, destdir=None, use_ninja=False, use_nmake=False, override_build_tool_check=False ): ''' Runs ``cmake``, ``make`` and optionally ``make install`` for all catkin packages in sourcespace_dir. It creates several folders in the current working directory. For non-catkin packages it runs ``cmake``, ``make`` and ``make install`` for each, installing it to the devel space or install space if the ``install`` option is specified. :param workspace: path to the current workspace, ``str`` :param sourcespace: workspace folder containing catkin packages, ``str`` :param buildspace: path to build space location, ``str`` :param develspace: path to devel space location, ``str`` :param installspace: path to install space (CMAKE_INSTALL_PREFIX), ``str`` :param merge: if True, build each catkin package into the same devel space (not affecting plain cmake packages), ``bool`` :param install: if True, install all packages to the install space, ``bool`` :param force_cmake: (optional), if True calls cmake explicitly for each package, ``bool`` :param colorize: if True, colorize cmake output and other messages, ``bool`` :param build_packages: specific packages to build (all parent packages in the topological order must have been built before), ``str`` :param quiet: if True, hides some build output, ``bool`` :param cmake_args: additional arguments for cmake, ``[str]`` :param make_args: additional arguments for make, ``[str]`` :param catkin_make_args: additional arguments for make but only for catkin packages, ``[str]`` :param continue_from_pkg: indicates whether or not cmi should continue when a package is reached, ``bool`` :param only_pkg_with_deps: only consider the specific packages and their recursive dependencies and ignore all other packages in the workspace, ``[str]`` :param destdir: define DESTDIR for cmake/invocation, ``string`` :param use_ninja: if True, use ninja instead of make, ``bool`` :param use_nmake: if True, use nmake instead of make, ``bool`` :param override_build_tool_check: if True, build even if a space was built by another tool previously. ''' if not colorize: disable_ANSI_colors() # Check workspace existance if not os.path.exists(workspace): sys.exit("Workspace path '{0}' does not exist.".format(workspace)) workspace = os.path.abspath(workspace) # Check source space existance if sourcespace is None: sourcespace = os.path.join(workspace, 'src') if not os.path.exists(sourcespace): sys.exit('Could not find source space: {0}'.format(sourcespace)) print('Base path: ' + str(workspace)) print('Source space: ' + str(sourcespace)) # Check build space if buildspace is None: buildspace = os.path.join(workspace, 'build_isolated') if not os.path.exists(buildspace): os.mkdir(buildspace) print('Build space: ' + str(buildspace)) # ensure the build space was previously built by catkin_make_isolated previous_tool = get_previous_tool_used_on_the_space(buildspace) if previous_tool is not None and previous_tool != 'catkin_make_isolated': if override_build_tool_check: print(fmt( "@{yf}Warning: build space at '%s' was previously built by '%s', " "but --override-build-tool-check was passed so continuing anyways." % (buildspace, previous_tool))) else: sys.exit(fmt( "@{rf}The build space at '%s' was previously built by '%s'. " "Please remove the build space or pick a different build space." % (buildspace, previous_tool))) mark_space_as_built_by(buildspace, 'catkin_make_isolated') # Check devel space if develspace is None: develspace = os.path.join(workspace, 'devel_isolated') print('Devel space: ' + str(develspace)) # ensure the devel space was previously built by catkin_make_isolated previous_tool = get_previous_tool_used_on_the_space(develspace) if previous_tool is not None and previous_tool != 'catkin_make_isolated': if override_build_tool_check: print(fmt( "@{yf}Warning: devel space at '%s' was previously built by '%s', " "but --override-build-tool-check was passed so continuing anyways." % (develspace, previous_tool))) else: sys.exit(fmt( "@{rf}The devel space at '%s' was previously built by '%s'. " "Please remove the devel space or pick a different devel space." % (develspace, previous_tool))) mark_space_as_built_by(develspace, 'catkin_make_isolated') # Check install space if installspace is None: installspace = os.path.join(workspace, 'install_isolated') print('Install space: ' + str(installspace)) if cmake_args: print("Additional CMake Arguments: " + " ".join(cmake_args)) else: cmake_args = [] if not [arg for arg in cmake_args if arg.startswith('-G')]: if use_ninja: cmake_args += ['-G', 'Ninja'] elif use_nmake: cmake_args += ['-G', 'NMake Makefiles'] else: cmake_args += ['-G', 'Unix Makefiles'] elif use_ninja or use_nmake: print(colorize_line("Error: either specify a generator using '-G...' or '--use-[ninja|nmake]' but not both")) sys.exit(1) if make_args: print("Additional make Arguments: " + " ".join(make_args)) else: make_args = [] if catkin_make_args: print("Additional make Arguments for catkin packages: " + " ".join(catkin_make_args)) else: catkin_make_args = [] # Find packages packages = find_packages(sourcespace, exclude_subspaces=True) if not packages: print(fmt("@{yf}No packages found in source space: %s@|" % sourcespace)) # whitelist packages and their dependencies in workspace if only_pkg_with_deps: package_names = [p.name for p in packages.values()] unknown_packages = [name for name in only_pkg_with_deps if name not in package_names] if unknown_packages: sys.exit('Packages not found in the workspace: %s' % ', '.join(unknown_packages)) whitelist_pkg_names = get_package_names_with_recursive_dependencies(packages, only_pkg_with_deps) print('Whitelisted packages: %s' % ', '.join(sorted(whitelist_pkg_names))) packages = {path: p for path, p in packages.items() if p.name in whitelist_pkg_names} # verify that specified package exists in workspace if build_packages: packages_by_name = {p.name: path for path, p in packages.items()} unknown_packages = [p for p in build_packages if p not in packages_by_name] if unknown_packages: sys.exit('Packages not found in the workspace: %s' % ', '.join(unknown_packages)) # Report topological ordering ordered_packages = topological_order_packages(packages) unknown_build_types = [] msg = [] msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_packages))))) msg.append('@{pf}~~@| traversing %d packages in topological order:' % len(ordered_packages)) for path, package in ordered_packages: if path is None: print(fmt('@{rf}Error: Circular dependency in subset of packages: @!%s@|' % package)) sys.exit('Can not build workspace with circular dependency') export_tags = [e.tagname for e in package.exports] if 'build_type' in export_tags: build_type_tag = [e.content for e in package.exports if e.tagname == 'build_type'][0] else: build_type_tag = 'catkin' if build_type_tag == 'catkin': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|') elif build_type_tag == 'cmake': msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@!@{cf}plain cmake@|)' ) else: msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@{rf}unknown@|)' ) unknown_build_types.append(package) msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_packages))))) for index in range(len(msg)): msg[index] = fmt(msg[index]) print('\n'.join(msg)) # Error if there are packages with unknown build_types if unknown_build_types: print(colorize_line('Error: Packages with unknown build types exist')) sys.exit('Can not build workspace with packages of unknown build_type') # Check to see if the workspace has changed cmake_args_with_spaces = list(cmake_args) if develspace: cmake_args_with_spaces.append('-DCATKIN_DEVEL_PREFIX=' + develspace) if installspace: cmake_args_with_spaces.append('-DCMAKE_INSTALL_PREFIX=' + installspace) if ( not force_cmake and cmake_input_changed(packages, buildspace, cmake_args=cmake_args_with_spaces, filename='catkin_make_isolated') ): print('The packages or cmake arguments have changed, forcing cmake invocation') force_cmake = True ensure_workspace_marker(workspace) # Build packages pkg_develspace = None last_env = None for index, path_package in enumerate(ordered_packages): path, package = path_package if merge: pkg_develspace = develspace else: pkg_develspace = os.path.join(develspace, package.name) if not build_packages or package.name in build_packages: if continue_from_pkg and build_packages and package.name in build_packages: build_packages = None try: print() last_env = build_package( path, package, workspace, buildspace, pkg_develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, catkin_make_args, destdir=destdir, use_ninja=use_ninja, number=index + 1, of=len(ordered_packages) ) except subprocess.CalledProcessError as e: _print_build_error(package, e) # Let users know how to reproduce # First add the cd to the build folder of the package cmd = 'cd ' + quote(os.path.join(buildspace, package.name)) + ' && ' # Then reproduce the command called if isinstance(e.cmd, list): # quote arguments to allow copy-n-paste of command cmd += ' '.join([quote(arg) for arg in e.cmd]) else: cmd += e.cmd print(fmt("\n@{rf}Reproduce this error by running:")) print(fmt("@{gf}@!==> @|") + cmd + "\n") sys.exit('Command failed, exiting.') except Exception as e: print("Unhandled exception of type '{0}':".format(type(e).__name__)) import traceback traceback.print_exc() _print_build_error(package, e) sys.exit('Command failed, exiting.') else: cprint("Skipping package: '@!@{bf}" + package.name + "@|'") last_env = get_new_env(package, pkg_develspace, installspace, install, last_env, destdir) # Provide a top level devel space environment setup script if not os.path.exists(develspace): os.makedirs(develspace) if not build_packages: generated_env_sh = os.path.join(develspace, 'env.sh') generated_setup_util_py = os.path.join(develspace, '_setup_util.py') if not merge and pkg_develspace: # generate env.sh and setup.sh|bash|zsh which relay to last devel space with open(generated_env_sh, 'w') as f: f.write("""\ #!/usr/bin/env sh # generated from catkin.builder module {0} "$@" """.format(os.path.join(pkg_develspace, 'env.sh'))) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) for shell in ['sh', 'bash', 'zsh']: with open(os.path.join(develspace, 'setup.%s' % shell), 'w') as f: f.write("""\ #!/usr/bin/env {1} # generated from catkin.builder module . "{0}/setup.{1}" """.format(pkg_develspace, shell)) # remove _setup_util.py file which might have been generated for an empty devel space before if os.path.exists(generated_setup_util_py): os.remove(generated_setup_util_py) elif not pkg_develspace: # generate env.sh and setup.sh|bash|zsh for an empty devel space if 'CMAKE_PREFIX_PATH' in os.environ.keys(): variables = { 'CATKIN_GLOBAL_BIN_DESTINATION': 'bin', 'CATKIN_LIB_ENVIRONMENT_PATHS': "'lib'", 'CATKIN_PKGCONFIG_ENVIRONMENT_PATHS': "os.path.join('lib', 'pkgconfig')", 'CMAKE_PREFIX_PATH_AS_IS': ';'.join(os.environ['CMAKE_PREFIX_PATH'].split(os.pathsep)), 'PYTHON_EXECUTABLE': sys.executable, 'PYTHON_INSTALL_DIR': get_python_install_dir(), } with open(generated_setup_util_py, 'w') as f: f.write(configure_file( os.path.join(get_cmake_path(), 'templates', '_setup_util.py.in'), variables)) os.chmod(generated_setup_util_py, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) else: sys.exit("Unable to process CMAKE_PREFIX_PATH from environment. Cannot generate environment files.") variables = {'SETUP_FILENAME': 'setup'} with open(generated_env_sh, 'w') as f: f.write(configure_file(os.path.join(get_cmake_path(), 'templates', 'env.sh.in'), variables)) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) variables = {'SETUP_DIR': develspace} for shell in ['sh', 'bash', 'zsh']: with open(os.path.join(develspace, 'setup.%s' % shell), 'w') as f: f.write(configure_file( os.path.join(get_cmake_path(), 'templates', 'setup.%s.in' % shell), variables))
def build_catkin_package( path, package, workspace, buildspace, develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, destdir=None, use_ninja=False, use_nmake=False ): cprint( "Processing @{cf}catkin@| package: '@!@{bf}" + package.name + "@|'" ) # Make the build dir build_dir = _check_build_dir(package.name, workspace, buildspace) # Check last_env if last_env is not None: cprint( blue_arrow + " Building with env: " + "'{0}'".format(last_env) ) # Check for Makefile and maybe call cmake if not use_ninja: makefile_name = 'Makefile' else: makefile_name = 'build.ninja' makefile = os.path.join(build_dir, makefile_name) if not os.path.exists(makefile) or force_cmake: package_dir = os.path.dirname(package.filename) if not os.path.exists(os.path.join(package_dir, 'CMakeLists.txt')): export_tags = [e.tagname for e in package.exports] if 'metapackage' not in export_tags: print(colorize_line('Error: Package "%s" does not have a CMakeLists.txt file' % package.name)) sys.exit('Can not build catkin package without CMakeLists.txt file') # generate CMakeLists.txt for metpackages without one print(colorize_line('Warning: metapackage "%s" should have a CMakeLists.txt file' % package.name)) cmake_code = configure_file( get_metapackage_cmake_template_path(), {'name': package.name, 'metapackage_arguments': 'DIRECTORY "%s"' % package_dir}) cmakelists_txt = os.path.join(build_dir, 'CMakeLists.txt') with open(cmakelists_txt, 'w') as f: f.write(cmake_code) package_dir = build_dir # Run cmake cmake_cmd = [ 'cmake', package_dir, '-DCATKIN_DEVEL_PREFIX=' + develspace, '-DCMAKE_INSTALL_PREFIX=' + installspace ] cmake_cmd.extend(cmake_args) add_env = get_additional_environment(install, destdir, installspace) isolation_print_command(' '.join(cmake_cmd), build_dir, add_env=add_env) if last_env is not None: cmake_cmd = [last_env] + cmake_cmd try: run_command_colorized(cmake_cmd, build_dir, quiet, add_env=add_env) except subprocess.CalledProcessError as e: if os.path.exists(makefile): # remove Makefile to force CMake invocation next time os.remove(makefile) raise else: print('%s exists, skipping explicit cmake invocation...' % makefile_name) # Check to see if cmake needs to be run via make if use_ninja: make_check_cmake_cmd = ['ninja', 'build.ninja'] elif use_nmake: make_check_cmake_cmd = ['nmake', 'cmake_check_build_system'] else: make_check_cmake_cmd = ['make', 'cmake_check_build_system'] add_env = get_additional_environment(install, destdir, installspace) isolation_print_command(' '.join(make_check_cmake_cmd), build_dir, add_env=add_env) if last_env is not None: make_check_cmake_cmd = [last_env] + make_check_cmake_cmd run_command_colorized( make_check_cmake_cmd, build_dir, quiet, add_env=add_env ) # Run make if use_ninja: make_executable = 'ninja' elif use_nmake: make_executable = 'nmake' else: make_executable = 'make' make_cmd = [make_executable] make_cmd.extend(handle_make_arguments(make_args)) isolation_print_command(' '.join(make_cmd), build_dir) if last_env is not None: make_cmd = [last_env] + make_cmd run_command(make_cmd, build_dir, quiet) # Make install # NMake doesn't have an option to list target so try it anyway if install or use_nmake: if has_make_target(build_dir, 'install', use_ninja=use_ninja): make_install_cmd = [make_executable, 'install'] isolation_print_command(' '.join(make_install_cmd), build_dir) if last_env is not None: make_install_cmd = [last_env] + make_install_cmd run_command(make_install_cmd, build_dir, quiet) else: print(fmt( '@{yf}Package has no "@{boldon}install@{boldoff}" target, skipping "%s install" invocation...' % make_executable))
def build_workspace_isolated( workspace='.', sourcespace=None, buildspace=None, develspace=None, installspace=None, merge=False, install=False, jobs=None, force_cmake=False, colorize=True, quiet=False, cmake_args=[] ): ''' Runs ``cmake``, ``make`` and optionally ``make install`` for all catkin packages in sourcespace_dir. It creates several folders in the current working directory. For non-catkin packages it runs ``cmake``, ``make`` and ``make install`` for each, installing it to the devel space or install space if the ``install`` option is specified. :param workspace: path to the current workspace, ``str`` :param sourcespace: workspace folder containing catkin packages, ``str`` :param buildspace: path to build space location, ``str`` :param develspace: path to devel space location, ``str`` :param installspace: path to install space (CMAKE_INSTALL_PREFIX), ``str`` :param merge: if True, build each catkin package into the same devel space. does not work with non-catkin packages, ``bool`` :param install: if True, install all packages to the install space, ``bool`` :param jobs: number of parallel build jobs to run (make -jN -lN), ``int`` :param force_cmake: (optional), if True calls cmake explicitly for each package, ``bool`` :param colorize: if True, colorize cmake output and other messages, ``bool`` :param quiet: if True, hides some build output, ``bool`` :param cmake_args: additional arguments for cmake, ``[str]`` ''' if not colorize: disable_ANSI_colors() # Check workspace existance if not os.path.exists(workspace): sys.exit("Workspace path '{0}' does not exist.".format(workspace)) workspace = os.path.abspath(workspace) if cmake_args: print("Additional CMake Arguments: " + " ".join(cmake_args)) # Check source space existance if sourcespace is None: ws_sourcespace = os.path.join(workspace, 'src') if not os.path.exists(ws_sourcespace): sys.exit("Could not find source space: {0}".format(sourcespace)) sourcespace = ws_sourcespace sourcespace = os.path.abspath(sourcespace) print('Base path: ' + str(workspace)) print('Source space: ' + str(sourcespace)) # Check build space if buildspace is None: buildspace = os.path.join(workspace, 'build_isolated') buildspace = os.path.abspath(buildspace) if not os.path.exists(buildspace): os.mkdir(buildspace) print('Build space: ' + str(buildspace)) # Check devel space if develspace is None: develspace = os.path.join(workspace, 'devel_isolated') develspace = os.path.abspath(develspace) print('Devel space: ' + str(develspace)) # Check install space if installspace is None: installspace = os.path.join(workspace, 'install_isolated') installspace = os.path.abspath(installspace) print('Install space: ' + str(installspace)) # Check jobs if not jobs: try: jobs = multiprocessing.cpu_count() except NotImplementedError: jobs = 1 jobs = int(jobs) # Find packages packages = topological_order(sourcespace) if not packages: sys.exit("No packages found in source space: {0}".format(sourcespace)) # Report topological ordering unknown_build_types = [] msg = [] msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') msg.append('@{pf}~~@| traversing packages in topological order:') for path, package in packages: export_tags = [e.tagname for e in package.exports] if 'metapackage' in export_tags: msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@{cf}metapackage@|)' ) else: if 'build_type' in export_tags: build_type_tag = [e.content for e in package.exports if e.tagname == 'build_type'][0] else: build_type_tag = 'catkin' if build_type_tag == 'catkin': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|') elif build_type_tag == 'cmake': msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@!@{cf}plain cmake@|)' ) else: msg.append( '@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@{rf}unknown@|)' ) unknown_build_types.append(package) msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') for index in range(len(msg)): msg[index] = fmt(msg[index]) print('\n'.join(msg)) # Error if there are packages with unknown build_types if unknown_build_types: print(colorize_line('Error: Packages with unknown build types exist')) sys.exit('Can not build workspace with packages of unknown build_type') # Check to see if the workspace has changed if cmake_input_changed( sourcespace, buildspace, cmake_args, 'catkin_make_isolated' ): force_cmake = True print(colorize_line( 'Warning: packages or cmake arguments have changed, forcing cmake' )) # Build packages original_develspace = copy.deepcopy(develspace) last_env = None for index, path_package in enumerate(packages): path, package = path_package if not merge: develspace = os.path.join(original_develspace, package.name) try: last_env = build_package( path, package, workspace, buildspace, develspace, installspace, install, jobs, force_cmake, quiet, last_env, cmake_args, number=index + 1, of=len(packages) ) except Exception as e: cprint( '@{rf}@!<==@| ' + 'Failed to process package \'@!@{bf}' + package.name + '@|\': \n ' + ('KeyboardInterrupt' if isinstance(e, KeyboardInterrupt) else str(e)) ) if isinstance(e, subprocess.CalledProcessError): cmd = ' '.join(e.cmd) if isinstance(e.cmd, list) else e.cmd print(fmt("\n@{rf}Reproduce this error by running:")) print(fmt("@{gf}@!==> @|") + cmd + "\n") sys.exit('Command failed, exiting.') # Provide a top level devel space environment setup script if not merge: target_setup = os.path.join(original_develspace, 'setup') with open(target_setup + '.sh', 'w') as f: f.write("""\ # Generated from catkin.builder module . "{0}/setup.sh" """.format(develspace)) with open(target_setup + '.bash', 'w') as f: f.write("""\ # Generated from catkin.builder module CATKIN_SHELL=bash source "{0}" """.format(target_setup + '.sh')) with open(target_setup + '.zsh', 'w') as f: f.write("""\ # Generated from catkin.builder module CATKIN_SHELL=zsh emulate sh # emulate POSIX . "{0}" emulate zsh # back to zsh mode """.format(target_setup + '.sh'))
def build_catkin_package(path, package, workspace, buildspace, develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, destdir=None, use_ninja=False): cprint("Processing @{cf}catkin@| package: '@!@{bf}" + package.name + "@|'") # Make the build dir build_dir = _check_build_dir(package.name, workspace, buildspace) # Check last_env if last_env is not None: cprint(blue_arrow + " Building with env: " + "'{0}'".format(last_env)) # Check for Makefile and maybe call cmake if not use_ninja: makefile_name = 'Makefile' else: makefile_name = 'build.ninja' makefile = os.path.join(build_dir, makefile_name) if not os.path.exists(makefile) or force_cmake: package_dir = os.path.dirname(package.filename) if not os.path.exists(os.path.join(package_dir, 'CMakeLists.txt')): export_tags = [e.tagname for e in package.exports] if 'metapackage' not in export_tags: print( colorize_line( 'Error: Package "%s" does not have a CMakeLists.txt file' % package.name)) sys.exit( 'Can not build catkin package without CMakeLists.txt file') # generate CMakeLists.txt for metpackages without one print( colorize_line( 'Warning: metapackage "%s" should have a CMakeLists.txt file' % package.name)) cmake_code = configure_file( get_metapackage_cmake_template_path(), { 'name': package.name, 'metapackage_arguments': 'DIRECTORY "%s"' % package_dir }) cmakelists_txt = os.path.join(build_dir, 'CMakeLists.txt') with open(cmakelists_txt, 'w') as f: f.write(cmake_code) package_dir = build_dir # Run cmake cmake_cmd = [ 'cmake', package_dir, '-DCATKIN_DEVEL_PREFIX=' + develspace, '-DCMAKE_INSTALL_PREFIX=' + installspace ] cmake_cmd.extend(cmake_args) add_env = get_additional_environment(install, destdir, installspace) isolation_print_command(' '.join(cmake_cmd), build_dir, add_env=add_env) if last_env is not None: cmake_cmd = [last_env] + cmake_cmd try: run_command_colorized(cmake_cmd, build_dir, quiet, add_env=add_env) except subprocess.CalledProcessError as e: if os.path.exists(makefile): # remove Makefile to force CMake invocation next time os.remove(makefile) raise else: print('%s exists, skipping explicit cmake invocation...' % makefile_name) # Check to see if cmake needs to be run via make if not use_ninja: make_check_cmake_cmd = ['make', 'cmake_check_build_system'] else: make_check_cmake_cmd = ['ninja', 'build.ninja'] add_env = get_additional_environment(install, destdir, installspace) isolation_print_command(' '.join(make_check_cmake_cmd), build_dir, add_env=add_env) if last_env is not None: make_check_cmake_cmd = [last_env] + make_check_cmake_cmd run_command_colorized(make_check_cmake_cmd, build_dir, quiet, add_env=add_env) # Run make if not use_ninja: make_executable = 'make' else: make_executable = 'ninja' make_cmd = [make_executable] make_cmd.extend(handle_make_arguments(make_args)) isolation_print_command(' '.join(make_cmd), build_dir) if last_env is not None: make_cmd = [last_env] + make_cmd run_command(make_cmd, build_dir, quiet) # Make install if install: if has_make_target(build_dir, 'install', use_ninja=use_ninja): make_install_cmd = [make_executable, 'install'] isolation_print_command(' '.join(make_install_cmd), build_dir) if last_env is not None: make_install_cmd = [last_env] + make_install_cmd run_command(make_install_cmd, build_dir, quiet) else: print( fmt('@{yf}Package has no "@{boldon}install@{boldoff}" target, skipping "%s install" invocation...' % make_executable))
def build_workspace_isolated( workspace=".", sourcespace=None, buildspace=None, develspace=None, installspace=None, merge=False, install=False, force_cmake=False, colorize=True, build_packages=None, quiet=False, cmake_args=None, make_args=None, catkin_make_args=None, continue_from_pkg=False, only_pkg_with_deps=None, destdir=None, ): """ Runs ``cmake``, ``make`` and optionally ``make install`` for all catkin packages in sourcespace_dir. It creates several folders in the current working directory. For non-catkin packages it runs ``cmake``, ``make`` and ``make install`` for each, installing it to the devel space or install space if the ``install`` option is specified. :param workspace: path to the current workspace, ``str`` :param sourcespace: workspace folder containing catkin packages, ``str`` :param buildspace: path to build space location, ``str`` :param develspace: path to devel space location, ``str`` :param installspace: path to install space (CMAKE_INSTALL_PREFIX), ``str`` :param merge: if True, build each catkin package into the same devel space (not affecting plain cmake packages), ``bool`` :param install: if True, install all packages to the install space, ``bool`` :param force_cmake: (optional), if True calls cmake explicitly for each package, ``bool`` :param colorize: if True, colorize cmake output and other messages, ``bool`` :param build_packages: specific packages to build (all parent packages in the topological order must have been built before), ``str`` :param quiet: if True, hides some build output, ``bool`` :param cmake_args: additional arguments for cmake, ``[str]`` :param make_args: additional arguments for make, ``[str]`` :param catkin_make_args: additional arguments for make but only for catkin packages, ``[str]`` :param continue_from_pkg: indicates whether or not cmi should continue when a package is reached, ``bool`` :param only_pkg_with_deps: only consider the specific packages and their recursive dependencies and ignore all other packages in the workspace, ``[str]`` :param destdir: define DESTDIR for cmake/invocation, ``string`` """ if not colorize: disable_ANSI_colors() # Check workspace existance if not os.path.exists(workspace): sys.exit("Workspace path '{0}' does not exist.".format(workspace)) workspace = os.path.abspath(workspace) # Check source space existance if sourcespace is None: sourcespace = os.path.join(workspace, "src") if not os.path.exists(sourcespace): sys.exit("Could not find source space: {0}".format(sourcespace)) print("Base path: " + str(workspace)) print("Source space: " + str(sourcespace)) # Check build space if buildspace is None: buildspace = os.path.join(workspace, "build_isolated") if not os.path.exists(buildspace): os.mkdir(buildspace) print("Build space: " + str(buildspace)) # Check devel space if develspace is None: develspace = os.path.join(workspace, "devel_isolated") print("Devel space: " + str(develspace)) # Check install space if installspace is None: installspace = os.path.join(workspace, "install_isolated") print("Install space: " + str(installspace)) if cmake_args: print("Additional CMake Arguments: " + " ".join(cmake_args)) else: cmake_args = [] if make_args: print("Additional make Arguments: " + " ".join(make_args)) else: make_args = [] if catkin_make_args: print("Additional make Arguments for catkin packages: " + " ".join(catkin_make_args)) else: catkin_make_args = [] # Find packages packages = find_packages(sourcespace, exclude_subspaces=True) if not packages: print(fmt("@{yf}No packages found in source space: %s@|" % sourcespace)) # whitelist packages and their dependencies in workspace if only_pkg_with_deps: package_names = [p.name for p in packages.values()] unknown_packages = [name for name in only_pkg_with_deps if name not in package_names] if unknown_packages: sys.exit("Packages not found in the workspace: %s" % ", ".join(unknown_packages)) whitelist_pkg_names = get_package_names_with_recursive_dependencies(packages, only_pkg_with_deps) print("Whitelisted packages: %s" % ", ".join(sorted(whitelist_pkg_names))) packages = {path: p for path, p in packages.items() if p.name in whitelist_pkg_names} # verify that specified package exists in workspace if build_packages: packages_by_name = {p.name: path for path, p in packages.items()} unknown_packages = [p for p in build_packages if p not in packages_by_name] if unknown_packages: sys.exit("Packages not found in the workspace: %s" % ", ".join(unknown_packages)) # Report topological ordering ordered_packages = topological_order_packages(packages) unknown_build_types = [] msg = [] msg.append("@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + ("~" * len(str(len(ordered_packages))))) msg.append("@{pf}~~@| traversing %d packages in topological order:" % len(ordered_packages)) for path, package in ordered_packages: if path is None: print(fmt("@{rf}Error: Circular dependency in subset of packages: @!%s@|" % package)) sys.exit("Can not build workspace with circular dependency") export_tags = [e.tagname for e in package.exports] if "build_type" in export_tags: build_type_tag = [e.content for e in package.exports if e.tagname == "build_type"][0] else: build_type_tag = "catkin" if build_type_tag == "catkin": msg.append("@{pf}~~@| - @!@{bf}" + package.name + "@|") elif build_type_tag == "cmake": msg.append("@{pf}~~@| - @!@{bf}" + package.name + "@|" + " (@!@{cf}plain cmake@|)") else: msg.append("@{pf}~~@| - @!@{bf}" + package.name + "@|" + " (@{rf}unknown@|)") unknown_build_types.append(package) msg.append("@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + ("~" * len(str(len(ordered_packages))))) for index in range(len(msg)): msg[index] = fmt(msg[index]) print("\n".join(msg)) # Error if there are packages with unknown build_types if unknown_build_types: print(colorize_line("Error: Packages with unknown build types exist")) sys.exit("Can not build workspace with packages of unknown build_type") # Check to see if the workspace has changed cmake_args_with_spaces = list(cmake_args) if develspace: cmake_args_with_spaces.append("-DCATKIN_DEVEL_PREFIX=" + develspace) if installspace: cmake_args_with_spaces.append("-DCMAKE_INSTALL_PREFIX=" + installspace) if not force_cmake and cmake_input_changed( packages, buildspace, cmake_args=cmake_args_with_spaces, filename="catkin_make_isolated" ): print("The packages or cmake arguments have changed, forcing cmake invocation") force_cmake = True ensure_workspace_marker(workspace) # Build packages pkg_develspace = None last_env = None for index, path_package in enumerate(ordered_packages): path, package = path_package if merge: pkg_develspace = develspace else: pkg_develspace = os.path.join(develspace, package.name) if not build_packages or package.name in build_packages: if continue_from_pkg and build_packages and package.name in build_packages: build_packages = None try: print() last_env = build_package( path, package, workspace, buildspace, pkg_develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, catkin_make_args, destdir=destdir, number=index + 1, of=len(ordered_packages), ) except subprocess.CalledProcessError as e: _print_build_error(package, e) # Let users know how to reproduce # First add the cd to the build folder of the package cmd = "cd " + os.path.join(buildspace, package.name) + " && " # Then reproduce the command called cmd += " ".join(e.cmd) if isinstance(e.cmd, list) else e.cmd print(fmt("\n@{rf}Reproduce this error by running:")) print(fmt("@{gf}@!==> @|") + cmd + "\n") sys.exit("Command failed, exiting.") except Exception as e: print("Unhandled exception of type '{0}':".format(type(e).__name__)) import traceback traceback.print_exc() _print_build_error(package, e) sys.exit("Command failed, exiting.") else: cprint("Skipping package: '@!@{bf}" + package.name + "@|'") last_env = get_new_env(package, pkg_develspace, installspace, install, last_env, destdir) # Provide a top level devel space environment setup script if not os.path.exists(develspace): os.makedirs(develspace) if not build_packages: generated_env_sh = os.path.join(develspace, "env.sh") generated_setup_util_py = os.path.join(develspace, "_setup_util.py") if not merge and pkg_develspace: # generate env.sh and setup.sh|bash|zsh which relay to last devel space with open(generated_env_sh, "w") as f: f.write( """\ #!/usr/bin/env sh # generated from catkin.builder module {0} "$@" """.format( os.path.join(pkg_develspace, "env.sh") ) ) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) for shell in ["sh", "bash", "zsh"]: with open(os.path.join(develspace, "setup.%s" % shell), "w") as f: f.write( """\ #!/usr/bin/env {1} # generated from catkin.builder module . "{0}/setup.{1}" """.format( pkg_develspace, shell ) ) # remove _setup_util.py file which might have been generated for an empty devel space before if os.path.exists(generated_setup_util_py): os.remove(generated_setup_util_py) elif not pkg_develspace: # generate env.sh and setup.sh|bash|zsh for an empty devel space if "CMAKE_PREFIX_PATH" in os.environ.keys(): variables = { "CATKIN_GLOBAL_BIN_DESTINATION": "bin", "CATKIN_GLOBAL_LIB_DESTINATION": "lib", "CMAKE_PREFIX_PATH_AS_IS": ";".join(os.environ["CMAKE_PREFIX_PATH"].split(os.pathsep)), "PYTHON_INSTALL_DIR": get_python_install_dir(), } with open(generated_setup_util_py, "w") as f: f.write(configure_file(os.path.join(get_cmake_path(), "templates", "_setup_util.py.in"), variables)) os.chmod(generated_setup_util_py, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) else: sys.exit("Unable to process CMAKE_PREFIX_PATH from environment. Cannot generate environment files.") variables = {"SETUP_FILENAME": "setup"} with open(generated_env_sh, "w") as f: f.write(configure_file(os.path.join(get_cmake_path(), "templates", "env.sh.in"), variables)) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) variables = {"SETUP_DIR": develspace} for shell in ["sh", "bash", "zsh"]: with open(os.path.join(develspace, "setup.%s" % shell), "w") as f: f.write( configure_file(os.path.join(get_cmake_path(), "templates", "setup.%s.in" % shell), variables) )
def build_catkin_package( path, package, workspace, buildspace, develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, destdir=None, ): cprint("Processing @{cf}catkin@| package: '@!@{bf}" + package.name + "@|'") # Make the build dir build_dir = _check_build_dir(package.name, workspace, buildspace) # Check last_env if last_env is not None: cprint(blue_arrow + " Building with env: " + "'{0}'".format(last_env)) # Check for Makefile and maybe call cmake makefile = os.path.join(build_dir, "Makefile") if not os.path.exists(makefile) or force_cmake: package_dir = os.path.dirname(package.filename) if not os.path.exists(os.path.join(package_dir, "CMakeLists.txt")): export_tags = [e.tagname for e in package.exports] if "metapackage" not in export_tags: print(colorize_line('Error: Package "%s" does not have a CMakeLists.txt file' % package.name)) sys.exit("Can not build catkin package without CMakeLists.txt file") # generate CMakeLists.txt for metpackages without one print(colorize_line('Warning: metapackage "%s" should have a CMakeLists.txt file' % package.name)) cmake_code = configure_file( get_metapackage_cmake_template_path(), {"name": package.name, "metapackage_arguments": 'DIRECTORY "%s"' % package_dir}, ) cmakelists_txt = os.path.join(build_dir, "CMakeLists.txt") with open(cmakelists_txt, "w") as f: f.write(cmake_code) package_dir = build_dir # Run cmake cmake_cmd = [ "cmake", package_dir, "-DCATKIN_DEVEL_PREFIX=" + develspace, "-DCMAKE_INSTALL_PREFIX=" + installspace, ] cmake_cmd.extend(cmake_args) add_env = get_additional_environment(install, destdir, installspace) isolation_print_command(" ".join(cmake_cmd), build_dir, add_env=add_env) if last_env is not None: cmake_cmd = [last_env] + cmake_cmd try: run_command_colorized(cmake_cmd, build_dir, quiet, add_env=add_env) except subprocess.CalledProcessError as e: if os.path.exists(makefile): # remove Makefile to force CMake invocation next time os.remove(makefile) raise else: print("Makefile exists, skipping explicit cmake invocation...") # Check to see if cmake needs to be run via make make_check_cmake_cmd = ["make", "cmake_check_build_system"] add_env = get_additional_environment(install, destdir, installspace) isolation_print_command(" ".join(make_check_cmake_cmd), build_dir, add_env=add_env) if last_env is not None: make_check_cmake_cmd = [last_env] + make_check_cmake_cmd run_command_colorized(make_check_cmake_cmd, build_dir, quiet, add_env=add_env) # Run make make_cmd = ["make"] make_cmd.extend(handle_make_arguments(make_args, force_single_threaded_when_running_tests=True)) isolation_print_command(" ".join(make_cmd), build_dir) if last_env is not None: make_cmd = [last_env] + make_cmd run_command(make_cmd, build_dir, quiet) # Make install if install: if has_make_target(build_dir, "install"): make_install_cmd = ["make", "install"] isolation_print_command(" ".join(make_install_cmd), build_dir) if last_env is not None: make_install_cmd = [last_env] + make_install_cmd run_command(make_install_cmd, build_dir, quiet) else: print(fmt('@{yf}Package has no "@{boldon}install@{boldoff}" target, skipping "make install" invocation...'))
def build_workspace_isolated(workspace='.', sourcespace=None, buildspace=None, develspace=None, installspace=None, merge=False, install=False, force_cmake=False, colorize=True, build_packages=None, quiet=False, cmake_args=None, make_args=None, catkin_make_args=None, continue_from_pkg=False, only_pkg_with_deps=None, destdir=None, use_ninja=False): ''' Runs ``cmake``, ``make`` and optionally ``make install`` for all catkin packages in sourcespace_dir. It creates several folders in the current working directory. For non-catkin packages it runs ``cmake``, ``make`` and ``make install`` for each, installing it to the devel space or install space if the ``install`` option is specified. :param workspace: path to the current workspace, ``str`` :param sourcespace: workspace folder containing catkin packages, ``str`` :param buildspace: path to build space location, ``str`` :param develspace: path to devel space location, ``str`` :param installspace: path to install space (CMAKE_INSTALL_PREFIX), ``str`` :param merge: if True, build each catkin package into the same devel space (not affecting plain cmake packages), ``bool`` :param install: if True, install all packages to the install space, ``bool`` :param force_cmake: (optional), if True calls cmake explicitly for each package, ``bool`` :param colorize: if True, colorize cmake output and other messages, ``bool`` :param build_packages: specific packages to build (all parent packages in the topological order must have been built before), ``str`` :param quiet: if True, hides some build output, ``bool`` :param cmake_args: additional arguments for cmake, ``[str]`` :param make_args: additional arguments for make, ``[str]`` :param catkin_make_args: additional arguments for make but only for catkin packages, ``[str]`` :param continue_from_pkg: indicates whether or not cmi should continue when a package is reached, ``bool`` :param only_pkg_with_deps: only consider the specific packages and their recursive dependencies and ignore all other packages in the workspace, ``[str]`` :param destdir: define DESTDIR for cmake/invocation, ``string`` :param use_ninja: if True, use ninja instead of make, ``bool`` ''' if not colorize: disable_ANSI_colors() # Check workspace existance if not os.path.exists(workspace): sys.exit("Workspace path '{0}' does not exist.".format(workspace)) workspace = os.path.abspath(workspace) # Check source space existance if sourcespace is None: sourcespace = os.path.join(workspace, 'src') if not os.path.exists(sourcespace): sys.exit('Could not find source space: {0}'.format(sourcespace)) print('Base path: ' + str(workspace)) print('Source space: ' + str(sourcespace)) # Check build space if buildspace is None: buildspace = os.path.join(workspace, 'build_isolated') if not os.path.exists(buildspace): os.mkdir(buildspace) print('Build space: ' + str(buildspace)) # Check devel space if develspace is None: develspace = os.path.join(workspace, 'devel_isolated') print('Devel space: ' + str(develspace)) # Check install space if installspace is None: installspace = os.path.join(workspace, 'install_isolated') print('Install space: ' + str(installspace)) if cmake_args: print("Additional CMake Arguments: " + " ".join(cmake_args)) else: cmake_args = [] if not [arg for arg in cmake_args if arg.startswith('-G')]: if not use_ninja: cmake_args += ['-G', 'Unix Makefiles'] else: cmake_args += ['-G', 'Ninja'] elif use_ninja: print( colorize_line( "Error: either specify a generator using '-G...' or '--use-ninja' but not both" )) sys.exit(1) if make_args: print("Additional make Arguments: " + " ".join(make_args)) else: make_args = [] if catkin_make_args: print("Additional make Arguments for catkin packages: " + " ".join(catkin_make_args)) else: catkin_make_args = [] # Find packages packages = find_packages(sourcespace, exclude_subspaces=True) if not packages: print(fmt("@{yf}No packages found in source space: %s@|" % sourcespace)) # whitelist packages and their dependencies in workspace if only_pkg_with_deps: package_names = [p.name for p in packages.values()] unknown_packages = [ name for name in only_pkg_with_deps if name not in package_names ] if unknown_packages: sys.exit('Packages not found in the workspace: %s' % ', '.join(unknown_packages)) whitelist_pkg_names = get_package_names_with_recursive_dependencies( packages, only_pkg_with_deps) print('Whitelisted packages: %s' % ', '.join(sorted(whitelist_pkg_names))) packages = { path: p for path, p in packages.items() if p.name in whitelist_pkg_names } # verify that specified package exists in workspace if build_packages: packages_by_name = {p.name: path for path, p in packages.items()} unknown_packages = [ p for p in build_packages if p not in packages_by_name ] if unknown_packages: sys.exit('Packages not found in the workspace: %s' % ', '.join(unknown_packages)) # Report topological ordering ordered_packages = topological_order_packages(packages) unknown_build_types = [] msg = [] msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_packages))))) msg.append('@{pf}~~@| traversing %d packages in topological order:' % len(ordered_packages)) for path, package in ordered_packages: if path is None: print( fmt('@{rf}Error: Circular dependency in subset of packages: @!%s@|' % package)) sys.exit('Can not build workspace with circular dependency') export_tags = [e.tagname for e in package.exports] if 'build_type' in export_tags: build_type_tag = [ e.content for e in package.exports if e.tagname == 'build_type' ][0] else: build_type_tag = 'catkin' if build_type_tag == 'catkin': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|') elif build_type_tag == 'cmake': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@!@{cf}plain cmake@|)') else: msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@{rf}unknown@|)') unknown_build_types.append(package) msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_packages))))) for index in range(len(msg)): msg[index] = fmt(msg[index]) print('\n'.join(msg)) # Error if there are packages with unknown build_types if unknown_build_types: print(colorize_line('Error: Packages with unknown build types exist')) sys.exit('Can not build workspace with packages of unknown build_type') # Check to see if the workspace has changed cmake_args_with_spaces = list(cmake_args) if develspace: cmake_args_with_spaces.append('-DCATKIN_DEVEL_PREFIX=' + develspace) if installspace: cmake_args_with_spaces.append('-DCMAKE_INSTALL_PREFIX=' + installspace) if not force_cmake and cmake_input_changed( packages, buildspace, cmake_args=cmake_args_with_spaces, filename='catkin_make_isolated'): print( 'The packages or cmake arguments have changed, forcing cmake invocation' ) force_cmake = True ensure_workspace_marker(workspace) # Build packages pkg_develspace = None last_env = None for index, path_package in enumerate(ordered_packages): path, package = path_package if merge: pkg_develspace = develspace else: pkg_develspace = os.path.join(develspace, package.name) if not build_packages or package.name in build_packages: if continue_from_pkg and build_packages and package.name in build_packages: build_packages = None try: print() last_env = build_package(path, package, workspace, buildspace, pkg_develspace, installspace, install, force_cmake, quiet, last_env, cmake_args, make_args, catkin_make_args, destdir=destdir, use_ninja=use_ninja, number=index + 1, of=len(ordered_packages)) except subprocess.CalledProcessError as e: _print_build_error(package, e) # Let users know how to reproduce # First add the cd to the build folder of the package cmd = 'cd ' + quote(os.path.join(buildspace, package.name)) + ' && ' # Then reproduce the command called if isinstance(e.cmd, list): # quote arguments to allow copy-n-paste of command cmd += ' '.join([quote(arg) for arg in e.cmd]) else: cmd += e.cmd print(fmt("\n@{rf}Reproduce this error by running:")) print(fmt("@{gf}@!==> @|") + cmd + "\n") sys.exit('Command failed, exiting.') except Exception as e: print("Unhandled exception of type '{0}':".format( type(e).__name__)) import traceback traceback.print_exc() _print_build_error(package, e) sys.exit('Command failed, exiting.') else: cprint("Skipping package: '@!@{bf}" + package.name + "@|'") last_env = get_new_env(package, pkg_develspace, installspace, install, last_env, destdir) # Provide a top level devel space environment setup script if not os.path.exists(develspace): os.makedirs(develspace) if not build_packages: generated_env_sh = os.path.join(develspace, 'env.sh') generated_setup_util_py = os.path.join(develspace, '_setup_util.py') if not merge and pkg_develspace: # generate env.sh and setup.sh|bash|zsh which relay to last devel space with open(generated_env_sh, 'w') as f: f.write("""\ #!/usr/bin/env sh # generated from catkin.builder module {0} "$@" """.format(os.path.join(pkg_develspace, 'env.sh'))) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) for shell in ['sh', 'bash', 'zsh']: with open(os.path.join(develspace, 'setup.%s' % shell), 'w') as f: f.write("""\ #!/usr/bin/env {1} # generated from catkin.builder module . "{0}/setup.{1}" """.format(pkg_develspace, shell)) # remove _setup_util.py file which might have been generated for an empty devel space before if os.path.exists(generated_setup_util_py): os.remove(generated_setup_util_py) elif not pkg_develspace: # generate env.sh and setup.sh|bash|zsh for an empty devel space if 'CMAKE_PREFIX_PATH' in os.environ.keys(): variables = { 'CATKIN_GLOBAL_BIN_DESTINATION': 'bin', 'CATKIN_LIB_ENVIRONMENT_PATHS': "'lib'", 'CATKIN_PKGCONFIG_ENVIRONMENT_PATHS': "os.path.join('lib', 'pkgconfig')", 'CMAKE_PREFIX_PATH_AS_IS': ';'.join(os.environ['CMAKE_PREFIX_PATH'].split( os.pathsep)), 'PYTHON_EXECUTABLE': sys.executable, 'PYTHON_INSTALL_DIR': get_python_install_dir(), } with open(generated_setup_util_py, 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', '_setup_util.py.in'), variables)) os.chmod(generated_setup_util_py, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) else: sys.exit( "Unable to process CMAKE_PREFIX_PATH from environment. Cannot generate environment files." ) variables = {'SETUP_FILENAME': 'setup'} with open(generated_env_sh, 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', 'env.sh.in'), variables)) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) variables = {'SETUP_DIR': develspace} for shell in ['sh', 'bash', 'zsh']: with open(os.path.join(develspace, 'setup.%s' % shell), 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', 'setup.%s.in' % shell), variables))
def build_workspace_isolated(workspace='.', sourcespace=None, buildspace=None, develspace=None, installspace=None, merge=False, install=False, force_cmake=False, colorize=True, build_packages=None, quiet=False, cmake_args=None, make_args=None, catkin_make_args=None): ''' Runs ``cmake``, ``make`` and optionally ``make install`` for all catkin packages in sourcespace_dir. It creates several folders in the current working directory. For non-catkin packages it runs ``cmake``, ``make`` and ``make install`` for each, installing it to the devel space or install space if the ``install`` option is specified. :param workspace: path to the current workspace, ``str`` :param sourcespace: workspace folder containing catkin packages, ``str`` :param buildspace: path to build space location, ``str`` :param develspace: path to devel space location, ``str`` :param installspace: path to install space (CMAKE_INSTALL_PREFIX), ``str`` :param merge: if True, build each catkin package into the same devel space. does not work with non-catkin packages, ``bool`` :param install: if True, install all packages to the install space, ``bool`` :param force_cmake: (optional), if True calls cmake explicitly for each package, ``bool`` :param colorize: if True, colorize cmake output and other messages, ``bool`` :param build_packages: specific packages to build (all parent packages in the topological order must have been built before), ``str`` :param quiet: if True, hides some build output, ``bool`` :param cmake_args: additional arguments for cmake, ``[str]`` :param make_args: additional arguments for make, ``[str]`` :param catkin_make_args: additional arguments for make but only for catkin packages, ``[str]`` ''' if not colorize: disable_ANSI_colors() # Check workspace existance if not os.path.exists(workspace): sys.exit("Workspace path '{0}' does not exist.".format(workspace)) workspace = os.path.abspath(workspace) # Check source space existance if sourcespace is None: ws_sourcespace = os.path.join(workspace, 'src') if not os.path.exists(ws_sourcespace): sys.exit("Could not find source space: {0}".format(sourcespace)) sourcespace = ws_sourcespace sourcespace = os.path.abspath(sourcespace) print('Base path: ' + str(workspace)) print('Source space: ' + str(sourcespace)) # Check build space if buildspace is None: buildspace = os.path.join(workspace, 'build_isolated') buildspace = os.path.abspath(buildspace) if not os.path.exists(buildspace): os.mkdir(buildspace) print('Build space: ' + str(buildspace)) # Check devel space if develspace is None: develspace = os.path.join(workspace, 'devel_isolated') develspace = os.path.abspath(develspace) print('Devel space: ' + str(develspace)) # Check install space if installspace is None: installspace = os.path.join(workspace, 'install_isolated') installspace = os.path.abspath(installspace) print('Install space: ' + str(installspace)) if cmake_args: print("Additional CMake Arguments: " + " ".join(cmake_args)) else: cmake_args = [] if make_args: print("Additional make Arguments: " + " ".join(make_args)) else: make_args = [] if catkin_make_args: print("Additional make Arguments for catkin packages: " + " ".join(catkin_make_args)) else: catkin_make_args = [] # Find packages packages = find_packages(sourcespace, exclude_subspaces=True) if not packages: print(fmt("@{yf}No packages found in source space: %s@|" % sourcespace)) # verify that specified package exists in workspace if build_packages: packages_by_name = {p.name: path for path, p in packages.iteritems()} unknown_packages = [ p for p in build_packages if p not in packages_by_name ] if unknown_packages: sys.exit('Packages not found in the workspace: %s' % ', '.join(unknown_packages)) # Report topological ordering ordered_packages = topological_order_packages(packages) unknown_build_types = [] msg = [] msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_packages))))) msg.append('@{pf}~~@| traversing %d packages in topological order:' % len(ordered_packages)) for path, package in ordered_packages: export_tags = [e.tagname for e in package.exports] if 'build_type' in export_tags: build_type_tag = [ e.content for e in package.exports if e.tagname == 'build_type' ][0] else: build_type_tag = 'catkin' if build_type_tag == 'catkin': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|') elif build_type_tag == 'cmake': msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@!@{cf}plain cmake@|)') else: msg.append('@{pf}~~@| - @!@{bf}' + package.name + '@|' + ' (@{rf}unknown@|)') unknown_build_types.append(package) msg.append('@{pf}~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' + ('~' * len(str(len(ordered_packages))))) for index in range(len(msg)): msg[index] = fmt(msg[index]) print('\n'.join(msg)) # Error if there are packages with unknown build_types if unknown_build_types: print(colorize_line('Error: Packages with unknown build types exist')) sys.exit('Can not build workspace with packages of unknown build_type') # Check to see if the workspace has changed if not force_cmake: force_cmake, install_toggled = cmake_input_changed( packages, buildspace, install=install, cmake_args=cmake_args, filename='catkin_make_isolated') if force_cmake: print( 'The packages or cmake arguments have changed, forcing cmake invocation' ) elif install_toggled: print( 'The install argument has been toggled, forcing cmake invocation on plain cmake package' ) # Build packages pkg_develspace = None last_env = None for index, path_package in enumerate(ordered_packages): path, package = path_package if merge: pkg_develspace = develspace else: pkg_develspace = os.path.join(develspace, package.name) if not build_packages or package.name in build_packages: try: export_tags = [e.tagname for e in package.exports] is_cmake_package = 'cmake' in [ e.content for e in package.exports if e.tagname == 'build_type' ] last_env = build_package( path, package, workspace, buildspace, pkg_develspace, installspace, install, force_cmake or (install_toggled and is_cmake_package), quiet, last_env, cmake_args, make_args, catkin_make_args, number=index + 1, of=len(ordered_packages)) except Exception as e: import traceback traceback.print_exc() cprint('@{rf}@!<==@| ' + 'Failed to process package \'@!@{bf}' + package.name + '@|\': \n ' + ('KeyboardInterrupt' if isinstance( e, KeyboardInterrupt) else str(e))) if isinstance(e, subprocess.CalledProcessError): cmd = ' '.join(e.cmd) if isinstance(e.cmd, list) else e.cmd print(fmt("\n@{rf}Reproduce this error by running:")) print(fmt("@{gf}@!==> @|") + cmd + "\n") sys.exit('Command failed, exiting.') else: cprint("Skipping package: '@!@{bf}" + package.name + "@|'") last_env = get_new_env(package, pkg_develspace, installspace, install, last_env) # Provide a top level devel space environment setup script if not os.path.exists(develspace): os.makedirs(develspace) if not build_packages: generated_env_sh = os.path.join(develspace, 'env.sh') generated_setup_sh = os.path.join(develspace, 'setup.sh') generated_setup_util_py = os.path.join(develspace, '_setup_util.py') if not merge and pkg_develspace: # generate env.sh and setup.sh which relay to last devel space with open(generated_env_sh, 'w') as f: f.write("""\ #!/usr/bin/env sh # generated from catkin.builder module {0} "$@" """.format(os.path.join(pkg_develspace, 'env.sh'))) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) with open(generated_setup_sh, 'w') as f: f.write("""\ #!/usr/bin/env sh # generated from catkin.builder module . "{0}/setup.sh" """.format(pkg_develspace)) elif not pkg_develspace: # generate env.sh and setup.sh for an empty devel space if 'CMAKE_PREFIX_PATH' in os.environ.keys(): variables = { 'CATKIN_GLOBAL_BIN_DESTINATION': 'bin', 'CATKIN_GLOBAL_LIB_DESTINATION': 'lib', 'CMAKE_PREFIX_PATH_AS_IS': ';'.join(os.environ['CMAKE_PREFIX_PATH'].split( os.pathsep)), 'PYTHON_INSTALL_DIR': get_python_install_dir(), 'SETUP_DIR': '', } with open(generated_setup_util_py, 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', '_setup_util.py.in'), variables)) os.chmod(generated_setup_util_py, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) else: sys.exit( "Unable to process CMAKE_PREFIX_PATH from environment. Cannot generate environment files." ) variables = {'SETUP_DIR': develspace, 'SETUP_FILENAME': 'setup'} with open(generated_env_sh, 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', 'env.sh.in'), variables)) os.chmod(generated_env_sh, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR) variables = {'SETUP_DIR': develspace} with open(generated_setup_sh, 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', 'setup.sh.in'), variables)) if not merge and pkg_develspace: # remove _setup_util.py file which might have been generated for an empty if os.path.exists(generated_setup_util_py): os.remove(generated_setup_util_py) if not merge or not pkg_develspace: # generate setup.bash and setup.zsh for convenience variables = {'SETUP_DIR': develspace} with open(os.path.join(develspace, 'setup.bash'), 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', 'setup.bash.in'), variables)) with open(os.path.join(develspace, 'setup.zsh'), 'w') as f: f.write( configure_file( os.path.join(get_cmake_path(), 'templates', 'setup.zsh.in'), variables))