def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_repos_file_urls(parser, required=True) add_argument_test_branch(parser) parser.add_argument('--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) with Scope('SUBSECTION', 'fetch repos files(s)'): repos_files = [] for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): # if a repo has been rebased against the default branch vcs can't detect the remote export_repositories(args.workspace_root, check=not args.test_branch)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_rosdistro_name(parser) add_argument_repos_file_urls(parser) add_argument_repository_names(parser, optional=True) add_argument_test_branch(parser) parser.add_argument('--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) assert args.repos_file_urls or args.repository_names ensure_workspace_exists(args.workspace_root) repos_files = [] if args.repository_names: with Scope('SUBSECTION', 'get repository information from rosdistro'): index = get_index(get_index_url()) dist = get_distribution(index, args.rosdistro_name) data = {} for repo_name in args.repository_names: repo = dist.repositories[repo_name] src_repo = repo.source_repository repo_data = { 'type': src_repo.type, 'url': src_repo.url, } if src_repo.version is not None: repo_data['version'] = src_repo.version data[repo_name] = repo_data repos_file = os.path.join(args.workspace_root, 'repositories-from-rosdistro.repos') with open(repos_file, 'w') as h: h.write( yaml.safe_dump({'repositories': data}, default_flow_style=False)) repos_files.append(repos_file) with Scope('SUBSECTION', 'fetch repos files(s)'): for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): # if a repo has been rebased against the default branch vcs can't detect the remote export_repositories(args.workspace_root, check=not args.test_branch)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_build_ignore(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser, required=True) add_argument_test_branch(parser) parser.add_argument( '--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) os.chdir(args.workspace_root) with Scope('SUBSECTION', 'fetch repos files(s)'): repos_files = [] for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): export_repositories(args.workspace_root) with Scope('SUBSECTION', 'mark package(s) to ignore'): if args.build_ignore: source_space = os.path.join(args.workspace_root, 'src') packages = locate_packages(source_space, args.build_ignore) for package_name, package_root in packages.items(): print("Ignoring package '%s'" % (package_name,)) Path(package_root, 'COLCON_IGNORE').touch() with Scope('SUBSECTION', 'select target package(s) in workspace'): packages = locate_packages(source_space) if args.package_selection_args: selected_packages = locate_packages( source_space, extra_args=args.package_selection_args) to_ignore = packages.keys() - selected_packages.keys() print('Ignoring %d packages to scope workspace' % len(to_ignore)) for package in to_ignore: package_root = packages.pop(package) Path(package_root, 'COLCON_IGNORE').touch() print('There are %d packages which meet selection criteria' % len(packages))
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Invoke the build tool on a workspace') parser.add_argument( '--rosdistro-name', required=True, help='The name of the ROS distro to identify the setup file to be ' 'sourced (if available)') add_argument_build_tool(parser, required=True) add_argument_build_tool_args(parser) parser.add_argument('--workspace-root', required=True, help='The root path of the workspace to compile') parser.add_argument('--parent-result-space', nargs='*', help='The paths of the parent result spaces') parser.add_argument( '--clean-before', action='store_true', help='The flag if the workspace should be cleaned before the ' 'invocation') parser.add_argument( '--clean-after', action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) if args.clean_before: clean_workspace(args.workspace_root) try: with Scope('SUBSECTION', 'build workspace in isolation and install'): parent_result_spaces = None if args.parent_result_space: parent_result_spaces = args.parent_result_space env = dict(os.environ) env.setdefault('MAKEFLAGS', '-j1') rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=['-DBUILD_TESTING=0', '-DCATKIN_SKIP_TESTING=1'], args=args.build_tool_args, install=True, parent_result_spaces=parent_result_spaces, env=env) finally: if args.clean_after: clean_workspace(args.workspace_root) return rc
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser, required=True) add_argument_test_branch(parser) parser.add_argument('--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) os.chdir(args.workspace_root) with Scope('SUBSECTION', 'fetch repos files(s)'): repos_files = [] for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): export_repositories(args.workspace_root) with Scope('SUBSECTION', 'mark packages with IGNORE files'): packages = locate_packages(source_space) if args.package_selection_args: print('Using package selection arguments:', args.package_selection_args) selected_packages = locate_packages( source_space, extra_args=args.package_selection_args) to_ignore = packages.keys() - selected_packages.keys() print('Ignoring %d packages' % len(to_ignore)) for package in sorted(to_ignore): print('-', package) package_root = packages.pop(package) Path(package_root, 'COLCON_IGNORE').touch() print('There are %d packages which meet selection criteria' % len(packages))
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Invoke the build tool on a workspace') parser.add_argument( '--rosdistro-name', required=True, help='The name of the ROS distro to identify the setup file to be ' 'sourced (if available)') add_argument_build_tool(parser, required=True) parser.add_argument( '--workspace-root', required=True, help='The root path of the workspace to compile') parser.add_argument( '--parent-result-space', nargs='*', help='The paths of the parent result spaces') parser.add_argument( '--clean-before', action='store_true', help='The flag if the workspace should be cleaned before the ' 'invocation') parser.add_argument( '--clean-after', action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) if args.clean_before: clean_workspace(args.workspace_root) try: with Scope('SUBSECTION', 'build workspace in isolation and install'): parent_result_spaces = None if args.parent_result_space: parent_result_spaces = args.parent_result_space env = dict(os.environ) env['MAKEFLAGS'] = '-j1' rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=['-DBUILD_TESTING=0', '-DCATKIN_SKIP_TESTING=1'], install=True, parent_result_spaces=parent_result_spaces, env=env) finally: if args.clean_after: clean_workspace(args.workspace_root) return rc
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Invoke the build tool on a workspace') parser.add_argument( '--rosdistro-name', required=True, help='The name of the ROS distro to identify the setup file to be ' 'sourced (if available)') add_argument_build_tool(parser, required=True) add_argument_build_tool_args(parser) add_argument_run_abichecker(parser) parser.add_argument('--workspace-root', required=True, help='The root path of the workspace to compile') parser.add_argument('--parent-result-space', nargs='*', help='The paths of the parent result spaces') parser.add_argument( '--clean-before', action='store_true', help='The flag if the workspace should be cleaned before the ' 'invocation') parser.add_argument( '--clean-after', action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') add_argument_ros_version(parser) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) if args.clean_before: clean_workspace(args.workspace_root) env = dict(os.environ) env.setdefault('MAKEFLAGS', '-j1') env.setdefault('ROS_DISTRO', args.rosdistro_name) try: with Scope('SUBSECTION', 'build workspace in isolation and install'): parent_result_spaces = None if args.parent_result_space: parent_result_spaces = args.parent_result_space rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=['-DBUILD_TESTING=0', '-DCATKIN_SKIP_TESTING=1'], args=args.build_tool_args, install=True, parent_result_spaces=parent_result_spaces, env=env) finally: if args.clean_after: clean_workspace(args.workspace_root) # only run abi-checker after successful builds and when requested if not rc and args.run_abichecker: with Scope('SUBSECTION', 'use abi checker'): abi_rc = call_abi_checker([args.workspace_root], args.ros_version, env) # Never fail a build because of abi errors but make them # unstable by printing MAKE_BUILD_UNSTABLE. Jenkins will # use a plugin to make it if abi_rc: print('MAKE_BUILD_UNSTABLE') return rc
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Invoke 'rosdoc_lite' on each package of a workspace") parser.add_argument( '--rosdistro-name', required=True, help='The name of the ROS distro to identify the setup file to be ' 'sourced (if available)') parser.add_argument('--os-code-name', required=True, help="The OS code name (e.g. 'xenial')") parser.add_argument('--arch', required=True, help="The architecture (e.g. 'amd64')") add_argument_build_tool(parser, required=True) parser.add_argument('--workspace-root', required=True, help='The root path of the workspace to compile') parser.add_argument('--rosdoc-lite-dir', required=True, help='The root path of the rosdoc_lite repository') parser.add_argument('--catkin-sphinx-dir', required=True, help='The root path of the catkin-sphinx repository') parser.add_argument('--rosdoc-index-dir', required=True, help='The root path of the rosdoc_index folder') parser.add_argument( '--canonical-base-url', help='The canonical base URL to add to all generated HTML files') parser.add_argument( 'pkg_tuples', nargs='*', help='A list of package tuples in topological order, each containing ' 'the name, the relative path and optionally the package-relative ' 'path of the rosdoc config file separated by a colon') add_argument_output_dir(parser, required=True) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) clean_workspace(args.workspace_root) with Scope('SUBSECTION', 'build workspace in isolation and install'): env = dict(os.environ) env.setdefault('MAKEFLAGS', '-j1') rc = call_build_tool(args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=['-DCATKIN_SKIP_TESTING=1'], install=True, env=env) # TODO compile error should still allow to generate doc from static parts if rc: return rc rosdoc_index = RosdocIndex([ os.path.join(args.output_dir, args.rosdistro_name), os.path.join(args.rosdoc_index_dir, args.rosdistro_name) ]) source_space = os.path.join(args.workspace_root, 'src') for pkg_tuple in args.pkg_tuples: pkg_name, pkg_subfolder, pkg_rosdoc_config = pkg_tuple.split(':', 2) with Scope('SUBSECTION', 'rosdoc_lite - %s' % pkg_name): pkg_path = os.path.join(source_space, pkg_subfolder) pkg_doc_path = os.path.join(args.output_dir, 'api_rosdoc', pkg_name) pkg_tag_path = os.path.join(args.output_dir, 'symbols', '%s.tag' % pkg_name) source_cmd = [ '.', os.path.join(args.workspace_root, 'install_isolated', 'setup.sh'), ] # for workspaces with only plain cmake packages the setup files # generated by cmi won't implicitly source the underlays setup_file = '/opt/ros/%s/setup.sh' % args.rosdistro_name if os.path.exists(setup_file): source_cmd = ['.', setup_file, '&&'] + source_cmd rosdoc_lite_cmd = [ os.path.join(args.rosdoc_lite_dir, 'scripts', 'rosdoc_lite'), pkg_path, '-o', pkg_doc_path, '-g', pkg_tag_path, '-t', os.path.join(args.output_dir, 'rosdoc_tags', '%s.yaml' % pkg_name), ] if '3' == os.environ.get('ROS_PYTHON_VERSION'): rosdoc_lite_cmd.insert(0, 'python3') print("Invoking `rosdoc_lite` for package '%s': %s" % (pkg_name, ' '.join(rosdoc_lite_cmd))) pkg_rc = subprocess.call([ 'sh', '-c', ' '.join(source_cmd) + ' && ' + 'PYTHONPATH=%s/src:%s/src:$PYTHONPATH ' % (args.rosdoc_lite_dir, args.catkin_sphinx_dir) + ' '.join(rosdoc_lite_cmd) ], stderr=subprocess.STDOUT, cwd=pkg_path) if pkg_rc: rc = pkg_rc # only if rosdoc runs generates a symbol file # create the corresponding location file if os.path.exists(pkg_tag_path): data = { 'docs_url': '../../../api/%s/html' % pkg_name, 'location': '%s/symbols/%s.tag' % (args.rosdistro_name, pkg_name), 'package': pkg_name, } # fetch generator specific output folders from rosdoc_lite if pkg_rosdoc_config: output_folders = get_generator_output_folders( pkg_rosdoc_config, pkg_name) for generator, output_folder in output_folders.items(): data['%s_output_folder' % generator] = output_folder rosdoc_index.locations[pkg_name] = [data] if args.canonical_base_url: add_canonical_link( pkg_doc_path, '%s/%s/api/%s' % (args.canonical_base_url, args.rosdistro_name, pkg_name)) # merge manifest.yaml files rosdoc_manifest_yaml_file = os.path.join(pkg_doc_path, 'manifest.yaml') job_manifest_yaml_file = os.path.join(args.output_dir, 'manifests', pkg_name, 'manifest.yaml') if os.path.exists(rosdoc_manifest_yaml_file): with open(rosdoc_manifest_yaml_file, 'r') as h: rosdoc_data = yaml.safe_load(h) else: # if rosdoc_lite failed to generate the file rosdoc_data = {} with open(job_manifest_yaml_file, 'r') as h: job_data = yaml.safe_load(h) rosdoc_data.update(job_data) with open(rosdoc_manifest_yaml_file, 'w') as h: yaml.safe_dump(rosdoc_data, h, default_flow_style=False) rosdoc_index.write_modified_data(args.output_dir, ['locations']) return rc
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Invoke the build tool on a workspace while enabling and ' 'running the tests') parser.add_argument( '--rosdistro-name', required=True, help='The name of the ROS distro to identify the setup file to be ' 'sourced (if available)') add_argument_build_tool(parser, required=True) a1 = add_argument_build_tool_args(parser) a2 = add_argument_build_tool_test_args(parser) parser.add_argument('--workspace-root', required=True, help='The root path of the workspace to compile') parser.add_argument('--parent-result-space', nargs='*', help='The paths of the parent result spaces') parser.add_argument( '--clean-before', action='store_true', help='The flag if the workspace should be cleaned before the ' 'invocation') parser.add_argument( '--clean-after', action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') add_argument_require_gpu_support(parser) remainder_args = extract_multiple_remainders(argv, (a1, a2)) args = parser.parse_args(argv) for k, v in remainder_args.items(): setattr(args, k, v) ensure_workspace_exists(args.workspace_root) if args.clean_before: clean_workspace(args.workspace_root) parent_result_spaces = None if args.parent_result_space: parent_result_spaces = args.parent_result_space try: with Scope('SUBSECTION', 'build workspace in isolation'): test_results_dir = os.path.join(args.workspace_root, 'test_results') cmake_args = [ '-DBUILD_TESTING=1', '-DCATKIN_ENABLE_TESTING=1', '-DCATKIN_SKIP_TESTING=0', '-DCATKIN_TEST_RESULTS_DIR=%s' % test_results_dir ] additional_args = args.build_tool_args or [] if args.build_tool == 'colcon': additional_args += ['--test-result-base', test_results_dir] env = dict(os.environ) env.setdefault('MAKEFLAGS', '-j1') rc = call_build_tool(args.build_tool, args.rosdistro_name, args.workspace_root, cmake_clean_cache=True, cmake_args=cmake_args, args=additional_args, parent_result_spaces=parent_result_spaces, env=env) if not rc: with Scope('SUBSECTION', 'build tests'): additional_args = args.build_tool_args or [] if args.build_tool == 'colcon': additional_args += ['--cmake-target-skip-unavailable'] rc = call_build_tool(args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=cmake_args, make_args=['tests'], args=additional_args, parent_result_spaces=parent_result_spaces, env=env) if not rc: make_args = ['run_tests'] additional_args = args.build_tool_args or [] if args.build_tool == 'colcon': cmake_args = None make_args = None additional_args = ['--test-result-base', test_results_dir] additional_args += args.build_tool_test_args or [] # for workspaces with only plain cmake packages the setup files # generated by cmi won't implicitly source the underlays if parent_result_spaces is None: parent_result_spaces = [ '/opt/ros/%s' % args.rosdistro_name ] if args.build_tool == 'catkin_make_isolated': devel_space = os.path.join(args.workspace_root, 'devel_isolated') parent_result_spaces.append(devel_space) # since catkin_make_isolated doesn't provide a custom # environment to run tests this needs to source the devel space # and force a CMake run ro use the new environment with Scope('SUBSECTION', 'run tests'): rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=cmake_args, force_cmake=args.build_tool == 'catkin_make_isolated', make_args=make_args, args=additional_args, parent_result_spaces=parent_result_spaces, env=env, colcon_verb='test') finally: if args.clean_after: clean_workspace(args.workspace_root) return rc
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Invoke the build tool on a workspace while enabling and ' 'running the tests') parser.add_argument( '--rosdistro-name', required=True, help='The name of the ROS distro to identify the setup file to be ' 'sourced (if available)') add_argument_build_tool(parser, required=True) parser.add_argument( '--workspace-root', required=True, help='The root path of the workspace to compile') parser.add_argument( '--parent-result-space', nargs='*', help='The paths of the parent result spaces') parser.add_argument( '--clean-before', action='store_true', help='The flag if the workspace should be cleaned before the ' 'invocation') parser.add_argument( '--clean-after', action='store_true', help='The flag if the workspace should be cleaned after the ' 'invocation') args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) if args.clean_before: clean_workspace(args.workspace_root) parent_result_spaces = None if args.parent_result_space: parent_result_spaces = args.parent_result_space try: with Scope('SUBSECTION', 'build workspace in isolation'): test_results_dir = os.path.join( args.workspace_root, 'test_results') cmake_args = [ '-DBUILD_TESTING=1', '-DCATKIN_ENABLE_TESTING=1', '-DCATKIN_SKIP_TESTING=0', '-DCATKIN_TEST_RESULTS_DIR=%s' % test_results_dir] additional_args = None if args.build_tool == 'colcon': additional_args = ['--test-result-base', test_results_dir] env = dict(os.environ) env['MAKEFLAGS'] = '-j1' rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_clean_cache=True, cmake_args=cmake_args, args=additional_args, parent_result_spaces=parent_result_spaces, env=env) if not rc: with Scope('SUBSECTION', 'build tests'): additional_args = None if args.build_tool == 'colcon': additional_args = ['--cmake-target-skip-unavailable'] rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=cmake_args, make_args=['tests'], args=additional_args, parent_result_spaces=parent_result_spaces, env=env) if not rc: make_args = ['run_tests'] additional_args = None if args.build_tool == 'colcon': cmake_args = None make_args = None additional_args = ['--test-result-base', test_results_dir] # for workspaces with only plain cmake packages the setup files # generated by cmi won't implicitly source the underlays if parent_result_spaces is None: parent_result_spaces = ['/opt/ros/%s' % args.rosdistro_name] if args.build_tool == 'catkin_make_isolated': devel_space = os.path.join( args.workspace_root, 'devel_isolated') parent_result_spaces.append(devel_space) # since catkin_make_isolated doesn't provide a custom # environment to run tests this needs to source the devel space # and force a CMake run ro use the new environment with Scope('SUBSECTION', 'run tests'): rc = call_build_tool( args.build_tool, args.rosdistro_name, args.workspace_root, cmake_args=cmake_args, force_cmake=args.build_tool == 'catkin_make_isolated', make_args=make_args, args=additional_args, parent_result_spaces=parent_result_spaces, env=env, colcon_verb='test') finally: if args.clean_after: clean_workspace(args.workspace_root) return rc