def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_build_ignore(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser, required=True) add_argument_test_branch(parser) parser.add_argument( '--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) os.chdir(args.workspace_root) with Scope('SUBSECTION', 'fetch repos files(s)'): repos_files = [] for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): export_repositories(args.workspace_root) with Scope('SUBSECTION', 'mark package(s) to ignore'): if args.build_ignore: source_space = os.path.join(args.workspace_root, 'src') packages = locate_packages(source_space, args.build_ignore) for package_name, package_root in packages.items(): print("Ignoring package '%s'" % (package_name,)) Path(package_root, 'COLCON_IGNORE').touch() with Scope('SUBSECTION', 'select target package(s) in workspace'): packages = locate_packages(source_space) if args.package_selection_args: selected_packages = locate_packages( source_space, extra_args=args.package_selection_args) to_ignore = packages.keys() - selected_packages.keys() print('Ignoring %d packages to scope workspace' % len(to_ignore)) for package in to_ignore: package_root = packages.pop(package) Path(package_root, 'COLCON_IGNORE').touch() print('There are %d packages which meet selection criteria' % len(packages))
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser, required=True) add_argument_test_branch(parser) parser.add_argument('--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) ensure_workspace_exists(args.workspace_root) os.chdir(args.workspace_root) with Scope('SUBSECTION', 'fetch repos files(s)'): repos_files = [] for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): export_repositories(args.workspace_root) with Scope('SUBSECTION', 'mark packages with IGNORE files'): packages = locate_packages(source_space) if args.package_selection_args: print('Using package selection arguments:', args.package_selection_args) selected_packages = locate_packages( source_space, extra_args=args.package_selection_args) to_ignore = packages.keys() - selected_packages.keys() print('Ignoring %d packages' % len(to_ignore)) for package in sorted(to_ignore): print('-', package) package_root = packages.pop(package) Path(package_root, 'COLCON_IGNORE').touch() print('There are %d packages which meet selection criteria' % len(packages))
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Run the 'CI' job") # Positional add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_ignore(parser) add_argument_build_tool(parser, required=True) add_argument_distribution_repository_key_files(parser) add_argument_distribution_repository_urls(parser) add_argument_dockerfile_dir(parser) add_argument_env_vars(parser) add_argument_install_packages(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser, required=True) add_argument_ros_version(parser) add_argument_skip_rosdep_keys(parser) add_argument_test_branch(parser) parser.add_argument( '--workspace-mount-point', nargs='*', help='Locations within the docker image where the workspace(s) ' 'will be mounted when the docker image is run.') args = parser.parse_args(argv) data = copy.deepcopy(args.__dict__) data.update({ 'distribution_repository_urls': args.distribution_repository_urls, 'distribution_repository_keys': get_distribution_repository_keys( args.distribution_repository_urls, args.distribution_repository_key_files), 'uid': get_user_id(), }) create_dockerfile( 'ci/ci_create_tasks.Dockerfile.em', data, args.dockerfile_dir)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser(description="Run the 'CI' job") # Positional add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_tool(parser, required=True) add_argument_distribution_repository_key_files(parser) add_argument_distribution_repository_urls(parser) add_argument_dockerfile_dir(parser) add_argument_env_vars(parser) add_argument_install_packages(parser) a1 = add_argument_package_selection_args(parser) a2 = add_argument_build_tool_args(parser) add_argument_repos_file_urls(parser) add_argument_repository_names(parser, optional=True) add_argument_ros_version(parser) add_argument_skip_rosdep_keys(parser) add_argument_test_branch(parser) parser.add_argument( '--workspace-mount-point', nargs='*', help='Locations within the docker image where the workspace(s) ' 'will be mounted when the docker image is run.') remainder_args = extract_multiple_remainders(argv, (a1, a2)) args = parser.parse_args(argv) for k, v in remainder_args.items(): setattr(args, k, v) assert args.repos_file_urls or args.repository_names data = copy.deepcopy(args.__dict__) data.update({ 'distribution_repository_urls': args.distribution_repository_urls, 'distribution_repository_keys': get_distribution_repository_keys( args.distribution_repository_urls, args.distribution_repository_key_files), 'uid': get_user_id(), }) create_dockerfile('ci/ci_create_tasks.Dockerfile.em', data, args.dockerfile_dir)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'Dockerfile' for the CI job") # Positional add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_distribution_repository_key_files(parser) add_argument_distribution_repository_urls(parser) add_argument_dockerfile_dir(parser) add_argument_env_vars(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser) add_argument_repository_names(parser, optional=True) add_argument_skip_rosdep_keys(parser) add_argument_test_branch(parser) parser.add_argument('--workspace-root', nargs='+', help='The root path of the workspace to compile') args = parser.parse_args(argv) assert args.repos_file_urls or args.repository_names debian_pkg_names = [ 'git', 'python3-apt', 'python3-colcon-metadata', 'python3-colcon-package-information', 'python3-colcon-package-selection', 'python3-colcon-recursive-crawl', 'python3-colcon-ros', 'python3-rosdep', 'python3-vcstool', ] # get versions for build dependencies apt_cache = Cache() debian_pkg_versions = get_binary_package_versions(apt_cache, debian_pkg_names) # generate Dockerfile data = { 'os_name': args.os_name, 'os_code_name': args.os_code_name, 'arch': args.arch, 'distribution_repository_urls': args.distribution_repository_urls, 'distribution_repository_keys': get_distribution_repository_keys( args.distribution_repository_urls, args.distribution_repository_key_files), 'rosdistro_name': args.rosdistro_name, 'custom_rosdep_urls': [], 'uid': get_user_id(), 'build_environment_variables': ['%s=%s' % key_value for key_value in args.env_vars.items()], 'dependencies': debian_pkg_names, 'dependency_versions': debian_pkg_versions, 'repos_file_urls': args.repos_file_urls, 'repository_names': args.repository_names, 'test_branch': args.test_branch, 'skip_rosdep_keys': args.skip_rosdep_keys, 'package_selection_args': args.package_selection_args, 'workspace_root': args.workspace_root, } create_dockerfile('ci/create_workspace.Dockerfile.em', data, args.dockerfile_dir)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Lists available binary packages and versions which are' 'needed to satisfy rosdep keys for ROS packages in the workspace') # Positional add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_output_dir(parser) add_argument_package_selection_args(parser) add_argument_skip_rosdep_keys(parser) parser.add_argument('--package-root', nargs='+', help='The path to the directory containing packages') args = parser.parse_args(argv) workspace_root = args.package_root[-1] os.chdir(workspace_root) with Scope('SUBSECTION', 'mark packages with IGNORE files'): all_packages = locate_packages(workspace_root) selected_packages = all_packages if args.package_selection_args: print('Using package selection arguments:', args.package_selection_args) selected_packages = locate_packages( workspace_root, extra_args=args.package_selection_args) to_ignore = all_packages.keys() - selected_packages.keys() print('Ignoring %d packages' % len(to_ignore)) for package in sorted(to_ignore): print('-', package) package_root = all_packages[package] Path(package_root, 'COLCON_IGNORE').touch() print('There are %d packages which meet selection criteria' % len(selected_packages)) with Scope('SUBSECTION', 'Enumerating packages needed to build'): # find all of the underlay packages underlay_pkgs = {} all_underlay_pkg_names = set() for package_root in args.package_root[0:-1]: print("Crawling for packages in '%s'" % package_root) underlay_pkgs.update(find_packages(package_root)) # Check for a colcon index for non-ROS package detection colcon_index = os.path.join(package_root, 'colcon-core', 'packages') try: all_underlay_pkg_names.update(os.listdir(colcon_index)) except FileNotFoundError: pass underlay_pkg_names = [pkg.name for pkg in underlay_pkgs.values()] print('Found the following ROS underlay packages:') for pkg_name in sorted(underlay_pkg_names): print(' -', pkg_name) # get direct build dependencies package_root = args.package_root[-1] print("Crawling for packages in '%s'" % package_root) pkgs = find_packages(package_root) pkg_names = [pkg.name for pkg in pkgs.values()] print('Found the following ROS packages:') for pkg_name in sorted(pkg_names): print(' -', pkg_name) # get build dependencies and map them to binary packages all_pkgs = set(pkgs.values()).union(underlay_pkgs.values()) for pkg in all_pkgs: pkg.evaluate_conditions(os.environ) for pkg in all_pkgs: for group_depend in pkg.group_depends: if group_depend.evaluated_condition: group_depend.extract_group_members(all_pkgs) dependency_keys_build = get_dependencies( all_pkgs, 'build', _get_build_and_recursive_run_dependencies, pkgs.values()) dependency_keys_test = get_dependencies( all_pkgs, 'run and test', _get_test_and_recursive_run_dependencies, pkgs.values()) if args.skip_rosdep_keys: dependency_keys_build.difference_update(args.skip_rosdep_keys) dependency_keys_test.difference_update(args.skip_rosdep_keys) # remove all non-ROS packages and packages which are present but # specifically ignored every_package_name = all_packages.keys() | all_underlay_pkg_names dependency_keys_build -= every_package_name dependency_keys_test -= every_package_name context = initialize_resolver(args.rosdistro_name, args.os_name, args.os_code_name) os_pkg_names_build = resolve_names(dependency_keys_build, **context) os_pkg_names_test = resolve_names(dependency_keys_test, **context) os_pkg_names_test -= os_pkg_names_build with Scope('SUBSECTION', 'Resolving packages versions using apt cache'): apt_cache = Cache() os_pkg_versions = get_binary_package_versions( apt_cache, os_pkg_names_build | os_pkg_names_test) with open(os.path.join(args.output_dir, 'install_list_build.txt'), 'w') as out_file: for package in sorted(os_pkg_names_build): out_file.write('%s=%s\n' % (package, os_pkg_versions[package])) with open(os.path.join(args.output_dir, 'install_list_test.txt'), 'w') as out_file: for package in sorted(os_pkg_names_test): out_file.write('%s=%s\n' % (package, os_pkg_versions[package]))
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser(description="Generate a 'CI' script") # Positional add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'ci') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_tool(parser) a1 = add_argument_package_selection_args(parser) a2 = add_argument_build_tool_args(parser) a3 = add_argument_build_tool_test_args(parser) add_argument_repos_file_urls(parser) add_argument_skip_cleanup(parser) add_argument_test_branch(parser) parser.add_argument( '--underlay-source-path', nargs='*', metavar='DIR_NAME', help='Path to one or more install spaces to use as an underlay') remainder_args = extract_multiple_remainders(argv, (a1, a2, a3)) args = parser.parse_args(argv) for k, v in remainder_args.items(): setattr(args, k, v) # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scms = [] self.scripts = [] self.parameters = {} if args.skip_cleanup: self.parameters['skip_cleanup'] = 'true' if args.repos_file_urls is not None: self.parameters['repos_file_urls'] = ' '.join( args.repos_file_urls) if args.test_branch is not None: self.parameters['test_branch'] = args.test_branch if args.package_selection_args is not None: self.parameters['package_selection_args'] = ' '.join( args.package_selection_args) if args.build_tool_args is not None: self.parameters['build_tool_args'] = ' '.join( args.build_tool_args) if args.build_tool_test_args is not None: self.parameters['build_tool_test_args'] = ' '.join( args.build_tool_test_args) def beforeInclude(self, *_, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/snippet/scm.xml.em'): self.scms.append( (kwargs['locals']['repo_spec'], kwargs['locals']['path'])) if template_path.endswith('/snippet/builder_shell.xml.em'): script = kwargs['locals']['script'] # reuse existing ros_buildfarm folder if it exists if 'Clone ros_buildfarm' in script: lines = script.splitlines() lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then') lines += [ 'else', 'echo "Using existing ros_buildfarm folder"', 'fi', ] script = '\n'.join(lines) if args.build_tool and ' --build-tool ' in script: script = script.replace( ' --build-tool catkin_make_isolated', ' --build-tool ' + args.build_tool) self.scripts.append(script) if template_path.endswith( '/snippet/property_parameters-definition.xml.em'): for parameter in reversed(kwargs['locals']['parameters']): name = parameter['name'] value_type = parameter['type'] if value_type in ['string', 'text']: default_value = parameter['default_value'] elif value_type == 'boolean': default_value = 'true' if parameter.get( 'default_value', False) else 'false' else: continue self.parameters.setdefault(name, default_value) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] config = get_config_index(args.config_url) build_files = get_ci_build_files(config, args.rosdistro_name) build_file = build_files[args.ci_build_name] underlay_source_paths = [ os.path.abspath(p) for p in args.underlay_source_path or [] ] configure_ci_job(args.config_url, args.rosdistro_name, args.ci_build_name, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, jenkins=False, views=False, underlay_source_paths=underlay_source_paths) templates.template_hooks = None ci_job_name = get_ci_job_name(args.rosdistro_name, args.os_name, args.os_code_name, args.arch, 'script') value = expand_template('ci/ci_script.sh.em', { 'ci_job_name': ci_job_name, 'scms': hook.scms, 'scripts': hook.scripts, 'build_tool': args.build_tool or build_file.build_tool, 'parameters': hook.parameters }, options={BANGPATH_OPT: False}) value = value.replace('python3 ', sys.executable + ' ') print(value)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'CI' script") # Positional add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'ci') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_ignore(parser) add_argument_build_tool(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser) add_argument_skip_cleanup(parser) add_argument_test_branch(parser) parser.add_argument( '--underlay-source-path', nargs='*', metavar='DIR_NAME', help='Path to one or more install spaces to use as an underlay') args = parser.parse_args(argv) # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scms = [] self.scripts = [] self.parameters = {} if args.skip_cleanup: self.parameters['skip_cleanup'] = 'true' if args.repos_file_urls is not None: self.parameters['repos_file_urls'] = ' '.join(args.repos_file_urls) if args.test_branch is not None: self.parameters['test_branch'] = args.test_branch if args.build_ignore is not None: self.parameters['build_ignore'] = ' '.join(args.build_ignore) if args.package_selection_args is not None: self.parameters['package_selection_args'] = ' '.join(args.package_selection_args) def beforeInclude(self, *_, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/snippet/scm.xml.em'): self.scms.append( (kwargs['locals']['repo_spec'], kwargs['locals']['path'])) if template_path.endswith('/snippet/builder_shell.xml.em'): script = kwargs['locals']['script'] # reuse existing ros_buildfarm folder if it exists if 'Clone ros_buildfarm' in script: lines = script.splitlines() lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then') lines += [ 'else', 'echo "Using existing ros_buildfarm folder"', 'fi', ] script = '\n'.join(lines) if args.build_tool and ' --build-tool ' in script: script = script.replace( ' --build-tool catkin_make_isolated', ' --build-tool ' + args.build_tool) self.scripts.append(script) if template_path.endswith('/snippet/property_parameters-definition.xml.em'): for parameter in reversed(kwargs['locals']['parameters']): name = parameter['name'] value_type = parameter['type'] if value_type in ['string', 'text']: default_value = parameter['default_value'] elif value_type == 'boolean': default_value = 'true' if parameter.get( 'default_value', False) else 'false' else: continue self.parameters.setdefault(name, default_value) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] config = get_config_index(args.config_url) build_files = get_ci_build_files(config, args.rosdistro_name) build_file = build_files[args.ci_build_name] underlay_source_paths = [os.path.abspath(p) for p in args.underlay_source_path or []] configure_ci_job( args.config_url, args.rosdistro_name, args.ci_build_name, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, jenkins=False, views=False, underlay_source_paths=underlay_source_paths) templates.template_hooks = None ci_job_name = get_ci_job_name( args.rosdistro_name, args.os_name, args.os_code_name, args.arch, 'script') value = expand_template( 'ci/ci_script.sh.em', { 'ci_job_name': ci_job_name, 'scms': hook.scms, 'scripts': hook.scripts, 'build_tool': args.build_tool or build_file.build_tool, 'parameters': hook.parameters}, options={BANGPATH_OPT: False}) value = value.replace('python3 ', sys.executable + ' ') print(value)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'Dockerfile' for the CI job") # Positional add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_ignore(parser) add_argument_distribution_repository_key_files(parser) add_argument_distribution_repository_urls(parser) add_argument_dockerfile_dir(parser) add_argument_env_vars(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser, required=True) add_argument_skip_rosdep_keys(parser) add_argument_test_branch(parser) parser.add_argument( '--workspace-root', nargs='+', help='The root path of the workspace to compile') args = parser.parse_args(argv) debian_pkg_names = [ 'git', 'python3-apt', 'python3-colcon-common-extensions', 'python3-rosdep', 'python3-vcstool', ] # get versions for build dependencies apt_cache = Cache() debian_pkg_versions = get_binary_package_versions( apt_cache, debian_pkg_names) # generate Dockerfile data = { 'os_name': args.os_name, 'os_code_name': args.os_code_name, 'arch': args.arch, 'distribution_repository_urls': args.distribution_repository_urls, 'distribution_repository_keys': get_distribution_repository_keys( args.distribution_repository_urls, args.distribution_repository_key_files), 'rosdistro_name': args.rosdistro_name, 'custom_rosdep_urls': [], 'uid': get_user_id(), 'build_environment_variables': args.env_vars, 'dependencies': debian_pkg_names, 'dependency_versions': debian_pkg_versions, 'repos_file_urls': args.repos_file_urls, 'test_branch': args.test_branch, 'skip_rosdep_keys': args.skip_rosdep_keys, 'build_ignore': args.build_ignore, 'package_selection_args': args.package_selection_args, 'workspace_root': args.workspace_root, } create_dockerfile( 'ci/create_workspace.Dockerfile.em', data, args.dockerfile_dir)