def _check_platform_helper() -> Tuple[str, dict, dict]: """ Check ROS_DISTRO environment variables and distribution installed. :return: string of distro name, dict of distribution info, dict of release platforms info """ distro_name = os.environ.get('ROS_DISTRO') if not distro_name: doctor_error('ROS_DISTRO is not set.') return distro_name = distro_name.lower() u = rosdistro.get_index_url() if not u: doctor_error( 'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL. ' 'Check network setting to make sure machine is connected to internet.' ) return i = rosdistro.get_index(u) distro_info = i.distributions.get(distro_name) if not distro_info: doctor_warn(f'Distribution name {distro_name} is not found') return try: distro_data = rosdistro.get_distribution(i, distro_name).get_data() except AttributeError: distro_data = '' return distro_name, distro_info, distro_data
def execute(self, rosdistro_path, distro): self.rosdistro_path = rosdistro_path internal_index_path = (rosdistro_path / 'rosdistro' / 'index.yaml').resolve().as_uri() self.internal_index = get_index(internal_index_path) self.internal_distro = get_distribution(self.internal_index, distro) self.internal_distro_file = self.internal_index.distributions[distro][ 'distribution'][-1]
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Create a workspace from vcs repos files.') add_argument_rosdistro_name(parser) add_argument_repos_file_urls(parser) add_argument_repository_names(parser, optional=True) add_argument_test_branch(parser) parser.add_argument('--workspace-root', help='The path of the desired workspace', required=True) args = parser.parse_args(argv) assert args.repos_file_urls or args.repository_names ensure_workspace_exists(args.workspace_root) repos_files = [] if args.repository_names: with Scope('SUBSECTION', 'get repository information from rosdistro'): index = get_index(get_index_url()) dist = get_distribution(index, args.rosdistro_name) data = {} for repo_name in args.repository_names: repo = dist.repositories[repo_name] src_repo = repo.source_repository repo_data = { 'type': src_repo.type, 'url': src_repo.url, } if src_repo.version is not None: repo_data['version'] = src_repo.version data[repo_name] = repo_data repos_file = os.path.join(args.workspace_root, 'repositories-from-rosdistro.repos') with open(repos_file, 'w') as h: h.write( yaml.safe_dump({'repositories': data}, default_flow_style=False)) repos_files.append(repos_file) with Scope('SUBSECTION', 'fetch repos files(s)'): for repos_file_url in args.repos_file_urls: repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url)) print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file)) urlretrieve(repos_file_url, repos_file) repos_files += [repos_file] with Scope('SUBSECTION', 'import repositories'): source_space = os.path.join(args.workspace_root, 'src') for repos_file in repos_files: print('Importing repositories from \'%s\'' % (repos_file)) import_repositories(source_space, repos_file, args.test_branch) with Scope('SUBSECTION', 'vcs export --exact'): # if a repo has been rebased against the default branch vcs can't detect the remote export_repositories(args.workspace_root, check=not args.test_branch)
def init_environment(): global os_name, os_version, rdistro, ctx, os_installers, default_os_installer, dist_data, rindex, rcache, rview ctx = create_default_installer_context() os_installers = ctx.get_os_installer_keys(os_name) default_os_installer = ctx.get_default_os_installer_key(os_name) rindex = get_index(get_index_url()) dist_data = _get_dist_file_data(rindex, rdistro, 'distribution') rcache = get_distribution(rindex, rdistro) rview = get_catkin_view(rdistro, os_name, os_version, False)
def load_upstream(self, distro, upstream_index, upstream_distro): recipes = yaml.safe_load( (self.rosdistro_path / 'config' / 'recipes.yaml').open()) try: info = recipes['common']['distributions'][distro]['upstream'] except KeyError: info = None index = get_index( upstream_index if upstream_index is not None else info['url']) self.upstream_distro = get_distribution( index, upstream_distro if upstream_distro is not None else info['name'])
def get_upstream_distro(self, upstream_index, upstream_distro): recipes = self.read_yaml_file('config/recipes.yaml') try: info = recipes['common']['distributions'][ self.distro_name]['upstream'] except KeyError: info = None index = get_index( upstream_index if upstream_index is not None else info['url']) return get_distribution( index, upstream_distro if upstream_distro is not None else info['name'])
def get_rosdistro(distroname): index_url = get_index_url() index = get_index(index_url) if index_url == DEFAULT_INDEX_URL: logger.error( 'ROSDISTRO_INDEX_URL is set to the default (did you forget to source a workspace?)' ) exit(1) # load rosdistro with patched SourceRepositorySpecification class with patch.object(repository, 'SourceRepositorySpecification', SourceRepositorySpecificationMock): return get_distribution(index, distroname)
def __init__(self, rosdistro_path, distro_name): self.rosdistro_path = pathlib.Path(rosdistro_path) self.distro_name = distro_name internal_index_path = (self.rosdistro_path / 'rosdistro' / 'index.yaml').resolve() if not internal_index_path.exists(): raise FileNotFoundError(f'No such file: {internal_index_path}') self.internal_index = get_index(internal_index_path.as_uri()) self.internal_distro = get_distribution(self.internal_index, distro_name) self.internal_distro_path = self.internal_index.distributions[ distro_name]['distribution'][-1][len('file://'):]
def update_repo_settings(rosdistro_index: pathlib.Path, recipes: Mapping[str, Any], github_key: str, deploy: bool, release_track: str): index = rosdistro.get_index(rosdistro_index.resolve().as_uri()) github_client = github.Github( github_key) # TODO(pbovbel) support more than just github? common_options = recipes['common'] for distro_name, _ in common_options['distributions'].items(): distro = rosdistro.get_distribution(index, distro_name) for repo_name, repository_data in distro.repositories.items(): if not repository_data.source_repository or not repository_data.source_repository.test_commits: continue click.echo(f"Updating settings for repo {repo_name}", err=True) url = repository_data.source_repository.url gh_repo_name = urlsplit(url).path[len('/'):-len('.git')] gh_repo = github_client.get_repo(gh_repo_name, lazy=False) # Remove wikis and projects if deploy: gh_repo.edit(has_wiki=False, has_projects=False) # Set PR merge to squash if deploy: gh_repo.edit(allow_merge_commit=False, allow_rebase_merge=False, allow_squash_merge=True) # Protect branch branch = gh_repo.get_branch( repository_data.get_data()["source"]["version"]) if deploy: branch.edit_protection(strict=True, required_approving_review_count=1) # Create label if deploy: try: gh_repo.get_label(release_track) except UnknownObjectException: gh_repo.create_label(release_track, color="00ff00")
def get_rosdistro(): if 'ROS_DISTRO' in os.environ: distroname = os.environ['ROS_DISTRO'] else: raise AssertionError('ROS_DISTRO is not defined in environment') index_url = get_index_url() index = get_index(index_url) if index_url == DEFAULT_INDEX_URL: logger.error( 'ROSDISTRO_INDEX_URL is set to the default (did you forget to source a workspace?)' ) exit(1) # load rosdistro with patched SourceRepositorySpecification class with patch.object(repository, 'SourceRepositorySpecification', SourceRepositorySpecificationMock): return get_distribution(index, distroname)
def get_distro_package_versions() -> dict: """ Return repos info using rosdistro API. :return: dictionary of rosdistro package name and version """ distro_name = os.environ.get('ROS_DISTRO') if not distro_name: doctor_error('ROS_DISTRO is not set.') return distro_name = distro_name.lower() url = rosdistro.get_index_url() if not url: doctor_error( 'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL. ' 'Check network setting to make sure machine is connected to internet.' ) return i = rosdistro.get_index(url) distro_info = rosdistro.get_distribution(i, distro_name) if not distro_info: doctor_warn(f'Distribution name {distro_name} is not found') return try: repos_info = distro_info.get_data().get('repositories') except AttributeError: doctor_warn('No repository information found.') return distro_package_vers = {} for package_name, info in repos_info.items(): try: release = info['release'] ver = release.get('version') if 'packages' in release: # Metapackage for package in release['packages']: distro_package_vers[package] = ver else: distro_package_vers[package_name] = ver except KeyError: pass return distro_package_vers
def check_platform_helper(): """Check ROS_DISTRO related environment variables and distribution name.""" distro_name = os.environ.get('ROS_DISTRO') if not distro_name: sys.stderr.write('WARNING: ROS_DISTRO is not set.\n') return else: distro_name = distro_name.lower() u = rosdistro.get_index_url() if not u: sys.stderr.write('WARNING: Unable to access ROSDISTRO_INDEX_URL ' 'or DEFAULT_INDEX_URL.\n') return i = rosdistro.get_index(u) distro_info = i.distributions.get(distro_name) if not distro_info: sys.stderr.write("WARNING: Distribution name '%s' is not found\n" % distro_name) return distro_data = rosdistro.get_distribution(i, distro_name).get_data() return distro_name, distro_info, distro_data
def get_distro_package_versions() -> dict: """ Return repos info using rosdistro API. :return: dictionary of rosdistro package name and version """ distro_name = os.environ.get('ROS_DISTRO') distro_name = distro_name.lower() url = rosdistro.get_index_url() i = rosdistro.get_index(url) distro_data = rosdistro.get_distribution(i, distro_name).get_data() repos_info = distro_data.get('repositories') distro_package_vers = {} for _, info in repos_info.items(): try: release = info['release'] packages = release['packages'] ver = release.get('version') for p in packages: distro_package_vers[p] = ver except KeyError: pass return distro_package_vers
def _check_platform_helper() -> Tuple[str, dict, dict]: """ Check ROS_DISTRO environment variables and distribution installed. :return: string of distro name, dict of distribution info, dict of release platforms info """ distro_name = os.environ.get('ROS_DISTRO') if not distro_name: doctor_warn('ROS_DISTRO is not set.') return else: distro_name = distro_name.lower() u = rosdistro.get_index_url() if not u: doctor_warn( 'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL.') return i = rosdistro.get_index(u) distro_info = i.distributions.get(distro_name) if not distro_info: doctor_warn("Distribution name '%s' is not found" % distro_name) return distro_data = rosdistro.get_distribution(i, distro_name).get_data() return distro_name, distro_info, distro_data
def pull_distro_repositories(src_dir: pathlib.Path, recipes: Mapping[str, Any], rosdistro_index: pathlib.Path, github_key: str, clean: bool) -> int: """Pull all the packages in all ROS distributions to disk :param src_dir: Directory where sources should be pulled. :param recipes: Recipe configuration defining distributions. :param rosdistro_index: Path to rosdistro index. :param github_key: Github API key. :param clean: Whether to delete distro folders before pulling. :returns: Result code """ index = rosdistro.get_index(rosdistro_index.resolve().as_uri()) github_client = github.Github(github_key) common_options = recipes['common'] results = {} with ThreadPoolExecutor() as executor: for distro_name, distro_options in common_options['distributions'].items(): distro = rosdistro.get_distribution(index, distro_name) target_dir = src_dir / distro_name if clean and target_dir.exists(): click.echo(f"Deleting {target_dir} ...", err=True) rmtree(str(target_dir)) target_dir.mkdir(parents=True, exist_ok=not clean) for repo_name, distro_data in distro.repositories.items(): # release.url overrides source.url. In most cases they should be equivalent, but sometimes we want to # pull from a bloomed repository with patches try: url = distro_data.release_repository.url except AttributeError: url = distro_data.source_repository.url # We're fitting to the rosdistro standard here, release.tags.release is a template that can take # parameters, though in our case it's usually just '{{ version }}'. if distro_data.release_repository and distro_data.release_repository.version is not None: version_template = distro_data.release_repository.tags['release'] context = { 'package': repo_name, 'upstream': distro_options['upstream']['name'], 'version': distro_data.release_repository.version } version = Environment(loader=BaseLoader()).from_string(version_template).render(**context) else: version = distro_data.source_repository.version # Repurpose the rosdistro 'release.packages' field as an optional whitelist to prevent building # packages we don't want. if distro_data.release_repository and distro_data.release_repository.package_names != [repo_name]: package_whitelist = distro_data.release_repository.package_names else: package_whitelist = None repo_dir = target_dir / repo_name results[repo_name] = executor.submit( pull_repository, repo_name, url, version, package_whitelist, repo_dir, github_client) # TODO(pbovbel) Handle errors and retry? We're definitely hitting rate limits sometimes exceptions = {name: result.exception() for name, result in results.items() if result.exception() is not None} if exceptions: for repo_name, exception in exceptions.items(): click.echo(click.style(f"Unable to pull {repo_name}: {exception}", fg="red"), err=True) return 1 return 0