def _check_platform_helper() -> Tuple[str, dict, dict]:
"""
Check ROS_DISTRO environment variables and distribution installed.
:return: string of distro name, dict of distribution info, dict of release platforms info
"""
distro_name = os.environ.get('ROS_DISTRO')
if not distro_name:
doctor_error('ROS_DISTRO is not set.')
return
distro_name = distro_name.lower()
u = rosdistro.get_index_url()
if not u:
doctor_error(
'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL. '
'Check network setting to make sure machine is connected to internet.'
)
return
i = rosdistro.get_index(u)
distro_info = i.distributions.get(distro_name)
if not distro_info:
doctor_warn(f'Distribution name {distro_name} is not found')
return
try:
distro_data = rosdistro.get_distribution(i, distro_name).get_data()
except AttributeError:
distro_data = ''
return distro_name, distro_info, distro_data
def execute(self, rosdistro_path, distro):
self.rosdistro_path = rosdistro_path
internal_index_path = (rosdistro_path / 'rosdistro' /
'index.yaml').resolve().as_uri()
self.internal_index = get_index(internal_index_path)
self.internal_distro = get_distribution(self.internal_index, distro)
self.internal_distro_file = self.internal_index.distributions[distro][
'distribution'][-1]
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Create a workspace from vcs repos files.')
add_argument_rosdistro_name(parser)
add_argument_repos_file_urls(parser)
add_argument_repository_names(parser, optional=True)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
help='The path of the desired workspace',
required=True)
args = parser.parse_args(argv)
assert args.repos_file_urls or args.repository_names
ensure_workspace_exists(args.workspace_root)
repos_files = []
if args.repository_names:
with Scope('SUBSECTION', 'get repository information from rosdistro'):
index = get_index(get_index_url())
dist = get_distribution(index, args.rosdistro_name)
data = {}
for repo_name in args.repository_names:
repo = dist.repositories[repo_name]
src_repo = repo.source_repository
repo_data = {
'type': src_repo.type,
'url': src_repo.url,
}
if src_repo.version is not None:
repo_data['version'] = src_repo.version
data[repo_name] = repo_data
repos_file = os.path.join(args.workspace_root,
'repositories-from-rosdistro.repos')
with open(repos_file, 'w') as h:
h.write(
yaml.safe_dump({'repositories': data},
default_flow_style=False))
repos_files.append(repos_file)
with Scope('SUBSECTION', 'fetch repos files(s)'):
for repos_file_url in args.repos_file_urls:
repos_file = os.path.join(args.workspace_root,
os.path.basename(repos_file_url))
print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file))
urlretrieve(repos_file_url, repos_file)
repos_files += [repos_file]
with Scope('SUBSECTION', 'import repositories'):
source_space = os.path.join(args.workspace_root, 'src')
for repos_file in repos_files:
print('Importing repositories from \'%s\'' % (repos_file))
import_repositories(source_space, repos_file, args.test_branch)
with Scope('SUBSECTION', 'vcs export --exact'):
# if a repo has been rebased against the default branch vcs can't detect the remote
export_repositories(args.workspace_root, check=not args.test_branch)
def init_environment():
global os_name, os_version, rdistro, ctx, os_installers, default_os_installer, dist_data, rindex, rcache, rview
ctx = create_default_installer_context()
os_installers = ctx.get_os_installer_keys(os_name)
default_os_installer = ctx.get_default_os_installer_key(os_name)
rindex = get_index(get_index_url())
dist_data = _get_dist_file_data(rindex, rdistro, 'distribution')
rcache = get_distribution(rindex, rdistro)
rview = get_catkin_view(rdistro, os_name, os_version, False)
def load_upstream(self, distro, upstream_index, upstream_distro):
recipes = yaml.safe_load(
(self.rosdistro_path / 'config' / 'recipes.yaml').open())
try:
info = recipes['common']['distributions'][distro]['upstream']
except KeyError:
info = None
index = get_index(
upstream_index if upstream_index is not None else info['url'])
self.upstream_distro = get_distribution(
index,
upstream_distro if upstream_distro is not None else info['name'])
def get_upstream_distro(self, upstream_index, upstream_distro):
recipes = self.read_yaml_file('config/recipes.yaml')
try:
info = recipes['common']['distributions'][
self.distro_name]['upstream']
except KeyError:
info = None
index = get_index(
upstream_index if upstream_index is not None else info['url'])
return get_distribution(
index,
upstream_distro if upstream_distro is not None else info['name'])
def get_rosdistro(distroname):
index_url = get_index_url()
index = get_index(index_url)
if index_url == DEFAULT_INDEX_URL:
logger.error(
'ROSDISTRO_INDEX_URL is set to the default (did you forget to source a workspace?)'
)
exit(1)
# load rosdistro with patched SourceRepositorySpecification class
with patch.object(repository, 'SourceRepositorySpecification',
SourceRepositorySpecificationMock):
return get_distribution(index, distroname)
def __init__(self, rosdistro_path, distro_name):
self.rosdistro_path = pathlib.Path(rosdistro_path)
self.distro_name = distro_name
internal_index_path = (self.rosdistro_path / 'rosdistro' /
'index.yaml').resolve()
if not internal_index_path.exists():
raise FileNotFoundError(f'No such file: {internal_index_path}')
self.internal_index = get_index(internal_index_path.as_uri())
self.internal_distro = get_distribution(self.internal_index,
distro_name)
self.internal_distro_path = self.internal_index.distributions[
distro_name]['distribution'][-1][len('file://'):]
def update_repo_settings(rosdistro_index: pathlib.Path, recipes: Mapping[str,
Any],
github_key: str, deploy: bool, release_track: str):
index = rosdistro.get_index(rosdistro_index.resolve().as_uri())
github_client = github.Github(
github_key) # TODO(pbovbel) support more than just github?
common_options = recipes['common']
for distro_name, _ in common_options['distributions'].items():
distro = rosdistro.get_distribution(index, distro_name)
for repo_name, repository_data in distro.repositories.items():
if not repository_data.source_repository or not repository_data.source_repository.test_commits:
continue
click.echo(f"Updating settings for repo {repo_name}", err=True)
url = repository_data.source_repository.url
gh_repo_name = urlsplit(url).path[len('/'):-len('.git')]
gh_repo = github_client.get_repo(gh_repo_name, lazy=False)
# Remove wikis and projects
if deploy:
gh_repo.edit(has_wiki=False, has_projects=False)
# Set PR merge to squash
if deploy:
gh_repo.edit(allow_merge_commit=False,
allow_rebase_merge=False,
allow_squash_merge=True)
# Protect branch
branch = gh_repo.get_branch(
repository_data.get_data()["source"]["version"])
if deploy:
branch.edit_protection(strict=True,
required_approving_review_count=1)
# Create label
if deploy:
try:
gh_repo.get_label(release_track)
except UnknownObjectException:
gh_repo.create_label(release_track, color="00ff00")
def get_rosdistro():
if 'ROS_DISTRO' in os.environ:
distroname = os.environ['ROS_DISTRO']
else:
raise AssertionError('ROS_DISTRO is not defined in environment')
index_url = get_index_url()
index = get_index(index_url)
if index_url == DEFAULT_INDEX_URL:
logger.error(
'ROSDISTRO_INDEX_URL is set to the default (did you forget to source a workspace?)'
)
exit(1)
# load rosdistro with patched SourceRepositorySpecification class
with patch.object(repository, 'SourceRepositorySpecification',
SourceRepositorySpecificationMock):
return get_distribution(index, distroname)
def get_distro_package_versions() -> dict:
"""
Return repos info using rosdistro API.
:return: dictionary of rosdistro package name and version
"""
distro_name = os.environ.get('ROS_DISTRO')
if not distro_name:
doctor_error('ROS_DISTRO is not set.')
return
distro_name = distro_name.lower()
url = rosdistro.get_index_url()
if not url:
doctor_error(
'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL. '
'Check network setting to make sure machine is connected to internet.'
)
return
i = rosdistro.get_index(url)
distro_info = rosdistro.get_distribution(i, distro_name)
if not distro_info:
doctor_warn(f'Distribution name {distro_name} is not found')
return
try:
repos_info = distro_info.get_data().get('repositories')
except AttributeError:
doctor_warn('No repository information found.')
return
distro_package_vers = {}
for package_name, info in repos_info.items():
try:
release = info['release']
ver = release.get('version')
if 'packages' in release:
# Metapackage
for package in release['packages']:
distro_package_vers[package] = ver
else:
distro_package_vers[package_name] = ver
except KeyError:
pass
return distro_package_vers
def check_platform_helper():
"""Check ROS_DISTRO related environment variables and distribution name."""
distro_name = os.environ.get('ROS_DISTRO')
if not distro_name:
sys.stderr.write('WARNING: ROS_DISTRO is not set.\n')
return
else:
distro_name = distro_name.lower()
u = rosdistro.get_index_url()
if not u:
sys.stderr.write('WARNING: Unable to access ROSDISTRO_INDEX_URL '
'or DEFAULT_INDEX_URL.\n')
return
i = rosdistro.get_index(u)
distro_info = i.distributions.get(distro_name)
if not distro_info:
sys.stderr.write("WARNING: Distribution name '%s' is not found\n" %
distro_name)
return
distro_data = rosdistro.get_distribution(i, distro_name).get_data()
return distro_name, distro_info, distro_data
def get_distro_package_versions() -> dict:
"""
Return repos info using rosdistro API.
:return: dictionary of rosdistro package name and version
"""
distro_name = os.environ.get('ROS_DISTRO')
distro_name = distro_name.lower()
url = rosdistro.get_index_url()
i = rosdistro.get_index(url)
distro_data = rosdistro.get_distribution(i, distro_name).get_data()
repos_info = distro_data.get('repositories')
distro_package_vers = {}
for _, info in repos_info.items():
try:
release = info['release']
packages = release['packages']
ver = release.get('version')
for p in packages:
distro_package_vers[p] = ver
except KeyError:
pass
return distro_package_vers
def _check_platform_helper() -> Tuple[str, dict, dict]:
"""
Check ROS_DISTRO environment variables and distribution installed.
:return: string of distro name, dict of distribution info, dict of release platforms info
"""
distro_name = os.environ.get('ROS_DISTRO')
if not distro_name:
doctor_warn('ROS_DISTRO is not set.')
return
else:
distro_name = distro_name.lower()
u = rosdistro.get_index_url()
if not u:
doctor_warn(
'Unable to access ROSDISTRO_INDEX_URL or DEFAULT_INDEX_URL.')
return
i = rosdistro.get_index(u)
distro_info = i.distributions.get(distro_name)
if not distro_info:
doctor_warn("Distribution name '%s' is not found" % distro_name)
return
distro_data = rosdistro.get_distribution(i, distro_name).get_data()
return distro_name, distro_info, distro_data
def pull_distro_repositories(src_dir: pathlib.Path, recipes: Mapping[str, Any], rosdistro_index: pathlib.Path,
github_key: str, clean: bool) -> int:
"""Pull all the packages in all ROS distributions to disk
:param src_dir: Directory where sources should be pulled.
:param recipes: Recipe configuration defining distributions.
:param rosdistro_index: Path to rosdistro index.
:param github_key: Github API key.
:param clean: Whether to delete distro folders before pulling.
:returns: Result code
"""
index = rosdistro.get_index(rosdistro_index.resolve().as_uri())
github_client = github.Github(github_key)
common_options = recipes['common']
results = {}
with ThreadPoolExecutor() as executor:
for distro_name, distro_options in common_options['distributions'].items():
distro = rosdistro.get_distribution(index, distro_name)
target_dir = src_dir / distro_name
if clean and target_dir.exists():
click.echo(f"Deleting {target_dir} ...", err=True)
rmtree(str(target_dir))
target_dir.mkdir(parents=True, exist_ok=not clean)
for repo_name, distro_data in distro.repositories.items():
# release.url overrides source.url. In most cases they should be equivalent, but sometimes we want to
# pull from a bloomed repository with patches
try:
url = distro_data.release_repository.url
except AttributeError:
url = distro_data.source_repository.url
# We're fitting to the rosdistro standard here, release.tags.release is a template that can take
# parameters, though in our case it's usually just '{{ version }}'.
if distro_data.release_repository and distro_data.release_repository.version is not None:
version_template = distro_data.release_repository.tags['release']
context = {
'package': repo_name,
'upstream': distro_options['upstream']['name'],
'version': distro_data.release_repository.version
}
version = Environment(loader=BaseLoader()).from_string(version_template).render(**context)
else:
version = distro_data.source_repository.version
# Repurpose the rosdistro 'release.packages' field as an optional whitelist to prevent building
# packages we don't want.
if distro_data.release_repository and distro_data.release_repository.package_names != [repo_name]:
package_whitelist = distro_data.release_repository.package_names
else:
package_whitelist = None
repo_dir = target_dir / repo_name
results[repo_name] = executor.submit(
pull_repository, repo_name, url, version, package_whitelist, repo_dir, github_client)
# TODO(pbovbel) Handle errors and retry? We're definitely hitting rate limits sometimes
exceptions = {name: result.exception() for name, result in results.items()
if result.exception() is not None}
if exceptions:
for repo_name, exception in exceptions.items():
click.echo(click.style(f"Unable to pull {repo_name}: {exception}", fg="red"), err=True)
return 1
return 0