class RosMeta(object):
def __init__(self, repo_dir, do_clone, org='allenh1', repo='meta-ros'):
self.repo = RepoInstance(org, repo, repo_dir, do_clone)
self.branch_name = 'yocto-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def clean_ros_recipe_dirs(self, distro=None):
if distro:
info('Cleaning up recipes-ros-{0} directory...'.format(distro))
self.repo.git.rm('-rf', 'recipes-ros-{0}'.format(distro))
else:
info('Cleaning up recipes-ros-* directories...')
self.repo.git.rm('-rf', 'recipes-ros-*')
def commit_changes(self, distro):
info('Adding changes...')
if distro == 'all' or distro == 'update':
self.repo.git.add('recipes-ros-*')
else:
self.repo.git.add('recipes-ros-{0}'.format(distro))
commit_msg = {
'update': 'rosdistro sync, {0}',
'all': 'regenerate all distros, {0}',
'lunar': 'regenerate ros-lunar, {0}',
'indigo': 'regenerate ros-indigo, {0}',
'kinetic': 'regenerate ros-kinetic, {0}',
}[distro].format(time.ctime())
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m='{0}'.format(commit_msg))
def pull_request(self, message):
pr_title = 'rosdistro sync, {0}'.format(time.ctime())
self.repo.pull_request(message, pr_title)
def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'):
self.repo = RepoInstance(
org, repo, repo_dir=repo_dir, do_clone=do_clone
)
self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def __init__(
self, dir, do_clone, branch, org='ros', repo='meta-ros', from_branch=''
):
self.repo = RepoInstance(
org, repo, dir, do_clone, from_branch=from_branch)
self.branch_name = branch
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
class RosOverlay(object):
def __init__(self,
repo_dir,
do_clone,
org='ros',
repo='ros-overlay',
from_branch=''):
self.repo = RepoInstance(org,
repo,
repo_dir=repo_dir,
do_clone=do_clone,
from_branch=from_branch)
self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def commit_changes(self, distro):
info('Adding changes...')
self.repo.git.add(self.repo.repo_dir)
info('Committing to branch {0}...'.format(self.branch_name))
if distro == 'all':
commit_msg = 'regenerate all distros, {0}'
elif distro:
commit_msg = 'regenerate ros-{1}, {0}'
else:
commit_msg = 'rosdistro sync, {0}'
commit_msg = commit_msg.format(time.ctime(), distro)
self.repo.git.commit(m='{0}'.format(commit_msg))
def regenerate_manifests(self,
regen_dict,
image_owner='allenh1',
image_name='ros_gentoo_base'):
info("Pulling docker image '%s/%s:latest'..." %
(image_owner, image_name))
dock = Docker()
dock.pull(image_owner, image_name)
info('Running docker image...')
info('Generating manifests...')
dock.map_directory('/home/%s/.gnupg' % os.getenv('USER'),
'/root/.gnupg')
dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay')
for key in regen_dict.keys():
for pkg in regen_dict[key]:
pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(key, pkg)
dock.add_bash_command('cd {0}'.format(pkg_dir))
dock.add_bash_command('repoman manifest')
try:
dock.run(show_cmd=True)
except docker.errors.ContainerError:
print(dock.log)
raise
def pull_request(self, message, overlay=None, title=''):
if not title:
title = 'rosdistro sync, {0}'.format(time.ctime())
self.repo.pull_request(message, title)
def __init__(self,
repo_dir,
do_clone,
org='allenh1',
repo='meta-ros',
from_branch=''):
self.repo = RepoInstance(org,
repo,
repo_dir,
do_clone,
from_branch=from_branch)
self.branch_name = 'yocto-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
class RosMeta(object):
def __init__(
self, dir, do_clone, branch, org='ros', repo='meta-ros', from_branch=''
):
self.repo = RepoInstance(
org, repo, dir, do_clone, from_branch=from_branch)
self.branch_name = branch
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def clean_ros_recipe_dirs(self, distro=None):
if distro:
info(
'Cleaning up generated-recipes-{} directory...'.format(distro))
self.repo.git.rm('-rf', 'generated-recipes-{}'.format(distro))
else:
info('Cleaning up generated-recipes-* directories...')
self.repo.git.rm('-rf', 'generated-recipes-*')
def commit_changes(self, distro, commit_msg):
info('Adding changes...')
self.repo.git.add('generated-recipes-{0}'.format(distro))
self.repo.git.add('conf/ros-distro/include/{0}/*.inc'.format(distro))
self.repo.git.add('files/{0}/cache.*'.format(distro))
self.repo.git.add('files/{0}/rosdep-resolve.yaml'.format(distro))
self.repo.git.add(
'files/{0}/newer-platform-components.list'.format(distro))
self.repo.git.add(
'files/{0}/superflore-change-summary.txt'.format(distro))
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m=commit_msg)
def pull_request(self, message, distro=None, title=''):
self.repo.pull_request(message, title, branch=distro)
def get_file_revision_logs(self, *file_path):
return self.repo.git.log('--oneline', '--', *file_path)
def get_change_summary(self):
self.repo.git.add('generated-recipes-*')
sep = '-' * 5
return '\n'.join([
sep, self.repo.git.status('--porcelain'), sep,
self.repo.git.diff('conf'), sep, self.repo.git.diff(
'files/*/cache.diffme',
'files/*/newer-platform-components.list',
'files/*/rosdep-resolve.yaml'
),
])
def __init__(self,
repo_dir,
do_clone,
org='ros',
repo='ros-overlay',
from_branch='',
new_branch=True):
self.repo = RepoInstance(org,
repo,
repo_dir=repo_dir,
do_clone=do_clone,
from_branch=from_branch)
if new_branch:
self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
else:
self.branch_name = None
class RosOverlay(object):
def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'):
self.repo = RepoInstance(org,
repo,
repo_dir=repo_dir,
do_clone=do_clone)
self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def commit_changes(self, distro):
info('Adding changes...')
self.repo.git.add(self.repo.repo_dir)
info('Committing to branch {0}...'.format(self.branch_name))
commit_msg = {
'update': 'rosdistro sync, ',
'all': 'regenerate all distros, ',
'lunar': 'regenerate ros-lunar, ',
'indigo': 'regenerate ros-indigo, ',
'kinetic': 'regenerate ros-kinetic, ',
}[distro or 'update'] + time.ctime()
self.repo.git.commit(m='{0}'.format(commit_msg))
def regenerate_manifests(self, regen_dict):
info('Building docker image...')
docker_file = resource_filename('repoman_docker', 'Dockerfile')
dock = Docker(docker_file, 'gentoo_repoman')
dock.build()
info('Running docker image...')
info('Generating manifests...')
dock.map_directory('/home/%s/.gnupg' % os.getenv('USER'),
'/root/.gnupg')
dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay')
for key in regen_dict.keys():
for pkg in regen_dict[key]:
pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(key, pkg)
dock.add_bash_command('cd {0}'.format(pkg_dir))
dock.add_bash_command('repoman manifest')
dock.add_bash_command('cd /tmp/ros-overlay')
dock.run(show_cmd=True)
def pull_request(self, message):
pr_title = 'rosdistro sync, {0}'.format(time.ctime())
self.repo.pull_request(message, pr_title)
def main():
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into Gentoo Linux')
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = args.skip_keys or []
selected_targets = None
if not args.dry_run:
if 'SUPERFLORE_GITHUB_TOKEN' not in os.environ:
raise NoGitHubAuthToken()
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
elif args.all:
warn('"All" mode detected... This may take a while!')
preserve_existing = False
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.only:
parser.error('Invalid args! --only requires specifying --ros-distro')
if not selected_targets:
selected_targets = get_distros_by_status('active')
repo_org = 'ros'
repo_name = 'ros-overlay'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosOverlay(
_repo,
not args.output_repository_path,
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
new_branch=(not args.no_branch),
)
if not preserve_existing and not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of all'
' packages from ROS distro %s from ROS-Overlay commit %s.' %
(selected_targets, overlay.repo.get_last_hash()))
elif not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator ran update from ROS-Overlay ' +
'commit %s.' % (overlay.repo.get_last_hash()))
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of ' +
'package(s) %s from commit %s.' %
(args.only, overlay.repo.get_last_hash()))
missing_depends = set()
to_commit = set()
will_file_pr = False
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
ebuild, deps, version = regenerate_pkg(
overlay, pkg, get_distro(args.ros_distro),
preserve_existing)
if not ebuild:
for dep in deps:
missing_depends.add(dep)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
continue
if ebuild:
to_commit.add(pkg)
will_file_pr = True
# if no packages succeeded, exit with error
if not will_file_pr:
err("No packages generated successfully, exiting.")
sys.exit(1)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = to_commit
overlay.regenerate_manifests(regen_dict)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
save_pr(overlay,
args.only,
missing_deps=gen_missing_deps_msg(missing_depends),
comment=pr_comment)
sys.exit(0)
delta = "Regenerated: '%s'\n" % args.only
file_pr(overlay, delta, gen_missing_deps_msg(missing_depends),
pr_comment)
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
get_distro(distro),
overlay=overlay,
gen_pkg_func=regenerate_pkg,
preserve_existing=preserve_existing,
skip_keys=skip_keys,
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes)
missing_deps = gen_missing_deps_msg(total_broken)
# Commit changes and file pull request
overlay.regenerate_manifests(total_installers)
overlay.commit_changes('all' if args.all else args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(overlay,
delta,
missing_deps=missing_deps,
comment=pr_comment)
sys.exit(0)
file_pr(overlay, delta, missing_deps, comment=pr_comment)
clean_up()
ok('Successfully synchronized repositories!')
def main():
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into Gentoo Linux')
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
selected_targets = None
if args.all:
warn('"All" mode detected... This may take a while!')
preserve_existing = False
elif args.ros_distro:
selected_targets = [args.ros_distro]
set_index_for_distro(args.ros_distro)
preserve_existing = False
elif args.dry_run and args.pr_only:
parser.error('Invalid args! cannot dry-run and file PR')
elif args.pr_only and not args.output_repository_path:
parser.error('Invalid args! no repository specified')
elif args.pr_only:
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
if not selected_targets:
selected_targets = active_distros + ros2_distros
repo_org = 'ros'
repo_name = 'ros-overlay'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosOverlay(
_repo,
not args.output_repository_path,
org=repo_org,
repo=repo_name
)
if not preserve_existing and not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of all'
' packages from ROS distro %s from ROS-Overlay commit %s.' % (
selected_targets,
overlay.repo.get_last_hash()
)
)
elif not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator ran update from ROS-Overlay ' +
'commit %s.' % (overlay.repo.get_last_hash())
)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of ' +
'package(s) %s from commit %s.' % (
args.only,
overlay.repo.get_last_hash()
)
)
for pkg in args.only:
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
get_distro(args.ros_distro),
preserve_existing
)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
sys.exit(1)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
overlay.regenerate_manifests(regen_dict)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
save_pr(
overlay, args.only, missing_deps=None, comment=pr_comment
)
sys.exit(0)
delta = "Regenerated: '%s'\n" % args.only
file_pr(overlay, delta, '', pr_comment)
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
set_index_for_distro(distro)
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro_name=distro,
overlay=overlay,
gen_pkg_func=regenerate_pkg,
preserve_existing=preserve_existing
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes)
missing_deps = gen_missing_deps_msg(total_broken)
# Commit changes and file pull request
overlay.regenerate_manifests(total_installers)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay, delta, missing_deps=missing_deps, comment=pr_comment
)
sys.exit(0)
file_pr(overlay, delta, missing_deps, comment=pr_comment)
clean_up()
ok('Successfully synchronized repositories!')
class RosMeta(object):
def __init__(
self, dir, do_clone, branch, org='ros', repo='meta-ros',
from_branch='', branch_name=''
):
self.repo = RepoInstance(
org, repo, dir, do_clone, from_branch=from_branch)
self.branch_name = branch
if branch:
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def clean_ros_recipe_dirs(self, distro=None):
if distro:
info(
'Cleaning up meta-ros{}-{}/generated-recipes directory...'
.format(yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
else:
info('Cleaning up generated-recipes directories...')
self.repo.git.rm('-rf', 'generated-recipes')
def commit_changes(self, distro, commit_msg):
info('Adding changes...')
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'superflore-change-summary.txt'.format(
yoctoRecipe._get_ros_version(distro), distro))
if self.repo.git.status('--porcelain') == '':
info('Nothing changed; no commit done')
else:
if self.branch_name:
info('Committing to branch {0}...'.format(self.branch_name))
else:
info('Committing to current branch')
self.repo.git.commit(m=commit_msg)
def pull_request(self, message, distro=None, title=''):
self.repo.pull_request(message, title, branch=distro)
def get_file_revision_logs(self, *file_path):
return self.repo.git.log('--oneline', '--', *file_path)
def get_change_summary(self, distro):
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
sep = '-' * 5
return '\n'.join([
sep,
self.repo.git.status('--porcelain'),
sep,
self.repo.git.diff(
'meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro)),
sep,
self.repo.git.diff(
'meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro),
'meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro)
),
]) + '\n'
def main():
overlay = None
parser = get_parser('Generate OpenEmbedded recipes for ROS packages',
exclude_all=True,
require_rosdistro=True,
require_dryrun=True)
parser.add_argument('--tar-archive-dir',
help='location to store archived packages',
type=str)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = set(args.skip_keys) if args.skip_keys else set()
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
warn('"{0}" distro detected...'.format(args.ros_distro))
"""
No longer supporting generation for multiple targets, but left the code in
place to handle them in case it might be needed again in the future.
"""
selected_targets = [args.ros_distro]
preserve_existing = args.only
now = os.getenv('SUPERFLORE_GENERATION_DATETIME',
get_utcnow_timestamp_str())
repo_org = 'ros'
repo_name = 'meta-ros'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
branch=(('superflore/{}'.format(now))
if not args.no_branch else None),
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
)
if not args.only:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for all packages in ROS '
'distribution {}.\n'.format(selected_targets[0]))
else:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for package(s) {} in ROS '
'distribution {}.\n'.format(args.only, args.ros_distro))
# generate installers
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir
else:
srcrev_filename = None
with CacheManager(srcrev_filename) as srcrev_cache:
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
distro,
False, # preserve_existing
srcrev_cache,
skip_keys=skip_keys,
)
except KeyError:
err("No package to satisfy key '%s' available "
"packages in selected distro: %s" %
(pkg, get_package_names(distro)))
sys.exit(1)
# Commit changes and file pull request
title =\
'{{{0}}} Selected recipes generated from '\
'files/{0}/generated/cache.yaml '\
'as of {1}\n'.format(
args.ros_distro,
now)
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
delta = "Regenerated: '%s'\n" % args.only
overlay.add_generated_files(args.ros_distro)
commit_msg = '\n'.join([
get_pr_text(
title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
if args.dry_run:
save_pr(overlay, args.only, '', pr_comment, title=title)
sys.exit(0)
file_pr(overlay,
delta,
'',
pr_comment,
distro=args.ros_distro,
title=title)
ok('Successfully synchronized repositories!')
sys.exit(0)
overlay.clean_ros_recipe_dirs(args.ros_distro)
for adistro in selected_targets:
yoctoRecipe.reset()
distro = get_distro(adistro)
distro_installers, _, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_pkg,
preserve_existing,
srcrev_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
)
total_changes[adistro] = distro_changes
total_installers[adistro] = distro_installers
yoctoRecipe.generate_ros_distro_inc(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'meta-ros{0}-{1}/files/{1}/generated/cache.yaml'.
format(yoctoRecipe._get_ros_version(args.ros_distro),
args.ros_distro)), distro.release_platforms,
skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
overlay.add_generated_files(args.ros_distro)
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
summary = overlay.get_change_summary(args.ros_distro)
if len(summary) == 0:
info('Exiting...')
clean_up()
sys.exit(0)
else:
info('But there are some changes in other regenerated files:'
'%s' % summary)
# remove duplicates
delta = gen_delta_msg(total_changes, markup='')
# Commit changes and file pull request
title = '{{{0}}} Sync to files/{0}/generated/'\
'cache.yaml as of {1}\n'.format(
args.ros_distro,
now)
commit_msg = '\n'.join([
get_pr_text(title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
delta = gen_delta_msg(total_changes)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay,
delta,
'',
pr_comment,
title=title,
)
sys.exit(0)
file_pr(overlay, delta, '', comment=pr_comment, title=title)
clean_up()
ok('Successfully synchronized repositories!')
class RosMeta(object):
def __init__(
self, dir, do_clone, branch, org='ros', repo='meta-ros',
from_branch='', branch_name=''
):
self.repo = RepoInstance(
org, repo, dir, do_clone, from_branch=from_branch)
self.branch_name = branch
if branch:
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def clean_ros_recipe_dirs(self, distro):
# superflore-change-summary.txt is no longer being generated since:
# https://github.com/ros-infrastructure/superflore/pull/273
# but remove it here to make sure it gets deleted when new distro
# release is being generated
files = 'meta-ros{0}-{1}/generated-recipes '\
'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\
'meta-ros{0}-{1}/files/{1}/generated/'\
'newer-platform-components.list '\
'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\
'meta-ros{0}-{1}/files/{1}/generated/'\
'superflore-change-summary.txt '.format(
yoctoRecipe._get_ros_version(distro), distro)
info(
'Cleaning up:\n{0}'
.format(files))
self.repo.git.rm('-rf', '--ignore-unmatch', files)
def commit_changes(self, distro, commit_msg):
info('Commit changes...')
if self.repo.git.status('--porcelain') == '':
info('Nothing changed; no commit done')
else:
if self.branch_name:
info('Committing to branch {0}...'.format(self.branch_name))
else:
info('Committing to current branch')
self.repo.git.commit(m=commit_msg)
def pull_request(self, message, distro=None, title=''):
self.repo.pull_request(message, title, branch=distro)
def get_file_revision_logs(self, *file_path):
return self.repo.git.log('--oneline', '--', *file_path)
def add_generated_files(self, distro):
info('Adding changes...')
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro))
def get_change_summary(self, distro):
sep = '-' * 5
return '\n'.join([
sep,
self.repo.git.status('--porcelain'),
sep,
self.repo.git.diff(
'HEAD',
'meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro)),
sep,
self.repo.git.diff(
'HEAD',
'meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro),
'meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro)
),
]) + '\n'
def main():
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into Yocto Linux')
parser.add_argument(
'--tar-archive-dir',
help='location to store archived packages',
type=str
)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
selected_targets = None
if args.all:
warn('"All" mode detected... this may take a while!')
preserve_existing = False
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.dry_run and args.pr_only:
parser.error('Invalid args! cannot dry-run and file PR')
elif args.pr_only and not args.output_repository_path:
parser.error('Invalid args! no repository specified')
elif args.pr_only:
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
if not selected_targets:
selected_targets = ros1_distros + ros2_distros
repo_org = 'lgsvl'
repo_name = 'meta-ros2'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output-repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
org=repo_org,
repo=repo_name
)
if not args.only:
pr_comment = pr_comment or (
'Superflore yocto generator began regeneration of all '
'packages form ROS distribution(s) %s on Meta-ROS from '
'commit %s.' % (
selected_targets,
overlay.repo.get_last_hash()
)
)
else:
pr_comment = pr_comment or (
'Superflore yocto generator began regeneration of package(s)'
' %s from ROS distro %s from Meta-ROS from commit %s.' % (
args.only,
args.ros_distro,
overlay.repo.get_last_hash()
)
)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.tar_archive_dir:
sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir
md5_filename = '%s/md5_cache.pickle' % args.tar_archive_dir
else:
sha256_filename = None
md5_filename = None
with TempfileManager(args.tar_archive_dir) as tar_dir,\
CacheManager(sha256_filename) as sha256_cache,\
CacheManager(md5_filename) as md5_cache: # noqa
if args.only:
for pkg in args.only:
info("Regenerating package '%s'..." % pkg)
try:
regenerate_installer(
overlay,
pkg,
get_distro(args.ros_distro),
preserve_existing,
tar_dir,
md5_cache,
sha256_cache
)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
sys.exit(1)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
overlay.commit_changes(args.ros_distro)
if args.dry_run:
save_pr(overlay, args.only, None, pr_comment)
sys.exit(0)
delta = "Regenerated: '%s'\n" % args.only
file_pr(overlay, delta, '', pr_comment, distro=args.ros_distro)
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_installer,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes)
missing_deps = gen_missing_deps_msg(total_broken)
# Commit changes and file pull request
overlay.commit_changes(args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(overlay, delta, missing_deps, pr_comment)
clean_up()
sys.exit(0)
file_pr(overlay, delta, missing_deps, pr_comment)
clean_up()
ok('Successfully synchronized repositories!')
def main():
os.environ["ROS_OS_OVERRIDE"] = "openembedded"
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into OpenEmbedded Linux',
exclude_all=True)
parser.add_argument('--tar-archive-dir',
help='location to store archived packages',
type=str)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = set(args.skip_keys) if args.skip_keys else set()
selected_targets = None
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.only:
parser.error('Invalid args! --only requires specifying --ros-distro')
if not selected_targets:
selected_targets = get_distros_by_status('active')
now = get_utcnow_timestamp_str()
repo_org = 'ros'
repo_name = 'meta-ros'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
branch='superflore/{}'.format(now),
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
)
if not args.only:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for all packages in ROS '
'distribution {}.\n'.format(selected_targets[0]))
else:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for package(s) {} in ROS '
'distribution {}.\n'.format(args.only, args.ros_distro))
# generate installers
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir
md5_filename = '%s/md5_cache.pickle' % args.tar_archive_dir
else:
sha256_filename = None
md5_filename = None
with TempfileManager(args.tar_archive_dir) as tar_dir,\
CacheManager(sha256_filename) as sha256_cache,\
CacheManager(md5_filename) as md5_cache: # noqa
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
distro,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache,
skip_keys=skip_keys,
)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
sys.exit(1)
yoctoRecipe.generate_rosdistro_conf(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'files/{0}/cache.yaml'.format(args.ros_distro)),
distro.release_platforms, skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_distro_cache(_repo, args.ros_distro)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
yoctoRecipe.generate_superflore_change_summary(
_repo, args.ros_distro, overlay.get_change_summary())
# Commit changes and file pull request
title = '{{{0}}} Sync to {0}-cache.yaml as of {1}\n'.format(
args.ros_distro, now)
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
delta = "Regenerated: '%s'\n" % args.only
commit_msg = '\n'.join([
get_pr_text(
title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
if args.dry_run:
save_pr(overlay, args.only, '', pr_comment, title=title)
sys.exit(0)
file_pr(overlay,
delta,
'',
pr_comment,
distro=args.ros_distro,
title=title)
ok('Successfully synchronized repositories!')
sys.exit(0)
for adistro in selected_targets:
yoctoRecipe.reset()
distro = get_distro(adistro)
distro_installers, _, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_pkg,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
)
total_changes[adistro] = distro_changes
total_installers[adistro] = distro_installers
yoctoRecipe.generate_rosdistro_conf(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'files/{0}/cache.yaml'.format(args.ros_distro)),
distro.release_platforms, skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_distro_cache(_repo, args.ros_distro)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
yoctoRecipe.generate_superflore_change_summary(
_repo, args.ros_distro, overlay.get_change_summary())
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes, markup='')
# Commit changes and file pull request
title = '{{{0}}} Sync to {0}-cache.yaml as of {1}\n'.format(
args.ros_distro, now)
commit_msg = '\n'.join([
get_pr_text(title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
delta = gen_delta_msg(total_changes)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay,
delta,
'',
pr_comment,
title=title,
)
sys.exit(0)
file_pr(overlay, delta, '', comment=pr_comment, title=title)
clean_up()
ok('Successfully synchronized repositories!')
def main():
global overlay
global preserve_existing
parser = argparse.ArgumentParser('Deploy ROS packages into Gentoo Linux')
parser.add_argument('--ros-distro',
help='regenerate packages for the specified distro',
type=str)
parser.add_argument('--all',
help='regenerate all packages in all distros',
action="store_true")
parser.add_argument('--dry-run',
help='run without filing a PR to remote',
action="store_true")
parser.add_argument('--pr-only',
help='ONLY file a PR to remote',
action='store_true')
parser.add_argument('--output-repository-path',
help='location of the Git repo',
type=str)
parser.add_argument('--only',
nargs='+',
help='generate only the specified packages')
args = parser.parse_args(sys.argv[1:])
selected_targets = None
if args.all:
warn('"All" mode detected... This may take a while!')
preserve_existing = False
elif args.ros_distro:
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.dry_run and args.pr_only:
parser.error('Invalid args! cannot dry-run and file PR')
elif args.pr_only and not args.output_repository_path:
parser.error('Invalid args! no repository specified')
elif args.pr_only:
try:
with open('.pr-message.tmp', 'r') as msg_file:
msg = msg_file.read().rstrip('\n')
with open('.pr-title.tmp', 'r') as title_file:
title = title_file.read().rstrip('\n')
except OSError:
err('Failed to open PR title/message file!')
err('Please supply the %s and %s files' %
('.pr_message.tmp', '.pr_title.tmp'))
raise
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
info('PR message:\n"%s"\n' % msg)
info('PR title:\n"%s"\n' % title)
prev_overlay.pull_request(msg, title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
if not selected_targets:
selected_targets = active_distros
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosOverlay(_repo, not args.output_repository_path)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.only:
for pkg in args.only:
info("Regenerating package '%s'..." % pkg)
regenerate_pkg(overlay, pkg, get_distro(args.ros_distro),
preserve_existing)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
overlay.regenerate_manifests(regen_dict)
overlay.commit_changes(args.ros_distro)
delta = "Regenerated: '%s'\n" % args.only
if args.dry_run:
info('Running in dry mode, not filing PR')
title_file = open('.pr-title.tmp', 'w')
title_file.write('rosdistro sync, {0}\n'.format(time.ctime()))
pr_message_file = open('.pr-message.tmp', 'w')
pr_message_file.write('%s\n%s\n' % (delta, ''))
sys.exit(0)
file_pr(overlay, delta, '')
clean_up()
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro_name=distro,
overlay=overlay,
gen_pkg_func=regenerate_pkg,
preserve_existing=preserve_existing
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
inst_list = total_broken
delta = "Changes:\n"
delta += "========\n"
if 'indigo' in total_changes and len(total_changes['indigo']) > 0:
delta += "Indigo Changes:\n"
delta += "---------------\n"
for d in sorted(total_changes['indigo']):
delta += '* {0}\n'.format(d)
delta += "\n"
if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0:
delta += "Kinetic Changes:\n"
delta += "----------------\n"
for d in sorted(total_changes['kinetic']):
delta += '* {0}\n'.format(d)
delta += "\n"
if 'lunar' in total_changes and len(total_changes['lunar']) > 0:
delta += "Lunar Changes:\n"
delta += "--------------\n"
for d in sorted(total_changes['lunar']):
delta += '* {0}\n'.format(d)
delta += "\n"
missing_deps = ''
if len(inst_list) > 0:
missing_deps = "Missing Dependencies:\n"
missing_deps += "=====================\n"
for pkg in sorted(inst_list):
missing_deps += " * [ ] {0}\n".format(pkg)
# Commit changes and file pull request
overlay.regenerate_manifests(total_installers)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
title_file = open('.pr-title.tmp', 'w')
title_file.write('rosdistro sync, {0}\n'.format(time.ctime()))
pr_message_file = open('.pr-message.tmp', 'w')
pr_message_file.write('%s\n%s\n' % (delta, missing_deps))
sys.exit(0)
file_pr(overlay, delta, missing_deps)
clean_up()
ok('Successfully synchronized repositories!')
