def clean_ros_recipe_dirs(self, distro=None):
if distro:
info(
'Cleaning up meta-ros{}-{}/generated-recipes directory...'
.format(yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
else:
info('Cleaning up generated-recipes directories...')
self.repo.git.rm('-rf', 'generated-recipes')
def add_generated_files(self, distro):
info('Adding changes...')
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro))
def get_change_summary(self, distro):
sep = '-' * 5
return '\n'.join([
sep,
self.repo.git.status('--porcelain'),
sep,
self.repo.git.diff(
'HEAD', 'meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(yoctoRecipe._get_ros_version(distro),
distro)),
sep,
self.repo.git.diff(
'HEAD', 'meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro),
'meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro)),
]) + '\n'
def commit_changes(self, distro, commit_msg):
info('Adding changes...')
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'superflore-change-summary.txt'.format(
yoctoRecipe._get_ros_version(distro), distro))
if self.repo.git.status('--porcelain') == '':
info('Nothing changed; no commit done')
else:
if self.branch_name:
info('Committing to branch {0}...'.format(self.branch_name))
else:
info('Committing to current branch')
self.repo.git.commit(m=commit_msg)
def clean_ros_recipe_dirs(self, distro):
# superflore-change-summary.txt is no longer being generated since:
# https://github.com/ros-infrastructure/superflore/pull/273
# but remove it here to make sure it gets deleted when new distro
# release is being generated
files = 'meta-ros{0}-{1}/generated-recipes '\
'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\
'meta-ros{0}-{1}/files/{1}/generated/'\
'newer-platform-components.list '\
'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\
'meta-ros{0}-{1}/files/{1}/generated/'\
'superflore-change-summary.txt '.format(
yoctoRecipe._get_ros_version(distro), distro)
info('Cleaning up:\n{0}'.format(files))
self.repo.git.rm('-rf', '--ignore-unmatch', files.split())
def regenerate_pkg(overlay, pkg, rosdistro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(rosdistro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s' available packages"
" in selected distro: %s" %
(pkg, get_package_names(rosdistro)))
try:
version = get_pkg_version(rosdistro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
rosdistro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe which was removed by clean_ros_recipe_dirs
prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format(
yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe)
existing = overlay.repo.git.status('--porcelain', '--', prefix)
if existing:
# The git status --porcelain output will look like this:
# D meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb
# we want just the path with filename
if len(existing.split('\n')) > 1:
warn('More than 1 recipe was output by "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
if existing.split()[0] != 'D':
err('Unexpected output from "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing.split()[1]
else:
# If it isn't shown in git status, it could still exist as normal
# unchanged file when --only option is being used
import glob
existing = glob.glob('{0}/{1}'.format(repo_dir, prefix))
if existing:
if len(existing) > 1:
err('More than 1 recipe was output by "git status '
'--porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.
format(yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing[0]
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(rosdistro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
def main():
overlay = None
parser = get_parser('Generate OpenEmbedded recipes for ROS packages',
exclude_all=True,
require_rosdistro=True,
require_dryrun=True)
parser.add_argument('--tar-archive-dir',
help='location to store archived packages',
type=str)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = set(args.skip_keys) if args.skip_keys else set()
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
warn('"{0}" distro detected...'.format(args.ros_distro))
"""
No longer supporting generation for multiple targets, but left the code in
place to handle them in case it might be needed again in the future.
"""
selected_targets = [args.ros_distro]
preserve_existing = args.only
now = os.getenv('SUPERFLORE_GENERATION_DATETIME',
get_utcnow_timestamp_str())
repo_org = 'ros'
repo_name = 'meta-ros'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
branch=(('superflore/{}'.format(now))
if not args.no_branch else None),
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
)
if not args.only:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for all packages in ROS '
'distribution {}.\n'.format(selected_targets[0]))
else:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for package(s) {} in ROS '
'distribution {}.\n'.format(args.only, args.ros_distro))
# generate installers
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir
else:
srcrev_filename = None
with CacheManager(srcrev_filename) as srcrev_cache:
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
distro,
False, # preserve_existing
srcrev_cache,
skip_keys=skip_keys,
)
except KeyError:
err("No package to satisfy key '%s' available "
"packages in selected distro: %s" %
(pkg, get_package_names(distro)))
sys.exit(1)
# Commit changes and file pull request
title =\
'{{{0}}} Selected recipes generated from '\
'files/{0}/generated/cache.yaml '\
'as of {1}\n'.format(
args.ros_distro,
now)
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
delta = "Regenerated: '%s'\n" % args.only
overlay.add_generated_files(args.ros_distro)
commit_msg = '\n'.join([
get_pr_text(
title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
if args.dry_run:
save_pr(overlay, args.only, '', pr_comment, title=title)
sys.exit(0)
file_pr(overlay,
delta,
'',
pr_comment,
distro=args.ros_distro,
title=title)
ok('Successfully synchronized repositories!')
sys.exit(0)
overlay.clean_ros_recipe_dirs(args.ros_distro)
for adistro in selected_targets:
yoctoRecipe.reset()
distro = get_distro(adistro)
distro_installers, _, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_pkg,
preserve_existing,
srcrev_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
)
total_changes[adistro] = distro_changes
total_installers[adistro] = distro_installers
yoctoRecipe.generate_ros_distro_inc(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'meta-ros{0}-{1}/files/{1}/generated/cache.yaml'.
format(yoctoRecipe._get_ros_version(args.ros_distro),
args.ros_distro)), distro.release_platforms,
skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
overlay.add_generated_files(args.ros_distro)
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
summary = overlay.get_change_summary(args.ros_distro)
if len(summary) == 0:
info('Exiting...')
clean_up()
sys.exit(0)
else:
info('But there are some changes in other regenerated files:'
'%s' % summary)
# remove duplicates
delta = gen_delta_msg(total_changes, markup='')
# Commit changes and file pull request
title = '{{{0}}} Sync to files/{0}/generated/'\
'cache.yaml as of {1}\n'.format(
args.ros_distro,
now)
commit_msg = '\n'.join([
get_pr_text(title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
delta = gen_delta_msg(total_changes)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay,
delta,
'',
pr_comment,
title=title,
)
sys.exit(0)
file_pr(overlay, delta, '', comment=pr_comment, title=title)
clean_up()
ok('Successfully synchronized repositories!')
def regenerate_pkg(overlay, pkg, distro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
)
existing = glob.glob('{}_*.bb'.format(prefix))
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
existing = existing[0]
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(distro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'ros{1}-{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes['ros{0}-{1}'.format(
yoctoRecipe._get_ros_version(distro.name),
recipe)] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
