def clean_ros_recipe_dirs(self, distro=None): if distro: info( 'Cleaning up meta-ros{}-{}/generated-recipes directory...' .format(yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format( yoctoRecipe._get_ros_version(distro), distro)) else: info('Cleaning up generated-recipes directories...') self.repo.git.rm('-rf', 'generated-recipes')
def add_generated_files(self, distro): info('Adding changes...') self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/' 'generated/*.inc'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'rosdep-resolve.yaml'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'newer-platform-components.list'.format( yoctoRecipe._get_ros_version(distro), distro))
def get_change_summary(self, distro): sep = '-' * 5 return '\n'.join([ sep, self.repo.git.status('--porcelain'), sep, self.repo.git.diff( 'HEAD', 'meta-ros{0}-{1}/conf/ros-distro/include/{1}/' 'generated/*.inc'.format(yoctoRecipe._get_ros_version(distro), distro)), sep, self.repo.git.diff( 'HEAD', 'meta-ros{0}-{1}/files/{1}/generated/' 'newer-platform-components.list'.format( yoctoRecipe._get_ros_version(distro), distro), 'meta-ros{0}-{1}/files/{1}/generated/' 'rosdep-resolve.yaml'.format( yoctoRecipe._get_ros_version(distro), distro)), ]) + '\n'
def commit_changes(self, distro, commit_msg): info('Adding changes...') self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/' 'generated/*.inc'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'rosdep-resolve.yaml'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'newer-platform-components.list'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'superflore-change-summary.txt'.format( yoctoRecipe._get_ros_version(distro), distro)) if self.repo.git.status('--porcelain') == '': info('Nothing changed; no commit done') else: if self.branch_name: info('Committing to branch {0}...'.format(self.branch_name)) else: info('Committing to current branch') self.repo.git.commit(m=commit_msg)
def clean_ros_recipe_dirs(self, distro): # superflore-change-summary.txt is no longer being generated since: # https://github.com/ros-infrastructure/superflore/pull/273 # but remove it here to make sure it gets deleted when new distro # release is being generated files = 'meta-ros{0}-{1}/generated-recipes '\ 'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\ 'meta-ros{0}-{1}/files/{1}/generated/'\ 'newer-platform-components.list '\ 'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\ 'meta-ros{0}-{1}/files/{1}/generated/'\ 'superflore-change-summary.txt '.format( yoctoRecipe._get_ros_version(distro), distro) info('Cleaning up:\n{0}'.format(files)) self.repo.git.rm('-rf', '--ignore-unmatch', files.split())
def regenerate_pkg(overlay, pkg, rosdistro, preserve_existing, srcrev_cache, skip_keys): pkg_names = get_package_names(rosdistro)[0] if pkg not in pkg_names: yoctoRecipe.not_generated_recipes.add(pkg) raise RuntimeError("Unknown package '%s' available packages" " in selected distro: %s" % (pkg, get_package_names(rosdistro))) try: version = get_pkg_version(rosdistro, pkg, is_oe=True) except KeyError as ke: yoctoRecipe.not_generated_recipes.add(pkg) raise ke repo_dir = overlay.repo.repo_dir component_name = yoctoRecipe.convert_to_oe_name( rosdistro.release_packages[pkg].repository_name) recipe = yoctoRecipe.convert_to_oe_name(pkg) # check for an existing recipe which was removed by clean_ros_recipe_dirs prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format( yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe) existing = overlay.repo.git.status('--porcelain', '--', prefix) if existing: # The git status --porcelain output will look like this: # D meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb # we want just the path with filename if len(existing.split('\n')) > 1: warn('More than 1 recipe was output by "git status --porcelain ' 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format( yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe, existing)) if existing.split()[0] != 'D': err('Unexpected output from "git status --porcelain ' 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format( yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe, existing)) existing = existing.split()[1] else: # If it isn't shown in git status, it could still exist as normal # unchanged file when --only option is being used import glob existing = glob.glob('{0}/{1}'.format(repo_dir, prefix)) if existing: if len(existing) > 1: err('More than 1 recipe was output by "git status ' '--porcelain ' 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'. format(yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe, existing)) existing = existing[0] previous_version = None if preserve_existing and existing: ok("recipe for package '%s' up to date, skipping..." % pkg) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None elif existing: overlay.repo.remove_file(existing, True) idx_version = existing.rfind('_') + len('_') previous_version = existing[idx_version:].rstrip('.bb') try: current = oe_recipe(rosdistro, pkg, srcrev_cache, skip_keys) except InvalidPackage as e: err('Invalid package: ' + str(e)) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None except Exception as e: err('Failed generating recipe for {}! {}'.format(pkg, str(e))) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None try: recipe_text = current.recipe_text() except NoPkgXml as nopkg: err("Could not fetch pkg! {}".format(str(nopkg))) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None except KeyError as ke: err("Failed to parse data for package {}! {}".format(pkg, str(ke))) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format( repo_dir, yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, component_name)) success_msg = 'Successfully generated recipe for package' ok('{0} \'{1}\'.'.format(success_msg, pkg)) recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \ '{4}_{5}.bb'.format( repo_dir, yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, component_name, recipe, version ) try: with open('{0}'.format(recipe_file_name), "w") as recipe_file: ok('Writing recipe {0}'.format(recipe_file_name)) recipe_file.write(recipe_text) yoctoRecipe.generated_components.add(component_name) yoctoRecipe.generated_recipes[recipe] = (version, component_name) except Exception: err("Failed to write recipe to disk!") yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None return current, previous_version, recipe
def main(): overlay = None parser = get_parser('Generate OpenEmbedded recipes for ROS packages', exclude_all=True, require_rosdistro=True, require_dryrun=True) parser.add_argument('--tar-archive-dir', help='location to store archived packages', type=str) args = parser.parse_args(sys.argv[1:]) pr_comment = args.pr_comment skip_keys = set(args.skip_keys) if args.skip_keys else set() if args.pr_only: if args.dry_run: parser.error('Invalid args! cannot dry-run and file PR') if not args.output_repository_path: parser.error('Invalid args! no repository specified') try: prev_overlay = RepoInstance(args.output_repository_path, False) msg, title = load_pr() prev_overlay.pull_request(msg, title=title) clean_up() sys.exit(0) except Exception as e: err('Failed to file PR!') err('reason: {0}'.format(e)) sys.exit(1) warn('"{0}" distro detected...'.format(args.ros_distro)) """ No longer supporting generation for multiple targets, but left the code in place to handle them in case it might be needed again in the future. """ selected_targets = [args.ros_distro] preserve_existing = args.only now = os.getenv('SUPERFLORE_GENERATION_DATETIME', get_utcnow_timestamp_str()) repo_org = 'ros' repo_name = 'meta-ros' if args.upstream_repo: repo_org, repo_name = url_to_repo_org(args.upstream_repo) # open cached tar file if it exists with TempfileManager(args.output_repository_path) as _repo: if not args.output_repository_path: # give our group write permissions to the temp dir os.chmod(_repo, 17407) # clone if args.output_repository_path is None overlay = RosMeta( _repo, not args.output_repository_path, branch=(('superflore/{}'.format(now)) if not args.no_branch else None), org=repo_org, repo=repo_name, from_branch=args.upstream_branch, ) if not args.only: pr_comment = pr_comment or ( 'Recipes generated by **superflore** for all packages in ROS ' 'distribution {}.\n'.format(selected_targets[0])) else: pr_comment = pr_comment or ( 'Recipes generated by **superflore** for package(s) {} in ROS ' 'distribution {}.\n'.format(args.only, args.ros_distro)) # generate installers total_installers = dict() total_changes = dict() if args.tar_archive_dir: srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir else: srcrev_filename = None with CacheManager(srcrev_filename) as srcrev_cache: if args.only: distro = get_distro(args.ros_distro) for pkg in args.only: if pkg in skip_keys: warn("Package '%s' is in skip-keys list, skipping..." % pkg) continue info("Regenerating package '%s'..." % pkg) try: regenerate_pkg( overlay, pkg, distro, False, # preserve_existing srcrev_cache, skip_keys=skip_keys, ) except KeyError: err("No package to satisfy key '%s' available " "packages in selected distro: %s" % (pkg, get_package_names(distro))) sys.exit(1) # Commit changes and file pull request title =\ '{{{0}}} Selected recipes generated from '\ 'files/{0}/generated/cache.yaml '\ 'as of {1}\n'.format( args.ros_distro, now) regen_dict = dict() regen_dict[args.ros_distro] = args.only delta = "Regenerated: '%s'\n" % args.only overlay.add_generated_files(args.ros_distro) commit_msg = '\n'.join([ get_pr_text( title + '\n' + pr_comment.replace('**superflore**', 'superflore'), markup=''), delta ]) overlay.commit_changes(args.ros_distro, commit_msg) if args.dry_run: save_pr(overlay, args.only, '', pr_comment, title=title) sys.exit(0) file_pr(overlay, delta, '', pr_comment, distro=args.ros_distro, title=title) ok('Successfully synchronized repositories!') sys.exit(0) overlay.clean_ros_recipe_dirs(args.ros_distro) for adistro in selected_targets: yoctoRecipe.reset() distro = get_distro(adistro) distro_installers, _, distro_changes =\ generate_installers( distro, overlay, regenerate_pkg, preserve_existing, srcrev_cache, skip_keys, skip_keys=skip_keys, is_oe=True, ) total_changes[adistro] = distro_changes total_installers[adistro] = distro_installers yoctoRecipe.generate_ros_distro_inc( _repo, args.ros_distro, overlay.get_file_revision_logs( 'meta-ros{0}-{1}/files/{1}/generated/cache.yaml'. format(yoctoRecipe._get_ros_version(args.ros_distro), args.ros_distro)), distro.release_platforms, skip_keys) yoctoRecipe.generate_superflore_datetime_inc( _repo, args.ros_distro, now) yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro) yoctoRecipe.generate_newer_platform_components( _repo, args.ros_distro) overlay.add_generated_files(args.ros_distro) num_changes = 0 for distro_name in total_changes: num_changes += len(total_changes[distro_name]) if num_changes == 0: info('ROS distro is up to date.') summary = overlay.get_change_summary(args.ros_distro) if len(summary) == 0: info('Exiting...') clean_up() sys.exit(0) else: info('But there are some changes in other regenerated files:' '%s' % summary) # remove duplicates delta = gen_delta_msg(total_changes, markup='') # Commit changes and file pull request title = '{{{0}}} Sync to files/{0}/generated/'\ 'cache.yaml as of {1}\n'.format( args.ros_distro, now) commit_msg = '\n'.join([ get_pr_text(title + '\n' + pr_comment.replace('**superflore**', 'superflore'), markup=''), delta ]) overlay.commit_changes(args.ros_distro, commit_msg) delta = gen_delta_msg(total_changes) if args.dry_run: info('Running in dry mode, not filing PR') save_pr( overlay, delta, '', pr_comment, title=title, ) sys.exit(0) file_pr(overlay, delta, '', comment=pr_comment, title=title) clean_up() ok('Successfully synchronized repositories!')
def regenerate_pkg(overlay, pkg, distro, preserve_existing, srcrev_cache, skip_keys): pkg_names = get_package_names(distro)[0] if pkg not in pkg_names: yoctoRecipe.not_generated_recipes.add(pkg) raise RuntimeError("Unknown package '%s'" % pkg) try: version = get_pkg_version(distro, pkg, is_oe=True) except KeyError as ke: yoctoRecipe.not_generated_recipes.add(pkg) raise ke repo_dir = overlay.repo.repo_dir component_name = yoctoRecipe.convert_to_oe_name( distro.release_packages[pkg].repository_name) recipe = yoctoRecipe.convert_to_oe_name(pkg) # check for an existing recipe prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format( repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name, component_name, recipe, ) existing = glob.glob('{}_*.bb'.format(prefix)) previous_version = None if preserve_existing and existing: ok("recipe for package '%s' up to date, skipping..." % pkg) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None elif existing: existing = existing[0] overlay.repo.remove_file(existing, True) idx_version = existing.rfind('_') + len('_') previous_version = existing[idx_version:].rstrip('.bb') try: current = oe_recipe(distro, pkg, srcrev_cache, skip_keys) except InvalidPackage as e: err('Invalid package: ' + str(e)) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None except Exception as e: err('Failed generating recipe for {}! {}'.format(pkg, str(e))) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None try: recipe_text = current.recipe_text() except NoPkgXml as nopkg: err("Could not fetch pkg! {}".format(str(nopkg))) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None except KeyError as ke: err("Failed to parse data for package {}! {}".format(pkg, str(ke))) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format( repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name, component_name)) success_msg = 'Successfully generated recipe for package' ok('{0} \'{1}\'.'.format(success_msg, pkg)) recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \ 'ros{1}-{4}_{5}.bb'.format( repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name, component_name, recipe, version ) try: with open('{0}'.format(recipe_file_name), "w") as recipe_file: ok('Writing recipe {0}'.format(recipe_file_name)) recipe_file.write(recipe_text) yoctoRecipe.generated_components.add(component_name) yoctoRecipe.generated_recipes['ros{0}-{1}'.format( yoctoRecipe._get_ros_version(distro.name), recipe)] = (version, component_name) except Exception: err("Failed to write recipe to disk!") yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None return current, previous_version, recipe