def generate_superflore_datetime_inc(basepath, dist):
datetime_dir = '{}/conf/ros-distro/include/{}/'.format(basepath, dist)
datetime_file_name = 'generated-superflore-datetime.inc'
datetime_path = '{}{}'.format(datetime_dir, datetime_file_name)
try:
make_dir(datetime_dir)
with open(datetime_path, 'w') as datetime_file:
datetime_file.write('# {}/{}\n'.format(
dist, datetime_file_name))
datetime_file.write('# Generated by superflore -- DO NOT EDIT')
datetime_file.write(
' (except ROS_DISTRO_METADATA_VERSION_REVISION)\n#\n')
datetime_file.write(
'# Copyright ' + strftime("%Y", gmtime())
+ ' Open Source Robotics Foundation\n\n')
datetime_file.write(
'# The start time, in UTC, of the last superflore run that'
+ ' resulted in a change to the generated files. The date'
+ ' portion is used as\n# the second version field of '
+ 'ROS_DISTRO_METADATA_VERSION prior to the first release'
+ ' of a ROS_DISTRO.\n')
now = datetime.utcnow().strftime('%Y%m%d%H%M%S')
datetime_file.write(
'ROS_SUPERFLORE_GENERATION_DATETIME = "{}"'.format(now))
ok('Wrote {0}'.format(datetime_path))
except OSError as e:
err('Failed to write SuperFlore datetime {} to disk! {}'.format(
datetime_path, e))
raise e
def generate_superflore_datetime_inc(basepath, dist, now):
datetime_dir = '{0}/meta-ros{1}-{2}/conf/ros-distro/include/{2}/' \
'generated/'.format(
basepath, yoctoRecipe._get_ros_version(dist), dist)
datetime_file_name = 'superflore-datetime.inc'
datetime_path = '{}{}'.format(datetime_dir, datetime_file_name)
try:
make_dir(datetime_dir)
with open(datetime_path, 'w') as datetime_file:
datetime_file.write('# {}/generated/{}\n'.format(
dist, datetime_file_name))
datetime_file.write('# Generated by superflore -- DO NOT EDIT')
datetime_file.write(
'\n#\n# Copyright Open Source Robotics Foundation\n\n')
datetime_file.write(
'\n# The time, in UTC, associated with the last superflore'
+
' run that resulted in a change to the generated files.' +
' The date portion is\n# used as the third version field' +
' of ROS_DISTRO_METADATA_VERSION prior to the first' +
' release of a ROS_DISTRO.\n')
datetime_file.write(
'ROS_SUPERFLORE_GENERATION_DATETIME = "{}"\n'.format(now))
ok('Wrote {0}'.format(datetime_path))
except OSError as e:
err('Failed to write SuperFlore datetime {} to disk! {}'.format(
datetime_path, e))
raise e
def pull_request(self, message, title, branch='master', remote='origin'):
info('Forking repository if a fork does not exist...')
self.github = Github(os.environ['SUPERFLORE_GITHUB_TOKEN'])
self.gh_user = self.github.get_user()
self.gh_upstream = self.github.get_repo(
'%s/%s' % (self.repo_owner, self.repo_name))
# TODO(allenh1): Don't fork if you're authorized for repo
forked_repo = self.gh_user.create_fork(self.gh_upstream)
info('Pushing changes to fork...')
self.git.remote('add', 'github', forked_repo.html_url)
retry_on_exception(
self.git.push,
'-u',
'github',
self.branch or branch,
retry_msg='Could not push',
error_msg='Error during push',
sleep_secs=0.0,
)
info('Filing pull-request...')
pr_head = '%s:%s' % (self.gh_user.login, self.branch)
pr = self.gh_upstream.create_pull(title=title,
body=message,
base=self.from_branch or branch,
head=pr_head)
ok('Successfully filed a pull request.')
ok(' %s' % pr.html_url)
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
ebuild_name =\
'/ros-{0}/{1}/{1}-{2}.ebuild'.format(distro.name, pkg, version)
ebuild_name = overlay.repo.repo_dir + ebuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing ebuild.
existing = glob.glob('{0}/ros-{1}/{2}/*.ebuild'.format(
overlay.repo.repo_dir, distro.name, pkg))
if preserve_existing and os.path.isfile(ebuild_name):
ok("ebuild for package '%s' up to date, skipping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg)
overlay.repo.remove_file(manifest_file)
try:
current = gentoo_installer(distro, pkg, has_patches)
current.ebuild.name = pkg
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
ebuild_text = current.ebuild_text()
metadata_text = current.metadata_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.ebuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.ebuild.get_unresolved()
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir("{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
try:
ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
overlay.repo.repo_dir, distro.name, pkg, version)
ebuild_file = open(ebuild_file, "w")
metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format(
overlay.repo.repo_dir, distro.name, pkg)
metadata_file = open(metadata_file, "w")
ebuild_file.write(ebuild_text)
metadata_file.write(metadata_text)
except Exception as e:
err("Failed to write ebuild/metadata to disk!")
raise e
return current, []
def generate_newer_platform_components(basepath, distro):
newer_sys_comps_dir = '{0}/meta-ros{1}-{2}/files/{2}/' \
'generated/'.format(
basepath,
yoctoRecipe._get_ros_version(distro), distro)
newer_sys_comps_path = '{0}newer-platform-components.list'.format(
newer_sys_comps_dir)
ros_version = yoctoRecipe._get_ros_version(distro)
str_distro = 'ros' if ros_version == 1 else 'ros{}'.format(ros_version)
args1_wget = ['wget', '-O', '-', 'http://packages.ros.org/'
+ str_distro
+ '/ubuntu/dists/bionic/main/source/Sources.gz']
args2_gunzip = ['gunzip', '-']
args3_grep = ['grep', '-E', '^(Package|Version|Build-Depends)']
args4_awk = ['awk', '$1 ~ /^Package:/ && $2 !~ /^ros-/ '
+ '{ printf "%s;", $2; getline; '
+ 'printf "%s;", $2; getline; '
+ 'gsub(/Build-Depends: /, ""); gsub(/, /, ","); print}']
args5_sort = ['sort', '-t', ';', '-k', '1,1']
try:
make_dir(newer_sys_comps_dir)
wget = Popen(args1_wget, stdout=PIPE, stderr=DEVNULL)
gunzip = Popen(args2_gunzip, stdin=wget.stdout,
stdout=PIPE, stderr=DEVNULL)
grep = Popen(args3_grep, stdin=gunzip.stdout,
stdout=PIPE, stderr=DEVNULL)
awk = Popen(args4_awk, stdin=grep.stdout,
stdout=PIPE, stderr=DEVNULL)
sort = Popen(args5_sort, env={'LC_ALL': 'C'},
stdin=awk.stdout, stdout=PIPE, stderr=DEVNULL)
cmds = [wget, gunzip, grep, awk]
# Allow previous process to receive a SIGPIPE
# if the next one in the pipeline exits.
for cmd in cmds:
cmd.stdout.close()
# Run the pipeline and collect the output
txt_output = sort.communicate()[0].decode()
# Consume the return value of the other processes
for cmd in cmds:
cmd.wait()
cmds.append(sort)
if any([cmd.returncode for cmd in cmds]):
errors = ['{}[{}]'.format(cmd.args[0], cmd.returncode)
for cmd in cmds]
raise RuntimeError('Error codes ' + ' '.join(errors))
with open(newer_sys_comps_path, 'w') as newer_sys_comps_file:
newer_sys_comps_file.write(
'# {}/newer-platform-components.list\n'.format(distro))
newer_sys_comps_file.write(txt_output)
ok('Wrote {0}'.format(newer_sys_comps_path))
except (OSError, RuntimeError) as e:
err('Failed to write {0} to disk! {1}'.format(
newer_sys_comps_path, e))
raise e
def regenerate_installer(overlay, pkg, distro, preserve_existing, tar_dir):
make_dir("{0}/recipes-ros-{1}".format(overlay.repo.repo_dir, distro.name))
version = get_pkg_version(distro, pkg)
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % pkg)
# check for an existing recipe
existing = glob.glob('{0}/recipes-ros-{1}/{2}/*.bb'.format(
overlay.repo.repo_dir, distro.name, pkg))
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skpping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
try:
current = oe_installer(distro, pkg, tar_dir)
current.recipe.name = pkg.replace('_', '-')
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
recipe_text = current.recipe_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.recipe.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.recipe.get_unresolved()
except NoPkgXml:
err("Could not fetch pkg!")
return None, []
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir("{0}/recipes-ros-{1}/{2}".format(overlay.repo.repo_dir,
distro.name,
pkg.replace('_', '-')))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_name = '{0}/recipes-ros-{1}/{2}/{2}_{3}.bb'.format(
overlay.repo.repo_dir, distro.name, pkg.replace('_', '-'), version)
try:
with open('{0}'.format(recipe_name), "w") as recipe_file:
recipe_file.write(recipe_text)
except Exception as e:
err("Failed to write recipe to disk!")
raise e
return current, []
def generate_superflore_change_summary(basepath, distro, change_summary):
change_summary_dir = '{0}/files/{1}/'.format(basepath, distro)
change_summary_path = '{0}superflore-change-summary.txt'.format(
change_summary_dir)
try:
make_dir(change_summary_dir)
with open(change_summary_path, 'w') as change_summary_file:
change_summary_file.write(
'{}/superflore-change-summary.txt\n'.format(distro))
change_summary_file.write(change_summary)
ok('Wrote {0}'.format(change_summary_path))
except OSError as e:
err('Failed to write change summary {} to disk! {}'.format(
change_summary_path, e))
raise e
def pull_request(self, message, title, branch='master', remote='origin'):
info('Forking repository if a fork does not exist...')
# TODO(allenh1): Don't fork if you're authorized for repo
forked_repo = self.gh_user.create_fork(self.gh_upstream)
info('Pushing changes to fork...')
self.git.remote('add', 'github', forked_repo.html_url)
self.git.push('-u', 'github', self.branch or 'master')
info('Filing pull-request...')
pr_head = '%s:%s' % (self.gh_user.login, self.branch)
pr = self.gh_upstream.create_pull(title=title,
body=message,
base='master',
head=pr_head)
ok('Successfully filed a pull request.')
ok(' %s' % pr.html_url)
def generate_distro_cache(basepath, distro, skip_keys=[]):
distro_cache_dir = '{0}/files/{1}/'.format(basepath, distro)
distro_cache_path = '{0}cache.yaml'.format(distro_cache_dir)
try:
index = get_index(get_index_url())
yaml_str = get_distribution_cache_string(index, distro)
make_dir(distro_cache_dir)
with open(distro_cache_path, 'w') as distro_cache_file:
distro_cache_file.write('# {}/cache.yaml\n'.format(distro))
distro_cache_file.write(yaml_str)
ok('Wrote {0}'.format(distro_cache_path))
except OSError as e:
err('Failed to write distro cache {} to disk! {}'.format(
distro_cache_path, e))
raise e
# Generate a diff'able cache file
distro_cache_diff_path = '{}cache.diffme'.format(distro_cache_dir)
try:
def replace_all_patterns(d, text):
for k, v in d.items():
text = re.sub(k, v, text, flags=re.M)
return text
replacement_table = {
r"{([^ }][^ }]*)}": r'[[\1]]',
r"{": r"{\n",
r"}": r"\n}",
r"\[\[": r"{",
r"\]\]": r"}",
r", ": r",\n",
r"^ ": r"-----\n",
r"<version>[^<]*</version>": r"",
r"><": r">\n<",
r"^ ": r"-----\n",
r"^(source_repo_package_xmls:)": r"-----\n\1",
}
with open(distro_cache_diff_path, 'w') as distro_cache_diff_file:
distro_cache_diff_file.write(
'# {}/cache.diffme\n'.format(distro))
yaml_str = replace_all_patterns(replacement_table, yaml_str)
distro_cache_diff_file.write(yaml_str)
ok('Wrote {0}'.format(distro_cache_diff_path))
except OSError as e:
err('Failed to write diffme distro cache {} to disk! {}'.format(
distro_cache_diff_path, e))
raise e
def run(self, rm=True, show_cmd=False):
cmd_string = "bash -c '"
for i, bash_cmd in enumerate(self.bash_cmds):
cmd_string += bash_cmd
if i != len(self.bash_cmds) - 1:
cmd_string += ' && '
cmd_string += "'"
msg = "Running container with command string '%s'..."
info(msg % cmd_string)
self.client.containers.run(
image=self.image,
remove=rm,
command=cmd_string,
volumes=self.directory_map,
)
ok("Docker container exited.")
def generate_rosdep_resolve(basepath, distro):
rosdep_resolve_dir = '{0}/files/{1}/'.format(basepath, distro)
rosdep_resolve_path = '{0}rosdep-resolve.yaml'.format(
rosdep_resolve_dir)
try:
make_dir(rosdep_resolve_dir)
with open(rosdep_resolve_path, 'w') as rosdep_resolve_file:
rosdep_resolve_file.write(
'# {}/rosdep-resolve.yaml\n'.format(distro))
cache_as_dict_of_list = {
k: list(v) for k, v in yoctoRecipe.rosdep_cache.items()}
rosdep_resolve_file.write(yaml.dump(
cache_as_dict_of_list, default_flow_style=False))
ok('Wrote {0}'.format(rosdep_resolve_path))
except OSError as e:
err('Failed to write rosdep resolve cache {} to disk! {}'.format(
rosdep_resolve_path, e))
raise e
def run(self, verbose=True, log_file=None):
# TODO(allenh1): add the ability to check out a non-master
# branch of the overlay (for CI).
info('testing gentoo package integrity')
for pkg in sorted(self.package_list.keys()):
self.container.add_bash_command('emaint sync -r ros-overlay')
self.container.add_bash_command('emerge %s' % pkg)
try:
self.container.run(rm=True,
show_cmd=True,
privileged=True,
log_file=log_file)
self.package_list[pkg] = 'building'
ok(" '%s': building" % pkg)
except ContainerError:
self.package_list[pkg] = 'failing'
err(" '%s': failing" % pkg)
if verbose:
print(self.container.log)
self.container.clear_commands()
return self.package_list
def run(self, rm=True, show_cmd=False, privileged=False, log_file=None):
if log_file:
# get the location to store the log
log_path = os.path.dirname(log_file)
# get the file name
log_name = log_file.replace(log_path, '').lstrip('/')
else:
log_path = None
log_name = 'log.txt'
with TempfileManager(log_path) as tmp:
if log_file:
# change access to the directory so docker can read it
os.chmod(tmp, 17407)
# map into the container
self.map_directory(tmp)
cmd_string = self.get_command(tmp, log_name)
if show_cmd:
msg = "Running container with command string '%s'..."
info(msg % cmd_string)
try:
self.client.containers.run(
image=self.image,
remove=rm,
command=cmd_string,
privileged=privileged,
volumes=self.directory_map,
)
ok("Docker container exited.")
if log_file:
info("Log file: '%s/%s'" % (tmp, log_name))
except docker.errors.ContainerError:
err("Docker container exited with errors.")
if log_file:
info("Log file: '%s/%s'" % (tmp, log_name))
# save log, then raise.
with open('%s/%s' % (tmp, log_name), 'r') as logfile:
self.log = logfile.read()
raise
with open('%s/%s' % (tmp, log_name), 'r') as logfile:
self.log = logfile.read()
def main():
global overlay
global preserve_existing
parser = argparse.ArgumentParser('Deploy ROS packages into Yocto Linux')
parser.add_argument(
'--ros-distro',
help='regenerate packages for the specified distro',
type=str
)
parser.add_argument(
'--all',
help='regenerate all packages in all distros',
action="store_true"
)
parser.add_argument(
'--output-repository-path',
help='location of the Git repo',
type=str
)
parser.add_argument(
'--tar-archive-dir',
help='location to store archived packages',
type=str
)
selected_targets = active_distros
args = parser.parse_args(sys.argv[1:])
if args.all:
warn('"All" mode detected... this may take a while!')
preserve_existing = False
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output-repository_path is None
overlay = RosMeta(_repo, not args.output_repository_path)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
with TempfileManager(args.tar_archive_dir) as tar_dir:
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_installer,
preserve_existing,
tar_dir
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
sys.exit(0)
# remove duplicates
inst_list = total_broken
delta = "Changes:\n"
delta += "========\n"
if 'indigo' in total_changes and len(total_changes['indigo']) > 0:
delta += "Indigo Changes:\n"
delta += "---------------\n"
for d in sorted(total_changes['indigo']):
delta += '* {0}\n'.format(d)
delta += "\n"
if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0:
delta += "Kinetic Changes:\n"
delta += "----------------\n"
for d in sorted(total_changes['kinetic']):
delta += '* {0}\n'.format(d)
delta += "\n"
if 'lunar' in total_changes and len(total_changes['lunar']) > 0:
delta += "Lunar Changes:\n"
delta += "--------------\n"
for d in sorted(total_changes['lunar']):
delta += '* {0}\n'.format(d)
delta += "\n"
missing_deps = ''
if len(inst_list) > 0:
missing_deps = "Missing Dependencies:\n"
missing_deps += "=====================\n"
for pkg in sorted(inst_list):
missing_deps += " * [ ] {0}\n".format(pkg)
# Commit changes and file pull request
overlay.commit_changes(args.ros_distro)
file_pr(overlay, delta, missing_deps)
ok('Successfully synchronized repositories!')
def generate_installers(
distro, # ros distro
overlay, # repo instance
gen_pkg_func, # function to call for generating
preserve_existing=True, # don't regenerate if installer exists
*args, # any additional args for gen_pkg_func
**kwargs # any additional keyword arguments
):
distro_name = distro.name
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
bad_installers = []
succeeded = 0
failed = 0
info("Generating installers for distro '%s'" % distro_name)
for i, pkg in enumerate(sorted(pkg_names[0])):
if 'skip_keys' in kwargs and pkg in kwargs['skip_keys']:
warn("Package '%s' is in skip-keys list, skipping..." % pkg)
continue
version = get_pkg_version(distro, pkg)
percent = '%.1f' % (100 * (float(i) / total))
try:
current, current_info = gen_pkg_func(overlay, pkg, distro,
preserve_existing, *args)
if not current and current_info:
# we are missing dependencies
failed_msg = "{0}%: Failed to generate".format(percent)
failed_msg += " installer for package '%s'!" % pkg
err(failed_msg)
borkd_pkgs[pkg] = current_info
failed = failed + 1
continue
elif not current and preserve_existing:
# don't replace the installer
succeeded = succeeded + 1
continue
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1
if current_info:
changes.append('*{0} {1} --> {2}*'.format(
pkg, current_info, version))
else:
changes.append('*{0} {1}*'.format(pkg, version))
installers.append(pkg)
except UnknownBuildType as ub:
err("{0}%: Unknown Build type '{1}' for package '{2}'".format(
percent, str(ub), pkg))
failed = failed + 1
except KeyError:
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(pkg)
failed = failed + 1
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
info("------ {0} ------\n".format(results))
if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))
return installers, borkd_pkgs, changes
def generate_installers(
distro_name, # ros distro name
overlay, # repo instance
gen_pkg_func, # function to call for generating
preserve_existing=True, # don't regenerate if installer exists
*args # any aditional args for gen_pkg_func
):
distro = get_distro(distro_name)
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
bad_installers = []
succeeded = 0
failed = 0
info("Generating installers for distro '%s'" % distro_name)
for i, pkg in enumerate(sorted(pkg_names[0])):
version = get_pkg_version(distro, pkg)
percent = '%.1f' % (100 * (float(i) / total))
try:
current, bad_deps = gen_pkg_func(overlay, pkg, distro,
preserve_existing, *args)
if not current and bad_deps:
# we are missing dependencies
failed_msg = "{0}%: Failed to generate".format(percent)
failed_msg += " installer for package '%s'!" % pkg
err(failed_msg)
borkd_pkgs[pkg] = bad_deps
failed = failed + 1
continue
elif not current and preserve_existing:
# don't replace the installer
succeeded = succeeded + 1
continue
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1
changes.append('*{0} --> {1}*'.format(pkg, version))
installers.append(pkg)
except UnknownLicense as ul:
err("{0}%: Unknown License '{1}'.".format(percent, str(ul)))
bad_installers.append(pkg)
failed = failed + 1
except KeyError:
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(pkg)
failed = failed + 1
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
info("------ {0} ------\n".format(results))
if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))
return installers, borkd_pkgs, changes
def main():
global overlay
global preserve_existing
parser = argparse.ArgumentParser('Deploy ROS packages into Gentoo Linux')
parser.add_argument('--ros-distro',
help='regenerate packages for the specified distro',
type=str)
parser.add_argument('--all',
help='regenerate all packages in all distros',
action="store_true")
parser.add_argument('--dry-run',
help='run without filing a PR to remote',
action="store_true")
parser.add_argument('--pr-only',
help='ONLY file a PR to remote',
action='store_true')
parser.add_argument('--output-repository-path',
help='location of the Git repo',
type=str)
parser.add_argument('--only',
nargs='+',
help='generate only the specified packages')
args = parser.parse_args(sys.argv[1:])
selected_targets = None
if args.all:
warn('"All" mode detected... This may take a while!')
preserve_existing = False
elif args.ros_distro:
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.dry_run and args.pr_only:
parser.error('Invalid args! cannot dry-run and file PR')
elif args.pr_only and not args.output_repository_path:
parser.error('Invalid args! no repository specified')
elif args.pr_only:
try:
with open('.pr-message.tmp', 'r') as msg_file:
msg = msg_file.read().rstrip('\n')
with open('.pr-title.tmp', 'r') as title_file:
title = title_file.read().rstrip('\n')
except OSError:
err('Failed to open PR title/message file!')
err('Please supply the %s and %s files' %
('.pr_message.tmp', '.pr_title.tmp'))
raise
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
info('PR message:\n"%s"\n' % msg)
info('PR title:\n"%s"\n' % title)
prev_overlay.pull_request(msg, title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
if not selected_targets:
selected_targets = active_distros
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosOverlay(_repo, not args.output_repository_path)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.only:
for pkg in args.only:
info("Regenerating package '%s'..." % pkg)
regenerate_pkg(overlay, pkg, get_distro(args.ros_distro),
preserve_existing)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
overlay.regenerate_manifests(regen_dict)
overlay.commit_changes(args.ros_distro)
delta = "Regenerated: '%s'\n" % args.only
if args.dry_run:
info('Running in dry mode, not filing PR')
title_file = open('.pr-title.tmp', 'w')
title_file.write('rosdistro sync, {0}\n'.format(time.ctime()))
pr_message_file = open('.pr-message.tmp', 'w')
pr_message_file.write('%s\n%s\n' % (delta, ''))
sys.exit(0)
file_pr(overlay, delta, '')
clean_up()
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro_name=distro,
overlay=overlay,
gen_pkg_func=regenerate_pkg,
preserve_existing=preserve_existing
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
inst_list = total_broken
delta = "Changes:\n"
delta += "========\n"
if 'indigo' in total_changes and len(total_changes['indigo']) > 0:
delta += "Indigo Changes:\n"
delta += "---------------\n"
for d in sorted(total_changes['indigo']):
delta += '* {0}\n'.format(d)
delta += "\n"
if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0:
delta += "Kinetic Changes:\n"
delta += "----------------\n"
for d in sorted(total_changes['kinetic']):
delta += '* {0}\n'.format(d)
delta += "\n"
if 'lunar' in total_changes and len(total_changes['lunar']) > 0:
delta += "Lunar Changes:\n"
delta += "--------------\n"
for d in sorted(total_changes['lunar']):
delta += '* {0}\n'.format(d)
delta += "\n"
missing_deps = ''
if len(inst_list) > 0:
missing_deps = "Missing Dependencies:\n"
missing_deps += "=====================\n"
for pkg in sorted(inst_list):
missing_deps += " * [ ] {0}\n".format(pkg)
# Commit changes and file pull request
overlay.regenerate_manifests(total_installers)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
title_file = open('.pr-title.tmp', 'w')
title_file.write('rosdistro sync, {0}\n'.format(time.ctime()))
pr_message_file = open('.pr-message.tmp', 'w')
pr_message_file.write('%s\n%s\n' % (delta, missing_deps))
sys.exit(0)
file_pr(overlay, delta, missing_deps)
clean_up()
ok('Successfully synchronized repositories!')
def main():
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into Gentoo Linux')
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
selected_targets = None
if args.all:
warn('"All" mode detected... This may take a while!')
preserve_existing = False
elif args.ros_distro:
selected_targets = [args.ros_distro]
set_index_for_distro(args.ros_distro)
preserve_existing = False
elif args.dry_run and args.pr_only:
parser.error('Invalid args! cannot dry-run and file PR')
elif args.pr_only and not args.output_repository_path:
parser.error('Invalid args! no repository specified')
elif args.pr_only:
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
if not selected_targets:
selected_targets = active_distros + ros2_distros
repo_org = 'ros'
repo_name = 'ros-overlay'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosOverlay(
_repo,
not args.output_repository_path,
org=repo_org,
repo=repo_name
)
if not preserve_existing and not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of all'
' packages from ROS distro %s from ROS-Overlay commit %s.' % (
selected_targets,
overlay.repo.get_last_hash()
)
)
elif not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator ran update from ROS-Overlay ' +
'commit %s.' % (overlay.repo.get_last_hash())
)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of ' +
'package(s) %s from commit %s.' % (
args.only,
overlay.repo.get_last_hash()
)
)
for pkg in args.only:
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
get_distro(args.ros_distro),
preserve_existing
)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
sys.exit(1)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
overlay.regenerate_manifests(regen_dict)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
save_pr(
overlay, args.only, missing_deps=None, comment=pr_comment
)
sys.exit(0)
delta = "Regenerated: '%s'\n" % args.only
file_pr(overlay, delta, '', pr_comment)
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
set_index_for_distro(distro)
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro_name=distro,
overlay=overlay,
gen_pkg_func=regenerate_pkg,
preserve_existing=preserve_existing
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes)
missing_deps = gen_missing_deps_msg(total_broken)
# Commit changes and file pull request
overlay.regenerate_manifests(total_installers)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay, delta, missing_deps=missing_deps, comment=pr_comment
)
sys.exit(0)
file_pr(overlay, delta, missing_deps, comment=pr_comment)
clean_up()
ok('Successfully synchronized repositories!')
def main():
os.environ["ROS_OS_OVERRIDE"] = "openembedded"
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into OpenEmbedded Linux',
exclude_all=True)
parser.add_argument('--tar-archive-dir',
help='location to store archived packages',
type=str)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = set(args.skip_keys) if args.skip_keys else set()
selected_targets = None
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.only:
parser.error('Invalid args! --only requires specifying --ros-distro')
if not selected_targets:
selected_targets = get_distros_by_status('active')
now = get_utcnow_timestamp_str()
repo_org = 'ros'
repo_name = 'meta-ros'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
branch='superflore/{}'.format(now),
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
)
if not args.only:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for all packages in ROS '
'distribution {}.\n'.format(selected_targets[0]))
else:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for package(s) {} in ROS '
'distribution {}.\n'.format(args.only, args.ros_distro))
# generate installers
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir
md5_filename = '%s/md5_cache.pickle' % args.tar_archive_dir
else:
sha256_filename = None
md5_filename = None
with TempfileManager(args.tar_archive_dir) as tar_dir,\
CacheManager(sha256_filename) as sha256_cache,\
CacheManager(md5_filename) as md5_cache: # noqa
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
distro,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache,
skip_keys=skip_keys,
)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
sys.exit(1)
yoctoRecipe.generate_rosdistro_conf(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'files/{0}/cache.yaml'.format(args.ros_distro)),
distro.release_platforms, skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_distro_cache(_repo, args.ros_distro)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
yoctoRecipe.generate_superflore_change_summary(
_repo, args.ros_distro, overlay.get_change_summary())
# Commit changes and file pull request
title = '{{{0}}} Sync to {0}-cache.yaml as of {1}\n'.format(
args.ros_distro, now)
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
delta = "Regenerated: '%s'\n" % args.only
commit_msg = '\n'.join([
get_pr_text(
title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
if args.dry_run:
save_pr(overlay, args.only, '', pr_comment, title=title)
sys.exit(0)
file_pr(overlay,
delta,
'',
pr_comment,
distro=args.ros_distro,
title=title)
ok('Successfully synchronized repositories!')
sys.exit(0)
for adistro in selected_targets:
yoctoRecipe.reset()
distro = get_distro(adistro)
distro_installers, _, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_pkg,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
)
total_changes[adistro] = distro_changes
total_installers[adistro] = distro_installers
yoctoRecipe.generate_rosdistro_conf(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'files/{0}/cache.yaml'.format(args.ros_distro)),
distro.release_platforms, skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_distro_cache(_repo, args.ros_distro)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
yoctoRecipe.generate_superflore_change_summary(
_repo, args.ros_distro, overlay.get_change_summary())
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes, markup='')
# Commit changes and file pull request
title = '{{{0}}} Sync to {0}-cache.yaml as of {1}\n'.format(
args.ros_distro, now)
commit_msg = '\n'.join([
get_pr_text(title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
delta = gen_delta_msg(total_changes)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay,
delta,
'',
pr_comment,
title=title,
)
sys.exit(0)
file_pr(overlay, delta, '', comment=pr_comment, title=title)
clean_up()
ok('Successfully synchronized repositories!')
def generate_rosdistro_conf(
basepath, distro, version, platforms, skip_keys=[]):
conf_dir = '{}/conf/ros-distro/include/{}/'.format(basepath, distro)
conf_file_name = 'generated-ros-distro.inc'
conf_path = '{}{}'.format(conf_dir, conf_file_name)
try:
make_dir(conf_dir)
with open(conf_path, 'w') as conf_file:
conf_file.write('# {}/{}\n'.format(distro, conf_file_name))
conf_file.write('# Generated by superflore -- DO NOT EDIT')
conf_file.write(
' (except ROS_DISTRO_METADATA_VERSION_REVISION)\n#\n')
conf_file.write(
'# Copyright ' + strftime("%Y", gmtime())
+ ' Open Source Robotics Foundation\n\n')
conf_file.write(
'# Increment every time meta-ros is released because of '
+ 'a manually created change, ie, NOT as a result of a '
+ 'superflore run (which\n# resets it to "0").')
conf_file.write(
'\nROS_DISTRO_METADATA_VERSION_REVISION = "0"\n')
conf_file.write(
'\nROS_SUPERFLORE_PROGRAM_VERSION = "{}"\n'
.format(get_superflore_version()))
conf_file.write('ROS_SUPERFLORE_GENERATION_SCHEME = "1"\n')
ros_version = yoctoRecipe._get_ros_version(distro)
conf_file.write(
'\nROS_DISTRO_TYPE = "ros{}"\n'.format(ros_version))
conf_file.write('ROS_VERSION = "{}"\n'.format(ros_version))
conf_file.write('# DO NOT OVERRIDE ROS_PYTHON_VERSION\n')
ros_python_version = 3
if ros_version == 1:
ros_python_version = 2
conf_file.write(
'ROS_PYTHON_VERSION = "{}"\n\n'.format(ros_python_version))
oe_skip_keys = map(
lambda skip_key: yoctoRecipe.convert_to_oe_name(skip_key),
skip_keys
)
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_SKIP_LIST', oe_skip_keys)
+ '\n')
conf_file.write(
'# Superflore was unable to generate recipes for these '
+ 'packages, eg, because their repositories are not on '
+ 'GitHub.\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_NOT_POSSIBLE',
yoctoRecipe.not_generated_recipes) + '\n')
conf_file.write(
'# Number of commits that will be returned by'
+ ' "git log files/ROS_DISTRO-cache.yaml" when the '
+ 'generated files are committed. This is\n# used for the'
+ ' third version field of DISTRO_VERSION.\n')
version = 1 if not version else len(version.splitlines()) + 1
conf_file.write(
'ROS_NUM_CACHE_YAML_COMMITS = "{}"'.format(version)
+ '\n\n')
conf_file.write(
'# Iterated values of '
+ 'ROS_DISTRO-cache.distribution_file.release_platforms.'
+ '<LINUX-DISTRO>.[ <NAME> ... ] .\n')
release_platforms = []
for p in sorted(platforms.items()):
for release in p[1]:
release_platforms.append(p[0] + '-' + release)
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_DISTRO_RELEASE_PLATFORMS', release_platforms) + '\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES',
yoctoRecipe.generated_recipes.keys()) + '\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPE_BASENAMES_WITH_COMPONENT',
[(yoctoRecipe.max_component_name - len(component)) * ' '
+ component + '/' + recipe + '_' + version
for recipe, (version, component)
in yoctoRecipe.generated_recipes.items()],
key=lambda recipe: recipe.split('/')[1].split('_')[0]))
conf_file.write(
'\n# What\'s built by packagegroup-ros-world. Does not '
+ 'include packages that appear solely in '
+ 'ROS_SUPERFLORE_GENERATED_BUILDTOOLS\n# (with a -native'
+ ' suffix) or ROS_SUPERFLORE_GENERATED_TESTS.\n')
recipes_set = set(yoctoRecipe.generated_recipes.keys())
test_deps = set(map(
lambda test_dep: yoctoRecipe.convert_to_oe_name(test_dep),
yoctoRecipe.generated_test_deps
- yoctoRecipe.generated_non_test_deps
))
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_WORLD_PACKAGES', recipes_set
- yoctoRecipe.generated_native_recipes - test_deps))
conf_file.write(
'\n# Packages found in the <buildtool_depend> and '
+ '<buildtool_export_depend> items, ie, ones for which a '
+ '-native is built. Does not\n# include those found in '
+ 'the ROS_EXEC_DEPENDS values in the recipes of build '
+ 'tools.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_BUILDTOOLS',
yoctoRecipe.generated_native_recipes) + '\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_PLATFORM_PACKAGE_DEPENDENCIES',
yoctoRecipe.platform_deps))
conf_file.write(
'\n# Packages found only in <test_depend> items. Does not'
+ ' include those found only in the ROS_*_DEPENDS of '
+ 'recipes of tests.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_TESTS',
test_deps) + '\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES_FOR_COMPONENTS',
yoctoRecipe.generated_components))
ok('Wrote {0}'.format(conf_path))
except OSError as e:
err('Failed to write conf {} to disk! {}'.format(conf_path, e))
raise e
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
pkgbuild_name =\
'/ros-{0}/{1}/{1}.pkgbuild'.format(distro.name, pkg)
pkgbuild_name = overlay.repo.repo_dir + pkgbuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
is_ros2 = get_distros()[distro.name]['distribution_type'] == 'ros2'
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]
patches = None
if os.path.exists(patch_path):
patches = [
f for f in glob.glob('%s/*.patch' % patch_path)
]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing pkgbuild.
prefix = '{0}/ros-{1}/{2}/'.format(overlay.repo.repo_dir, distro.name, pkg)
existing = glob.glob('%s*.pkgbuild' % prefix)
previous_version = None
if preserve_existing and os.path.isfile(pkgbuild_name):
ok("pkgbuild for package '%s' up to date, skipping..." % pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing[0])
previous_version = existing[0].lstrip(prefix).rstrip('.pkgbuild')
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg
)
overlay.repo.remove_file(manifest_file)
try:
current = arch_pkgbuild(distro, pkg, has_patches)
current.pkgbuild.name = pkg
current.pkgbuild.version = version
current.pkgbuild.patches = patches
current.pkgbuild.is_ros2 = is_ros2
except Exception as e:
err('Failed to generate pkgbuild for package {}!'.format(pkg))
raise e
try:
pkgbuild_text = current.pkgbuild_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.pkgbuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.pkgbuild.get_unresolved(), None
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir(
"{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg)
)
success_msg = 'Successfully generated pkgbuild for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
try:
pkgbuild_file = '{0}/ros-{1}/{2}/PKGBUILD'.format(
overlay.repo.repo_dir,
distro.name,
pkg,
)
ok(f"writing {pkgbuild_file}")
with open(pkgbuild_file, "w") as pkgbuild_file_f:
pkgbuild_file_f.write(pkgbuild_text)
except Exception as e:
err(f"Failed to write f{pkgbuild_file} to disk!")
raise e
return current, previous_version, pkg
def pull_request(self, message, title):
info('Filing pull-request...')
self.git.pull_request(m='{0}'.format(message),
title='{0}'.format(title))
ok('Successfully filed a pull request.')
def main():
overlay = None
parser = get_parser('Generate OpenEmbedded recipes for ROS packages',
exclude_all=True,
require_rosdistro=True,
require_dryrun=True)
parser.add_argument('--tar-archive-dir',
help='location to store archived packages',
type=str)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = set(args.skip_keys) if args.skip_keys else set()
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
warn('"{0}" distro detected...'.format(args.ros_distro))
"""
No longer supporting generation for multiple targets, but left the code in
place to handle them in case it might be needed again in the future.
"""
selected_targets = [args.ros_distro]
preserve_existing = args.only
now = os.getenv('SUPERFLORE_GENERATION_DATETIME',
get_utcnow_timestamp_str())
repo_org = 'ros'
repo_name = 'meta-ros'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
branch=(('superflore/{}'.format(now))
if not args.no_branch else None),
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
)
if not args.only:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for all packages in ROS '
'distribution {}.\n'.format(selected_targets[0]))
else:
pr_comment = pr_comment or (
'Recipes generated by **superflore** for package(s) {} in ROS '
'distribution {}.\n'.format(args.only, args.ros_distro))
# generate installers
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir
else:
srcrev_filename = None
with CacheManager(srcrev_filename) as srcrev_cache:
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
regenerate_pkg(
overlay,
pkg,
distro,
False, # preserve_existing
srcrev_cache,
skip_keys=skip_keys,
)
except KeyError:
err("No package to satisfy key '%s' available "
"packages in selected distro: %s" %
(pkg, get_package_names(distro)))
sys.exit(1)
# Commit changes and file pull request
title =\
'{{{0}}} Selected recipes generated from '\
'files/{0}/generated/cache.yaml '\
'as of {1}\n'.format(
args.ros_distro,
now)
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
delta = "Regenerated: '%s'\n" % args.only
overlay.add_generated_files(args.ros_distro)
commit_msg = '\n'.join([
get_pr_text(
title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
if args.dry_run:
save_pr(overlay, args.only, '', pr_comment, title=title)
sys.exit(0)
file_pr(overlay,
delta,
'',
pr_comment,
distro=args.ros_distro,
title=title)
ok('Successfully synchronized repositories!')
sys.exit(0)
overlay.clean_ros_recipe_dirs(args.ros_distro)
for adistro in selected_targets:
yoctoRecipe.reset()
distro = get_distro(adistro)
distro_installers, _, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_pkg,
preserve_existing,
srcrev_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
)
total_changes[adistro] = distro_changes
total_installers[adistro] = distro_installers
yoctoRecipe.generate_ros_distro_inc(
_repo, args.ros_distro,
overlay.get_file_revision_logs(
'meta-ros{0}-{1}/files/{1}/generated/cache.yaml'.
format(yoctoRecipe._get_ros_version(args.ros_distro),
args.ros_distro)), distro.release_platforms,
skip_keys)
yoctoRecipe.generate_superflore_datetime_inc(
_repo, args.ros_distro, now)
yoctoRecipe.generate_rosdep_resolve(_repo, args.ros_distro)
yoctoRecipe.generate_newer_platform_components(
_repo, args.ros_distro)
overlay.add_generated_files(args.ros_distro)
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
summary = overlay.get_change_summary(args.ros_distro)
if len(summary) == 0:
info('Exiting...')
clean_up()
sys.exit(0)
else:
info('But there are some changes in other regenerated files:'
'%s' % summary)
# remove duplicates
delta = gen_delta_msg(total_changes, markup='')
# Commit changes and file pull request
title = '{{{0}}} Sync to files/{0}/generated/'\
'cache.yaml as of {1}\n'.format(
args.ros_distro,
now)
commit_msg = '\n'.join([
get_pr_text(title + '\n' +
pr_comment.replace('**superflore**', 'superflore'),
markup=''), delta
])
overlay.commit_changes(args.ros_distro, commit_msg)
delta = gen_delta_msg(total_changes)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(
overlay,
delta,
'',
pr_comment,
title=title,
)
sys.exit(0)
file_pr(overlay, delta, '', comment=pr_comment, title=title)
clean_up()
ok('Successfully synchronized repositories!')
def regenerate_pkg(overlay, pkg, rosdistro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(rosdistro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s' available packages"
" in selected distro: %s" %
(pkg, get_package_names(rosdistro)))
try:
version = get_pkg_version(rosdistro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
rosdistro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe which was removed by clean_ros_recipe_dirs
prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format(
yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe)
existing = overlay.repo.git.status('--porcelain', '--', prefix)
if existing:
# The git status --porcelain output will look like this:
# D meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb
# we want just the path with filename
if len(existing.split('\n')) > 1:
warn('More than 1 recipe was output by "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
if existing.split()[0] != 'D':
err('Unexpected output from "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing.split()[1]
else:
# If it isn't shown in git status, it could still exist as normal
# unchanged file when --only option is being used
import glob
existing = glob.glob('{0}/{1}'.format(repo_dir, prefix))
if existing:
if len(existing) > 1:
err('More than 1 recipe was output by "git status '
'--porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.
format(yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing[0]
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(rosdistro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
def regenerate_pkg(overlay, pkg, distro, preserve_existing, tar_dir, md5_cache,
sha256_cache, skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
glob_pattern = '{0}/generated-recipes-{1}/{2}/{3}*.bb'.format(
repo_dir, distro.name, component_name, recipe)
existing = glob.glob(glob_pattern)
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0], True)
try:
current = oe_installer(distro, pkg, tar_dir, md5_cache, sha256_cache,
skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
except Exception as e:
err('Failed generating installer for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
make_dir("{0}/generated-recipes-{1}/{2}".format(repo_dir, distro.name,
component_name))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/generated-recipes-{1}/{2}/{3}_{4}.bb'.format(
repo_dir, distro.name, component_name, recipe, version)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
return current, []
def generate_ros_distro_inc(basepath,
distro,
version,
platforms,
skip_keys=[]):
conf_dir = '{0}/meta-ros{1}-{2}/conf/ros-distro/include/{2}/' \
'generated/'.format(
basepath, yoctoRecipe._get_ros_version(distro), distro)
conf_file_name = 'superflore-ros-distro.inc'
conf_path = '{}{}'.format(conf_dir, conf_file_name)
try:
make_dir(conf_dir)
with open(conf_path, 'w') as conf_file:
conf_file.write('# {}/generated/{}\n'.format(
distro, conf_file_name))
conf_file.write('# Generated by superflore -- DO NOT EDIT')
conf_file.write(
' (except ROS_DISTRO_METADATA_VERSION_REVISION)\n#\n')
conf_file.write(
'# Copyright Open Source Robotics Foundation\n\n')
conf_file.write(
'# Increment every time meta-ros is released because of ' +
'a manually created change, ie, NOT as a result of a ' +
'superflore run (which\n# resets it to "0").')
conf_file.write(
'\nROS_DISTRO_METADATA_VERSION_REVISION = "0"\n')
conf_file.write(
'\nROS_SUPERFLORE_PROGRAM_VERSION = "{}"\n'.format(
get_superflore_version()))
conf_file.write('ROS_SUPERFLORE_GENERATION_SCHEME = "2"\n')
ros_version = yoctoRecipe._get_ros_version(distro)
conf_file.write(
'\nROS_DISTRO_TYPE = "ros{}"\n'.format(ros_version))
conf_file.write('ROS_VERSION = "{}"\n'.format(ros_version))
conf_file.write('# DO NOT OVERRIDE ROS_PYTHON_VERSION\n')
ros_python_version = 3
if ros_version == 1:
ros_python_version = 2
conf_file.write(
'ROS_PYTHON_VERSION = "{}"\n\n'.format(ros_python_version))
oe_skip_keys = map(
lambda skip_key: yoctoRecipe.convert_to_oe_name(skip_key),
skip_keys)
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_SKIP_LIST', oe_skip_keys) +
'\n')
conf_file.write(
'# Superflore was unable to generate recipes for these ' +
'packages, eg, because their repositories are not on ' +
'GitHub.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_NOT_POSSIBLE',
yoctoRecipe.not_generated_recipes) + '\n')
conf_file.write(
'# Number of commits that will be returned by '
'"git log meta-ros{0}-{1}/files/{1}/generated/'
'cache.yaml" when the\n# generated files are committed. '
'This is used for the fourth version field of '
'DISTRO_VERSION.\n'.format(
yoctoRecipe._get_ros_version(distro), distro))
version = 1 if not version else len(version.splitlines()) + 1
conf_file.write(
'ROS_NUM_CACHE_YAML_COMMITS = "{}"'.format(version) +
'\n\n')
conf_file.write(
'# Iterated values of ' +
'ROS_DISTRO-cache.distribution_file.release_platforms.' +
'<LINUX-DISTRO>.[ <NAME> ... ] .\n')
release_platforms = []
for p in sorted(platforms.items()):
for release in p[1]:
release_platforms.append(p[0] + '-' + release)
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_DISTRO_RELEASE_PLATFORMS', release_platforms) +
'\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES',
yoctoRecipe.generated_recipes.keys()) + '\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPE_BASENAMES_WITH_COMPONENT',
[(yoctoRecipe.max_component_name - len(component)) *
' ' + component + '/' + recipe + '_' + version
for recipe, (version, component) in
yoctoRecipe.generated_recipes.items()],
key=lambda recipe: recipe.split('/')[1].split('_')[0]))
conf_file.write(
'\n# What\'s built by packagegroup-ros-world. Does not ' +
'include packages that appear solely in ' +
'ROS_SUPERFLORE_GENERATED_BUILDTOOLS\n# (with a -native' +
' suffix) or ROS_SUPERFLORE_GENERATED_TESTS.\n')
recipes_set = set(yoctoRecipe.generated_recipes.keys())
test_deps = set(
map(
lambda test_dep: yoctoRecipe.convert_to_oe_name(
test_dep), yoctoRecipe.generated_test_deps -
yoctoRecipe.generated_non_test_deps))
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_WORLD_PACKAGES',
recipes_set - yoctoRecipe.generated_native_recipes -
test_deps))
conf_file.write(
'\n# Packages found in the <buildtool_depend> and ' +
'<buildtool_export_depend> items, ie, ones for which a ' +
'-native is built. Does not\n# include those found in ' +
'the ROS_EXEC_DEPENDS values in the recipes of build ' +
'tools.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_BUILDTOOLS_%s' %
distro.upper(), yoctoRecipe.generated_native_recipes) +
'\n')
conf_file.write('ROS_SUPERFLORE_GENERATED_BUILDTOOLS_append ='
' " ${ROS_SUPERFLORE_GENERATED_BUILDTOOLS_%s}"'
'\n\n' % distro.upper())
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_PLATFORM_PACKAGE_DEPENDENCIES',
yoctoRecipe.platform_deps))
conf_file.write(
'\n# Packages found only in <test_depend> items. Does not'
+ ' include those found only in the ROS_*_DEPENDS of ' +
'recipes of tests.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_TESTS', test_deps) + '\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES_FOR_COMPONENTS',
yoctoRecipe.generated_components))
conf_file.write(
'\n# Platform packages without a OE-RECIPE@OE-LAYER' +
' mapping in base.yaml, python.yaml, or ruby.yaml. Until' +
' they are added, override\n# the settings in' +
' ros-distro.inc .\n')
"""
Drop trailing "}" so that "..._foo-native" sorts after
"..._foo".
"""
unresolved = [
p[0:-1] for p in yoctoRecipe.platform_deps
if p.startswith(UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX)
]
for p in sorted(unresolved):
"""
PN is last underscore-separated field. NB the trailing '}'
has already been removed.
"""
pn = p.split('_')[-1]
conf_file.write(UNRESOLVED_PLATFORM_PKG_PREFIX + pn +
' = "UNRESOLVED-' + pn + '"\n')
ok('Wrote {0}'.format(conf_path))
except OSError as e:
err('Failed to write conf {} to disk! {}'.format(conf_path, e))
raise e
def regenerate_pkg(overlay, pkg, distro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
)
existing = glob.glob('{}_*.bb'.format(prefix))
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
existing = existing[0]
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(distro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'ros{1}-{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes['ros{0}-{1}'.format(
yoctoRecipe._get_ros_version(distro.name),
recipe)] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
def main():
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into Gentoo Linux')
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
skip_keys = args.skip_keys or []
selected_targets = None
if not args.dry_run:
if 'SUPERFLORE_GITHUB_TOKEN' not in os.environ:
raise NoGitHubAuthToken()
if args.pr_only:
if args.dry_run:
parser.error('Invalid args! cannot dry-run and file PR')
if not args.output_repository_path:
parser.error('Invalid args! no repository specified')
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title=title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
elif args.all:
warn('"All" mode detected... This may take a while!')
preserve_existing = False
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.only:
parser.error('Invalid args! --only requires specifying --ros-distro')
if not selected_targets:
selected_targets = get_distros_by_status('active')
repo_org = 'ros'
repo_name = 'ros-overlay'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output_repository_path is None
overlay = RosOverlay(
_repo,
not args.output_repository_path,
org=repo_org,
repo=repo_name,
from_branch=args.upstream_branch,
new_branch=(not args.no_branch),
)
if not preserve_existing and not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of all'
' packages from ROS distro %s from ROS-Overlay commit %s.' %
(selected_targets, overlay.repo.get_last_hash()))
elif not args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator ran update from ROS-Overlay ' +
'commit %s.' % (overlay.repo.get_last_hash()))
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.only:
pr_comment = pr_comment or (
'Superflore ebuild generator began regeneration of ' +
'package(s) %s from commit %s.' %
(args.only, overlay.repo.get_last_hash()))
missing_depends = set()
to_commit = set()
will_file_pr = False
for pkg in args.only:
if pkg in skip_keys:
warn("Package '%s' is in skip-keys list, skipping..." %
pkg)
continue
info("Regenerating package '%s'..." % pkg)
try:
ebuild, deps, version = regenerate_pkg(
overlay, pkg, get_distro(args.ros_distro),
preserve_existing)
if not ebuild:
for dep in deps:
missing_depends.add(dep)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
continue
if ebuild:
to_commit.add(pkg)
will_file_pr = True
# if no packages succeeded, exit with error
if not will_file_pr:
err("No packages generated successfully, exiting.")
sys.exit(1)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = to_commit
overlay.regenerate_manifests(regen_dict)
overlay.commit_changes(args.ros_distro)
if args.dry_run:
save_pr(overlay,
args.only,
missing_deps=gen_missing_deps_msg(missing_depends),
comment=pr_comment)
sys.exit(0)
delta = "Regenerated: '%s'\n" % args.only
file_pr(overlay, delta, gen_missing_deps_msg(missing_depends),
pr_comment)
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
get_distro(distro),
overlay=overlay,
gen_pkg_func=regenerate_pkg,
preserve_existing=preserve_existing,
skip_keys=skip_keys,
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes)
missing_deps = gen_missing_deps_msg(total_broken)
# Commit changes and file pull request
overlay.regenerate_manifests(total_installers)
overlay.commit_changes('all' if args.all else args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(overlay,
delta,
missing_deps=missing_deps,
comment=pr_comment)
sys.exit(0)
file_pr(overlay, delta, missing_deps, comment=pr_comment)
clean_up()
ok('Successfully synchronized repositories!')
def main():
overlay = None
preserve_existing = True
parser = get_parser('Deploy ROS packages into Yocto Linux')
parser.add_argument(
'--tar-archive-dir',
help='location to store archived packages',
type=str
)
args = parser.parse_args(sys.argv[1:])
pr_comment = args.pr_comment
selected_targets = None
if args.all:
warn('"All" mode detected... this may take a while!')
preserve_existing = False
elif args.ros_distro:
warn('"{0}" distro detected...'.format(args.ros_distro))
selected_targets = [args.ros_distro]
preserve_existing = False
elif args.dry_run and args.pr_only:
parser.error('Invalid args! cannot dry-run and file PR')
elif args.pr_only and not args.output_repository_path:
parser.error('Invalid args! no repository specified')
elif args.pr_only:
try:
prev_overlay = RepoInstance(args.output_repository_path, False)
msg, title = load_pr()
prev_overlay.pull_request(msg, title)
clean_up()
sys.exit(0)
except Exception as e:
err('Failed to file PR!')
err('reason: {0}'.format(e))
sys.exit(1)
if not selected_targets:
selected_targets = ros1_distros + ros2_distros
repo_org = 'lgsvl'
repo_name = 'meta-ros2'
if args.upstream_repo:
repo_org, repo_name = url_to_repo_org(args.upstream_repo)
# open cached tar file if it exists
with TempfileManager(args.output_repository_path) as _repo:
if not args.output_repository_path:
# give our group write permissions to the temp dir
os.chmod(_repo, 17407)
# clone if args.output-repository_path is None
overlay = RosMeta(
_repo,
not args.output_repository_path,
org=repo_org,
repo=repo_name
)
if not args.only:
pr_comment = pr_comment or (
'Superflore yocto generator began regeneration of all '
'packages form ROS distribution(s) %s on Meta-ROS from '
'commit %s.' % (
selected_targets,
overlay.repo.get_last_hash()
)
)
else:
pr_comment = pr_comment or (
'Superflore yocto generator began regeneration of package(s)'
' %s from ROS distro %s from Meta-ROS from commit %s.' % (
args.only,
args.ros_distro,
overlay.repo.get_last_hash()
)
)
# generate installers
total_installers = dict()
total_broken = set()
total_changes = dict()
if args.tar_archive_dir:
sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir
md5_filename = '%s/md5_cache.pickle' % args.tar_archive_dir
else:
sha256_filename = None
md5_filename = None
with TempfileManager(args.tar_archive_dir) as tar_dir,\
CacheManager(sha256_filename) as sha256_cache,\
CacheManager(md5_filename) as md5_cache: # noqa
if args.only:
for pkg in args.only:
info("Regenerating package '%s'..." % pkg)
try:
regenerate_installer(
overlay,
pkg,
get_distro(args.ros_distro),
preserve_existing,
tar_dir,
md5_cache,
sha256_cache
)
except KeyError:
err("No package to satisfy key '%s'" % pkg)
sys.exit(1)
# Commit changes and file pull request
regen_dict = dict()
regen_dict[args.ros_distro] = args.only
overlay.commit_changes(args.ros_distro)
if args.dry_run:
save_pr(overlay, args.only, None, pr_comment)
sys.exit(0)
delta = "Regenerated: '%s'\n" % args.only
file_pr(overlay, delta, '', pr_comment, distro=args.ros_distro)
ok('Successfully synchronized repositories!')
sys.exit(0)
for distro in selected_targets:
distro_installers, distro_broken, distro_changes =\
generate_installers(
distro,
overlay,
regenerate_installer,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache
)
for key in distro_broken.keys():
for pkg in distro_broken[key]:
total_broken.add(pkg)
total_changes[distro] = distro_changes
total_installers[distro] = distro_installers
num_changes = 0
for distro_name in total_changes:
num_changes += len(total_changes[distro_name])
if num_changes == 0:
info('ROS distro is up to date.')
info('Exiting...')
clean_up()
sys.exit(0)
# remove duplicates
delta = gen_delta_msg(total_changes)
missing_deps = gen_missing_deps_msg(total_broken)
# Commit changes and file pull request
overlay.commit_changes(args.ros_distro)
if args.dry_run:
info('Running in dry mode, not filing PR')
save_pr(overlay, delta, missing_deps, pr_comment)
clean_up()
sys.exit(0)
file_pr(overlay, delta, missing_deps, pr_comment)
clean_up()
ok('Successfully synchronized repositories!')