def generate_superflore_datetime_inc(basepath, dist):
datetime_dir = '{}/conf/ros-distro/include/{}/'.format(basepath, dist)
datetime_file_name = 'generated-superflore-datetime.inc'
datetime_path = '{}{}'.format(datetime_dir, datetime_file_name)
try:
make_dir(datetime_dir)
with open(datetime_path, 'w') as datetime_file:
datetime_file.write('# {}/{}\n'.format(
dist, datetime_file_name))
datetime_file.write('# Generated by superflore -- DO NOT EDIT')
datetime_file.write(
' (except ROS_DISTRO_METADATA_VERSION_REVISION)\n#\n')
datetime_file.write(
'# Copyright ' + strftime("%Y", gmtime())
+ ' Open Source Robotics Foundation\n\n')
datetime_file.write(
'# The start time, in UTC, of the last superflore run that'
+ ' resulted in a change to the generated files. The date'
+ ' portion is used as\n# the second version field of '
+ 'ROS_DISTRO_METADATA_VERSION prior to the first release'
+ ' of a ROS_DISTRO.\n')
now = datetime.utcnow().strftime('%Y%m%d%H%M%S')
datetime_file.write(
'ROS_SUPERFLORE_GENERATION_DATETIME = "{}"'.format(now))
ok('Wrote {0}'.format(datetime_path))
except OSError as e:
err('Failed to write SuperFlore datetime {} to disk! {}'.format(
datetime_path, e))
raise e
def generate_superflore_datetime_inc(basepath, dist, now):
datetime_dir = '{0}/meta-ros{1}-{2}/conf/ros-distro/include/{2}/' \
'generated/'.format(
basepath, yoctoRecipe._get_ros_version(dist), dist)
datetime_file_name = 'superflore-datetime.inc'
datetime_path = '{}{}'.format(datetime_dir, datetime_file_name)
try:
make_dir(datetime_dir)
with open(datetime_path, 'w') as datetime_file:
datetime_file.write('# {}/generated/{}\n'.format(
dist, datetime_file_name))
datetime_file.write('# Generated by superflore -- DO NOT EDIT')
datetime_file.write(
'\n#\n# Copyright Open Source Robotics Foundation\n\n')
datetime_file.write(
'\n# The time, in UTC, associated with the last superflore'
+
' run that resulted in a change to the generated files.' +
' The date portion is\n# used as the third version field' +
' of ROS_DISTRO_METADATA_VERSION prior to the first' +
' release of a ROS_DISTRO.\n')
datetime_file.write(
'ROS_SUPERFLORE_GENERATION_DATETIME = "{}"\n'.format(now))
ok('Wrote {0}'.format(datetime_path))
except OSError as e:
err('Failed to write SuperFlore datetime {} to disk! {}'.format(
datetime_path, e))
raise e
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
ebuild_name =\
'/ros-{0}/{1}/{1}-{2}.ebuild'.format(distro.name, pkg, version)
ebuild_name = overlay.repo.repo_dir + ebuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing ebuild.
existing = glob.glob('{0}/ros-{1}/{2}/*.ebuild'.format(
overlay.repo.repo_dir, distro.name, pkg))
if preserve_existing and os.path.isfile(ebuild_name):
ok("ebuild for package '%s' up to date, skipping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg)
overlay.repo.remove_file(manifest_file)
try:
current = gentoo_installer(distro, pkg, has_patches)
current.ebuild.name = pkg
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
ebuild_text = current.ebuild_text()
metadata_text = current.metadata_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.ebuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.ebuild.get_unresolved()
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir("{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
try:
ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
overlay.repo.repo_dir, distro.name, pkg, version)
ebuild_file = open(ebuild_file, "w")
metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format(
overlay.repo.repo_dir, distro.name, pkg)
metadata_file = open(metadata_file, "w")
ebuild_file.write(ebuild_text)
metadata_file.write(metadata_text)
except Exception as e:
err("Failed to write ebuild/metadata to disk!")
raise e
return current, []
def test_mkdir(self):
"""Tests the make directory function"""
with TempfileManager(None) as temp_dir:
created = '%s/test' % temp_dir
make_dir(created)
self.assertTrue(os.path.isdir(created))
# try and create the directory again, should pass
make_dir(created)
self.assertTrue(os.path.isdir(created))
def generate_newer_platform_components(basepath, distro):
newer_sys_comps_dir = '{0}/meta-ros{1}-{2}/files/{2}/' \
'generated/'.format(
basepath,
yoctoRecipe._get_ros_version(distro), distro)
newer_sys_comps_path = '{0}newer-platform-components.list'.format(
newer_sys_comps_dir)
ros_version = yoctoRecipe._get_ros_version(distro)
str_distro = 'ros' if ros_version == 1 else 'ros{}'.format(ros_version)
args1_wget = ['wget', '-O', '-', 'http://packages.ros.org/'
+ str_distro
+ '/ubuntu/dists/bionic/main/source/Sources.gz']
args2_gunzip = ['gunzip', '-']
args3_grep = ['grep', '-E', '^(Package|Version|Build-Depends)']
args4_awk = ['awk', '$1 ~ /^Package:/ && $2 !~ /^ros-/ '
+ '{ printf "%s;", $2; getline; '
+ 'printf "%s;", $2; getline; '
+ 'gsub(/Build-Depends: /, ""); gsub(/, /, ","); print}']
args5_sort = ['sort', '-t', ';', '-k', '1,1']
try:
make_dir(newer_sys_comps_dir)
wget = Popen(args1_wget, stdout=PIPE, stderr=DEVNULL)
gunzip = Popen(args2_gunzip, stdin=wget.stdout,
stdout=PIPE, stderr=DEVNULL)
grep = Popen(args3_grep, stdin=gunzip.stdout,
stdout=PIPE, stderr=DEVNULL)
awk = Popen(args4_awk, stdin=grep.stdout,
stdout=PIPE, stderr=DEVNULL)
sort = Popen(args5_sort, env={'LC_ALL': 'C'},
stdin=awk.stdout, stdout=PIPE, stderr=DEVNULL)
cmds = [wget, gunzip, grep, awk]
# Allow previous process to receive a SIGPIPE
# if the next one in the pipeline exits.
for cmd in cmds:
cmd.stdout.close()
# Run the pipeline and collect the output
txt_output = sort.communicate()[0].decode()
# Consume the return value of the other processes
for cmd in cmds:
cmd.wait()
cmds.append(sort)
if any([cmd.returncode for cmd in cmds]):
errors = ['{}[{}]'.format(cmd.args[0], cmd.returncode)
for cmd in cmds]
raise RuntimeError('Error codes ' + ' '.join(errors))
with open(newer_sys_comps_path, 'w') as newer_sys_comps_file:
newer_sys_comps_file.write(
'# {}/newer-platform-components.list\n'.format(distro))
newer_sys_comps_file.write(txt_output)
ok('Wrote {0}'.format(newer_sys_comps_path))
except (OSError, RuntimeError) as e:
err('Failed to write {0} to disk! {1}'.format(
newer_sys_comps_path, e))
raise e
def regenerate_installer(overlay, pkg, distro, preserve_existing, tar_dir):
make_dir("{0}/recipes-ros-{1}".format(overlay.repo.repo_dir, distro.name))
version = get_pkg_version(distro, pkg)
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % pkg)
# check for an existing recipe
existing = glob.glob('{0}/recipes-ros-{1}/{2}/*.bb'.format(
overlay.repo.repo_dir, distro.name, pkg))
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skpping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
try:
current = oe_installer(distro, pkg, tar_dir)
current.recipe.name = pkg.replace('_', '-')
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
recipe_text = current.recipe_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.recipe.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.recipe.get_unresolved()
except NoPkgXml:
err("Could not fetch pkg!")
return None, []
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir("{0}/recipes-ros-{1}/{2}".format(overlay.repo.repo_dir,
distro.name,
pkg.replace('_', '-')))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_name = '{0}/recipes-ros-{1}/{2}/{2}_{3}.bb'.format(
overlay.repo.repo_dir, distro.name, pkg.replace('_', '-'), version)
try:
with open('{0}'.format(recipe_name), "w") as recipe_file:
recipe_file.write(recipe_text)
except Exception as e:
err("Failed to write recipe to disk!")
raise e
return current, []
def generate_superflore_change_summary(basepath, distro, change_summary):
change_summary_dir = '{0}/files/{1}/'.format(basepath, distro)
change_summary_path = '{0}superflore-change-summary.txt'.format(
change_summary_dir)
try:
make_dir(change_summary_dir)
with open(change_summary_path, 'w') as change_summary_file:
change_summary_file.write(
'{}/superflore-change-summary.txt\n'.format(distro))
change_summary_file.write(change_summary)
ok('Wrote {0}'.format(change_summary_path))
except OSError as e:
err('Failed to write change summary {} to disk! {}'.format(
change_summary_path, e))
raise e
def generate_distro_cache(basepath, distro, skip_keys=[]):
distro_cache_dir = '{0}/files/{1}/'.format(basepath, distro)
distro_cache_path = '{0}cache.yaml'.format(distro_cache_dir)
try:
index = get_index(get_index_url())
yaml_str = get_distribution_cache_string(index, distro)
make_dir(distro_cache_dir)
with open(distro_cache_path, 'w') as distro_cache_file:
distro_cache_file.write('# {}/cache.yaml\n'.format(distro))
distro_cache_file.write(yaml_str)
ok('Wrote {0}'.format(distro_cache_path))
except OSError as e:
err('Failed to write distro cache {} to disk! {}'.format(
distro_cache_path, e))
raise e
# Generate a diff'able cache file
distro_cache_diff_path = '{}cache.diffme'.format(distro_cache_dir)
try:
def replace_all_patterns(d, text):
for k, v in d.items():
text = re.sub(k, v, text, flags=re.M)
return text
replacement_table = {
r"{([^ }][^ }]*)}": r'[[\1]]',
r"{": r"{\n",
r"}": r"\n}",
r"\[\[": r"{",
r"\]\]": r"}",
r", ": r",\n",
r"^ ": r"-----\n",
r"<version>[^<]*</version>": r"",
r"><": r">\n<",
r"^ ": r"-----\n",
r"^(source_repo_package_xmls:)": r"-----\n\1",
}
with open(distro_cache_diff_path, 'w') as distro_cache_diff_file:
distro_cache_diff_file.write(
'# {}/cache.diffme\n'.format(distro))
yaml_str = replace_all_patterns(replacement_table, yaml_str)
distro_cache_diff_file.write(yaml_str)
ok('Wrote {0}'.format(distro_cache_diff_path))
except OSError as e:
err('Failed to write diffme distro cache {} to disk! {}'.format(
distro_cache_diff_path, e))
raise e
def generate_rosdep_resolve(basepath, distro):
rosdep_resolve_dir = '{0}/files/{1}/'.format(basepath, distro)
rosdep_resolve_path = '{0}rosdep-resolve.yaml'.format(
rosdep_resolve_dir)
try:
make_dir(rosdep_resolve_dir)
with open(rosdep_resolve_path, 'w') as rosdep_resolve_file:
rosdep_resolve_file.write(
'# {}/rosdep-resolve.yaml\n'.format(distro))
cache_as_dict_of_list = {
k: list(v) for k, v in yoctoRecipe.rosdep_cache.items()}
rosdep_resolve_file.write(yaml.dump(
cache_as_dict_of_list, default_flow_style=False))
ok('Wrote {0}'.format(rosdep_resolve_path))
except OSError as e:
err('Failed to write rosdep resolve cache {} to disk! {}'.format(
rosdep_resolve_path, e))
raise e
def generate_ros_distro_inc(basepath,
distro,
version,
platforms,
skip_keys=[]):
conf_dir = '{0}/meta-ros{1}-{2}/conf/ros-distro/include/{2}/' \
'generated/'.format(
basepath, yoctoRecipe._get_ros_version(distro), distro)
conf_file_name = 'superflore-ros-distro.inc'
conf_path = '{}{}'.format(conf_dir, conf_file_name)
try:
make_dir(conf_dir)
with open(conf_path, 'w') as conf_file:
conf_file.write('# {}/generated/{}\n'.format(
distro, conf_file_name))
conf_file.write('# Generated by superflore -- DO NOT EDIT')
conf_file.write(
' (except ROS_DISTRO_METADATA_VERSION_REVISION)\n#\n')
conf_file.write(
'# Copyright Open Source Robotics Foundation\n\n')
conf_file.write(
'# Increment every time meta-ros is released because of ' +
'a manually created change, ie, NOT as a result of a ' +
'superflore run (which\n# resets it to "0").')
conf_file.write(
'\nROS_DISTRO_METADATA_VERSION_REVISION = "0"\n')
conf_file.write(
'\nROS_SUPERFLORE_PROGRAM_VERSION = "{}"\n'.format(
get_superflore_version()))
conf_file.write('ROS_SUPERFLORE_GENERATION_SCHEME = "2"\n')
ros_version = yoctoRecipe._get_ros_version(distro)
conf_file.write(
'\nROS_DISTRO_TYPE = "ros{}"\n'.format(ros_version))
conf_file.write('ROS_VERSION = "{}"\n'.format(ros_version))
conf_file.write('# DO NOT OVERRIDE ROS_PYTHON_VERSION\n')
ros_python_version = 3
if ros_version == 1:
ros_python_version = 2
conf_file.write(
'ROS_PYTHON_VERSION = "{}"\n\n'.format(ros_python_version))
oe_skip_keys = map(
lambda skip_key: yoctoRecipe.convert_to_oe_name(skip_key),
skip_keys)
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_SKIP_LIST', oe_skip_keys) +
'\n')
conf_file.write(
'# Superflore was unable to generate recipes for these ' +
'packages, eg, because their repositories are not on ' +
'GitHub.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_NOT_POSSIBLE',
yoctoRecipe.not_generated_recipes) + '\n')
conf_file.write(
'# Number of commits that will be returned by '
'"git log meta-ros{0}-{1}/files/{1}/generated/'
'cache.yaml" when the\n# generated files are committed. '
'This is used for the fourth version field of '
'DISTRO_VERSION.\n'.format(
yoctoRecipe._get_ros_version(distro), distro))
version = 1 if not version else len(version.splitlines()) + 1
conf_file.write(
'ROS_NUM_CACHE_YAML_COMMITS = "{}"'.format(version) +
'\n\n')
conf_file.write(
'# Iterated values of ' +
'ROS_DISTRO-cache.distribution_file.release_platforms.' +
'<LINUX-DISTRO>.[ <NAME> ... ] .\n')
release_platforms = []
for p in sorted(platforms.items()):
for release in p[1]:
release_platforms.append(p[0] + '-' + release)
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_DISTRO_RELEASE_PLATFORMS', release_platforms) +
'\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES',
yoctoRecipe.generated_recipes.keys()) + '\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPE_BASENAMES_WITH_COMPONENT',
[(yoctoRecipe.max_component_name - len(component)) *
' ' + component + '/' + recipe + '_' + version
for recipe, (version, component) in
yoctoRecipe.generated_recipes.items()],
key=lambda recipe: recipe.split('/')[1].split('_')[0]))
conf_file.write(
'\n# What\'s built by packagegroup-ros-world. Does not ' +
'include packages that appear solely in ' +
'ROS_SUPERFLORE_GENERATED_BUILDTOOLS\n# (with a -native' +
' suffix) or ROS_SUPERFLORE_GENERATED_TESTS.\n')
recipes_set = set(yoctoRecipe.generated_recipes.keys())
test_deps = set(
map(
lambda test_dep: yoctoRecipe.convert_to_oe_name(
test_dep), yoctoRecipe.generated_test_deps -
yoctoRecipe.generated_non_test_deps))
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_WORLD_PACKAGES',
recipes_set - yoctoRecipe.generated_native_recipes -
test_deps))
conf_file.write(
'\n# Packages found in the <buildtool_depend> and ' +
'<buildtool_export_depend> items, ie, ones for which a ' +
'-native is built. Does not\n# include those found in ' +
'the ROS_EXEC_DEPENDS values in the recipes of build ' +
'tools.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_BUILDTOOLS_%s' %
distro.upper(), yoctoRecipe.generated_native_recipes) +
'\n')
conf_file.write('ROS_SUPERFLORE_GENERATED_BUILDTOOLS_append ='
' " ${ROS_SUPERFLORE_GENERATED_BUILDTOOLS_%s}"'
'\n\n' % distro.upper())
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_PLATFORM_PACKAGE_DEPENDENCIES',
yoctoRecipe.platform_deps))
conf_file.write(
'\n# Packages found only in <test_depend> items. Does not'
+ ' include those found only in the ROS_*_DEPENDS of ' +
'recipes of tests.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_TESTS', test_deps) + '\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES_FOR_COMPONENTS',
yoctoRecipe.generated_components))
conf_file.write(
'\n# Platform packages without a OE-RECIPE@OE-LAYER' +
' mapping in base.yaml, python.yaml, or ruby.yaml. Until' +
' they are added, override\n# the settings in' +
' ros-distro.inc .\n')
"""
Drop trailing "}" so that "..._foo-native" sorts after
"..._foo".
"""
unresolved = [
p[0:-1] for p in yoctoRecipe.platform_deps
if p.startswith(UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX)
]
for p in sorted(unresolved):
"""
PN is last underscore-separated field. NB the trailing '}'
has already been removed.
"""
pn = p.split('_')[-1]
conf_file.write(UNRESOLVED_PLATFORM_PKG_PREFIX + pn +
' = "UNRESOLVED-' + pn + '"\n')
ok('Wrote {0}'.format(conf_path))
except OSError as e:
err('Failed to write conf {} to disk! {}'.format(conf_path, e))
raise e
def regenerate_pkg(overlay, pkg, rosdistro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(rosdistro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s' available packages"
" in selected distro: %s" %
(pkg, get_package_names(rosdistro)))
try:
version = get_pkg_version(rosdistro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
rosdistro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe which was removed by clean_ros_recipe_dirs
prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format(
yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name, recipe)
existing = overlay.repo.git.status('--porcelain', '--', prefix)
if existing:
# The git status --porcelain output will look like this:
# D meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb
# we want just the path with filename
if len(existing.split('\n')) > 1:
warn('More than 1 recipe was output by "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
if existing.split()[0] != 'D':
err('Unexpected output from "git status --porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.format(
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing.split()[1]
else:
# If it isn't shown in git status, it could still exist as normal
# unchanged file when --only option is being used
import glob
existing = glob.glob('{0}/{1}'.format(repo_dir, prefix))
if existing:
if len(existing) > 1:
err('More than 1 recipe was output by "git status '
'--porcelain '
'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'.
format(yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name, recipe, existing))
existing = existing[0]
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(rosdistro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(rosdistro.name), rosdistro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(rosdistro.name),
rosdistro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
def regenerate_pkg(overlay, pkg, distro, preserve_existing, srcrev_cache,
skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
)
existing = glob.glob('{}_*.bb'.format(prefix))
previous_version = None
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
elif existing:
existing = existing[0]
overlay.repo.remove_file(existing, True)
idx_version = existing.rfind('_') + len('_')
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(distro, pkg, srcrev_cache, skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except Exception as e:
err('Failed generating recipe for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
make_dir("{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
repo_dir, yoctoRecipe._get_ros_version(distro.name), distro.name,
component_name))
success_msg = 'Successfully generated recipe for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
'ros{1}-{4}_{5}.bb'.format(
repo_dir,
yoctoRecipe._get_ros_version(distro.name),
distro.name,
component_name,
recipe,
version
)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes['ros{0}-{1}'.format(
yoctoRecipe._get_ros_version(distro.name),
recipe)] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, [], None
return current, previous_version, recipe
def regenerate_pkg(overlay, pkg, distro, preserve_existing, tar_dir, md5_cache,
sha256_cache, skip_keys):
pkg_names = get_package_names(distro)[0]
if pkg not in pkg_names:
yoctoRecipe.not_generated_recipes.add(pkg)
raise RuntimeError("Unknown package '%s'" % pkg)
try:
version = get_pkg_version(distro, pkg, is_oe=True)
except KeyError as ke:
yoctoRecipe.not_generated_recipes.add(pkg)
raise ke
repo_dir = overlay.repo.repo_dir
component_name = yoctoRecipe.convert_to_oe_name(
distro.release_packages[pkg].repository_name)
recipe = yoctoRecipe.convert_to_oe_name(pkg)
# check for an existing recipe
glob_pattern = '{0}/generated-recipes-{1}/{2}/{3}*.bb'.format(
repo_dir, distro.name, component_name, recipe)
existing = glob.glob(glob_pattern)
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skipping..." % pkg)
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0], True)
try:
current = oe_installer(distro, pkg, tar_dir, md5_cache, sha256_cache,
skip_keys)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
except Exception as e:
err('Failed generating installer for {}! {}'.format(pkg, str(e)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
try:
recipe_text = current.recipe_text()
except NoPkgXml as nopkg:
err("Could not fetch pkg! {}".format(str(nopkg)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
except KeyError as ke:
err("Failed to parse data for package {}! {}".format(pkg, str(ke)))
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
make_dir("{0}/generated-recipes-{1}/{2}".format(repo_dir, distro.name,
component_name))
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_file_name = '{0}/generated-recipes-{1}/{2}/{3}_{4}.bb'.format(
repo_dir, distro.name, component_name, recipe, version)
try:
with open('{0}'.format(recipe_file_name), "w") as recipe_file:
ok('Writing recipe {0}'.format(recipe_file_name))
recipe_file.write(recipe_text)
yoctoRecipe.generated_components.add(component_name)
yoctoRecipe.generated_recipes[recipe] = (version, component_name)
except Exception:
err("Failed to write recipe to disk!")
yoctoRecipe.not_generated_recipes.add(pkg)
return None, []
return current, []
def generate_rosdistro_conf(
basepath, distro, version, platforms, skip_keys=[]):
conf_dir = '{}/conf/ros-distro/include/{}/'.format(basepath, distro)
conf_file_name = 'generated-ros-distro.inc'
conf_path = '{}{}'.format(conf_dir, conf_file_name)
try:
make_dir(conf_dir)
with open(conf_path, 'w') as conf_file:
conf_file.write('# {}/{}\n'.format(distro, conf_file_name))
conf_file.write('# Generated by superflore -- DO NOT EDIT')
conf_file.write(
' (except ROS_DISTRO_METADATA_VERSION_REVISION)\n#\n')
conf_file.write(
'# Copyright ' + strftime("%Y", gmtime())
+ ' Open Source Robotics Foundation\n\n')
conf_file.write(
'# Increment every time meta-ros is released because of '
+ 'a manually created change, ie, NOT as a result of a '
+ 'superflore run (which\n# resets it to "0").')
conf_file.write(
'\nROS_DISTRO_METADATA_VERSION_REVISION = "0"\n')
conf_file.write(
'\nROS_SUPERFLORE_PROGRAM_VERSION = "{}"\n'
.format(get_superflore_version()))
conf_file.write('ROS_SUPERFLORE_GENERATION_SCHEME = "1"\n')
ros_version = yoctoRecipe._get_ros_version(distro)
conf_file.write(
'\nROS_DISTRO_TYPE = "ros{}"\n'.format(ros_version))
conf_file.write('ROS_VERSION = "{}"\n'.format(ros_version))
conf_file.write('# DO NOT OVERRIDE ROS_PYTHON_VERSION\n')
ros_python_version = 3
if ros_version == 1:
ros_python_version = 2
conf_file.write(
'ROS_PYTHON_VERSION = "{}"\n\n'.format(ros_python_version))
oe_skip_keys = map(
lambda skip_key: yoctoRecipe.convert_to_oe_name(skip_key),
skip_keys
)
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_SKIP_LIST', oe_skip_keys)
+ '\n')
conf_file.write(
'# Superflore was unable to generate recipes for these '
+ 'packages, eg, because their repositories are not on '
+ 'GitHub.\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATION_NOT_POSSIBLE',
yoctoRecipe.not_generated_recipes) + '\n')
conf_file.write(
'# Number of commits that will be returned by'
+ ' "git log files/ROS_DISTRO-cache.yaml" when the '
+ 'generated files are committed. This is\n# used for the'
+ ' third version field of DISTRO_VERSION.\n')
version = 1 if not version else len(version.splitlines()) + 1
conf_file.write(
'ROS_NUM_CACHE_YAML_COMMITS = "{}"'.format(version)
+ '\n\n')
conf_file.write(
'# Iterated values of '
+ 'ROS_DISTRO-cache.distribution_file.release_platforms.'
+ '<LINUX-DISTRO>.[ <NAME> ... ] .\n')
release_platforms = []
for p in sorted(platforms.items()):
for release in p[1]:
release_platforms.append(p[0] + '-' + release)
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_DISTRO_RELEASE_PLATFORMS', release_platforms) + '\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES',
yoctoRecipe.generated_recipes.keys()) + '\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPE_BASENAMES_WITH_COMPONENT',
[(yoctoRecipe.max_component_name - len(component)) * ' '
+ component + '/' + recipe + '_' + version
for recipe, (version, component)
in yoctoRecipe.generated_recipes.items()],
key=lambda recipe: recipe.split('/')[1].split('_')[0]))
conf_file.write(
'\n# What\'s built by packagegroup-ros-world. Does not '
+ 'include packages that appear solely in '
+ 'ROS_SUPERFLORE_GENERATED_BUILDTOOLS\n# (with a -native'
+ ' suffix) or ROS_SUPERFLORE_GENERATED_TESTS.\n')
recipes_set = set(yoctoRecipe.generated_recipes.keys())
test_deps = set(map(
lambda test_dep: yoctoRecipe.convert_to_oe_name(test_dep),
yoctoRecipe.generated_test_deps
- yoctoRecipe.generated_non_test_deps
))
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_WORLD_PACKAGES', recipes_set
- yoctoRecipe.generated_native_recipes - test_deps))
conf_file.write(
'\n# Packages found in the <buildtool_depend> and '
+ '<buildtool_export_depend> items, ie, ones for which a '
+ '-native is built. Does not\n# include those found in '
+ 'the ROS_EXEC_DEPENDS values in the recipes of build '
+ 'tools.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_BUILDTOOLS',
yoctoRecipe.generated_native_recipes) + '\n')
conf_file.write(yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_PLATFORM_PACKAGE_DEPENDENCIES',
yoctoRecipe.platform_deps))
conf_file.write(
'\n# Packages found only in <test_depend> items. Does not'
+ ' include those found only in the ROS_*_DEPENDS of '
+ 'recipes of tests.\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_TESTS',
test_deps) + '\n')
conf_file.write(
yoctoRecipe.generate_multiline_variable(
'ROS_SUPERFLORE_GENERATED_RECIPES_FOR_COMPONENTS',
yoctoRecipe.generated_components))
ok('Wrote {0}'.format(conf_path))
except OSError as e:
err('Failed to write conf {} to disk! {}'.format(conf_path, e))
raise e
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
pkgbuild_name =\
'/ros-{0}/{1}/{1}.pkgbuild'.format(distro.name, pkg)
pkgbuild_name = overlay.repo.repo_dir + pkgbuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
is_ros2 = get_distros()[distro.name]['distribution_type'] == 'ros2'
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]
patches = None
if os.path.exists(patch_path):
patches = [
f for f in glob.glob('%s/*.patch' % patch_path)
]
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing pkgbuild.
prefix = '{0}/ros-{1}/{2}/'.format(overlay.repo.repo_dir, distro.name, pkg)
existing = glob.glob('%s*.pkgbuild' % prefix)
previous_version = None
if preserve_existing and os.path.isfile(pkgbuild_name):
ok("pkgbuild for package '%s' up to date, skipping..." % pkg)
return None, [], None
elif existing:
overlay.repo.remove_file(existing[0])
previous_version = existing[0].lstrip(prefix).rstrip('.pkgbuild')
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg
)
overlay.repo.remove_file(manifest_file)
try:
current = arch_pkgbuild(distro, pkg, has_patches)
current.pkgbuild.name = pkg
current.pkgbuild.version = version
current.pkgbuild.patches = patches
current.pkgbuild.is_ros2 = is_ros2
except Exception as e:
err('Failed to generate pkgbuild for package {}!'.format(pkg))
raise e
try:
pkgbuild_text = current.pkgbuild_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.pkgbuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.pkgbuild.get_unresolved(), None
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir(
"{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg)
)
success_msg = 'Successfully generated pkgbuild for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
try:
pkgbuild_file = '{0}/ros-{1}/{2}/PKGBUILD'.format(
overlay.repo.repo_dir,
distro.name,
pkg,
)
ok(f"writing {pkgbuild_file}")
with open(pkgbuild_file, "w") as pkgbuild_file_f:
pkgbuild_file_f.write(pkgbuild_text)
except Exception as e:
err(f"Failed to write f{pkgbuild_file} to disk!")
raise e
return current, previous_version, pkg
