def test_replaces_regular_shebang(self):
shebang_scripts = [('python_shebang.sh', 'python_shebang_expected.sh'),
('template_shebang.sh', 'template_shebang.sh'),
('regular_python_shebang.sh',
'regular_python_shebang_expected.sh'),
('regular_python3_shebang.sh',
'regular_python3_shebang_expected.sh'),
('regular_node_shebang.sh',
'regular_node_shebang_expected.sh')]
for script in shebang_scripts:
regular_shebang_script = script[0]
expected_shebang_script = script[1]
assets_directory = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'assets')
shutil.copy(os.path.join(assets_directory, regular_shebang_script),
self.tmpdir)
update_shebang(self.tmpdir)
actual_file = os.path.join(self.tmpdir, regular_shebang_script)
expected_file = os.path.join(assets_directory,
expected_shebang_script)
assert filecmp.cmp(actual_file, expected_file), \
"{} did not match expected file {}".format(
regular_shebang_script, expected_shebang_script)
def create_workspace_overlay(install_base: str, workspace_staging_path: str,
overlay_path: str):
"""
Create overlay from user's built workspace install directory.
:param str install_base: Path to built workspace install directory
:param str workspace_staging_path: Path to stage the overlay build at
:param str overlay_path: Name of the overlay file (.tar.gz)
"""
ws_install_path = Path(workspace_staging_path) / 'opt' / 'built_workspace'
shutil.rmtree(workspace_staging_path, ignore_errors=True)
# install_base: Directory with built artifacts from the workspace
os.mkdir(workspace_staging_path)
shellscript_dest = Path(workspace_staging_path) / 'setup.sh'
_render_template(Path('v2_workspace_setup.jinja2.sh'), shellscript_dest,
_CONTEXT_VAR_SH)
shellscript_dest.chmod(0o755)
shellscript_dest_bash = Path(workspace_staging_path) / 'setup.bash'
_render_template(Path('v2_workspace_setup.jinja2.sh'),
shellscript_dest_bash, _CONTEXT_VAR_BASH)
shellscript_dest_bash.chmod(0o755)
shutil.copytree(install_base, str(ws_install_path))
# This is required because python3 shell scripts use a hard
# coded shebang
update_shebang(workspace_staging_path)
recursive_tar_gz_in_path(Path(overlay_path), Path(workspace_staging_path))
def create_workspace_overlay(install_base: str, workspace_staging_path: str,
overlay_path: str):
"""
Create overlay from user's built workspace install directory.
:param str install_base: Path to built workspace install directory
:param str workspace_staging_path: Path to stage the overlay build at
:param str overlay_path: Name of the overlay file (.tar.gz)
"""
workspace_install_path = os.path.join(workspace_staging_path, 'opt',
'built_workspace')
shutil.rmtree(workspace_staging_path, ignore_errors=True)
assets_directory = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'assets')
shellscript_path = os.path.join(assets_directory, 'v2_workspace_setup.sh')
# install_base: Directory with built artifacts from the workspace
os.mkdir(workspace_staging_path)
shutil.copy2(shellscript_path,
os.path.join(workspace_staging_path, 'setup.sh'))
shutil.copytree(install_base, workspace_install_path)
# This is required because python3 shell scripts use a hard
# coded shebang
update_shebang(workspace_staging_path)
recursive_tar_gz_in_path(overlay_path, workspace_staging_path)
def run_installers(self, *, include_sources=False):
"""
Invoke all installers to install packages into the bundle.
:param include_sources: creates a sources tarball
for all packages being installed that have sources
available
:return: true if all packages installed in the
bundle are the same as the previous run
"""
print('Collecting dependency information...')
logger.info('Collecting dependency information...')
print('Fetching and installing dependencies...')
logger.info('Fetching and installing dependencies...')
installer_metadata = {}
for name, installer in self.installers.items():
installer_metadata[name] = installer.install()
installer_metadata_string = json.dumps(installer_metadata,
sort_keys=True)
installer_metadata_path = self._path_context.installer_metadata_path()
dependency_match = False
if os.path.exists(installer_metadata_path):
with open(installer_metadata_path, 'r') as f:
previous_metadata = f.read()
if previous_metadata == installer_metadata_string:
dependency_match = True
with open(installer_metadata_path, 'w') as f:
f.write(installer_metadata_string)
if include_sources:
sources_tar_gz_path = self._path_context.sources_tar_gz_path()
with tarfile.open(sources_tar_gz_path, 'w:gz',
compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
update_symlinks(self.prefix_path)
# TODO: Update pkgconfig files?
update_shebang(self.prefix_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(self.prefix_path)
return dependency_match
def test_replaces_py3_shebang(self):
regular_python_shebang_script = 'regular_python3_shebang.sh'
assets_directory = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'assets')
shutil.copy(
os.path.join(assets_directory, regular_python_shebang_script),
self.tmpdir)
update_shebang(self.tmpdir)
actual_file = os.path.join(self.tmpdir, regular_python_shebang_script)
expected_file = os.path.join(assets_directory,
'regular_python3_shebang_expected.sh')
assert filecmp.cmp(actual_file, expected_file)
def _manage_dependencies(self, context, path_context, upgrade_deps_graph):
bundle_base = path_context.bundle_base()
check_and_mark_install_layout(path_context.install_base(),
merge_install=context.args.merge_install)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run', ))
installers = self._setup_installers(context, path_context)
print('Checking if dependency tarball exists...')
logger.info('Checking if dependency tarball exists...')
if not os.path.exists(path_context.dependencies_tar_gz_path()):
self._check_package_dependency_update(path_context, decorators)
self._check_installer_dependency_update(context, decorators,
installers, path_context)
elif upgrade_deps_graph:
self._check_package_dependency_update(path_context, decorators)
print('Checking if dependency graph has changed since last '
'bundle...')
logger.info('Checking if dependency graph has changed since last'
' bundle...')
if self._check_installer_dependency_update(context, decorators,
installers,
path_context):
print('All dependencies in dependency graph not changed, '
'skipping dependencies update...')
logger.info('All dependencies in dependency graph not changed,'
' skipping dependencies update...')
return False
else:
print('Checking if local dependencies have changed since last'
' bundle...')
logger.info(
'Checking if local dependencies have changed since last'
' bundle...')
if self._check_package_dependency_update(path_context, decorators):
print('Local dependencies not changed, skipping dependencies'
' update...')
logger.info(
'Local dependencies not changed, skipping dependencies'
' update...')
return False
self._check_installer_dependency_update(context, decorators,
installers, path_context)
if context.args.include_sources:
sources_tar_gz_path = path_context.sources_tar_gz_path()
with tarfile.open(sources_tar_gz_path, 'w:gz',
compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
staging_path = path_context.staging_path()
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return True
def _manage_dependencies(self, context, path_context, upgrade_deps_graph):
bundle_base = path_context.bundle_base()
bundle_version = context.args.bundle_version
check_and_mark_install_layout(path_context.install_base(),
merge_install=context.args.merge_install)
# This must be first for backwards compatibility
# reasons. We assume the folder was previously
# used for v1 if it exists.
check_and_mark_bundle_version(bundle_base,
this_bundle_version=bundle_version)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run', ))
if len(decorators) == 0:
estr = 'We did not find any packages to add to the '\
'bundle. This might be because you are not '\
'in the right directory, or your workspace is '\
'not setup correctly for colcon. Please see '\
'https://github.com/colcon/colcon-ros-bundle/issues/13' \
'for some possible suggestions. If you are still having ' \
'trouble please post to our' \
'issues: https://github.com/colcon/colcon-bundle/issues '\
'and we will be happy to help.'
raise RuntimeError(estr)
installers = self._setup_installers(context, path_context)
print('Checking if dependency tarball exists...')
logger.info('Checking if dependency tarball exists...')
if not os.path.exists(path_context.dependencies_tar_gz_path()):
self._check_package_dependency_update(path_context, decorators)
self._check_installer_dependency_update(context, decorators,
installers, path_context)
elif upgrade_deps_graph:
self._check_package_dependency_update(path_context, decorators)
print('Checking if dependency graph has changed since last '
'bundle...')
logger.info('Checking if dependency graph has changed since last'
' bundle...')
if self._check_installer_dependency_update(context, decorators,
installers,
path_context):
print('All dependencies in dependency graph not changed, '
'skipping dependencies update...')
logger.info('All dependencies in dependency graph not changed,'
' skipping dependencies update...')
return False
else:
print('Checking if local dependencies have changed since last'
' bundle...')
logger.info(
'Checking if local dependencies have changed since last'
' bundle...')
if self._check_package_dependency_update(path_context, decorators):
print('Local dependencies not changed, skipping dependencies'
' update...')
logger.info(
'Local dependencies not changed, skipping dependencies'
' update...')
return False
self._check_installer_dependency_update(context, decorators,
installers, path_context)
if context.args.include_sources:
sources_tar_gz_path = path_context.sources_tar_gz_path()
with tarfile.open(sources_tar_gz_path, 'w:gz',
compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
staging_path = path_context.staging_path()
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return True
def _manage_dependencies(self, context,
install_base, bundle_base,
staging_path, installer_metadata_path):
check_and_mark_install_layout(
install_base, merge_install=context.args.merge_install)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run',))
installers = self._setup_installers(context)
print('Collecting dependency information...')
jobs = self._get_jobs(context.args, installers, decorators)
rc = execute_jobs(context, jobs)
if rc != 0:
return rc
print('Fetching and installing dependencies...')
installer_metadata = {}
for name, installer in installers.items():
installer_metadata[name] = installer.install()
installer_metadata_string = json.dumps(installer_metadata,
sort_keys=True)
dependencies_changed = True
if os.path.exists(installer_metadata_path):
with open(installer_metadata_path, 'r') as f:
previous_metadata = f.read()
if previous_metadata == installer_metadata_string:
dependencies_changed = False
with open(installer_metadata_path, 'w') as f:
f.write(installer_metadata_string)
if context.args.include_sources and dependencies_changed:
sources_tar_gz_path = os.path.join(bundle_base, 'sources.tar.gz')
with tarfile.open(
sources_tar_gz_path, 'w:gz', compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(
file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
if dependencies_changed:
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return dependencies_changed