def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(context.args.install_base,
merge_install=context.args.merge_install)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs, unselected_packages = self._get_jobs(context.args, decorators,
install_base)
on_error = OnError.interrupt \
if not context.args.continue_on_error else OnError.skip_downstream
def post_unselected_packages(*, event_queue):
nonlocal unselected_packages
names = [pkg.name for pkg in unselected_packages]
for name in sorted(names):
event_queue.put((JobUnselected(name), None))
rc = execute_jobs(context,
jobs,
on_error=on_error,
pre_execution_callback=post_unselected_packages)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
jobs, unselected_packages = self._get_jobs(context.args, decorators)
def post_unselected_packages(*, event_queue):
nonlocal unselected_packages
names = [pkg.name for pkg in unselected_packages]
for name in sorted(names):
event_queue.put((JobUnselected(name), None))
on_error = OnError.continue_
rc = execute_jobs(context,
jobs,
on_error=on_error,
pre_execution_callback=post_unselected_packages)
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
base_handler_extensions = get_base_handler_extensions()
base_paths = set()
args = context.args
decorators = get_packages(args)
for base_name in args.base_select:
if base_name in base_handler_extensions:
base_handler_extension = base_handler_extensions[base_name]
for decorator in decorators:
if not decorator.selected:
continue
pkg = decorator.descriptor
package_paths = \
base_handler_extension.get_package_paths(
args=args, pkg=pkg)
for package_path in package_paths:
package_path = Path(os.path.abspath(package_path))
base_paths.add(package_path)
else:
logger.warning(
"No base handler for selction '{base_name}'"
.format_map(locals()))
confirmed = args.yes
clean_paths(base_paths, confirmed)
return 0
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(os.path.join(
os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
return execute_jobs(
context, jobs, abort_on_error=context.args.abort_on_error)
def test_get_packages():
args = Namespace()
d1 = PackageDescriptor('/some/path')
d1.name = 'one'
d2 = PackageDescriptor('/other/path')
d2.name = 'two'
with patch('colcon_core.package_selection.discover_packages',
return_value=[d1, d2]):
decos = get_packages(args)
assert len(decos) == 2
assert decos[0].descriptor.name == 'one'
assert decos[0].selected is True
assert decos[1].descriptor.name == 'two'
assert decos[1].selected is True
d2.name = 'one'
with patch('colcon_core.package_selection.discover_packages',
return_value=[d1, d2]):
with pytest.raises(RuntimeError) as e:
get_packages(args)
assert 'Duplicate package names not supported:' in str(e.value)
assert '- one:' in str(e.value)
assert '- {sep}some{sep}path'.format(sep=os.sep) in str(e.value)
assert '- {sep}other{sep}path'.format(sep=os.sep) in str(e.value)
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(
context.args.install_base,
merge_install=context.args.merge_install)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(os.path.join(
os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
if context.args.return_code_on_test_failure:
# watch published events on all jobs to detect any test failures
any_test_failures = False
def check_for_test_failures(put_event_into_queue):
nonlocal any_test_failures
def put_event_into_queue_(self, event):
nonlocal any_test_failures
nonlocal put_event_into_queue
if isinstance(event, TestFailure):
any_test_failures = True
return put_event_into_queue(event)
return put_event_into_queue_
for job in jobs.values():
job.put_event_into_queue = types.MethodType(
check_for_test_failures(job.put_event_into_queue), job)
on_error = OnError.continue_ \
if not context.args.abort_on_error else OnError.interrupt
rc = execute_jobs(context, jobs, on_error=on_error)
if context.args.return_code_on_test_failure:
if not rc and any_test_failures:
return 1
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
rc = execute_jobs(context, jobs)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(context.args.install_base,
merge_install=context.args.merge_install)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
on_error = OnError.interrupt \
if not context.args.continue_on_error else OnError.skip_downstream
rc = execute_jobs(context, jobs, on_error=on_error)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def _manage_dependencies(self, context,
path_context,
upgrade_deps_graph):
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run',))
if len(decorators) == 0:
estr = 'We did not find any packages to add to the '\
'bundle. This might be because you are not '\
'in the right directory, or your workspace is '\
'not setup correctly for colcon. Please see '\
'https://github.com/colcon/colcon-ros-bundle/issues/13' \
'for some possible suggestions. If you are still having ' \
'trouble please post to our' \
'issues: https://github.com/colcon/colcon-bundle/issues '\
'and we will be happy to help.'
raise RuntimeError(estr)
self._installer_manager.setup_installers(context)
print('Checking if dependency tarball exists...')
logger.info('Checking if dependency tarball exists...')
jobs = self._get_jobs(context.args,
self._installer_manager.installers,
decorators)
rc = execute_jobs(context, jobs)
if rc != 0:
return rc
direct_dependencies_changed = package_dependencies_changed(
path_context, decorators)
installer_parameters_changed = \
self._installer_manager.cache_invalid()
if not os.path.exists(path_context.dependencies_overlay_path()):
self._installer_manager.run_installers(
include_sources=context.args.include_sources)
return True
elif upgrade_deps_graph:
print('Checking if dependency graph has changed since last '
'bundle...')
logger.info('Checking if dependency graph has changed since last'
' bundle...')
if self._installer_manager.run_installers(
include_sources=context.args.include_sources):
print('All dependencies in dependency graph not changed, '
'skipping dependencies update...')
logger.info('All dependencies in dependency graph not changed,'
' skipping dependencies update...')
return False
else:
print('Checking if local dependencies have changed since last'
' bundle...')
logger.info(
'Checking if local dependencies have changed since last'
' bundle...')
if not direct_dependencies_changed and \
not installer_parameters_changed:
print('Local dependencies not changed, skipping dependencies'
' update...')
logger.info(
'Local dependencies not changed, skipping dependencies'
' update...')
return False
self._installer_manager.run_installers(
include_sources=context.args.include_sources)
return True
def _manage_dependencies(self, context, path_context, upgrade_deps_graph):
bundle_base = path_context.bundle_base()
check_and_mark_install_layout(path_context.install_base(),
merge_install=context.args.merge_install)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run', ))
installers = self._setup_installers(context, path_context)
print('Checking if dependency tarball exists...')
logger.info('Checking if dependency tarball exists...')
if not os.path.exists(path_context.dependencies_tar_gz_path()):
self._check_package_dependency_update(path_context, decorators)
self._check_installer_dependency_update(context, decorators,
installers, path_context)
elif upgrade_deps_graph:
self._check_package_dependency_update(path_context, decorators)
print('Checking if dependency graph has changed since last '
'bundle...')
logger.info('Checking if dependency graph has changed since last'
' bundle...')
if self._check_installer_dependency_update(context, decorators,
installers,
path_context):
print('All dependencies in dependency graph not changed, '
'skipping dependencies update...')
logger.info('All dependencies in dependency graph not changed,'
' skipping dependencies update...')
return False
else:
print('Checking if local dependencies have changed since last'
' bundle...')
logger.info(
'Checking if local dependencies have changed since last'
' bundle...')
if self._check_package_dependency_update(path_context, decorators):
print('Local dependencies not changed, skipping dependencies'
' update...')
logger.info(
'Local dependencies not changed, skipping dependencies'
' update...')
return False
self._check_installer_dependency_update(context, decorators,
installers, path_context)
if context.args.include_sources:
sources_tar_gz_path = path_context.sources_tar_gz_path()
with tarfile.open(sources_tar_gz_path, 'w:gz',
compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
staging_path = path_context.staging_path()
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return True
def _manage_dependencies(self, context, path_context, upgrade_deps_graph):
bundle_base = path_context.bundle_base()
bundle_version = context.args.bundle_version
check_and_mark_install_layout(path_context.install_base(),
merge_install=context.args.merge_install)
# This must be first for backwards compatibility
# reasons. We assume the folder was previously
# used for v1 if it exists.
check_and_mark_bundle_version(bundle_base,
this_bundle_version=bundle_version)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run', ))
if len(decorators) == 0:
estr = 'We did not find any packages to add to the '\
'bundle. This might be because you are not '\
'in the right directory, or your workspace is '\
'not setup correctly for colcon. Please see '\
'https://github.com/colcon/colcon-ros-bundle/issues/13' \
'for some possible suggestions. If you are still having ' \
'trouble please post to our' \
'issues: https://github.com/colcon/colcon-bundle/issues '\
'and we will be happy to help.'
raise RuntimeError(estr)
installers = self._setup_installers(context, path_context)
print('Checking if dependency tarball exists...')
logger.info('Checking if dependency tarball exists...')
if not os.path.exists(path_context.dependencies_tar_gz_path()):
self._check_package_dependency_update(path_context, decorators)
self._check_installer_dependency_update(context, decorators,
installers, path_context)
elif upgrade_deps_graph:
self._check_package_dependency_update(path_context, decorators)
print('Checking if dependency graph has changed since last '
'bundle...')
logger.info('Checking if dependency graph has changed since last'
' bundle...')
if self._check_installer_dependency_update(context, decorators,
installers,
path_context):
print('All dependencies in dependency graph not changed, '
'skipping dependencies update...')
logger.info('All dependencies in dependency graph not changed,'
' skipping dependencies update...')
return False
else:
print('Checking if local dependencies have changed since last'
' bundle...')
logger.info(
'Checking if local dependencies have changed since last'
' bundle...')
if self._check_package_dependency_update(path_context, decorators):
print('Local dependencies not changed, skipping dependencies'
' update...')
logger.info(
'Local dependencies not changed, skipping dependencies'
' update...')
return False
self._check_installer_dependency_update(context, decorators,
installers, path_context)
if context.args.include_sources:
sources_tar_gz_path = path_context.sources_tar_gz_path()
with tarfile.open(sources_tar_gz_path, 'w:gz',
compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
staging_path = path_context.staging_path()
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return True
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(context.args.install_base,
merge_install=context.args.merge_install)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs, unselected_packages = self._get_jobs(context.args, decorators,
install_base)
underlay_packages = {}
for prefix_path in get_chained_prefix_path():
packages = find_installed_packages(Path(prefix_path))
if packages:
for pkg, path in packages.items():
if pkg not in underlay_packages:
underlay_packages[pkg] = []
underlay_packages[pkg].append(str(path))
override_messages = {}
for overlay_package in jobs.keys():
if overlay_package in underlay_packages:
if overlay_package not in context.args.allow_overriding:
override_messages[overlay_package] = (
"'{overlay_package}'".format_map(locals()) +
' is in: ' +
', '.join(underlay_packages[overlay_package]))
if override_messages:
override_msg = (
'Some selected packages are already built in one or more'
' underlay workspaces:'
'\n\t' + '\n\t'.join(override_messages.values()) +
'\nIf a package in a merged underlay workspace is overridden'
' and it installs headers, then all packages in the overlay'
' must sort their include directories by workspace order.'
' Failure to do so may result in build failures or undefined'
' behavior at run time.'
'\nIf the overridden package is used by another package'
' in any underlay, then the overriding package in the'
' overlay must be API and ABI compatible or undefined'
' behavior at run time may occur.'
'\n\nIf you understand the risks and want to override a'
' package anyways, add the following to the command'
' line:'
'\n\t--allow-overriding ' +
' '.join(sorted(override_messages.keys())))
logger.warn(override_msg +
'\n\nThis may be promoted to an error in a'
' future release of colcon-core.')
on_error = OnError.interrupt \
if not context.args.continue_on_error else OnError.skip_downstream
def post_unselected_packages(*, event_queue):
nonlocal unselected_packages
names = [pkg.name for pkg in unselected_packages]
for name in sorted(names):
event_queue.put((JobUnselected(name), None))
rc = execute_jobs(context,
jobs,
on_error=on_error,
pre_execution_callback=post_unselected_packages)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def _manage_dependencies(self, context,
install_base, bundle_base,
staging_path, installer_metadata_path):
check_and_mark_install_layout(
install_base, merge_install=context.args.merge_install)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run',))
installers = self._setup_installers(context)
print('Collecting dependency information...')
jobs = self._get_jobs(context.args, installers, decorators)
rc = execute_jobs(context, jobs)
if rc != 0:
return rc
print('Fetching and installing dependencies...')
installer_metadata = {}
for name, installer in installers.items():
installer_metadata[name] = installer.install()
installer_metadata_string = json.dumps(installer_metadata,
sort_keys=True)
dependencies_changed = True
if os.path.exists(installer_metadata_path):
with open(installer_metadata_path, 'r') as f:
previous_metadata = f.read()
if previous_metadata == installer_metadata_string:
dependencies_changed = False
with open(installer_metadata_path, 'w') as f:
f.write(installer_metadata_string)
if context.args.include_sources and dependencies_changed:
sources_tar_gz_path = os.path.join(bundle_base, 'sources.tar.gz')
with tarfile.open(
sources_tar_gz_path, 'w:gz', compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(
file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
if dependencies_changed:
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return dependencies_changed
