def test_execute_jobs():
context = Mock()
context.args = Mock()
context.args.event_handlers = None
task_context = Mock()
task_context.pkg = Mock()
task_context.pkg.name = 'name'
jobs = {
'one':
Job(identifier='id',
dependencies=set(),
task=None,
task_context=task_context)
}
event_reactor = Mock()
event_reactor.__enter__ = lambda self: self
event_reactor.__exit__ = lambda self, *args: None
with patch('colcon_core.executor.create_event_reactor',
return_value=event_reactor):
with EntryPointContext(extension1=Extension1, extension2=Extension2):
# no extension selected
with pytest.raises(AssertionError):
execute_jobs(context, jobs)
# execute method not implemented and sending skipped job event
context.args.executor = 'extension2'
with patch('colcon_core.executor.logger.error') as error:
rc = execute_jobs(context, jobs)
assert rc == 1
assert error.call_count == 1
assert len(error.call_args[0]) == 1
assert error.call_args[0][0].startswith(
"Exception in executor extension 'extension2': \n")
assert event_reactor.get_queue().put.call_count == 2
assert isinstance(
event_reactor.get_queue().put.call_args_list[0][0][0][0],
JobQueued)
assert isinstance(
event_reactor.get_queue().put.call_args_list[1][0][0][0],
JobSkipped)
# successful execution
event_reactor.get_queue().put.reset_mock()
jobs['one'].returncode = 0
extensions = get_executor_extensions()
extensions[110]['extension2'].execute = \
lambda args, jobs, on_error: 0
callback = Mock()
rc = execute_jobs(context,
jobs,
on_error=OnError.interrupt,
pre_execution_callback=callback)
assert rc == 0
assert event_reactor.get_queue().put.call_count == 1
assert isinstance(event_reactor.get_queue().put.call_args[0][0][0],
JobQueued)
assert callback.call_count == 1
def _check_installer_dependency_update(self, context, decorators,
installers, path_context):
print('Collecting dependency information...')
logger.info('Collecting dependency information...')
jobs = self._get_jobs(context.args, installers, decorators)
rc = execute_jobs(context, jobs)
if rc != 0:
return rc
print('Fetching and installing dependencies...')
logger.info('Fetching and installing dependencies...')
installer_metadata = {}
for name, installer in installers.items():
installer_metadata[name] = installer.install()
installer_metadata_string = json.dumps(installer_metadata,
sort_keys=True)
installer_metadata_path = path_context.installer_metadata_path()
dependency_match = False
if os.path.exists(installer_metadata_path):
with open(installer_metadata_path, 'r') as f:
previous_metadata = f.read()
if previous_metadata == installer_metadata_string:
dependency_match = True
with open(installer_metadata_path, 'w') as f:
f.write(installer_metadata_string)
return dependency_match
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
jobs, unselected_packages = self._get_jobs(context.args, decorators)
def post_unselected_packages(*, event_queue):
nonlocal unselected_packages
names = [pkg.name for pkg in unselected_packages]
for name in sorted(names):
event_queue.put((JobUnselected(name), None))
on_error = OnError.continue_
rc = execute_jobs(context,
jobs,
on_error=on_error,
pre_execution_callback=post_unselected_packages)
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(context.args.install_base,
merge_install=context.args.merge_install)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs, unselected_packages = self._get_jobs(context.args, decorators,
install_base)
on_error = OnError.interrupt \
if not context.args.continue_on_error else OnError.skip_downstream
def post_unselected_packages(*, event_queue):
nonlocal unselected_packages
names = [pkg.name for pkg in unselected_packages]
for name in sorted(names):
event_queue.put((JobUnselected(name), None))
rc = execute_jobs(context,
jobs,
on_error=on_error,
pre_execution_callback=post_unselected_packages)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(os.path.join(
os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
return execute_jobs(
context, jobs, abort_on_error=context.args.abort_on_error)
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(
context.args.install_base,
merge_install=context.args.merge_install)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(os.path.join(
os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
if context.args.return_code_on_test_failure:
# watch published events on all jobs to detect any test failures
any_test_failures = False
def check_for_test_failures(put_event_into_queue):
nonlocal any_test_failures
def put_event_into_queue_(self, event):
nonlocal any_test_failures
nonlocal put_event_into_queue
if isinstance(event, TestFailure):
any_test_failures = True
return put_event_into_queue(event)
return put_event_into_queue_
for job in jobs.values():
job.put_event_into_queue = types.MethodType(
check_for_test_failures(job.put_event_into_queue), job)
on_error = OnError.continue_ \
if not context.args.abort_on_error else OnError.interrupt
rc = execute_jobs(context, jobs, on_error=on_error)
if context.args.return_code_on_test_failure:
if not rc and any_test_failures:
return 1
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
rc = execute_jobs(context, jobs)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(context.args.install_base,
merge_install=context.args.merge_install)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs = self._get_jobs(context.args, decorators, install_base)
on_error = OnError.interrupt \
if not context.args.continue_on_error else OnError.skip_downstream
rc = execute_jobs(context, jobs, on_error=on_error)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def _manage_dependencies(self, context,
path_context,
upgrade_deps_graph):
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run',))
if len(decorators) == 0:
estr = 'We did not find any packages to add to the '\
'bundle. This might be because you are not '\
'in the right directory, or your workspace is '\
'not setup correctly for colcon. Please see '\
'https://github.com/colcon/colcon-ros-bundle/issues/13' \
'for some possible suggestions. If you are still having ' \
'trouble please post to our' \
'issues: https://github.com/colcon/colcon-bundle/issues '\
'and we will be happy to help.'
raise RuntimeError(estr)
self._installer_manager.setup_installers(context)
print('Checking if dependency tarball exists...')
logger.info('Checking if dependency tarball exists...')
jobs = self._get_jobs(context.args,
self._installer_manager.installers,
decorators)
rc = execute_jobs(context, jobs)
if rc != 0:
return rc
direct_dependencies_changed = package_dependencies_changed(
path_context, decorators)
installer_parameters_changed = \
self._installer_manager.cache_invalid()
if not os.path.exists(path_context.dependencies_overlay_path()):
self._installer_manager.run_installers(
include_sources=context.args.include_sources)
return True
elif upgrade_deps_graph:
print('Checking if dependency graph has changed since last '
'bundle...')
logger.info('Checking if dependency graph has changed since last'
' bundle...')
if self._installer_manager.run_installers(
include_sources=context.args.include_sources):
print('All dependencies in dependency graph not changed, '
'skipping dependencies update...')
logger.info('All dependencies in dependency graph not changed,'
' skipping dependencies update...')
return False
else:
print('Checking if local dependencies have changed since last'
' bundle...')
logger.info(
'Checking if local dependencies have changed since last'
' bundle...')
if not direct_dependencies_changed and \
not installer_parameters_changed:
print('Local dependencies not changed, skipping dependencies'
' update...')
logger.info(
'Local dependencies not changed, skipping dependencies'
' update...')
return False
self._installer_manager.run_installers(
include_sources=context.args.include_sources)
return True
def main(self, *, context):
check_and_mark_build_tool(context.args.build_base)
lcov_base_abspath = Path(os.path.abspath(context.args.lcov_base))
lcov_base_abspath.mkdir(exist_ok=True)
gcc_pkgs = self._get_gcc_packages(context,
additional_argument_names=['*'])
jobs = OrderedDict()
for pkg in gcc_pkgs:
task_context = TaskContext(pkg=pkg,
args=context.args,
dependencies=OrderedDict())
if context.args.zero_counters:
extension = LcovZeroCountersTask()
else:
extension = LcovCaptureTask()
extension.PACKAGE_TYPE = pkg.type
job = Job(
identifier=pkg.name,
dependencies=set(), # Can be generated in any order
task=extension,
task_context=task_context)
jobs[pkg.name] = job
rc = execute_jobs(context, jobs)
if context.args.initial or context.args.zero_counters:
return rc
print("\nCalculating total coverage... ")
total_output_file = str(lcov_base_abspath / 'total_coverage.info')
if rc == 0:
output_files = []
for pkg in gcc_pkgs:
output_file = os.path.abspath(
os.path.join(context.args.build_base, pkg.name,
'coverage.info'))
if os.stat(output_file).st_size != 0:
output_files.append(output_file)
if len(output_files) == 0:
logger.error(
'No valid coverage.info files found. Did you run tests?')
return 1
rc = lcov_add(context,
output_files,
total_output_file,
verbose=context.args.verbose)
if rc != 0:
return rc
if context.args.filter:
print("\nApplying filters... ")
rc = lcov_remove(context, total_output_file)
if rc != 0:
return rc
print("\nGenerating HTML: ", end='')
# Check that genhtml exists
if GENHTML_EXECUTABLE is None:
raise RuntimeError("Could not find 'genhtml' executable")
# Generate html
cmd = [
GENHTML_EXECUTABLE, '--quiet', '--output-directory',
str(lcov_base_abspath), total_output_file, '--config-file',
str(context.args.lcov_config_file)
]
if CPP_FILT_EXECUTABLE is not None:
cmd.extend(['--demangle-cpp'])
# Strip paths to packages
for path in context.args.base_paths:
cmd.extend(['--prefix', str(os.path.abspath(path))])
rc = subprocess.run(cmd).returncode
print("Done")
return rc
def main(self, *, context): # noqa: D102
check_and_mark_build_tool(context.args.build_base)
check_and_mark_install_layout(context.args.install_base,
merge_install=context.args.merge_install)
self._create_paths(context.args)
decorators = get_packages(
context.args,
additional_argument_names=self.task_argument_destinations,
recursive_categories=('run', ))
install_base = os.path.abspath(
os.path.join(os.getcwd(), context.args.install_base))
jobs, unselected_packages = self._get_jobs(context.args, decorators,
install_base)
underlay_packages = {}
for prefix_path in get_chained_prefix_path():
packages = find_installed_packages(Path(prefix_path))
if packages:
for pkg, path in packages.items():
if pkg not in underlay_packages:
underlay_packages[pkg] = []
underlay_packages[pkg].append(str(path))
override_messages = {}
for overlay_package in jobs.keys():
if overlay_package in underlay_packages:
if overlay_package not in context.args.allow_overriding:
override_messages[overlay_package] = (
"'{overlay_package}'".format_map(locals()) +
' is in: ' +
', '.join(underlay_packages[overlay_package]))
if override_messages:
override_msg = (
'Some selected packages are already built in one or more'
' underlay workspaces:'
'\n\t' + '\n\t'.join(override_messages.values()) +
'\nIf a package in a merged underlay workspace is overridden'
' and it installs headers, then all packages in the overlay'
' must sort their include directories by workspace order.'
' Failure to do so may result in build failures or undefined'
' behavior at run time.'
'\nIf the overridden package is used by another package'
' in any underlay, then the overriding package in the'
' overlay must be API and ABI compatible or undefined'
' behavior at run time may occur.'
'\n\nIf you understand the risks and want to override a'
' package anyways, add the following to the command'
' line:'
'\n\t--allow-overriding ' +
' '.join(sorted(override_messages.keys())))
logger.warn(override_msg +
'\n\nThis may be promoted to an error in a'
' future release of colcon-core.')
on_error = OnError.interrupt \
if not context.args.continue_on_error else OnError.skip_downstream
def post_unselected_packages(*, event_queue):
nonlocal unselected_packages
names = [pkg.name for pkg in unselected_packages]
for name in sorted(names):
event_queue.put((JobUnselected(name), None))
rc = execute_jobs(context,
jobs,
on_error=on_error,
pre_execution_callback=post_unselected_packages)
self._create_prefix_scripts(install_base, context.args.merge_install)
return rc
def main(self, *, context): # noqa: D102
build_base = context.args.build_base
check_and_mark_build_tool(build_base)
# Check once if the 'coverage' command is available,
# otherwise we will need to fall back on using the Python module
has_command = has_coverage_command()
logger.info("'coverage' command available: {has_command}".format_map(
locals()))
# Get packages
coveragepy_pkgs = self._get_coveragepy_packages(context)
if not coveragepy_pkgs:
logger.warning('No packages selected or found')
return 0
# Combine each package's .coverage files
jobs = OrderedDict()
for pkg in coveragepy_pkgs:
task_context = TaskContext(
pkg=pkg,
args=context.args,
dependencies=OrderedDict(),
)
task = CoveragePyTask(has_command)
job = Job(
identifier=pkg.name,
dependencies=set(),
task=task,
task_context=task_context,
)
jobs[pkg.name] = job
rc = execute_jobs(context, jobs)
# Get all packages' .coverage files
coverage_files = [
str(
Path(
CoveragePyTask.get_package_combine_dir(
build_base, pkg.name)) / '.coverage')
for pkg in coveragepy_pkgs
]
# Filter out non-existing files in case processing failed for some packages
coverage_files = list(filter(os.path.exists, coverage_files))
if 0 == len(coverage_files):
logger.warning('No coverage files found')
return 0
logger.info('Coverage files: {coverage_files}'.format_map(locals()))
# Combine .coverage files
coveragepy_base_dir = str(os.path.abspath(
context.args.coveragepy_base))
Path(coveragepy_base_dir).mkdir(exist_ok=True)
rc, stdout, _ = coverage_combine(coverage_files, coveragepy_base_dir,
has_command)
if 0 == rc and context.args.verbose:
# Print report
rc, stdout, _ = coverage_report(
coveragepy_base_dir,
context.args.coverage_report_args,
has_command,
)
if 0 == rc:
print('\n' + stdout)
# Generate HTML report
rc, stdout, _ = coverage_html(
coveragepy_base_dir,
context.args.coverage_html_args,
has_command,
)
return rc
def _manage_dependencies(self, context,
install_base, bundle_base,
staging_path, installer_metadata_path):
check_and_mark_install_layout(
install_base, merge_install=context.args.merge_install)
self._create_path(bundle_base)
check_and_mark_bundle_tool(bundle_base)
destinations = self.task_argument_destinations
decorators = get_packages(context.args,
additional_argument_names=destinations,
recursive_categories=('run',))
installers = self._setup_installers(context)
print('Collecting dependency information...')
jobs = self._get_jobs(context.args, installers, decorators)
rc = execute_jobs(context, jobs)
if rc != 0:
return rc
print('Fetching and installing dependencies...')
installer_metadata = {}
for name, installer in installers.items():
installer_metadata[name] = installer.install()
installer_metadata_string = json.dumps(installer_metadata,
sort_keys=True)
dependencies_changed = True
if os.path.exists(installer_metadata_path):
with open(installer_metadata_path, 'r') as f:
previous_metadata = f.read()
if previous_metadata == installer_metadata_string:
dependencies_changed = False
with open(installer_metadata_path, 'w') as f:
f.write(installer_metadata_string)
if context.args.include_sources and dependencies_changed:
sources_tar_gz_path = os.path.join(bundle_base, 'sources.tar.gz')
with tarfile.open(
sources_tar_gz_path, 'w:gz', compresslevel=5) as archive:
for name, directory in self.installer_cache_dirs.items():
sources_path = os.path.join(directory, 'sources')
if not os.path.exists(sources_path):
continue
for filename in os.listdir(sources_path):
file_path = os.path.join(sources_path, filename)
archive.add(
file_path,
arcname=os.path.join(
name, os.path.basename(file_path)))
if dependencies_changed:
update_symlinks(staging_path)
# TODO: Update pkgconfig files?
update_shebang(staging_path)
# TODO: Move this to colcon-ros-bundle
rewrite_catkin_package_path(staging_path)
return dependencies_changed
