def determine_packages_to_be_cleaned(context, include_dependents, packages):
"""Returns list of packages which should be cleaned, and those packages' deps.
:param context: Workspace context
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:param packages: list of package names to be cleaned
:type packages: list
:returns: full list of package names to be cleaned
:rtype: list
"""
# Get all the cached packages in the context source space
workspace_packages = find_packages(context.package_metadata_path(),
exclude_subspaces=True,
warnings=[])
# Order the packages by topology
ordered_packages = topological_order_packages(workspace_packages)
# Create a dict of all packages in the workspace by name
workspace_packages_by_name = dict([(pkg.name, (path, pkg))
for path, pkg in ordered_packages])
# Initialize empty output
packages_to_be_cleaned = set()
# Expand glob patterns in packages
expanded_packages = []
for package_name in packages:
expanded_packages.extend(
expand_glob_package(package_name, workspace_packages_by_name))
packages = expanded_packages
# Expand metapackages into their constituents
for package_name in packages:
# This is ok if it's orphaned
if package_name not in workspace_packages_by_name:
packages_to_be_cleaned.add(package_name)
else:
# Get the package object
package = workspace_packages_by_name[package_name][1]
# If metapackage, include run depends which are in the workspace
if 'metapackage' in [e.tagname for e in package.exports]:
for rdep in package.run_depends:
if rdep.name in workspace_packages_by_name:
packages_to_be_cleaned.add(rdep.name)
else:
packages_to_be_cleaned.add(package_name)
# Determine the packages that depend on the given packages
if include_dependents:
for package_name in list(packages_to_be_cleaned):
# Get the packages that depend on the packages to be cleaned
dependents = get_recursive_build_dependents_in_workspace(
package_name, ordered_packages)
packages_to_be_cleaned.update([pkg.name for _, pkg in dependents])
return [
workspace_packages_by_name[n] for n in packages_to_be_cleaned
if n in workspace_packages_by_name
]
def determine_packages_to_be_built(packages, context, workspace_packages):
"""Returns list of packages which should be built, and those package's deps.
:param packages: list of packages to be built, if None all packages are built
:type packages: list
:param context: Workspace context
:type context: :py:class:`catkin_tools.verbs.catkin_build.context.Context`
:returns: tuple of packages to be built and those package's deps
:rtype: tuple
"""
start = time.time()
# If there are no packages raise
if not workspace_packages:
log("[build] No packages were found in the source space '{0}'".format(
context.source_space_abs))
else:
wide_log("[build] Found '{0}' packages in {1}.".format(
len(workspace_packages), format_time_delta(time.time() - start)))
# Order the packages by topology
ordered_packages = topological_order_packages(workspace_packages)
# Set the packages in the workspace for the context
context.packages = ordered_packages
# Determine the packages which should be built
packages_to_be_built = []
packages_to_be_built_deps = []
# Check if topological_order_packages determined any circular dependencies, if so print an error and fail.
# If this is the case, the last entry of ordered packages is a tuple that starts with nil.
if ordered_packages and ordered_packages[-1][0] is None:
guilty_packages = ", ".join(ordered_packages[-1][1:])
sys.exit(
"[build] Circular dependency detected in the following packages: {}"
.format(guilty_packages))
workspace_package_names = dict([(pkg.name, (path, pkg))
for path, pkg in ordered_packages])
# Determine the packages to be built
if packages:
# First assert all of the packages given are in the workspace
for package in packages:
if package not in workspace_package_names:
# Try whether package is a pattern and matches
glob_packages = expand_glob_package(package,
workspace_package_names)
if len(glob_packages) > 0:
packages.extend(glob_packages)
continue
else:
sys.exit(
"[build] Given package '{0}' is not in the workspace "
"and pattern does not match any package".format(
package))
# If metapackage, include run depends which are in the workspace
package_obj = workspace_package_names[package][1]
if 'metapackage' in [e.tagname for e in package_obj.exports]:
for rdep in package_obj.run_depends:
if rdep.name in workspace_package_names:
packages.append(rdep.name)
# Limit the packages to be built to just the provided packages
for pkg_path, package in ordered_packages:
if package.name in packages:
packages_to_be_built.append((pkg_path, package))
# Get the recursive dependencies for each of these packages
pkg_deps = get_cached_recursive_build_depends_in_workspace(
package, ordered_packages)
packages_to_be_built_deps.extend(pkg_deps)
else:
# Only use whitelist when no other packages are specified
if len(context.whitelist) > 0:
# Expand glob patterns in whitelist
whitelist = []
for whitelisted_package in context.whitelist:
whitelist.extend(
expand_glob_package(whitelisted_package,
workspace_package_names))
packages_to_be_built = [
p for p in ordered_packages if (p[1].name in whitelist)
]
else:
packages_to_be_built = ordered_packages
# Filter packages with blacklist
if len(context.blacklist) > 0:
# Expand glob patterns in blacklist
blacklist = []
for blacklisted_package in context.blacklist:
blacklist.extend(
expand_glob_package(blacklisted_package,
workspace_package_names))
# Apply blacklist to packages and dependencies
packages_to_be_built = [
(path, pkg) for path, pkg in packages_to_be_built
if (pkg.name not in blacklist or pkg.name in packages)
]
packages_to_be_built_deps = [
(path, pkg) for path, pkg in packages_to_be_built_deps
if (pkg.name not in blacklist or pkg.name in packages)
]
return packages_to_be_built, packages_to_be_built_deps, ordered_packages
def test_workspace(
context,
packages=None,
n_jobs=None,
quiet=False,
interleave_output=False,
no_status=False,
limit_status_rate=10.0,
no_notify=False,
continue_on_failure=False,
summarize_build=False,
catkin_test_target='run_tests',
cmake_test_target='test',
):
"""Tests a catkin workspace
:param context: context in which to test the catkin workspace
:type context: :py:class:`catkin_tools.context.Context`
:param packages: list of packages to test
:type packages: list
:param n_jobs: number of parallel package test jobs
:type n_jobs: int
:param quiet: suppresses verbose build or test information
:type quiet: bool
:param interleave_output: prints the output of commands as they are received
:type interleave_output: bool
:param no_status: suppresses the bottom status line
:type no_status: bool
:param limit_status_rate: rate to which status updates are limited; the default 0, places no limit.
:type limit_status_rate: float
:param no_notify: suppresses system notifications
:type no_notify: bool
:param continue_on_failure: do not stop testing other packages on error
:type continue_on_failure: bool
:param summarize_build: summarizes the build at the end
:type summarize_build: bool
:param catkin_test_target: make target for tests in catkin packages
:type catkin_test_target: str
:param cmake_test_target: make target for tests in cmake packages
:type cmake_test_target: str
"""
pre_start_time = time.time()
# Assert that the limit_status_rate is valid
if limit_status_rate < 0:
sys.exit(
"[test] @!@{rf}Error:@| The value of --limit-status-rate must be greater than or equal to zero."
)
# Get all the packages in the context source space
# Suppress warnings since this is a utility function
try:
workspace_packages = find_packages(context.source_space_abs,
exclude_subspaces=True,
warnings=[])
except InvalidPackage as ex:
sys.exit(
clr("@{rf}Error:@| The file {} is an invalid package.xml file."
" See below for details:\n\n{}").format(
ex.package_path, ex.msg))
# Get all build type plugins
test_job_creators = {
ep.name: ep.load()['create_test_job']
for ep in pkg_resources.iter_entry_points(group='catkin_tools.jobs')
}
# It's a problem if there aren't any build types available
if len(test_job_creators) == 0:
sys.exit(
'Error: No build types available. Please check your catkin_tools installation.'
)
# Get list of packages to test
ordered_packages = topological_order_packages(workspace_packages)
# Check if topological_order_packages determined any circular dependencies, if so print an error and fail.
# If this is the case, the last entry of ordered packages is a tuple that starts with nil.
if ordered_packages and ordered_packages[-1][0] is None:
guilty_packages = ", ".join(ordered_packages[-1][1:])
sys.exit(
"[test] Circular dependency detected in the following packages: {}"
.format(guilty_packages))
workspace_packages = dict([(pkg.name, (path, pkg))
for path, pkg in ordered_packages])
packages_to_test = []
if packages:
for package in packages:
if package not in workspace_packages:
# Try whether package is a pattern and matches
glob_packages = expand_glob_package(package,
workspace_packages)
if len(glob_packages) > 0:
packages.extend(glob_packages)
else:
sys.exit(
"[test] Given packages '{}' is not in the workspace "
"and pattern does not match any package".format(
package))
for pkg_path, package in ordered_packages:
if package.name in packages:
packages_to_test.append((pkg_path, package))
else:
# Only use buildlist when no other packages are specified
if len(context.buildlist) > 0:
# Expand glob patterns in buildlist
buildlist = []
for buildlisted_package in context.buildlist:
buildlist.extend(
expand_glob_package(buildlisted_package,
workspace_packages))
packages_to_test = [
p for p in ordered_packages if (p[1].name in buildlist)
]
else:
packages_to_test = ordered_packages
# Filter packages on skiplist
if len(context.skiplist) > 0:
# Expand glob patterns in skiplist
skiplist = []
for skiplisted_package in context.skiplist:
skiplist.extend(
expand_glob_package(skiplisted_package, workspace_packages))
# Apply skiplist to packages and dependencies
packages_to_test = [
(path, pkg) for path, pkg in packages_to_test
if (pkg.name not in skiplist or pkg.name in packages)
]
# Check if all packages to test are already built
built_packages = set([
pkg.name
for (path, pkg) in find_packages(context.package_metadata_path(),
warnings=[]).items()
])
packages_to_test_names = set(pkg.name for path, pkg in packages_to_test)
if not built_packages.issuperset(packages_to_test_names):
wide_log(
clr("@{rf}Error: Packages have to be built before they can be tested.@|"
))
wide_log(clr("The following requested packages are not built yet:"))
for package_name in packages_to_test_names.difference(built_packages):
wide_log(' - ' + package_name)
sys.exit(1)
# Construct jobs
jobs = []
for pkg_path, pkg in packages_to_test:
# Determine the job parameters
test_job_kwargs = dict(context=context,
package=pkg,
package_path=pkg_path,
verbose=not quiet)
# Create the job based on the build type
build_type = pkg.get_build_type()
if build_type == 'catkin':
test_job_kwargs['test_target'] = catkin_test_target
elif build_type == 'cmake':
test_job_kwargs['test_target'] = cmake_test_target
if build_type in test_job_creators:
jobs.append(test_job_creators[build_type](**test_job_kwargs))
# Queue for communicating status
event_queue = Queue()
# Initialize job server
job_server.initialize(
max_jobs=n_jobs,
max_load=None,
gnu_make_enabled=context.use_internal_make_jobserver,
)
try:
# Spin up status output thread
status_thread = ConsoleStatusController(
'test',
['package', 'packages'],
jobs,
n_jobs,
[pkg.name for path, pkg in packages_to_test],
[p for p in context.buildlist],
[p for p in context.skiplist],
event_queue,
show_notifications=not no_notify,
show_active_status=not no_status,
show_buffered_stdout=not interleave_output,
show_buffered_stderr=not interleave_output,
show_live_stdout=interleave_output,
show_live_stderr=interleave_output,
show_full_summary=summarize_build,
show_stage_events=not quiet,
pre_start_time=pre_start_time,
active_status_rate=limit_status_rate,
)
status_thread.start()
locks = {}
# Block while running N jobs asynchronously
try:
all_succeeded = run_until_complete(
execute_jobs('test',
jobs,
locks,
event_queue,
context.log_space_abs,
max_toplevel_jobs=n_jobs,
continue_on_failure=continue_on_failure,
continue_without_deps=False))
except Exception:
status_thread.keep_running = False
all_succeeded = False
status_thread.join(1.0)
wide_log(str(traceback.format_exc()))
status_thread.join(1.0)
if all_succeeded:
return 0
else:
return 1
except KeyboardInterrupt:
wide_log("[test] Interrupted by user!")
event_queue.put(None)
return 130