def test_topological_order_packages():
d1 = PackageDescriptor('/some/path')
d1.name = 'a'
d1.dependencies['build'].add('c')
d2 = PackageDescriptor('/other/path')
d2.name = 'b'
d2.dependencies['run'].add('c')
d3 = PackageDescriptor('/another/path')
d3.name = 'c'
d3.dependencies['build'].add('e')
d3.dependencies['run'].add('f')
d3.dependencies['test'].add('d')
d4 = PackageDescriptor('/yet-another/path')
d4.name = 'd'
d4.dependencies['run'].add('f')
d5 = PackageDescriptor('/more/path')
d5.name = 'e'
d5.dependencies['run'].add('f')
d6 = PackageDescriptor('/yet-more/path')
d6.name = 'f'
decos = topological_order_packages({d1, d2, d3, d4, d5, d6})
names = [d.descriptor.name for d in decos]
assert names == ['f', 'd', 'e', 'c', 'a', 'b']
# ensure that input order doesn't affect the result
decos = topological_order_packages({d6, d5, d4, d3, d2, d1})
names = [d.descriptor.name for d in decos]
assert names == ['f', 'd', 'e', 'c', 'a', 'b']
def test_topological_order_packages_with_circular_dependency():
d1 = PackageDescriptor('/some/path')
d1.name = 'one'
d1.dependencies['run'].add('two')
d2 = PackageDescriptor('/other/path')
d2.name = 'two'
d2.dependencies['run'].add('three')
d3 = PackageDescriptor('/another/path')
d3.name = 'three'
d3.dependencies['run'].add('one')
d3.dependencies['run'].add('six')
d4 = PackageDescriptor('/yet-another/path')
d4.name = 'four'
d5 = PackageDescriptor('/more/path')
d5.name = 'five'
d5.dependencies['run'].add('four')
d6 = PackageDescriptor('/yet-more/path')
d6.name = 'six'
with pytest.raises(RuntimeError) as e:
topological_order_packages({d1, d2, d3, d4})
lines = str(e.value).splitlines()
assert len(lines) == 4
assert lines[0] == 'Unable to order packages topologically:'
assert lines[1] == "one: ['three', 'two']"
assert lines[2] == "three: ['one', 'two']"
assert lines[3] == "two: ['one', 'three']"
def main(self, *, context): # noqa: D102
args = context.args
descriptors = get_package_descriptors(args)
# always perform topological order for the select package extensions
decorators = topological_order_packages(descriptors,
recursive_categories=('run', ))
select_package_decorators(args, decorators)
if not args.topological_order:
decorators = sorted(decorators, key=lambda d: d.descriptor.name)
lines = []
for decorator in decorators:
if not decorator.selected:
continue
pkg = decorator.descriptor
if args.names_only:
lines.append(pkg.name)
elif args.paths_only:
lines.append(str(pkg.path))
else:
lines.append(pkg.name + '\t' + str(pkg.path) +
'\t(%s)' % pkg.type)
if not args.topological_order:
# output names and / or paths in alphabetical order
lines.sort()
for line in lines:
print(line)
def get_packages(args,
*,
additional_argument_names=None,
direct_categories=None,
recursive_categories=None):
"""
Get the selected package decorators in topological order.
The overview of the process:
* Get the package descriptors
* Order them topologically
* Select the packages based on the command line arguments
:param additional_argument_names: A list of additional arguments to
consider
:param Iterable[str] direct_categories: The names of the direct categories
:param Iterable[str] recursive_categories: The names of the recursive
categories
:rtype: list
:raises RuntimeError: if the returned set of packages contains duplicates
package names
"""
descriptors = get_package_descriptors(
args, additional_argument_names=additional_argument_names)
decorators = topological_order_packages(
descriptors,
direct_categories=direct_categories,
recursive_categories=recursive_categories)
select_package_decorators(args, decorators)
# check for duplicate package names
pkgs = [m.descriptor for m in decorators if m.selected]
if len({d.name for d in pkgs}) < len(pkgs):
pkg_paths = defaultdict(list)
for d in pkgs:
pkg_paths[d.name].append(' - {d.path}'.format_map(locals()))
raise RuntimeError('Duplicate package names not supported:\n' +
'\n'.join(('- ' + name + ':\n' +
'\n'.join(sorted(pkg_paths[name])))
for name in sorted(pkg_paths.keys())
if len(pkg_paths[name]) > 1))
return decorators
def _get_gcc_packages(context, additional_argument_names=None):
descriptors = get_package_descriptors(
context.args, additional_argument_names=additional_argument_names)
# always perform topological order for the select package extensions
decorators = topological_order_packages(descriptors,
recursive_categories=('run', ))
select_package_decorators(context.args, decorators)
gcc_pkgs = []
for decorator in decorators:
if not decorator.selected:
continue
pkg = decorator.descriptor
if pkg.type in ['ros.ament_cmake', 'ros.cmake', 'cmake']:
gcc_pkgs.append(pkg)
else:
logger.info("Specified package {} is not a gcc package. Not "
"collecting coverage".format(pkg.name))
return gcc_pkgs
def _get_coveragepy_packages(context, additional_argument_names=None):
"""Get packages that could have coverage.py results."""
descriptors = get_package_descriptors(
context.args,
additional_argument_names=additional_argument_names,
)
decorators = topological_order_packages(descriptors,
recursive_categories=('run', ))
select_package_decorators(context.args, decorators)
coveragepy_pkgs = []
for decorator in decorators:
if not decorator.selected:
continue
pkg = decorator.descriptor
if pkg.type in ['ros.ament_cmake', 'ros.ament_python']:
coveragepy_pkgs.append(pkg)
else:
logger.info(
"Specified package '{pkg.name}' is not a coverage.py-compatible "
'package. Not collecting coverage information.'.format_map(
locals()))
return coveragepy_pkgs
def main(self, *, context): # noqa: D102
args = context.args
if args.topological_graph or args.topological_graph_dot:
args.topological_order = True
descriptors = get_package_descriptors(args)
# always perform topological order for the select package extensions
decorators = topological_order_packages(descriptors,
recursive_categories=('run', ))
select_package_decorators(args, decorators)
if args.topological_graph:
if args.topological_graph_legend:
print('+ marks when the package in this row can be processed')
print('* marks a direct dependency '
'from the package indicated by the + in the same column '
'to the package in this row')
print('. marks a transitive dependency')
print()
# draw dependency graph in ASCII
shown_decorators = list(filter(lambda d: d.selected, decorators))
max_length = max(
[len(m.descriptor.name) for m in shown_decorators] + [0])
lines = [
m.descriptor.name.ljust(max_length + 2)
for m in shown_decorators
]
depends = [
m.descriptor.get_dependencies() for m in shown_decorators
]
rec_depends = [
m.descriptor.get_recursive_dependencies(
[d.descriptor for d in decorators],
recursive_categories=('run', )) for m in shown_decorators
]
empty_cells = 0
for i, decorator in enumerate(shown_decorators):
for j in range(len(lines)):
if j == i:
# package i is being processed
lines[j] += '+'
elif shown_decorators[j].descriptor.name in depends[i]:
# package i directly depends on package j
lines[j] += '*'
elif shown_decorators[j].descriptor.name in rec_depends[i]:
# package i recursively depends on package j
lines[j] += '.'
else:
# package i doesn't depend on package j
lines[j] += ' '
empty_cells += 1
if args.topological_graph_density:
empty_fraction = \
empty_cells / (len(lines) * (len(lines) - 1)) \
if len(lines) > 1 else 1.0
# normalize to 200% since half of the matrix should be empty
density_percentage = 200.0 * (1.0 - empty_fraction)
print('dependency density %.2f %%' % density_percentage)
print()
elif args.topological_graph_dot:
lines = ['digraph graphname {']
decorators_by_name = defaultdict(set)
for deco in decorators:
decorators_by_name[deco.descriptor.name].add(deco)
selected_pkg_names = [
m.descriptor.name for m in decorators if m.selected
]
has_duplicate_names = \
len(selected_pkg_names) != len(set(selected_pkg_names))
selected_pkg_names = set(selected_pkg_names)
# collect selected package descriptors and their parent path
nodes = OrderedDict()
for deco in reversed(decorators):
if not deco.selected:
continue
nodes[deco.descriptor] = Path(deco.descriptor.path).parent
# collect direct dependencies
direct_edges = defaultdict(set)
for deco in reversed(decorators):
if not deco.selected:
continue
# iterate over dependency categories
for category, deps in deco.descriptor.dependencies.items():
# iterate over dependencies
for dep in deps:
if dep not in selected_pkg_names:
continue
# store the category of each dependency
# use the decorator descriptor
# since there might be packages with the same name
direct_edges[(deco.descriptor, dep)].add(category)
# collect indirect dependencies
indirect_edges = defaultdict(set)
for deco in reversed(decorators):
if not deco.selected:
continue
# iterate over dependency categories
for category, deps in deco.descriptor.dependencies.items():
# iterate over dependencies
for dep in deps:
# ignore direct dependencies
if dep in selected_pkg_names:
continue
# ignore unknown dependencies
if dep not in decorators_by_name.keys():
continue
# iterate over recursive dependencies
for rdep in itertools.chain.from_iterable(
d.recursive_dependencies
for d in decorators_by_name[dep]):
if rdep not in selected_pkg_names:
continue
# skip edges which are redundant to direct edges
if (deco.descriptor, rdep) in direct_edges:
continue
indirect_edges[(deco.descriptor,
rdep)].add(category)
try:
# HACK Python 3.5 can't handle Path objects
common_path = os.path.commonpath(
[str(p) for p in nodes.values()])
except ValueError:
common_path = None
def get_node_data(descriptor):
nonlocal has_duplicate_names
if not has_duplicate_names:
# use name where possible so the dot code is easy to read
return descriptor.name, ''
# otherwise append the descriptor id to make each node unique
descriptor_id = id(descriptor)
return (
'{descriptor.name}_{descriptor_id}'.format_map(locals()),
' [label = "{descriptor.name}"]'.format_map(locals()),
)
if not args.topological_graph_dot_cluster or common_path is None:
# output nodes
for desc in nodes.keys():
node_name, attributes = get_node_data(desc)
lines.append(' "{node_name}"{attributes};'.format_map(
locals()))
else:
# output clusters
clusters = defaultdict(set)
for desc, path in nodes.items():
clusters[path.relative_to(common_path)].add(desc)
for i, cluster in zip(range(len(clusters)), clusters.items()):
path, descs = cluster
if path.name:
# wrap cluster in subgraph
lines.append(' subgraph cluster_{i} {{'.format_map(
locals()))
lines.append(' label = "{path}";'.format_map(
locals()))
indent = ' '
else:
indent = ' '
for desc in descs:
node_name, attributes = get_node_data(desc)
lines.append(
'{indent}"{node_name}"{attributes};'.format_map(
locals()))
if path.name:
lines.append(' }')
# output edges
color_mapping = OrderedDict((
('build', 'blue'),
('run', 'red'),
('test', 'tan'),
))
for style, edges in zip(
('', ', style="dashed"'),
(direct_edges, indirect_edges),
):
for (desc_start, node_end), categories in edges.items():
colors = ':'.join([
color for category, color in color_mapping.items()
if category in categories
])
start_name, _ = get_node_data(desc_start)
for deco in decorators_by_name[node_end]:
end_name, _ = get_node_data(deco.descriptor)
lines.append(' "{start_name}" -> "{end_name}" '
'[color="{colors}"{style}];'.format_map(
locals()))
lines.append('}')
else:
if not args.topological_order:
decorators = sorted(decorators,
key=lambda d: d.descriptor.name)
lines = []
for decorator in decorators:
if not decorator.selected:
continue
pkg = decorator.descriptor
if args.names_only:
lines.append(pkg.name)
elif args.paths_only:
lines.append(str(pkg.path))
else:
lines.append(pkg.name + '\t' + str(pkg.path) +
'\t(%s)' % pkg.type)
if not args.topological_order:
# output names and / or paths in alphabetical order
lines.sort()
for line in lines:
print(line)
def main(self, *, context): # noqa: D102
args = context.args
descriptors = get_package_descriptors(args)
decorators = topological_order_packages(descriptors,
recursive_categories=('run', ))
select_package_decorators(args, decorators)
if not args.dot:
if args.legend:
print('+ marks when the package in this row can be processed')
print('* marks a direct dependency '
'from the package indicated by the + in the same column '
'to the package in this row')
print('. marks a transitive dependency')
print()
# draw dependency graph in ASCII
shown_decorators = list(filter(lambda d: d.selected, decorators))
max_length = max(
[len(m.descriptor.name) for m in shown_decorators] + [0])
lines = [
m.descriptor.name.ljust(max_length + 2)
for m in shown_decorators
]
depends = [
m.descriptor.get_dependencies() for m in shown_decorators
]
rec_depends = [
m.descriptor.get_recursive_dependencies(
[d.descriptor for d in decorators],
recursive_categories=('run', )) for m in shown_decorators
]
empty_cells = 0
for i, decorator in enumerate(shown_decorators):
for j in range(len(lines)):
if j == i:
# package i is being processed
lines[j] += '+'
elif shown_decorators[j].descriptor.name in depends[i]:
# package i directly depends on package j
lines[j] += '*'
elif shown_decorators[j].descriptor.name in rec_depends[i]:
# package i recursively depends on package j
lines[j] += '.'
else:
# package i doesn't depend on package j
lines[j] += ' '
empty_cells += 1
if args.density:
empty_fraction = \
empty_cells / (len(lines) * (len(lines) - 1)) \
if len(lines) > 1 else 1.0
# normalize to 200% since half of the matrix should be empty
density_percentage = 200.0 * (1.0 - empty_fraction)
print('dependency density %.2f %%' % density_percentage)
print()
else: # --dot
lines = ['digraph graphname {']
decorators_by_name = defaultdict(set)
for deco in decorators:
decorators_by_name[deco.descriptor.name].add(deco)
selected_pkg_names = [
m.descriptor.name for m in decorators
if m.selected or args.dot_include_skipped
]
has_duplicate_names = \
len(selected_pkg_names) != len(set(selected_pkg_names))
selected_pkg_names = set(selected_pkg_names)
# collect selected package decorators and their parent path
nodes = OrderedDict()
for deco in reversed(decorators):
if deco.selected or args.dot_include_skipped:
nodes[deco] = Path(deco.descriptor.path).parent
# collect direct dependencies
direct_edges = defaultdict(set)
for deco in reversed(decorators):
if (not deco.selected and not args.dot_include_skipped):
continue
# iterate over dependency categories
for category, deps in deco.descriptor.dependencies.items():
# iterate over dependencies
for dep in deps:
if dep not in selected_pkg_names:
continue
# store the category of each dependency
# use the decorator
# since there might be packages with the same name
direct_edges[(deco, dep)].add(category)
# collect indirect dependencies
indirect_edges = defaultdict(set)
for deco in reversed(decorators):
if not deco.selected:
continue
# iterate over dependency categories
for category, deps in deco.descriptor.dependencies.items():
# iterate over dependencies
for dep in deps:
# ignore direct dependencies
if dep in selected_pkg_names:
continue
# ignore unknown dependencies
if dep not in decorators_by_name.keys():
continue
# iterate over recursive dependencies
for rdep in itertools.chain.from_iterable(
d.recursive_dependencies
for d in decorators_by_name[dep]):
if rdep not in selected_pkg_names:
continue
# skip edges which are redundant to direct edges
if (deco, rdep) in direct_edges:
continue
indirect_edges[(deco, rdep)].add(category)
try:
# HACK Python 3.5 can't handle Path objects
common_path = os.path.commonpath(
[str(p) for p in nodes.values()])
except ValueError:
common_path = None
def get_node_data(decorator):
nonlocal args
nonlocal has_duplicate_names
if not has_duplicate_names:
# use name where possible so the dot code is easy to read
return decorator.descriptor.name, \
'' if (
decorator.selected or
not args.dot_include_skipped
) else '[color = "gray" fontcolor = "gray"]'
# otherwise append the descriptor id to make each node unique
descriptor_id = id(decorator.descriptor)
return (
'{decorator.descriptor.name}_{descriptor_id}'.format_map(
locals()),
' [label = "{decorator.descriptor.name}"]'.format_map(
locals()),
)
if not args.dot_cluster or common_path is None:
# output nodes
for deco in nodes.keys():
if (not deco.selected and not args.dot_include_skipped):
continue
node_name, attributes = get_node_data(deco)
lines.append(' "{node_name}"{attributes};'.format_map(
locals()))
else:
# output clusters
clusters = defaultdict(set)
for deco, path in nodes.items():
clusters[path.relative_to(common_path)].add(deco)
for i, cluster in zip(range(len(clusters)), clusters.items()):
path, decos = cluster
if path.name:
# wrap cluster in subgraph
lines.append(' subgraph cluster_{i} {{'.format_map(
locals()))
lines.append(' label = "{path}";'.format_map(
locals()))
indent = ' '
else:
indent = ' '
for deco in decos:
node_name, attributes = get_node_data(deco)
lines.append(
'{indent}"{node_name}"{attributes};'.format_map(
locals()))
if path.name:
lines.append(' }')
# output edges
color_mapping = OrderedDict((
('build', '#0000ff'), # blue
('run', '#ff0000'), # red
('test', '#d2b48c'), # tan
))
for style, edges in zip(
('', ', style="dashed"'),
(direct_edges, indirect_edges),
):
for (deco_start, node_end), categories in edges.items():
start_name, _ = get_node_data(deco_start)
for deco in decorators_by_name[node_end]:
end_name, _ = get_node_data(deco)
edge_alpha = '' \
if deco_start.selected and deco.selected else '77'
colors = ':'.join([
color + edge_alpha
for category, color in color_mapping.items()
if category in categories
])
lines.append(' "{start_name}" -> "{end_name}" '
'[color="{colors}"{style}];'.format_map(
locals()))
if args.legend:
lines.append(' subgraph cluster_legend {')
lines.append(' color=gray')
lines.append(' label="Legend";')
lines.append(' margin=0;')
# invisible nodes between the dependency edges
lines.append(' node [label="", shape=none];')
previous_node = '_legend_first'
# an edge for each dependency type
for dependency_type, color in color_mapping.items():
next_node = '_legend_' + dependency_type
lines.append(
' {previous_node} -> {next_node} '
'[label="{dependency_type} dep.", color="{color}"];'.
format_map(locals()))
previous_node = next_node
lines.append(
' {previous_node} -> _legend_last '
'[label="indirect dep.", style="dashed"];'.format_map(
locals()))
# layout all legend nodes on the same rank
lines.append(' {')
lines.append(' rank=same;')
lines.append(' _legend_first;')
for dependency_type in color_mapping.keys():
lines.append(' _legend_{dependency_type};'.format_map(
locals()))
lines.append(' _legend_last;')
lines.append(' }')
lines.append(' }')
lines.append('}')
for line in lines:
print(line)
