def test_topological_order_packages(): d1 = PackageDescriptor('/some/path') d1.name = 'a' d1.dependencies['build'].add('c') d2 = PackageDescriptor('/other/path') d2.name = 'b' d2.dependencies['run'].add('c') d3 = PackageDescriptor('/another/path') d3.name = 'c' d3.dependencies['build'].add('e') d3.dependencies['run'].add('f') d3.dependencies['test'].add('d') d4 = PackageDescriptor('/yet-another/path') d4.name = 'd' d4.dependencies['run'].add('f') d5 = PackageDescriptor('/more/path') d5.name = 'e' d5.dependencies['run'].add('f') d6 = PackageDescriptor('/yet-more/path') d6.name = 'f' decos = topological_order_packages({d1, d2, d3, d4, d5, d6}) names = [d.descriptor.name for d in decos] assert names == ['f', 'd', 'e', 'c', 'a', 'b'] # ensure that input order doesn't affect the result decos = topological_order_packages({d6, d5, d4, d3, d2, d1}) names = [d.descriptor.name for d in decos] assert names == ['f', 'd', 'e', 'c', 'a', 'b']
def test_topological_order_packages_with_circular_dependency(): d1 = PackageDescriptor('/some/path') d1.name = 'one' d1.dependencies['run'].add('two') d2 = PackageDescriptor('/other/path') d2.name = 'two' d2.dependencies['run'].add('three') d3 = PackageDescriptor('/another/path') d3.name = 'three' d3.dependencies['run'].add('one') d3.dependencies['run'].add('six') d4 = PackageDescriptor('/yet-another/path') d4.name = 'four' d5 = PackageDescriptor('/more/path') d5.name = 'five' d5.dependencies['run'].add('four') d6 = PackageDescriptor('/yet-more/path') d6.name = 'six' with pytest.raises(RuntimeError) as e: topological_order_packages({d1, d2, d3, d4}) lines = str(e.value).splitlines() assert len(lines) == 4 assert lines[0] == 'Unable to order packages topologically:' assert lines[1] == "one: ['three', 'two']" assert lines[2] == "three: ['one', 'two']" assert lines[3] == "two: ['one', 'three']"
def main(self, *, context): # noqa: D102 args = context.args descriptors = get_package_descriptors(args) # always perform topological order for the select package extensions decorators = topological_order_packages(descriptors, recursive_categories=('run', )) select_package_decorators(args, decorators) if not args.topological_order: decorators = sorted(decorators, key=lambda d: d.descriptor.name) lines = [] for decorator in decorators: if not decorator.selected: continue pkg = decorator.descriptor if args.names_only: lines.append(pkg.name) elif args.paths_only: lines.append(str(pkg.path)) else: lines.append(pkg.name + '\t' + str(pkg.path) + '\t(%s)' % pkg.type) if not args.topological_order: # output names and / or paths in alphabetical order lines.sort() for line in lines: print(line)
def get_packages(args, *, additional_argument_names=None, direct_categories=None, recursive_categories=None): """ Get the selected package decorators in topological order. The overview of the process: * Get the package descriptors * Order them topologically * Select the packages based on the command line arguments :param additional_argument_names: A list of additional arguments to consider :param Iterable[str] direct_categories: The names of the direct categories :param Iterable[str] recursive_categories: The names of the recursive categories :rtype: list :raises RuntimeError: if the returned set of packages contains duplicates package names """ descriptors = get_package_descriptors( args, additional_argument_names=additional_argument_names) decorators = topological_order_packages( descriptors, direct_categories=direct_categories, recursive_categories=recursive_categories) select_package_decorators(args, decorators) # check for duplicate package names pkgs = [m.descriptor for m in decorators if m.selected] if len({d.name for d in pkgs}) < len(pkgs): pkg_paths = defaultdict(list) for d in pkgs: pkg_paths[d.name].append(' - {d.path}'.format_map(locals())) raise RuntimeError('Duplicate package names not supported:\n' + '\n'.join(('- ' + name + ':\n' + '\n'.join(sorted(pkg_paths[name]))) for name in sorted(pkg_paths.keys()) if len(pkg_paths[name]) > 1)) return decorators
def _get_gcc_packages(context, additional_argument_names=None): descriptors = get_package_descriptors( context.args, additional_argument_names=additional_argument_names) # always perform topological order for the select package extensions decorators = topological_order_packages(descriptors, recursive_categories=('run', )) select_package_decorators(context.args, decorators) gcc_pkgs = [] for decorator in decorators: if not decorator.selected: continue pkg = decorator.descriptor if pkg.type in ['ros.ament_cmake', 'ros.cmake', 'cmake']: gcc_pkgs.append(pkg) else: logger.info("Specified package {} is not a gcc package. Not " "collecting coverage".format(pkg.name)) return gcc_pkgs
def _get_coveragepy_packages(context, additional_argument_names=None): """Get packages that could have coverage.py results.""" descriptors = get_package_descriptors( context.args, additional_argument_names=additional_argument_names, ) decorators = topological_order_packages(descriptors, recursive_categories=('run', )) select_package_decorators(context.args, decorators) coveragepy_pkgs = [] for decorator in decorators: if not decorator.selected: continue pkg = decorator.descriptor if pkg.type in ['ros.ament_cmake', 'ros.ament_python']: coveragepy_pkgs.append(pkg) else: logger.info( "Specified package '{pkg.name}' is not a coverage.py-compatible " 'package. Not collecting coverage information.'.format_map( locals())) return coveragepy_pkgs
def main(self, *, context): # noqa: D102 args = context.args if args.topological_graph or args.topological_graph_dot: args.topological_order = True descriptors = get_package_descriptors(args) # always perform topological order for the select package extensions decorators = topological_order_packages(descriptors, recursive_categories=('run', )) select_package_decorators(args, decorators) if args.topological_graph: if args.topological_graph_legend: print('+ marks when the package in this row can be processed') print('* marks a direct dependency ' 'from the package indicated by the + in the same column ' 'to the package in this row') print('. marks a transitive dependency') print() # draw dependency graph in ASCII shown_decorators = list(filter(lambda d: d.selected, decorators)) max_length = max( [len(m.descriptor.name) for m in shown_decorators] + [0]) lines = [ m.descriptor.name.ljust(max_length + 2) for m in shown_decorators ] depends = [ m.descriptor.get_dependencies() for m in shown_decorators ] rec_depends = [ m.descriptor.get_recursive_dependencies( [d.descriptor for d in decorators], recursive_categories=('run', )) for m in shown_decorators ] empty_cells = 0 for i, decorator in enumerate(shown_decorators): for j in range(len(lines)): if j == i: # package i is being processed lines[j] += '+' elif shown_decorators[j].descriptor.name in depends[i]: # package i directly depends on package j lines[j] += '*' elif shown_decorators[j].descriptor.name in rec_depends[i]: # package i recursively depends on package j lines[j] += '.' else: # package i doesn't depend on package j lines[j] += ' ' empty_cells += 1 if args.topological_graph_density: empty_fraction = \ empty_cells / (len(lines) * (len(lines) - 1)) \ if len(lines) > 1 else 1.0 # normalize to 200% since half of the matrix should be empty density_percentage = 200.0 * (1.0 - empty_fraction) print('dependency density %.2f %%' % density_percentage) print() elif args.topological_graph_dot: lines = ['digraph graphname {'] decorators_by_name = defaultdict(set) for deco in decorators: decorators_by_name[deco.descriptor.name].add(deco) selected_pkg_names = [ m.descriptor.name for m in decorators if m.selected ] has_duplicate_names = \ len(selected_pkg_names) != len(set(selected_pkg_names)) selected_pkg_names = set(selected_pkg_names) # collect selected package descriptors and their parent path nodes = OrderedDict() for deco in reversed(decorators): if not deco.selected: continue nodes[deco.descriptor] = Path(deco.descriptor.path).parent # collect direct dependencies direct_edges = defaultdict(set) for deco in reversed(decorators): if not deco.selected: continue # iterate over dependency categories for category, deps in deco.descriptor.dependencies.items(): # iterate over dependencies for dep in deps: if dep not in selected_pkg_names: continue # store the category of each dependency # use the decorator descriptor # since there might be packages with the same name direct_edges[(deco.descriptor, dep)].add(category) # collect indirect dependencies indirect_edges = defaultdict(set) for deco in reversed(decorators): if not deco.selected: continue # iterate over dependency categories for category, deps in deco.descriptor.dependencies.items(): # iterate over dependencies for dep in deps: # ignore direct dependencies if dep in selected_pkg_names: continue # ignore unknown dependencies if dep not in decorators_by_name.keys(): continue # iterate over recursive dependencies for rdep in itertools.chain.from_iterable( d.recursive_dependencies for d in decorators_by_name[dep]): if rdep not in selected_pkg_names: continue # skip edges which are redundant to direct edges if (deco.descriptor, rdep) in direct_edges: continue indirect_edges[(deco.descriptor, rdep)].add(category) try: # HACK Python 3.5 can't handle Path objects common_path = os.path.commonpath( [str(p) for p in nodes.values()]) except ValueError: common_path = None def get_node_data(descriptor): nonlocal has_duplicate_names if not has_duplicate_names: # use name where possible so the dot code is easy to read return descriptor.name, '' # otherwise append the descriptor id to make each node unique descriptor_id = id(descriptor) return ( '{descriptor.name}_{descriptor_id}'.format_map(locals()), ' [label = "{descriptor.name}"]'.format_map(locals()), ) if not args.topological_graph_dot_cluster or common_path is None: # output nodes for desc in nodes.keys(): node_name, attributes = get_node_data(desc) lines.append(' "{node_name}"{attributes};'.format_map( locals())) else: # output clusters clusters = defaultdict(set) for desc, path in nodes.items(): clusters[path.relative_to(common_path)].add(desc) for i, cluster in zip(range(len(clusters)), clusters.items()): path, descs = cluster if path.name: # wrap cluster in subgraph lines.append(' subgraph cluster_{i} {{'.format_map( locals())) lines.append(' label = "{path}";'.format_map( locals())) indent = ' ' else: indent = ' ' for desc in descs: node_name, attributes = get_node_data(desc) lines.append( '{indent}"{node_name}"{attributes};'.format_map( locals())) if path.name: lines.append(' }') # output edges color_mapping = OrderedDict(( ('build', 'blue'), ('run', 'red'), ('test', 'tan'), )) for style, edges in zip( ('', ', style="dashed"'), (direct_edges, indirect_edges), ): for (desc_start, node_end), categories in edges.items(): colors = ':'.join([ color for category, color in color_mapping.items() if category in categories ]) start_name, _ = get_node_data(desc_start) for deco in decorators_by_name[node_end]: end_name, _ = get_node_data(deco.descriptor) lines.append(' "{start_name}" -> "{end_name}" ' '[color="{colors}"{style}];'.format_map( locals())) lines.append('}') else: if not args.topological_order: decorators = sorted(decorators, key=lambda d: d.descriptor.name) lines = [] for decorator in decorators: if not decorator.selected: continue pkg = decorator.descriptor if args.names_only: lines.append(pkg.name) elif args.paths_only: lines.append(str(pkg.path)) else: lines.append(pkg.name + '\t' + str(pkg.path) + '\t(%s)' % pkg.type) if not args.topological_order: # output names and / or paths in alphabetical order lines.sort() for line in lines: print(line)
def main(self, *, context): # noqa: D102 args = context.args descriptors = get_package_descriptors(args) decorators = topological_order_packages(descriptors, recursive_categories=('run', )) select_package_decorators(args, decorators) if not args.dot: if args.legend: print('+ marks when the package in this row can be processed') print('* marks a direct dependency ' 'from the package indicated by the + in the same column ' 'to the package in this row') print('. marks a transitive dependency') print() # draw dependency graph in ASCII shown_decorators = list(filter(lambda d: d.selected, decorators)) max_length = max( [len(m.descriptor.name) for m in shown_decorators] + [0]) lines = [ m.descriptor.name.ljust(max_length + 2) for m in shown_decorators ] depends = [ m.descriptor.get_dependencies() for m in shown_decorators ] rec_depends = [ m.descriptor.get_recursive_dependencies( [d.descriptor for d in decorators], recursive_categories=('run', )) for m in shown_decorators ] empty_cells = 0 for i, decorator in enumerate(shown_decorators): for j in range(len(lines)): if j == i: # package i is being processed lines[j] += '+' elif shown_decorators[j].descriptor.name in depends[i]: # package i directly depends on package j lines[j] += '*' elif shown_decorators[j].descriptor.name in rec_depends[i]: # package i recursively depends on package j lines[j] += '.' else: # package i doesn't depend on package j lines[j] += ' ' empty_cells += 1 if args.density: empty_fraction = \ empty_cells / (len(lines) * (len(lines) - 1)) \ if len(lines) > 1 else 1.0 # normalize to 200% since half of the matrix should be empty density_percentage = 200.0 * (1.0 - empty_fraction) print('dependency density %.2f %%' % density_percentage) print() else: # --dot lines = ['digraph graphname {'] decorators_by_name = defaultdict(set) for deco in decorators: decorators_by_name[deco.descriptor.name].add(deco) selected_pkg_names = [ m.descriptor.name for m in decorators if m.selected or args.dot_include_skipped ] has_duplicate_names = \ len(selected_pkg_names) != len(set(selected_pkg_names)) selected_pkg_names = set(selected_pkg_names) # collect selected package decorators and their parent path nodes = OrderedDict() for deco in reversed(decorators): if deco.selected or args.dot_include_skipped: nodes[deco] = Path(deco.descriptor.path).parent # collect direct dependencies direct_edges = defaultdict(set) for deco in reversed(decorators): if (not deco.selected and not args.dot_include_skipped): continue # iterate over dependency categories for category, deps in deco.descriptor.dependencies.items(): # iterate over dependencies for dep in deps: if dep not in selected_pkg_names: continue # store the category of each dependency # use the decorator # since there might be packages with the same name direct_edges[(deco, dep)].add(category) # collect indirect dependencies indirect_edges = defaultdict(set) for deco in reversed(decorators): if not deco.selected: continue # iterate over dependency categories for category, deps in deco.descriptor.dependencies.items(): # iterate over dependencies for dep in deps: # ignore direct dependencies if dep in selected_pkg_names: continue # ignore unknown dependencies if dep not in decorators_by_name.keys(): continue # iterate over recursive dependencies for rdep in itertools.chain.from_iterable( d.recursive_dependencies for d in decorators_by_name[dep]): if rdep not in selected_pkg_names: continue # skip edges which are redundant to direct edges if (deco, rdep) in direct_edges: continue indirect_edges[(deco, rdep)].add(category) try: # HACK Python 3.5 can't handle Path objects common_path = os.path.commonpath( [str(p) for p in nodes.values()]) except ValueError: common_path = None def get_node_data(decorator): nonlocal args nonlocal has_duplicate_names if not has_duplicate_names: # use name where possible so the dot code is easy to read return decorator.descriptor.name, \ '' if ( decorator.selected or not args.dot_include_skipped ) else '[color = "gray" fontcolor = "gray"]' # otherwise append the descriptor id to make each node unique descriptor_id = id(decorator.descriptor) return ( '{decorator.descriptor.name}_{descriptor_id}'.format_map( locals()), ' [label = "{decorator.descriptor.name}"]'.format_map( locals()), ) if not args.dot_cluster or common_path is None: # output nodes for deco in nodes.keys(): if (not deco.selected and not args.dot_include_skipped): continue node_name, attributes = get_node_data(deco) lines.append(' "{node_name}"{attributes};'.format_map( locals())) else: # output clusters clusters = defaultdict(set) for deco, path in nodes.items(): clusters[path.relative_to(common_path)].add(deco) for i, cluster in zip(range(len(clusters)), clusters.items()): path, decos = cluster if path.name: # wrap cluster in subgraph lines.append(' subgraph cluster_{i} {{'.format_map( locals())) lines.append(' label = "{path}";'.format_map( locals())) indent = ' ' else: indent = ' ' for deco in decos: node_name, attributes = get_node_data(deco) lines.append( '{indent}"{node_name}"{attributes};'.format_map( locals())) if path.name: lines.append(' }') # output edges color_mapping = OrderedDict(( ('build', '#0000ff'), # blue ('run', '#ff0000'), # red ('test', '#d2b48c'), # tan )) for style, edges in zip( ('', ', style="dashed"'), (direct_edges, indirect_edges), ): for (deco_start, node_end), categories in edges.items(): start_name, _ = get_node_data(deco_start) for deco in decorators_by_name[node_end]: end_name, _ = get_node_data(deco) edge_alpha = '' \ if deco_start.selected and deco.selected else '77' colors = ':'.join([ color + edge_alpha for category, color in color_mapping.items() if category in categories ]) lines.append(' "{start_name}" -> "{end_name}" ' '[color="{colors}"{style}];'.format_map( locals())) if args.legend: lines.append(' subgraph cluster_legend {') lines.append(' color=gray') lines.append(' label="Legend";') lines.append(' margin=0;') # invisible nodes between the dependency edges lines.append(' node [label="", shape=none];') previous_node = '_legend_first' # an edge for each dependency type for dependency_type, color in color_mapping.items(): next_node = '_legend_' + dependency_type lines.append( ' {previous_node} -> {next_node} ' '[label="{dependency_type} dep.", color="{color}"];'. format_map(locals())) previous_node = next_node lines.append( ' {previous_node} -> _legend_last ' '[label="indirect dep.", style="dashed"];'.format_map( locals())) # layout all legend nodes on the same rank lines.append(' {') lines.append(' rank=same;') lines.append(' _legend_first;') for dependency_type in color_mapping.keys(): lines.append(' _legend_{dependency_type};'.format_map( locals())) lines.append(' _legend_last;') lines.append(' }') lines.append(' }') lines.append('}') for line in lines: print(line)