def antichains(G, topo_order=None): """Generates antichains from a directed acyclic graph (DAG). An antichain is a subset of a partially ordered set such that any two elements in the subset are incomparable. Parameters ---------- G : NetworkX DiGraph A directed acyclic graph (DAG) topo_order: list or tuple, optional A topological order for G (if None, the function will compute one) Returns ------- generator object Raises ------ NetworkXNotImplemented If `G` is not directed NetworkXUnfeasible If `G` contains a cycle Notes ----- This function was originally developed by Peter Jipsen and Franco Saliola for the SAGE project. It's included in NetworkX with permission from the authors. Original SAGE code at: https://github.com/sagemath/sage/blob/master/src/sage/combinat/posets/hasse_diagram.py References ---------- .. [1] Free Lattices, by R. Freese, J. Jezek and J. B. Nation, AMS, Vol 42, 1995, p. 226. """ if topo_order is None: topo_order = list(nx.topological_sort(G)) TC = nx.transitive_closure_dag(G, topo_order) antichains_stacks = [([], list(reversed(topo_order)))] while antichains_stacks: (antichain, stack) = antichains_stacks.pop() # Invariant: # - the elements of antichain are independent # - the elements of stack are independent from those of antichain yield antichain while stack: x = stack.pop() new_antichain = antichain + [x] new_stack = [ t for t in stack if not ((t in TC[x]) or (x in TC[t])) ] antichains_stacks.append((new_antichain, new_stack))
def all_dominators(sdfg: SDFG, idom: Dict[SDFGState, SDFGState] = None) -> Dict[SDFGState, Set[SDFGState]]: """ Returns a mapping between each state and all its dominators. """ idom = idom or nx.immediate_dominators(sdfg.nx, sdfg.start_state) # Create a dictionary of all dominators of each node by using the # transitive closure of the DAG induced by the idoms g = nx.DiGraph() for node, dom in idom.items(): if node is dom: # Skip root continue g.add_edge(node, dom) tc = nx.transitive_closure_dag(g) alldoms: Dict[SDFGState, Set[SDFGState]] = {sdfg.start_state: set()} for node in tc: alldoms[node] = set(dst for _, dst in tc.out_edges(node)) return alldoms
def remove_very_indirect_dependencies( g: nx.DiGraph, attrs: List[Attr], modified_files: List[str] ) -> nx.DiGraph: """This is fairly subtle, and it requires a bit of tuning and just looking at the results on multiple PRs to see if it's reasonable. It's supposed to do a few things: 1. `g` is the build graph. it's a DAG. each edge points from an more leaf-like package to a more core package that the leaf-like package depends on. For example, there's an edge from a numpy derivation to a python derivation. 2. the graph `g` is over .drvs. many of these .drvs correspond to named attributes in nixpkgs, like python38Packages.numpy. Many of them do not. .drvs that are not attrs include `src`s and other stuff that's not addressable as a nixpkgs derivation. generally this is stuff that users do not care about. 3. for each attr, with some execptions, we have the position filename and line number where it was created. We also know the set of modified files in this PR. From that, we can form a reasonable _guess_ of which attrs were actually _directly modified_ in the pr. the other attrs are ones that are downstream from the modification. 4. The identification in (3) is both under- and over- inclusive. it's over- inclusive because we only look for a match at the level of the filename, so if a file contains many attrs then all of them will be flaged. it's under- inclusive because the PR may have changed a file that's used in attrs, but is not itself where any attr is declared. for example if someone changes a shell hook function and only edits a `.sh` file. 5. so, because of (3) and (4), we have two schemes for identifying the "root" attrs that were actually changed by the PR. the first is simply which attrs are in files that were changed. the second is more graph theoretical. we look for the set of all *longest paths* in the DAG of drvs to be built, measuring the length of the path by the number of edges that include at least 1 attr in the edge. so edges that purely include non-attr .drvs don't count. and then we look at the deepest attr in each of these longest paths. these seem to capture the roots of the DAG pretty well. 6. then, armed with these roots, we look for all nodes that are within 2 hops of a root, and throw away everything else. so we keep the roots themselves. we keep packages that directly link against these roots. and we keep edges that directly link against the packages that link to the roots. dependencies that are futher away than that get discarded. 7. for each of these packages that are going to be retained, we record the number of ancestors they have. that is, how many attrs in the build graph depend directly or indirectly on them. note that this includes attrs in the *full graph* -- do this calculation before discarding the set that need to be discarded as described in (6). the purpose of this calculation is so that we can assign a relative importance to every package that we're keeping. things that are depended on by more packages are more important. So finally, the result is a new graph """ non_autogenerated_modified_files = { m for m in modified_files if m not in BIG_AUTOGENERATED_FILES } # # Determine which attrs were modified directly by the git commit, rather than modified # indirectly because their inputs changed # drv_to_attr = { a.drv_path: a for a in attrs if a.drv_path is not None and a.position is not None } # Record if an edge connects to Attrs. There are some really long un-interesting # paths that relate to multi-lib or cross-compilation or something. for e in g.edges(): g.edges[e]["is_attr"] = int(e[0] in drv_to_attr or e[1] in drv_to_attr) build_roots = { n: drv_to_attr[n].name for n in g.nodes if ( n in drv_to_attr and drv_to_attr[n].filename() in non_autogenerated_modified_files ) } log.info("Directly modified attrs", roots=json.dumps(sorted(build_roots.values()))) # # Sometimes the scheme above might give no build roots, if for example # the only file edited was a hook or something and not where the attrs are # named. We want to have at least 1 build root, so if this happens lets # find the longest path in the dag counting only attrs, so like attr a depends # on b depends on c depends on d, and then let's call attr d the build root. # longest_path_build_roots = {} for long_path in dag_longest_paths(g, weight="is_attr"): *_, long_path_end = (n for n in long_path if n in drv_to_attr) long_path_end_name = drv_to_attr[long_path_end].name # log.info("Longest path", name=long_path_end_name) build_roots[long_path_end] = long_path_end_name longest_path_build_roots[long_path_end] = long_path_end_name log.info( "Longest-path build roots", roots=sorted(longest_path_build_roots.values()) ) log.info('Consensus build "roots"', roots=sorted(build_roots.values())) assert len(build_roots) > 0 or len(g) == 0 path_length_counts: Counter_t[int] = Counter() to_keep = set() for root in build_roots: g.nodes[root]["is_root"] = True for node, path in nx.single_target_shortest_path(g, root, 2).items(): # If node == root, len(path) == 1. Keep that, of course # If node is a direct dependency of root, len(path) == 2. Keep that too. # If node ia a 1-step indirect dependency of root, len(path) == 3. Keep that. # Otherwise, throw away. to_keep.add(node) path_length_counts[len(path) - 1] += 1 # # Compute the closure of everything in to_keep, because we've gotta keep # those too. before_ancestor = time.time() transitive_closure = nx.transitive_closure_dag(g) if time.time() - before_ancestor > 1: log.info( f"Computing transitive closure of build graph: {time.time() - before_ancestor:.2f} sec" ) # # Since the changed attrs were not necesarily actually roots of the build graph # (this happens if we're on staging and some unchanged dependencies of the changed # attrs actually haven't been built yet), we need to keep them in the graph. this # confused be at first, but if we don't keep these things in the build graph, # then the precedence-constrained knapsack problem won't be working with the right # information, because it won't know that these packages need to be built in order # to build the stuff we actually care about. if the "build roots" are really roots, # then these should be empty. # to_keep_closure = set() for k in to_keep: to_keep_closure.update(transitive_closure.succ[k]) to_keep.update(to_keep_closure) if len(g) < 500: # Ehhhh, just keep everything if the number of nodes is small enough to_keep.update(g.nodes()) for n in to_keep: n_ancestors = sum(nn in drv_to_attr for nn in transitive_closure.pred[n]) g.nodes[n]["n_ancestors"] = n_ancestors log.info( "Removing remote dependencies", kept=len(to_keep), removed=g.number_of_nodes() - len(to_keep), total=g.number_of_nodes(), ) if DEBUG: print("To remove") import IPython IPython.embed() return g.subgraph(to_keep), transitive_closure.subgraph(to_keep)