def sorted_ephemeral_ancestors( self, manifest: Manifest, unique_id: str ) -> Iterable[str]: """Get the ephemeral ancestors of unique_id, stopping at the first non-ephemeral node in each chain, in graph-topological order. """ to_check: Set[str] = {unique_id} ephemerals: Set[str] = set() visited: Set[str] = set() while to_check: # note that this avoids collecting unique_id itself nextval = to_check.pop() for pred in self.graph.predecessors(nextval): if pred in visited: continue visited.add(pred) node = manifest.expect(pred) if node.resource_type != NodeType.Model: continue if node.get_materialization() != 'ephemeral': # type: ignore continue # this is an ephemeral model! We have to find everything it # refs and do it all over again until we exhaust them all ephemerals.add(pred) to_check.add(pred) ephemeral_graph = self.build_subset_graph(ephemerals) # we can just topo sort this because we know there are no cycles. return nx.topological_sort(ephemeral_graph)
def write_graph(self, outfile: str, manifest: Manifest): """Write the graph to a gpickle file. Before doing so, serialize and include all nodes in their corresponding graph entries. """ out_graph = self.graph.copy() for node_id in self.graph.nodes(): data = manifest.expect(node_id).to_dict(omit_none=True) out_graph.add_node(node_id, **data) nx.write_gpickle(out_graph, outfile)