def test_reversed_directed_traversal(graph: DiGraph):
result = list(dependent_node_iterator(graph))
assert len(result) == 3 # 3 steps to complete
assert result == [
[3, 5, 7, 10, 11, 13], # step 1
[2, 6, 9, 12], # step 2
[1, 4, 8], # step 3
]
def cleanup(self) -> None:
if not ArgumentParser.args.cleanup:
log.error(
("Cleanup called but --cleanup flag not provided at startup"
" - ignoring call"))
return
log.info("Running cleanup")
# create a subgraph of all the nodes that have a delete edge
delete_graph = DiGraph(self.graph.edge_type_subgraph(EdgeType.delete))
# from that graph delete all the nodes not marked for cleanup
for node in list(delete_graph.nodes):
if not node.clean:
delete_graph.remove_node(node)
# add all the nodes that are supposed to be cleaned
# but do not have a delete edge so weren't part of the
# subgraph
for node in self.graph.nodes:
if node.clean and node not in delete_graph:
delete_graph.add_node(node)
cleanup_nodes = list(delete_graph.nodes)
for node in cleanup_nodes:
log.debug(f"Adding {node.rtdname} to cleanup plan")
log.debug(f"Sending {len(cleanup_nodes)} nodes to pre-cleanup pool")
with ThreadPoolExecutor(
max_workers=ArgumentParser.args.cleanup_pool_size,
thread_name_prefix="pre_cleaner",
) as executor:
executor.map(self.pre_clean, cleanup_nodes)
log.debug(f"Running parallel cleanup on {len(cleanup_nodes)} nodes")
parallel_pass_num = 1
for nodes in dependent_node_iterator(delete_graph):
log.debug(
f"Cleaning {len(nodes)} nodes in {ordinal(parallel_pass_num)} pass"
)
with ThreadPoolExecutor(
max_workers=ArgumentParser.args.cleanup_pool_size,
thread_name_prefix="cleaner",
) as executor:
executor.map(self.clean, nodes)
parallel_pass_num += 1
def test_empty_graph():
assert list(dependent_node_iterator(DiGraph())) == []
def test_delete_nodes(graph: DiGraph):
to_delete = graph.copy()
for parallel in dependent_node_iterator(graph):
for node in parallel:
to_delete.remove_node(node)
assert len(to_delete.nodes) == 0