def gen_top_choices(src):
source_node = [n for n in src.nodes() if is_source_node(src.nodes[n])][0]
sink_node = [n for n in src.nodes() if is_sink_node(src.nodes[n])][0]
branch_nodes = [n for n in src.nodes() if is_branch_node(src.nodes[n])]
merge_nodes = [n for n in src.nodes() if is_merge_node(src.nodes[n])]
gtr = nx.transitive_closure(src)
branches_gr = nx.subgraph(gtr, branch_nodes + merge_nodes)
paths = []
min_branches = [b for b in branch_nodes if len(branches_gr.in_edges(b)) == 0]
for branch in min_branches:
for (b, choice) in src.out_edges(branch):
new_graph = nx.DiGraph(src)
for (in_node, b1) in new_graph.in_edges(branch):
new_graph.add_edge(in_node, choice, branch_src = branch)
new_graph = nx.DiGraph(nx.subgraph(new_graph, [n for n in new_graph.nodes() if n != branch]))
matching_merges = [m for (a,m) in branches_gr.out_edges(branch) if m in merge_nodes and len(branches_gr.out_edges(m)) == 0]
path = {branch: (choice, matching_merges)}
for merge in matching_merges:
for (in_node, a) in new_graph.in_edges(merge):
for (a, out_node) in new_graph.out_edges(merge):
new_graph.add_edge(in_node, out_node, merge_src = merge)
new_graph = nx.subgraph(new_graph, [n for n in new_graph.nodes() if n != merge])
new_graph_tr = nx.transitive_closure(new_graph)
reachable = [x for (a,x) in new_graph_tr.out_edges(source_node)]+[source_node]
new_graph = nx.subgraph(new_graph, reachable)
paths.append((path, new_graph))
return paths
def test_transitive_closure(self):
G = nx.DiGraph([(1, 2), (2, 3), (3, 4)])
solution = [(1, 2), (1, 3), (1, 4), (2, 3), (2, 4), (3, 4)]
assert_edges_equal(nx.transitive_closure(G).edges(), solution)
G = nx.DiGraph([(1, 2), (2, 3), (2, 4)])
solution = [(1, 2), (1, 3), (1, 4), (2, 3), (2, 4)]
assert_edges_equal(nx.transitive_closure(G).edges(), solution)
G = nx.DiGraph([(1, 2), (2, 3), (3, 1)])
solution = [(1, 2), (2, 1), (2, 3), (3, 2), (1, 3), (3, 1)]
soln = sorted(solution + [(n, n) for n in G])
assert_edges_equal(sorted(nx.transitive_closure(G).edges()), soln)
G = nx.Graph([(1, 2), (2, 3), (3, 4)])
pytest.raises(nx.NetworkXNotImplemented, nx.transitive_closure, G)
# test if edge data is copied
G = nx.DiGraph([(1, 2, {"a": 3}), (2, 3, {"b": 0}), (3, 4)])
H = nx.transitive_closure(G)
for u, v in G.edges():
assert G.get_edge_data(u, v) == H.get_edge_data(u, v)
k = 10
G = nx.DiGraph((i, i + 1, {"f": "b", "weight": i}) for i in range(k))
H = nx.transitive_closure(G)
for u, v in G.edges():
assert G.get_edge_data(u, v) == H.get_edge_data(u, v)
def getmaxcommonsubgraphsize(G, H):
numnodes = find_num_nodes(G, H)
G = nx.transitive_closure(G)
H = nx.transitive_closure(H)
matching_graph = makematchinggraph(G, H)
labels = [v for v in matching_graph.nodes()]
mg_nodes = dict([(n, [e for e in matching_graph.neighbors(n)])
for n in matching_graph])
Gn, Hn = zip(*labels)
def pick_nodes(G_list, node_dict):
if len(G_list) == 0:
return (numnodes + num_edges(node_dict))
sizes = 0
for node in mg_nodes:
if node[0] == G_list[0] and not node_in(node[1], node_dict):
#check if node in matching graph corresponds to our node in G and check if node choice is valid
sizes = max(
sizes,
pick_nodes(G_list[1:], {
**node_dict, node: mg_nodes[node]
}))
#pick this node and then make recursive call to pick other nodes
if numnodes < len(node_dict) + len(G_list):
sizes = max(sizes, pick_nodes(G_list[1:], node_dict))
#Check the case in which no node in the mathcing graph corrisponds to G_list[0]
return (sizes)
#Gn is the list of nodes in G, for each node in G we must pick a node in the matching graph.
return pick_nodes(list(set(Gn)), {})
def dag_distance(G,H):
'''
G and H are nx.DiGraph objects with node labels, representing ACYCLIC graphs.
Returns the distance between G and H.
'''
G = nx.transitive_closure(G)
H = nx.transitive_closure(H)
return 1 - getmaxcommonsubgraphsize(G,H)/(max(len(G.edges), len(H.edges)))
def mmd(G, k=2, already_tc=False):
"""
Calculate the Myrheim-Meyer dimension of a DAG
Parameters
----------
G : Networkx DiGraph
k : int
Length of chains to count - default to 2
"""
if G.number_of_edges() == 0:
return 0
if not already_tc:
G = nx.transitive_closure(G)
N = G.number_of_nodes()
if k == 2:
# this is a special case where we can use the inbuilt
# number_of_edges function
S = G.number_of_edges()
f_D = float(S) / (N ** 2.0)
else:
S = dag.count_chains(G, k)
f_D = float(S) / (N ** k)
# lookup inverse of f_D(k) to find D estimate
D = mmd_lookup(f_D, k)
return D
def trans_close(G):
#trans_closure_G = nx.transitive_closure(G, False) # for newer versions of networkx
trans_closure_G = nx.transitive_closure(G)
A_star = nx.to_scipy_sparse_matrix(trans_closure_G, dtype = np.uint8)
return A_star
def reachability_using_kron(pda_graph: nx.Graph, graph: nx.Graph,
get_nonterminals, grammar: CFGrammar):
if grammar.produces_eps():
for i in range(len(graph.nodes)):
graph.add_edge(i,
i,
label=grammar.get_eps_producing_nonterminals())
changes = True
while changes:
changes = False
m3 = my_tensor_prod(pda_graph, graph)
m3_closure = nx.transitive_closure(m3)
for edge in m3_closure.edges(data='label'):
s = edge[0][0]
f = edge[1][0]
if pda_graph.nodes[s]['is_start'] and pda_graph.nodes[f][
'is_final']:
x, y = edge[0][1], edge[1][1]
if y not in graph[x].keys():
graph.add_edge(x, y, label=[get_nonterminals[s]])
changes = True
elif get_nonterminals[s] not in graph[x][y]['label']:
changes = True
graph[x][y]['label'].append(get_nonterminals[s])
return graph
def test_issue_2127(self):
"""Test from issue 2127"""
# Build the example DAG
G = nx.DiGraph()
G.add_edge("A", "C")
G.add_edge("A", "B")
G.add_edge("C", "E")
G.add_edge("C", "D")
G.add_edge("E", "G")
G.add_edge("E", "F")
G.add_edge("G", "I")
G.add_edge("G", "H")
tc = nx.transitive_closure(G)
btc = nx.Graph()
# Create a bipartite graph based on the transitive closure of G
for v in tc.nodes():
btc.add_node((0, v))
btc.add_node((1, v))
for u, v in tc.edges():
btc.add_edge((0, u), (1, v))
top_nodes = {n for n in btc if n[0] == 0}
matching = hopcroft_karp_matching(btc, top_nodes)
vertex_cover = to_vertex_cover(btc, matching, top_nodes)
independent_set = set(G) - {v for _, v in vertex_cover}
assert_equal({'B', 'D', 'F', 'I', 'H'}, independent_set)
def test_issue_2127(self):
"""Test from issue 2127"""
# Build the example DAG
G = nx.DiGraph()
G.add_edge("A", "C")
G.add_edge("A", "B")
G.add_edge("C", "E")
G.add_edge("C", "D")
G.add_edge("E", "G")
G.add_edge("E", "F")
G.add_edge("G", "I")
G.add_edge("G", "H")
tc = nx.transitive_closure(G)
btc = nx.Graph()
# Create a bipartite graph based on the transitive closure of G
for v in tc.nodes():
btc.add_node((0, v))
btc.add_node((1, v))
for u, v in tc.edges():
btc.add_edge((0, u), (1, v))
top_nodes = set([n for n in btc if n[0] == 0])
matching = hopcroft_karp_matching(btc, top_nodes)
vertex_cover = to_vertex_cover(btc, matching, top_nodes)
independent_set = set(G) - set([v for _, v in vertex_cover])
assert_equal(set(['B', 'D', 'F', 'I', 'H']), independent_set)
def ilp(G, k):
dist = dict(nx.all_pairs_shortest_path_length(G))
closure = nx.transitive_closure(G)
candidates = {v: {v} | set(closure.predecessors(v)) for v in G.nodes}
m = gurobi.Model()
y = {}
for j in G.nodes:
y[j] = m.addVar(vtype=gurobi.GRB.BINARY)
x = {}
for i in G.nodes:
for j in candidates[i]:
x[i, j] = m.addVar(vtype=gurobi.GRB.BINARY)
m.setObjective(sum(x[i, j] * dist[j][i] for i, j in x.keys()))
m.addConstr(sum(y[j] for j in G.nodes) <= k)
for i in G.nodes:
m.addConstr(sum(x[i, j] for j in candidates[i]) == 1)
for i, j in x.keys():
m.addConstr(x[i, j] <= y[j])
m.optimize()
return [j for j in G.nodes if y[j].x > 0.5]
def setUp(self):
g = nx.DiGraph()
g.add_path(range(5))
g = nx.transitive_closure(g)
for s, t in g.edges_iter():
g[s][t]['c'] = 1
self.g = g
def part_one(puzz_input):
orbits = nx.DiGraph()
for l in puzz_input:
p, s = l.split(')')
orbits.add_edge(p, s)
print(nx.transitive_closure(orbits).size())
def order_inscriptions(graph):
transitive_closure = networkx.transitive_closure(graph)
logging.info("{0} nodes, {1} edges in transtive closure.".format(transitive_closure.number_of_nodes(),
transitive_closure.number_of_edges()))
num_predecessors_nodes = [(len(transitive_closure.predecessors(node)), node) for node in transitive_closure]
for n in sorted(num_predecessors_nodes, key=lambda tup: tup[0]):
print '%i %s' % (n[0], n[1])
def apply_pass(self, sdfg: SDFG, _) -> Dict[SDFGState, Set[SDFGState]]:
"""
:return: A dictionary mapping each state to its other reachable states.
"""
reachable: Dict[SDFGState, Set[SDFGState]] = {}
tc: nx.DiGraph = nx.transitive_closure(sdfg.nx)
for state in sdfg.nodes():
reachable[state] = set(tc.successors(state))
return reachable
def antichains(G, topo_order=None):
"""Generates antichains from a directed acyclic graph (DAG).
An antichain is a subset of a partially ordered set such that any
two elements in the subset are incomparable.
Parameters
----------
G : NetworkX DiGraph
A directed acyclic graph (DAG)
topo_order: list or tuple, optional
A topological order for G (if None, the function will compute one)
Returns
-------
generator object
Raises
------
NetworkXNotImplemented
If `G` is not directed
NetworkXUnfeasible
If `G` contains a cycle
Notes
-----
This function was originally developed by Peter Jipsen and Franco Saliola
for the SAGE project. It's included in NetworkX with permission from the
authors. Original SAGE code at:
https://github.com/sagemath/sage/blob/master/src/sage/combinat/posets/hasse_diagram.py
References
----------
.. [1] Free Lattices, by R. Freese, J. Jezek and J. B. Nation,
AMS, Vol 42, 1995, p. 226.
"""
if topo_order is None:
topo_order = nx.topological_sort(G)
TC = nx.transitive_closure(G)
antichains_stacks = [([], list(topo_order)[-1::-1])]
while antichains_stacks:
(antichain, stack) = antichains_stacks.pop()
# Invariant:
# - the elements of antichain are independent
# - the elements of stack are independent from those of antichain
yield antichain
while stack:
x = stack.pop()
new_antichain = antichain + [x]
new_stack = [
t for t in stack if not ((t in TC[x]) or (x in TC[t]))
]
antichains_stacks.append((new_antichain, new_stack))
def make_move(self, a, f_testing=False):
if params.debug_mode >= 2:
print('making move', a)
self.G.add_edges_from([a])
self.E.remove_edges_from([a])
# Remove inconsistent edges from E
G_transitive_closure = nx.transitive_closure(self.G)
Ec = self.E.copy().edges()
for e in Ec:
if G_transitive_closure.has_edge(e[1], e[0]):
self.E.remove_edges_from([e])
# Add bridge edges from E to G
T = self.get_legal_actions()
Gc = self.G.copy()
Gc.add_edges_from(T)
scc = [
list(g.edges())
for g in nx.strongly_connected_component_subgraphs(Gc, copy=True)
if len(g.edges()) != 0
]
bridges = set(Gc.edges()) - set(itertools.chain(*scc))
self.G.add_edges_from(bridges)
self.E.remove_edges_from(bridges)
T = list(set(T) - bridges)
# Remove "redundant edges": if there is already path from e[0] to e[1], can immediately add e
redundant_edges = set()
for e in T:
if G_transitive_closure.has_edge(e[0], e[1]):
redundant_edges.add(e)
self.G.add_edges_from([e])
self.E.remove_edges_from([e])
self.stats.num_nodes += 1
if params.use_visited:
self.update_visited()
if not f_testing:
self.running_nodes += 1
if self.running_nodes % params.print_loss_every == 0:
print("*******LOSS:",
self.running_loss / params.print_loss_every)
if self.loss_output_file:
self.loss_output_file.write(
str(self.running_nodes) + '\t' +
str(self.running_loss / params.print_loss_every) +
'\n')
self.loss_output_file.flush()
self.running_loss = 0
def hypernym_transitive_closure(G):
hyperedges = [
e for e in G.edges(data=True) if e[-1]['relation'] == 'hypernym'
]
hyperedgenodes = set(flatten([(n1, n2) for n1, n2, __ in hyperedges]))
hypergraph = nx.DiGraph(G.subgraph(hyperedgenodes))
tc = nx.transitive_closure(hypergraph)
for n1, n2 in set(tc.edges()) - set(hypergraph.edges()):
G.add_edges_from([(n1, n2, {'relation': 'hypernym'})])
return G
def bipartite_transitive_closure(self):
"""Create the bipartite graph of the transitive closure of this pipeline."""
self.btc = Graph()
tc = transitive_closure(self.dag)
for task in tc.nodes_iter():
self.btc.add_node((0, task))
self.btc.add_node((1, task))
for u, v in tc.edges_iter():
self.btc.add_edge((0, u), (1, v))
def recalculate_function_can_throw_info(module: ir3.Module):
if not module.function_defns:
return module
function_dependency_graph = nx.DiGraph()
function_defn_by_name = {
function_defn.name: function_defn
for function_defn in module.function_defns
}
for function_defn in module.function_defns:
function_dependency_graph.add_node(function_defn.name)
for global_function_name in get_referenced_global_function_names(
function_defn):
if global_function_name in function_defn_by_name.keys():
function_dependency_graph.add_edge(function_defn.name,
global_function_name)
condensed_graph = nx.condensation(function_dependency_graph)
assert isinstance(condensed_graph, nx.DiGraph)
function_dependency_graph_transitive_closure = nx.transitive_closure(
function_dependency_graph)
assert isinstance(function_dependency_graph_transitive_closure, nx.DiGraph)
# Determine which connected components can throw.
condensed_node_can_throw = defaultdict(lambda: False)
for connected_component_index in nx.topological_sort(condensed_graph,
reverse=True):
condensed_node = condensed_graph.node[connected_component_index]
# If a function in this connected component can throw, the whole component can throw.
for function_name in condensed_node['members']:
if function_contains_raise_stmt(
function_defn_by_name[function_name]):
condensed_node_can_throw[connected_component_index] = True
# If a function in this connected component calls a function in a connected component that can throw, this
# connected component can also throw.
for called_condensed_node_index in condensed_graph.successors(
connected_component_index):
if condensed_node_can_throw[called_condensed_node_index]:
condensed_node_can_throw[connected_component_index] = True
function_can_throw = dict()
for connected_component_index in condensed_graph:
for function_name in condensed_graph.node[connected_component_index][
'members']:
function_can_throw[function_name] = condensed_node_can_throw[
connected_component_index]
return apply_function_can_throw_info(module, function_can_throw)
def antichains(G):
"""Generates antichains from a DAG.
An antichain is a subset of a partially ordered set such that any
two elements in the subset are incomparable.
Parameters
----------
G : NetworkX DiGraph
Graph
Returns
-------
antichain : generator object
Raises
------
NetworkXNotImplemented
If G is not directed
NetworkXUnfeasible
If G contains a cycle
Notes
-----
This function was originally developed by Peter Jipsen and Franco Saliola
for the SAGE project. It's included in NetworkX with permission from the
authors. Original SAGE code at:
https://sage.informatik.uni-goettingen.de/src/combinat/posets/hasse_diagram.py
References
----------
.. [1] Free Lattices, by R. Freese, J. Jezek and J. B. Nation,
AMS, Vol 42, 1995, p. 226.
"""
TC = nx.transitive_closure(G)
antichains_stacks = [([], nx.topological_sort(G, reverse=True))]
while antichains_stacks:
(antichain, stack) = antichains_stacks.pop()
# Invariant:
# - the elements of antichain are independent
# - the elements of stack are independent from those of antichain
yield antichain
while stack:
x = stack.pop()
new_antichain = antichain + [x]
new_stack = [
t for t in stack if not ((t in TC[x]) or (x in TC[t]))
]
antichains_stacks.append((new_antichain, new_stack))
def __init__(
self,
debug=False,
dim=2,
loss_fn='ec',
title_text='',
weights_to_load='/cluster/scratch/adhall/exp/ethec/final_ec_full/load_emb_5k/50-50_hide_levels/ec_2d_2xlr_init_5k/weights/best_model.pth'
):
# weights_to_load='/home/ankit/Desktop/emb_weights/joint_2xlr/best_model_model.pth'):
torch.manual_seed(0)
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.title_text = title_text
labelmap = ETHECLabelMapMerged()
if debug:
labelmap = ETHECLabelMapMergedSmall()
path_to_folder = '../database/ETHEC/ETHECSmall_embeddings/graphs'
else:
path_to_folder = '../database/ETHEC/ETHEC_embeddings/graphs'
G = nx.read_gpickle(os.path.join(path_to_folder, 'G'))
self.G = G
self.G_tc = nx.transitive_closure(self.G)
self.labelmap = labelmap
if loss_fn == 'ec':
self.model = Embedder(embedding_dim=dim,
labelmap=labelmap,
normalize=False,
K=3.0)
elif loss_fn == 'oe':
self.model = Embedder(embedding_dim=dim,
labelmap=labelmap,
normalize=False) #, K=3.0)
self.model = nn.DataParallel(self.model)
self.loss_fn = loss_fn
self.weights_to_load = weights_to_load
self.load_model()
self.title_text = title_text
if self.title_text == '':
self.title_text = 'F1 score: {:.4f} Accuracy: {:.4f} \n Precision: {:.4f} Recall: {:.4f} | Threshold: {:.4f}'.format(
self.reconstruction_f1, self.reconstruction_accuracy,
self.reconstruction_prec, self.reconstruction_recall,
self.reconstruction_threshold)
# run vizualize
self.vizualize()
def recalculate_function_can_throw_info(module: ir.Module, context_object_file_content: ObjectFileContent):
if not module.function_defns:
return module
function_dependency_graph = nx.DiGraph()
function_defn_by_name = {function_defn.name: function_defn
for function_defn in module.function_defns}
for function_defn in module.function_defns:
function_dependency_graph.add_node(function_defn.name)
for global_function_name in get_referenced_global_function_names(function_defn):
if global_function_name in function_defn_by_name.keys():
function_dependency_graph.add_edge(function_defn.name, global_function_name)
condensed_graph = nx.condensation(function_dependency_graph)
assert isinstance(condensed_graph, nx.DiGraph)
function_dependency_graph_transitive_closure = nx.transitive_closure(function_dependency_graph)
assert isinstance(function_dependency_graph_transitive_closure, nx.DiGraph)
# Determine which connected components can throw.
condensed_node_can_throw = defaultdict(lambda: False)
for connected_component_index in reversed(list(nx.lexicographical_topological_sort(condensed_graph))):
condensed_node = condensed_graph.node[connected_component_index]
# If a function in this connected component can throw, the whole component can throw.
for function_name in condensed_node['members']:
if function_contains_raise_stmt(function_defn_by_name[function_name]):
condensed_node_can_throw[connected_component_index] = True
# If a function in this connected component calls a function in a connected component that can throw, this
# connected component can also throw.
for called_condensed_node_index in condensed_graph.successors(connected_component_index):
if condensed_node_can_throw[called_condensed_node_index]:
condensed_node_can_throw[connected_component_index] = True
function_can_throw = dict()
for connected_component_index in condensed_graph:
for function_name in condensed_graph.node[connected_component_index]['members']:
function_can_throw[function_name] = condensed_node_can_throw[connected_component_index]
external_function_can_throw = dict()
for module_name, module_info in context_object_file_content.modules_by_name.items():
for elem in itertools.chain(module_info.ir2_module.custom_types, module_info.ir2_module.function_defns):
if elem.name in module_info.ir2_module.public_names:
external_function_can_throw[(module_name, elem.name)] = (isinstance(elem, ir.FunctionDefn)
and (function_contains_raise_stmt(elem)
or function_contains_var_reference_that_can_throw(elem)))
return apply_function_can_throw_info(module, function_can_throw, external_function_can_throw)
def recalculate_template_instantiation_can_trigger_static_asserts_info(
header: ir.Header):
if not header.template_defns:
return header
template_defn_by_name = {
template_defn.name: template_defn
for template_defn in header.template_defns
}
template_defn_dependency_graph = compute_template_dependency_graph(
header.template_defns, template_defn_by_name)
condensed_graph = nx.condensation(template_defn_dependency_graph)
assert isinstance(condensed_graph, nx.DiGraph)
template_defn_dependency_graph_transitive_closure = nx.transitive_closure(
template_defn_dependency_graph)
assert isinstance(template_defn_dependency_graph_transitive_closure,
nx.DiGraph)
# Determine which connected components can trigger static assert errors.
condensed_node_can_trigger_static_asserts = defaultdict(lambda: False)
for connected_component_index in reversed(
list(nx.lexicographical_topological_sort(condensed_graph))):
condensed_node = condensed_graph.node[connected_component_index]
# If a template defn in this connected component can trigger a static assert, the whole component can.
for template_defn_name in condensed_node['members']:
if _template_defn_contains_static_assert_stmt(
template_defn_by_name[template_defn_name]):
condensed_node_can_trigger_static_asserts[
connected_component_index] = True
# If a template defn in this connected component references a template defn in a connected component that can
# trigger static asserts, this connected component can also trigger them.
for called_condensed_node_index in condensed_graph.successors(
connected_component_index):
if condensed_node_can_trigger_static_asserts[
called_condensed_node_index]:
condensed_node_can_trigger_static_asserts[
connected_component_index] = True
template_defn_can_trigger_static_asserts = dict()
for connected_component_index in condensed_graph:
for template_defn_name in condensed_graph.node[
connected_component_index]['members']:
template_defn_can_trigger_static_asserts[
template_defn_name] = condensed_node_can_trigger_static_asserts[
connected_component_index]
return _apply_template_instantiation_can_trigger_static_asserts_info(
header, template_defn_can_trigger_static_asserts)
def antichains(G):
"""Generates antichains from a DAG.
An antichain is a subset of a partially ordered set such that any
two elements in the subset are incomparable.
Parameters
----------
G : NetworkX DiGraph
Graph
Returns
-------
antichain : generator object
Raises
------
NetworkXNotImplemented
If G is not directed
NetworkXUnfeasible
If G contains a cycle
Notes
-----
This function was originally developed by Peter Jipsen and Franco Saliola
for the SAGE project. It's included in NetworkX with permission from the
authors. Original SAGE code at:
https://sage.informatik.uni-goettingen.de/src/combinat/posets/hasse_diagram.py
References
----------
.. [1] Free Lattices, by R. Freese, J. Jezek and J. B. Nation,
AMS, Vol 42, 1995, p. 226.
"""
TC = nx.transitive_closure(G)
antichains_stacks = [([], list(reversed(list(nx.topological_sort(G)))))]
while antichains_stacks:
(antichain, stack) = antichains_stacks.pop()
# Invariant:
# - the elements of antichain are independent
# - the elements of stack are independent from those of antichain
yield antichain
while stack:
x = stack.pop()
new_antichain = antichain + [x]
new_stack = [
t for t in stack if not ((t in TC[x]) or (x in TC[t]))]
antichains_stacks.append((new_antichain, new_stack))
def order_inscriptions(graph):
"""Print a list of inscriptions heuristically ordered by their date of writing"""
output_dir = faust.config.get("macrogenesis", "output-dir")
order_file_name = 'order.txt'
with open(os.path.join(output_dir, order_file_name), mode='w') as order_file:
transitive_closure = networkx.transitive_closure(graph)
logging.info("{0} nodes, {1} edges in transtive closure.".format(transitive_closure.number_of_nodes(),
transitive_closure.number_of_edges()))
num_predecessors_nodes = [(len(transitive_closure.predecessors(node)), node) for node in transitive_closure]
for n in sorted(num_predecessors_nodes, key=lambda tup: tup[0]):
order_file.write('%i %s\n' % (n[0], n[1]))
# print '%i %s' % (n[0], n[1])
return [("Inscription order", order_file_name)]
def main():
input_filename = 'input.txt'
with open(input_filename, 'r') as input:
G = nx.DiGraph()
for line in input:
add_edges(line, G)
# star 1:
transitive_closure = nx.transitive_closure(G)
num_pred = len(list(transitive_closure.predecessors('shiny gold')))
print('We have {} predecessors of "shiny gold"'.format(num_pred))
#star 2:
needed_bags = get_total_weight(G, 'shiny gold')
print('The shiny gold bag contains {} bags.'.format(needed_bags))
def analyse_graph(graph):
print "graph"
print "{0} nodes, {1} edges in macrogenetic graph.".format(graph.number_of_nodes(),
graph.number_of_edges())
absolutely_dated_notes_sorted = macrogenesis.absolutely_dated_nodes_sorted(graph)
print "{0} nodes with absolute datings.".format(len(absolutely_dated_notes_sorted))
num_equal_absolute_datings = macrogenesis.insert_minimal_edges_from_absolute_datings(graph)
print "{0} equal absolute datings.".format(num_equal_absolute_datings)
print "{0} strongly connected components (conflicts).".format(networkx.number_strongly_connected_components(graph))
transitive_closure = networkx.transitive_closure(graph)
print "transitive closure"
print "{0} nodes, {1} edges in transtive closure.".format(transitive_closure.number_of_nodes(),
transitive_closure.number_of_edges())
def load_graphs(self, label_embeddings):
if self.check_graph_embedding_neg_graph is None:
start_time = time.time()
path_to_folder = '../database/ETHEC/ETHEC_embeddings/graphs'
self.G = nx.read_gpickle(os.path.join(path_to_folder, 'G'))
self.G = self.G
self.G_tc = nx.transitive_closure(self.G)
# make negative graph
n_nodes = len(list(self.G.nodes()))
A = np.ones((n_nodes, n_nodes), dtype=np.bool)
for u, v in list(self.G.edges()):
# remove edges that are in G_train_tc
A[u, v] = 0
np.fill_diagonal(A, 0)
self.check_graph_embedding_neg_graph = A
self.edges_in_G = self.G.edges()
self.n_nodes_in_G = len(self.G.nodes())
self.nodes_in_G = [i for i in range(self.n_nodes_in_G)]
self.pos_u_list, self.pos_v_list = [], []
for edge in self.edges_in_G:
self.pos_u_list.append(edge[0])
self.pos_v_list.append(edge[1])
self.neg_u_list, self.neg_v_list = [], []
for i_ix in range(n_nodes):
for j_ix in range(n_nodes):
if self.check_graph_embedding_neg_graph[i_ix, j_ix] == 1:
self.neg_u_list.append(i_ix)
self.neg_v_list.append(j_ix)
print('created negative graph in {}'.format(time.time() - start_time))
positive_e = self.dot_operator(label_embeddings[self.pos_u_list, :], label_embeddings[self.pos_v_list, :])
negative_e = self.dot_operator(label_embeddings[self.neg_u_list, :], label_embeddings[self.neg_v_list, :])
metrics = EmbeddingMetrics(positive_e, negative_e, 0.0,
'val', n_proc=self.n_proc)
best_score, best_threshold, best_accuracy, best_precision, best_recall = metrics.calculate_metrics()
print('Checking graph reconstruction: +ve edges {}, -ve edges {}'.format(len(self.edges_in_G),
self.check_graph_embedding_neg_graph.size))
return best_score, best_threshold, best_accuracy, best_precision, best_recall
def __init__(self, multi_graph: nx.MultiDiGraph) -> None:
self.class_graph = nx.transitive_closure(
nx.DiGraph(multi_graph.edges()))
for _u, _v, data in self.class_graph.edges(data=True):
data["eval"] = Evaluation(EdgeClass.UNKNOWN, None)
for u, v, data in multi_graph.edges(data=True):
if self.get_class(u, v) != EdgeClass.UNKNOWN:
continue
measurement = data["measurement"]
if measurement.trust_reason:
self.add_good_edge(u, v, measurement.trust_reason)
else:
ip = getattr(data["record"].data, "ip", None)
if ip and not ip.is_global:
self.add_bad_edge(u, v, "Non global IP")
def extract(cls):
try:
transfers_df = read_csv(
"transfers.txt",
usecols=["from_stop_id", "to_stop_id", "min_transfer_time"],
dtype={
"from_stop_id": str,
"to_stop_id": str
},
)
except FileNotFoundError:
# Create a dummy transfer_df in case of absence of the optional file transfers.txt
transfers_df = pd.DataFrame(
[],
columns=["from_stop_id", "to_stop_id", "min_transfer_time"])
# There might be NaN values in the file, thus we can only change the dtype of min_transfer_time
# to int after removing NaN values
transfers_df = transfers_df.dropna()
transfers_df["min_transfer_time"] = transfers_df[
"min_transfer_time"].astype(int)
g = nx.from_pandas_edgelist(
transfers_df,
"from_stop_id",
"to_stop_id",
"min_transfer_time",
create_using=nx.DiGraph(),
)
print("\nFinding the transitive closure of the transfer graph...")
gt = nx.transitive_closure(g)
dijkstra_lengths = dict(
nx.all_pairs_dijkstra_path_length(g, weight="min_transfer_time"))
# If u and v are the same node, the path length will be 0. In that case,
# we need to take the transfer time from the original graph
transfers = [[
u, v,
dijkstra_lengths[u][v] if u != v else data["min_transfer_time"]
] for u, v, data in gt.edges(data=True)]
Data.transfers = pd.DataFrame(
transfers,
columns=["from_stop_id", "to_stop_id", "min_transfer_time"])
def prepare_naive_strategy(model):
trans_closure = nx.transitive_closure( model['net'] )
naive_score_perts = {}
for pert in model['perturbation_set']:
targets = model['perturbations'][pert]
naive_perturbation_score = sum( len(list(trans_closure.successors(target))) for target in targets )
if naive_perturbation_score in naive_score_perts:
naive_score_perts[naive_perturbation_score].add(pert)
else:
naive_score_perts[naive_perturbation_score] = {pert}
model['naive_score_perts'] = naive_score_perts
model['sorted_scores'] = sorted(naive_score_perts.keys(), reverse=True)
return
def compute_and_print_transitive_closure(self, G):
transG = nx.transitive_closure(G)
nodeCnt = len(transG.nodes)
nodeSet = set([i for i in range(nodeCnt)])
cnt=0
nodecnt=0;
for node_id in transG:
#if this node is not a CNOT, we dont care about it
thisGate = G.nodes[node_id]['details']
if not "CNOT" in thisGate['type']:
continue
nodecnt += 1
print nodecnt
for node_id in transG:
#if this node is not a CNOT, we dont care about it
thisGate = G.nodes[node_id]['details']
if not "CNOT" in thisGate['type']:
continue
parents = [p for p in transG.predecessors(node_id)]
children = [c for c in transG.successors(node_id)]
bad_set = set(parents+children)
bad_set.add(node_id)
good_set = nodeSet.difference(bad_set)
#prune good_set to include only CNOTs
good_set_new = set()
for item in good_set:
if "CNOT" in G.nodes[item]['details']['type']:
if item > node_id: #symmetry breaking - add only (i,j) or (j,i) depending on value
good_set_new.add(item)
good_set = good_set_new
cnt += len(good_set)
print node_id, len(good_set),
for item in list(good_set):
print item,
print ""
for item in good_set:
#check if edge exists in transitive closure :)
assert transG.has_edge(node_id, item)==False
assert transG.has_edge(item, node_id)==False
def get_tier(self, G, E, depth):
if len(E.edges()) == 0:
# There are no possible actions
return []
G_transitive_closure = nx.transitive_closure(G)
max_weight = max([(d['weight']) for (u, v, d) in E.edges(data=True)])
T = [(u, v) for (u, v, d) in E.edges(data=True) if d['weight'] == max_weight]
tier = []
for e in T:
if not G_transitive_closure.has_edge(e[1], e[0]):
Gc = G.copy()
Ec = E.copy()
Gc.add_edges_from([e])
Ec.remove_edges_from([e])
tier.append((e, Gc, Ec, depth+1))
return tier
def get_legal_actions(self, G, E):
if len(E.edges()) == 0:
# There are no possible actions
return []
Gc = G.copy()
G_transitive_closure = nx.transitive_closure(G)
max_weight = max([(d['weight']) for (u, v, d) in E.edges(data=True)])
T = [(u, v) for (u, v, d) in E.edges(data=True)
if d['weight'] == max_weight]
legal_actions = []
for e in T:
if not G_transitive_closure.has_edge(e[1], e[0]):
Gc.add_edges_from([e])
legal_actions.append(e)
Gc.remove_edges_from([e])
return legal_actions
def graph_statistics(macrogenetic_graph):
"""Print statistics about the macrogenenesis graph"""
output_dir = faust.config.get("macrogenesis", "output-dir")
statistics_file_name = 'statistics.txt'
with open(os.path.join(output_dir, statistics_file_name), mode='w') as statistics_file:
statistics_file.write("graph\n")
statistics_file.write("{0} nodes, {1} edges in macrogenetic graph.\n".format(macrogenetic_graph.number_of_nodes(),
macrogenetic_graph.number_of_edges()))
absolutely_dated_notes_sorted = graph._absolutely_dated_nodes_sorted(macrogenetic_graph)
statistics_file.write("{0} nodes with absolute datings.\n".format(len(absolutely_dated_notes_sorted)))
num_equal_absolute_datings = graph.insert_minimal_edges_from_absolute_datings(macrogenetic_graph)
statistics_file.write("{0} equal absolute datings.\n".format(num_equal_absolute_datings))
statistics_file.write("{0} strongly connected components (conflicts).\n".format(
networkx.number_strongly_connected_components(macrogenetic_graph)))
transitive_closure = networkx.transitive_closure(macrogenetic_graph)
statistics_file.write("\ntransitive closure\n")
statistics_file.write("{0} nodes, {1} edges in transitive closure.\n".format(transitive_closure.number_of_nodes(),
transitive_closure.number_of_edges()))
return [("Graph statistics", statistics_file_name)]
def ACTest():
G = nxdr.read_dot(f'dag3/random8.dot')
TC = nx.transitive_closure(G) # Complexity of TC
nxdr.write_dot(TC, f'dump.dot')
antichains_stacks = [([], list(reversed(list(nx.topological_sort(G)))))]
pprint.pprint(antichains_stacks)
i = 0
while antichains_stacks:
(antichain, stack) = antichains_stacks.pop()
print(f'i({i}):AC:{antichain},stack:{stack}')
#yield antichain
j = 0
while stack:
x = stack.pop()
print(f'j({j}),x:{x},stack:{stack},TC[{x}]:{TC[x]}')
new_antichain = antichain + [x]
new_stack = [
t for t in stack if not ((t in TC[x]) or (x in TC[t]))
]
antichains_stacks.append((new_antichain, new_stack))
j = j + 1
i = i + 1
def get_path_graph(self, uris, depth=2):
"""
Based on the paper: https://arxiv.org/pdf/1707.05288.pdf
:param uris: uris to build graph for
:param depth: depth of the paths to search
:return:
"""
edges = set()
nodes = set(uris)
log.info('linker: started building subgraph on {} nodes with depth {}'.
format(len(nodes), depth))
mmap = map # todo: make parallel queries
for i in range(depth - 1):
new_nodes = set()
for uri_nodes, uri_edges in mmap(self._resolve_nodes, nodes):
new_nodes.update(uri_nodes)
edges.update(uri_edges)
nodes = new_nodes
log.info('linker: finished iter {}/{} with {} new nodes, {} edges'.
format(i + 1, depth, len(new_nodes), len(edges)))
# Last step can be done easier
edges.update(self._resolve_edges(uris, nodes))
log.info('linker: finished building subgraph: {} edges'.format(
len(edges)))
graph = nx.DiGraph()
graph.add_nodes_from(uris) # need only original entities
graph.add_edges_from(edges)
subgraph = nx.transitive_closure(graph).subgraph(nbunch=uris)
log.info('linker: ended extracting subgraph: {} edges'.format(
len(subgraph.edges())))
return subgraph
def main():
output_dir = faust.config.get("macrogenesis", "output-dir")
# copy resources
try:
shutil.copytree('macrogenesis/resources/js', os.path.join(output_dir, 'js'))
except OSError as e:
logging.warn(e)
#collect hyperlinks to selected graphs for the TOC as [(link_text_1, relative_link_to_file_1), ...]
links = []
# draw raw input data
graph_imported = macrogenesis.import_graph()
logging.info("Generating raw data graph.")
#UUU write_agraph_layout(agraph_from(graph_imported), output_dir, '00_raw_data')
# highlight a single node and its neighbors
# highlighted_node = 'faust://document/wa/2_I_H.17'
for highlighted_node in graph_imported:
highlighted_bunch = list(networkx.all_neighbors(graph_imported, highlighted_node))
highlighted_bunch.append(highlighted_node)
graph_highlighted_subgraph = graph_imported.subgraph(nbunch=highlighted_bunch).copy()
graph_highlighted_subgraph.node[highlighted_node][KEY_HIGHLIGHT]= VALUE_TRUE
macrogenesis.insert_minimal_edges_from_absolute_datings(graph_highlighted_subgraph)
#, edge_labels=True)
#agraph_highlighted_subgraph.node_attr[highlighted_node]['color'] = 'red'
write_agraph_layout(agraph_from(graph_highlighted_subgraph), output_dir,
highlighted_base_filename(highlighted_node))
# add relationships implicit in absolute datings
logging.info("Generating graph with implicit absolute date relationships.")
graph_absolute_edges = graph_imported
macrogenesis.insert_minimal_edges_from_absolute_datings(graph_absolute_edges)
del graph_imported
base_filename_absolute_edges = '10_absolute_edges'
write_agraph_layout(agraph_from(graph_absolute_edges), output_dir, base_filename_absolute_edges)
links.append(('Raw datings (relative and absolute)', base_filename_absolute_edges))
# again with edge labels
# TODO this breaks graphviz
# agraph_absolute_edges_edge_labels = agraph_from(graph_absolute_edges, edge_labels=True)
# write_agraph_layout(agraph_absolute_edges_edge_labels, output_dir, '15_absolute_edges_edge_labels')
logging.info("Generating condensation.")
strongly_connected_components = list(networkx.strongly_connected_components(graph_absolute_edges))
#condensation
graph_condensation = networkx.condensation(graph_absolute_edges, scc=strongly_connected_components)
for node in graph_condensation:
label = ', '.join([label_from_uri(uri) for uri in graph_condensation.node[node]['members']])
label_width = int(2 * math.sqrt(len(label)))
graph_condensation.node[node]['label'] = textwrap.fill(label, label_width, break_long_words=False).replace('\n','\\n')
component_filename_pattern = '16_strongly_connected_component_%i'
# make a hyperlink to subgraph of the component
if len(graph_condensation.node[node]['members']) > 1:
set_node_url(graph_condensation.node[node], component_filename_pattern % node)
else:
# TODO just link to normal neighborhood subgraph for single nodes
pass
base_filename_condensation = '15_condensation'
write_agraph_layout(agraph_from(graph_condensation), output_dir, base_filename_condensation)
links.append(('Condensation', base_filename_condensation))
for (component_index, component) in enumerate(strongly_connected_components):
# don't generate subgraphs consisting of a single node
if len(component) > 1:
graph_component = graph_absolute_edges.subgraph(nbunch=component).copy()
#macrogenesis.insert_minimal_edges_from_absolute_datings(graph_component)
# , edge_labels=True)
write_agraph_layout(agraph_from(graph_component), output_dir,
component_filename_pattern % (component_index))
# transitive closure, don't draw
logging.info("Generating transitive closure graph.")
transitive_closure = networkx.transitive_closure(graph_absolute_edges)
logging.info("{0} nodes, {1} edges in transtive closure.".format(transitive_closure.number_of_nodes(),
transitive_closure.number_of_edges()))
agraph_transitive_closure = agraph_from(transitive_closure)
# draw transitive reduction
logging.info("Generating transitive reduction graph.")
agraph_transitive_reduction = agraph_transitive_closure.tred(copy=True)
logging.info("{0} nodes, {1} edges in transtive reduction.".format(agraph_transitive_reduction.number_of_nodes(),
agraph_transitive_reduction.number_of_edges()))
base_filename_transitive_reduction = '30_transitive_reduction'
write_agraph_layout(agraph_transitive_reduction, output_dir, base_filename_transitive_reduction)
links.append(('Transitive reduction', base_filename_transitive_reduction))
# generate index.html
logging.info("Generating index.html")
html_links = ['<a href="{1}.html">{0}</a>'.format(*link) for link in links]
with open(os.path.join(output_dir, 'index.html'), mode='w') as html_file:
html_file.write(html_template('<h1>macrogenesis graphs</h1>' + ('<br/> '.join(html_links))))
def test_complete(self):
G = nx.path_graph(10, create_using=nx.DiGraph())
G = nx.transitive_closure(G)
for k in range(2, 11):
S = dag.count_chains(G, k)
assert_equal(S, comb(10, k))
def visualize():
"""
Lay out, style and write to disk the macrogenesis graphs.
:return: list of pairs ``(link_text, relative_path)`` to the generated html pages
"""
output_dir = faust.config.get("macrogenesis", "output-dir")
# copy resources
try:
shutil.copytree('macrogenesis/resources/js', os.path.join(output_dir, 'js'))
except OSError as e:
logging.warn(e)
#collect hyperlinks to selected graphs for the TOC as [(link_text_1, relative_link_to_file_1), ...]
links = []
# draw raw input data
graph_imported = graph.import_graph()
logging.info("Generating raw data graph.")
#UUU write_agraph_layout(agraph_from(graph_imported), output_dir, '00_raw_data')
# highlight a single node and its neighbors
# highlighted_node = 'faust://document/wa/2_I_H.17'
for highlighted_node in graph_imported:
highlighted_bunch = list(networkx.all_neighbors(graph_imported, highlighted_node))
highlighted_bunch.append(highlighted_node)
graph_highlighted_subgraph = graph_imported.subgraph(nbunch=highlighted_bunch).copy()
graph_highlighted_subgraph.node[highlighted_node][KEY_HIGHLIGHT]= VALUE_TRUE
graph.insert_minimal_edges_from_absolute_datings(graph_highlighted_subgraph)
#agraph_highlighted_subgraph.node_attr[highlighted_node]['color'] = 'red'
_write_agraph_layout(_agraph_from(graph_highlighted_subgraph), output_dir,
_highlighted_base_filename(highlighted_node))
# try again with edge labels, sometimes this genereats segfaults in dot, then there is the already
# generated graph without edge labels already there
try:
_write_agraph_layout(_agraph_from(graph_highlighted_subgraph, edge_labels=True), output_dir,
_highlighted_base_filename(highlighted_node))
except Exception:
logging.error("Dot failed generating a graph")
# add relationships implicit in absolute datings
logging.info("Generating graph with implicit absolute date relationships.")
graph_absolute_edges = graph_imported
graph.insert_minimal_edges_from_absolute_datings(graph_absolute_edges)
del graph_imported
base_filename_absolute_edges = '10_absolute_edges'
_write_agraph_layout(_agraph_from(graph_absolute_edges), output_dir, base_filename_absolute_edges)
links.append(('Raw datings (relative and absolute)', '%s.html' % base_filename_absolute_edges))
# again with edge labels
# TODO this breaks graphviz
# agraph_absolute_edges_edge_labels = agraph_from(graph_absolute_edges, edge_labels=True)
# write_agraph_layout(agraph_absolute_edges_edge_labels, output_dir, '15_absolute_edges_edge_labels')
logging.info("Generating condensation.")
strongly_connected_components = list(networkx.strongly_connected_components(graph_absolute_edges))
#condensation
graph_condensation = networkx.condensation(graph_absolute_edges, scc=strongly_connected_components)
for node in graph_condensation:
label = ', '.join([_label_from_uri(uri) for uri in graph_condensation.node[node]['members']])
label_width = int(2 * math.sqrt(len(label)))
graph_condensation.node[node]['label'] = textwrap.fill(label, label_width, break_long_words=False).replace('\n','\\n')
component_filename_pattern = '16_strongly_connected_component_%i'
# make a hyperlink to subgraph of the component
if len(graph_condensation.node[node]['members']) > 1:
_set_node_url(graph_condensation.node[node], component_filename_pattern % node)
else:
# TODO just link to normal neighborhood subgraph for single nodes
pass
base_filename_condensation = '15_condensation'
_write_agraph_layout(_agraph_from(graph_condensation), output_dir, base_filename_condensation)
links.append(('Condensation', '%s.html' % base_filename_condensation))
for (component_index, component) in enumerate(strongly_connected_components):
# don't generate subgraphs consisting of a single node
if len(component) > 1:
graph_component = graph_absolute_edges.subgraph(nbunch=component).copy()
#macrogenesis.insert_minimal_edges_from_absolute_datings(graph_component)
# , edge_labels=True)
_write_agraph_layout(_agraph_from(graph_component), output_dir,
component_filename_pattern % (component_index))
# transitive closure, don't draw
logging.info("Generating transitive closure graph.")
transitive_closure = networkx.transitive_closure(graph_absolute_edges)
logging.info("{0} nodes, {1} edges in transtive closure.".format(transitive_closure.number_of_nodes(),
transitive_closure.number_of_edges()))
agraph_transitive_closure = _agraph_from(transitive_closure)
# draw transitive reduction
logging.info("Generating transitive reduction graph.")
agraph_transitive_reduction = agraph_transitive_closure.tred(copy=True)
logging.info("{0} nodes, {1} edges in transtive reduction.".format(agraph_transitive_reduction.number_of_nodes(),
agraph_transitive_reduction.number_of_edges()))
base_filename_transitive_reduction = '30_transitive_reduction'
_write_agraph_layout(agraph_transitive_reduction, output_dir, base_filename_transitive_reduction)
links.append(('Transitive reduction', '%s.html' % base_filename_transitive_reduction))
return links
