def test_union_all_and_compose_all():
K3=nx.complete_graph(3)
P3=nx.path_graph(3)
G1=nx.DiGraph()
G1.add_edge('A','B')
G1.add_edge('A','C')
G1.add_edge('A','D')
G2=nx.DiGraph()
G2.add_edge('1','2')
G2.add_edge('1','3')
G2.add_edge('1','4')
G=nx.union_all([G1,G2])
H=nx.compose_all([G1,G2])
assert_edges_equal(G.edges(),H.edges())
assert_false(G.has_edge('A','1'))
assert_raises(nx.NetworkXError, nx.union, K3, P3)
H1=nx.union_all([H,G1],rename=('H','G1'))
assert_equal(sorted(H1.nodes()),
['G1A', 'G1B', 'G1C', 'G1D',
'H1', 'H2', 'H3', 'H4', 'HA', 'HB', 'HC', 'HD'])
H2=nx.union_all([H,G2],rename=("H",""))
assert_equal(sorted(H2.nodes()),
['1', '2', '3', '4',
'H1', 'H2', 'H3', 'H4', 'HA', 'HB', 'HC', 'HD'])
assert_false(H1.has_edge('NB','NA'))
G=nx.compose_all([G,G])
assert_edges_equal(G.edges(),H.edges())
G2=nx.union_all([G2,G2],rename=('','copy'))
assert_equal(sorted(G2.nodes()),
['1', '2', '3', '4', 'copy1', 'copy2', 'copy3', 'copy4'])
assert_equal(G2.neighbors('copy4'),[])
assert_equal(sorted(G2.neighbors('copy1')),['copy2', 'copy3', 'copy4'])
assert_equal(len(G),8)
assert_equal(nx.number_of_edges(G),6)
E=nx.disjoint_union_all([G,G])
assert_equal(len(E),16)
assert_equal(nx.number_of_edges(E),12)
E=nx.disjoint_union_all([G1,G2])
assert_equal(sorted(E.nodes()),[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
G1=nx.DiGraph()
G1.add_edge('A','B')
G2=nx.DiGraph()
G2.add_edge(1,2)
G3=nx.DiGraph()
G3.add_edge(11,22)
G4=nx.union_all([G1,G2,G3],rename=("G1","G2","G3"))
assert_equal(sorted(G4.nodes()),
['G1A', 'G1B', 'G21', 'G22',
'G311', 'G322'])
def test_union_all_and_compose_all():
K3 = nx.complete_graph(3)
P3 = nx.path_graph(3)
G1 = nx.DiGraph()
G1.add_edge("A", "B")
G1.add_edge("A", "C")
G1.add_edge("A", "D")
G2 = nx.DiGraph()
G2.add_edge("1", "2")
G2.add_edge("1", "3")
G2.add_edge("1", "4")
G = nx.union_all([G1, G2])
H = nx.compose_all([G1, G2])
assert_edges_equal(G.edges(), H.edges())
assert_false(G.has_edge("A", "1"))
assert_raises(nx.NetworkXError, nx.union, K3, P3)
H1 = nx.union_all([H, G1], rename=("H", "G1"))
assert_equal(sorted(H1.nodes()), ["G1A", "G1B", "G1C", "G1D", "H1", "H2", "H3", "H4", "HA", "HB", "HC", "HD"])
H2 = nx.union_all([H, G2], rename=("H", ""))
assert_equal(sorted(H2.nodes()), ["1", "2", "3", "4", "H1", "H2", "H3", "H4", "HA", "HB", "HC", "HD"])
assert_false(H1.has_edge("NB", "NA"))
G = nx.compose_all([G, G])
assert_edges_equal(G.edges(), H.edges())
G2 = nx.union_all([G2, G2], rename=("", "copy"))
assert_equal(sorted(G2.nodes()), ["1", "2", "3", "4", "copy1", "copy2", "copy3", "copy4"])
assert_equal(G2.neighbors("copy4"), [])
assert_equal(sorted(G2.neighbors("copy1")), ["copy2", "copy3", "copy4"])
assert_equal(len(G), 8)
assert_equal(nx.number_of_edges(G), 6)
E = nx.disjoint_union_all([G, G])
assert_equal(len(E), 16)
assert_equal(nx.number_of_edges(E), 12)
E = nx.disjoint_union_all([G1, G2])
assert_equal(sorted(E.nodes()), [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
G1 = nx.DiGraph()
G1.add_edge("A", "B")
G2 = nx.DiGraph()
G2.add_edge(1, 2)
G3 = nx.DiGraph()
G3.add_edge(11, 22)
G4 = nx.union_all([G1, G2, G3], rename=("G1", "G2", "G3"))
assert_equal(sorted(G4.nodes()), ["G1A", "G1B", "G21", "G22", "G311", "G322"])
def get_size_biggest_physical_groups_unify(network):
"""
Returns the size of the biggest physical group unifying layers.
:param network: the network data.
"""
new_network = Network(nx.compose_all(network.graph), list(), 0)
return max(fast_get_connected_components(new_network))
def get_amount_physical_groups_unify(network):
"""
Returns the amount of physical groups unifying layers.
:param network: the network data.
"""
new_network = Network(nx.compose_all(network.graph), list(), 0)
return len(fast_get_connected_components(new_network))
def nonfinal_compute_order(table):
"""
Returns which nodes to compute first, and what inputs are needed
>>> from dtool.depcache_control import * # NOQA
>>> from dtool.example_depcache import testdata_depc
>>> import plottool as pt
>>> pt.ensure_pylab_qt4()
>>> depc = testdata_depc()
>>> tablename = 'neighbs'
>>> tablename = 'multitest_score'
>>> table = depc[tablename]
>>> nonfinal_compute_order = table.nonfinal_compute_order()
>>> print(ut.repr3(nonfinal_compute_order))
"""
import networkx as nx
expanded_input_graph = table.expanded_input_graph
composed_graph = nx.compose_all(expanded_input_graph.values())
#pt.show_nx(composed_graph)
topsort = nx.topological_sort(composed_graph)
type_to_dependlevels = ut.map_dict_vals(ut.level_order,
expanded_input_graph)
level_orders = type_to_dependlevels
# Find computation order for all dependencies
nonfinal_compute_order = ut.merge_level_order(level_orders, topsort)
return nonfinal_compute_order
def draw_stmt_graph(stmts):
"""Render the attributes of a list of Statements as directed graphs.
The layout works well for a single Statement or a few Statements at a time.
This function displays the plot of the graph using plt.show().
Parameters
----------
stmts : list[indra.statements.Statement]
A list of one or more INDRA Statements whose attribute graph should
be drawn.
"""
import networkx
try:
import matplotlib.pyplot as plt
except Exception:
logger.error('Could not import matplotlib, not drawing graph.')
return
try: # This checks whether networkx has this package to work with.
import pygraphviz
except Exception:
logger.error('Could not import pygraphviz, not drawing graph.')
return
import numpy
g = networkx.compose_all([stmt.to_graph() for stmt in stmts])
plt.figure()
plt.ion()
g.graph['graph'] = {'rankdir': 'LR'}
pos = networkx.drawing.nx_agraph.graphviz_layout(g, prog='dot')
g = g.to_undirected()
# Draw nodes
options = {
'marker': 'o',
's': 200,
'c': [0.85, 0.85, 1],
'facecolor': '0.5',
'lw': 0,
}
ax = plt.gca()
nodelist = list(g)
xy = numpy.asarray([pos[v] for v in nodelist])
node_collection = ax.scatter(xy[:, 0], xy[:, 1], **options)
node_collection.set_zorder(2)
# Draw edges
networkx.draw_networkx_edges(g, pos, arrows=False, edge_color='0.5')
# Draw labels
edge_labels = {(e[0], e[1]): e[2].get('label') for e in g.edges(data=True)}
networkx.draw_networkx_edge_labels(g, pos, edge_labels=edge_labels)
node_labels = {n[0]: n[1].get('label') for n in g.nodes(data=True)}
for key, label in node_labels.items():
if len(label) > 25:
parts = label.split(' ')
parts.insert(int(len(parts)/2), '\n')
label = ' '.join(parts)
node_labels[key] = label
networkx.draw_networkx_labels(g, pos, labels=node_labels)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
def clean_up_geometry(self, threshold, byblock=True):
""" function cleans up geometry, and returns a _copy_ of the graph,
cleaned up nicely. Does not change original graph. connected considers
graph by connected components only for clean up.
"""
Gs = []
if byblock:
for i in self.connected_components():
i.G.remove_edges_from(i.G.selfloop_edges())
i.__combine_near_nodes(threshold)
i.__find_bad_edges(threshold)
i.__remove_bad_edges(i.bad_edge_dict)
Gs.append(i.G)
else:
i = self.copy()
i.G.remove_edges_from(i.G.selfloop_edges())
i.__combine_near_nodes(threshold)
i.__find_bad_edges(threshold)
i.__remove_bad_edges(i.bad_edge_dict)
Gs.append(i.G)
nxG = nx.compose_all(Gs)
newG = MyGraph(nxG, name=self.name)
newG.cleaned = True
return newG
def get_info_physical_groups_unify(network):
"""
Returns the amount of physical groups and the biggest one unifying layers.
:param network: the network data.
"""
new_network = Network(nx.compose_all(network.graph), list(), 0)
info = fast_get_connected_components(new_network)
return len(info), max(info)
def _merge_markups(rmarkups):
if rmarkups:
raw_text = " ".join([r.graph["__rawTxt"] for r in rmarkups])
newgraph = nx.compose_all(rmarkups)
newgraph.graph["__rawTxt"] = raw_text
return newgraph
else:
return nx.DiGraph()
def make_unigraph(all_graphs): """ Makes the collaboration graph for the full university, by composing the given list of all the school graphs. Returns full university graph """ unigraph = nx.compose_all(all_graphs) unigraph = add_metrics(unigraph) return unigraph
def test_input_output():
l = [nx.Graph([(1,2)]),nx.Graph([(3,4)])]
U = nx.disjoint_union_all(l)
assert_equal(len(l),2)
C = nx.compose_all(l)
assert_equal(len(l),2)
l = [nx.Graph([(1,2)]),nx.Graph([(1,2)])]
R = nx.intersection_all(l)
assert_equal(len(l),2)
def complete(cls, context, **kwargs):
"""
Build a complete graph showing all devices.
:param `Context` context: a udev context
:param kwargs: arguments for filtering the devices.
:returns: a graph
:rtype: `DiGraph`
"""
devices = (d for d in context.list_devices(**kwargs))
graphs = (cls.parents_and_children(context, d) for d in devices)
return nx.compose_all(chain([nx.DiGraph()], graphs), name="sysfs")
def getMergedDag(start, end, path_list):
"""
Given a list of paths, returns the loop-free merged DAG
forcing all (possible) paths in path_list.
"""
pathsDags = []
for path in path_list:
tmp_dag = nx.DiGraph()
tmp_dag.add_edges_from(zip(path[:-1], path[1:]))
pathsDags.append(tmp_dag)
## Merge them all into one single DAG
composedDag = nx.compose_all(pathsDags)
while not nx.is_directed_acyclic_graph(composedDag):
## Eliminate loop edges at random
# Get simple cycles
simple_cycles = [c for c in nx.simple_cycles(composedDag)]
## Eliminate loops
for cycle in simple_cycles:
# Remove cycle edge at random
cycle_edges = zip(cycle[:-1], cycle[1:])
random.shuffle(cycle_edges)
(x, y) = cycle_edges[0]
if (x,y) in composedDag.edges():
composedDag.remove_edge(x, y)
# Compute all paths on randomDag (to eliminate dummy paths)
all_final_paths = getAllPathsLim(composedDag, start, end, k=0)
pathsDags = []
for path in all_final_paths:
tmp_dag = nx.DiGraph()
tmp_dag.add_edges_from(zip(path[:-1], path[1:]))
pathsDags.append(tmp_dag)
finalMergedDag = nx.compose_all(pathsDags)
return finalMergedDag
def create_sample_DCP_instance(node_count=100, tree_count=10, tree_span=float('infinity')):
"""
Generates a sample DCP problem instance:
- A directed graph with attribute 'weight' on all edges
- A dictionary from (node, time) to existence {True, False}
- A list of connectivity demands (source, target, time)
The graph is created by sampling trees (each on at most tree_span nodes) from a pool of nodes,
then taking their union.
"""
assert tree_span > 1, 'Sampled trees must contain at least two nodes each.'
nodes = create_node_pool(node_count)
# Map each (node, time) to its existence, initially 0
existence_for_node_time = {(node,time): 0 for node in nodes for time in range(tree_count)}
# List of connectivity demands in the form (source, target, time)
connectivity_demands = []
# Sample a tree at each time point
trees = []
for time in range(tree_count):
# Choose a random subset of the nodes from which to build the tree
tree_nodes = random.sample(nodes, min(len(nodes), tree_span))
# Sample a tree
tree, source, terminals = create_sample_tree(tree_nodes)
# Record tree
trees += [tree]
# Record existence of tree nodes at the current time
for node in tree.nodes_iter():
existence_for_node_time[node,time] = 1
# Record connectivity demands for this tree/time
for target in terminals:
connectivity_demands += [(source, target, time)]
# Union of trees
graph = networkx.compose_all(trees)
# Weights on edges
# TODO: make weights non-unit
for u,v in graph.edges_iter():
graph[u][v]['weight'] = 1
return graph, existence_for_node_time, connectivity_demands
def merge_RRI_new(stage_ls, sample_ls):
inter_RRI_ls = ['/Share/home/zhangqf7/gongjing/zebrafish/data/paris/shi-zp-%s-rep-combine/27-DG.inter.element.txt'%(i) for i in stage_ls]
nx_inter_RRI_network_ls = []
for inter_RRI,sample in zip(inter_RRI_ls, sample_ls):
nx_inter_RRI, df_inter_RRI = read_inter_RRI(inter_RRI, only_mRNA_lncRNA=True, support_read_num=3)
nx_inter_RRI_network_ls.append(nx_inter_RRI)
nx_inter_RRI_merge = nx.compose_all(nx_inter_RRI_network_ls)
print nx.info(nx_inter_RRI_merge)
trans_dict = loadTransGtfBed2()
maternal_decay_file = '/Share/home/zhangqf7/gongjing/zebrafish/result/zhangting_provided/supplementary/decay.txt'
maternal_decay_dict = read_maternal(maternal_list_file=maternal_decay_file)
maternal_stable_file = '/Share/home/zhangqf7/gongjing/zebrafish/result/zhangting_provided/supplementary/stable.txt'
maternal_stable_dict = read_maternal(maternal_list_file=maternal_stable_file)
def _decayorstable(tx_id):
if tx_id in maternal_decay_dict:
if tx_id in maternal_stable_dict:
return 'mystic'
else:
return 'decay'
elif tx_id in maternal_stable_dict:
return 'stable'
else:
return 'undetermined'
out_dir = '/Share/home/zhangqf7/gongjing/zebrafish/result/network/merge/_select_example_for_fig4'
savefn = os.path.join(out_dir, 'nx_merge_%s.mrna_lncrna.nodes.new.decay.txt' % "".join(map(str,stage_ls)))
savefn2 = os.path.join(out_dir, 'nx_merge_%s.mrna_lncrna.edges.new.decay.txt' % "".join(map(str,stage_ls)))
SAVEFN2 = open(savefn2, 'w')
SAVEFN = open(savefn, 'w')
header_ls = ['tx_id', 'gene_name', 'type', 'Maternal_Decay'] + sample_ls
print >>SAVEFN, '\t'.join(header_ls)
for node in nx_inter_RRI_merge.nodes():
node_status = ['YES' if n.has_node(node) else 'NO' for n in nx_inter_RRI_network_ls]
maternalorNot = _decayorstable(node)
print >>SAVEFN, '\t'.join([node, trans_dict[node]['gene'].split('=')[0], trans_dict[node]['type'], maternalorNot] + node_status)
SAVEFN.close()
header_ls2 = ['source', 'target'] + sample_ls + [ "%s(read)" % i for i in sample_ls] + ["%s(group)" % i for i in sample_ls] + ['type_source', 'type_target', 'name_source', 'name_target',
'source_Maternal_Decay', 'target_Maternal_Decay']
print >>SAVEFN2, '\t'.join(header_ls2)
for i in nx_inter_RRI_merge.edges():
edge_status = ['YES' if n.has_edge(i[0], i[1]) else 'NO' for n in nx_inter_RRI_network_ls]
edge_support_read = [n[i[0]][i[1]]['support'] if n.has_edge(i[0], i[1]) else 0 for n in nx_inter_RRI_network_ls]
edge_group = [n[i[0]][i[1]]['Group'] if n.has_edge(i[0], i[1]) else '.' for n in nx_inter_RRI_network_ls]
print >> SAVEFN2, '\t'.join([i[0], i[1]] + edge_status + map(str, edge_support_read) + map(str, edge_group) + [trans_dict[i[0]]['type'], trans_dict[i[1]]['type']]
+ [trans_dict[i[0]]['gene'].split('=')[0], trans_dict[i[1]]['gene'].split('=')[0]] + [_decayorstable(i[0]), _decayorstable(i[1])])
SAVEFN2.close()
def complete(cls, context):
"""
Build a complete graph showing device mapper, partition relationships.
:param `Context` context: a udev context
:returns: a graph
:rtype: `DiGraph`
"""
block_devices = context.list_devices(subsystem="block")
partitions = block_devices.match_property('DEVTYPE', 'partition')
graphs = (cls.congruence_graph(context, d) for d in partitions)
return nx.compose_all(
chain([nx.DiGraph()], graphs),
name='congruence'
)
def complete(cls, context):
"""
Build a complete graph showing path/spindle relationships.
:param `Context` context: a udev context
:returns: a graph
:rtype: `DiGraph`
"""
block_devices = context.list_devices(subsystem="block")
disks = block_devices.match_property('DEVTYPE', 'disk')
graphs = (cls.spindle_graph(d) for d in disks)
return nx.compose_all(
chain([nx.DiGraph()], graphs),
name='spindle'
)
def NetworkAnalysis(jsonGraph):
"""gets graph defined by for export json and computes the top 3
most connected subgraphs"""
G = json_graph.node_link_graph(jsonGraph)
graphs = sorted(nx.connected_component_subgraphs(G), key = len, reverse=True)
(GC1, GC2, GC3, GC4, GC5) = graphs[0:5]
top5 = nx.compose_all([GC1,GC2,GC3,GC4,GC5])
deg = top5.degree()
nx.set_node_attributes(top5, "degree", deg)
take = {
"nodes": json_graph.node_link_data(top5)["nodes"],
"links": json_graph.node_link_data(top5)["links"]
}
return take
def graph(context, name, classes):
"""
Build a graph using the designated classes.
:param context: a context
:param str name: a name for the graph
:param classes: a list of graph classes
:type classes: list of type
:returns: a graph
:rtype: nx.DiGraph
"""
return nx.compose_all(
(t.complete(context) for t in classes),
name=name
)
def complete(cls, context):
"""
Build a complete graph of all partitions.
:param `Context` context: a udev context
:returns: a graph
:rtype: `DiGraph`
"""
block_devices = context.list_devices(subsystem="block")
partitions = block_devices.match_property('DEVTYPE', 'partition')
graphs = (cls.partition_graph(d) for d in partitions)
return nx.compose_all(
chain([nx.DiGraph()], graphs),
name="partiton"
)
def add_school_info():
nameurlpath = "../nameurls/"
graphpath = "../grc/graphs/collab/"
nufiles = os.listdir(nameurlpath)
graphfiles = os.listdir(graphpath)
allgraphs = []
school_lists = {}
for nufile in nufiles:
schoolname = nufile.split(".")[0]
with open(nameurlpath + nufile) as f:
nameurls = json.load(f)
just_urls = [nameurl[1] for nameurl in nameurls]
school_lists[schoolname] = just_urls
for gfile in graphfiles:
if "University" in gfile or "Inter School" in gfile:
continue
with open(graphpath + gfile) as f:
gdata = json.load(f)
g = json_graph.node_link_graph(gdata)
for author in g.nodes():
found = False
for school, urls in school_lists.items():
if author == "http://eprints.gla.ac.uk/view/author/3545.html":
g.node[author]["school"] = "School of Computing Science"
found = True
elif author in urls:
found = True
g.node[author]["school"] = school
if not found:
g.node[author]["school"] = False
allgraphs.append(g)
writedata = json_graph.node_link_data(g)
with open(graphpath + gfile, 'w') as f:
json.dump(writedata, f)
unigraph = nx.compose_all(allgraphs)
gm = gfd.GraphMaker(unigraph)
gm.add_metrics()
gm.write_to_file(graphpath + "The University of Glasgow.json")
def plot_all_paths(self, all_paths, update=False):
""" plots the shortest paths from all interior parcels to the road.
Optional to update road geometery based on changes in network geometry.
"""
plt.figure()
if len(all_paths) == 0:
self.plot_roads(update=update)
else:
Gs = []
for p in all_paths:
G = nx.subgraph(self.G, p)
Gs.append(G)
Gpaths = nx.compose_all(Gs, name="shortest paths")
myGpaths = MyGraph(Gpaths)
self.plot_roads(update=update)
myGpaths.plot(edge_color='purple', width=6, node_size=1)
def get_commits_from_forks(forks, old_commits, auth_key, user, repo):
new_commits = []
graphs = []
for fork in forks:
graph = nx.DiGraph()
branch_request_url = fork[:-5] + "git/refs/heads"
commit_request_url = fork[:-5] + "git/commits"
branches = requests.get(branch_request_url, auth=auth_key)
for branch in branches.json():
if branch[u'object'][u'sha'] not in old_commits:
new_commits.append(branch[u'object'][u'sha'])
diff_url_base = 'http://github.com/' + user + '/' + repo + '/commit/'
for commit in new_commits:
queue = deque()
queue.append(commit)
while len(queue) > 0:
to_examine = queue.popleft()
request_url = commit_request_url + '/' + to_examine
commit_info = requests.get(request_url, auth=auth_key).json()
parents = commit_info['parents']
for parent in parents:
if parent['sha'] not in old_commits:
queue.append(parent['sha'])
attributes = dict(authored_date=commit_info['author']['date'], author=commit_info['author']['email'],
committer=commit_info['committer']['email'],
committed_date=commit_info['committer']['date'], message=commit_info['message'],
hexsha=to_examine, parents=[i['sha'] for i in parents])
diff_file = requests.get(diff_url_base + to_examine + ".diff").text
attributes['diff'] = diff_file
graph.add_node(attributes['hexsha'], **attributes)
for node in graph.nodes():
for parent in graph.node[node]['parents']:
graph.add_edge(parent, node)
graphs.append(graph)
if len(new_commits) == 0:
return None, False
else:
if len(graphs) > 1:
return nx.compose_all(graphs), True
else:
return graphs[0], True
def prune_paths(self, beta):
"""
Heuristic pruning:
1. Normalize weights in interval [-1,1]
2. Set all negative weights to 0
3. Compute beta shortest paths (beta is a paramater)
https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.algorithms.simple_paths.shortest_simple_paths.html
4. Remove all edges that do not belong to these paths
"""
# Normalize weights
max_weight = max(
[self.G.edges[i, j]["weight"] for (i, j) in self.G.edges()])
min_weight = min(
[self.G.edges[i, j]["weight"] for (i, j) in self.G.edges()])
for edge in self.G.edges(data=True):
edge[2]["pos_weight"] = (-max_weight - min_weight +
2 * edge[2]["weight"]) / (max_weight -
min_weight)
edge[2]["pos_weight"] = max(0, edge[2]["pos_weight"])
# Compute beta shortest paths
best_paths = list(
islice(
shortest_simple_paths(self.G,
"Source",
"Sink",
weight="pos_weight"),
beta,
))
# Store these paths as a list of DiGraphs
best_paths_list = []
for path in best_paths:
H = DiGraph()
add_path(H, path)
best_paths_list.append(H)
# Merge the paths into one graph
induced_graph = compose_all(best_paths_list)
# Create subgraph induced by the edges of this graph
self.sub_G = self.G.edge_subgraph(induced_graph.edges()).copy()
def agglomerate_group_graph(seg_map,
G_neighbor,
merge_output,
edge_indices=None):
# G_neighbor = get_neighbor_graph(seg_map)
# agglomerate each edge and generate global remap dict
full_edge_arr = np.array(G_neighbor.edges())
local_edge_set = full_edge_arr[edge_indices]
pbar = tqdm(local_edge_set, total=len(edge_indices), desc='find overlap')
# pbar = tqdm(G_neighbor.edges(), total=G_neighbor.number_of_edges(), desc='find overlap')
global_merge_dict = {}
# counter = 0
G_overlaps = []
for k1, k2 in pbar:
G_overlap = nx.Graph()
# logging.warning('test %s %s', k1, k2)
p1 = seg_map[k1]['output']
p2 = seg_map[k2]['output']
#if relabel:
remap_label = agglomerate_v2(p1,
p2,
contiguous=False,
inplace=False,
no_zero=True)
# a dict of object ids from p2 -> p1
#src_attr = {k:{'cube': k2} for k in remap_label.keys()}
#tgt_attr = {k:{'cube': k1} for k in remap_label.keys()}
G_overlap.add_nodes_from(remap_label.keys(), cube=k2)
G_overlap.add_nodes_from(remap_label.values(), cube=k1)
edges = [(k, v) for k, v in remap_label.items()]
G_overlap.add_edges_from(edges)
G_overlaps.append(G_overlap)
# counter += 1
# if counter == 3:
# break
if len(G_overlaps):
G_overlaps_merged = nx.compose_all(G_overlaps)
else:
G_overlaps_merged = nx.Graph()
global_merge_dict = get_merge_dict(G_overlaps_merged)
return G_overlaps_merged, global_merge_dict
def severalStarGraphs(m, k, highSubsets):
graphs = []
centers = []
for i in range(k):
center = i*(m+1)
centers.append(center)
nodes = np.array([j+center for j in range(m+1)])
if (highSubsets[i] == 1):
graphs.append(simpleStar(nodes, center, direction=ut.Direction.OUTGOING))
else:
graphs.append(simpleStar(nodes, center, direction=ut.Direction.INCOMING))
graphs.append(simpleCycleGraph(centers))
G = nx.compose_all(graphs)
ut.normalizeGraph(G)
return G
def refine_to_chain(g, from_attr, to_attr):
'''can be used to refine basic blocks into blocks - the dual of contract_chains()
assume g.node[n][attr] is a list
returns a graph whose nodes are the refinement of the lists into paths
the elements of the lists are held as to_attr
the nodes become tuples (node_index, list_index)'''
paths = []
for n in g.nodes_iter():
block = g.node[n][from_attr]
size = len(block)
path = nx.path_graph(size, create_using=nx.DiGraph())
nx.relabel_nodes(path, mapping={x:(n, x) for x in path.nodes()}, copy=False)
path.add_edges_from(((n, size - 1), (s, 0)) for s in g.successors_iter(n))
paths.append(path)
values = {(n, x): block
for n in g.nodes_iter()
for x, block in enumerate(g.node[n][from_attr])}
res = nx.compose_all(paths)
nx.set_node_attributes(res, to_attr, values)
return res
def randomize(fname: str,
target: str,
iterations: int,
per_cc: bool = False,
edge_predicate: str = edge_predicate):
"""Write in file of given name a randomized version of input graph.
"""
fname = commons.normalize_filename(fname)
target = commons.normalize_filename(target)
graph = graph_from_file(fname, edge_predicate=edge_predicate)
if per_cc:
graphs = (graph.subgraph(nodes).copy()
for nodes in networkx.connected_components(graph))
else:
graphs = [graph]
def run():
for graph in graphs:
print(tuple(graph.edges))
nb_edge = graph.number_of_edges()
total_iterations = iterations * graph.number_of_edges()
try:
yield networkx.algorithms.double_edge_swap(
graph,
nswap=total_iterations,
max_tries=100 * total_iterations)
except networkx.exception.NetworkXError as err:
print(err.args[0])
yield graph
except networkx.exception.NetworkXAlgorithmError:
print(
"Maximum number of swap attempts reached, or graph can't be swapped. Ignored."
)
yield graph
if per_cc:
graph = networkx.compose_all(run())
else:
graph = next(run())
return graph_to_file(graph, target, edge_predicate=edge_predicate)
def merge_graphs(self, graphs: List[nx.MultiDiGraph]) -> None:
"""
Merge all graphs with self.graph
- If two nodes with same 'id' exist in two graphs, the nodes will be merged based on the 'id'
- If two nodes with the same 'id' exists in two graphs and they both have conflicting values
for a property, then the value is overwritten from left to right
- If two edges with the same 'key' exists in two graphs, the edge will be merged based on the
'key' property
- If two edges with the same 'key' exists in two graphs and they both have one or more conflicting
values for a property, then the value is overwritten from left to right
Parameters
----------
graphs: List[networkx.MultiDiGraph]
List of graphs that are to be merged with self.graph
"""
# TODO: Check behavior and consistency
graphs.insert(0, self.graph)
self.graph = nx.compose_all(graphs)
def join_two_trees_v2(G, tree1, tree2):
'''Joins two trees by brute force scanning. Outputs joint tree'''
#tree1 = tree1.copy()
tree2 = tree2.copy()
terminals_1 = get_terminals(tree1)
terminals_2 = get_terminals(tree2)
all_pairs = list(itertools.product(terminals_1, terminals_2))
def my_distance(pair):
return nx.shortest_path(G, *pair, weight='length')
bridge = sorted(map(my_distance, all_pairs), key=len)[0]
nx.add_path(tree2, bridge)
joint_graph = nx.compose_all([tree1, tree2])
return joint_graph
def _combine_nets(*nets: nx.DiGraph):
net = nx.compose_all(nets)
ns = [list(this.nodes) for this in nets]
cs = [list(map(lambda it: (it.x, it.y), map(lambda it: transform(Point(it.point)), n))) for n in ns]
for (t, tn, tc), (o, on, oc) in tqdm(combinations(zip(nets, ns, cs), 2)):
bkdtree, skdtree = KDTree(tc), KDTree(oc)
for i, js in tqdm(enumerate(bkdtree.query_ball_tree(skdtree, 500))): # any pair of nodes within 500 meters
for j in js:
if tn[i].modal != on[j].modal:
ix, iy = tc[i]
jx, jy = oc[j]
dist = ((ix - jx) ** 2 + (iy - jy) ** 2) ** .5
net.add_edge(tn[i], on[j], **{
'distance': dist,
'time': 5,
'price': 0,
'transfer_time': 1,
})
return net
def _get_iterative_props(self):
r"""
Finds and returns properties that need to be iterated while
running the algorithm.
"""
import networkx as nx
phase = self.project[self.settings.phase]
physics = self.project.find_physics(phase=phase)
geometries = self.project.geometries().values()
# Generate global dependency graph
dg = nx.compose_all([x.models.dependency_graph(deep=True)
for x in [phase, *geometries, *physics]])
base = [self.settings["quantity"]] + self.settings["variable_props"]
# Find all props downstream that depend on base props
dg = nx.DiGraph(nx.edge_dfs(dg, source=base))
if len(dg.nodes) == 0:
return []
iterative_props = list(nx.dag.lexicographical_topological_sort(dg))
# "variable_props" should be in the returned list but not "quantity"
iterative_props.remove(self.settings["quantity"])
return iterative_props
def test_example_graph(self):
qc = {'$text': {'$search': 'methicillin'}}
g1 = self.qry.get_connections_graph(qc, "targets")
self.assertAlmostEqual(82, g1.number_of_edges(), delta=10)
self.assertAlmostEqual(73, g1.number_of_nodes(), delta=10)
g2 = self.qry.get_connections_graph(qc, "enzymes")
self.assertAlmostEqual(16, g2.number_of_edges(), delta=4)
self.assertAlmostEqual(12, g2.number_of_nodes(), delta=4)
g3 = self.qry.get_connections_graph(qc, "transporters")
self.assertAlmostEqual(30, g3.number_of_edges(), delta=14)
self.assertAlmostEqual(22, g3.number_of_nodes(), delta=4)
g4 = self.qry.get_connections_graph(qc, "carriers")
self.assertAlmostEqual(7, g4.number_of_edges(), delta=4)
self.assertAlmostEqual(8, g4.number_of_nodes(), delta=4)
r = nx.compose_all([g1, g2, g3, g4])
self.assertAlmostEqual(125, r.number_of_edges(), delta=20)
self.assertAlmostEqual(94, r.number_of_nodes(), delta=14)
remove_small_subgraphs(r)
save_graph(r, EX_GRAPHS + 'drugbank-methicillin.json')
r = neighbors_graph(r, "Ticarcillin", beamwidth=8, maxnodes=100)
assert 2 == r.number_of_nodes()
def GenerateGraph(self) -> nx.Graph:
"""Creates a connected graph.
The graphs are geographic threshold graphs, but with added edges via a
minimum spanning tree algorithm, to ensure all nodes are connected.
Returns:
The graph.
"""
# Sample num_nodes.
num_nodes = self.rand.randint(*self.options.num_nodes_min_max)
# Create geographic threshold graph.
pos_array = self.rand.uniform(size=(num_nodes, self.options.dimensions))
pos = dict(enumerate(pos_array))
weight = dict(enumerate(self.rand.exponential(
self.options.rate, size=num_nodes)))
geo_graph = nx.geographical_threshold_graph(
num_nodes, self.options.theta, pos=pos, weight=weight)
# Create minimum spanning tree across geo_graph's nodes.
distances = spatial.distance.squareform(spatial.distance.pdist(pos_array))
i_, j_ = np.meshgrid(range(num_nodes), range(num_nodes), indexing="ij")
weighted_edges = list(zip(i_.ravel(), j_.ravel(), distances.ravel()))
mst_graph = nx.Graph()
mst_graph.add_weighted_edges_from(
weighted_edges, weight=self.options.weight_name)
mst_graph = nx.minimum_spanning_tree(
mst_graph, weight=self.options.weight_name)
# Put geo_graph's node attributes into the mst_graph.
for i in mst_graph.nodes():
mst_graph.node[i].update(geo_graph.node[i])
# Compose the graphs.
combined_graph = nx.compose_all([mst_graph, geo_graph.copy()])
# Put all distance weights into edge attributes.
for i, j in combined_graph.edges():
combined_graph.get_edge_data(i, j).setdefault(
self.options.weight_name, distances[i, j])
return combined_graph
def get_build_graph(drvpath_targets: Iterable[str],
drvpath_universe: List[str] = None) -> nx.DiGraph:
"""Build graph a a DiGraph.
Each edge (i, j) means that drv `i` is required as input to drv `j`.
"""
start_time = time.time()
targets = list(drvpath_targets)
if len(targets) == 0:
return nx.DiGraph()
def graph_ml_chunks():
# Work around `OSError: [Errno 7] Argument list too long: 'nix-store'`
# by chunking.
for chunk in _chunker(targets, 5000):
graph_ml = subprocess.run(["nix-store", "--query", "--graphml"] +
chunk,
stdout=subprocess.PIPE)
yield nx.read_graphml(io.BytesIO(graph_ml.stdout))
g = nx.compose_all(graph_ml_chunks())
# At this point, the nodes to not have /nix/store in their names
# And it includes _everything_, down to glibc
PFX = "/nix/store/"
if drvpath_universe is not None:
to_remove = set(g.nodes) - {
removeprefix(drv_path, PFX)
for drv_path in drvpath_universe
}
to_remove -= {removeprefix(t, PFX) for t in targets}
g.remove_nodes_from(to_remove)
nx.relabel_nodes(g, {n: f"{PFX}{n}" for n in g.nodes}, copy=False)
if time.time() - start_time > 2:
log.info(f"Computing build graph: {time.time() - start_time:.2f} sec")
return g
def create_dep_graph_device(self, dags):
DAG = nx.compose_all(dags)
DAG.add_node(self.name + ' multiplexed readout')
DAG.add_node(self.name + ' resonator frequencies coarse')
DAG.add_node('AWG8 MW-staircase')
DAG.add_node('AWG8 Flux-staircase')
DAG.add_node('Chevron q0-q1')
DAG.add_node('Chevron q1-q2')
DAG.add_node('CZ q0-q1')
DAG.add_node('CZ q1-q2')
DAG.add_edge('CZ q0-q1', 'Chevron q0-q1')
DAG.add_edge('CZ q1-q2', 'Chevron q1-q2')
DAG.add_edge('CZ q0-q1', 'q0 cryo dist. corr.')
DAG.add_edge('CZ q0-q1', 'q1 cryo dist. corr.')
DAG.add_edge('CZ q1-q2', 'q1 cryo dist. corr.')
DAG.add_edge('CZ q1-q2', 'q2 cryo dist. corr.')
DAG.add_edge('Chevron q0-q1', 'q0 gates restless')
DAG.add_edge('Chevron q0-q1', 'AWG8 Flux-staircase')
DAG.add_edge('Chevron q0-q1', 'q1 gates restless')
DAG.add_edge('Chevron q0-q1', self.name + ' multiplexed readout')
DAG.add_edge('Chevron q1-q2', 'q0 gates restless')
DAG.add_edge('Chevron q1-q2', 'AWG8 Flux-staircase')
DAG.add_edge('Chevron q1-q2', 'q1 gates restless')
DAG.add_edge('Chevron q1-q2', self.name + ' multiplexed readout')
for qubit in self.qubits:
q_name = qubit.name
DAG.add_edge(q_name + ' room temp. dist. corr.', 'AWG8 Flux-staircase')
DAG.add_edge(self.name + ' multiplexed readout',
q_name + ' optimal weights')
DAG.add_edge(q_name + ' resonator frequency',
self.name + ' resonator frequencies coarse')
DAG.add_edge(q_name + ' pulse amplitude coarse', 'AWG8 MW-staircase')
return DAG
def generate_connected_graph(generator, num_nodes, theta, rate):
"""Generates graph ensures all nodes are connected
Args:
generator: A numpy random generator
num_nodes: An integer number of nodes in the graph
theta: A `float` threshold parameters for the geographic threshold graph's threshold.
rate: A rate parameter for the node weight exponential sampling distribution.
Returns:
combined_graph: A generated graph
"""
pos_array = generator.uniform(size=(num_nodes, 2))
pos = dict(enumerate(pos_array))
weight = dict(enumerate(generator.exponential(rate, size=num_nodes)))
geo_graph = nx.geographical_threshold_graph(num_nodes,
theta,
pos=pos,
weight=weight)
distances = spatial.distance.squareform(spatial.distance.pdist(pos_array))
i_, j_ = np.meshgrid(range(num_nodes), range(num_nodes), indexing="ij")
weighted_edges = list(zip(i_.ravel(), j_.ravel(), distances.ravel()))
mst_graph = nx.Graph()
mst_graph.add_weighted_edges_from(weighted_edges,
weight=DISTANCE_WEIGHT_NAME)
mst_graph = nx.minimum_spanning_tree(mst_graph,
weight=DISTANCE_WEIGHT_NAME)
# Put geo_graph's node attributes into the mst_graph.
for i in mst_graph.nodes():
mst_graph.nodes[i].update(geo_graph.nodes[i])
# Compose the graphs.
combined_graph = nx.compose_all([mst_graph, geo_graph.copy()])
# Put all distance weights into edge attributes.
for i, j in combined_graph.edges():
combined_graph.get_edge_data(i, j).setdefault(DISTANCE_WEIGHT_NAME,
distances[i, j])
return combined_graph
def prune_network(self, verbose=False):
"""
Remove subgraphs of the network that do not connect to the coastline.
"""
# weakly_connected_component_subgraphs doesn't work with the
# RiverGraph subclass, so we have to switch it back to DiGraph
sg = self.graph
coast_n = 0
noncoast_n = 0
g_list = []
cps = nx.weakly_connected_component_subgraphs(sg)
for cp in cps:
if has_rivermouth(cp.nodes(), sg):
g_list.append(cp)
coast_n += 1
else:
noncoast_n += 1
if verbose:
print "{} graphs with coastal nodes, {} without.".format(
coast_n, noncoast_n)
return RiverGraph(data=nx.compose_all(g_list),
coastline_shp=self.coast_fn)
def compile(self, reduce: bool = True):
"""
Simplify the graph and load it into the networkx graph.
Arguments:
reduce (bool : True): Whether to attempt to reduce degree=2 edges
to a simplified, more optimized graph.
Returns:
None
"""
self.loaded = True
self._graph = nx.DiGraph()
if reduce:
self._graph = nx.compose_all(
[neuron.reduce() for neuron in self._neurons])
for synapse in self._synapses:
self._graph.add_edge("/".join(synapse[0]),
"/".join(synapse[1]),
**{"synapse": synapse[2]})
def graph_to_file(network, file_name, print_amount_nodes=True, delimeter=","):
"""
:param network:
:param file_name:
:param print_amount_nodes:
:param delimeter:
:return:
"""
output_f = open(file_name, 'w')
new_network = nx.compose_all(network.graph)
nodes = dict()
i = 1
for n in new_network.nodes():
nodes[n] = i
i += 1
if print_amount_nodes:
output_f.write(str(i-1) + "\n")
for e1, e2 in new_network.edges():
output_f.write(delimeter.join([str(nodes[e1]), str(nodes[e2])]) + "\n")
output_f.close()
def graph_to_file(network, file_name, print_amount_nodes=True, delimeter=","):
"""
:param network:
:param file_name:
:param print_amount_nodes:
:param delimeter:
:return:
"""
output_f = open(file_name, 'w')
new_network = nx.compose_all(network.graph)
nodes = dict()
i = 1
for n in new_network.nodes():
nodes[n] = i
i += 1
if print_amount_nodes:
output_f.write(str(i - 1) + "\n")
for e1, e2 in new_network.edges():
output_f.write(delimeter.join([str(nodes[e1]), str(nodes[e2])]) + "\n")
output_f.close()
def findtour(inputfile):
data = read_file(inputfile)
# need this to create output file
name = inputfile.name
numl, numh, listl, listh, start, matrix = data_parser(data)
G, message = adjacency_matrix_to_graph(matrix)
nodes = G.__iter__()
nodedict = {}
for i in range(numl):
nodedict[listl[i]] = next(nodes)
paths = []
graphs = []
distances = []
disttonode = {}
nodetodist = {}
for home in listh:
homenode = nodedict.get(home)
distance, shortestpath = nx.single_source_dijkstra(
G, homenode, nodedict.get(start))
paths += [shortestpath]
shortestgraph = nx.path_graph(shortestpath)
graphs += [shortestgraph]
distances.append(distance)
# fix duplicate keys
if not distance in disttonode:
disttonode[distance] = [homenode]
else:
disttonode[distance].append(homenode)
nodetodist[homenode] = distance
distances.sort(reverse=True)
mst = nx.compose_all(graphs)
leaves = [x for x in mst.nodes() if mst.degree(x) == 1]
visited = []
drive = []
adjlist = mst._adj
startnode = nodedict.get(start)
tour = traverse(startnode, visited, leaves, drive, distances,
adjlist, nodedict, disttonode, nodetodist, G, startnode, mst, [], listh)
return tour
def aggregate(
self,
bin=None
): #aggregate with a bin of size bin. if bin==None, aggreagte everything
toReturn = deepcopy(self)
if bin == None:
bin = len(toReturn._snapshots)
i = 0
while i < len(toReturn._snapshots):
t = toReturn._snapshots.iloc[i]
toMerge = []
toDelete = []
for j in range(bin):
if i + j < len(toReturn._snapshots):
toMerge.append(toReturn._snapshots.peekitem(i + j)[1])
if j >= 1:
toDelete.append(toReturn._snapshots.iloc[i + j])
toReturn._snapshots[t] = nx.compose_all(toMerge)
for k in toDelete:
del toReturn._snapshots[k]
i += 1
return toReturn
def unify(self, edge_weights={}):
"""Return a unified DiGraph containing all
subgraphs, with edges weighted differently per subgraph.
"""
social_following = self.social_following()
weight = edge_weights.get('social_following', 1)
for u, v, d in social_following.edges(data=True):
d['weight'] = weight
content_tree = self.content_tree()
weight = edge_weights.get('content_tree', 1)
for u, v, d in content_tree.edges(data=True):
d['weight'] = weight
content_authors = self.content_authors()
weight = edge_weights.get('content_authors', 1)
for u, v, d in content_authors.edges(data=True):
d['weight'] = weight
content_tags = self.content_tags()
weight = edge_weights.get('content_tags', 1)
for u, v, d in content_tags.edges(data=True):
d['weight'] = weight
return nx.compose_all(
[social_following, content_tree, content_authors, content_tags])
def run_fast_edge_tests():
graph_size = 32
#fast_edge_pcts = [pct*0.01 for pct in range(5,100,5)]
fast_edge_pcts = [pct * 0.01 for pct in range(2, 100, 2)]
iterations = 40
results_for_pct = {}
for fast_edge_pct in fast_edge_pcts:
results = {
'random': [],
'individual_steiner': [],
'iterative_steiner': [],
'individual_mst': [],
'iterative_mst': [],
'est_lower_bound': []
}
for iteration in range(0, iterations):
print(f'{fast_edge_pct*100} % fast edges, iteration {iteration}')
model_params = placer.get_default_model_params(
graph_size, fast_edge_pct)
model, weights, capacities = placer.get_model(**model_params)
spec = placer.get_random_pipe_spec(
model.nodes,
8, #depth
3, #num inputs per stage
1) #reqd capacity per stage
to_run = placer.prepare_functions(spec, model)
for name, func in to_run.items():
placements, tree = func()
if name == 'individual_steiner':
composed = nx.compose_all((p.tree for p in placements))
results['est_lower_bound'] += [
placer.total_weight(composed)
]
results[name].append(placer.total_weight(tree))
results_for_pct[fast_edge_pct] = results
with open('performance_fast_edge_pct.pickled', 'wb') as pickleout:
pickle.dump(results_for_pct, pickleout)
def superStars(k, m, highSubsets, kernelsSubsets):
graphs = []
graphs.append(severalStarGraphs(m, k, highSubsets))
s = len(kernelsSubsets)
kernels = []
for i in range(s):
kernel = (k+i)*(m+1)
kernels.append(kernel)
nodes = np.array([j*(m+1) for j in range(k)]+[kernel])
if (kernelsSubsets[i] == 1):
graphs.append(simpleStar(nodes, kernel, direction=ut.Direction.OUTGOING))
else:
graphs.append(simpleStar(nodes, kernel, direction=ut.Direction.INCOMING))
if s > 1:
graphs.append(simpleCycleGraph(kernels))
G = nx.compose_all(graphs)
ut.normalizeGraph(G)
return G
def NetworkAnalysis(jsonGraph,subgraphs):
"""gets graph defined by for export json and computes the top 3
most connected subgraphs"""
G = json_graph.node_link_graph(jsonGraph)
graphs = sorted(nx.connected_component_subgraphs(G), key = len, reverse=True)
nodes = [len(graph.nodes()) for graph in graphs]
subgraphs_nodes = [len(graph.nodes()) for graph in graphs[0:subgraphs]]
print nodes
print subgraphs_nodes
frac = sum(subgraphs_nodes)/sum(nodes)
print subgraphs, "represents", frac*100, "% of nodes"
topn = nx.compose_all(graphs[0:subgraphs])
deg = topn.degree()
cent_deg = nx.degree_centrality(topn)
nx.set_node_attributes(topn, "degree", deg)
nx.set_node_attributes(topn, "cent_deg", cent_deg)
take = {
"nodes": json_graph.node_link_data(topn)["nodes"],
"links": json_graph.node_link_data(topn)["links"],
"frac" : frac
}
return take
def run_pipe_depth_tests():
graph_size = 64
iterations = 40
results_for_depth = {}
for depth in range(1, 20):
results = {
'random': [],
'individual_steiner': [],
'iterative_steiner': [],
'individual_mst': [],
'iterative_mst': [],
'est_lower_bound': []
}
for iteration in range(0, iterations):
print(f'{depth} pipe depth, iteration {iteration}')
#choose fast edge pct based on point of max separation between
#steiner and mst methods
model_params = placer.get_default_model_params(graph_size, 0.05)
model, weights, capacities = placer.get_model(**model_params)
spec = placer.get_random_pipe_spec(
model.nodes,
depth, #depth
3, #num inputs per stage
1) #reqd capacity per stage
to_run = placer.prepare_functions(spec, model)
for name, func in to_run.items():
placements, tree = func()
results[name].append(placer.total_weight(tree))
if name == 'individual_steiner':
composed = nx.compose_all((p.tree for p in placements))
results['est_lower_bound'] += [
placer.total_weight(composed)
]
results_for_depth[depth] = results
print(results_for_depth)
with open('performance_pipe_depth.pickled', 'wb') as pickleout:
pickle.dump(results_for_depth, pickleout)
def get_composed_charity_network(
self,
run_index: int = -1,
*args: QueryParameters,
# records: Optional[int] = None,
# start_entry: int = 0,
**kwargs: QueryParameters,
) -> Graph:
"""Iterate over charities, compose networks and return."""
try:
if not self._charity_networks_cached:
prior_runs: int = len(self._charity_runs)
composed_network: Graph = compose_all(
graph for _, graph in self.get_charity_networks())
self._charity_composed_runs.append(composed_network)
first_run: CharityRunConfig = self._charity_runs[prior_runs +
1]
last_run: CharityRunConfig = self._charity_runs[-1]
self._charity_runs.append(
CharityRunConfig(
start_time=first_run.start_time,
end_time=last_run.end_time,
kinds_ids_dict=first_run.kinds_ids_dict,
parameter_state=first_run.parameter_state,
root_id=tuple(self.active_charity_ids),
connected_components_count=number_connected_components(
composed_network),
composed_runs=[
r for r in self._charity_runs[prior_runs:]
],
))
return self._charity_composed_runs[-1]
return self._charity_composed_runs[run_index]
# return compose_all(n.charity_network for n in self
# if hasattr(n, 'charity_network'))
except StopIteration:
logger.warning("No charity networks to compose.")
def chop_subgraph(G):
if len(G.nodes) == 2 or is_clique(G):
return [G]
# Create set of edges and randomly try
# deleting and see if we get broken clique
# move good subgraphs to a list and keep trying
good_subgraphs = []
edges = set(G.edges)
while edges:
H = G.copy()
random_edge = choice(list(edges))
edges.remove(random_edge)
H.remove_edge(*random_edge)
subgraphs = list(gen_subgraphs(H))
if len(subgraphs) == 1:
_ = [delete_min_connected(x) for x in subgraphs]
good = [x for x in subgraphs if is_clique(x)]
if good:
good_subgraphs.extend(good)
bad = [x for x in subgraphs if not is_clique(x)]
if not bad:
break
G = nx.compose_all(bad)
edges = set(G.edges)
# Add recursive call when large subgraphs get split
elif len(subgraphs) > 1:
_ = [delete_min_connected(x) for x in subgraphs]
sgs = []
for g in subgraphs:
sgs.extend(chop_subgraph(g))
good_subgraphs.extend(sgs)
# If any successful splitting, return
# Else, graph is valid
return good_subgraphs or [G]
def run_inputs_per_stage_tests():
graph_size = 96
inputs_per_stage = [i for i in range(1, 11)]
iterations = 40
results_for_pct = {}
for num_inputs in inputs_per_stage:
results = {
'random': [],
'individual_steiner': [],
'iterative_steiner': [],
'individual_mst': [],
'iterative_mst': [],
'est_lower_bound': []
}
for iteration in range(0, iterations):
print(f'{num_inputs} inputs per stage, iteration {iteration}')
model_params = placer.get_default_model_params(graph_size, 0.05)
model, weights, capacities = placer.get_model(**model_params)
spec = placer.get_random_pipe_spec(
model.nodes,
8, #depth
num_inputs, #num inputs per stage
1) #reqd capacity per stage
to_run = placer.prepare_functions(spec, model)
for name, func in to_run.items():
placements, tree = func()
if name == 'individual_steiner':
composed = nx.compose_all((p.tree for p in placements))
results['est_lower_bound'] += [
placer.total_weight(composed)
]
results[name].append(placer.total_weight(tree))
results_for_pct[num_inputs] = results
with open('performance_inputs_per_stage.pickled', 'wb') as pickleout:
pickle.dump(results_for_pct, pickleout)
def unify(self, edge_weights={}):
"""Return a unified DiGraph containing all
subgraphs, with edges weighted differently per subgraph.
"""
social_following = self.social_following()
weight = edge_weights.get('social_following', 1)
for u, v, d in social_following.edges(data=True):
d['weight'] = weight
content_tree = self.content_tree()
weight = edge_weights.get('content_tree', 1)
for u, v, d in content_tree.edges(data=True):
d['weight'] = weight
content_authors = self.content_authors()
weight = edge_weights.get('content_authors', 1)
for u, v, d in content_authors.edges(data=True):
d['weight'] = weight
content_tags = self.content_tags()
weight = edge_weights.get('content_tags', 1)
for u, v, d in content_tags.edges(data=True):
d['weight'] = weight
return nx.compose_all([social_following,
content_tree,
content_authors,
content_tags])
def run_randomized_model_tests():
graph_size = 32
iterations = 40
pipe_depth = 8
results_for_sigma = {}
for sigma in (i * 0.1 for i in range(10, 50, 1)):
results = {
'random': [],
'individual_steiner': [],
'iterative_steiner': [],
'individual_mst': [],
'iterative_mst': [],
'est_lower_bound': []
}
for iteration in range(0, iterations):
print(f'Sigma:{sigma}, iteration {iteration}')
model = placer.get_randomized_model(graph_size, 10, sigma)
spec = placer.get_random_pipe_spec(
model.nodes,
pipe_depth, #depth
3, #num inputs per stage
1) #reqd capacity per stage
to_run = placer.prepare_functions(spec, model)
for name, func in to_run.items():
placements, tree = func()
results[name].append(placer.total_weight(tree))
if name == 'individual_steiner':
composed = nx.compose_all((p.tree for p in placements))
results['est_lower_bound'] += [
placer.total_weight(composed)
]
results_for_sigma[sigma] = results
print(results_for_sigma)
with open('performance_randomized_sigma.pickled', 'wb') as pickleout:
pickle.dump(results_for_sigma, pickleout)
def join_two_trees(G, tree1, tree2):
'''Joins two trees by brute force scanning. Outputs joint tree'''
tree1 = tree1.copy()
tree2 = tree2.copy()
terminals_1 = get_terminals(tree1)
old_bridge = nx.path_graph(100)
best_bridge = old_bridge
for node_1 in terminals_1:
new_bridge = shortest_path_node_subgraph(G, node_1, tree2)
if len(new_bridge) < len(best_bridge):
best_bridge = new_bridge
old_bridge = new_bridge
nx.add_path(tree2, best_bridge)
joint_graph = nx.compose_all([tree1, tree2])
return joint_graph
def unify(self, con:str='s', **kwargs):
'''Unification.'''
self.semantics = []
self.syntax = []
for x, y in self.xref:
if 'x' in y: x, y = y, x
ntok, xsrc = xsourceSplit(x)
self.tokens[ntok - 1].add_xsource(xsrc)
pres = [g.cat for g in self.tokens]
sorts = [g.sort for g in self.tokens]
for con, pres in al.deAbbr(con, pres, self.abbr, self.calc):
con, pres, parse, idxDic = al.searchLinks(self.calc, con, pres, **kwargs)
if parse.proofs:
_tokens = self.tokens.copy()
for i, g in enumerate(_tokens):
if _tokens[i].cat == 'conj':
_tokens[i] = g.conj_expand(idxDic)
sorts[i] = _tokens[i].sort
for p in parse.proofs:
qset = quotSet(p, idxDic, self.xref, sorts)
Gs = []
for g in _tokens:
relabel = {}
for v in g.nodes:
for i, S in qset.items():
if v in S: relabel[v] = 'i%s' % i
Gs.append(g.iso(relabel))
self.semantics.append(compose_all(Gs))
self.syntax.append(PackSyntax(
insight=parse, links=p, idxDic=idxDic))
list of Bool values"""
return [search_seq(i, seq, check_key) for i in graph_frames]
def first_search_seq_temporal(graph_frames, seq, check_key=True):
l = search_seq_temporal(graph_frames, seq, check_key=check_key)
return l.index(True) if True in l else None
if __name__ == "__main__":
x_a = graphcombinator.combine_scene_graphs_list('../../test/video1_out')
y_a = sgs_list_to_dgs_list(x_a)
z_a = get_list_of_sgNx(y_a)
seq_composed_a = list(map(nx.compose_all, z_a))
full_composed_a = nx.compose_all(seq_composed_a)
x_c = graphcombinator.combine_scene_graphs_list('../../test/video3_out')
y_c = sgs_list_to_dgs_list(x_c)
z_c = get_list_of_sgNx(y_c)
seq_composed_c = list(map(nx.compose_all, z_c))
full_composed_c = nx.compose_all(seq_composed_c)
x_d = graphcombinator.combine_scene_graphs_list('../../test/video4_out')
y_d = sgs_list_to_dgs_list(x_d)
z_d = get_list_of_sgNx(y_d)
seq_composed_d = list(map(nx.compose_all, z_d))
full_composed_d = nx.compose_all(seq_composed_d)
x_e = graphcombinator.combine_scene_graphs_list('../../test/video5_out')
y_e = sgs_list_to_dgs_list(x_e)
def play(self):
"""Play recorded graphs.
If the animation constructor has been called with ``width=None``, checks
if all recorded graphs have the same width. If they do, such width is
used for displaying the animation. Otherwise, the default value for
graph width is used. Same for ``height=None``.
At least two graphs must have been recorded.
"""
if len(self.graphs) < 2:
raise ValueError('animation must have at least two recs')
h = None
width = self.width
height = self.height
last = self.graphs[-1]
number_of_nodes = last.number_of_nodes()
number_of_edges = last.number_of_edges()
local_width = _build_graph_width(last)
local_height = _build_graph_height(last)
for g in self.graphs[:-1]:
if h is None and (g.number_of_nodes() != number_of_nodes or g.number_of_edges() != number_of_edges):
h = nx.compose_all(self.graphs)
if width is None and _build_graph_width(g) != local_width:
width = graph_width
if height is None and _build_graph_height(g) != local_height:
height = graph_height
if width is None:
width = local_width
if height is None:
height = local_height
frames = []
for i, g in enumerate(self.graphs):
if h is None:
frames.append(self._render(g, g, width, height))
else:
if g != last:
next = self.graphs[i + 1]
for n in next.nodes:
h.nodes[n].update(next.nodes[n])
for n, m in next.edges:
h.edges[n, m].update(next.edges[n, m])
frames.append(self._render(g, h, width, height))
# parameters estimated from screenshots
width = 1.05 * width + 72
height = 1.00 * height + 76
steps = []
for i, frame in enumerate(frames):
frame['name'] = i
step = {
'args': [[i], {'frame': {'redraw': False}, 'mode': 'immediate'}],
'label': '',
'method': 'animate',
}
steps.append(step)
layout = _build_layout(width, height)
layout.update({
'updatemenus': [
{
'buttons': [
{
'args': [None, {'frame': {'redraw': False}, 'fromcurrent': True}],
'label': 'Play',
'method': 'animate',
},
{
'args': [[None], {'frame': {'redraw': False}, 'mode': 'immediate'}],
'label': 'Pause',
'method': 'animate',
},
],
'showactive': True,
'type': 'buttons',
},
],
'sliders': [
{
'currentvalue': {'visible': False},
'steps': steps,
},
],
})
figure = {
'data': frames[0]['data'],
'layout': layout,
'frames': frames,
}
plotly.offline.iplot(figure, config={'staticPlot': True}, show_link=False)
def test_mixed_type_compose():
G = nx.Graph()
H = nx.MultiGraph()
I = nx.Graph()
U = nx.compose_all([G,H,I])
def to_networkx(self):
graph = nx.compose_all(self.components())
return graph
from gameplay import play_game
from policies import RandomPolicy, MCTSPolicy
import numpy as np
import networkx as nx
player_policies = [MCTSPolicy(), RandomPolicy()]
# For reproducibility
np.random.seed(0)
games = []
for i in range(100):
games.append(play_game(player_policies))
graphs = [game[0] for game in games]
dot_graph_combined = nx.compose_all(graphs)
dot_graph = nx.to_pydot(dot_graph_combined)
dot_graph.set_graph_defaults(fontname='Courier')
dot_graph.write_png('multiple_game_graph.png')
