def _get_sys_graph(self):
"""Return the subsystem graph for this Group."""
sgraph = self._relevance._sgraph
if self.pathname:
path = self.pathname.split('.')
start = self.pathname + '.'
else:
path = []
start = ''
graph = sgraph.subgraph([n for n in sgraph
if n.startswith(start)])
renames = {}
for node in graph.nodes_iter():
renames[node] = '.'.join(node.split('.')[:len(path)+1])
if renames[node] == node:
del renames[node]
# get the graph of direct children of current group
nx.relabel_nodes(graph, renames, copy=False)
# remove self loops created by renaming
graph.remove_edges_from([(u, v) for u, v in graph.edges()
if u == v])
return graph
def grid_2d(dim):
"""Creates a 2d grid of dimension dim"""
graph = nx.grid_2d_graph(dim, dim)
for node in graph:
graph.node[node]['asn'] = 1
graph.node[node]['x'] = node[0] * 150
graph.node[node]['y'] = node[1] * 150
graph.node[node]['device_type'] = 'router'
graph.node[node]['platform'] = 'cisco'
graph.node[node]['syntax'] = 'ios_xr'
graph.node[node]['host'] = 'internal'
graph.node[node]['ibgp_role'] = "Peer"
mapping = {node: "%s_%s" % (node[0], node[1]) for node in graph}
# Networkx wipes data if remap with same labels
nx.relabel_nodes(graph, mapping, copy=False)
for src, dst in graph.edges():
graph[src][dst]['type'] = "physical"
# add global index for sorting
SETTINGS['General']['deploy'] = True
SETTINGS['Deploy Hosts']['internal'] = {
'cisco': {
'deploy': True,
},
}
return graph
def load_graphs(fname, atlas):
"""Load a graph and return both the original and the common-nodes one.
This reads a pickled graph and returns a pair: the graph just loaded and
one that contains only nodes common with those in the atlas.
"""
f = open(fname, 'rb')
f.seek(0) # make sure we're at the beginning of the file
g_coco = pickle.load(f)
f.close()
# relabel nodes to remove 'PHT00-' prefix
remove_pht_map = dict([(lab, lab.replace('PHT00-','')) for lab in g_coco])
g_coco = nx.relabel_nodes(g_coco, remove_pht_map)
if hasattr(atlas, 'cocomac'):
# relabel graph according to cocomac nodes
name_map = dict(zip(atlas.cocomac, atlas.label))
g = nx.relabel_nodes(g_coco, name_map)
else:
g = g_coco
common = set(g.nodes()).intersection(set(atlas.label))
gnorm = nx.DiGraph()
for node in common:
gnorm.add_node(node)
for u,v,data in g.edges(data=True):
if u in common and v in common:
gnorm.add_edge(u, v, data)
return g, gnorm
def get_clusters(self, num_clusters=None):
if num_clusters == None:
index, value = max(enumerate(self.quality_history), key=lambda iv: iv[1])
num_clusters = len(self.quality_history) - index
nx.relabel_nodes(self.dendrogram, self.rename_map.integer, copy=False)
start_node = max(self.dendrogram)
priors, fringe = self.dendrogram_crawl(start=start_node, max_steps=num_clusters - 1)
# Double check we got the right number of values
if len(fringe) != num_clusters:
raise ValueError(
"get_clusters failed to retrieve "
+ "%d clusters correctly (got %d instead)" % (num_clusters, len(fringe))
)
clusters = []
for neg_clust_start in fringe:
clust_start = -neg_clust_start
cprior, cfringe = self.dendrogram_crawl(start=clust_start, priors=priors.copy())
clusters.append(
set(self.rename_map.original[n] for n in cprior if n <= clust_start and self.orig.has_node(n))
)
nx.relabel_nodes(self.dendrogram, self.rename_map.original, copy=False)
return sorted(clusters, key=lambda c: -len(c))
def grid_2d(dim):
"""Creates a 2d grid of dimension dim"""
graph = nx.grid_2d_graph(dim, dim)
for node in graph:
graph.node[node]["asn"] = 1
graph.node[node]["x"] = node[0] * 150
graph.node[node]["y"] = node[1] * 150
graph.node[node]["device_type"] = "router"
graph.node[node]["platform"] = "cisco"
graph.node[node]["syntax"] = "ios_xr"
graph.node[node]["host"] = "internal"
graph.node[node]["ibgp_role"] = "Peer"
mapping = {node: "%s_%s" % (node[0], node[1]) for node in graph}
# Networkx wipes data if remap with same labels
nx.relabel_nodes(graph, mapping, copy=False)
for src, dst in graph.edges():
graph[src][dst]["type"] = "physical"
# add global index for sorting
SETTINGS["General"]["deploy"] = True
SETTINGS["Deploy Hosts"]["internal"] = {"cisco": {"deploy": True}}
return graph
def collapse_nodes(graph, node_map, copy=False):
"""
Args
----
graph : nx.DiGraph
Graph with nodes we want to collapse.
node_map : dict
A map of existing node names to collapsed names.
copy : bool(False)
If True, copy the graph before collapsing the nodes.
Returns
-------
nx.DiGraph
The graph with the nodes collapsed.
"""
nx.relabel_nodes(graph, node_map, copy=copy)
# remove any self edges created by the relabeling
graph.remove_edges_from([(u, v) for u, v in graph.edges_iter()
if u == v])
return graph
def createtree(n):
T=nx.DiGraph()
slevel=[1]*(len(G.node[n]['haplotype'][0])+1)
T.add_node(1)
for i, hap in enumerate(G.node[n]['haplotype']):
cnode = 1
for j, h in enumerate(hap):
if h in [(edata['allele']) for u,v,edata in T.out_edges(cnode,data=True) if 'allele' in edata]:
T[cnode][v]['weight'] = T[cnode][v]['weight'] + G.node[n]['frequency'][i]
cnode = v
else:
if j == len(hap)-1 or slevel[j]==slevel[j+1]:
pass
else:
#Define function to relabel node names
def mapping(x):
if x > slevel[j+1]:
return x+1
else:
return x
#Relabel node names which are above the level of the node being added.
nx.relabel_nodes(T,mapping,copy=False)
T.add_edge(cnode, slevel[j+1]+1, weight=G.node[n]['frequency'][i], allele=h)
cnode = slevel[j+1]+1
for a in range(j+1,len(slevel)):
slevel[a] = slevel[a] + 1
return T
def __init__(self, *args, basepath=None, **kwargs):
super(CandidateGraph, self).__init__(*args, **kwargs)
self.node_counter = 0
node_labels = {}
self.node_name_map = {}
# the node_name is the relative path for the image
for node_name, node in self.nodes_iter(data=True):
image_name = os.path.basename(node_name)
image_path = node_name
# Replace the default node dict with an object
self.node[node_name] = Node(image_name, image_path)
# fill the dictionary used for relabelling nodes with relative path keys
node_labels[node_name] = self.node_counter
# fill the dictionary used for mapping base name to node index
self.node_name_map[self.node[node_name].image_name] = self.node_counter
self.node_counter += 1
nx.relabel_nodes(self, node_labels, copy=False)
# Add the Edge class as a edge data structure
for s, d, edge in self.edges_iter(data=True):
self.edge[s][d] = Edge(self.node[s], self.node[d])
def get_dot(G_orig):
"""Change labels and colors to be presented with graphviz"""
G = G_orig.copy()
cluster_networks.relabel_graph(G)
tr = {}
for node in G.nodes_iter():
tr[node] = '"{}"'.format(node)
nx.relabel_nodes(G, tr, copy=False)
for node in G.nodes_iter():
label = str(node)
if len(label) > MAX_LABEL:
label = u'{}..."'.format(label[:MAX_LABEL])
G.node[node]['label'] = label
for node in cluster_networks.get_leaf_nodes(G):
G.node[node]['color'] = "blue"
for node in cluster_networks.hybrid_nodes(G):
G.node[node]['color'] = "#7BFF74" # light green
G.node[node]['style'] = 'filled'
for node in cluster_networks.get_root_nodes(G):
G.node[node]['color'] = "orange"
for node in cluster_networks.problematic_treechild_nodes(G):
G.node[node]['color'] = "#FF77EB" # light pink
G.node[node]['style'] = 'filled'
for u, v in cluster_networks.removable_edges(G):
G.edge[u][v]['color'] = "red"
for root in cluster_networks.get_root_nodes(G):
G.node[root]['label'] = '"R"'
dot = nx.to_pydot(G).to_string()
return dot.strip()
def spingraph_from_graph(graph):
# even_graph = nx.relabel_nodes(graph, lambda x:x*2)
# odd_graph = nx.relabel_nodes(graph, lambda x:2*x+1)
# union_graph = nx.union(even_graph, odd_graph)
# on the fly union saves about 20% memory, ugly but more efficient
union_graph = nx.union(nx.relabel_nodes(graph, lambda x:x*2),nx.relabel_nodes(graph, lambda x:2*x+1))
# from pudb import set_trace; set_trace()
for spin_down_node in xrange(1,union_graph.order(),2):
spin_up_node = spin_down_node -1
for spin_down_node_neighbour in union_graph[spin_down_node].keys():
if spin_down_node_neighbour % 2 ==0:
continue
if spin_down_node_neighbour < spin_down_node: # is either top or left neighbour
if spin_down_node_neighbour == spin_down_node-2: # is left neighbour
union_graph.add_edge(spin_up_node,spin_down_node_neighbour,weight=-p.tso)
union_graph.add_edge(spin_down_node_neighbour,spin_up_node,weight=-p.tso)
else:
union_graph.add_edge(spin_up_node,spin_down_node_neighbour,weight=+1j*p.tso)
union_graph.add_edge(spin_down_node_neighbour,spin_up_node,weight=-1j*p.tso)
if spin_down_node_neighbour > spin_down_node: # is either right or bottom neighbour
if spin_down_node_neighbour == spin_down_node+2: # is right neighbour
union_graph.add_edge(spin_up_node,spin_down_node_neighbour,weight=p.tso)
union_graph.add_edge(spin_down_node_neighbour,spin_up_node,weight=p.tso)
else:
union_graph.add_edge(spin_up_node,spin_down_node_neighbour,weight=-1j*p.tso)
union_graph.add_edge(spin_down_node_neighbour,spin_up_node,weight=+1j*p.tso)
return union_graph
def sub_values(tree, var_values):
"""Substitute given values for variables.
@param tree: AST
@type var_values: C{dict}
@return: AST with L{Var} nodes replaces by
L{Num}, L{Const}, or L{Bool}
"""
old2new = dict()
for u in tree.nodes_iter():
if u.type != 'var':
continue
val = var_values[u.value]
# instantiate appropriate value type
if isinstance(val, bool):
v = nodes.Bool(val)
elif isinstance(val, int):
v = nodes.Num(val)
elif isinstance(val, str):
v = nodes.Str(val)
old2new[u] = v
# replace variable by value
nx.relabel_nodes(tree, old2new, copy=False)
def __init__(self, *args, basepath=None, **kwargs):
super(CandidateGraph, self).__init__(*args, **kwargs)
self.node_counter = 0
node_labels = {}
self.node_name_map = {}
for node_name in self.nodes():
image_name = os.path.basename(node_name)
image_path = node_name
# Replace the default attr dict with a Node object
self.node[node_name] = Node(image_name, image_path, self.node_counter)
# fill the dictionary used for relabelling nodes with relative path keys
node_labels[node_name] = self.node_counter
# fill the dictionary used for mapping base name to node index
self.node_name_map[self.node[node_name].image_name] = self.node_counter
self.node_counter += 1
nx.relabel_nodes(self, node_labels, copy=False)
for s, d in self.edges():
if s > d:
s, d = d, s
e = self.edge[s][d]
e.source = self.node[s]
e.destination = self.node[d]
self.creationdate = strftime("%Y-%m-%d %H:%M:%S", gmtime())
self.modifieddate = strftime("%Y-%m-%d %H:%M:%S", gmtime())
def add_edge_from_cnode(G,c,h,l):
#If node is being created at last level
if l == haplolength - 1:
#Add edge from current node to node with name one higher than existing nodes.
#Set weight and allele to edge data.
G.add_edge(c, (level[l+1]+1), weight=frequency[h], allele=haplotype[h][l])
#If node is being created that is not at the last level
else:
#Define function to relabel node names
def mapping(x):
if x > level[l+1]:
return x+1
else:
return x
#Relabel node names which are above the level of the node being added.
nx.relabel_nodes(G,mapping,copy=False)
#Add edge from current node to node with name one higher than the last node on the next level.
#Set weight and allele to edge data.
G.add_edge(c, (level[l+1]+1), weight=frequency[h], allele=haplotype[h][l])
def coword_network(mesh_df, start, end,topic_count=0):
"""
constructs a coword network for the years supplied;
nodes will be labelled by topic, have a 'weight' of co-occurrence,
a 'start_year' attribute,
and an 'end_year' attribute which is the end year of the search
Parameters
----------------
mesh_df: a dataframe with at least the topics and years columns
start: start year
end: end year
topic_count: the number of the topics to use
(not too big, otherwise coword matrix will be huge
"""
# determine the number of topics to count
all_topics = [t for top in mesh_df.topics.dropna() for t in top]
topic_collection = collections.Counter(all_topics)
if topic_count > 0 and topic_count < len(topic_collection):
common_topics = [k[0] for k in topic_collection.most_common(topic_count)]
else:
common_topics = sorted(topic_collection.keys())
cow_df = coword_matrix_years(mesh_df, start, end, common_topics)
cow_nx = nx.from_numpy_matrix(cow_df.as_matrix())
col_names = cow_df.columns.tolist()
labels = {col_names.index(l): l for l in col_names}
start_year = {i: end for i in range(0, len(col_names))}
end_year = {i: start for i in range(0, len(col_names))}
nx.set_node_attributes(cow_nx, 'start_year', start_year)
nx.set_node_attributes(cow_nx, 'end_year', end_year)
nx.relabel_nodes(cow_nx, labels, copy=False)
return cow_nx
def get_clusters(self, num_clusters=None):
if num_clusters == None:
index, value = max(enumerate(self.quality_history), key=lambda iv: iv[1])
num_clusters = len(self.quality_history) - index
num_clusters = max(min(num_clusters, self.max_clusters), 0)
clusters = [set([n]) for n in self.orphans]
if self.dendrogram and num_clusters:
nx.relabel_nodes(self.dendrogram, self.rename_map.integer, copy=False)
try:
start_node = max(self.dendrogram)
priors, fringe = self.dendrogram_crawl(start=start_node, max_fringe_size=num_clusters)
# Double check we got the right number of values
if len(fringe) != num_clusters:
raise ValueError("Failed to retrieve %d clusters correctly (got %d instead)"
% (num_clusters, len(fringe)))
for neg_clust_start in fringe:
clust_start = -neg_clust_start
cprior, cfringe = self.dendrogram_crawl(start=clust_start, priors=priors.copy())
cluster_set = set(self.rename_map.original[n] for n in cprior
if n <= clust_start and self.orig.has_node(n))
if cluster_set:
clusters.append(cluster_set)
finally:
nx.relabel_nodes(self.dendrogram, self.rename_map.original, copy=False)
return sorted(clusters, key=lambda c: -len(c))
def coalesce(G, node1, node2):
"""Performs Briggs coalescing. Takes in the graph and two nodes.
Returns 1 if unable to coalesce, 0 otherwise."""
# we need to get a modified degrees list for pre-colored nodes
degs = nx.degree(G)
for i in range(1, k + 1):
degs[str(i)] = 9999
if node1 in G.neighbors(node2) or node2 in G.neighbors(node1):
print "ERROR:", node1, "and", node2, "share an edge. \
Check your moves list"
return # intentionally void to cause error
elif degs[node1] + degs[node2] >= k:
print "Failure:", node1, "and", node2, "have combined degree \
= " + str(
degs[node1] + degs[node2]
) + ", which is too high for k =", k
return 1
else:
newedge = []
for i in range(len(G.neighbors(node2))):
newedge.append((node1, G.neighbors(node2)[i]))
G.add_edges_from(newedge)
G.remove_node(node2)
nx.relabel_nodes(G, {node1: node1 + node2}, copy=False)
print "Success:", node1, "and", node2, "have been coalesced"
return 0
def grid_2d(dim):
import networkx as nx
graph = nx.grid_2d_graph(dim, dim)
for n in graph:
graph.node[n]['asn'] = 1
graph.node[n]['x'] = n[0] * 150
graph.node[n]['y'] = n[1] * 150
graph.node[n]['device_type'] = 'router'
graph.node[n]['platform'] = 'cisco'
graph.node[n]['syntax'] = 'ios_xr'
graph.node[n]['host'] = 'internal'
graph.node[n]['ibgp_level'] = 0
mapping = {n: "%s_%s" % (n[0], n[1]) for n in graph}
nx.relabel_nodes(graph, mapping, copy=False) # Networkx wipes data if remap with same labels
for index, (src, dst) in enumerate(graph.edges()):
graph[src][dst]['type'] = "physical"
graph[src][dst]['edge_id'] = "%s_%s_%s" % (index, src, dst) # add global index for sorting
SETTINGS['General']['deploy'] = True
SETTINGS['Deploy Hosts']['internal'] = {
'cisco': {
'deploy': True,
},
}
return graph
def synthetic_three_level(n,p1,p2,p3,J_isolates=False,F_isolates=False,D_isolates=False):#,isolate_up=True,isolate_down=True):
k=n
J=nx.erdos_renyi_graph(n,p1) #The first layer graph
Jis = nx.isolates(J)
F=nx.erdos_renyi_graph(n,p2) #The second layer graph
Fis = nx.isolates(F)
D=nx.erdos_renyi_graph(n,p3) #The third layer graph
Dis = nx.isolates(D)
def translation_graph(J,F,D):
H1=nx.Graph()
H2=nx.Graph()
for i in range(n):
H1.add_edges_from([(J.nodes()[i],F.nodes()[i])])
H2.add_edges_from([(F.nodes()[i],D.nodes()[i])])
return H1, H2
Jed = set(J.edges())
Fed = set(F.edges())
Ded = set(D.edges())
l=[Jed,Fed,Ded]
lu = list(set.union(*l))
JFD=nx.Graph()
JFD.add_edges_from(lu)
G=nx.Graph() #The synthetic two-layer graph
# Relabing nodes maps
mappingF={}
for i in range(2*n):
mappingF[i]=n+i
FF=nx.relabel_nodes(F,mappingF,copy=True)
mappingD={}
for i in range(2*n):
if i >n-1:
mappingD[i]=i-n
else:
mappingD[i]=2*n+i
DD=nx.relabel_nodes(D,mappingD,copy=True)
H1, HH2 = translation_graph(J,FF,DD)
G.add_edges_from(J.edges())
G.add_edges_from(H1.edges())
G.add_edges_from(DD.edges())
G.add_edges_from(HH2.edges())
G.add_edges_from(FF.edges())
edgeList = []
for e in H1.edges():
edgeList.append(e)
for e in HH2.edges():
edgeList.append(e)
return G, J, FF, DD, JFD, edgeList
def as_nx_graph(self):
result = nx.from_numpy_matrix(self.matrix)
nx.relabel_nodes(result,
dict(zip(
numpy.arange(len(result.nodes())),
self.rows)),
False)
return result
def expand(G_in):
""" Expands out graph products. G is the source "backbone" graph. H_x is the "PoP template" graphs
"""
graph_unwrapped = ank_utils.unwrap_graph(G_in)
G = graph_unwrapped.copy()
ank.set_node_default(G_in, G_in)
template_names = set(node.pop_template for node in G_in)
template_names.remove(None)
if not len(template_names):
return # no templates set
# Load these templates
templates = {}
for template in template_names:
print "TEMplate is", template
template_filename = os.path.join("pop_templates", "%s.graphml" % template)
pop_graph = ank.load_graphml(template_filename) #TODO: pass in properties eg edge type = physical
pop_graph = pop_graph.to_undirected() # Undirected for now TODO: document this
templates[template] = pop_graph
# construct new graph
G_out = nx.Graph() #TODO: what about bidirectional graphs?
G_out.add_nodes_from(expand_nodes(G, templates))
G_out.add_edges_from(intra_pop_links(G, templates))
G_out.add_edges_from(inter_pop_links(G, templates))
for s, t in G_out.edges():
G_out[s][t]['type'] = 'physical' # ensure copied across
# Update properties based on co-ordinates
for node in G_out:
u, v = node
template = G.node[u]['pop_template']
u_properties = dict(G.node[u])
v_properties = dict(templates[template].node[v]) # create copy to append with
x = float(u_properties.get('x')) + float(v_properties.get('x'))
y = float(u_properties.get('y')) + float(v_properties.get('y'))
asn = u_properties['asn']
u_properties.update(v_properties)
u_properties['x'] = x
u_properties['y'] = y
u_properties['label'] = "%s_%s" % (v, u)
u_properties['id'] = "%s_%s" % (v, u)
u_properties['pop'] = u
u_properties['asn'] = asn # restore, don't inherit from pop
del u_properties['pop_template']
G_out.node[node] = u_properties
nx.relabel_nodes(G_out, dict( ((u, v), "%s_%s" % (v, u)) for (u, v) in G_out), copy = False)
#TODO: set edge_ids
for s, t in G_out.edges():
G_out[s][t]['edge_id'] = "%s_%s" % (s, t)
G_in._replace_graph(G_out)
return
def _setup_graphs(self, group, connections):
"""
Set up dependency graphs for variables and components in the Problem.
Returns
-------
tuple of (nx.DiGraph, nx.DiGraph)
The variable graph and the component graph.
"""
params_dict = self.params_dict
unknowns_dict = self.unknowns_dict
vgraph = nx.DiGraph() # var graph
sgraph = nx.DiGraph() # subsystem graph
compins = {} # maps input vars to components
compouts = {} # maps output vars to components
promote_map = {}
# ensure we have system graph nodes even for unconnected subsystems
sgraph.add_nodes_from([s.pathname for s in group.subsystems(recurse=True)])
for meta in params_dict.values():
param = meta['pathname']
tcomp = param.rsplit('.', 1)[0]
compins.setdefault(tcomp, []).append(param)
if param in connections and meta['promoted_name'] != param:
promote_map[param] = meta['promoted_name']
for meta in unknowns_dict.values():
unknown = meta['pathname']
scomp = unknown.rsplit('.', 1)[0]
compouts.setdefault(scomp, []).append(unknown)
if meta['promoted_name'] != unknown:
promote_map[unknown] = meta['promoted_name']
for target, source in connections.items():
vgraph.add_edge(source, target)
sgraph.add_edge(source.rsplit('.', 1)[0], target.rsplit('.', 1)[0])
# connect inputs to outputs on same component in order to fully
# connect the variable graph.
for comp, inputs in compins.items():
for inp in inputs:
for out in compouts.get(comp, ()):
vgraph.add_edge(inp, out)
# now collapse any var nodes with implicit connections
nx.relabel_nodes(vgraph, promote_map, copy=False)
# remove any self edges created by the relabeling
for u, v in vgraph.edges():
if u == v:
vgraph.remove_edge(u, v)
return vgraph, sgraph
def _build_network(self, demand_nodes):
"""
project demand nodes onto optional existing supply network and
network generation algorithm on it
Args:
demand_nodes: GeoGraph of demand nodes
Returns:
GeoGraph minimum spanning forest proposed by the chosen
network algorithm
"""
geo_graph = subgraphs = rtree = None
existing = None
if 'existing_networks' in self.config:
existing = networker_runner.load_existing_networks(
**self.config['existing_networks'])
# rename existing nodes so that they don't intersect with metrics
nx.relabel_nodes(existing,
{n: 'grid-' + str(n) for n in existing.nodes()}, copy=False)
existing.coords = {'grid-' + str(n): c for n, c in
existing.coords.items()}
geo_graph, subgraphs, rtree = \
networker_runner.merge_network_and_nodes(existing, \
demand_nodes)
else:
geo_graph = demand_nodes
# now run the selected algorithm
network_algo = networker_runner.NetworkerRunner.ALGOS[\
self.config['network_algorithm']]
result_geo_graph = network_algo(geo_graph, subgraphs=subgraphs,\
rtree=rtree)
# now filter out subnetworks via minimum node count
min_node_count = self.config['network_parameters']\
['minimum_node_count']
filtered_graph = nx.union_all(filter(
lambda sub: len(sub.node) >= min_node_count,
nx.connected_component_subgraphs(result_geo_graph)))
# map coords back to geograph
# NOTE: explicit relabel to int as somewhere in filtering above, some
# node ids are set to numpy types which screws up comparisons to
# tuples in write op
# TODO: Google problem and report to networkx folks if needed
# NOTE: relabeling nodes in-place here drops node attributes for some
# reason so create a copy for now
# NOTE: use i+1 as node id in graph because dataset_store node ids
# start at 1 (this is the realignment noted in _get_demand_nodes)
coords = {i+1: result_geo_graph.coords[i] for i in filtered_graph}
relabeled = nx.relabel_nodes(filtered_graph, {i: int(i+1)
for i in filtered_graph}, copy=True)
msf = GeoGraph(result_geo_graph.srs, coords=coords, data=relabeled)
return existing, msf
def _run_interface(self, runtime):
if not have_cv:
raise ImportError("cviewer library is not available")
THRESH = self.inputs.threshold
K = self.inputs.number_of_permutations
TAIL = self.inputs.t_tail
edge_key = self.inputs.edge_key
details = edge_key + '-thresh-' + str(THRESH) + '-k-' + str(K) + '-tail-' + TAIL + '.pck'
# Fill in the data from the networks
X = ntwks_to_matrices(self.inputs.in_group1, edge_key)
Y = ntwks_to_matrices(self.inputs.in_group2, edge_key)
PVAL, ADJ, _ = nbs.compute_nbs(X, Y, THRESH, K, TAIL)
iflogger.info('p-values:')
iflogger.info(PVAL)
pADJ = ADJ.copy()
for idx, _ in enumerate(PVAL):
x, y = np.where(ADJ == idx + 1)
pADJ[x, y] = PVAL[idx]
# Create networkx graphs from the adjacency matrix
nbsgraph = nx.from_numpy_matrix(ADJ)
nbs_pval_graph = nx.from_numpy_matrix(pADJ)
# Relabel nodes because they should not start at zero for our convention
nbsgraph = nx.relabel_nodes(nbsgraph, lambda x: x + 1)
nbs_pval_graph = nx.relabel_nodes(nbs_pval_graph, lambda x: x + 1)
if isdefined(self.inputs.node_position_network):
node_ntwk_name = self.inputs.node_position_network
else:
node_ntwk_name = self.inputs.in_group1[0]
node_network = nx.read_gpickle(node_ntwk_name)
iflogger.info('Populating node dictionaries with attributes from %s',
node_ntwk_name)
for nid, ndata in node_network.nodes(data=True):
nbsgraph.nodes[nid] = ndata
nbs_pval_graph.nodes[nid] = ndata
path = op.abspath('NBS_Result_' + details)
iflogger.info(path)
nx.write_gpickle(nbsgraph, path)
iflogger.info('Saving output NBS edge network as %s', path)
pval_path = op.abspath('NBS_P_vals_' + details)
iflogger.info(pval_path)
nx.write_gpickle(nbs_pval_graph, pval_path)
iflogger.info('Saving output p-value network as %s', pval_path)
return runtime
def test___eq__(self):
S3 = nx.DiGraph.copy(S1)
# Structure equality should not depend on object identifiers
# because they are arbitrary.
nx.relabel_nodes(S3, {"N2": "N3"}, copy=False)
assert S1 == S3
# Two structures should not be equal if at least one function
# value pair is different.
S3.node["N3"]["label"] = "cat"
assert S1 != S3
def _check_graph(self, out_stream=sys.stdout):
# Cycles in group w/o solver
cgraph = self.root._relevance._cgraph
for grp in self.root.subgroups(recurse=True, include_self=True):
path = [] if not grp.pathname else grp.pathname.split('.')
graph = cgraph.subgraph([n for n in cgraph if n.startswith(grp.pathname)])
renames = {}
for node in graph.nodes_iter():
renames[node] = '.'.join(node.split('.')[:len(path)+1])
if renames[node] == node:
del renames[node]
# get the graph of direct children of current group
nx.relabel_nodes(graph, renames, copy=False)
# remove self loops created by renaming
graph.remove_edges_from([(u,v) for u,v in graph.edges()
if u==v])
strong = [s for s in nx.strongly_connected_components(graph)
if len(s)>1]
if strong and isinstance(grp.nl_solver, RunOnce): # no solver, cycles BAD
relstrong = []
for slist in strong:
relstrong.append([])
for s in slist:
relstrong[-1].append(name_relative_to(grp.pathname, s))
relstrong[-1] = sorted(relstrong[-1])
print("Group '%s' has the following cycles: %s" %
(grp.pathname, relstrong), file=out_stream)
# Components/Systems/Groups are not in the right execution order
subnames = [s.pathname for s in grp.subsystems()]
while strong:
# break cycles to check order
lsys = [s for s in subnames if s in strong[0]]
for p in graph.predecessors(lsys[0]):
if p in lsys:
graph.remove_edge(p, lsys[0])
strong = [s for s in nx.strongly_connected_components(graph)
if len(s)>1]
visited = set()
out_of_order = set()
for sub in grp.subsystems():
visited.add(sub.pathname)
for u,v in nx.dfs_edges(graph, sub.pathname):
if v in visited:
out_of_order.add(v)
if out_of_order:
print("In group '%s', the following subsystems are out-of-order: %s" %
(grp.pathname, sorted([name_relative_to(grp.pathname, n)
for n in out_of_order])), file=out_stream)
def add_all_inodes(iTree):
"""function to construct interaction tree, given suitably annotated gene tree"""
# first we build a list of nodes ordered by their birth time
nodes_t_births = [(n, iTree.node[n]['t_birth']) for n in iTree.nodes()]
ordered_nodes = [n for n, d in sorted(nodes_t_births, key=lambda x:x[1])]
for gene in ordered_nodes:
for fellow in get_fellow_extants(iTree, gene):
# if doesn't already exist, add an interaction between node and gene
if get_inode_name(gene, fellow) not in iTree.nodes():
new_interaction = get_inode_name(gene, fellow)
new_int_name = get_inode_name(iTree.node[gene]['name'],
iTree.node[fellow]['name'])
iTree.add_node(new_interaction,
node_type='interaction',
S=iTree.node[gene]['S'],
name=new_int_name
)
iTree.add_edge(gene, new_interaction)
iTree.add_edge(fellow, new_interaction)
parent_interaction, evol_dist = get_parent_interaction(iTree, new_interaction)
if parent_interaction:
iTree.add_edge(parent_interaction, new_interaction, evol_dist=evol_dist)
# we can use the remaining gene nodes to mark all the extant interactions
# we first list all the extant gene nodes in the present time - ie. no children
extant_gnodes = set([n for n in iTree.nodes() if not iTree.successors(n)])
for inode in [n for n in iTree.nodes() if iTree.node[n]['node_type'] == 'interaction']:
parent_genes = [n for n in iTree.predecessors(inode)
if iTree.node[n]['node_type'] == 'gene']
# if the parent gene(s) is / are both extant, then so is the interaction
if set(parent_genes).issubset(extant_gnodes):
iTree.node[inode]['extant'] = True
# we don't want the gTree nodes actually remaining as part of the iTree
iTree.remove_nodes_from([n for n in iTree.nodes() if iTree.node[n]['node_type'] == 'gene'])
# instead of inscrutable numbers as the nodes,
# we relabel using the 'name' property
nx.relabel_nodes(iTree,
{n: iTree.node[n]['name'] for n in iTree.nodes()},
copy=False)
return iTree
def rename_method(self, method_name):
new_name = 'method%d' % self.counter
self.counter += 1
method_info = self._method_call_graph.node[method_name]
method_info['method'].name = new_name
change_size = 1
for inv in self.method_invocations(method_name):
inv.name = new_name
change_size += 1
relabel_nodes(self._method_call_graph, {method_name: new_name}, copy=False)
method_info['fitness'] = random()
return change_size, new_name
def outputGraph(self, filename, _filter=None):
if filename not in self._graphTime:
self._graphTime[filename] = 0
if (time.time() - self._graphTime[filename]) > 10:
# print 'pre-Update Graph: %s' % filename
G = nx.DiGraph()
plt.clf()
for edge in self._edges:
if _filter is None:
G.add_edge(edge[0], edge[1])
elif _filter(edge[0]) or _filter(edge[1]):
G.add_edge(edge[0], edge[1])
try:
G1 = dfs_tree(G, u"0015")
G2 = dfs_tree(G, u"0013")
G3 = nx.compose(G1, G2)
G4 = dfs_tree(G, u"0017")
G = nx.compose(G3, G4)
except:
pass
relabel = {}
newToOld = {}
for n in G.nodes():
if n in self._berths and self._berths[n] is not None:
relabel[n] = self._berths[n]
newToOld[self._berths[n]] = n
nx.relabel_nodes(G, relabel, False)
colours = []
for n in G.nodes():
if n in newToOld:
n = newToOld[n]
if n in self._berths and self._berths[n] is not None:
colours.append("r")
else:
colours.append("g")
pos = nx.graphviz_layout(G, prog="dot")
nx.draw_networkx_nodes(G, pos, node_color=colours, node_shape="s", node_size=900)
nx.draw_networkx_edges(G, pos)
nx.draw_networkx_labels(G, pos)
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(16.0, 25.0)
plt.axis("off")
self.outputFile(plt, filename)
self._graphTime[filename] = time.time()
def get_graph(graph_file, map_file, trim=False):
"""
graph_file --> is the file to convert to a networkx graph
trim --> either takes an integer or 'False'. If integer, returned graph
will call return graph with nodes having degree greater than integer
"""
the_graph = net.DiGraph(net.read_pajek(graph_file))
id_map = make_id_map(map_file)
net.relabel_nodes(the_graph, id_map, copy=False)
if trim:
the_graph = trim_degrees(the_graph, degree=trim)
return the_graph
def barabasi_albert():
G = nx.barabasi_albert_graph(121, BARABASI_NEIGHBORS)
G = nx.convert_node_labels_to_integers(G)
mapping = {}
for node in G.nodes():
i = node / 11
j = node % 11
mapping[node] = (i, j)
nx.relabel_nodes(G, mapping, copy=False)
G.graph['name'] = 'barabasi'
nests = [(0, 5), (10, 5)]
nest, target = nests
G.graph['nests'] = nests
G.graph['init path'] = []
'''
if not nx.has_path(G, nest, target):
print "no barabasi path"
return None
short_path = nx.shortest_path(G, nest, target)
if len(short_path) <= 3:
print "barabasi path too short"
return None
#print short_path
idx = choice(range(1, len(short_path) - 1))
#print idx
G.remove_edge(short_path[idx], short_path[idx + 1])
for i in xrange(idx):
G.graph['init path'].append((short_path[i], short_path[i + 1]))
for i in xrange(idx + 1, len(short_path) - 1):
G.graph['init path'].append((short_path[i], short_path[i + 1]))
#print P
if not nx.has_path(G, nest, target):
print "now no barabasi path"
return None
'''
for i in xrange(10):
if i != 4:
G.add_edge((i, 5), (i + 1, 5))
G.graph['init path'].append(((i, 5), (i + 1, 5)))
if G.has_edge((4, 5), (5, 5)):
G.remove_edge((4, 5), (5, 5))
#print nx.shortest_path_length(G, (4, 5), (5, 5))
if not nx.has_path(G, nest, target):
return None
elif nx.shortest_path_length(G, (4, 5), (5, 5)) > 8:
return None
return G
def edger(df, node, link, graphtype, attr_cols, node_sep, link_sep, file_path):
"""Main function. Called buy GUI, and can be used standalone by supplying a
pandas dataframe and relevant attributes."""
df = clean(df, node, link, graphtype, node_sep, link_sep)
if graphtype == 'normal':
indptr = np.fromiter(chain((0, ), map(len, df[link])), int,
len(df[link]) + 1).cumsum()
unq, idx = np.unique(np.concatenate(df[link]), return_inverse=True)
node_link_matrix = sparse.csr_matrix(
(np.ones(idx.size, int), idx, indptr), (len(df[link]), len(unq)))
node_node_matrix = (node_link_matrix @ node_link_matrix.T).tocoo()
G = nx.convert_matrix.from_scipy_sparse_matrix(node_node_matrix)
labels = df[node]
if graphtype == 'citation':
indptr = np.fromiter(chain((0, ), map(len, df[link])), int,
len(df[link]) + 1).cumsum()
unq = np.unique(df[node])
idx = np.where(np.concatenate(df[link])[:, None] == unq[None, :])[1]
node_node_matrix = sparse.csr_matrix(
(np.ones(np.concatenate(df[link]).size, int), idx, indptr),
(len(df[link]), len(unq))).rint()
G = nx.convert_matrix.from_scipy_sparse_matrix(node_node_matrix,
create_using=nx.DiGraph)
labels = df[node]
if graphtype == 'bipartite':
indptr1 = np.fromiter(chain((0, ), map(len, df[node])), int,
len(df[node]) + 1).cumsum()
unq, idx = np.unique(np.concatenate(df[node]), return_inverse=True)
node1_matrix = sparse.csr_matrix(
(np.ones(idx.size, int), idx, indptr1), (len(df[node]), len(unq)))
indptr2 = np.fromiter(chain((0, ), map(len, df[link])), int,
len(df[link]) + 1).cumsum()
unq, idx = np.unique(np.concatenate(df[link]), return_inverse=True)
node2_matrix = sparse.csr_matrix(
(np.ones(idx.size, int), idx, indptr2), (len(df[link]), len(unq)))
node1_node2_matrix = (node1_matrix.T @ node2_matrix).tocoo()
G = nx.algorithms.bipartite.matrix.from_biadjacency_matrix(
node1_node2_matrix)
labels = np.concatenate([
np.unique(np.concatenate(df[node])),
np.unique(np.concatenate(df[link]))
])
nx.relabel_nodes(G, dict(zip(G.nodes, labels)), copy=False)
if attr_cols:
if graphtype == 'bipartite':
attrdf = df.explode(node).explode(link)[[node] + [link] +
attr_cols]
node_attrs = attrdf.set_index(node).drop_duplicates().drop(link,
axis=1)
node_attrs = node_attrs.groupby(
node_attrs.index).agg(lambda x: '|'.join(x)).to_dict('index')
link_attrs = attrdf.set_index(link).drop_duplicates().drop(node,
axis=1)
link_attrs = link_attrs.groupby(
link_attrs.index).agg(lambda x: '|'.join(x)).to_dict('index')
nx.set_node_attributes(G, node_attrs)
nx.set_node_attributes(G, link_attrs)
else:
node_attr_dict = df.set_index(node)[attr_cols].to_dict('index')
nx.set_node_attributes(G, node_attr_dict)
outfile = f'{file_path[:-4]}.gexf'
nx.write_gexf(G, outfile)
return outfile
yield do_run, method, g
def do_run(method, g):
print method
cda = getattr(algs, method)
args = default_args.copy()
args.update(method_args.get(method, {}))
r = cda(g, **args)
assert_true(hasattr(r, 'cmtys'))
assert_true(hasattr(r, 'results'))
nclq = 10 # clique size
g1 = networkx.complete_graph(nclq)
g2 = networkx.relabel_nodes(g1, dict(zip(range(0,nclq),range(nclq,nclq*2))), copy=True)
g_2clique = networkx.union(g1, g2)
g_2clique.add_edge(0, nclq)
g_2clique_cmtys = pcd.cmty.Communities({0:set(range(0,nclq)), 1:set(range(nclq,nclq*2))})
@attr('slow')
def test_2clique(options={}):
"""Test that algorithms run and give reasonable results."""
for method in methods:
if method in methods_exclude:
continue
if method in methods_dontwork:
continue
g = g_2clique.copy()
def networkx_to_igraph(G): mapping = dict(zip(G.nodes(),range(G.number_of_nodes()))) reverse_mapping = dict(zip(range(G.number_of_nodes()),G.nodes())) G = nx.relabel_nodes(G,mapping) G_ig = ig.Graph(len(G), list(zip(*list(zip(*nx.to_edgelist(G)))[:2]))) return G_ig, reverse_mapping
def add_sent_id(t): # t: AMR text
""" add sentence ID to node labels ('x' -> 'i.x') """
for i in range(len(t)):
mapping = dict(
zip(t[i].nodes(), [str(i) + '.' + l for l in t[i].nodes()]))
t[i] = nx.relabel_nodes(t[i], mapping)
def main():
tic = time.perf_counter()
gn = Granatum()
clustersvsgenes = gn.pandas_from_assay(gn.get_import('clustersvsgenes'))
max_dist = gn.get_arg('max_dist')
min_zscore = gn.get_arg('min_zscore')
clustercomparisonstotest = list(clustersvsgenes.index)
G = nx.MultiDiGraph()
clusternames = list(clustersvsgenes.T.columns)
individualclusters = [
n[:n.index(" vs rest")] for n in clusternames if n.endswith("vs rest")
]
print(individualclusters, flush=True)
for cl in individualclusters:
G.add_node(cl)
# {pathway : {"cluster1":score1, "cluster2":score2}, pathway2 : {}}
# resultsmap = {}
relabels = {}
keys = {}
currentkeyindex = 0
maxexpression = np.max(np.max(clustersvsgenes))
print("Max expression = {}".format(maxexpression))
print("Number to analyze = {}".format(
len(clustersvsgenes.columns) * len(clustercomparisonstotest)),
flush=True)
gene_count = 0
for gene_id in clustersvsgenes.columns:
gene_count = gene_count + 1
print("Genecount = {}/{}".format(gene_count,
len(clustersvsgenes.columns)),
flush=True)
add_all_edges_for_current_gene = True
for cluster in clustercomparisonstotest:
score = clustersvsgenes.loc[cluster, gene_id]
if score >= min_zscore:
add_edges = True
if not gene_id in keys:
# First check if within distance of another group
closestkey = None
closestkeyvalue = 1.0e12
for key in keys:
gene_values = clustersvsgenes.loc[:, gene_id]
ref_values = clustersvsgenes.loc[:, key]
sc = np.sqrt(
np.nansum(np.square(gene_values - ref_values)) /
len(gene_values))
if sc <= max_dist and sc < closestkeyvalue:
closestkeyvalue = sc
closestkey = key
break
if closestkey == None:
keys[gene_id] = currentkeyindex + 1
else:
keys[gene_id] = keys[closestkey]
add_edges = False
add_all_edges_for_current_gene = False
print("Found a near gene: {}".format(closestkey),
flush=True)
else:
add_edges = add_all_edges_for_current_gene
# print("Score = {}".format(score), flush=True)
# olddict = resultsmap.get(gene_id, {})
# olddict[cluster] = score
# resultsmap[gene_id] = olddict
if add_edges:
from_to = re.split(' vs ', cluster)
if from_to[1] != 'rest':
G.add_weighted_edges_from(
[(from_to[1], from_to[0],
score / maxexpression * 1.0)],
label=str(keys[gene_id]),
penwidth=str(score / maxexpression * 1.0))
else:
relabel_dict = relabels.get(from_to[0], "")
if relabel_dict == "":
relabel_dict = from_to[0] + ": " + str(
keys[gene_id])
else:
relabel_dict = relabel_dict + ", " + str(
keys[gene_id])
relabels[from_to[0]] = relabel_dict
currentkeyindex = max(currentkeyindex, keys[gene_id])
print("Relabels {}".format(relabels), flush=True)
G = nx.relabel_nodes(G, relabels)
pos = nx.spring_layout(G)
edge_labels = nx.get_edge_attributes(G, 'label')
write_dot(G, 'plot.dot')
os.system('dot plot.dot -Kcirco -Tpng -Gsize="6,6" -Gdpi=600 > plot.png')
with open('plot.png', "rb") as f:
image_b64 = b64encode(f.read()).decode("utf-8")
gn.results.append({
"type": "png",
"width": 650,
"height": 480,
"description": 'Network of clusters based on expression',
"data": image_b64,
})
footnote = ""
inv_map = {}
for k, v in keys.items():
inv_map[v] = inv_map.get(v, []) + [k]
for k, v in sorted(inv_map.items(), key=lambda item: item[0]):
newv = map(lambda gene: "[{}]({})".format(gene, geturl(gene)), v)
vliststr = ", ".join(newv)
newstr = "{}: {} {}".format(
k, (clustersvsgenes.loc[clustersvsgenes[v[0]] > min_zscore,
v[0]]).to_dict(), vliststr)
if footnote == "":
footnote = newstr
else:
footnote = footnote + " \n" + newstr
gn.add_result(footnote, "markdown")
# gn.export(return_df.T.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True)
toc = time.perf_counter()
time_passed = round(toc - tic, 2)
timing = "* Finished differential expression sets step in {} seconds*".format(
time_passed)
gn.add_result(timing, "markdown")
gn.commit()
g = nx.complete_graph(4)
g.add_edge(0, 4)
g.add_edge(3, 5)
### First test our helper functions
if not __debug__:
print "You should run these simple tests without the '-O' flag that optimizes Python" \
" as we do assert statements to check correctness!"
assert path_exists(g, [4, 0, 2, 1, 3, 5])
assert path_exists(g, [2, 3])
assert not path_exists(g, [0, 5])
assert not path_exists(g, [100, 101])
### Now test disjoint path algorithms
print 'test disjoint path algorithms via manual visual inspection...'
# Need to relabel to strings since we assume nodes are strings
nx.relabel_nodes(g, {i: str(i) for i in g.nodes()}, copy=False)
target = '5'
paths = get_redundant_paths(g, '4', target, npaths)
print paths
multi_sources = ['0', '3', '4', '5']
target2 = '1'
paths2 = get_multi_source_disjoint_paths(g, multi_sources, target2)
print paths2
print 'drawing get_multi_source_disjoint_paths(g, %s, %s) output...' % (
multi_sources, target2)
draw_paths(g, paths2)
def weighted_directed_stochastic_block_model(N, relative_group_sizes,
connectivity_block_matrix,
weight_distribution_parameter_matrix,
degree_distribution_parameter_vector,
negative_weight_fraction_matrix=None,
weight_distribution="gamma",
degree_distribution="poisson",
correlated_inout_degree=True,
self_loops=False,
other_edge_block_attributes=[],
seed=None,
random_state=None):
"""
N - number of nodes in the graph
relative_group_sizes - sequence with magnitudes (will be normalized internally)
connectivity_block_matrix - matrix, elements are connection probabilities
(i,j element is from group-j to group-i)
degree_distribution_parameter_vecdegree_distribution="poisson"tor - degree distribution parameters
(tuple or seq) for each group
weight_distribution_parameter_matrix - weight distribution parameters
(tuple of seq) for each edge bundle (i,j)
weight_distribution - key for distribution
degree_distribution - key for distribution
negative_weight_fraction_matrix - vector of [0,1] values
specifying the fraction of sign swaps for each edge bundle (i,j)
Generate expected node degrees for all nodes based on degree distribution
parameters. The degree distribution is drawn from scipy's random
discrete distributions: poisson, uniform (randint), powerlaw (zipf).
The parameter's given are expected to match those required
by the numpy RNG for the chosen distribution.
Then uses a poisson distribution to draw the number of edges for the graph.
These edges are then connected according to the connectivity block matrix
using the degree corrected SBM (Karrer & Newman, 2010).
Finally weight distribution parameters are used to draw weights from the
desired class of distributions: general normal (gennorm), uniform,
pareto, log-normal (lognorm), or gamma.
kwargs: currently supports additional edge attributes
other_edge_block_attributes: default []
requires a list of dictionarys with
{'distribution_param_matrix', 'key',
'distribution', 'distribution_type'}
distribution_param_matrix - params for each i.j bundle
key - key to be assigned to the edge attribute
distribution - name of distribution
distribution_type - continuous/discrete
:param seed: None, else an integer
:param random_state: None, else a np.random.RandomState. If not set then
one is generated with a seed, else with the default seed of RandomState
"""
if random_state is None:
if seed is None:
random_state = np.random.RandomState()
else:
random_state = np.random.RandomState(seed)
num_groups = len(relative_group_sizes)
group_sizes = np.array(list(map(int, N * relative_group_sizes
/ np.sum(relative_group_sizes))))
N = np.sum(group_sizes)
expected_node_indegrees_by_group, expected_node_outdegrees_by_group = \
calculate_expected_degrees(group_sizes,
degree_distribution_parameter_vector,
degree_distribution,
correlated_inout_degree)
in_connection_prob_by_group = \
[ calculate_node_connection_probabilities(group_indegress)
for group_indegress in expected_node_indegrees_by_group ]
out_connection_prob_by_group = \
[ calculate_node_connection_probabilities(group_outdegrees)
for group_outdegrees in expected_node_outdegrees_by_group ]
node_membership_dictionary = { np.sum(group_sizes[:group]) + node :
group for group in range(num_groups)
for node in range(group_sizes[group]) }
community_to_nodelist_dictionary = { group : [ node for node in range(N)
if node_membership_dictionary[node] == group ]
for group in range(num_groups) }
graph = nx.DiGraph()
graph.add_nodes_from(node_membership_dictionary.keys())
nx.set_node_attributes(graph, 'community', node_membership_dictionary)
# Add connections
edge_bundles_by_group = {}
for source_group in range(num_groups):
for target_group in range(num_groups):
edge_bundles_by_group[(source_group, target_group)] = \
connect_edge_bundle(graph,
group_sizes[source_group], group_sizes[target_group],
community_to_nodelist_dictionary[source_group],
community_to_nodelist_dictionary[target_group],
connectivity_block_matrix[source_group, target_group],
out_connection_prob_by_group[source_group],
in_connection_prob_by_group[target_group],
random_state)
# Add weights
for bundle, edges in edge_bundles_by_group.items():
add_connection_weights(graph, edges,
weight_distribution_parameter_matrix[bundle[0], bundle[1]],
weight_distribution, negative_weight_fraction_matrix[bundle[0],
bundle[1]],
random_state)
# Add other attributes
for edge_block_attributes in other_edge_block_attributes:
add_edge_attributes(graph, edge_bundles_by_group,
edge_block_attributes,
random_state)
if not self_loops:
remove_self_loops(graph)
# Relabel all the nodes
mapping = { old_label: new_label
for old_label, new_label in enumerate(random_state.permutation(N)) }
graph = nx.relabel_nodes(graph, mapping, copy=True)
return graph
def get_lcc(di_graph):
di_graph = max(nx.weakly_connected_component_subgraphs(di_graph), key=len)
tdl_nodes = list(di_graph.nodes())
nodeListMap = dict(zip(tdl_nodes, range(len(tdl_nodes))))
nx.relabel_nodes(di_graph, nodeListMap, copy=False)
return di_graph, nodeListMap
def write_gml(G,T,outputfile="reference",partition=False,hwm=4000):
G=G.copy()
mapping={}
if 'paths' in G.graph:
totn=len(G.graph['paths'])
logging.debug("Graph contains %d samples"%totn)
else:
totn=0
for key in G.graph:
G.graph[key]=str(G.graph[key])
if type(G)==nx.MultiDiGraph:
for n1,n2,k,d in G.edges(keys=True,data=True):
for key in d:
v=d[key]
if not isinstance(v,str) and not isinstance(v,int):
G[n1][n2][k][key]=str(v)
else:
for n1,n2,d in G.edges(data=True):
for key in d:
v=d[key]
if not isinstance(v,str) and not isinstance(v,int):
G[n1][n2][key]=str(v)
for n,d in G.nodes(data=True):
mapping[n]=str(n)
d=G.node[n]
if 'offsets' in d:
G.node[n]['n']=len(d['offsets'])
for key in d:
v=d[key]
if type(v)!=str and type(v)!=int:
G.node[n][key]=str(v)
if 'seq' not in G.node[n]:
if isinstance(n,Interval):
G.node[n]['seq']=T[n.begin:n.end].upper()
else:
G.node[n]['seq']=""
G.node[n]['l']=len(G.node[n]['seq'])
# G.node[n]['seqstart']=G.node[n]['seq'][:20]
G.node[n]['seqend']=G.node[n]['seq'][-20:]
G=nx.relabel_nodes(G,mapping)
outputfiles=[]
if partition:
logging.debug("Trying to partion graph into subgraphs of size %d."%hwm)
i=0
# for sgi,subset in enumerate(nx.connected_components(G.to_undirected())):
for sgi,subset in enumerate(nx.weakly_connected_component_subgraphs(G)):
logging.debug("Partitioning connected component: %d"%sgi)
sgn=[]
g=G.subgraph(subset)
gn=G.graph['paths']
for n in nx.topological_sort(g):
sgn.append(n)
if G.node[n]['n']==totn: #join/split node
logging.debug("Can split graph at node: %s."%n)
if len(sgn)>=hwm:
logging.debug("Splitting graph at node: %s"%n)
sg=G.subgraph(sgn)
fn=outputfile+'.'+str(i)+'.gml'
nx.write_gml(sg,fn)
outputfiles.append(fn)
sgn=[n]
i+=1
if len(sgn)>1:
sg=G.subgraph(sgn)
fn=outputfile+'.'+str(i)+'.gml'
nx.write_gml(sg,fn)
i+=1
outputfiles.append(fn)
else:
if not outputfile.endswith(".gml"):
outputfile=outputfile+'.gml'
nx.write_gml(G,outputfile)
outputfiles.append(outputfile)
return outputfiles
print(dominating_set)
print(len(dominating_set))
return
def test_rand_gnm(n,m):
g = rand_gnm_undirected(n,m)
print(nx.info(g))
return g
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--file', required=False, type=str, help="File that contains edgelist of graph to be loaded.")
args = parser.parse_args()
g = nx.relabel_nodes(nx.read_edgelist('./graphs/'+args.file+'.txt', create_using=nx.DiGraph), int)
# print("="*35)
# print("Loaded graph")
# print(nx.info(g))
print("="*35)
print("Driver nodes")
test_maximum_matching(g)
print("="*35)
print("All possible driver nodes")
# test_all_maximum_matching(g)
# print("="*35)
# print("Random ER graph generation")
# test_randER(n = 100, p = 0.2, seed = 5)
# print("="*35)
# print("Fast random ER graph generation")
# test_randER_fast(n = 688, p = 0.002282844, seed = 5)
def edge_current_flow_betweenness_centrality_subset(G,
sources,
targets,
normalized=True,
weight=None,
dtype=float,
solver="lu"):
r"""Compute current-flow betweenness centrality for edges using subsets
of nodes.
Current-flow betweenness centrality uses an electrical current
model for information spreading in contrast to betweenness
centrality which uses shortest paths.
Current-flow betweenness centrality is also known as
random-walk betweenness centrality [2]_.
Parameters
----------
G : graph
A NetworkX graph
sources: list of nodes
Nodes to use as sources for current
targets: list of nodes
Nodes to use as sinks for current
normalized : bool, optional (default=True)
If True the betweenness values are normalized by b=b/(n-1)(n-2) where
n is the number of nodes in G.
weight : string or None, optional (default=None)
Key for edge data used as the edge weight.
If None, then use 1 as each edge weight.
dtype: data type (float)
Default data type for internal matrices.
Set to np.float32 for lower memory consumption.
solver: string (default='lu')
Type of linear solver to use for computing the flow matrix.
Options are "full" (uses most memory), "lu" (recommended), and
"cg" (uses least memory).
Returns
-------
nodes : dict
Dictionary of edge tuples with betweenness centrality as the value.
See Also
--------
betweenness_centrality
edge_betweenness_centrality
current_flow_betweenness_centrality
Notes
-----
Current-flow betweenness can be computed in $O(I(n-1)+mn \log n)$
time [1]_, where $I(n-1)$ is the time needed to compute the
inverse Laplacian. For a full matrix this is $O(n^3)$ but using
sparse methods you can achieve $O(nm{\sqrt k})$ where $k$ is the
Laplacian matrix condition number.
The space required is $O(nw)$ where $w$ is the width of the sparse
Laplacian matrix. Worse case is $w=n$ for $O(n^2)$.
If the edges have a 'weight' attribute they will be used as
weights in this algorithm. Unspecified weights are set to 1.
References
----------
.. [1] Centrality Measures Based on Current Flow.
Ulrik Brandes and Daniel Fleischer,
Proc. 22nd Symp. Theoretical Aspects of Computer Science (STACS '05).
LNCS 3404, pp. 533-544. Springer-Verlag, 2005.
http://algo.uni-konstanz.de/publications/bf-cmbcf-05.pdf
.. [2] A measure of betweenness centrality based on random walks,
M. E. J. Newman, Social Networks 27, 39-54 (2005).
"""
import numpy as np
if not nx.is_connected(G):
raise nx.NetworkXError("Graph not connected.")
n = G.number_of_nodes()
ordering = list(reverse_cuthill_mckee_ordering(G))
# make a copy with integer labels according to rcm ordering
# this could be done without a copy if we really wanted to
mapping = dict(zip(ordering, range(n)))
H = nx.relabel_nodes(G, mapping)
edges = (tuple(sorted((u, v))) for u, v in H.edges())
betweenness = dict.fromkeys(edges, 0.0)
if normalized:
nb = (n - 1.0) * (n - 2.0) # normalization factor
else:
nb = 2.0
for row, (e) in flow_matrix_row(H,
weight=weight,
dtype=dtype,
solver=solver):
for ss in sources:
i = mapping[ss]
for tt in targets:
j = mapping[tt]
betweenness[e] += 0.5 * np.abs(row[i] - row[j])
betweenness[e] /= nb
return {(ordering[s], ordering[t]): v for (s, t), v in betweenness.items()}
if pointsScored[i, k] == 0:
df.loc[ind, 'Points'] = np.NaN
else:
df.loc[ind,
'Points'] = (df.loc[ind, 'Points']) / (pointsScored[i,
k])
dfAverages = df.groupby(['From country', 'To country']).Points.mean()
dfAverages = dfAverages.unstack().fillna(0)
#Find louvian groups
np_matrix = dfAverages.values
names = dfAverages.columns
print(names)
G = nx.from_numpy_matrix(np_matrix)
G = nx.relabel_nodes(G, dict(enumerate(names)))
partition = community_louvain.best_partition(G)
#Asign groups on pandaframe
partition = collections.OrderedDict(sorted(partition.items()))
dfAverages['group'] = partition.values()
dfAverages = dfAverages.sort_values(by='group')
#Calculate stats of clusters
clusters = dfAverages['group'].unique()
for c in clusters:
names = dfAverages[dfAverages['group'] == c].index
temp = dfAverages[names]
m = np.zeros((len(names), len(names)))
for i, n in enumerate(names):
for i2, n2 in enumerate(names):
def sample_graph(G,
output_dir,
s_n,
times=10,
with_test=False,
radio=0.8,
feature_path=None):
if s_n is None:
s_n = int(np.sqrt(G.number_of_nodes()))
for t in range(times):
t_dir = os.path.join(output_dir, 's{}'.format(t))
n = random.randint(int(s_n / 2), 2 * s_n)
Gs = utils.random_walk_induced_graph_sampling(G, n)
mapping = dict(zip(Gs.nodes(), range(Gs.number_of_nodes())))
if feature_path is not None:
feats = sparse.load_npz(feature_path)
row = []
col = []
data = []
fr, fc = feats.nonzero()
for i, j in zip(fr, fc):
if i in mapping:
row.append(mapping[i])
col.append(j)
data.append(feats[i, j])
feats = sparse.csr_matrix((data, (row, col)),
shape=(len(mapping), feats.shape[1]))
Gs = nx.relabel_nodes(Gs, mapping)
file_path = os.path.join(t_dir, 'graph.edgelist')
file_test_path = os.path.join(t_dir, 'graph_test.edgelist')
label_path = os.path.join(t_dir, 'label.txt')
feature_save_path = os.path.join(t_dir, 'features.npz')
if feature_path is not None:
utils.write_with_create(feature_save_path)
sparse.save_npz(feature_save_path, feats)
if not with_test:
print("sample graph, nodes: {}, edges: {}, save into {}".format(
Gs.number_of_nodes(), Gs.number_of_edges(), t_dir))
with utils.write_with_create(file_path) as f:
for i, j in Gs.edges():
print(i, j, file=f)
with utils.write_with_create(label_path) as f:
for i, data in Gs.nodes(data=True):
if 'label' in data:
for j in data['label']:
print(i, j, file=f)
else:
G_train = nx.Graph()
G_test = nx.Graph()
edges = np.random.permutation(list(Gs.edges()))
nodes = set()
for a, b in edges:
if a not in nodes or b not in nodes:
G_train.add_edge(a, b)
nodes.add(a)
nodes.add(b)
else:
G_test.add_edge(a, b)
assert len(nodes) == Gs.number_of_nodes()
assert len(nodes) == G_train.number_of_nodes()
num_test_edges = int((1 - radio) * Gs.number_of_edges())
now_number = G_test.number_of_edges()
if num_test_edges < now_number:
test_edges = list(G_test.edges())
G_train.add_edges_from(test_edges[:now_number -
num_test_edges])
G_test.remove_edges_from(test_edges[:now_number -
num_test_edges])
print(
"sample graph,origin: {} {}, train: {} {}, test: {} {}".format(
Gs.number_of_nodes(), Gs.number_of_edges(),
G_train.number_of_nodes(), G_train.number_of_edges(),
G_test.number_of_nodes(), G_test.number_of_edges()))
with utils.write_with_create(file_path) as f:
for i, j in G_train.edges():
print(i, j, file=f)
with utils.write_with_create(file_test_path) as f:
for i, j in G_test.edges():
print(i, j, file=f)
for v in pvals_significant:
i = v[0]
j = v[1]
Gw_bh[i,j] = Gw[i,j]
print("Results:", len(pvals_significant),"discoveries at a false-discovery-rate of", false_discovery_rate)
if kmax == -1:
print("No significant results")
return [Gw, Gw_bh, pvals_significant]
def matrix_to_gml_file(Gw, L, name):
'''
Takes and adjacency matrix Gw with a list of labels (one label per row) and saves the corresponding labeled graph as a gml file with name "name".
'''
Gw = np.array(Gw)
n = len(Gw)
G = nx.from_numpy_matrix(Gw, create_using=nx.MultiDiGraph())
ll = {}
for k in list(range(n)):
ll[k] = str(k) +": " + L[k]
G = nx.relabel_nodes(G, ll)
nx.write_gml(G, name + ".gml")
def get_feature_order(X, eps=.25):
A = squareform(pdist(X.T, metric=lambda x, y: pearsonr(x, y)[0]))
A[A < eps] = 0
G = nx.relabel_nodes(nx.from_numpy_array(A), {i:s for i,s in enumerate(X.columns)})
return nx.spectral_ordering(G)
def draw_alert_entity_graph(
nx_graph: nx.Graph,
font_size: int = 12,
height: int = 15,
width: int = 15,
margin: float = 0.3,
scale: int = 1,
):
"""
Draw networkX graph with matplotlib.
Parameters
----------
nx_graph : nx.Graph
The NetworkX graph to draw
font_size : int, optional
base font size (the default is 12)
height : int, optional
Image height (the default is 15)
width : int, optional
Image width (the default is 15)
margin : float, optional
Image margin (the default is 0.3)
scale : int, optional
Position scale (the default is 1)
"""
alert_node = [
n for (n, node_type
) in nx.get_node_attributes(nx_graph, "node_type").items()
if node_type == "alert"
]
entity_nodes = [
n for (n, node_type
) in nx.get_node_attributes(nx_graph, "node_type").items()
if node_type == "entity"
]
# now draw them in subsets using the `nodelist` arg
plt.rcParams["figure.figsize"] = (width, height)
plt.margins(x=margin, y=margin)
pos = nx.kamada_kawai_layout(nx_graph, scale=scale, weight="weight")
nx.draw_networkx_nodes(nx_graph,
pos,
nodelist=alert_node,
node_color="red",
alpha=0.5,
node_shape="o")
nx.draw_networkx_nodes(
nx_graph,
pos,
nodelist=entity_nodes,
node_color="green",
alpha=0.5,
node_shape="s",
s=200,
)
nlabels = nx.get_node_attributes(nx_graph, "description")
nx.relabel_nodes(nx_graph, nlabels)
nx.draw_networkx_labels(nx_graph, pos, nlabels, font_size=font_size)
nx.draw_networkx_edges(nx_graph, pos)
elabels = nx.get_edge_attributes(nx_graph, "description")
nx.draw_networkx_edge_labels(nx_graph,
pos,
edge_labels=elabels,
font_size=font_size * 2 / 3,
alpha=0.6)
def maf2graph(maffile):
files = set()
G = nx.MultiDiGraph()
startnode = uuid.uuid4().hex
endnode = uuid.uuid4().hex
G.graph['startnodes'] = set([startnode])
G.graph['endnodes'] = set([endnode])
G.graph['path2id'] = dict()
G.add_node(startnode, offsets=dict())
G.add_node(endnode, offsets=dict())
nid = 0
with open(maffile, "r") as maf:
for line in maf:
if line.startswith("#"):
continue
elif line.startswith("a"): #start of an aligned segment
nid += 1
G.add_node(nid, data=dict())
elif line.startswith("s"):
cols = line.rstrip().split()
if '.' in cols[
1]: #TODO: use db parameter to specificy a single mfa file with all sequence
file, name = cols[1][:cols[1].find('.')], cols[1][
cols[1].find('.') + 1:]
files.add(file)
else:
file = None #args.db?
name = cols[1]
if name not in G.graph['path2id']:
G.graph['path2id'][name] = len(G.graph['path2id'])
G.node[startnode]['offsets'][G.graph['path2id'][name]] = 0
G.node[nid]['data'][(file, name)] = {
'start': int(cols[2]),
'end': int(cols[2]) + int(cols[3]),
'orientation': cols[4],
'aln': cols[6]
}
nid += 1
remove = []
for node, d in G.nodes(data=True):
if 'data' in d and len(
d['data']) == 1: #multiplicity of 1, strictly not an alignment
remove.append(node)
G.remove_nodes_from(remove)
db = dict() #map name to sequence
for file in files:
for name, seq in utils.fasta_reader(
file + ".fasta"
): #guess that the original file has a ".fasta" extension
name = name.split()[0]
key = (file, name)
if key in db:
logging.fatal("Non unique contig-name: %s. quit." % name)
sys.exit(1)
else:
db[key] = seq
remove = []
#for every sequence, check that none of the alignments overlap, otherwise assignment is not 1-1
for file, name in db:
seq = db[(file, name)]
intvs = []
for node in G:
if 'data' in G.node[
node]: #does the node represent an aligned segment?
if (file, name) in G.node[node]['data']:
intvs.append(
(G.node[node]['data'][(file, name)]['start'],
G.node[node]['data'][(file, name)]['end'], node))
intvs.sort() #sort by start position
pstart = 0
pend = 0
pnode = startnode
unaligned = []
for start, end, node in intvs:
if start > pend:
unaligned.append((pend, start))
G.add_node(nid, intv=(pend, start), seq=seq[pend:start])
G.add_edge(pnode,
nid,
paths=set([G.graph['path2id'][name]]),
ofrom="+",
oto="+")
G.add_edge(nid,
node,
paths=set([G.graph['path2id'][name]]),
ofrom="+",
oto="+")
nid += 1
elif start < pend:
logging.fatal(
"Overlapping alignments for sequence: %s.%s --> (%d,%d) and (%d,%d)."
% (file, name, pstart, pend, start, end))
remove.append(node)
# sys.exit(1)
else: #no gap, just connect subsequent intervals
G.add_edge(pnode,
node,
paths=set([G.graph['path2id'][name]]),
ofrom="+",
oto="+")
pstart, pend, pnode = start, end, node
if len(seq) != pend:
unaligned.append((pend, len(seq)))
G.add_node(nid, intv=((pend, len(seq))), seq=seq[pend:len(seq)])
G.add_edge(pnode,
nid,
paths=set([G.graph['path2id'][name]]),
ofrom="+",
oto="+")
G.add_edge(nid,
endnode,
paths=set([G.graph['path2id'][name]]),
ofrom="+",
oto="+")
nid += 1
else:
G.add_edge(pnode,
endnode,
paths=set([G.graph['path2id'][name]]),
ofrom="+",
oto="+")
G.remove_nodes_from(remove)
# print "Unaligned segments",unaligned
alignments = [node for node in G if 'data' in G.node[node]]
for node in alignments: #expand all alignments in the graph
if 'data' in G.node[node]:
seqs = []
names = []
offsets = {}
for file, name in G.node[node]['data']:
seqs.append(G.node[node]['data'][(file, name)]['aln'])
offsets[G.graph['path2id'][name]] = G.node[node]['data'][(
file, name)]['start']
names.append(name)
sg, nid = utils.aln2graph(seqs,
names,
idoffset=nid,
path2id=G.graph['path2id'],
offsets=offsets)
nid += 1
G.add_nodes_from(sg.nodes(data=True))
G.add_edges_from(sg.edges(data=True))
assert (len(sg.graph['startnodes']) == 1)
assert (len(sg.graph['endnodes']) == 1)
sgstart = sg.graph['startnodes'][0]
sgend = sg.graph['endnodes'][0]
for v, t, d in G.in_edges(node, data=True):
G.add_edge(v, sgstart, paths=d['paths'], ofrom="+", oto="+")
for v, t, d in G.out_edges(node, data=True):
G.add_edge(sgend, t, paths=d['paths'], ofrom="+", oto="+")
#hack this in here so we can continue
G.node[sgstart]['seq'] = ""
G.node[sgend]['seq'] = ""
nx.relabel_nodes(G, {sgstart: nid, sgend: nid + 1}, copy=False)
nid += 2
G.remove_node(node)
return G
n_G, # The number of nodes
k_G, # Each node is connected to k nearest neighbors in ring topology
0.3, # The probability of rewiring each edge
seed=None) # Seed for random number generator (default=None)
H = nx.watts_strogatz_graph(
n_H, # The number of nodes
k_H, # Each node is connected to k nearest neighbors in ring topology
0.3, # The probability of rewiring each edge
seed=None) # Seed for random number generator (default=None)
def mapping(x):
return x + n_G
H = nx.relabel_nodes(H, mapping)
G = nx.compose(G, H)
pos = nx.spring_layout(G)
nx.set_node_attributes(G, 'NA', 'TimeClass')
nx.set_node_attributes(G, 'NA', 'Color')
lab = ["a", "b", "c", "d", "e", "f", "g", "h"]
color = ['1', '2', '3', '4', '5', '6', '7', '8']
for i in G.nodes:
for node in G.neighbors(i):
if G.nodes[node]['TimeClass'] == 'NA':
G.nodes[node]['TimeClass'] = lab[floor(i / 133)]
G.nodes[node]['Color'] = color[floor(i / 133)]
pos = nx.spring_layout(G)
def terminate(G, count):
flag1 = terminate_1('A', G)
flag2 = terminate_1('B', G)
if flag1 == 1 or flag2 == 1 or count >= 100:
return 1
else:
return 0
G = nx.read_gml('random_graph.gml')
# We need to convert the node's labels to integer since they are in string format
pass_dict = dict()
for i in range(10):
pass_dict[str(i)] = i
nx.relabel_nodes(G, pass_dict, copy=False) # Function to relablel nodes
G = set_all_B(G)
list1 = [3, 7]
G = set_A(G, list1)
colors = get_colors(G)
nx.draw(G, node_color=colors, node_size=800, with_labels=True)
plt.show()
flag = 0
count = 0
while (1):
flag = terminate(G, count)
if flag == 1:
break
count += 1
action_dict = recalculate_options(G)
def iso(self, mapping):
return nx.relabel_nodes(self, mapping)
def ControlFlowGraphFromDotSource(
dot_source: str,
remove_blocks_without_predecessors: bool = True,
tag_hook: typing.Optional[TagHook] = None,
) -> LlvmControlFlowGraph:
"""Create a control flow graph from an LLVM-generated dot file.
The control flow graph generated from the dot source is not guaranteed to
be valid. That is, it may contain fusible basic blocks. This can happen if
the creating the graph from unoptimized bytecode. To disable this generate
the bytecode with optimizations enabled, e.g. clang -emit-llvm -O3 -S ...
Args:
dot_source: The dot source generated by the LLVM -dot-cfg pass.
remove_blocks_without_predecessors: If true, remove CFG blocks without
predecessors (except for the entry block). This is similar to the
-simplify-cfg opt pass.
tag_hook: An optional object that can tag specific statements in the CFG
according to some logic.
Returns:
A ControlFlowGraph instance.
Raises:
pyparsing.ParseException: If dotfile could not be parsed.
ValueError: If dotfile could not be interpretted / is malformed.
"""
try:
parsed_dots = pydot.graph_from_dot_data(dot_source)
except TypeError as e:
raise pyparsing.ParseException("Failed to parse dot source") from e
if len(parsed_dots) != 1:
raise ValueError(f"Expected 1 Dot in source, found {len(parsed_dots)}")
dot = parsed_dots[0]
if tag_hook:
tag_hook.OnGraphBegin(dot)
function_name_match = re.match(_DOT_CFG_FUNCTION_NAME_RE, dot.get_name())
if not function_name_match:
raise ValueError(f"Could not interpret graph name '{dot.get_name()}'")
# Create the ControlFlowGraph instance.
graph = LlvmControlFlowGraph(
name=function_name_match.group(1), tag_hook=tag_hook
)
# Opt names nodes like 'Node0x7f86c670c590'. We discard those names and assign
# nodes simple integer names.
# Create the nodes and build a map from node names to indices.
node_name_to_index_map = {}
for i, node in enumerate(dot.get_nodes()):
if node.get_name() in node_name_to_index_map:
raise ValueError(f"Duplicate node name: '{node.get_name()}'")
node_name_to_index_map[node.get_name()] = i
if tag_hook:
other_attrs = tag_hook.OnNode(node) or {}
else:
other_attrs = {}
graph.add_node(
i, **NodeAttributesToBasicBlock(node.get_attributes()), **other_attrs
)
# Create edges and encode their position. The position is an integer starting
# at zero and increasing for each outgoing edge, e.g. a switch with `n` cases
# will have 0..(n-1) unique positions.
for edge in dot.get_edges():
# In the dot file, an edge looks like this:
# Node0x7f86c670c590:s0 -> Node0x7f86c65001a0;
src_components = edge.get_source().split(":")
if len(src_components) > 2:
raise ValueError(
f"Cannot interpret edge source name `{edge.get_source()}`"
)
elif len(src_components) == 2:
# Case: Node0x7f87aaf14520:s0
src_name, position_name = src_components
position = int(position_name[1:])
else:
# Case: Node0x7f87aaf14520
src_name, position = src_components[0], 0
src = node_name_to_index_map[src_name]
dst = node_name_to_index_map[edge.get_destination()]
graph.add_edge(src, dst, position=position)
# Optionally remove blocks without predecessors (except the entry block).
# This emulates the behaviour of the -simplify-cfg opt pass.
if remove_blocks_without_predecessors:
removed_blocks_count = 0
changed = True
while changed:
changed = False
nodes_to_remove = []
for i, node in enumerate(graph.nodes()):
if i and graph.in_degree(node) == 0:
nodes_to_remove.append(node)
removed_blocks_count += len(nodes_to_remove)
for node in nodes_to_remove:
graph.remove_node(node)
changed = True
if removed_blocks_count:
app.Log(
1,
"Removed %s without predecessors from `%s`, " "%d blocks remaining",
humanize.Plural(removed_blocks_count, "block"),
graph.name,
graph.number_of_nodes(),
)
# Rename nodes to retain a contiguous numeric sequence.
node_rename_map = {
oldname: newname for newname, oldname in enumerate(graph.nodes())
}
nx.relabel_nodes(graph, node_rename_map, copy=False)
# The first block is always the function entry.
graph.nodes[0]["entry"] = True
# Mark the exit node.
exit_count = 0
for node, data in graph.nodes(data=True):
if graph.out_degree(node) == 0:
exit_count += 1
data["exit"] = True
# CFGs may have multiple exit blocks, e.g. unreachables, exceptions, etc.
# However, they should have at least one block.
if exit_count < 1:
raise ValueError(f"Function `{graph.name}` has no exit blocks")
return graph
def _init_graph(self, basetype):
if basetype[0:7] == 'grid2d:':
side = int(basetype[7:])
print 'Loading network graph from basetype grid2d with side', side
self._graph = networkx.generators.classic.grid_2d_graph(side, side)
elif basetype[0:9] == 'clusters:':
cluster_size = int(basetype[9:])
print 'Loading network graph from basetype clusters with size', cluster_size
#
# cluster 1
# / \
# cluster 2 --- cluster3
#
conn_radius = 0.3
# cluster1 = networkx.random_geometric_graph(cluster_size, conn_radius)
cluster1 = networkx.complete_graph(cluster_size)
# pos = networkx.spring_layout(cluster1, iterations=1000)
# pos = networkx.spectral_layout(cluster1)
pos = circle_pos(cluster_size)
networkx.set_node_attributes(cluster1, 'pos', pos)
# cluster2 = networkx.random_geometric_graph(cluster_size, conn_radius)
cluster2 = networkx.complete_graph(cluster_size)
# pos = networkx.spring_layout(cluster2, iterations=1000)
# pos = networkx.spectral_layout(cluster2)
pos = circle_pos(cluster_size)
networkx.set_node_attributes(cluster2, 'pos', pos)
relabel_map = dict(
zip(range(0, cluster_size),
range(cluster_size, cluster_size * 2)))
cluster2 = networkx.relabel_nodes(cluster2, relabel_map)
# cluster3 = networkx.random_geometric_graph(cluster_size, conn_radius)
cluster3 = networkx.complete_graph(cluster_size)
# pos = networkx.spring_layout(cluster3, iterations=1000)
# pos = networkx.spectral_layout(cluster3)
pos = circle_pos(cluster_size)
networkx.set_node_attributes(cluster3, 'pos', pos)
relabel_map = dict(
zip(range(0, cluster_size),
range(cluster_size * 2, cluster_size * 3)))
cluster3 = networkx.relabel_nodes(cluster3, relabel_map)
hubnode1to2 = find_nearest_node_to_point(cluster1, 0.1, 0.)
hubnode2to1 = find_nearest_node_to_point(cluster2, 0.9, 1.)
hubnode2to3 = find_nearest_node_to_point(cluster2, 1., 0.5)
hubnode3to2 = find_nearest_node_to_point(cluster3, 0., 0.5)
hubnode3to1 = find_nearest_node_to_point(cluster3, 0.35, 1)
hubnode1to3 = find_nearest_node_to_point(cluster1, 0.9, 0.)
shift_graph_position(cluster1, 0.6, 1.)
shift_graph_position(cluster3, 1.2, 0)
self._hubnodes = [
hubnode1to2, hubnode2to1, hubnode2to3, hubnode3to2,
hubnode3to1, hubnode1to3
]
cluster_links = [(hubnode1to2, hubnode2to1),
(hubnode2to3, hubnode3to2),
(hubnode3to1, hubnode1to3)]
# cluster1.add_nodes_from(cluster2)
# cluster1.add_nodes_from(cluster3)
# cluster1.add_edges_from(cluster2.edges())
# cluster1.add_edges_from(cluster3.edges())
# cluster1.add_edges_from(cluster_links)
cluster1 = networkx.union(cluster1, cluster2)
cluster1 = networkx.union(cluster1, cluster3)
cluster1.add_edges_from(cluster_links)
self._graph = cluster1
print "hubnodes : ", self._hubnodes
print "cluster_links : ", cluster_links
print "graph diameter : ", networkx.diameter(self._graph)
if self._graph is None:
return
# init agents
for n in self._graph.nodes():
self._agents[n] = Agent(n, self._M, self._D, self._N, self._barN,
self._alpha_0, self._sigma,
self._track_exact_size)
G.add_node(link,bipartite = 1)
G.add_edge(user,link)
user_nodes = [n for n,d in G.nodes(data=True) if d['bipartite'] == 0]
A = nx.algorithms.bipartite.basic.biadjacency_matrix(G,user_nodes,dtype=np.float)
A = sparse.csr_matrix(A)
S = np.array( [1./r.sum() for r in A]);
D = sparse.csr_matrix(np.diag(S))
Norm_A = D.dot(A)
U = (Norm_A.dot(Norm_A.T)).todense()
#np.fill_diagonal(U,0.0) I think OSLOM ignores self-loops
UU = nx.from_numpy_matrix(U)
mapping = dict(enumerate(user_nodes))
UU = nx.relabel_nodes(UU,mapping,copy=False)
# OSLOM
# export the graph in a temporary .dat file
oslom_filename = './temp/{}.dat'.format(temp_name)
node_index = UU.nodes()
with open(oslom_filename,'w') as f :
for x,y,d in UU.edges(data=True) :
xi = node_index.index(x)
yi = node_index.index(y)
f.write('{}\t{}\t{}\n'.format(str(xi),str(yi),str(d['weight'])))
with open(os.devnull, "w") as fnull:
call(['OSLOM2/oslom_undir','-cp',str(resolution),'-w','-f',oslom_filename],stdout=fnull)
def main(simulated_time):
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)
"""
TOPOLOGY from a json
"""
t = Topology()
t.G = nx.read_graphml("Euclidean.graphml")
ls = list(t.G.nodes)
li = {x: int(x) for x in ls}
nx.relabel_nodes(t.G, li, False) #Transform str-labels to int-labels
print "Nodes: %i" % len(t.G.nodes())
print "Edges: %i" % len(t.G.edges())
#MANDATORY fields of a link
# Default values = {"BW": 1, "PR": 1}
valuesOne = dict(itertools.izip(t.G.edges(), np.ones(len(t.G.edges()))))
nx.set_edge_attributes(t.G, name='BW', values=valuesOne)
nx.set_edge_attributes(t.G, name='PR', values=valuesOne)
centrality = nx.betweenness_centrality(t.G)
nx.set_node_attributes(t.G, name="centrality", values=centrality)
sorted_clustMeasure = sorted(centrality.items(),
key=operator.itemgetter(1),
reverse=True)
top20_devices = sorted_clustMeasure[0:20]
main_fog_device = copy.copy(top20_devices[0][0])
# df = pd.read_csv("pos_network.csv")
# pos = {}
# for r in df.iterrows():
# lat = r[1].x
# lng = r[1].y
# pos[r[0]] = (lat, lng)
# fig = plt.figure(figsize=(10, 8), dpi=100)
# nx.draw(t.G, with_labels=True,pos=pos,node_size=60,node_color="orange", font_size=8)
# plt.savefig('labels.png')
# exit()
print "-" * 20
print "Best top centralized device: ", main_fog_device
print "-" * 20
"""
APPLICATION
"""
app1 = create_application("app1")
"""
PLACEMENT algorithm
"""
#There are not modules to place.
placement = NoPlacementOfModules("NoPlacement")
"""
POPULATION algorithm
"""
number_generators = int(len(t.G) * 0.1)
print "Number of generators %i" % number_generators
#you can use whatever funciton to change the topology
dStart = deterministicDistributionStartPoint(500,
400,
name="Deterministic")
pop = Population_Move(name="mttf-nodes",
srcs=number_generators,
node_dst=main_fog_device,
activation_dist=dStart)
pop.set_sink_control({
"id": main_fog_device,
"number": number_generators,
"module": app1.get_sink_modules()
})
dDistribution = deterministicDistribution(name="Deterministic", time=100)
pop.set_src_control({
"number": 1,
"message": app1.get_message("M.Action"),
"distribution": dDistribution
})
#In addition, a source includes a distribution function:
"""--
SELECTOR algorithm
"""
selectorPath = CloudPath_RR()
"""
SIMULATION ENGINE
"""
s = Sim(t, default_results_path="Results_%s" % (simulated_time))
s.deploy_app(app1, placement, pop, selectorPath)
s.run(simulated_time,
test_initial_deploy=False,
show_progress_monitor=False)
# s.draw_allocated_topology() # for debugging
s.print_debug_assignaments()
name: str = opinion.cluster.case_name
name_arr = name.split("v. ")
name_arr[0] = (" ".join(name_arr[0].split(" ")[:3])
if len(name_arr[0]) > 25 else name_arr[0])
name_arr[1] = (" ".join(name_arr[1].split(" ")[:3])
if len(name_arr[1]) > 25 else name_arr[1])
name_arr.insert(1, " v. \n")
name_map[opinion.resource_id] = "".join(name_arr)
output_str += f"{i + 1}: {opinion.resource_id}, {opinion.cluster.case_name}\n"
except:
name_map[opinion_id] = "Unknown"
pass
print(output_str)
top_opinion_names = [name_map[op] for op in top_opinions]
subgraph = nx.subgraph(roe_ego, top_opinions)
subgraph = nx.relabel_nodes(subgraph, name_map)
graph_pos = nx.shell_layout(
subgraph,
[
top_opinion_names[:5],
top_opinion_names[5:10],
top_opinion_names[10:15],
top_opinion_names[15:],
],
)
nx.draw_networkx_nodes(subgraph, graph_pos, node_size=100)
nx.draw_networkx_edges(subgraph, graph_pos, width=0.6, edge_color="#585858")
nx.draw_networkx_labels(
subgraph,
graph_pos,
font_size=16,
def parse_gml_lines(lines, label, destringizer):
"""Parse GML `lines` into a graph.
"""
def tokenize():
patterns = [
r"[A-Za-z][0-9A-Za-z_]*\b", # keys
# reals
r"[+-]?(?:[0-9]*\.[0-9]+|[0-9]+\.[0-9]*)(?:[Ee][+-]?[0-9]+)?",
r"[+-]?[0-9]+", # ints
r'".*?"', # strings
r"\[", # dict start
r"\]", # dict end
r"#.*$|\s+", # comments and whitespaces
]
tokens = re.compile("|".join("(" + pattern + ")"
for pattern in patterns))
lineno = 0
for line in lines:
length = len(line)
pos = 0
while pos < length:
match = tokens.match(line, pos)
if match is not None:
for i in range(len(patterns)):
group = match.group(i + 1)
if group is not None:
if i == 0: # keys
value = group.rstrip()
elif i == 1: # reals
value = float(group)
elif i == 2: # ints
value = int(group)
else:
value = group
if i != 6: # comments and whitespaces
yield (i, value, lineno + 1, pos + 1)
pos += len(group)
break
else:
raise NetworkXError("cannot tokenize %r at (%d, %d)" %
(line[pos:], lineno + 1, pos + 1))
lineno += 1
yield (None, None, lineno + 1, 1) # EOF
def unexpected(curr_token, expected):
category, value, lineno, pos = curr_token
raise NetworkXError(
"expected %s, found %s at (%d, %d)" %
(expected, repr(value) if value is not None else "EOF", lineno,
pos))
def consume(curr_token, category, expected):
if curr_token[0] == category:
return next(tokens)
unexpected(curr_token, expected)
def parse_kv(curr_token):
dct = defaultdict(list)
while curr_token[0] == 0: # keys
key = curr_token[1]
curr_token = next(tokens)
category = curr_token[0]
if category == 1 or category == 2: # reals or ints
value = curr_token[1]
curr_token = next(tokens)
elif category == 3: # strings
value = unescape(curr_token[1][1:-1])
if destringizer:
try:
value = destringizer(value)
except ValueError:
pass
curr_token = next(tokens)
elif category == 4: # dict start
curr_token, value = parse_dict(curr_token)
else:
# Allow for string convertible id and label values
if key in ("id", "label", "source", "target"):
try:
# String convert the token value
value = unescape(str(curr_token[1]))
if destringizer:
try:
value = destringizer(value)
except ValueError:
pass
curr_token = next(tokens)
except Exception:
msg = ("an int, float, string, '[' or string" +
" convertable ASCII value for node id or label")
unexpected(curr_token, msg)
else: # Otherwise error out
unexpected(curr_token, "an int, float, string or '['")
dct[key].append(value)
dct = {
key: (value if not isinstance(value, list) or len(value) != 1 else
value[0])
for key, value in dct.items()
}
return curr_token, dct
def parse_dict(curr_token):
curr_token = consume(curr_token, 4, "'['") # dict start
curr_token, dct = parse_kv(curr_token)
curr_token = consume(curr_token, 5, "']'") # dict end
return curr_token, dct
def parse_graph():
curr_token, dct = parse_kv(next(tokens))
if curr_token[0] is not None: # EOF
unexpected(curr_token, "EOF")
if "graph" not in dct:
raise NetworkXError("input contains no graph")
graph = dct["graph"]
if isinstance(graph, list):
raise NetworkXError("input contains more than one graph")
return graph
tokens = tokenize()
graph = parse_graph()
directed = graph.pop("directed", False)
multigraph = graph.pop("multigraph", False)
if not multigraph:
G = nx.DiGraph() if directed else nx.Graph()
else:
G = nx.MultiDiGraph() if directed else nx.MultiGraph()
G.graph.update((key, value) for key, value in graph.items()
if key != "node" and key != "edge")
def pop_attr(dct, category, attr, i):
try:
return dct.pop(attr)
except KeyError:
raise NetworkXError("%s #%d has no '%s' attribute" %
(category, i, attr))
nodes = graph.get("node", [])
mapping = {}
node_labels = set()
for i, node in enumerate(nodes if isinstance(nodes, list) else [nodes]):
id = pop_attr(node, "node", "id", i)
if id in G:
raise NetworkXError("node id %r is duplicated" % (id, ))
if label is not None and label != "id":
node_label = pop_attr(node, "node", label, i)
if node_label in node_labels:
raise NetworkXError("node label %r is duplicated" %
(node_label, ))
node_labels.add(node_label)
mapping[id] = node_label
G.add_node(id, **node)
edges = graph.get("edge", [])
for i, edge in enumerate(edges if isinstance(edges, list) else [edges]):
source = pop_attr(edge, "edge", "source", i)
target = pop_attr(edge, "edge", "target", i)
if source not in G:
raise NetworkXError("edge #%d has an undefined source %r" %
(i, source))
if target not in G:
raise NetworkXError("edge #%d has an undefined target %r" %
(i, target))
if not multigraph:
if not G.has_edge(source, target):
G.add_edge(source, target, **edge)
else:
msg = "edge #%d (%r%s%r) is duplicated.\n"
msg2 = 'Hint: If multigraph add "multigraph 1" to file header.'
info = (i, source, "->" if directed else "--", target)
raise nx.NetworkXError((msg % info) + msg2)
else:
key = edge.pop("key", None)
if key is not None and G.has_edge(source, target, key):
raise nx.NetworkXError(
"edge #%d (%r%s%r, %r) is duplicated" %
(i, source, "->" if directed else "--", target, key))
G.add_edge(source, target, key, **edge)
if label is not None and label != "id":
G = nx.relabel_nodes(G, mapping)
return G
def get_truncated_complex(r_complex, bond_rearrangement):
"""
From a truncated reactant complex by removing non core atoms and adding
capping atoms where appropriate
r_complex -> t_complex
Arguments:
r_complex (autode.complex.ReactantComplex):
bond_rearrangement (autode.bond_rearrangement.BondRearrangement):
Returns:
(autode.complex.ReactantComplex)
"""
active_atoms = bond_rearrangement.active_atoms
t_complex = deepcopy(r_complex)
logger.info(f'Truncating {r_complex.name} with {r_complex.n_atoms} atoms '
f'around core atoms: {active_atoms}')
t_graph = nx.Graph()
# Add all the core active atoms to the graphs, their nearest neighbours
# and the bonds between them
t_graph.add_nodes_from([(i, r_complex.graph.nodes[i])
for i in active_atoms])
for i in active_atoms:
t_graph.add_nodes_from([(j, r_complex.graph.nodes[j])
for j in r_complex.graph.neighbors(i)])
t_graph.add_edges_from([(i, j, r_complex.graph.edges[(i, j)])
for j in r_complex.graph.neighbors(i)])
# Add all the π bonds that are associated with the core atoms, then close
# those those etc.
curr_nodes = add_core_pi_bonds(r_complex,
t_complex,
truncated_graph=t_graph)
# Swap all saturated carbons and the attached fragment for H
logger.warning('Truncation is only implemented over C-X single bonds')
add_capping_atoms(r_complex,
t_complex,
truncated_graph=t_graph,
curr_nodes=curr_nodes)
add_remaining_bonds(t_graph, full_graph=r_complex.graph)
add_remaining_atoms(t_graph,
full_graph=r_complex.graph,
s_molecule=t_complex)
# Delete all atoms not in the truncated graph and reset the graph
t_complex.graph = t_graph
t_complex.atoms = [
atom for i, atom in enumerate(t_complex.atoms)
if i in sorted(t_graph.nodes)
]
# Relabel the nodes so they correspond to the new set of atoms
mapping = {
node_label: i
for i, node_label in enumerate(sorted(t_graph.nodes))
}
t_complex.graph = nx.relabel_nodes(t_complex.graph, mapping=mapping)
logger.info(f'Truncated to {t_complex.n_atoms} atoms')
t_complex.name += '_truncated'
return t_complex
for user_id in graph.nodes():
for att in graph_attributes:
if att in graph.node[user_id]:
del graph.node[user_id][att]
df = pd.read_csv("../data/users_all.csv", usecols=["user_id"])
mapping = dict()
for old_idx, new_idx in zip(df["user_id"].values, np.array(range(len(df["user_id"].values)))):
mapping[str(old_idx)] = int(new_idx)
graph = nx.relabel_nodes(graph, mapping)
nx.write_graphml(graph, "../data/users_clean.graphml")
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = graph sage input = = = = = = = = = = = = = = = = = = = = =
graph = nx.read_graphml("../data/users_clean.graphml")
nx.write_edgelist(graph, "../data/graph-input/users.edges", data=False)
df = pd.read_csv("../data/users_anon.csv")
print(len(cols_attr))
feats = np.nan_to_num(df[cols_glove].values)
ids = df["user_id"].values
import networkx as nx
from the_traffic_magic import get_pareto_traffic
G = nx.Graph()
with open('p2p-Gnutella04_Nodes_10876_edges_39994.txt', 'r') as f:
for line in f:
l = [each.strip() for each in line.strip().split('\t')]
G.add_edge(l[0], l[1])
nodes = G.nodes()
import itertools
mapping = {value: int(value) for value in nodes}
G = nx.relabel_nodes(G, mapping, copy=False)
nodes = G.nodes()
edges = G.edges()
edge_dict = {}
edge_traffic = {}
def initialize_edge_traffic():
global edge_traffic
for each in edges:
edge_traffic[each] = 0
def update_edge_traffic(val1, val2, traffic):
global edge_traffic
if (val1, val2) in edge_traffic.keys():
def main():
tic = time.perf_counter()
gn = Granatum()
clustersvsgenes = gn.pandas_from_assay(gn.get_import('clustersvsgenes'))
gset_group_id = gn.get_arg('gset_group_id')
min_zscore = gn.get_arg('min_zscore')
clustercomparisonstotest = list(clustersvsgenes.index)
# Load all gene sets
gsets = load_gsets(gset_group_id)
G = nx.MultiDiGraph()
clusternames = list(clustersvsgenes.T.columns)
individualclusters = [
n[:n.index(" vs rest")] for n in clusternames if n.endswith("vs rest")
]
print(individualclusters, flush=True)
for cl in individualclusters:
G.add_node(cl)
# {pathway : {"cluster1":score1, "cluster2":score2}, pathway2 : {}}
resultsmap = {}
relabels = {}
keys = {}
urlsforkeys = {}
currentkeyindex = 0
for gset in gsets:
urlsforkeys[gset["name"]] = gset["url"]
for cluster in clustercomparisonstotest:
try:
resultdf = clustersvsgenes.loc[cluster, gset["gene_ids"]]
resultdf = np.nan_to_num(resultdf)
score = np.nanmean(resultdf)
if score >= min_zscore:
keys[gset["name"]] = keys.get(gset["name"],
currentkeyindex + 1)
print("Score = {}".format(score), flush=True)
olddict = resultsmap.get(gset["name"], {})
olddict[cluster] = score
resultsmap[gset["name"]] = olddict
from_to = re.split(' vs ', cluster)
if from_to[1] != 'rest':
G.add_weighted_edges_from(
[(from_to[1], from_to[0], score * 2.0)],
label=str(keys[gset["name"]]),
penwidth=str(score * 2.0))
else:
relabel_dict = relabels.get(from_to[0], "")
if relabel_dict == "":
relabel_dict = from_to[0] + ": " + str(
keys[gset["name"]])
else:
relabel_dict = relabel_dict + ", " + str(
keys[gset["name"]])
relabels[from_to[0]] = relabel_dict
currentkeyindex = max(currentkeyindex, keys[gset["name"]])
except Exception as inst:
print("Key error with {}".format(gset["name"]), flush=True)
print("Exception: {}".format(inst), flush=True)
print("Relabels {}".format(relabels), flush=True)
G = nx.relabel_nodes(G, relabels)
pos = nx.spring_layout(G)
edge_labels = nx.get_edge_attributes(G, 'label')
write_dot(G, 'plot.dot')
os.system("dot plot.dot -Tpng -Gdpi=600 > plot.png")
with open('plot.png', "rb") as f:
image_b64 = b64encode(f.read()).decode("utf-8")
gn.results.append({
"type": "png",
"width": 650,
"height": 480,
"description": 'Network of clusters based on expression',
"data": image_b64,
})
footnote = ""
for k, v in sorted(keys.items(), key=lambda item: item[1]):
newstr = "{}: [{}]({})".format(v, k, urlsforkeys[k])
if footnote == "":
footnote = newstr
else:
footnote = footnote + " \n" + newstr
gn.add_result(footnote, "markdown")
# gn.export(return_df.T.to_csv(), 'differential_gene_sets.csv', kind='raw', meta=None, raw=True)
toc = time.perf_counter()
time_passed = round(toc - tic, 2)
timing = "* Finished differential expression sets step in {} seconds*".format(
time_passed)
gn.add_result(timing, "markdown")
gn.commit()
