def copy_layout(from_fname, to_fname):
if not from_fname[-4:] =='.gml': from_name +='.gml'
if not to_fname[-4:] =='.gml': to_name +='.gml'
print 'reading A=', from_fname,'..',
g1 = NX.read_gml(from_fname)
labels1 = NX.get_node_attributes(g1, 'label')
n1 = set(labels1.values())
print len(n1),'nodes'
print 'reading B=', to_fname,'..',
g2 = NX.read_gml(to_fname)
labels2 = NX.get_node_attributes(g2, 'label')
n2 = set(labels2.values())
print len(n2),'nodes'
intersection = len(n2.intersection(n1))
percent=100.*intersection/len(n2)
print 'B.intersect(A)=',intersection,'(%.1f%%)'%percent
print 'copying layout..',
mapping = {}
for L1 in labels1:
for L2 in labels2:
if labels1[L1]==labels2[L2]:
mapping[L1] = L2
break
layout = NX.get_node_attributes(g1, 'graphics')
attr = dict([ ( mapping[ID], {'x':layout[ID]['x'],'y':layout[ID]['y']} ) for ID in mapping])
NX.set_node_attributes(g2, 'graphics', attr)
NX.write_gml(g2, to_fname)
print 'done.'
def build_pi_graph(criterion_uri):
G = nx.DiGraph()
q_up = render_template('pi_to_pi_upwards.q', criterion_uri = criterion_uri)
q_down = render_template('pi_to_pi_downwards.q', criterion_uri = criterion_uri)
q_trial = render_template('pi_originating_trial.q', criterion_uri = criterion_uri)
G = build_graph(G, criterion_uri, source="child", target="trial", query=q_up, intermediate = "parent")
G = build_graph(G, criterion_uri, source="parent", target="trial", query=q_down, intermediate = "child")
G = build_graph(G, criterion_uri, source="child", target="trial", query=q_trial)
types = {}
names = {}
for n, nd in G.nodes(data=True):
print nd['type']
if nd['type'] != 'trial' :
label = nd['label'][:-3]
types[n] = 'criterion_cg'
names[n] = label
else :
label = nd['label'].replace('Trial','').upper()
types[n] = 'trial_cg'
names[n] = label
nx.set_node_attributes(G,'type', types)
nx.set_node_attributes(G,'label', names)
g_json = json_graph.node_link_data(G) # node-link format to serialize
return g_json
def detect_communities(self):
partition = community.best_partition(self.G)
for n in partition:
nx.set_node_attributes(self.G, 'community', {n: partition[n]})
self.l.append("community")
return self
def randomly_sample_coherent_paths(self,p_start,num_samples, related_param):
#Find related papers in the influence graph
#Weight according to sum of influence graph edge scores
edge_sum = 0
related_paper_dict = {}
for neighbor in self.influence_graph.neighbors(p_start):
edge_sum += sum([self.influence_graph.edge[p_start][neighbor]['weights'][c] for c in self.active_concepts])
#print "p_start has influence of ", edge_sum, "with neighbor " , neighbor
related_paper_dict[neighbor] = edge_sum
self.setup_random_dict_for_sampling(related_paper_dict)
#Sample chains in cohGraph
for s in range(num_samples):
if (rand.random() < related_param):
#get_rand_related_paper
related_paper = self.sample_dict_randomly()
#print "Walking from",related_paper
self.tally_walk_from(related_paper)
else: #start at p_start
#print "Walking from ",p_start
self.tally_walk_from(p_start)
#Add atribute to influence graph for coverage
nx.set_node_attributes(self.influence_graph, 'coverage',0)
#Collect walk tallies for all chains, sum and add to nodes in the influence graph
for chain in self.cohGraph.nodes():
# print "Chain:",chain," - ",self.cohGraph.node[chain]['walk-tally']/float(num_samples)
for node in chain:
self.influence_graph.node[node]['coverage'] += self.cohGraph.node[chain]['walk-tally']
def calculate_indegree(graph):
# will only work on DiGraph (directed graph)
print "Calculating indegree..."
g = graph
indeg = g.in_degree()
nx.set_node_attributes(g, 'indegree', indeg)
return g, indeg
def from_trial_data(cls, circuit, cycles, pos, resultinfo, trialid, step):
# several parts of this script would require reconsideration to support trials
# where the vertices weren't actually deleted
if resultinfo['defect_mode']['mode'] != 'direct removal':
raise RuntimeError('This script was only written with "remove" mode in mind')
# Find change in currents
before = analysis.trial_edge_currents_at_step(circuit, cycles, resultinfo, trialid, step)
after = analysis.trial_edge_currents_at_step(circuit, cycles, resultinfo, trialid, step+1)
before = edict_remove_redundant_entries(before)
delta = {e: after.get(e, 0.0) - before[e] for e in before}
# NOTE: this reconstruction will lack some isolated vertices
defects = set(get_added_defects(resultinfo, trialid, step))
defect_dict = {v:False for v in edict_vertices(delta)}
defect_dict.update({v:True for v in defects})
sources = {(s,t):s for (s,t) in delta}
g = nx.Graph()
g.add_edges_from(delta)
g.add_nodes_from(defects) # in case any of the defects are isolated vertices
# ``pos`` contains all vertices from the graph's initial state. Limit it to those currently
# in the modified graph (because set_node_attributes crashes on nodes that don't exist)
pos = {k:v for k,v in pos.items() if k in g}
nx.set_edge_attributes(g, EATTR_CHANGE, delta)
nx.set_edge_attributes(g, EATTR_SOURCE, sources)
nx.set_node_attributes(g, VATTR_POS, pos)
nx.set_node_attributes(g, VATTR_DEFECT, defect_dict)
return cls(g)
def __init__(self, n=1000, k=10, p=0.02947368):
self.n = n
self.k = k
self.p = p
self.ws = nx.watts_strogatz_graph(self.n, self.k, self.p, seed='nsll')
nx.set_node_attributes(self.ws, 'SIR', 'S')
self.clustering = nx.clustering(self.ws)
self.betweenness = nx.betweenness_centrality(self.ws)
p_r_0 = 0.001
r_0 = int(self.n * p_r_0)
if r_0 < 1:
r_0 = 1
random.seed('nsll')
self.r = random.sample(self.ws.nodes(), r_0)
i_0 = 4
if i_0 < r_0:
i_0 += 1
random.seed('nsll')
self.infected = random.sample(self.ws.nodes(), i_0)
for n in self.infected:
self.ws.node[n]['SIR'] = 'I'
for n in self.r:
self.ws.node[n]['SIR'] = 'R'
self.s = self.n - len(self.infected) - len(self.r)
print(self.r)
print(self.infected)
def update_pos(self,n,p,now=0.,p_pe='p'):
"""
Update Position of a node
Parameters
----------
n : float/string (or a list of)
node ID
p : np.array ( or a list of )
node position
"""
if (isinstance(p,np.ndarray)) or (isinstance(n,list) and isinstance(p,list) ):
# Tranfrom input as list
if not(isinstance(n,list)):
n=[n]
p=[p]
if len(n) == len(p):
d=dict(zip(n,p)) # transform data to be complient with nx.set_node_attributes
nowd=dict(zip(n,[now]*len(n)))
else :
raise TypeError('n and p must have the same length')
# update position
nx.set_node_attributes(self,p_pe,d)
# update time of ground truth position
if p_pe=='p':
nx.set_node_attributes(self,'t',nowd)
else :
raise TypeError('n and p must be either: a key and a np.ndarray, or 2 lists')
def add_partitions_to_digraph(graph, partitiondict): ''' Add the partition numbers to a graph - in this case, using this to update the digraph, with partitions calc'd off the undirected graph. Yes, it's a bad hack. ''' g = graph nx.set_node_attributes(g, 'partition', partitiondict) nx.info(g) return
def new_vertex_property(self, name):
values = {v: None for v in self.nodes()}
nx.set_node_attributes(self, name=name, values=values)
if name == 'vertex_color':
self.vertex_color = [0 for v in range(self.number_of_nodes())]
if name == 'vertex_fill_color':
self.vertex_fill_color = [0 for v in range(self.number_of_nodes())]
def gen_data():
"""
generates test metrics and network, where the network is a
balanced graph of height 2 and branch factor 2
"""
network = nx.balanced_tree(2, 2)
metrics = DataFrame(network.node).T
metrics['Demand'] = [np.nan, 100, 50, 25, 12, 6, 3]
metrics['Population'] = [np.nan, 100, 50, 25, 12, 6, 3]
#level 0
metrics['coords'] = [np.array([125,10]) for x in metrics.index]
#level 2
metrics['coords'].ix[1] = metrics['coords'].ix[0] + [-.5, -.25]
metrics['coords'].ix[2] = metrics['coords'].ix[0] + [+.5, -.25]
#level 3
metrics['coords'].ix[3] = metrics['coords'].ix[1] + [-.25, -.25]
metrics['coords'].ix[4] = metrics['coords'].ix[1] + [+.25, -.25]
metrics['coords'].ix[5] = metrics['coords'].ix[2] + [-.25, -.25]
metrics['coords'].ix[6] = metrics['coords'].ix[2] + [+.25, -.25]
metrics['coords'] = metrics['coords'].apply(tuple)
nx.set_node_attributes(network, 'coords', metrics.coords.to_dict())
#nx.draw(network, nx.get_node_attributes(network, 'coords'))
return metrics, network.to_directed()
def build(self, matrix, skim_depth=10):
"""
Build graph, with PageRanks on nodes.
:param matrix: A term matrix.
:param skim_depth: The number of sibling edges.
"""
# Register nodes and edges.
for anchor in progress.bar(matrix.terms):
n1 = matrix.text.unstem(anchor)
# Heaviest pair scores:
pairs = matrix.anchored_pairs(anchor).items()
for term, weight in list(pairs)[:skim_depth]:
n2 = matrix.text.unstem(term)
self.graph.add_edge(n1, n2, weight=weight)
# Compute PageRanks.
ranks = nx.pagerank(self.graph)
first = max(ranks.values())
# Convert to 0->1 ratios.
ranks = {k: v/first for k, v in ranks.items()}
# Annotate the nodes.
nx.set_node_attributes(self.graph, 'pagerank', ranks)
def calculate_betweenness(graph): ''' Calculate betweenness centrality of a node, sets value on node as attribute; returns graph, and dict of the betweenness centrality values ''' g = graph bc=nx.betweenness_centrality(g) nx.set_node_attributes(g,'betweenness',bc) return g, bc
def import_layout(from_fname, to_graph):
if not from_fname[-4:] =='.gml': from_fname +='.gml'
print 'importing layout from', from_fname+'..'
g1 = NX.read_gml(from_fname)
labels1 = NX.get_node_attributes(g1, 'label')
n1 = set(labels1.values())
g2 = to_graph
n2 = set(g2.nodes())
if not n1:
print ' empty target graph'
return
if not n2:
print ' empty layout graph'
return
mapping = {}
for L1 in labels1:
for name in n2:
if labels1[L1]==name:
mapping[L1] = name
break
intersection = len(n2.intersection(n1))
percent=100.*intersection/len(n2)
print ' %.1f%%'%percent,'(%i positions)'%intersection
layout = NX.get_node_attributes(g1, 'graphics')
attr = dict([ ( mapping[ID], {'x':layout[ID]['x'],'y':layout[ID]['y']} ) for ID in mapping])
NX.set_node_attributes(g2, 'graphics', attr)
def coword_network(mesh_df, start, end,topic_count=0):
"""
constructs a coword network for the years supplied;
nodes will be labelled by topic, have a 'weight' of co-occurrence,
a 'start_year' attribute,
and an 'end_year' attribute which is the end year of the search
Parameters
----------------
mesh_df: a dataframe with at least the topics and years columns
start: start year
end: end year
topic_count: the number of the topics to use
(not too big, otherwise coword matrix will be huge
"""
# determine the number of topics to count
all_topics = [t for top in mesh_df.topics.dropna() for t in top]
topic_collection = collections.Counter(all_topics)
if topic_count > 0 and topic_count < len(topic_collection):
common_topics = [k[0] for k in topic_collection.most_common(topic_count)]
else:
common_topics = sorted(topic_collection.keys())
cow_df = coword_matrix_years(mesh_df, start, end, common_topics)
cow_nx = nx.from_numpy_matrix(cow_df.as_matrix())
col_names = cow_df.columns.tolist()
labels = {col_names.index(l): l for l in col_names}
start_year = {i: end for i in range(0, len(col_names))}
end_year = {i: start for i in range(0, len(col_names))}
nx.set_node_attributes(cow_nx, 'start_year', start_year)
nx.set_node_attributes(cow_nx, 'end_year', end_year)
nx.relabel_nodes(cow_nx, labels, copy=False)
return cow_nx
def _get_demand_nodes(self, input_proj=None):
"""
Converts the dataset_store metrics records to a GeoGraph of nodes
(prereq: _run_metric_model to populate store)
Args:
input_proj: projection of demand node coordinates
Returns:
GeoGraph: demand nodes as GeoGraph
"""
coords = [node.getCommonCoordinates() for node in
self.store.cycleNodes()]
# set default projection
if not input_proj:
input_proj = self._get_default_proj4(coords)
# NOTE: Although dataset_store nodes id sequence starts at 1
# leave the GeoGraph ids 0 based because there are places in the
# network algorithm that assume 0 based coords
# This will be realigned later
coords_dict = {i: coord for i, coord in enumerate(coords)}
budget_dict = {i: node.metric for i, node in
enumerate(self.store.cycleNodes())}
geo_nodes = GeoGraph(input_proj, coords_dict)
nx.set_node_attributes(geo_nodes, 'budget', budget_dict)
return geo_nodes
def create_protein_graph(ingraph):
pnodes = {}
pnodes = defaultdict(lambda: 0, pnodes)
pedges = {}
pedges = defaultdict(lambda: 0, pedges)
outgraph = nx.Graph()
for node in ingraph.nodes_iter():
pnodes[ingraph.node[node]['protein']]+=1
for u,v,d in ingraph.edges(data=True):
key=(ingraph.node[u]['protein'], ingraph.node[v]['protein'])
pedges[key]+=1
for key in pnodes.keys():
outgraph.add_node(key)
edges = combinations(pnodes.keys(), 2)
outgraph.add_nodes_from(pnodes.keys())
outgraph.add_edges_from(edges)
nx.set_node_attributes(outgraph, 'count', pnodes)
nx.set_edge_attributes(outgraph, 'weight', pedges)
return outgraph
def draw_network_by_years(df, start_year, end_year, trim):
""" Constructs and draws the co-word networks for the years
Parameters
-----------------------------------------
df: WoS references
start_year:
end_year:
trim: degree of nodes to include in the graph
Returns
----------------------------------
coword networkx object
"""
df_sub = df[(df.PY> start_year) & (df.PY <= end_year)]
keys = keyword_counts(df_sub)
print('Calculating co-word matrix')
coword_df = coword_matrix(df_sub,keys.keys())
coword_array = coword_df.as_matrix()
np.fill_diagonal(coword_array, 0)
coword_net = nx.from_numpy_matrix(coword_array)
col_names = coword_df.columns.tolist()
labels = {col_names.index(l):l for l in col_names}
nx.set_node_attributes(coword_net, 'keyword', labels)
nx.set_node_attributes(coword_net, 'between_central', nx.betweenness_centrality(coword_net))
if trim > 0:
coword_net = trim_nodes(coword_net, trim)
labels = {n:labels[n] for n in coword_net.nodes()}
return coword_net
def closeness_neighbors(seed_num, graph=None, graph_json_filename=None, graph_json_str=None):
if graph_json_filename is None and graph_json_str is None and graph is None:
return []
G = None
if graph is not None:
G = graph
elif graph_json_str is None:
G = util.load_graph(graph_json_filename=graph_json_filename)
else:
G = util.load_graph(graph_json_str=graph_json_str)
clse_cent = nx.get_node_attributes(G, "centrality")
if len(clse_cent) == 0:
clse_cent = nx.closeness_centrality(G)
nx.set_node_attributes(G, "centrality", clse_cent)
print "closeness neighbors"
collector = collections.Counter(clse_cent)
clse_cent = collector.most_common(SURROUND_TOP)
nodes = map(lambda (x, y): x, clse_cent)
current_seed = 0
rtn = []
while current_seed < seed_num:
current_node = nodes[current_seed % len(nodes)]
current_neighbors = G.neighbors(current_node)
rtn += random.sample(set(current_neighbors) - set(rtn) - set(nodes), 1)
current_seed += 1
return rtn
def findCommunities(G):
"""
Partition network with the Infomap algorithm.
Annotates nodes with 'community' id and return number of communities found.
"""
infomapWrapper = infomap.Infomap("--two-level")
print("Building Infomap network from a NetworkX graph...")
for e in G.edges_iter():
infomapWrapper.addLink(*e)
print("Find communities with Infomap...")
infomapWrapper.run();
tree = infomapWrapper.tree
print("Found %d top modules with codelength: %f" % (tree.numTopModules(), tree.codelength()))
communities = {}
for node in tree.leafIter():
communities[node.originalLeafIndex] = node.moduleIndex()
nx.set_node_attributes(G, 'community', communities)
return tree.numTopModules()
def build_graph(df, cluster_id, case_id, date_col, color, gen_mean, gen_sd):
"""
Generate a directed graph from data on transmission tree.
Node color is determined by node attributes, e.g. case severity or gender.
df = pandas dataframe
"""
clusters = basics.cluster_builder(df=df, cluster_id=cluster_id, \
case_id=case_id, date_col=date_col, attr_col=color, \
gen_mean=gen_mean, gen_sd=gen_sd)
G = nx.DiGraph()
G.add_nodes_from(clusters['case_id'])
edgelist = [pair for pair in clusters[['source_node']].dropna().itertuples()]
G.add_edges_from(edgelist)
nx.set_node_attributes(G, 'date', clusters['time'].to_dict())
nx.set_node_attributes(G, 'pltdate', clusters['pltdate'].to_dict())
nx.set_node_attributes(G, 'source_node', clusters['source_node'].to_dict())
nx.set_node_attributes(G, color, clusters[color].to_dict())
nx.set_node_attributes(G, 'index_node', clusters['index_node'].to_dict())
G = nx.DiGraph.reverse(G)
for i in G.nodes():
G.node[i]['generation'] = _generations(G, i)
return G
def compute_friends_installed(g, count=False, set_attribute=True):
"""
Analyzes neighborhood app installation ("app_installed" attribute).
Sets nodes' "at_least_one_friend" property (if set_attribute=True)
:param g: networkx graph
:param count: boolean,
if True, computes (for each node) the number of neighbors that
have the app installed;
otherwise, indicates whether any of the friends have the app installed
:param set_attribute: boolean,
if True, sets nodes' "at_least_one_friend" property and returns the graph;
otherwise, returns a vector with the results
:return: networkx graph or vector
"""
myfriends_installed = {}
for n, d in g.nodes_iter(data=True):
fs = g.neighbors(n)
myfriends_installed[n] = False
for f in fs:
if g.node[f]['app_installed']:
if count:
myfriends_installed[n] += 1
else:
myfriends_installed[n] = True
break
if set_attribute:
nx.set_node_attributes(g, 'at_least_one_friend', myfriends_installed)
return g
else:
return myfriends_installed
def __update_structure(self):
if self.seed:
random.seed(self.seed)
prob_out = self.avg_intercomm /\
(float)(self.comm_size * (self.num_comm - 1))
prob_in = 0.5 - (float)(self.avg_intercomm / self.comm_size)
self.structure.add_nodes_from(range(self.num_comm * self.comm_size))
for left_node in self.structure.nodes():
nx.set_node_attributes(
self.structure,
"community",
{left_node: left_node % self.num_comm}
)
for right_node in self.structure.nodes():
if left_node < right_node:
rand = random.random()
if left_node % self.num_comm == right_node % self.num_comm:
if rand <= prob_in:
self.structure.add_edge(
left_node,
right_node
)
else:
if rand <= prob_out:
self.structure.add_edge(
left_node,
right_node
)
def create_graph_df(vtask_paths, graphs_dir_out):
"""
Creates a frame that maps sourcefiles to networkx digraphs in terms of DOT files
:param source_path_list:
:param dest_dir_path:
:param relabel:
:return:
"""
if not isdir(graphs_dir_out):
raise ValueError('Invalid destination directory.')
data = []
graphgen_times = []
print('Writing graph representations of verification tasks to {}'.format(graphs_dir_out), flush=True)
common_prefix = commonprefix(vtask_paths)
for vtask in tqdm(vtask_paths):
short_prefix = dirname(common_prefix)
path = join(graphs_dir_out, vtask[len(short_prefix):][1:])
if not os.path.exists(dirname(path)):
os.makedirs(dirname(path))
ret_path = path + '.pickle'
# DEBUG
if isfile(ret_path):
data.append(ret_path)
continue
start_time = time.time()
graph_path, node_labels_path, edge_types_path, edge_truth_path, node_depths_path \
= _run_cpachecker(abspath(vtask))
nx_digraph = nx.read_graphml(graph_path)
node_labels = _read_node_labeling(node_labels_path)
nx.set_node_attributes(nx_digraph, 'label', node_labels)
edge_types = _read_edge_labeling(edge_types_path)
parsed_edge_types = _parse_edge(edge_types)
nx.set_edge_attributes(nx_digraph, 'type', parsed_edge_types)
edge_truth = _read_edge_labeling(edge_truth_path)
parsed_edge_truth = _parse_edge(edge_truth)
nx.set_edge_attributes(nx_digraph, 'truth', parsed_edge_truth)
node_depths = _read_node_labeling(node_depths_path)
parsed_node_depths = _parse_node_depth(node_depths)
nx.set_node_attributes(nx_digraph, 'depth', parsed_node_depths)
assert not isfile(ret_path)
assert node_labels and parsed_edge_types and parsed_edge_truth and parsed_node_depths
nx.write_gpickle(nx_digraph, ret_path)
data.append(ret_path)
gg_time = time.time() - start_time
graphgen_times.append(gg_time)
return pd.DataFrame({'graph_representation': data}, index=vtask_paths), graphgen_times
def _parse(self):
# Initialize variables
self.tree = nx.DiGraph()
self.name2taxon_id = defaultdict(int)
self.taxon_id2name = defaultdict(str)
ncbi_taxdmp_dir = self._downloaders[0].path
# csv.field_size_limit(sys.maxsize)
with open(os.path.join(ncbi_taxdmp_dir, 'names.dmp'), 'r') as handle:
# csv_handle = csv.reader(handle, delimiter="\t")
for cols in handle:
cols = cols.split('\t')
taxon_id = int(cols[0])
name = cols[2]
self.name2taxon_id[name] = taxon_id
if cols[-2] == 'scientific name':
self.taxon_id2name[taxon_id] = name
# construct node tree
edges = []
nodes = {}
with open(os.path.join(ncbi_taxdmp_dir, 'nodes.dmp'), 'r') as handle:
csv_handle = csv.reader(handle, delimiter="\t")
for cols in csv_handle:
parent_node = int(cols[2])
child_node = int(cols[0])
rank = cols[4]
nodes[child_node] = rank
if child_node != parent_node:
edges.append((child_node, parent_node))
self.tree.add_edges_from(edges)
nx.set_node_attributes(self.tree, 'rank', nodes)
def random_binary_dgm(n=10, p=0.2):
G = nx.gnr_graph(n, p)
dgm = DGM()
dgm.add_nodes_from(G.nodes())
dgm.add_edges_from(G.edges())
nx.set_node_attributes(dgm, 'CPD', { node: TableFactor(random_table_factor(dgm.in_degree(node) + 1), list(dgm.predecessors(node)) + [node]) for node in dgm.nodes() })
return dgm
def to_networkx(self):
"""Return a NetworkX DiGraph object representing the single linkage tree.
Edge weights in the graph are the distance values at which child nodes
merge to form the parent cluster.
Nodes have a `size` attribute attached giving the number of points
that are in the cluster.
"""
try:
from networkx import DiGraph, set_node_attributes
except ImportError:
raise ImportError('You must have networkx installed to export networkx graphs')
max_node = 2 * self._linkage.shape[0]
num_points = max_node - (self._linkage.shape[0] - 1)
result = DiGraph()
for parent, row in enumerate(self._linkage, num_points):
result.add_edge(parent, row[0], weight=row[2])
result.add_edge(parent, row[1], weight=row[2])
size_dict = {parent : row[3] for parent, row in enumerate(self._linkage, num_points)}
set_node_attributes(result, 'size', size_dict)
return result
def calculate_outdegree(graph):
# will only work on DiGraph (directed graph)
print "Calculating outdegree..."
g = graph
outdeg = g.out_degree()
nx.set_node_attributes(g, 'outdegree', outdeg)
return g, outdeg
def test_node_num_attribute(self):
g = nx.karate_club_graph()
attr = {n: {"even": int(n % 10)} for n in g.nodes()}
nx.set_node_attributes(g, attr)
model = gc.CompositeModel(g)
model.add_status("Susceptible")
model.add_status("Infected")
c = cpm.NodeNumericalAttribute("even", value=0, op="==", probability=1)
model.add_rule("Susceptible", "Infected", c)
config = mc.Configuration()
config.add_model_parameter('percentage_infected', 0.1)
model.set_initial_status(config)
iterations = model.iteration_bunch(10)
self.assertEqual(len(iterations), 10)
model = gc.CompositeModel(g)
model.add_status("Susceptible")
model.add_status("Infected")
c = cpm.NodeNumericalAttribute("even", value=[3, 5], op="IN", probability=1)
model.add_rule("Susceptible", "Infected", c)
config = mc.Configuration()
config.add_model_parameter('percentage_infected', 0.1)
model.set_initial_status(config)
iterations = model.iteration_bunch(10)
self.assertEqual(len(iterations), 10)
def detect_communities(graph, verbose=False):
graph = graph_from_csv(graph)
partition = community.best_partition(graph)
if verbose:
print "%i partitions" % len(set(partition.values()))
nx.set_node_attributes(graph, 'partition', partition)
return graph, partition
def _compute_optimistic_cost_table(_self, dag):
"""
Uses a basic BFS approach to traverse upwards through the graph building the optimistic cost table along the way
"""
optimistic_cost_table = {}
terminal_node = [
node for node in dag.nodes()
if not any(True for _ in dag.successors(node))
]
assert len(
terminal_node
) == 1, f"Expected a single terminal node, found {len(terminal_node)}"
terminal_node = terminal_node[0]
diagonal_mask = np.ones(_self.communication_matrix.shape, dtype=bool)
np.fill_diagonal(diagonal_mask, 0)
avgCommunicationCost = np.mean(_self.communication_matrix[diagonal_mask])
for edge in dag.edges():
logger.debug(
f"Assigning {edge}'s average weight based on average communication cost. {float(dag.get_edge_data(*edge)['weight'])} => {float(dag.get_edge_data(*edge)['weight']) / avgCommunicationCost}"
)
nx.set_edge_attributes(dag, {
edge:
float(dag.get_edge_data(*edge)['weight']) / avgCommunicationCost
}, 'avgweight')
optimistic_cost_table[
terminal_node] = _self.computation_matrix.shape[1] * [0]
# lol whoops dag.node doesn't exist
#dag.node[terminal_node]['rank'] = 0
nx.set_node_attributes(dag, {terminal_node: 0}, "rank")
visit_queue = deque(dag.predecessors(terminal_node))
node_can_be_processed = lambda node: all(successor in optimistic_cost_table
for successor in dag.successors(
node))
while visit_queue:
node = visit_queue.pop()
while node_can_be_processed(node) is not True:
try:
node2 = visit_queue.pop()
except IndexError:
raise RuntimeError(
f"Node {node} cannot be processed, and there are no other nodes in the queue to process instead!"
)
visit_queue.appendleft(node)
node = node2
optimistic_cost_table[node] = _self.computation_matrix.shape[1] * [0]
logger.debug(
f"Computing optimistic cost table entries for node: {node}")
# Perform OCT kernel
# Need to build the OCT entries for every task on each processor
for curr_proc in range(_self.computation_matrix.shape[1]):
# Need to maximize over all the successor nodes
max_successor_oct = -inf
for succnode in dag.successors(node):
logger.debug(f"\tLooking at successor node: {succnode}")
# Need to minimize over the costs across each processor
min_proc_oct = inf
for succ_proc in range(_self.computation_matrix.shape[1]):
successor_oct = optimistic_cost_table[succnode][succ_proc]
successor_comp_cost = _self.computation_matrix[succnode][
succ_proc]
successor_comm_cost = dag[node][succnode][
'avgweight'] if curr_proc != succ_proc else 0
cost = successor_oct + successor_comp_cost + successor_comm_cost
logger.debug(
f"If node {node} is on {curr_proc} and successor {succnode} is on {succ_proc}, the optimistic cost entry is {cost}"
)
if cost < min_proc_oct:
min_proc_oct = cost
if min_proc_oct > max_successor_oct:
max_successor_oct = min_proc_oct
assert max_successor_oct != -inf, f"No node should have a maximum successor OCT of {-inf} but {node} does when looking at processor {curr_proc}"
optimistic_cost_table[node][curr_proc] = max_successor_oct
# End OCT kernel
# lol whoops dag.node doesn't exist
#dag.node[node]['rank'] = np.mean(optimistic_cost_table[node])
nx.set_node_attributes(dag,
{node: np.mean(optimistic_cost_table[node])},
"rank")
visit_queue.extendleft([
prednode for prednode in dag.predecessors(node)
if prednode not in visit_queue
])
return optimistic_cost_table
def set_node_position(self, node: str, x, y) -> None:
node_attr = {"position": {"x": x, "y": y}}
nx.set_node_attributes(self._graph, {node: node_attr})
emails.MetadataFrom = emails.MetadataFrom.apply(unify_name)
emails.MetadataTo = emails.MetadataTo.apply(unify_name)
# 设置遍的权重等于发邮件的次数
edges_weights_temp = defaultdict(list)
for row in zip(emails.MetadataFrom, emails.MetadataTo, emails.RawText):
temp = (row[0], row[1])
if temp not in edges_weights_temp:
edges_weights_temp[temp] = 1
else:
edges_weights_temp[temp] = edges_weights_temp[temp] + 1
edges_weights = [(key[0], key[1], val)
for key, val in edges_weights_temp.items()]
graph = nx.DiGraph()
graph.add_weighted_edges_from(edges_weights)
pagerank = nx.pagerank(graph)
nx.set_node_attributes(graph, name='pagerank', values=pagerank)
show_graph(graph)
pagerank_threshold = 0.005
small_graph = graph.copy()
for n, p_rank in graph.nodes(data=True):
if p_rank['pagerank'] < pagerank_threshold:
small_graph.remove_node(n)
show_graph(small_graph, 'circular_layout')
def _route(root: TreeNode,
G: nx.Graph,
removeLCA=True,
remove_intra=False,
plot_tree=False):
# first convert to cartesian coordinates
idx2node = {}
node2pos = {}
def cartesian(node):
if node is None:
return
x = node.length * math.cos(node.angle)
y = node.length * math.sin(node.angle)
node2pos[node] = np.array([x, y])
idx2node[node.idx] = node
for child in node.children:
cartesian(child)
cartesian(root)
idx2pos = {i: node2pos[x] for i, x in idx2node.items()}
nx.set_node_attributes(G, idx2pos, 'pos')
colors = nx.get_node_attributes(G, 'color')
if plot_tree:
def drawtree(node):
if node is None:
return
x0, y0 = node2pos[node]
if node.parent is not None:
x1, y1 = node2pos[node.parent]
ax.plot([x0, x1], [-y0, -y1], color='k')
for child in node.children:
drawtree(child)
fig, ax = plt.subplots()
drawtree(root)
ax.axis('equal')
plt.show()
def LCA(src1, src2):
curr1 = src1
curr2 = src2
while curr1 is not curr2:
curr1 = curr1.parent if curr1 is not None else src2
curr2 = curr2.parent if curr2 is not None else src1
return curr1
def getpath(src, tgt):
path = []
while src is not tgt:
path.append(node2pos[src])
src = src.parent
return path
# then route edges through hierarchy
splines = {}
for src, tgt in G.edges:
lca = LCA(idx2node[src], idx2node[tgt])
assert lca is not None
src2lca = getpath(idx2node[src], lca)
tgt2lca = getpath(idx2node[tgt], lca)
if not removeLCA or len(src2lca) + len(tgt2lca) <= 2:
if not remove_intra:
splines[(src, tgt)] = np.array(src2lca + [node2pos[lca]] +
list(reversed(tgt2lca)))
else:
splines[(src, tgt)] = np.array(src2lca + list(reversed(tgt2lca)))
nx.set_edge_attributes(G, splines, 'spline')
args.retrieve) # receive a tuple, see how's used below
logging.info('Meta-Task ' + args.retrieve + ' retrieved')
dg = nx.DiGraph()
# note that we have to stringify some data elements. Doing it in the Oracle
# access method does not make it more elegant, unfortunately. Will change
# eventually to automatic conversion TBD.
for node in meta[
0]: # first element of the tuple is an array of nodes (tasks)
nn = str(node[0]) # "node name"
dg.add_node(nn) # dgn = DeftGraphNode() # yet to be developed
nx.set_node_attributes(dg, 'meta', {nn: node[1]})
nx.set_node_attributes(dg, 'state', {nn: node[2]})
nx.set_node_attributes(dg, 'tag', {nn: node[3]})
nx.set_node_attributes(dg, 'comment', {nn: node[4]})
for edge in meta[
1]: # second element of the tuple is an array of edges (datasets)
edge_source = str(edge[3])
edge_target = str(edge[4])
dg.add_edge(edge_source, edge_target)
et = (edge_source, edge_target) # "edge tuple"
nx.set_edge_attributes(dg, 'id', {et: edge[0]})
nx.set_edge_attributes(dg, 'meta', {et: edge[1]})
nx.set_edge_attributes(dg, 'state', {et: edge[2]})
osp_idx.append(tree.query([coord_osp[i]], k=1, return_distance=False)[0])
closest_node_to_osp.append(nodes.iloc[osp_idx[i]].index.values[0])
# Si aggiunge l'attributo ospedale ai nodi
# In[4]:
"""si creano due dizionari per i nuovi attributi ospedale e ID_ospedale. Attraverso questi
si valorizzeranno i nuovi attributi"""
nodes_ospedali = dict()
ID_nodes_ospedali = dict()
#si creano altri due dizionari per valorizzare i dati di default: '' per ospedale e -1 per ID
default_ospedale = {i: '' for i in set(g.nodes())}
default_ID_ospedale = {i: -1 for i in set(g.nodes())}
#si attribuiscono i valori di default ai nodi
nx.set_node_attributes(g, default_ospedale, 'ospedale')
nx.set_node_attributes(g, default_ID_ospedale, 'ID_ospedale')
#si assegna la descrizione dell'ospedale e l'ID all'interno del dizionario dei nodi relativi agli ospedali
for i in range(n_ospedali):
nodes_ospedali[closest_node_to_osp[i]] = df.loc[i, 'RAGIONE SOCIALE']
ID_nodes_ospedali[closest_node_to_osp[i]] = i
#si attribuiscono i valori dei dizionari ai nuovi attributi dei nodi della rete
nx.set_node_attributes(g, nodes_ospedali, 'ospedale')
nx.set_node_attributes(g, ID_nodes_ospedali, 'ID_ospedale')
nodes, edges = ox.graph_to_gdfs(g)
nodes.sort_values('ID_ospedale', ascending=False).head()
# Si rappresenta nella mappa in rosso gli ospedali e in verde i nodi più vicini ad essi nella mappa
def color_nodes_compact(g, n, p, q, limit, neighbors_dict):
g = g.copy()
all_nodes = g.nodes()
colors = {}
colored = []
clusters = {}
seeds = {}
for value in all_nodes:
colors[value] = 'g'
seeds[value] = 0
clusters[value] = 0
random_node = random.choice(all_nodes)
jump = True
step = 0
cluster_id = 1
cluster_nodes = []
cluster_neighbors = []
while (colors.values().count('r') < n):
# coloring of the node if it is uncolored
if colors[random_node] == 'g':
if jump == True:
seeds[random_node] = cluster_id
colors[random_node] = 'r'
colored.append(random_node)
clusters[random_node] = cluster_id
cluster_nodes.append(random_node)
cluster_neighbors += neighbors_dict[random_node]
cluster_neighbors = list(set(cluster_neighbors))
last_colored = True
step = 0
jump = False
else:
last_colored = False
step = step + 1
# chose node within cluster to color if there are none uncolored try to jump
if random.random() <= p or last_colored == False:
try:
potential_nodes = list(
set(cluster_neighbors).difference(set(colored)))
if len(potential_nodes) > 0:
random_node = random.choice(potential_nodes)
else:
potential_nodes = list(
set(all_nodes).difference(set(colored)))
random_node = random.choice(potential_nodes)
cluster_id += 1
cluster_nodes = []
cluster_neighbors = []
jump = True
except:
pass
# try to jump by chosing randomly uncolored cluster
else:
potential_nodes = list(set(all_nodes).difference(set(colored)))
random_node = random.choice(potential_nodes)
cluster_id += 1
cluster_nodes = []
cluster_neighbors = []
jump = True
# start new cluster if time limit is up
if step > limit:
potential_nodes = list(set(all_nodes).difference(set(colored)))
random_node = random.choice(potential_nodes)
cluster_id += 1
cluster_nodes = []
cluster_neighbors = []
jump = True
remaining_colors = {}
remaining_colors_clr = {}
for node, color in colors.items():
if color == 'g':
remaining_colors[node] = 'g'
remaining_colors_clr[node] = 'g'
elif color == 'r':
if random.random() <= q:
remaining_colors[node] = 'r'
remaining_colors_clr[node] = 'r'
else:
remaining_colors[node] = 'y'
remaining_colors_clr[node] = 'g'
nx.set_node_attributes(g, 'color_old', colors)
nx.set_node_attributes(g, 'color_clr', remaining_colors_clr)
nx.set_node_attributes(g, 'color', remaining_colors)
nx.set_node_attributes(g, 'cluster', clusters)
nx.set_node_attributes(g, 'seed', seeds)
return g, cluster_id
def color_nodes_hybrid(g, n, p, q, limit, neighbors_dict):
g = g.copy()
all_nodes = g.nodes()
colors = {}
colored = []
clusters = {}
seeds = {}
for value in all_nodes:
colors[value] = 'g'
seeds[value] = 0
clusters[value] = 0
random_node = random.choice(all_nodes)
step = 0
cluster_id = 1
cluster_nodes = []
cluster_neighbors = []
while (colors.values().count('r') < n):
if colors[random_node] == 'g':
colors[random_node] = 'r'
colored.append(random_node)
clusters[random_node] = cluster_id
cluster_nodes.append(random_node)
cluster_neighbors += neighbors_dict[random_node]
cluster_neighbors = list(set(cluster_neighbors))
last_colored = True
step = 0
else:
last_colored = False
step = step + 1
if random.random() <= p or last_colored == False:
try:
#random_node = random.choice(neighbors_dict[random_node])
potential_nodes = list(
set(neighbors_dict[random_node]).difference(set(colored)))
#potential_nodes = list(set(cluster_neighbors).difference(set(colored)))
if len(potential_nodes) > 0:
random_node = random.choice(potential_nodes)
else:
potential_nodes = list(
set(cluster_neighbors).difference(set(colored)))
if len(potential_nodes) > 0:
random_node = random.choice(potential_nodes)
else:
potential_nodes = list(
set(all_nodes).difference(set(colored)))
random_node = random.choice(potential_nodes)
cluster_id += 1
cluster_nodes = []
cluster_neighbors = []
seeds[random_node] = cluster_id
except:
pass
else:
potential_nodes = list(set(all_nodes).difference(set(colored)))
random_node = random.choice(potential_nodes)
cluster_id += 1
cluster_nodes = []
cluster_neighbors = []
if step > limit:
potential_nodes = list(set(all_nodes).difference(set(colored)))
random_node = random.choice(potential_nodes)
cluster_id += 1
cluster_nodes = []
cluster_neighbors = []
remaining_colors = {}
remaining_colors_clr = {}
for node, color in colors.items():
if color == 'g':
remaining_colors[node] = 'g'
remaining_colors_clr[node] = 'g'
elif color == 'r':
if random.random() <= q:
remaining_colors[node] = 'r'
remaining_colors_clr[node] = 'r'
else:
remaining_colors[node] = 'y'
remaining_colors_clr[node] = 'g'
nx.set_node_attributes(g, 'color_old', colors)
nx.set_node_attributes(g, 'color_clr', remaining_colors_clr)
nx.set_node_attributes(g, 'color', remaining_colors)
nx.set_node_attributes(g, 'cluster', clusters)
nx.set_node_attributes(g, 'seed', seeds)
return g, cluster_id
if number_class == 'binary':
# Correct the graph with emd
print("Correcting the graph with EMD")
new_adj, s, gamma, M = total_repair_emd(g, metric='euclidean',
case='weighted', log=False)
new_g = nx.from_numpy_matrix(new_adj)
# Filter out the smallest weights to keep a reasonable density
list_edge = [(u, v) for (u, v, d) in new_g.edges(data=True) if d['weight'] <= 0.5]
new_g.remove_edges_from (list_edge)
# Coefficient of assortativity
dict_s = {i: s[i] for i in range(0, len(s))}
nx.set_node_attributes(new_g, dict_s, 's')
ass_rep.append(nx.attribute_assortativity_coefficient(new_g, 's'))
# Density
density_old.append(nx.density(g))
density_rep.append(nx.density(new_g))
elif number_class == "multi":
X0 = []
X = nx.to_scipy_sparse_matrix(g)
if issparse(X):
X = X.todense()
X = np.squeeze(np.asarray (X))
X0.append(X)
n, d = X.shape
classes = np.unique(s)
def draw_twoday_count(ibs, visit_info_list_):
import copy
visit_info_list = copy.deepcopy(visit_info_list_)
aids_day1, aids_day2 = ut.take_column(visit_info_list_, 'aids')
nids_day1, nids_day2 = ut.take_column(visit_info_list_, 'unique_nids')
resight_nids = ut.isect(nids_day1, nids_day2)
if False:
# HACK REMOVE DATA TO MAKE THIS FASTER
num = 20
for info in visit_info_list:
non_resight_nids = list(set(info['unique_nids']) - set(resight_nids))
sample_nids2 = non_resight_nids[0:num] + resight_nids[:num]
info['grouped_aids'] = ut.dict_subset(info['grouped_aids'], sample_nids2)
info['unique_nids'] = sample_nids2
# Build a graph of matches
if False:
debug = False
for info in visit_info_list:
edges = []
grouped_aids = info['grouped_aids']
aids_list = list(grouped_aids.values())
ams_list = ibs.get_annotmatch_rowids_in_cliques(aids_list)
aids1_list = ibs.unflat_map(ibs.get_annotmatch_aid1, ams_list)
aids2_list = ibs.unflat_map(ibs.get_annotmatch_aid2, ams_list)
for ams, aids, aids1, aids2 in zip(ams_list, aids_list, aids1_list, aids2_list):
edge_nodes = set(aids1 + aids2)
##if len(edge_nodes) != len(set(aids)):
# #print('--')
# #print('aids = %r' % (aids,))
# #print('edge_nodes = %r' % (edge_nodes,))
bad_aids = edge_nodes - set(aids)
if len(bad_aids) > 0:
print('bad_aids = %r' % (bad_aids,))
unlinked_aids = set(aids) - edge_nodes
mst_links = list(ut.itertwo(list(unlinked_aids) + list(edge_nodes)[:1]))
bad_aids.add(None)
user_links = [(u, v) for (u, v) in zip(aids1, aids2) if u not in bad_aids and v not in bad_aids]
new_edges = mst_links + user_links
new_edges = [(int(u), int(v)) for u, v in new_edges if u not in bad_aids and v not in bad_aids]
edges += new_edges
info['edges'] = edges
# Add edges between days
grouped_aids1, grouped_aids2 = ut.take_column(visit_info_list, 'grouped_aids')
nids_day1, nids_day2 = ut.take_column(visit_info_list, 'unique_nids')
resight_nids = ut.isect(nids_day1, nids_day2)
resight_aids1 = ut.take(grouped_aids1, resight_nids)
resight_aids2 = ut.take(grouped_aids2, resight_nids)
#resight_aids3 = [list(aids1) + list(aids2) for aids1, aids2 in zip(resight_aids1, resight_aids2)]
ams_list = ibs.get_annotmatch_rowids_between_groups(resight_aids1, resight_aids2)
aids1_list = ibs.unflat_map(ibs.get_annotmatch_aid1, ams_list)
aids2_list = ibs.unflat_map(ibs.get_annotmatch_aid2, ams_list)
between_edges = []
for ams, aids1, aids2, rawaids1, rawaids2 in zip(ams_list, aids1_list, aids2_list, resight_aids1, resight_aids2):
link_aids = aids1 + aids2
rawaids3 = rawaids1 + rawaids2
badaids = ut.setdiff(link_aids, rawaids3)
assert not badaids
user_links = [(int(u), int(v)) for (u, v) in zip(aids1, aids2)
if u is not None and v is not None]
# HACK THIS OFF
user_links = []
if len(user_links) == 0:
# Hack in an edge
between_edges += [(rawaids1[0], rawaids2[0])]
else:
between_edges += user_links
assert np.all(0 == np.diff(np.array(ibs.unflat_map(ibs.get_annot_nids, between_edges)), axis=1))
import plottool as pt
import networkx as nx
#pt.qt4ensure()
#len(list(nx.connected_components(graph1)))
#print(ut.graph_info(graph1))
# Layout graph
layoutkw = dict(
prog='neato',
draw_implicit=False, splines='line',
#splines='curved',
#splines='spline',
#sep=10 / 72,
#prog='dot', rankdir='TB',
)
def translate_graph_to_origin(graph):
x, y, w, h = ut.get_graph_bounding_box(graph)
ut.translate_graph(graph, (-x, -y))
def stack_graphs(graph_list, vert=False, pad=None):
graph_list_ = [g.copy() for g in graph_list]
for g in graph_list_:
translate_graph_to_origin(g)
bbox_list = [ut.get_graph_bounding_box(g) for g in graph_list_]
if vert:
dim1 = 3
dim2 = 2
else:
dim1 = 2
dim2 = 3
dim1_list = np.array([bbox[dim1] for bbox in bbox_list])
dim2_list = np.array([bbox[dim2] for bbox in bbox_list])
if pad is None:
pad = np.mean(dim1_list) / 2
offset1_list = ut.cumsum([0] + [d + pad for d in dim1_list[:-1]])
max_dim2 = max(dim2_list)
offset2_list = [(max_dim2 - d2) / 2 for d2 in dim2_list]
if vert:
t_xy_list = [(d2, d1) for d1, d2 in zip(offset1_list, offset2_list)]
else:
t_xy_list = [(d1, d2) for d1, d2 in zip(offset1_list, offset2_list)]
for g, t_xy in zip(graph_list_, t_xy_list):
ut.translate_graph(g, t_xy)
nx.set_node_attributes(g, name='pin', values='true')
new_graph = nx.compose_all(graph_list_)
#pt.show_nx(new_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
return new_graph
# Construct graph
for count, info in enumerate(visit_info_list):
graph = nx.Graph()
edges = [(int(u), int(v)) for u, v in info['edges']
if u is not None and v is not None]
graph.add_edges_from(edges, attr_dict={'zorder': 10})
nx.set_node_attributes(graph, name='zorder', values=20)
# Layout in neato
_ = pt.nx_agraph_layout(graph, inplace=True, **layoutkw) # NOQA
# Extract components and then flatten in nid ordering
ccs = list(nx.connected_components(graph))
root_aids = []
cc_graphs = []
for cc_nodes in ccs:
cc = graph.subgraph(cc_nodes)
try:
root_aids.append(list(ut.nx_source_nodes(cc.to_directed()))[0])
except nx.NetworkXUnfeasible:
root_aids.append(list(cc.nodes())[0])
cc_graphs.append(cc)
root_nids = ibs.get_annot_nids(root_aids)
nid2_graph = dict(zip(root_nids, cc_graphs))
resight_nids_ = set(resight_nids).intersection(set(root_nids))
noresight_nids_ = set(root_nids) - resight_nids_
n_graph_list = ut.take(nid2_graph, sorted(noresight_nids_))
r_graph_list = ut.take(nid2_graph, sorted(resight_nids_))
if len(n_graph_list) > 0:
n_graph = nx.compose_all(n_graph_list)
_ = pt.nx_agraph_layout(n_graph, inplace=True, **layoutkw) # NOQA
n_graphs = [n_graph]
else:
n_graphs = []
r_graphs = [stack_graphs(chunk) for chunk in ut.ichunks(r_graph_list, 100)]
if count == 0:
new_graph = stack_graphs(n_graphs + r_graphs, vert=True)
else:
new_graph = stack_graphs(r_graphs[::-1] + n_graphs, vert=True)
#pt.show_nx(new_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
info['graph'] = new_graph
graph1_, graph2_ = ut.take_column(visit_info_list, 'graph')
if False:
_ = pt.show_nx(graph1_, layout='custom', node_labels=False, as_directed=False) # NOQA
_ = pt.show_nx(graph2_, layout='custom', node_labels=False, as_directed=False) # NOQA
graph_list = [graph1_, graph2_]
twoday_graph = stack_graphs(graph_list, vert=True, pad=None)
nx.set_node_attributes(twoday_graph, name='pin', values='true')
if debug:
ut.nx_delete_None_edge_attr(twoday_graph)
ut.nx_delete_None_node_attr(twoday_graph)
print('twoday_graph(pre) info' + ut.repr3(ut.graph_info(twoday_graph), nl=2))
# Hack, no idea why there are nodes that dont exist here
between_edges_ = [edge for edge in between_edges
if twoday_graph.has_node(edge[0]) and twoday_graph.has_node(edge[1])]
twoday_graph.add_edges_from(between_edges_, attr_dict={'alpha': .2, 'zorder': 0})
ut.nx_ensure_agraph_color(twoday_graph)
layoutkw['splines'] = 'line'
layoutkw['prog'] = 'neato'
agraph = pt.nx_agraph_layout(twoday_graph, inplace=True, return_agraph=True, **layoutkw)[-1] # NOQA
if False:
fpath = ut.truepath('~/ggr_graph.png')
agraph.draw(fpath)
ut.startfile(fpath)
if debug:
print('twoday_graph(post) info' + ut.repr3(ut.graph_info(twoday_graph)))
_ = pt.show_nx(twoday_graph, layout='custom', node_labels=False, as_directed=False) # NOQA
import networkx as nx
a = nx.Graph()
a.add_weighted_edges_from([(1, 2, 1), (2, 3, 1), (2, 4, 1), (3, 4, 1),
(3, 5, 1)])
b = nx.Graph()
b.add_weighted_edges_from([(1, 2, 1), (2, 3, 1), (1, 3, 1), (3, 4, 2)])
c = nx.Graph()
c.add_weighted_edges_from([(1, 2, 1), (2, 3, 1), (1, 3, 1)])
nx.set_node_attributes(a, 'atom', {1: 'N', 2: 'C', 3: 'C', 4: 'C', 5: 'O'})
nx.set_node_attributes(b, 'atom', {1: 'C', 2: 'C', 3: 'C', 4: 'O'})
nx.set_node_attributes(c, 'atom', {1: 'C', 2: 'C', 3: 'C'})
def ematch(G, H, eg, eh):
return G.node[eg[1]]['atom'] == H.node[eh[1]]['atom'] and G[eg[0]][
eg[1]]['weight'] == H[eh[0]][eh[1]]['weight']
def nmatch(G, H, ng, nh):
return G.node[ng]['atom'] == H.node[nh]['atom']
def n_uvisit_init(G):
return dict.fromkeys(G.nodes_iter(), False)
def e_uvisit_init(G):
return set(G.edges_iter())
a_n_uvisited = n_uvisit_init(a)
def network_graph(yearRange, AccountToSearch):
edge1 = pd.read_csv('edge1.csv')
node1 = pd.read_csv('node1.csv')
# filter the record by datetime, to enable interactive control through the input box
edge1['Datetime'] = "" # add empty Datetime column to edge1 dataframe
accountSet = set() # contain unique account
for index in range(0, len(edge1)):
edge1['Datetime'][index] = datetime.strptime(edge1['Date'][index],
'%d/%m/%Y')
if edge1['Datetime'][index].year < yearRange[0] or edge1['Datetime'][
index].year > yearRange[1]:
edge1.drop(axis=0, index=index, inplace=True)
continue
accountSet.add(edge1['Source'][index])
accountSet.add(edge1['Target'][index])
# to define the centric point of the networkx layout
shells = []
shell1 = []
shell1.append(AccountToSearch)
shells.append(shell1)
shell2 = []
for ele in accountSet:
if ele != AccountToSearch:
shell2.append(ele)
shells.append(shell2)
G = nx.from_pandas_edgelist(edge1,
'Source',
'Target',
['Source', 'Target', 'TransactionAmt', 'Date'],
create_using=nx.MultiDiGraph())
nx.set_node_attributes(
G,
node1.set_index('Account')['CustomerName'].to_dict(), 'CustomerName')
nx.set_node_attributes(G,
node1.set_index('Account')['Type'].to_dict(),
'Type')
# pos = nx.layout.spring_layout(G)
# pos = nx.layout.circular_layout(G)
# nx.layout.shell_layout only works for more than 3 nodes
if len(shell2) > 1:
pos = nx.drawing.layout.shell_layout(G, shells)
else:
pos = nx.drawing.layout.spring_layout(G)
for node in G.nodes:
G.nodes[node]['pos'] = list(pos[node])
if len(shell2) == 0:
traceRecode = [] # contains edge_trace, node_trace, middle_node_trace
node_trace = go.Scatter(x=tuple([1]),
y=tuple([1]),
text=tuple([str(AccountToSearch)]),
textposition="bottom center",
mode='markers+text',
marker={
'size': 50,
'color': 'LightSkyBlue'
})
traceRecode.append(node_trace)
node_trace1 = go.Scatter(x=tuple([1]),
y=tuple([1]),
mode='markers',
marker={
'size': 50,
'color': 'LightSkyBlue'
},
opacity=0)
traceRecode.append(node_trace1)
figure = {
"data":
traceRecode,
"layout":
go.Layout(title='Interactive Transaction Visualization',
showlegend=False,
margin={
'b': 40,
'l': 40,
'r': 40,
't': 40
},
xaxis={
'showgrid': False,
'zeroline': False,
'showticklabels': False
},
yaxis={
'showgrid': False,
'zeroline': False,
'showticklabels': False
},
height=600)
}
return figure
traceRecode = [] # contains edge_trace, node_trace, middle_node_trace
############################################################################################################################################################
colors = list(
Color('lightcoral').range_to(Color('darkred'), len(G.edges())))
colors = ['rgb' + str(x.rgb) for x in colors]
index = 0
for edge in G.edges:
x0, y0 = G.nodes[edge[0]]['pos']
x1, y1 = G.nodes[edge[1]]['pos']
weight = float(G.edges[edge]['TransactionAmt']) / max(
edge1['TransactionAmt']) * 10
trace = go.Scatter(x=tuple([x0, x1, None]),
y=tuple([y0, y1, None]),
mode='lines',
line={'width': weight},
marker=dict(color=colors[index]),
line_shape='spline',
opacity=1)
traceRecode.append(trace)
index = index + 1
###############################################################################################################################################################
node_trace = go.Scatter(x=[],
y=[],
hovertext=[],
text=[],
mode='markers+text',
textposition="bottom center",
hoverinfo="text",
marker={
'size': 50,
'color': 'LightSkyBlue'
})
index = 0
for node in G.nodes():
x, y = G.nodes[node]['pos']
hovertext = "CustomerName: " + str(
G.nodes[node]['CustomerName']) + "<br>" + "AccountType: " + str(
G.nodes[node]['Type'])
text = node1['Account'][index]
node_trace['x'] += tuple([x])
node_trace['y'] += tuple([y])
node_trace['hovertext'] += tuple([hovertext])
node_trace['text'] += tuple([text])
index = index + 1
traceRecode.append(node_trace)
################################################################################################################################################################
middle_hover_trace = go.Scatter(x=[],
y=[],
hovertext=[],
mode='markers',
hoverinfo="text",
marker={
'size': 20,
'color': 'LightSkyBlue'
},
opacity=0)
index = 0
for edge in G.edges:
x0, y0 = G.nodes[edge[0]]['pos']
x1, y1 = G.nodes[edge[1]]['pos']
hovertext = "From: " + str(
G.edges[edge]['Source']) + "<br>" + "To: " + str(
G.edges[edge]['Target']) + "<br>" + "TransactionAmt: " + str(
G.edges[edge]['TransactionAmt']
) + "<br>" + "TransactionDate: " + str(G.edges[edge]['Date'])
middle_hover_trace['x'] += tuple([(x0 + x1) / 2])
middle_hover_trace['y'] += tuple([(y0 + y1) / 2])
middle_hover_trace['hovertext'] += tuple([hovertext])
index = index + 1
traceRecode.append(middle_hover_trace)
#################################################################################################################################################################
figure = {
"data":
traceRecode,
"layout":
go.Layout(title='Interactive Transaction Visualization',
showlegend=False,
hovermode='closest',
margin={
'b': 40,
'l': 40,
'r': 40,
't': 40
},
xaxis={
'showgrid': False,
'zeroline': False,
'showticklabels': False
},
yaxis={
'showgrid': False,
'zeroline': False,
'showticklabels': False
},
height=600,
clickmode='event+select',
annotations=[
dict(ax=(G.nodes[edge[0]]['pos'][0] +
G.nodes[edge[1]]['pos'][0]) / 2,
ay=(G.nodes[edge[0]]['pos'][1] +
G.nodes[edge[1]]['pos'][1]) / 2,
axref='x',
ayref='y',
x=(G.nodes[edge[1]]['pos'][0] * 3 +
G.nodes[edge[0]]['pos'][0]) / 4,
y=(G.nodes[edge[1]]['pos'][1] * 3 +
G.nodes[edge[0]]['pos'][1]) / 4,
xref='x',
yref='y',
showarrow=True,
arrowhead=3,
arrowsize=4,
arrowwidth=1,
opacity=1) for edge in G.edges
])
}
return figure
for i in range(rownr):
if not i % 1000:
print(i)
mep = topicDF['name'].iloc[i]
topic = get_topics(ast.literal_eval(topicDF['topic'].iloc[i]))
for t in topic:
if t[0] not in [2, 3, 8, 11, 14] and t[1] > 0:
topics.add(t[0])
edge = (mep, t[0])
if edge in network.edges():
network[mep][t[0]]['weight'] += t[1]
else:
network.add_edge(mep, t[0], weight=t[1])
bp = dict((n, n in topics) for n in network.nodes())
nx.set_node_attributes(network, bp, 'bipartite')
top_nodes = [n for n, d in network.nodes(data=True) if d['bipartite'] == 1]
bottom_nodes = [n for n, d in network.nodes(data=True) if d['bipartite'] == 0]
#network = bipartite.generic_weighted_projected_graph(network,bottom_nodes,weight_function=weight_function)
w = [network[e[0]][e[1]]['weight'] for e in network.edges()]
thresh = min(w) + (max(w) - min(w)) * 0.5
print(thresh)
removeE = [
e for e in network.edges() if network[e[0]][e[1]]['weight'] < thresh
]
network.remove_edges_from(removeE)
removeN = [
node for node in network.nodes() if dict(network.degree())[node] == 0
def dependencies_detection(nb_graph: nx.DiGraph,
pipeline_parameters: dict = None,
imports_and_functions: str = ""):
"""Detect the data dependencies between nodes in the graph.
The data dependencies detection algorithm roughly works as follows:
1. Traversing the graph in topological order, for every node `step` do
2. Detect the `ins` of current `step` by running PyFlakes on the source
code. During this action the pipeline parameters are taken into
consideration
3. Parse `step`'s global function definitions to get free variables
(i.e. variables that would need to be marshalled in other steps that call
these functions) - in this action pipeline parameters are taken into
consideration.
4. Get all the function that `step` calls
5. For every `step`'s ancestor `anc` do
- Get all the potential names (objects, functions, ...) of `anc` that
could be marshalled (saved)
- Intersect this with the `step`'s `ins` (from action 2) and add the
result to `anc`'s `outs`.
- for every `step`'s function call (action 4), check if this function
was defined in `anc` and if it has free variables (action 3). If so,
add to `step`'s `ins` and to `anc`'s `outs` these free variables.
Args:
nb_graph: nx DiGraph with pipeline code blocks
pipeline_parameters: Pipeline parameters dict
imports_and_functions: Multiline Python source that is prepended to
every pipeline step
Returns: annotated graph
"""
# resolve the data dependencies between steps, looping through the graph
for step in nx.topological_sort(nb_graph):
step_data = nb_graph.nodes(data=True)[step]
# detect the INS dependencies of the CURRENT node----------------------
step_source_code = '\n'.join(step_data['source'])
# get the variables that this step is missing and the pipeline
# parameters that it actually needs.
ins, parameters = detect_in_dependencies(
source_code=step_source_code,
pipeline_parameters=pipeline_parameters)
fns_free_variables = detect_fns_free_variables(step_source_code,
imports_and_functions,
pipeline_parameters)
# Get all the function calls. This will be used below to check if any
# of the ancestors declare any of these functions. Is that is so, the
# free variables of those functions will have to be loaded.
fn_calls = kale_ast.get_function_calls(step_source_code)
# add OUT dependencies annotations in the PARENT nodes-----------------
# Intersect the missing names of this father's child with all
# the father's names. The intersection is the list of variables
# that the father need to serialize
# The ancestors are the the nodes that have a path to `step`, ordered
# by path length.
ins_left = ins.copy()
for anc in (graph_utils.get_ordered_ancestors(nb_graph, step)):
if not ins_left:
# if there are no more variables that need to be marshalled,
# stop the graph traverse
break
anc_data = nb_graph.nodes(data=True)[anc]
anc_source = '\n'.join(anc_data['source'])
# get all the marshal candidates from father's source and intersect
# with the required names of the current node
marshal_candidates = kale_ast.get_marshal_candidates(anc_source)
outs = ins_left.intersection(marshal_candidates)
# Remove the ins that have already been assigned to an ancestor.
ins_left.difference_update(outs)
# Include free variables
to_remove = set()
for fn_call in fn_calls:
anc_fns_free_vars = anc_data.get("fns_free_variables", {})
if fn_call in anc_fns_free_vars.keys():
# the current step needs to load these variables
fn_free_vars, used_params = anc_fns_free_vars[fn_call]
# search if this function calls other functions (i.e. if
# its free variables are found in the free variables dict)
_left = list(fn_free_vars)
while _left:
_cur = _left.pop(0)
# if the free var is itself a fn with free vars
if _cur in anc_fns_free_vars:
fn_free_vars.update(anc_fns_free_vars[_cur][0])
_left = _left + list(anc_fns_free_vars[_cur][0])
ins.update(fn_free_vars)
# the current ancestor needs to save these variables
outs.update(fn_free_vars)
# add the parameters used by the function to the list
# of pipeline parameters used by the step
for param in used_params:
parameters[param] = pipeline_parameters[param]
# Remove this function as it has been served. We don't want
# other ancestors to save free variables for this function.
# Using the helper to_remove because the set can not be
# resized during iteration.
to_remove.add(fn_call)
# add the function and its free variables to the current
# step as well. This is useful in case *another* function
# will call this one (`fn_call`) in a child step. In this
# way we can track the calls up to the last free variable.
# (refer to test `test_dependencies_detection_recursive`)
fns_free_variables[fn_call] = anc_fns_free_vars[fn_call]
fn_calls.difference_update(to_remove)
# Add to ancestor the new outs annotations. First merge the current
# outs present in the anc with the new ones
outs.update(anc_data.get('outs', []))
nx.set_node_attributes(nb_graph, {anc: {'outs': sorted(outs)}})
new_data = {
'ins': sorted(ins),
'fns_free_variables': fns_free_variables,
'parameters': parameters
}
nx.set_node_attributes(nb_graph, {step: new_data})
def untilhpathkernel(*args,
node_label='atom',
edge_label='bond_type',
depth=10,
k_func='tanimoto',
compute_method='trie',
n_jobs=None,
verbose=True):
"""Calculate path graph kernels up to depth/hight h between graphs.
Parameters
----------
Gn : List of NetworkX graph
List of graphs between which the kernels are calculated.
/
G1, G2 : NetworkX graphs
2 graphs between which the kernel is calculated.
node_label : string
Node attribute used as label. The default node label is atom.
edge_label : string
Edge attribute used as label. The default edge label is bond_type.
depth : integer
Depth of search. Longest length of paths.
k_func : function
A kernel function applied using different notions of fingerprint
similarity.
compute_method: string
Computation method, 'trie' or 'naive'.
Return
------
Kmatrix : Numpy matrix
Kernel matrix, each element of which is the path kernel up to h between
2 praphs.
"""
# pre-process
depth = int(depth)
Gn = args[0] if len(args) == 1 else [args[0], args[1]]
Kmatrix = np.zeros((len(Gn), len(Gn)))
ds_attrs = get_dataset_attributes(
Gn,
attr_names=['node_labeled', 'edge_labeled', 'is_directed'],
node_label=node_label,
edge_label=edge_label)
if not ds_attrs['node_labeled']:
for G in Gn:
nx.set_node_attributes(G, '0', 'atom')
if not ds_attrs['edge_labeled']:
for G in Gn:
nx.set_edge_attributes(G, '0', 'bond_type')
start_time = time.time()
# ---- use pool.imap_unordered to parallel and track progress. ----
# get all paths of all graphs before calculating kernels to save time,
# but this may cost a lot of memory for large datasets.
pool = Pool(n_jobs)
itr = zip(Gn, range(0, len(Gn)))
if len(Gn) < 100 * n_jobs:
chunksize = int(len(Gn) / n_jobs) + 1
else:
chunksize = 100
all_paths = [[] for _ in range(len(Gn))]
if compute_method == 'trie':
getps_partial = partial(wrapper_find_all_path_as_trie, depth, ds_attrs,
node_label, edge_label)
else:
getps_partial = partial(wrapper_find_all_paths_until_length, depth,
ds_attrs, node_label, edge_label)
if verbose:
iterator = tqdm(pool.imap_unordered(getps_partial, itr, chunksize),
desc='getting paths',
file=sys.stdout)
else:
iterator = pool.imap_unordered(getps_partial, itr, chunksize)
for i, ps in iterator:
all_paths[i] = ps
pool.close()
pool.join()
# for g in Gn:
# if compute_method == 'trie':
# find_all_path_as_trie(g, depth, ds_attrs, node_label, edge_label)
# else:
# find_all_paths_until_length(g, depth, ds_attrs, node_label, edge_label)
## size = sys.getsizeof(all_paths)
## for item in all_paths:
## size += sys.getsizeof(item)
## for pppps in item:
## size += sys.getsizeof(pppps)
## print(size)
#
## ttt = time.time()
## # ---- ---- use pool.map to parallel ----
## for i, ps in tqdm(
## pool.map(getps_partial, range(0, len(Gn))),
## desc='getting paths', file=sys.stdout):
## all_paths[i] = ps
## print(time.time() - ttt)
if compute_method == 'trie':
def init_worker(trie_toshare):
global G_trie
G_trie = trie_toshare
do_partial = partial(wrapper_uhpath_do_trie, k_func)
parallel_gm(do_partial,
Kmatrix,
Gn,
init_worker=init_worker,
glbv=(all_paths, ),
n_jobs=n_jobs,
verbose=verbose)
else:
def init_worker(plist_toshare):
global G_plist
G_plist = plist_toshare
do_partial = partial(wrapper_uhpath_do_naive, k_func)
parallel_gm(do_partial,
Kmatrix,
Gn,
init_worker=init_worker,
glbv=(all_paths, ),
n_jobs=n_jobs,
verbose=verbose)
# # ---- direct running, normally use single CPU core. ----
# all_paths = [
# find_all_paths_until_length(
# Gn[i],
# depth,
# ds_attrs,
# node_label=node_label,
# edge_label=edge_label) for i in tqdm(
# range(0, len(Gn)), desc='getting paths', file=sys.stdout)
# ]
#
# if compute_method == 'trie':
# pbar = tqdm(
# total=((len(Gn) + 1) * len(Gn) / 2),
# desc='calculating kernels',
# file=sys.stdout)
# for i in range(0, len(Gn)):
# for j in range(i, len(Gn)):
# Kmatrix[i][j] = _untilhpathkernel_do_trie(all_paths[i],
# all_paths[j], k_func)
# Kmatrix[j][i] = Kmatrix[i][j]
# pbar.update(1)
# else:
# pbar = tqdm(
# total=((len(Gn) + 1) * len(Gn) / 2),
# desc='calculating kernels',
# file=sys.stdout)
# for i in range(0, len(Gn)):
# for j in range(i, len(Gn)):
# Kmatrix[i][j] = _untilhpathkernel_do_naive(all_paths[i], all_paths[j],
# k_func)
# Kmatrix[j][i] = Kmatrix[i][j]
# pbar.update(1)
run_time = time.time() - start_time
if verbose:
print(
"\n --- kernel matrix of path kernel up to %d of size %d built in %s seconds ---"
% (depth, len(Gn), run_time))
# print(Kmatrix[0][0:10])
return Kmatrix, run_time
def setUpClass(cls):
cls.G = simple_weighted_graph()
cls.Gname = simple_weighted_graph()
nx.set_node_attributes(cls.Gname, {
0: 'a',
1: 'b',
2: 'c'
},
name='name')
cls.Gcentroids = simple_weighted_graph()
nx.set_node_attributes(cls.Gcentroids, {
0: (1, 0, 0),
1: (0, 1, 0),
2: (0, 0, 1)
},
name='centroids')
nx.set_node_attributes(cls.Gcentroids, {
0: '1',
1: '0',
2: '0'
},
name='x')
nx.set_node_attributes(cls.Gcentroids, {
0: '0',
1: '1',
2: '0'
},
name='y')
nx.set_node_attributes(cls.Gcentroids, {
0: '0',
1: '0',
2: '1'
},
name='z')
cls.H = simple_anatomical_graph()
cls.J = nx.Graph()
cls.J.add_nodes_from(cls.H.nodes)
mkg.copy_anatomical_data(cls.J, cls.H)
cls.K = simple_anatomical_graph()
cls.K.remove_edges_from(cls.H.edges)
cls.L = mkg.anatomical_copy(cls.H)
cls.R = mkg.anatomical_copy(cls.H)
nx.set_node_attributes(cls.R, 'stetson', name='hats')
def detect(self):
"""detect the source with GSBA.
Returns:
@rtype:int
the detected source
"""
self.reset_centrality()
self.prior_detector.set_data(self.data)
self.prior_detector.detect()
self.prior = nx.get_node_attributes(self.subgraph, 'centrality')
self.reset_centrality()
rc = rumor_center.RumorCenter()
rc.set_data(self.data)
rc.detect()
rumor_centralities = nx.get_node_attributes(self.subgraph,
'centrality')
self.reset_centrality()
infected_nodes = set(self.subgraph.nodes())
n = len(infected_nodes)
posterior = {}
included = set()
neighbours = set()
weights = self.data.weights
for v in infected_nodes:
"""find the approximate upper bound by greedy searching"""
included.clear()
neighbours.clear()
included.add(v)
neighbours.add(v)
likelihood = 1
w = {} # effective propagation probabilities: node->w
w_key_sorted = blist()
w[v] = 1
w_key_sorted.append(v)
while len(included) < n:
w_sum = sum([w[j] for j in neighbours])
u = w_key_sorted.pop(
) # pop out the last element from w_key_sorted with the largest w
likelihood *= w[u] / w_sum
included.add(u)
neighbours.remove(u)
new = nx.neighbors(self.data.graph, u)
for h in new:
if h in included:
continue
neighbours.add(h)
# compute w for h
w_h2u = weights[self.data.node2index[u],
self.data.node2index[h]]
# w_h2u = weights[self.data.node2index[u]][self.data.node2index[h]]
if h in w.keys():
w[h] = 1 - (1 - w[h]) * (1 - w_h2u)
else:
w[h] = w_h2u
# h_neighbor = nx.neighbors(self.data.graph, h)
# w_h = 1
# for be in included.intersection(h_neighbor):
# w_h *= 1 - self.data.get_weight(h, be)
# w[h] = 1 - w_h
"""insert h into w_key_sorted, ranking by w from small to large"""
if h in infected_nodes:
if h in w_key_sorted:
w_key_sorted.remove(h) # remove the old w[h]
k = 0
while k < len(w_key_sorted):
if w[w_key_sorted[k]] > w[h]:
break
k += 1
w_key_sorted.insert(k, h)
#w_key_sorted[k:k] = [h]
posterior[v] = (decimal.Decimal(self.prior[v]) *
decimal.Decimal(likelihood) *
rumor_centralities[v])
nx.set_node_attributes(self.subgraph, 'centrality', posterior)
return self.sort_nodes_by_centrality()
def build(self, savePickle=True, grabPickle=False):
startTime = time.time()
print 'building dig site graph'
a_star_heuristic = lambda a, b: np.linalg.norm(
np.array(a) - np.array(b))
if not grabPickle:
self.genMapFN = self.genMap(self.envFN)
else:
input = open('data_001_genMapFN.pkl', 'rb')
self.genMapFN = pickle.load(input)
input.close()
if savePickle:
output = open('data_001_genMapFN.pkl', 'wb')
pickle.dump(self.genMapFN, output, pickle.HIGHEST_PROTOCOL)
output.close()
self.pe = PolygonEnvironment()
self.pe.read_env(self.genMapFN)
if not grabPickle:
self.limits = self.pe.lims
self.map_limits = [[self.pe.x_min, self.pe.x_max],
[self.pe.y_min, self.pe.y_max]]
self.ranges = self.limits[:, 1] - self.limits[:, 0]
print '\tgenerating dig occupancy grid'
self.occupancyGrid = np.zeros(shape=(self.rows, self.cols),
dtype=np.uint)
self.obsNodes = []
self.startNodes = []
self.digNodes = []
self.dumpNodes = []
self.freeNodes = []
self.nodeTypes = {}
digOccStartTime = time.time()
for r in range(self.rows):
print '\t\trow {} of {}'.format(r, self.rows) #SOOOOOO SLOW
for c in range(self.cols):
if self.pe.test_collisions(
(r, c)) or (r == 0 or r == self.rows - 1 or c == 0 or c
== self.cols - 1): #REPLACE WITH BULLET
self.occupancyGrid[(r, c)] = cmn._OG_OBS #OR EDGE
self.obsNodes.append((r, c))
elif (r, c) == self.start:
self.occupancyGrid[(r, c)] = cmn._OG_START
self.startNodes.append((r, c))
self.freeNodes.append((r, c))
elif (r, c) in self.digSites:
self.occupancyGrid[(r, c)] = cmn._OG_DIG
self.digNodes.append((r, c))
self.freeNodes.append((r, c))
elif (r, c) in self.dumpSites:
self.occupancyGrid[(r, c)] = cmn._OG_DUMP
self.dumpNodes.append((r, c))
self.freeNodes.append((r, c))
else:
self.occupancyGrid[(r, c)] = cmn._OG_FREE
self.freeNodes.append((r, c))
self.nodeTypes.update({(r, c): self.occupancyGrid[(r, c)]})
print '\t\toccupancy grid took {} s'.format(time.time() -
digOccStartTime)
print '\tgenerating dig nx grid'
self.origBaseGrid = genFullyConGrid(maxr=self.rows,
maxc=self.cols,
rdim=cmn._sim_res)
nx.set_node_attributes(self.origBaseGrid, self.nodeTypes, 'type')
self.baseGrid = self.origBaseGrid.copy()
self.nodeSizes = []
self.nodeColors = []
self.edgeColors = []
self.edgeWidths = []
for node in self.origBaseGrid.nodes(data=True):
if node[1]['type'] == cmn._OG_FREE: # FREE SPACE
self.nodeSizes.append(freeNodeSize)
self.nodeColors.append(freeNodeColor)
if node[1]['type'] == cmn._OG_DIG: # DIG
self.nodeSizes.append(digNodeSize)
self.nodeColors.append(digNodeColor)
if node[1]['type'] == cmn._OG_OBS: # OBS
self.nodeSizes.append(obsNodeSize)
self.nodeColors.append(obsNodeColor)
if node[1]['type'] == cmn._OG_START: # START
self.nodeSizes.append(startNodeSize)
self.nodeColors.append(startNodeColor)
if node[1]['type'] == cmn._OG_DUMP: # Dump
self.nodeSizes.append(dumpNodeSize)
self.nodeColors.append(dumpNodeColor)
rmvEdgs = []
for edg in self.baseGrid.edges(data=True):
if (edg[0] in self.freeNodes and edg[1] in self.obsNodes) or \
(edg[1] in self.freeNodes and edg[0] in self.obsNodes):
rmvEdgs.append(edg)
self.baseGrid.remove_edges_from(rmvEdgs)
self.origBaseGrid.remove_edges_from(
rmvEdgs) # BOTH SHOULD BE SEPARATED
# COULD MERGE OBS NODES INTO ONE BUT WOULD
# REQUIRE DIFFERENT COLLISION CHECKING LOGIC BELOW
# The above does not find "cross-over" connections..
print '\tgenerating obstacle nx grid...'
obsGridStartTime = time.time()
self.obsGrid = self.baseGrid.subgraph(self.obsNodes).copy()
rmvEdgs = []
ndx = 0
obsSize = len(list(self.obsGrid.edges()))
edgeCheckSubStart = time.time()
# Skip crossover check for less accurate but MUCH faster execution. This just removes free connections that cross the line of an obstacle connection
checkCrossOver = False
if checkCrossOver:
for edg in self.obsGrid.edges(data=True):
if ndx % (int(np.ceil(obsSize * .01))) == 0:
print '\t\t{} of {} obs edges checked in {} S'.format(
ndx, obsSize,
time.time() - edgeCheckSubStart)
edgeCheckSubStart = time.time()
#np.any([(edg[0]==(r2,c2) or edg[1]==(r2,c2)) for ((r1,c1),(r2,c2)) in [cmn.getRCAfterAction(edg[0][0], edg[0][1], tmpAct) for tmpAct in cmn._actions]])
''' THIS IS SOOOO MUCH SLOWER.. THOUGHT I could look around node of interest rather than every node
for edgBase in [tmpedg for tmpedg in self.baseGrid.edges(data=True) if np.any([(edg[0]==(r2,c2) or edg[1]==(r2,c2)) for ((_,_),(r2,c2)) in [cmn.getRCAfterAction(edg[0][0], edg[0][1], tmpAct) for tmpAct in cmn._actions]])]:
'''
for edgBase in self.baseGrid.edges():
if not edgBase in self.obsGrid.edges():
if self.pe.line_line_collision(edg, edgBase):
rmvEdgs.append(edgBase)
ndx += 1
self.baseGrid.remove_edges_from(rmvEdgs)
for edg in self.origBaseGrid.edges():
if edg[0] in self.obsNodes or edg[1] in self.obsNodes:
self.edgeColors.append(obsEdgeColor)
self.edgeWidths.append(obsEdgeWidth)
else:
self.edgeColors.append(othrEdgeColor)
self.edgeWidths.append(othrEdgeWidth)
print '\t\tobstacle grid took {} s'.format(time.time() -
obsGridStartTime)
print '\tsaving initial base grid...'
#print self.origBaseGrid.nodes()
nx.draw_networkx(
self.origBaseGrid,
pos={k: np.array(k)
for k in self.origBaseGrid.nodes()},
node_size=self.nodeSizes,
node_color=self.nodeColors,
edge_color=self.edgeColors,
linewidths=self.edgeWidths,
with_labels=False)
plt.xlim((-1, self.cols))
plt.ylim((-1, self.rows))
fig = plt.gcf()
rc_ratio = float(self.rows) / self.cols
fig.set_size_inches(11, int(np.ceil(11 * rc_ratio)))
ax = plt.gca()
ax.set_facecolor('ghostwhite')
bbox_props = dict(boxstyle="square", fc="w", ec="0.7", alpha=0.5)
for digIdx in range(len(self.digNodes)):
tmpTxtPos = list(self.digNodes[digIdx])
tmpTxtPos[1] -= 1.5
ax.text(tmpTxtPos[0],
tmpTxtPos[1],
'Dig'.format(digIdx + 1),
ha="center",
va="center",
size=20,
bbox=bbox_props)
for startIdx in range(len(self.startNodes)):
tmpTxtPos = list(self.startNodes[startIdx])
tmpTxtPos[1] -= 1.5
ax.text(tmpTxtPos[0],
tmpTxtPos[1],
'Start',
ha="center",
va="center",
size=20,
bbox=bbox_props)
for dumpIdx in range(len(self.dumpNodes)):
tmpTxtPos = list(self.dumpNodes[dumpIdx])
tmpTxtPos[1] -= 1.5
ax.text(tmpTxtPos[0],
tmpTxtPos[1],
'Dump',
ha="center",
va="center",
size=20,
bbox=bbox_props)
plt.savefig(
os.path.join(
cmn._IMG_FLDR,
'dig_site_ordering_r{}_c{}_d{}_INIT.svg'.format(
self.rows, self.cols, len(self.digNodes))))
plt.draw()
plt.cla()
plt.clf()
plt.close()
print '\tgenerating free nx grid...'
self.freeGrid = self.baseGrid.subgraph(self.freeNodes).copy()
self.digGrid = self.baseGrid.subgraph(self.digNodes).copy()
else:
print '\topening pickles... '
sys.stdout.flush()
self.freeGrid = nx.read_gpickle('data_001_freeGrid.pkl')
self.digGrid = nx.read_gpickle('data_001_digGrid.pkl')
self.baseGrid = nx.read_gpickle('data_001_baseGrid.pkl')
self.origBaseGrid = nx.read_gpickle('data_001_origBaseGrid.pkl')
input = open('data_001.pkl', 'rb')
self.genMapFN = pickle.load(input)
self.limits = pickle.load(input)
self.map_limits = pickle.load(input)
self.ranges = pickle.load(input)
self.discreteObsMap = pickle.load(input)
self.rows = pickle.load(input)
self.cols = pickle.load(input)
self.nodeTypes = pickle.load(input)
self.occupancyGrid = pickle.load(input)
self.nodeSizes = pickle.load(input)
self.nodeColors = pickle.load(input)
self.edgeColors = pickle.load(input)
self.edgeWidths = pickle.load(input)
self.obsNodes = pickle.load(input)
self.startNodes = pickle.load(input)
self.digNodes = pickle.load(input)
self.dumpNodes = pickle.load(input)
self.freeNodes = pickle.load(input)
input.close()
if savePickle:
print '\tsaving pickles...'
sys.stdout.flush()
nx.write_gpickle(self.freeGrid, 'data_001_freeGrid.pkl')
nx.write_gpickle(self.digGrid, 'data_001_digGrid.pkl')
nx.write_gpickle(self.baseGrid, 'data_001_baseGrid.pkl')
nx.write_gpickle(self.origBaseGrid, 'data_001_origBaseGrid.pkl')
output = open('data_001.pkl', 'wb')
pickle.dump(self.genMapFN, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.limits, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.map_limits, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.ranges, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.discreteObsMap, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.rows, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.cols, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.nodeTypes, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.occupancyGrid, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.nodeSizes, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.nodeColors, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.edgeColors, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.edgeWidths, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.obsNodes, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.startNodes, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.digNodes, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.dumpNodes, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(self.freeNodes, output, pickle.HIGHEST_PROTOCOL)
output.close()
print '\tgrid generation took {} s...\n'.format(time.time() -
startTime)
digPathPlanningStartTime = time.time()
self.digOrderings = list(itertools.permutations(self.digNodes))
#self.digOrderings =[self.digNodes] # JUST ONCE
self.shortestRoverPathNodes = None
self.shortestRoverPathEdges = None
self.shortestRoverPathLength = 0.0
self.pathLengths = []
self.shortestDigOrder = []
allBroken = True
while allBroken:
print '\n\tFind dig paths for {} dig locations'.format(
len(self.digNodes))
for digOrderIdx in range(len(self.digOrderings)):
digOrder = self.digOrderings[digOrderIdx]
# print digOrder
print '\n\t\tChecking out combo {} of {}'.format(
digOrderIdx + 1, len(self.digOrderings))
startPos = self.startNodes[0]
currPos = startPos
dumpPos = self.dumpNodes[0]
roverPathNodes = []
roverPathEdges = []
MutatedBaseGrid = self.baseGrid.copy()
totalLength = 0.0
print '\t\tCalculating dig ',
sys.stdout.flush()
broken = False
for digIdx in range(len(digOrder)):
print '{} at {}.. '.format(digIdx + 1, digOrder[digIdx]),
sys.stdout.flush()
try:
tmpPath = nx.astar_path(MutatedBaseGrid,
source=currPos,
target=digOrder[digIdx],
heuristic=a_star_heuristic,
weight='r')
edgs = list(zip(tmpPath, tmpPath[1:]))
tmpLen = 0.0
for edg in edgs:
try:
tmpLen += nx.get_edge_attributes(
MutatedBaseGrid, 'r')[edg]
except:
pass
roverPathNodes.append(tmpPath)
roverPathEdges.append(edgs)
totalLength += tmpLen
currPos = digOrder[digIdx]
except:
# print '\t\tfailed for dig {} on {} --> {}\n{}\n'.format(digIdx+1,currPos,digOrder[digIdx],traceback.format_exc())
broken = True
break
try:
tmpPath = nx.astar_path(MutatedBaseGrid,
source=currPos,
target=dumpPos,
heuristic=a_star_heuristic,
weight='r')
edgs = list(zip(tmpPath, tmpPath[1:]))
tmpLen = 0.0
for edg in edgs:
try:
tmpLen += nx.get_edge_attributes(
MutatedBaseGrid, 'r')[edg]
except:
pass
roverPathNodes.append(tmpPath)
roverPathEdges.append(edgs)
totalLength += tmpLen
currPos = dumpPos
except:
# print '\t\tfailed for dump {} on {} --> {}\n{}\n'.format(digIdx+1,currPos,dumpPos,traceback.format_exc())
broken = True
break
MutatedBaseGrid.remove_node(digOrder[digIdx])
if broken: continue
self.pathLengths.append(totalLength)
if self.shortestRoverPathNodes is None:
self.shortestRoverPathNodes = copy.copy(roverPathNodes)
self.shortestRoverPathEdges = copy.copy(roverPathEdges)
self.shortestDigOrder = copy.copy(digOrder)
self.shortestRoverPathLength = totalLength
else:
if totalLength < self.shortestRoverPathLength:
self.shortestRoverPathNodes = copy.copy(roverPathNodes)
self.shortestRoverPathEdges = copy.copy(roverPathEdges)
self.shortestDigOrder = copy.copy(digOrder)
self.shortestRoverPathLength = totalLength
if self.shortestRoverPathNodes is None:
allBroken = True
print '\t\tFAILURE: Removing the furthest dig site... ',
sys.stdout.flush()
tmpLens = []
tmpDigs = []
for digPos in self.digOrderings[0]:
tmpPath = nx.astar_path(self.baseGrid,
source=self.startNodes[0],
target=digPos,
heuristic=a_star_heuristic,
weight='r')
edgs = list(zip(tmpPath, tmpPath[1:]))
tmpLen = 0.0
for edg in edgs:
try:
tmpLen += nx.get_edge_attributes(
self.baseGrid, 'r')[edg]
except:
pass
tmpLens.append(tmpLen)
tmpDigs.append(digPos)
maxIdx = np.nanargmax(tmpLens)
rmvNode = copy.copy(tmpDigs[maxIdx])
self.basegrid.remove_node(rmvNode)
self.digGrid.remove_node(rmvNode)
self.freeGrid.remove_node(rmvNode)
self.occupancyGrid[rmvNode] = cmn._OG_RMVD
self.digNodes.remove(rmvNode)
print 'removed {}'.format(rmvNode)
sys.stdout.flush()
else:
allBroken = False
print '\n\tPATH LENGTH STATS\n\n{}\n\n'.format(
stats.describe(self.pathLengths))
print '\tdig ordering took {} s...'.format(
time.time() - digPathPlanningStartTime)
print '\tfound dig order to be {}'.format(
self.shortestDigOrder)
print '\tplotting total path of {}...'.format(
self.shortestRoverPathLength)
nx.draw_networkx(
self.origBaseGrid,
pos={k: np.array(k)
for k in self.origBaseGrid.nodes()},
node_size=self.nodeSizes,
node_color=self.nodeColors,
edge_color=self.edgeColors,
linewidths=self.edgeWidths,
with_labels=False)
for i in range(len(self.shortestRoverPathNodes)):
try:
nx.draw_networkx_nodes(
self.origBaseGrid,
pos={
k: np.array(k)
for k in self.shortestRoverPathNodes[i]
},
nodelist=self.shortestRoverPathNodes[i],
node_color=[i] *
len(self.shortestRoverPathNodes[i]),
vmin=-5,
vmax=len(self.shortestRoverPathNodes) + 1,
cmap=pathColormap,
node_size=pathNodeSize)
except:
print 'skipping node {} in path due to\n{}\n'.format(
self.shortestRoverPathNodes[i],
traceback.format_exc())
for i in range(len(self.shortestRoverPathEdges)):
try:
nx.draw_networkx_edges(
self.origBaseGrid,
pos={
k: np.array(k)
for k in self.shortestRoverPathNodes[i]
},
edgelist=self.shortestRoverPathEdges[i],
width=pathEdgeWidth)
except:
print 'skipping edge {} in path due to\n{}\n'.format(
self.shortestRoverPathEdges[i],
traceback.format_exc())
nx.draw_networkx_nodes(
self.origBaseGrid,
pos={k: np.array(k)
for k in self.digNodes},
nodelist=self.digNodes,
node_color=digNodeColor,
node_size=digNodeSize)
nx.draw_networkx_nodes(
self.origBaseGrid,
pos={k: np.array(k)
for k in self.startNodes},
nodelist=self.startNodes,
node_color=startNodeColor,
node_size=startNodeSize)
nx.draw_networkx_nodes(
self.origBaseGrid,
pos={k: np.array(k)
for k in self.dumpNodes},
nodelist=self.dumpNodes,
node_color=dumpNodeColor,
node_size=dumpNodeSize)
plt.xlim((-1, self.rows))
plt.ylim((-1, self.cols))
fig = plt.gcf()
rc_ratio = float(self.rows) / self.cols
fig.set_size_inches(figWidth,
int(np.ceil(figWidth * rc_ratio)))
ax = plt.gca()
ax.set_facecolor(imgBGColor)
bbox_props = dict(boxstyle="square",
fc="w",
ec="0.7",
alpha=0.5)
for digIdx in range(len(self.shortestDigOrder)):
tmpTxtPos = list(self.shortestDigOrder[digIdx])
tmpTxtPos[1] -= 1.5
ax.text(tmpTxtPos[0],
tmpTxtPos[1],
'Dig {}'.format(digIdx + 1),
ha="center",
va="center",
size=20,
bbox=bbox_props)
for startIdx in range(len(self.startNodes)):
tmpTxtPos = list(self.startNodes[startIdx])
tmpTxtPos[1] -= 1.5
ax.text(tmpTxtPos[0],
tmpTxtPos[1],
'Start',
ha="center",
va="center",
size=20,
bbox=bbox_props)
for dumpIdx in range(len(self.dumpNodes)):
tmpTxtPos = list(self.dumpNodes[dumpIdx])
tmpTxtPos[1] -= 1.5
ax.text(tmpTxtPos[0],
tmpTxtPos[1],
'Dump',
ha="center",
va="center",
size=20,
bbox=bbox_props)
totTime = time.time() - startTime
print '\ttotal execution time: {} s\n'.format(totTime)
plt.savefig(
os.path.join(
cmn._IMG_FLDR,
'dig_site_ordering_r{}_c{}_d{}.svg'.format(
self.rows, self.cols, len(self.digNodes))))
plt.draw()
def generate_clique_mcf_problem(
n_nodes=50,
n_comm=10,
n_item_types=5,
max_degree=5,
prob_non_zero_inv=.1,
supply_demand_balance=.5,
max_item_inv=100,
max_comm=5,
avg_comm_per_node=2, # poisson lambda
edge_capacity=300 # we'll see if I end up using this. Don't want the system to be over complicated
):
"""
Generate an MCF problem on a graph of overlapping cliques.
"""
node_communities = {} #defaultdict(list)
community_nodes = defaultdict(list)
attr_d = defaultdict(Counter)
g_bipart = nx.Graph()
for n in range(n_nodes):
# determine community assignments
k = 0
while k == 0 or k > max_comm:
k = poisson.rvs(avg_comm_per_node, size=1)
comm = np.random.randint(n_comm, size=k)
node_communities[str(n)] = comm
for c in comm:
community_nodes[c].append(n)
g_bipart.add_edge(str(n), 'comm' + str(c))
# Add inventory
for i in range(n_item_types):
if random.random(
) > prob_non_zero_inv: # Does this node have non-zero inventory for this item?
continue
s = 1
if random.random(
) < supply_demand_balance: # What is the sign on the item's inventory?
s = -1
attr_d['item' +
str(i)][str(n)] = s * random.randint(1, max_item_inv)
g = bipartite.projected_graph(g_bipart, [str(n) for n in range(n_nodes)])
g_agents = bipartite.weighted_projected_graph(
g_bipart, [str(n) for n in range(n_nodes)])
g_comm = bipartite.weighted_projected_graph(
g_bipart, ['comm' + str(c) for c in range(n_comm)])
## Scavenged from generate_transport_problem()
##############
for _, d in g.nodes(data=True):
d['max_degree'] = max_degree
# Set all edges to default weight of '1'
for u, v, d in g.edges(data=True):
d['cost'] = 1 # I should play with this
for k, v in attr_d.items():
d = dict(v)
d.update({a: 0
for a in g.nodes() if a not in d.keys()
}) # Explicitly add node attributes for zero inventory nodes
nx.set_node_attributes(g, k, d)
##############
#nx.set_node_attributes(g, 'communities', node_communities)
return {
'community_skeleton': g_comm,
'agent_skeleton': g_agents,
'graph': g,
'node_communities': node_communities,
'community_nodes': community_nodes,
'attr_d': attr_d
}
def NLD_processing(df):
# get original column names from df
list_columns_names = df.columns
# change the column labels from strings to integers
list_columns_int = []
for number in range(0, len(df.index.values)):
list_columns_int.append(number)
df.columns = list_columns_int
# change the rows labels/ indexes from strings to integers
df['index'] = list_columns_int
df.set_index("index", inplace=True)
# Making a function to map color to edges
color_palette = list(reversed(Viridis11[:8]))
w_max = df.values.max()
w_min = df.values.min()
step = (w_max - w_min) / (len(color_palette) - 1)
colors = []
# Create a graph with 1-way edges for faster painting
g = nx.DiGraph()
for row in df.index.values:
g.add_node(row)
for column in df.index.values:
if row < column:
if (df[row][column] > 0):
color_index = int((df[row][column] - w_min) / step)
g.add_edge(row,
column,
weight=df[row][column],
color=color_palette[color_index])
colors.append(color_palette[color_index])
weights = []
# Create a separate graph with 2-way edges only to calculate weights
g_w = nx.DiGraph()
for row in df.index.values:
g_w.add_node(row)
for column in df.index.values:
if row != column:
if (df[row][column] > 0):
g_w.add_edge(row,
column,
weight=df[row][column],
color=color_palette[color_index])
weights.append(df[row][column])
# do not draw edges with different widths if the max weight is too big
if max(weights) > 30:
for index, w in enumerate(weights):
weights[index] = 1
# loop over all nodes to find neighbors and set min, max, sum for egdes weights connected to a node
node_w_dict = {}
for n in list_columns_int:
node_weight_list = []
for nb in nx.neighbors(g_w, n):
node_weight_list.append(
nx.get_edge_attributes(g_w, 'weight')[n, nb])
len_list = len(node_weight_list)
if len_list != 0:
node_min_weight = min(node_weight_list)
node_max_weight = max(node_weight_list)
node_sum_weight = sum(node_weight_list)
node_avr_weight = node_sum_weight / len_list
else:
node_min_weight = 0
node_max_weight = 0
node_sum_weight = 0
node_avr_weight = 0
node_w_dict.update({
n: {
'minweight': node_min_weight,
'maxweight': node_max_weight,
'avrweight': node_avr_weight,
'sumweight': node_sum_weight
}
})
nx.set_node_attributes(g, node_w_dict)
# Making a function to map node size
deg_node_size_list = [
5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30
]
deg_max = max(list(list(zip(*g.degree))[1]))
deg_min = min(list(list(zip(*g.degree))[1]))
deg_step = (deg_max - deg_min) / (len(deg_node_size_list) - 1)
i = 0
node_s_dict = {}
for node in list(list(zip(*g.degree))[0]):
deg_index = int(
(list(list(zip(*g.degree))[1])[i] - deg_min) / deg_step)
node_s_dict.update({node: {'nodesize': deg_node_size_list[deg_index]}})
i += 1
nx.set_node_attributes(g, node_s_dict)
# create a dictoinary with double for loop
mapping = {
old_label: new_label
for old_label, new_label in itertools.zip_longest(
sorted(g.nodes()), list_columns_names, fillvalue=1)
}
# relabel the names of the nodes from integers back to strings
nx.relabel_nodes(g, mapping, copy=False)
# Organize common layouts' size for NLD
NLD_width = 780
NLD_height = 690
color_mapper = LinearColorMapper(palette=color_palette,
low=w_min,
high=w_max)
color_bar = ColorBar(color_mapper=color_mapper,
border_line_color=None,
location=(0, 0))
# circular layout
plot_circle = Plot(plot_width=NLD_width,
plot_height=NLD_height,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
graph_circle = NLD_pocessing_graph(g, weights, colors, nx.circular_layout)
NLD_add_tools(plot_circle)
plot_circle.add_layout(color_bar, 'right')
plot_circle.renderers.append(graph_circle)
# spring layout
plot_spring = Plot(plot_width=NLD_width,
plot_height=NLD_height,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
graph_spring = NLD_pocessing_graph(g, weights, colors, nx.spring_layout)
NLD_add_tools(plot_spring)
plot_spring.add_layout(color_bar, 'right')
plot_spring.renderers.append(graph_spring)
# force-directed layout
plot_fd = Plot(plot_width=NLD_width,
plot_height=NLD_height,
x_range=Range1d(-1.1, 1.1),
y_range=Range1d(-1.1, 1.1))
graph_fd = NLD_FD_pocessing_graph(g, weights, colors)
NLD_add_tools(plot_fd)
plot_fd.add_layout(color_bar, 'right')
plot_fd.renderers.append(graph_fd)
# random layout
plot_random = Plot(plot_width=NLD_width,
plot_height=NLD_height,
x_range=Range1d(-0.1, 1.1),
y_range=Range1d(-0.1, 1.1))
graph_random = NLD_random_processing_graph(g, weights, colors,
nx.random_layout)
NLD_add_tools(plot_random)
plot_random.add_layout(color_bar, 'right')
plot_random.renderers.append(graph_random)
# Create panels for each layout
circle_panel = Panel(child=plot_circle, title='Circle layout')
spring_panel = Panel(child=plot_spring, title='Spring layout')
random_panel = Panel(child=plot_random, title='Random layout')
fd_panel = Panel(child=plot_fd, title='Force-Directed layout')
# Assign NLD panels to Tabs
tabsNLD_int = Tabs(
tabs=[circle_panel, spring_panel, fd_panel, random_panel])
return tabsNLD_int
def viz_netx_chipgraph(
ibs,
graph,
fnum=None,
use_image=False,
layout=None,
zoom=None,
prog='neato',
as_directed=False,
augment_graph=True,
layoutkw=None,
framewidth=True,
**kwargs,
):
r"""
DEPRICATE or improve
Args:
ibs (IBEISController): wbia controller object
graph (nx.DiGraph):
fnum (int): figure number(default = None)
use_image (bool): (default = False)
zoom (float): (default = 0.4)
Returns:
?: pos
CommandLine:
python -m wbia --tf viz_netx_chipgraph --show
Cand:
wbia review_tagged_joins --save figures4/mergecase.png --figsize=15,15
--clipwhite --diskshow
wbia compute_occurrence_groups --save figures4/occurgraph.png
--figsize=40,40 --clipwhite --diskshow
~/code/wbia/wbia/algo/preproc/preproc_occurrence.py
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.viz.viz_graph import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='PZ_MTEST')
>>> nid_list = ibs.get_valid_nids()[0:10]
>>> fnum = None
>>> use_image = True
>>> zoom = 0.4
>>> make_name_graph_interaction(ibs, nid_list, prog='neato')
>>> ut.show_if_requested()
"""
import wbia.plottool as pt
logger.info('[viz_graph] drawing chip graph')
fnum = pt.ensure_fnum(fnum)
pt.figure(fnum=fnum, pnum=(1, 1, 1))
ax = pt.gca()
if layout is None:
layout = 'agraph'
logger.info('layout = %r' % (layout, ))
if use_image:
ensure_node_images(ibs, graph)
nx.set_node_attributes(graph, name='shape', values='rect')
if layoutkw is None:
layoutkw = {}
layoutkw['prog'] = layoutkw.get('prog', prog)
layoutkw.update(kwargs)
if prog == 'neato':
graph = graph.to_undirected()
plotinfo = pt.show_nx(
graph,
ax=ax,
# img_dict=img_dict,
layout=layout,
# hacknonode=bool(use_image),
layoutkw=layoutkw,
as_directed=as_directed,
framewidth=framewidth,
)
return plotinfo
print(len(node_names))
print(edges)
print(len(edges))
G = nx.Graph()
G.add_nodes_from(node_names)
G.add_edges_from(edges)
print(nx.info(G))
id_dict = {}
for node in nodes: # Loop through the list, one row at a time
id_dict[node[0]] = node[1]
nx.set_node_attributes(G, id_dict, 'sdfb_id')
density = nx.density(G)
print("Network density:", density)
Sean_Eureka_path = nx.shortest_path(G, source="Sean Huang", target="Gia")
print("Shortest path between Fell and Whitehead:", Sean_Eureka_path)
print("Length of that path:", len(Sean_Eureka_path)-1)
print(nx.is_connected(G))
components = nx.connected_components(G)
largest_component = max(components, key=len)
subgraph = G.subgraph(largest_component)
def reset_random_placement(self):
d = self.get_random_placement()
for i, n in enumerate(self.nodes()):
self.node_embeddings[i].update_placement(d[n])
nx.set_node_attributes(self.G, d, 'placement')
return d
gyro_feature = np.squeeze(gyro_feature_array[indices_gyro]).tolist()
comp_feature = np.squeeze(comp_feature_array[indices_comp]).tolist()
graph_list = []
for i in range(len(all)):
nodes = ['HR', 'sound', 'step', 'gyro']
edges = combinations(nodes, 2)
g = nx.Graph()
g.add_nodes_from(nodes)
g.add_edges_from(edges)
attrs = {
'HR': {
"attr": HR_feature[i]
},
'sound': {
"attr": sound_feature[i]
},
'step': {
"attr": step_feature[i]
},
'gyro': {
"attr": gyro_feature[i]
},
'comp': {
"attr": comp_feature[i]
}
}
nx.set_node_attributes(g, attrs)
print(g.nodes['gyro']['attr'])
graph_list.append(g)
def init_zero_placement(self):
d = self.get_zero_placement()
for i, n in enumerate(self.nodes()):
self.node_embeddings[i].update_placement(0)
nx.set_node_attributes(self.G, d, 'placement')
def make_netx_graph_from_aid_groups(
ibs,
aids_list,
only_reviewed_matches=True,
invis_edges=None,
ensure_edges=None,
temp_nids=None,
allow_directed=False,
):
r"""
Args:
ibs (wbia.IBEISController): image analysis api
aids_list (list):
Example:
>>> # DISABLE_DOCTEST
>>> from wbia.viz.viz_graph import * # NOQA
>>> import wbia
>>> ibs = wbia.opendb(defaultdb='testdb1')
>>> aids_list = [[1, 2, 3, 4], [5, 6, 7]]
>>> invis_edges = [(1, 5)]
>>> only_reviewed_matches = True
>>> graph = make_netx_graph_from_aid_groups(ibs, aids_list,
>>> only_reviewed_matches,
>>> invis_edges)
>>> list(nx.connected_components(graph.to_undirected()))
"""
# aids_list, nid_list = ibs.group_annots_by_name(aid_list)
unique_aids = list(ut.flatten(aids_list))
# grouped version
unflat_edges = (list(itertools.product(aids, aids)) for aids in aids_list)
aid_pairs = [tup for tup in ut.iflatten(unflat_edges) if tup[0] != tup[1]]
aids1 = ut.get_list_column(aid_pairs, 0)
aids2 = ut.get_list_column(aid_pairs, 1)
if only_reviewed_matches:
annotmatch_rowids = ibs.get_annotmatch_rowid_from_undirected_superkey(
aids1, aids2)
annotmatch_rowids = ut.filter_Nones(annotmatch_rowids)
aids1 = ibs.get_annotmatch_aid1(annotmatch_rowids)
aids2 = ibs.get_annotmatch_aid2(annotmatch_rowids)
graph = make_netx_graph_from_aidpairs(ibs,
aids1,
aids2,
unique_aids=unique_aids)
if ensure_edges is not None:
if ensure_edges == 'all':
ensure_edges = list(ut.upper_diag_self_prodx(list(graph.nodes())))
ensure_edges_ = []
for edge in ensure_edges:
edge = tuple(edge)
redge = tuple(edge[::-1]) # HACK
if graph.has_edge(*edge):
ensure_edges_.append(edge)
pass
# nx.set_edge_attributes(graph, name='weight', values={edge: .001})
elif (not allow_directed) and graph.has_edge(*redge):
ensure_edges_.append(redge)
# nx.set_edge_attributes(graph, name='weight', values={redge: .001})
pass
else:
ensure_edges_.append(edge)
# graph.add_edge(*edge, weight=.001)
graph.add_edge(*edge)
if temp_nids is None:
unique_nids = ibs.get_annot_nids(list(graph.nodes()))
else:
# HACK
unique_nids = [1] * len(list(graph.nodes()))
# unique_nids = temp_nids
nx.set_node_attributes(graph,
name='nid',
value=ut.dzip(graph.nodes(), unique_nids))
import wbia.plottool as pt
ensure_names_are_connected(graph, aids_list)
# Color edges by nid
color_by_nids(graph, unique_nids=unique_nids)
if invis_edges:
for edge in invis_edges:
if graph.has_edge(*edge):
nx.set_edge_attributes(graph,
name='style',
values={edge: 'invis'})
nx.set_edge_attributes(graph,
name='invisible',
values={edge: True})
else:
graph.add_edge(*edge, style='invis', invisible=True)
# Hack color images orange
if ensure_edges:
nx.set_edge_attributes(
graph,
name='color',
values={tuple(edge): pt.ORANGE
for edge in ensure_edges_},
)
return graph
userposition = {}
deploymentservices = {}
for k in l:
cu = 0
cd = 0
for item in l[k]:
if "None" in item:
cu += 1
else:
cd += 1
deploymentservices[k] = cd
userposition[k] = cu
nx.set_node_attributes(s.topology.G,
values=deploymentservices,
name='services')
nx.set_node_attributes(s.topology.G,
values=userposition,
name='userposition')
# nx.write_gexf(s.topology.G, "network_assignments.gexf")
print selectorPath.dname
f = open(
selectorPath.dname + "/file_alloc_entities_%s_%i_%i.pkl" %
(case, stop_time, it), "wb")
pickle.dump(l, f)
f.close()
print "----"
controlServices = selectorPath.controlServices
def reset_placement(self, pl):
for i, n in enumerate(self.nodes()):
self.node_embeddings[i].update_placement(pl[n])
nx.set_node_attributes(self.G, pl, 'placement')
def greedy_coherence_graph_create(self, dag):
q = []
cohGraph = nx.DiGraph()
rev_dag = nx.reverse(dag, copy=True)
chain_list = []
num_chains = 0
#Add all nodes to priority queue
for node in dag.nodes():
#All nodes have initially 0 coherence as chains
#print "pushing node",node," to heap"
heappush(q, (-10, [node]))
num_chains += 1
while num_chains < self.threshold_num_chains:
if (len(q)) == 0:
#print "Out of choices to pop from heap"
break
best_choice = heappop(q)
#print best_choice, "is popped val"
best_chain = best_choice[1]
#print type(best_chain),"is type of chain of length ", len(best_chain)
# If chain is long enough, turn it into a vertex of G
if len(best_chain) >= self.m_chain_length:
chain_list.append(best_chain)
num_chains += 1
else:
#Generate all extensions of best_Choice and add to queue
#print "Generating all extensions of ",best_chain
#x=raw_input()
#Get last element of chain
last_ele = best_chain[-1]
#All extensions
num_ext = 0
for i, ext in enumerate(rev_dag.neighbors(last_ele)):
if (ext in best_chain):
continue
#print "Adding to chain",best_chain, " with ",ext
new_chain = best_chain[:]
new_chain.append(ext)
#print "Adding new chain as extension",new_chain
chain_coherence = self.calc_coherence(new_chain)
if (chain_coherence < -self.min_coherence):
heappush(q, (chain_coherence, new_chain))
num_ext += 1
#if (num_ext == 0):
#print "No more extensions possible"
#break
print "Done computing chains"
#Now convert nodes in coherence graph into paths, linking paths at all intersecting nodes
for chain in chain_list:
print str(chain), "coherence - ", self.calc_coherence(chain)
cohGraph.add_node(tuple(chain))
for chain in chain_list:
for other_chain in chain_list:
if (chain != other_chain):
if chain[1:] == other_chain[:-1]:
#print "Overlap",str(chain), " and ",other_chain
cohGraph.add_edge(tuple(chain), tuple(other_chain))
if other_chain[1:] == chain[:-1]:
#print "Overlap",chain, " and ",other_chain
cohGraph.add_edge(tuple(other_chain), tuple(chain))
print "Num overlaps (Edges)", len(
cohGraph.edges()), "num chains :", len(cohGraph.nodes())
#Make sure construction made no loops
assert (len(list(nx.simple_cycles(cohGraph))) == 0)
#Save copy to class variable
self.cohGraph = cohGraph
#Set node attribute
nx.set_node_attributes(self.cohGraph, 'walk-tally', 0)
def sub_graph_between_nodes(graph: nx.MultiDiGraph,
start_nodes: list,
end_nodes: list,
detect_extra_start_node: callable = None):
"""
Finds nodes of the sub-graph between 'start_nodes' and 'end_nodes'. Input nodes for the sub-graph nodes are also
added to the sub-graph. Constant inputs of the 'start_nodes' are also added to the sub-graph.
:param graph: graph to operate on.
:param start_nodes: list of nodes names that specifies start nodes.
:param end_nodes: list of nodes names that specifies end nodes.
:return: list of nodes of the identified sub-graph or None if the sub-graph cannot be extracted.
"""
sub_graph_nodes = list()
visited = set(start_nodes)
d = deque(start_nodes)
extra_start_nodes = []
nx.set_node_attributes(G=graph, name='prev', values=None)
while len(d) != 0:
cur_node_name = d.popleft()
sub_graph_nodes.append(cur_node_name)
if cur_node_name not in end_nodes: # do not add output nodes of the end_nodes
for _, dst_node_name in graph.out_edges(cur_node_name):
if dst_node_name not in visited:
d.append(dst_node_name)
visited.add(dst_node_name)
graph.node[dst_node_name]['prev'] = cur_node_name
for src_node_name, _ in graph.in_edges(cur_node_name):
# add input nodes for the non-start_nodes
if cur_node_name not in start_nodes and src_node_name not in visited:
if detect_extra_start_node is not None and detect_extra_start_node(
Node(graph, cur_node_name)):
extra_start_nodes.append(cur_node_name)
else:
d.append(src_node_name)
graph.node[src_node_name]['prev'] = cur_node_name
visited.add(src_node_name)
# use forward dfs to check that all end nodes are reachable from at least one of input nodes
forward_visited = set()
for start_node in start_nodes:
dfs(graph, start_node, forward_visited)
for end_node in end_nodes:
if end_node not in forward_visited:
raise Error('End node "{}" is not reachable from start nodes: {}. '
.format(end_node, start_nodes) + refer_to_faq_msg(74))
for node_name in sub_graph_nodes:
# sub-graph should not contain Placeholder nodes
if graph.node[node_name].get('op', '') == 'Placeholder':
path = list()
cur_node = node_name
while cur_node and 'prev' in graph.node[cur_node]:
path.append(str(cur_node))
cur_node = graph.node[cur_node]['prev']
log.debug("The path from input node is the following: {}".format(
'\n'.join(path)))
raise Error(
'The matched sub-graph contains network input node "{}". '.
format(node_name) + refer_to_faq_msg(75))
if detect_extra_start_node is None:
return sub_graph_nodes
else:
return sub_graph_nodes, extra_start_nodes
