def edge_threshold_sequence(G, min_wt, max_wt, winc):
import networkx as nx
"""
Accepts a (dense) graph and systematically redefine its
edges by edge-thresholding it in a loop of calls to
edge_threshold (above), the loop provided by a min, max,
and increment. Note (below) that the increment is added to
the max_wt to make sure max_wt is included in the range
(this is because of the way python does loops).
Returns a dictionary of graphs keyed by the threshold weights.
Usage: if G is a dense XGraph with edge weights <= 10000 m,
>>> Gts = edge_threshold_sequence(G,1000,10000,1000)
returns a dictionary of of 10 new graphs keyed by the numbers
1000-10000. To grab one:
>>> G4000 = Gts[4000]
DL Urban (22 Feb 2007)
"""
Gts = {}
nbunch = G.nodes()
edges = G.edges()
for wt in range(min_wt, max_wt+winc, winc):
tGw = nx.XGraph()
tGw.add_nodes_from(nbunch)
for e in edges:
(n1, n2, w) = e
if w <= wt:
tGw.add_edge(e)
Gts[wt] = tGw
return Gts
def run(data, prefix, count, iterCount, antAlg="reversePrice"): _workerCount = 4 _g = NX.XGraph() for _obj in data["nodes"]: _g.add_node(_obj["name"]) for _obj in data["edges"]: (_from , _to) = _obj["nodes"] _g.add_edge(_from, _to, _obj["price"]) _targets = [_o["node"] for _o in data["foods"]] _srcs = [_o["node"] for _o in data["lairs"]] assert len(_targets) == len(_srcs) == 1 _minPath = NX.dijkstra_path_length(_g, _targets[0], _srcs[0]) _workers = [] for _x in xrange(_workerCount): _outMask = prefix + ".worker" + str(_x) + ".iter%i" _worker = WorkerThread(copy.deepcopy(data), _minPath, count/_workerCount, _outMask, iterCount, antAlg) _workers.append(_worker) _startTime = time.time() [_t.start() for _t in _workers] _finished = False while not _finished: _finished = True for _th in _workers: if _th.isAlive(): _finished = False _th.join(1) [_t.join() for _t in _workers] print "Time elapsed: %s" % (time.time() - _startTime)
def stochastic(G, inplace=False):
"""Return a version of the weighted graph G converted to a (right)
stochastic representation. That is, make all of the weights for the
neighbors of a given node sum to 1.
If inplace=True the conversion is done in place - no copy of the graph is
made. This will destroy the original graph G.
"""
# this is a hack, better handling to come in networkx-0.36
if inplace:
W = G # copy, better be an XGraph
else:
if G.is_directed():
W = NX.XDiGraph(G) # make a new XDiGraph
else:
W = NX.XGraph(G) # make a new XGraph
for (u, v, d) in W.edges():
if d is None:
W.add_edge(u, v, 1.0)
# exposing graph/digraph internals here
for n in W:
print "W.adj[n].values():", W.adj[n].values()
deg = float(sum(W.adj[n].values()))
for p in W.adj[n]:
W.adj[n][p] /= deg
return W
def build_graph(self, pts, max_dist=maxint):
G = nx.XGraph()
for i in range(len(pts[0])):
for j in range(len(pts[0])):
if (self.map.is_visible(pts[:, i], pts[:, j])):
d_euclid = get_euclidean_distance(pts[:, i], pts[:, j])
if (d_euclid < max_dist):
G.add_edge((i, j, d_euclid))
return G
def network_graph(net, title, names=False, logodds_cutoff=0.):
"""
Draws a nice graph of all the genes with positive logodds for the relevant disease.
@param net: A Bayesnet object.
@param title: Titles of the graphs. Will be preappended with WTCCC and GBA.
@param names: Dictionary from the entrez gene ids used in the net to other names.
@param logodds_cutoff: Cutoff for logodds score needed to be drawn.
@return: Handles to the two figures.
"""
# Start out with the leery preds.
import networkx
import pylab
H = networkx.Graph()
edges = net.edges(
[node for node in net.nodes() if net.snps[node][1] > logodds_cutoff])
H.add_edges_from([(A, B) for (A, B, s) in edges])
# Remove stuff not in the candidate set.
for node in H.nodes():
try:
if net.snps[node][1] < logodds_cutoff:
H.delete_node(node)
except:
pass
if names is not False:
for node in H.nodes():
try:
if names[node] == '':
names[node] = node
except KeyError:
names[node] = node
f1, p = make_graph([(names[A], names[B]) for (A, B) in H.edges()])
else:
f1, p = make_graph(H.edges())
f1.suptitle('%s GBA Predictions (%s total)' % (title, len(H.nodes())))
H = networkx.XGraph()
edges = net.edges(
[node for node in net.nodes() if net.snps[node][0] > logodds_cutoff])
H.add_edges_from(edges)
# Remove stuff not in the candidate set.
for node in H.nodes():
try:
if net.snps[node][0] < logodds_cutoff:
H.delete_node(node)
except:
pass
if names is not False:
f2, p = make_graph([(names[A], names[B], s)
for (A, B, s) in H.edges()], p)
else:
f2, p = make_graph(H.edges(), p)
f2.suptitle('%s WTCCC Predictions (%s total)' % (title, len(H.nodes())))
return f1, f2
def construct_site_graph_out_of_ecotype_id_ls(ecotype_id_ls, ecotypeid2pos):
"""
2007-10-18
modified from construct_site_graph_out_of_strain_graph()
node weighted by the number of ecotype_id's
"""
sys.stderr.write("Constructing site graph out of ecotype_id_ls...")
site2pos = {}
site2weight = {}
import networkx as nx
site_g = nx.XGraph()
for e in ecotype_id_ls:
pos = (round(ecotypeid2pos[e][0], 2), round(ecotypeid2pos[e][1], 2))
site_g.add_node(pos)
if pos not in site2weight:
site2weight[pos] = 0
site2weight[pos] += 1
site2pos[pos] = pos
sys.stderr.write("Done.\n")
return site_g, site2weight, site2pos
def __init__(self, routers):
""" Constructor: pass the list of routers """
self.pickle_path = DATADIR + "network-model.pickle"
self.logfile = None # FileHandler(DATADIR + "proposals")
self.proposals = [] # Current list of path proposals
self.prefixes = {} # Prefixes for DFS
self.routers = routers # Link to the router-list
self.target_host = None
self.target_port = None
self.max_rtt = 0
try:
self.graph = self.load_graph()
self.up_to_date = False
except:
plog("INFO", "Could not load a model, creating a new one ..")
self.graph = networkx.XGraph(name="Tor Subnet")
self.graph.add_node(None)
self.up_to_date = True
self.print_info()
plog("INFO", "NetworkModel initiated")
def parse_pajek(lines):
"""Parse pajek format graph from string or iterable.."""
import shlex
if is_string_like(lines): lines = iter(lines.split('\n'))
lines = iter([line.rstrip('\n') for line in lines])
G = networkx.XDiGraph(selfloops=True) # are multiedges allowed in Pajek?
G.node_attr = {} # dictionary to hold node attributes
directed = True # assume this is a directed network for now
while lines:
try:
l = lines.next()
except: #EOF
break
if l.startswith("*network"):
label, name = l.split()
G.name = name
if l.startswith("*vertices"):
nodelabels = {}
l, nnodes = l.split()
for i in range(int(nnodes)):
splitline = shlex.split(lines.next())
id, label, x, y, shape = splitline[0:5]
G.add_node(label)
nodelabels[id] = label
G.node_attr[label] = {'id': id, 'x': x, 'y': y, 'shape': shape}
extra_attr = zip(splitline[5::2], splitline[6::2])
G.node_attr[label].update(extra_attr)
if l.startswith("*edges") or l.startswith("*arcs"):
if l.startswith("*edge"):
G = networkx.XGraph(G) # switch from digraph to graph
for l in lines:
splitline = shlex.split(l)
ui, vi, w = splitline[0:3]
u = nodelabels.get(ui, ui)
v = nodelabels.get(vi, vi)
edge_data = {'value': float(w)}
extra_attr = zip(splitline[3::2], splitline[4::2])
edge_data.update(extra_attr)
G.add_edge(u, v, edge_data)
return G
def edge_threshold(G, max_wt):
import networkx as nx
"""
Accepts a (dense) weighted graph (XGraph) and a threshold weight,
returnsa new graph with only edges for which the weight is
less than the threshold. The weights are general but this has been
designed for (and tested with) distances.
Usage: if G is a XGraph with edges < 5000 m,
>>> G2 = edge_threshold(G, 3000)
returns a new graph with edges < 3000 m.
DL Urban (22 Feb 2007)
"""
tG = nx.XGraph() # create an empty graph
nbunch = G.nodes()
tG.add_nodes_from(nbunch) # copy the nodes
for edge in G.edges():
(tn, fn, w) = edge
if w <= max_wt:
tG.add_edge(edge)
return tG
def __init__(self):
self.graph_ = networkx.XGraph()
self.distMat_ = {}
self.ptCnt_ = 0
return
in_network_txt = sys.argv[1] # the textfile that locates all the graph stuff
wt_min = sys.argv[2] # minimum edge weight
wt_max = sys.argv[3] # max weight
wt_inc = sys.argv[4] # increment for thresholding sequence (loop)
out_csv = sys.argv[5] # output CSV file (generates a plot, not an Arc object)
# initialize logging
log = '%s/log_thresholding.txt' % os.path.dirname(in_network_txt)
cm.log_init(log)
cm.log('Reading network configuration')
F = cm.read_network(in_network_txt) # read in full network
#nodes_shp = F.config[('shapefile', 'nodes')]
lcpaths_txt = F.config[('leastcostpaths', 'txt')]
G = NX.XGraph() # create fresh graph
cm.log('Reading least-cost paths into fresh graph with only centroids')
rdr = csv.reader(open(lcpaths_txt,'r'))
headers = rdr.next()
for row in rdr:
c1,c2,w = row[0:3]
G.add_edge(int(c1), int(c2), int(w))
# Process and write output:
cm.log('Creating sequence of thresholded networks')
Gts = edge_threshold_sequence(G, wt_min, wt_max, wt_inc)
cm.log('Finding NC and graph diameter of largest component for sequence')
Gcs = graph_comp_sequence(Gts)
cm.log('Writing output file')
def compareToBinary(complexSentencesById, classifications, exampleBuilder,
options):
# Load corpus and make sentence graphs
print >> sys.stderr, "Calculating performance on binary corpus"
classificationsBySentence = {}
for classification in classifications:
example = classification[0][0]
sentenceId = example[0].rsplit(".", 1)[0]
sentenceOrigId = complexSentencesById[sentenceId].sentence.attrib[
"origId"]
if not classificationsBySentence.has_key(sentenceOrigId):
classificationsBySentence[sentenceOrigId] = []
classificationsBySentence[sentenceOrigId].append(classification)
print >> sys.stderr, "Loading Binary corpus"
binaryCorpusElements = loadCorpus(options.binaryCorpus)
binaryClassifications = []
counter = ProgressCounter(len(binaryCorpusElements.sentences),
"Build binary classifications")
for binarySentence in binaryCorpusElements.sentences:
counter.update(
1, "Building binary classifications (" +
binarySentence.sentence.attrib["id"] + "): ")
if (classificationsBySentence.has_key(
binarySentence.sentence.attrib["origId"])):
complexClassificationGraph = NX.XGraph(multiedges=multiedges)
for token in binarySentence.sentenceGraph.tokens:
complexClassificationGraph.add_node(token)
for classification in classificationsBySentence[
binarySentence.sentence.attrib["origId"]]:
if classification[1] > 0:
example = classification[0][0]
t1 = example[3]["t1"]
t2 = example[3]["t2"]
t1Binary = None
for token in binarySentence.sentenceGraph.tokens:
if token.attrib["charOffset"] == t1.attrib[
"charOffset"]:
t1Binary = token
t2Binary = None
for token in binarySentence.sentenceGraph.tokens:
if token.attrib["charOffset"] == t2.attrib[
"charOffset"]:
t2Binary = token
assert (t1Binary != None and t2Binary != None)
complexClassificationGraph.add_edge(t1Binary, t2Binary)
paths = NX.all_pairs_shortest_path(complexClassificationGraph,
cutoff=999)
for pair in binarySentence.pairs:
t1 = binarySentence.sentenceGraph.entityHeadTokenByEntity[
pair.attrib["e1"]]
t2 = binarySentence.sentenceGraph.entityHeadTokenByEntity[
pair.attrib["e2"]]
assert (pair.attrib["interaction"] == "True"
or pair.attrib["interaction"] == "False")
if pair.attrib["interaction"] == "True":
pairClass = 1
else:
pairClass = -1
extra = {"xtype": "edge", "type": "i", "t1": t1, "t2": t2}
if paths.has_key(t1) and paths[t1].has_key(t2):
binaryClassifications.append(
[[pair.attrib["id"], pairClass, None, extra], 1,
"binary"])
else:
binaryClassifications.append(
[[pair.attrib["id"], pairClass, None, extra], -1,
"binary"])
print >> sys.stderr, "Evaluating binary classifications"
evaluation = Evaluation(predictions, classSet=exampleBuilder.classSet)
print >> sys.stderr, evaluation.toStringConcise()
if options.output != None:
evaluation.saveCSV(options.output + "/binary_comparison_results.csv")
def network_nexus(net, title, names={}, logodds_cutoff=0., boost_cutoff=1.):
"""
Draws a nice graph of all the genes with positive logodds for the relevant disease.
@param net: A Bayesnet object.
@param title: Titles of the graphs. Will be preappended with Leery and Naive.
@param names: Dictionary from the entrez gene ids used in the net to other names.
@param logodds_cutoff: Cutoff for logodds score needed to be drawn.
@param boost_cutoff: Cutoff for logodds improvement needed to be drawn.
@return: Handles to the two figures.
"""
import networkx
import pylab
all_nodes = networkx.XGraph()
edges = net.edges(
[node for node in net.nodes() if net.snps[node][1] > logodds_cutoff])
all_nodes.add_edges_from([(A, B, float(s)) for (A, B, s) in edges])
edges = net.edges([
node for node in net.nodes()
if (net.snps[node][1] - net.snps[node][0]) > boost_cutoff
])
all_nodes.add_edges_from([(A, B, float(s)) for (A, B, s) in edges])
# Remove stuff not in the candidate set.
for node in all_nodes.nodes():
try:
if net.snps[node][0] < logodds_cutoff and (
net.snps[node][1] - net.snps[node][0]) <= boost_cutoff:
all_nodes.delete_node(node)
except:
raise
all_nodes.size = {}
all_nodes.col = {}
dodds_hits = [] # Things that make both cutoffs
lodds_hits = [] # Things that only make the logodds cutoff
bodds_hits = [] # Things that only make the boost cutoff
for node in all_nodes.nodes():
try:
if len(all_nodes.edges(node)) == 0:
all_nodes.delete_node(node)
elif net.snps[node][0] >= logodds_cutoff and net.snps[node][
1] - net.snps[node][0] >= boost_cutoff:
dodds_hits.append(node)
elif net.snps[node][0] >= logodds_cutoff:
lodds_hits.append(node)
elif net.snps[node][1] - net.snps[node][0] >= boost_cutoff:
bodds_hits.append(node)
else:
raise Error
except:
raise
#pass
edges = all_nodes.edges()
# Trix with the names.
labels = {}
for node in all_nodes.nodes():
try:
if names[node] == '':
labels[node] = node
else:
labels[node] = names[node]
except KeyError:
labels[node] = node
# Now start drawing:
#f = pylab.figure(facecolor='slategrey')
#pylab.subplot(111, axisbg='slategrey')
for node in bodds_hits:
print '%s: %s\t%s' % (labels[node], net.snps[node][1] -
net.snps[node][0], net.snps[node][0])
f = pylab.figure()
#pos=networkx.graphviz_layout(all_nodes,prog="fdp")
pos = networkx.graphviz_layout(all_nodes, prog="neato")
#pos=networkx.graphviz_layout(all_nodes,prog="twopi",root=6934)
# draw nodes
# colors = [pylab.cm.hot(net.snps[v][1]-net.snps[v][0]) for v in all_nodes]
colors = dict([(v, pylab.cm.Blues(net.snps[v][1] - net.snps[v][0]))
for v in all_nodes.nodes()])
print len(pos)
networkx.draw_networkx_nodes(
all_nodes,
pos,
nodelist=dodds_hits,
node_size=[
1000 * pylab.sqrt(1 - logodds_cutoff + net.snps[v][0])
for v in lodds_hits
],
node_shape='h',
node_color=[colors[v] for v in dodds_hits],
vmin=0.0,
vmax=1.0,
alpha=0.9)
networkx.draw_networkx_nodes(
all_nodes,
pos,
nodelist=lodds_hits,
node_size=[
1000 * pylab.sqrt(1 - logodds_cutoff + net.snps[v][0])
for v in lodds_hits
],
node_color=[colors[v] for v in lodds_hits],
vmin=0.0,
vmax=1.0,
alpha=0.9)
networkx.draw_networkx_nodes(all_nodes,
pos,
nodelist=bodds_hits,
node_size=1000,
node_shape='d',
node_color=[colors[v] for v in bodds_hits],
vmin=0.0,
vmax=1.0,
alpha=0.9)
networkx.draw_networkx_labels(all_nodes,
pos,
labels=labels,
font_size=9,
font_color='r')
# draw edges of different thickness
networkx.draw_networkx_edges(all_nodes, pos, width=2)
# draw labels
pylab.xticks([])
pylab.yticks([])
f.suptitle('%s Predictions (%s total)' % (title, len(all_nodes.nodes())))
return f, all_nodes
def network_differences(net, title, names=False, logodds_cutoff=0.):
"""
Draws a nice graph of all the genes with positive logodds for the relevant disease.
@param net: A Bayesnet object.
@param title: Titles of the graphs. Will be preappended with Leery and Naive.
@param names: Dictionary from the entrez gene ids used in the net to other names.
@param logodds_cutoff: Cutoff for logodds score needed to be drawn.
@return: Handles to the two figures.
"""
import networkx
import pylab
all_nodes = networkx.XGraph()
edges = net.edges(
[node for node in net.nodes() if net.snps[node][1] > logodds_cutoff])
all_nodes.add_edges_from(edges)
# Remove stuff not in the candidate set.
for node in all_nodes.nodes():
try:
if net.snps[node][1] < logodds_cutoff:
all_nodes.delete_node(node)
except:
pass
for node in all_nodes.nodes():
try:
if len(all_nodes.edges(node)) == 0:
all_nodes.delete_node(node)
except:
pass
edges = all_nodes.edges()
# Boosted genes that would have made it anyway.
boost_nodes = networkx.XGraph()
boost_nodes.add_edges_from(edges)
# Genes that would not have made the cutoff except for the boost.
saved_nodes = networkx.XGraph()
saved_nodes.add_edges_from(edges)
# Unboosted genes that still made it.
raw_nodes = networkx.XGraph()
raw_nodes.add_edges_from(edges)
for node in all_nodes.nodes():
if net.snps[node][1] == net.snps[node][0]:
boost_nodes.delete_node(node)
saved_nodes.delete_node(node)
elif net.snps[node][0] > logodds_cutoff:
saved_nodes.delete_node(node)
raw_nodes.delete_node(node)
else:
boost_nodes.delete_node(node)
raw_nodes.delete_node(node)
# Trix with the names.
if names is not False:
for node in all_nodes.nodes():
try:
if names[node] == '':
names[node] = node
except KeyError:
names[node] = node
if names is not False:
an = networkx.XGraph()
an.add_edges_from([(names[A], names[B], float(s))
for (A, B, s) in all_nodes.edges()])
an.add_nodes_from([names[A] for A in all_nodes.nodes()])
sn = networkx.XGraph()
sn.add_edges_from([(names[A], names[B], float(s))
for (A, B, s) in saved_nodes.edges()])
sn.add_nodes_from([names[A] for A in saved_nodes.nodes()])
bn = networkx.XGraph()
bn.add_edges_from([(names[A], names[B], float(s))
for (A, B, s) in boost_nodes.edges()])
bn.add_nodes_from([names[A] for A in boost_nodes.nodes()])
rn = networkx.XGraph()
rn.add_edges_from([(names[A], names[B], float(s))
for (A, B, s) in raw_nodes.edges()])
rn.add_nodes_from([names[A] for A in raw_nodes.nodes()])
else:
an = networkx.XGraph()
an.add_edges_from([(A, B, float(s))
for (A, B, s) in all_nodes.edges()])
an.add_nodes_from([A for A in all_nodes.nodes()])
sn = networkx.XGraph()
sn.add_edges_from([(A, B, float(s))
for (A, B, s) in saved_nodes.edges()])
sn.add_nodes_from([A for A in saved_nodes.nodes()])
bn = networkx.XGraph()
bn.add_edges_from([(A, B, float(s))
for (A, B, s) in boost_nodes.edges()])
bn.add_nodes_from([A for A in boost_nodes.nodes()])
rn = networkx.XGraph()
rn.add_edges_from([(A, B, float(s))
for (A, B, s) in raw_nodes.edges()])
rn.add_nodes_from([A for A in raw_nodes.nodes()])
# Now start drawing:
f = pylab.figure()
pos = networkx.graphviz_layout(an, prog="neato")
# draw nodes
colors = ['yellowgreen', 'saddlebrown', 'slateblue', 'chartreuse']
for g in [rn, sn, bn]:
networkx.draw_networkx_nodes(g,
pos,
node_size=600,
node_color=colors.pop(),
vmin=0.0,
vmax=1.0,
alpha=0.8)
networkx.draw_networkx_labels(an, pos, font_size=8)
# draw edges of different thickness
networkx.draw_networkx_edges(an, pos, width=2)
# draw labels
pylab.xticks([])
pylab.yticks([])
f.suptitle('%s Predictions (%s total)' % (title, len(an.nodes())))
return f