def build_response(tree, races, root):
tree_data = json_graph.tree_data(tree, root=root)
resp = make_response(
render_template('sc2.html',
tree_data=tree_data,
race=races[0],
enemy_race=races[1]))
encoded_tree = encode_tree(tree)
encoded_data = json_graph.tree_data(encoded_tree, root=root)
resp.set_cookie('tree_data', json.dumps(encoded_data).replace(' ', ''))
resp.set_cookie('races', json.dumps(races))
return resp
def test_graph_attributes():
G = nx.DiGraph()
G.add_nodes_from([1, 2, 3], color="red")
G.add_edge(1, 2, foo=7)
G.add_edge(1, 3, foo=10)
G.add_edge(3, 4, foo=10)
H = tree_graph(tree_data(G, 1))
assert H.nodes[1]["color"] == "red"
d = json.dumps(tree_data(G, 1))
H = tree_graph(json.loads(d))
assert H.nodes[1]["color"] == "red"
def test_graph_attributes(self):
G = nx.DiGraph()
G.add_nodes_from([1, 2, 3], color='red')
G.add_edge(1, 2, foo=7)
G.add_edge(1, 3, foo=10)
G.add_edge(3, 4, foo=10)
H = tree_graph(tree_data(G, 1))
assert H.nodes[1]['color'] == 'red'
d = json.dumps(tree_data(G, 1))
H = tree_graph(json.loads(d))
assert H.nodes[1]['color'] == 'red'
def taxonomy_tree(abundance,filename,pipeline,analysis,dbname):
G=nx.DiGraph()
counts,tix=taxoLevels()
for i in abundance:
#i=i.split()
taxo=i[-1].replace("\n","").split(";") #last column is the taxonomy lineage
taxo=['R']+taxo
abundance=float(i[1])/2 #first column is the number of sites (unique sites)
matches=float(i[2])/2 #Second column is the number of matches
rpkm=float(i[3])/2
for tx in range(len(taxo)-1):
parent=taxo[tx].replace("unknown","unknown."+taxo[tx-1]).replace("uncultured","uncultured."+taxo[tx-1])
child=taxo[tx+1].replace("unknown","unknown."+taxo[tx]).replace("uncultured","uncultured."+taxo[tx])
#if "unknown" in parent or "unknown" in child: break
if not (parent,child) in G.edges():
if not child in G.nodes():
G.add_node(child,matches=matches,sites=abundance,rpkm=rpkm,level=tix[tx])
else:
G.node[child]['sites']+=abundance;G.node[child]['rpkm']+=rpkm; G.node[child]['matches'] +=matches;
if not parent in G.nodes():
G.add_node(parent,matches=matches,sites=abundance,rpkm=rpkm,level=tix[tx-1])
else:
G.node[parent]['sites'] +=abundance; G.node[parent]['rpkm']+=rpkm; G.node[parent]['matches'] +=matches;
if not G.predecessors(child):
G.add_edge(parent,child)
else:
G.node[parent]['sites']+=abundance;G.node[parent]['rpkm']+=rpkm; G.node[parent]['matches'] +=matches;
G.node[child]['sites'] +=abundance; G.node[child]['rpkm']+=rpkm; G.node[child]['matches'] +=matches;
try:
G.node['R']['rpkm']=2*G.node['R']['rpkm']
except:
G.add_node('R')
G.node['R']['rpkm']=1
G.node['R']['matches']=1
G.node['R']['sites']=1
G.node['R']['level']='Root'
# now take the different levels
#print G.node['R']['level']
#print G.edges('Bacteria')
if pipeline=="matches":
nx.write_gpickle(G,filename+"."+analysis+".abundance"+'.pk')
tree=json_graph.tree_data(G,root='R')
with open(filename+"."+analysis+".abundance"+'.json', 'w') as outfile:
json.dump(tree, outfile)
else:
nx.write_gpickle(G,filename+"."+analysis+".abundance"+'.pk')
tree=json_graph.tree_data(G,root='R')
with open(filename+"."+analysis+".abundance"+'.json', 'w') as outfile:
json.dump(tree, outfile)
return(G)
def test_attrs_deprecation():
G = nx.path_graph(3, create_using=nx.DiGraph)
# No warnings when `attrs` kwarg not used
with pytest.warns(None) as record:
data = tree_data(G, 0)
H = tree_graph(data)
assert len(record) == 0
# DeprecationWarning issued when `attrs` is used
attrs = {"id": "foo", "children": "bar"}
with pytest.warns(DeprecationWarning):
data = tree_data(G, 0, attrs=attrs)
with pytest.warns(DeprecationWarning):
H = tree_graph(data, attrs=attrs)
def createGraphJson(res):
g = nx.DiGraph()
edgeList = []
n = ""
d = ""
r = ""
if len(res) == 0:
return 1
r = res[0].name
g.add_node(r)
n = res[0].name
for i in res[0].relations:
d = i.destination
g.add_node(d)
edgeList = edgeList + [(n,d)]
g.add_edges_from(edgeList)
nx.draw(g)
pylab.show()
t = json_graph.tree_data(g, root = r)
x = str(t)
x=re.sub("id","name",x)
x=re.sub("\'","\"",x)
print x
return 0
def metaphlan_taxonomy_tree(filename):
with open(filename) as f:
content = f.readlines()
G=nx.DiGraph()
counts,tix=taxoLevels()
#add first node bacteria!
for i in content[2:-1]:
i=i.split()
taxo=['R']+i[0].split("|")
#print taxo
matches=int(i[4])
abundance=float(i[1])
G.add_node(taxo[-1], sites=abundance, matches=matches, rpkm=abundance, level=tix[len(taxo)-2])
for tx in range(0,len(taxo)-1):
parent=taxo[tx]
child=taxo[tx+1]
G.add_edge(parent,child)
try:
G.node['R']['rpkm']=G.node['k__Bacteria']['rpkm']
G.node['R']['matches']=G.node['k__Bacteria']['matches']
G.node['R']['sites']=G.node['k__Bacteria']['sites']
G.node['R']['level']='Root'
except:
G.add_node('R')
G.node['R']['rpkm']=1
G.node['R']['matches']=1
G.node['R']['sites']=1
G.node['R']['level']='Root'
tree=json_graph.tree_data(G,root='R')
with open(filename+".taxonomy.abundance.json", 'w') as outfile:
json.dump(tree, outfile)
nx.write_gpickle(G,filename+'.taxonomy.abundance.pk')
return G
def make_directed_json_graph_soc(gmt_fn, d_id_name, d_id_category, d_category_color, outfn=None):
# make directed graph based on SOC - PT
d_gmt = read_gmt(gmt_fn)
d_gmt_filt = {}
for term, genes in d_gmt.items():
if len(genes) >= 5:
d_gmt_filt[term] = genes
d_gmt = d_gmt_filt
print 'number of terms:', len(d_gmt)
umls_ids_kept = d_gmt.keys()
adj_matrix = jaccard_matrix(d_gmt)
m = adj_matrix > 0.2
adj_matrix = adj_matrix * m.astype(int)
Gu = nx.from_numpy_matrix(adj_matrix) # undirected Graph, to get size
G = nx.DiGraph()
for i in range(len(umls_ids_kept)):
umls_id = umls_ids_kept[i]
name = d_id_name[umls_id]
category = d_id_category[umls_id]
color = d_category_color[category]
G.add_edge('root', category)
G.add_edge(category, umls_id)
G.node[umls_id]['size'] = Gu.degree(i)
G.node[umls_id]['label'] = name
G.node[umls_id]['color'] = color
print G.number_of_nodes(), G.number_of_edges()
graph_data = json_graph.tree_data(G,root='root')
json.dump(graph_data, open(outfn, 'wb'))
return
def buildHierarchiesGraph(fileName):
"""
The main function which creats a graph for each country,
and then traverses the graphs for making a csv out of the graph information
"""
data = []
#Roots of each graph is the first division, here called "PROV"
roots = getSetOfName(fileName,"PROV")
graphs = []
for rs in roots:
g = nx.DiGraph()
g.add_node(1,label=rs)
graphs.append(g)
for g in graphs:
trav = list()
trav.append(''+ g.node[1]['label']+',')
cnt = 2
# to travese the grah and form individual paths rom the top most divisions to settlements
graphLevel(g,fileName,1,trav)
# Creates the JSON representation of the graph
data = json_graph.tree_data(g,root=1)
with io.open('../Data/tree'+g.node[1]['label']+'.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(data, ensure_ascii=False)))
def main():
beam_data = np.load(ARGS.data)
# Optionally load vocabulary data
if ARGS.vocab:
vocab_cls = vocab.Vocab(ARGS.vocab)
if not os.path.exists(ARGS.output_dir):
os.makedirs(ARGS.output_dir)
rel_script_dir = os.path.dirname(os.path.realpath(__file__))
beam_search_viz_dir = os.path.abspath(
os.path.join(rel_script_dir, "beam_search_viz"))
# Copy required files
shutil.copy2("{}/tree.css".format(beam_search_viz_dir), ARGS.output_dir)
shutil.copy2("{}/tree.js".format(beam_search_viz_dir), ARGS.output_dir)
for idx in range(len(beam_data["predicted_ids"])):
predicted_ids = beam_data["predicted_ids"][idx]
parent_ids = beam_data["beam_parent_ids"][idx]
scores = beam_data["scores"][idx]
graph = create_graph(predicted_ids=predicted_ids,
parent_ids=parent_ids,
scores=scores,
vocab=vocab_cls)
json_str = json.dumps(json_graph.tree_data(graph, (0, 0)),
ensure_ascii=False)
html_str = HTML_TEMPLATE.substitute(DATA=json_str)
output_path = os.path.join(ARGS.output_dir, "{:06d}.html".format(idx))
with open(output_path, "w") as file:
file.write(html_str)
print(output_path)
def buildHierarchiesGraph(fileName):
"""
Main functoin to build the graph and write it to a file.
It also creats a json representation of the hierarchies.
"""
global cnt
data = []
roots = getSetOfName(fileName,"PROV")
graphs = []
g = nx.DiGraph()
g.add_node(1,label="root")
for rs in roots:
trav=[]
g.add_edge(1,cnt, label = "child of root")
g.add_node(cnt,label=rs)
trav.append(''+ g.node[cnt]['label'])
#trav.append(lS[1])
cnt = cnt + 1
graphLevel(g, fileName, cnt-1, trav)
data = json_graph.tree_data(g,root=1)
with io.open('../Data/tree'+g.node[1]['label']+'.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(data, ensure_ascii=False)))
def generate(input_data, output_dir, include_pad=False):
path_base = os.path.dirname(os.path.realpath(__file__))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Copy required files
shutil.copy2(path_base+"/templates/tree.css", output_dir)
shutil.copy2(path_base+"/templates/tree.js", output_dir)
with open(input_data) as beams:
for i, line in enumerate(beams):
beam = json.loads(line)
graph = create_graph(predicted_ids=beam["predicted_tokens"],
parent_ids=beam["parent_ids"],
scores=beam["scores"],
normalized_scores=beam["normalized_scores"],
include_pad=include_pad)
json_str = json.dumps(
json_graph.tree_data(graph, (0, 0)),
ensure_ascii=True)
html_str = HTML_TEMPLATE.substitute(DATA=json_str, SENT=str(i))
output_path = os.path.join(output_dir, "{:06d}.html".format(i))
with open(output_path, "w", encoding="utf-8") as out:
out.write(html_str)
print("Output beams written to: {}".format(output_dir))
def tree_data_json(asn):
H = flare_tree_as_json_for_asn(asn)
try:
json_tree = json_graph.tree_data(H, root=asn)
return json.dumps(json_tree)
except TypeError:
return json.dumps({})
def as_tree(graph, root=OPENSTACK_CLUSTER, reverse=False):
linked_graph = json_graph.node_link_graph(graph)
if 0 == nx.number_of_nodes(linked_graph):
return {}
if reverse:
linked_graph = linked_graph.reverse()
return json_graph.tree_data(linked_graph, root=root)
def as_tree(graph, root=OPENSTACK_CLUSTER, reverse=False):
linked_graph = json_graph.node_link_graph(graph)
if 0 == nx.number_of_nodes(linked_graph):
return {}
if reverse:
linked_graph = linked_graph.reverse()
return json_graph.tree_data(linked_graph, root=root)
def generate(input_data, output_dir, include_pad=False):
path_base = os.path.dirname(os.path.realpath(__file__))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Copy required files
shutil.copy2(path_base + "/templates/tree.css", output_dir)
shutil.copy2(path_base + "/templates/tree.js", output_dir)
with open(input_data) as beams:
for i, line in enumerate(beams):
beam = json.loads(line)
graph = create_graph(predicted_ids=beam["predicted_tokens"],
parent_ids=beam["parent_ids"],
scores=beam["scores"],
normalized_scores=beam["normalized_scores"],
include_pad=include_pad)
json_str = json.dumps(json_graph.tree_data(graph, (0, 0)),
ensure_ascii=True)
html_str = HTML_TEMPLATE.substitute(DATA=json_str, SENT=str(i))
output_path = os.path.join(output_dir, "{:06d}.html".format(i))
with open(output_path, "w", encoding="utf-8") as out:
out.write(html_str)
print("Output beams written to: {}".format(output_dir))
def tree_data_json(asn):
H = flare_tree_as_json_for_asn(asn)
try:
json_tree = json_graph.tree_data(H, root=asn)
return json.dumps(json_tree)
except TypeError:
return json.dumps({})
def main():
beam_data = json.load(open(ARGS.data, 'r'))
# Optionally load vocabulary data
vocab = None
if not os.path.exists(ARGS.output_dir):
os.makedirs(ARGS.output_dir)
path_base = os.path.dirname(os.path.realpath(__file__))
# Copy required files
shutil.copy2(path_base + "/beam_search_viz/tree.css", ARGS.output_dir)
shutil.copy2(path_base + "/beam_search_viz/tree.js", ARGS.output_dir)
for idx in range(len(beam_data["predicted_ids"])):
predicted_ids = beam_data["predicted_ids"][idx]
parent_ids = beam_data["beam_parent_ids"][idx]
scores = beam_data["scores"][idx]
graph = create_graph(predicted_ids=predicted_ids,
parent_ids=parent_ids,
scores=scores,
vocab=vocab)
json_str = json.dumps(json_graph.tree_data(graph, (0, 0)),
ensure_ascii=True)
html_str = HTML_TEMPLATE.substitute(DATA=json_str)
output_path = os.path.join(ARGS.output_dir, "{:06d}.html".format(idx))
with open(output_path, "w") as file:
file.write(html_str)
print(output_path)
def createGraphJson(res):
g = nx.DiGraph()
edgeList = []
n = ""
d = ""
r = ""
if len(res) == 0:
return 1
r = res[0].name
g.add_node(r)
n = res[0].name
for i in res[0].relations:
d = i.destination
g.add_node(d)
edgeList = edgeList + [(n, d)]
g.add_edges_from(edgeList)
nx.draw(g)
pylab.show()
t = json_graph.tree_data(g, root=r)
x = str(t)
x = re.sub("id", "name", x)
x = re.sub("\'", "\"", x)
print x
return 0
def build_sunburst(sequences):
from djangophylocore.models import TaxonomyReference
import networkx as nx
from networkx.readwrite import json_graph
scores = sequences.values_list("score", flat=True)
scores_min = min(scores) if scores else 0
scores_max = max(scores) if scores else 100
green = Color("#66c2a5")
red = Color("#fc8d62")
color_range = list(red.range_to(green, 100))
def get_color_for_taxa(taxon):
avg_score = taxon.children.filter(sequence__all_model_scores__used_for_classification=True).aggregate(score=Avg("sequence__all_model_scores__score"))["score"]
avg_score = avg_score if avg_score else scores_min
scaled = int(floor((float(avg_score-scores_min)/float(scores_max-scores_min))*100))
color_index = scaled if scaled <= 99 else 99
color_index = color_index if color_index >= 0 else 0
return str(color_range[color_index])
taxa = list(sequences.values_list("taxonomy__parent__parent__parent", flat=True).distinct())
allow_ranks = ["kingdom", "phylum", "order"]
tree = TaxonomyReference().get_filtered_reference_graph(taxa, allow_ranks=allow_ranks)
nx.set_node_attributes(tree, "colour", {n:get_color_for_taxa(Taxonomy.objects.get(name=n)) for n,d in tree.out_degree_iter() if d==0})
return json_graph.tree_data(tree, "root", attrs={'children': 'children', 'id': 'name'})
def save_profile(Tree, args):
'''
Save the profile, after aggregation, to files. One for white and one for black.
'''
for c in range(len(Tree)):
setattr(args, 'colour', args.colours[c])
if args.v:
print('The', args.colour, 'tree has', Tree[c].number_of_nodes(),
'nodes.')
print('The', args.colour, 'tree has', Tree[c].number_of_edges(),
'edges.')
if args.tree:
# for a tree format and the graph needs to be created with DiGraph
data = json_graph.tree_data(Tree[c], root=args.root)
else:
Tree[c] = nx.convert_node_labels_to_integers(Tree[c],
first_label=0,
label_attribute='fen')
# name is changed to: "(Carlsen (black))_with_int_labels"
Tree[c].name = ''.join([args.player, ' (', args.colour, ')'
]) # above convert also converts the name
# for a Force directed graph
data = json_graph.node_link_data(Tree[c])
s = json.dumps(data)
js = open(profile_file_name(args), 'w')
js.write(s)
js.close()
def mytaxa_taxonomy_tree(data,filename):
G=nx.DiGraph()
counts,tix=taxoLevels()
for lineage in data:
taxo=['R']+lineage.split(";")
for tx in range(len(taxo)-1):
parent=taxo[tx]
child=taxo[tx+1]
matches=data[lineage]['genes']
abundance=data[lineage]['scaffold']
#if "unknown" in parent or "unknown" in child: break
if not (parent,child) in G.edges():
if not child in G.nodes():
G.add_node(child,matches=matches,sites=abundance,rpkm=1*float(matches),level=tix[tx])
else:
G.node[child]['sites']+=abundance;G.node[child]['rpkm']+=1*float(matches); G.node[child]['matches'] +=matches;
if not parent in G.nodes():
G.add_node(parent,matches=matches,sites=abundance,rpkm=1*float(matches),level=tix[tx-1])
else:
G.node[parent]['sites'] +=abundance; G.node[parent]['rpkm']+=1*float(matches); G.node[parent]['matches'] +=matches;
if not G.predecessors(child):
G.add_edge(parent,child)
else:
G.node[parent]['sites']+=abundance;G.node[parent]['rpkm']+=1*float(matches); G.node[parent]['matches'] +=matches;
G.node[child]['sites'] +=abundance; G.node[child]['rpkm']+=1*float(matches); G.node[child]['matches'] +=matches;
G.node['R']['rpkm']=2*G.node['R']['rpkm']
tree=json_graph.tree_data(G,root='R')
with open(filename+".json", 'w') as outfile:
json.dump(tree, outfile)
nx.write_gpickle(G,filename+'.pk')
return G
def _get_tree(graph, root_node, fmt):
tree = tree_from_dag(graph, root_node, False) # tree :: nx.DiGraph
if fmt:
size = tree_size(tree) if compute_size else tree.size()
tree = JS.tree_data(tree, root_node) # tree :: ! nx.DiGraph
tree["size"] = size
return tree
def as_tree(graph, root=OPENSTACK_CLUSTER, reverse=False):
if nx.__version__ >= '2.0':
linked_graph = json_graph.node_link_graph(
graph, attrs={'name': 'graph_index'})
else:
linked_graph = json_graph.node_link_graph(graph)
if 0 == nx.number_of_nodes(linked_graph):
return {}
if reverse:
linked_graph = linked_graph.reverse()
if nx.__version__ >= '2.0':
return json_graph.tree_data(
linked_graph,
root=root,
attrs={'id': 'graph_index', 'children': 'children'})
else:
return json_graph.tree_data(linked_graph, root=root)
def test_graph():
G = nx.DiGraph()
G.add_nodes_from([1, 2, 3], color="red")
G.add_edge(1, 2, foo=7)
G.add_edge(1, 3, foo=10)
G.add_edge(3, 4, foo=10)
H = tree_graph(tree_data(G, 1))
nx.is_isomorphic(G, H)
def _get_tree(graph, root_node, fmt):
tree = tree_from_dag(graph, root_node, False) # tree :: nx.DiGraph
if fmt:
size = tree_size(tree) if compute_size else tree.size()
tree = JS.tree_data(tree, root_node) # tree :: ! nx.DiGraph
tree["size"] = size
return tree
def main():
if not os.path.exists(ARGS.output_dir):
os.makedirs(ARGS.output_dir)
path_base = os.path.dirname(os.path.realpath(__file__))
# Copy required files
shutil.copy2(path_base + "/beam_search_viz/tree.css", ARGS.output_dir)
shutil.copy2(path_base + "/beam_search_viz/tree.js", ARGS.output_dir)
shutil.copy2(path_base + "/beam_search_viz/d3.v3.min.js", ARGS.output_dir)
with open(ARGS.data, 'r') as file:
idx = 0
for line in file:
logging.info("Process sentence %d" % idx)
beam_data = json.loads(line)
predicted_ids = beam_data["predicted_ids"]
parent_ids = beam_data["parent_ids"]
predicted_tokens = beam_data["predicted_tokens"]
scores = beam_data["scores"]
model_scores = beam_data[
"model_scores"] if "model_scores" in beam_data else None
alignment = beam_data[
"alignment"] if "alignment" in beam_data else None
graph, min_node = create_graph(predicted_ids=predicted_ids,
parent_ids=parent_ids,
scores=scores,
predicted_tokens=predicted_tokens,
model_scores=model_scores,
alignment=alignment)
translation = []
while min_node is not None:
graph.node[min_node]["best_path"] = True
translation.insert(0,
html.escape(graph.node[min_node]["name"]))
if min_node == (0, 0):
min_node = None
else:
min_node = list(graph.in_edges(min_node))[0][0]
json_str = json.dumps(json_graph.tree_data(graph, (0, 0)),
ensure_ascii=True)
html_str = HTML_TEMPLATE.substitute(
SENTENCE=" ".join(translation).replace("@@ ", ""),
DATA=json_str)
output_path = os.path.join(ARGS.output_dir,
"{:06d}.html".format(idx))
with open(output_path, "w") as file:
file.write(html_str)
logging.info("write output to \t%s" % output_path)
idx += 1
def sunburst(in_df, outname='sun_tree.json'):
# in_df has as indexes the pathway reactome ID, 'value' as the colors to be plot, 'ngenes' as the width, and 'descr' as the description of the pathway
in_df = in_df.loc[[x for x in in_df.index if 'HSA' in x]]
topPaths = rel_df.loc[(rel_df['parentId'] == 'HomoSapiens'), 'id']
homoNgenes = np.sum(
in_df.loc[[x in topPaths.tolist() for x in in_df.index], 'ngenes'])
homoNode = pd.DataFrame([[1, homoNgenes, "H**o Sapiens"]],
columns=["q", "ngenes", "description"]).xs(0)
homoNode.name = 'HomoSapiens'
in_df = in_df.append(homoNode)
topDict = in_df.to_dict()
pathways = in_df.index
n_path = len(pathways)
subset_vec = [x in pathways for x in rel_df.iloc[:, 0]
] and [x in pathways for x in rel_df.iloc[:, 1]]
sub_rel_df = rel_df[subset_vec]
G = nx.DiGraph()
G.add_nodes_from(pathways)
G.add_edges_from(sub_rel_df.values)
tree = nx.algorithms.dag.dag_to_branching(G)
secondDict = nx.get_node_attributes(tree, 'source')
thirdDict = {
'value': {},
'ngenes': {},
'description': {},
'timelineHigh': {},
'timelineLow': {},
'high': {},
'low': {}
}
for key, value in secondDict.items():
thirdDict['value'].update({key: topDict['q'][value]})
thirdDict['ngenes'].update({key: topDict['ngenes'][value]})
thirdDict['description'].update({key: topDict['description'][value]})
nx.set_node_attributes(tree, thirdDict['value'], name='value')
nx.set_node_attributes(tree, thirdDict['ngenes'], name='ngenes')
nx.set_node_attributes(tree, thirdDict['description'], name='description')
root = [v for v, d in tree.in_degree() if d == 0][0]
out_json = json_graph.tree_data(tree, root)
with open(outname, 'w') as outfile:
json.dump(out_json, outfile, default=default)
def saveJSON(self, concepts):
fileOutput = u"testData.json"
d0 = json_graph.tree_data(self.G, root=concepts.name)
with open(u"testData.json", "w") as f:
f.write(json.dumps(d0))
logger.info(u"Saved : %s" % fileOutput)
def sunburst(infile):
"""generate hierarchical json file"""
in_df = pd.read_csv(infile, sep="\t")
in_df.set_index(in_df["pathways"], inplace=True)
#set up 'pathways' colum as index
in_df = in_df.loc[[x for x in in_df.index if 'HSA' in x]]
topPaths = rel_df.loc[(rel_df['parentId'] == 'HomoSapiens'), 'id']
homoNgenes = np.sum(
in_df.loc[[x in topPaths.tolist() for x in in_df["pathways"]],
'ngenes'])
homoNode = pd.DataFrame([["H**o Sapiens", 1, homoNgenes]],
columns=["pathways", "explained_ratios",
"ngenes"]).xs(0)
homoNode.name = 'HomoSapiens'
# add up HomoSapiens info into matrix file (info include:"pathways", "explained_ratios", "ngenes"), and set which as homenode
in_df = in_df.append(homoNode)
topDict = in_df.to_dict() #set up dictionary as the the key for node tree
pathways = in_df["pathways"]
subset_vec = [i in pathways for i in rel_df.iloc[:, 0]] and [
x in pathways for x in rel_df.iloc[:, 1]
] #filter the pathways in realtion file corelated to those pathways in matrix file.
sub_rel_df = rel_df[subset_vec]
#sub_rel_df.to_csv("test.csv", sep="\t") #can test if the filter works here
G = nx.DiGraph()
G.add_nodes_from(pathways)
G.add_edges_from(sub_rel_df.values)
tree = nx.algorithms.dag.dag_to_branching(G)
secondDict = nx.get_node_attributes(
tree, 'source'
) #set up the value of second dictionary (hierarchical dictionary)
thirdDict = {
'explained_ratios': {},
'ngenes': {},
'description': {}
} #set up the frame of third dictionary
for key, value in secondDict.items():
thirdDict['explained_ratios'].update({
key:
topDict['explained_ratios'][value]
}) #valuation third dictionary with the key value from top dictionary
thirdDict['ngenes'].update({key: topDict['ngenes'][value]})
thirdDict['description'].update({key: topDict['pathways'][value]})
nx.set_node_attributes(tree,
thirdDict['explained_ratios'],
name='explained_ratios')
nx.set_node_attributes(tree, thirdDict['ngenes'], name='ngenes')
nx.set_node_attributes(tree, thirdDict['description'], name='description')
root = [v for v, d in tree.in_degree() if d == 0][0]
out_json = json_graph.tree_data(tree, root)
with open(outname, 'w') as outfile:
simplejson.dump(out_json, outfile, ignore_nan=True)
def render_trees(parsed_dir_path: Path, target_dir_path: Path,
work_distribution: list):
for ecli in work_distribution:
parsed_files_glob = parsed_dir_path.joinpath(ecli).glob('*.xml')
for parsed_file_path in parsed_files_glob:
if parsed_file_path.is_file(
) and parsed_file_path.stat().st_size != 0:
json_dir_path = target_dir_path.joinpath(ecli)
if not json_dir_path.is_dir():
mkdir(str(json_dir_path))
json_file_name = parsed_file_path.name + '.json'
json_file_path = json_dir_path.joinpath(json_file_name)
# draw_spring(tree)
if not json_file_path.is_file(
) or json_file_path.stat().st_size != 0:
tree = DiGraph()
xml_tree = parse(str(parsed_file_path))
sentence = SENTENCE_XPATH(xml_tree)[0].text
nodes = []
edges = []
for xml_node in XML_NODES(xml_tree):
lemma = xml_node.get('lemma')
pos = xml_node.get('pos')
if lemma is None:
lemma = '...'
node_id = int(xml_node.attrib['id'])
node_attributes = {'name': lemma, 'pos': pos}
node = (node_id, node_attributes)
nodes.append(node)
parent_node_id = int(xml_node.getparent().get('id'))
edges.append((parent_node_id, node_id))
tree.add_nodes_from(nodes)
tree.add_edges_from(edges)
tree_json = json_graph.tree_data(tree, root=0)
wrapper_json = {
'origin': parsed_file_path.as_uri(),
'sentence': sentence,
'tree': tree_json
}
with json_file_path.open(mode='wt') as json_file:
dump(wrapper_json, json_file, indent=True)
info("Rendered parse tree to '{json_file_path}'. ".format(
json_file_path=json_file_path))
else:
error("Empty or non-existent XML parse tree file at "
"'{parsed_file_path}'. ".format(
parsed_file_path=parsed_file_path))
def main():
args = read_arguments()
filename = args.filename
nodes = {}
edges = {}
actors = {}
roots = []
digraph = nx.DiGraph()
doc_title = filename.replace('.ann', '')
with open(filename, 'r') as file_:
for line in file_:
if line.startswith('T'):
name, label, text = line.split('\t')[:3]
nodes[name] = {'label': label.split(' ')[0], 'text': text}
digraph.add_node(name, {
'label': label.split(' ')[0],
'text': text
})
if 'major-claim' in label:
roots.append(name)
elif line.startswith('R'):
name, label = line.split('\t')[:2]
label, destination, origin = label.split(' ')
origin = origin.replace('Arg2:', '')
destination = destination.replace('Arg1:', '')
edges[name] = {
'label': label,
'origin': origin,
'destination': destination
}
digraph.add_edge(origin, destination, {'label': label})
elif line.startswith('A'):
_, attribute_name, source, attribute_value = line.split()
if attribute_name == 'ACTOR':
actors[source] = attribute_value
nx.set_node_attributes(digraph, 'actor', actors)
data = {'name': doc_title.replace('_', ' '), 'children': []}
# Transform digraph into a forest using the major-claims as tree roots.
for root in roots:
descendants = nx.descendants(digraph, root)
descendants.add(root)
subgraph = digraph.subgraph(descendants)
if not nx.is_tree(subgraph):
print('Subgraph that is not a tree founded')
continue
data['children'].append(
json_graph.tree_data(subgraph,
root=root,
attrs={
'children': 'children',
'id': 'name',
'actor': 'actor'
}))
with open('{}.json'.format(doc_title), 'w') as jf:
json.dump(data, jf)
def write_circular_packing_files():
typeset = CompleteSet()
graph = typeset.base_graph.copy()
nx.relabel_nodes(graph, {n: str(n) for n in graph.nodes}, copy=False)
data = json_graph.tree_data(graph, root="Generic")
data = update(data)
write_json(data)
write_html(data)
def main():
typeset = visions_complete_set()
graph = typeset.base_graph.copy()
nx.relabel_nodes(graph, {n: str(n) for n in graph.nodes}, copy=False)
data = json_graph.tree_data(graph, root="visions_generic")
data = update(data)
write_json(data)
write_html(data)
def write_evolution_trees():
logger.info("Starting to scrape evolution chains")
species, variants, typing, stats, urls = scrape_pokedex()
graphs = scrape_evolution_trees(list(zip(species, variants)))
trees = []
for tree in graphs:
sources = [n for n, d in tree.in_degree() if d == 0]
trees.append(tree_data(tree, sources[0]))
with open("evolutions.json", "w") as f:
f.write(str.encode(json.dumps(trees)).decode("unicode-escape"))
def searchTerm(query):
fl = open("Cluster_Names.csv", "r")
x = []
fx = object()
g = nx.DiGraph()
list_of_graphs = []
list_of_graph_node_lengths = []
cluster_name = ''
#query = " " + query + " "
#cluster_names = []
file_name = ''
for line in fl:
x = line.split(",")
cluster_name = x[1]
#cluster_names = cluster_names + [cluster_name]
file_name = x[0]
if cluster_name.find(query) != -1:
fx = open(os.getcwd() + "/Clusters/" + file_name, "r")
g = nx.DiGraph()
g.add_node(cluster_name, size=2000)
for ln in fx:
g.add_node(ln.strip("\n"), size=2000)
g.add_edge(cluster_name, ln.strip("\n"))
list_of_graphs = list_of_graphs + [g]
list_of_graph_node_lengths = list_of_graph_node_lengths + [
len(g.nodes())
]
i = list_of_graph_node_lengths.index(max(list_of_graph_node_lengths))
print "Here : ", list_of_graph_node_lengths[i]
g = list_of_graphs[i]
x = g.out_degree().values()
i = x.index(max(x))
nx.draw(g)
pylab.show()
t = json_graph.tree_data(g, root=g.out_degree().keys()[i])
x1 = g.nodes()
x1.remove(g.out_degree().keys()[i])
p = generate_list(x1)
relevant_tags = getRelevantTags(x1)
x = str(t)
x = re.sub("\'id\'", "\'name\'", x)
x = re.sub("\'", "\"", x)
print x
print relevant_tags
str1 = open("file1.txt", "r").read() + str(relevant_tags) + open(
"file2.txt", "r").read()
print str1
#fl=open("static/js/examples/simple.html","w")
#fl.write(str1)
#fl.close()
print str1
print p
def render_trees(parsed_dir_path: Path, target_dir_path: Path,
work_distribution: list):
for ecli in work_distribution:
parsed_files_glob = parsed_dir_path.joinpath(ecli).glob('*.xml')
for parsed_file_path in parsed_files_glob:
if parsed_file_path.is_file() and parsed_file_path.stat(
).st_size != 0:
json_dir_path = target_dir_path.joinpath(ecli)
if not json_dir_path.is_dir():
mkdir(str(json_dir_path))
json_file_name = parsed_file_path.name + '.json'
json_file_path = json_dir_path.joinpath(json_file_name)
# draw_spring(tree)
if not json_file_path.is_file() or json_file_path.stat(
).st_size != 0:
tree = DiGraph()
xml_tree = parse(str(parsed_file_path))
sentence = SENTENCE_XPATH(xml_tree)[0].text
nodes = []
edges = []
for xml_node in XML_NODES(xml_tree):
lemma = xml_node.get('lemma')
pos = xml_node.get('pos')
if lemma is None:
lemma = '...'
node_id = int(xml_node.attrib['id'])
node_attributes = {'name': lemma, 'pos': pos}
node = (node_id, node_attributes)
nodes.append(node)
parent_node_id = int(xml_node.getparent().get('id'))
edges.append((parent_node_id, node_id))
tree.add_nodes_from(nodes)
tree.add_edges_from(edges)
tree_json = json_graph.tree_data(tree, root=0)
wrapper_json = {'origin': parsed_file_path.as_uri(),
'sentence': sentence,
'tree': tree_json}
with json_file_path.open(mode='wt') as json_file:
dump(wrapper_json, json_file, indent=True)
info("Rendered parse tree to '{json_file_path}'. ".format(
json_file_path=json_file_path))
else:
error("Empty or non-existent XML parse tree file at "
"'{parsed_file_path}'. ".format(
parsed_file_path=parsed_file_path))
def generate_sunburst_json(stats_df, relation_file, root_node_id = 'Homo_Sapiens'):
'''
stats_df has a very specific format:
indexes should be the pathway IDs
'value' will be translated to colors in the plot.
'ngenes' is the size of the pathwat, and will be displayed as the size of each arc.
'description' as the description of the pathway
relation_file is downloaded from Reactome "ReactomePathwaysRelation.txt"
'''
# in_df = in_df.loc[[x for x in in_df.index if 'HSA' in x]]
rel_df = generate_root_node(relation_file, root_node_id = root_node_id)
topPaths = rel_df.loc[(rel_df['parentId'] == root_node_id), 'id']
rootNgenes = np.sum(stats_df.loc[[x in topPaths.tolist() for x in stats_df.index],'ngenes'])
rootNode = pd.DataFrame([[1,rootNgenes,"H**o Sapiens", 0,0,0,0]], columns = ["value", "ngenes", "description"]).xs(0)
rootNode.name = root_node_id
stats_df = stats_df.append(rootNode)
topDict = stats_df.to_dict()
pathways = stats_df.index
n_path = len(pathways)
subset_vec = [x in pathways for x in rel_df.iloc[:,0]] and [x in pathways for x in rel_df.iloc[:,1]]
sub_rel_df = rel_df[subset_vec]
G = nx.DiGraph()
G.add_nodes_from(pathways)
G.add_edges_from(sub_rel_df.values)
tree = nx.algorithms.dag.dag_to_branching(G)
secondDict = nx.get_node_attributes(tree,'source')
thirdDict = {'value':{}, 'ngenes':{}, 'description':{}}
for key, value in secondDict.items():
thirdDict['value'].update({key : topDict['value'][value]})
thirdDict['ngenes'].update({key : topDict['ngenes'][value]})
thirdDict['description'].update({key : topDict['description'][value]})
nx.set_node_attributes(tree, thirdDict['value'], name = 'value')
nx.set_node_attributes(tree, thirdDict['ngenes'], name = 'ngenes')
nx.set_node_attributes(tree, thirdDict['description'], name = 'description')
root = [v for v, d in tree.in_degree() if d == 0][0]
out_json = json_graph.tree_data(tree, root)
return out_json
def show_author(authname):
#try:
authname = authname.lower().replace(" ", "")
try:
authortree = buildfnauthortree(authname, query_db_graph, query_db_full, 2)
authornetwork = unicode(json.dumps(json_graph.tree_data(authortree, lookupfn(authname, query_db_full),
attrs={'children': 'children', 'id': 'name'})))
except TypeError:
authornetwork = ''
html = flask.render_template(
'authors.html',
JSONAUTHORNETWORK = authornetwork)
return html
def searchTerm(query):
fl = open("Cluster_Names.csv","r")
x=[]
fx=object()
g=nx.DiGraph()
list_of_graphs = []
list_of_graph_node_lengths = []
cluster_name = ''
#query = " " + query + " "
#cluster_names = []
file_name = ''
for line in fl :
x=line.split(",")
cluster_name = x[1]
#cluster_names = cluster_names + [cluster_name]
file_name = x[0]
if cluster_name.find(query) != -1:
fx=open(os.getcwd()+"/Clusters/"+file_name,"r")
g=nx.DiGraph()
g.add_node(cluster_name,size=2000)
for ln in fx:
g.add_node(ln.strip("\n"),size=2000)
g.add_edge(cluster_name,ln.strip("\n"))
list_of_graphs = list_of_graphs + [g]
list_of_graph_node_lengths = list_of_graph_node_lengths + [len(g.nodes())]
i=list_of_graph_node_lengths.index(max(list_of_graph_node_lengths))
print "Here : " , list_of_graph_node_lengths[i]
g=list_of_graphs[i]
x=g.out_degree().values()
i=x.index(max(x))
nx.draw(g)
pylab.show()
t=json_graph.tree_data(g,root=g.out_degree().keys()[i])
x1=g.nodes()
x1.remove(g.out_degree().keys()[i])
p=generate_list(x1)
relevant_tags = getRelevantTags(x1)
x=str(t)
x=re.sub("\'id\'","\'name\'",x)
x=re.sub("\'","\"",x)
print x
print relevant_tags
str1 = open("file1.txt","r").read() + str(relevant_tags) + open("file2.txt","r").read()
print str1
#fl=open("static/js/examples/simple.html","w")
#fl.write(str1)
#fl.close()
print str1
print p
def tree_from_dag(g, root_node, fmt=False, edge_attrs=_E_ATTRS):
"""
Make tree from DAG ``g``.
:param g: networkx.DiGraph instance of DAG, direct acyclic graph.
:param root_node: Root node
:param fmt: Return data in tree format that is suitable for JSON
serialization.
:param edge_attrs: Default edge attributes :: dict
:return: networkx.DiGraph instance or its JSON representation.
"""
assert NX.is_directed_acyclic_graph(g), "Given graph is not DAG."
nsattrs = NX.get_node_attributes(g, "names")
g = NX.DiGraph()
g.add_node(root_node,
name=root_node,
names=nsattrs.get(root_node, [root_node]))
visited = set([root_node])
parents = [root_node]
ids = {root_node: count()}
while parents:
next_parents = set()
for parent in parents:
for s in g.successors_iter(parent):
names = nsattrs.get(s, [s])
if s in visited:
new_s = _clone_nodeid(s, ids)
logging.debug("Clone visited node %s to %s" % (s, new_s))
g.add_node(new_s, name=s, names=names)
g.add_edge(parent, new_s, edge_attrs)
else:
logging.debug("Node=" + s)
visited.add(s)
next_parents.add(s)
ids[s] = count()
g.add_node(s, name=s, names=names)
g.add_edge(parent, s, edge_attrs)
parents = next_parents
return JS.tree_data(g, root_node) if fmt else g
def history_to_json(fname, snapshot, id):
"""Generate json version of merger history."""
id = np.longlong(id)
meraxes.set_little_h(fname)
gals = meraxes.read_gals(fname, snapshot, props=PROPS, quiet=True)
snaplist, _, _ = meraxes.read_snaplist(fname)
# get the ind of our start galaxy
ind = np.where(gals['ID'] == id)[0]
assert(len(ind) == 1)
ind = ind[0]
# read in the walk indices
fp_ind = deque()
np_ind = deque()
for snap in tqdm(snaplist[:snapshot+1], desc="Reading indices"):
try:
fp_ind.append(meraxes.read_firstprogenitor_indices(fname, snap))
except:
fp_ind.append([])
try:
np_ind.append(meraxes.read_nextprogenitor_indices(fname, snap))
except:
np_ind.append([])
# generate the graph (populates global variable `tree`)
walk(snapshot, ind, fp_ind, np_ind)
# attach the galaxies to the graph
for snap in tqdm(snaplist[:snapshot+1], desc="Generating graph"):
try:
gal = meraxes.read_gals(fname, snapshot=snap, quiet=True,
props=PROPS)
except IndexError:
continue
for nid in tree.nodes_iter():
node = tree.node[nid]
gal_snap, gal_ind = [int(v) for v in nid.split('|')]
if gal_snap == snap:
add_galaxy_to_node(node, gal[gal_ind])
# dump the tree to json
data = json_graph.tree_data(tree, root=node_id(snapshot, ind),
attrs=dict(id='name', children='children'))
fname_out = "tree_%09d.json" % id
with open(fname_out, "wb") as fd:
json.dump(data, fd)
print("Conversion complete: %s" % fname_out)
def default(self, obj):
if isinstance(obj, np.ndarray):
return obj.tolist()
elif isinstance(obj, np.generic):
return obj.item()
elif isinstance(obj, nx.Graph) or isinstance(obj, nx.DiGraph):
if nx.is_tree(obj):
# NOTE: the root must be the first node added. otherwise it won't work
return json_graph.tree_data(obj, obj.nodes_iter().next())
else:
return json_graph.node_link_data(obj)
elif isinstance(obj, pd.DataFrame):
return obj.to_dict(orient='records')
return json.JSONEncoder.default(self, obj)
def test_exceptions():
with pytest.raises(TypeError, match="is not a tree."):
G = nx.complete_graph(3)
tree_data(G, 0)
with pytest.raises(TypeError, match="is not directed."):
G = nx.path_graph(3)
tree_data(G, 0)
with pytest.raises(TypeError, match="is not weakly connected."):
G = nx.path_graph(3, create_using=nx.DiGraph)
G.add_edge(2, 0)
G.add_node(3)
tree_data(G, 0)
with pytest.raises(nx.NetworkXError, match="must be different."):
G = nx.MultiDiGraph()
G.add_node(0)
tree_data(G, 0, ident="node", children="node")
def save_graph(G, root_id, filename):
# Write the graph to a json file
from networkx.readwrite import json_graph
import json
datag = json_graph.tree_data(G, root=root_id)
#datag['links'] = [
# {
# 'source': datag['nodes'][link['source']]['id'],
# 'target': datag['nodes'][link['target']]['id']
# }
# for link in datag['links']]
s = json.dumps(datag)
with open(filename, "w") as f:
f.write(s)
def tree_from_dag(g, root_node, fmt=False, edge_attrs=_E_ATTRS):
"""
Make tree from DAG ``g``.
:param g: networkx.DiGraph instance of DAG, direct acyclic graph.
:param root_node: Root node
:param fmt: Return data in tree format that is suitable for JSON
serialization.
:param edge_attrs: Default edge attributes :: dict
:return: networkx.DiGraph instance or its JSON representation.
"""
assert NX.is_directed_acyclic_graph(g), "Given graph is not DAG."
nsattrs = NX.get_node_attributes(g, "names")
g = NX.DiGraph()
g.add_node(root_node, name=root_node, names=nsattrs.get(root_node, [root_node]))
visited = set([root_node])
parents = [root_node]
ids = {root_node: count()}
while parents:
next_parents = set()
for parent in parents:
for s in g.successors_iter(parent):
names = nsattrs.get(s, [s])
if s in visited:
new_s = _clone_nodeid(s, ids)
logging.debug("Clone visited node %s to %s" % (s, new_s))
g.add_node(new_s, name=s, names=names)
g.add_edge(parent, new_s, edge_attrs)
else:
logging.debug("Node=" + s)
visited.add(s)
next_parents.add(s)
ids[s] = count()
g.add_node(s, name=s, names=names)
g.add_edge(parent, s, edge_attrs)
parents = next_parents
return JS.tree_data(g, root_node) if fmt else g
def default(self, obj):
if isinstance(obj, np.ndarray):
return obj.tolist()
elif isinstance(obj, np.generic):
return obj.item()
elif isinstance(obj, nx.Graph) or isinstance(obj, nx.DiGraph):
if nx.is_tree(obj) and 'root' in obj.graph:
# NOTE: there must be a root graph attribute, or the root must be the first node added.
# otherwise it won't work
return json_graph.tree_data(obj, obj.graph['root'])
else:
return json_graph.node_link_data(obj)
elif isinstance(obj, pd.DataFrame):
return obj.to_dict(orient='records')
return json.JSONEncoder.default(self, obj)
def get_graph(self, graph_type):
graph_data = self.client.call(self.ctxt, 'get_topology', arg=None)
LOG.info(graph_data)
# graph_file = pecan.request.cfg.find_file('graph.sample.json')
try:
if graph_type == 'graph':
return json.loads(graph_data)
if graph_type == 'tree':
return json_graph.tree_data(
json_graph.node_link_graph(json.loads(graph_data)),
root='RESOURCE:node')
except Exception as e:
LOG.exception("failed to open file %s ", e)
abort(404, str(e))
def buildHierarchiesGraph(fileName):
data = []
roots = getSetOfName(fileName,"PROV")
graphs = []
for rs in roots:
g = nx.DiGraph()
g.add_node(1,label=rs)
graphs.append(g)
for g in graphs:
trav = list()
trav.append(''+ g.node[1]['label']+',')
cnt = 2
graphLevel(g,fileName,1,trav)
data = json_graph.tree_data(g,root=1)
with io.open('tree'+g.node[1]['label']+'.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(data, ensure_ascii=False)))
def combineEverything(fname, allNodes, toOutput = 2):
DG = level2Nodes(fname)
DGN_s1Nodes = DG.nodes()
tmpDict = level1Nodes()
for n in DGN_s1Nodes: #ex: Root1: {science:{[psychology, linguistics]}}
if n in tmpDict:
#print n, type(tmpDict[n]) == list
for v in tmpDict[n]:
DG.add_edge(n, v)
del tmpDict[n]
###print nodes have hierachy but not grouped
"""
for k, vd in tmpDict.iteritems():
if len(vd) > 1:
for v in vd:
DG.add_edge(k , v)
DG.add_edge("ROOT0", k)
"""
###
DGN_s2Nodes = DG.nodes()
isolatedNodes = allNodes - set(DGN_s2Nodes)
for d in DGN_s2Nodes:
if d.startswith("Root"):
#print d, DG.neighbors(d)
DG.add_edge("ROOT0", d)
#print DG.neighbors("ROOT0")
DG2 = nx.DiGraph()
for sn in isolatedNodes:
DG2.add_edge("ROOT0", sn)
#if sn in DG.nodes():
# DG.remove_node(sn)
if toOutput == 2:
try:
tmpout1 = str(json_graph.tree_data(DG, "ROOT0")).replace("'id'", "'name'")
out1 = "var data1 ="+tmpout1+";"
tmpout2 = str(json_graph.tree_data(DG2, "ROOT0")).replace("'id'", "'name'")
out2 = "var data2 ="+tmpout2+";"
fout = open("file"+str(file_id)+"_final.js", "w")
fout.write(out1+"\n"+out2)
fout.close()
except:
nx.write(DG, "out.gml")
else:
try:
print json_graph.tree_data(DG, "ROOT0")
print json_graph.tree_data(DG2, "ROOT0")
except:
nx.write_gml(DG,"out.gml")
def convert_graph_to_json(graph, filename):
"""
convert from a networkX graph object, to serialized json format.
:param graph: the networkX graph object.
:param filename: the name of the file to write to.
:return:
"""
networkx.write_gml(graph, filename)
# find the lowest degree node
# store [node_name, degree] and compare for the lowest degree
root_node = [graph.nodes()[0], graph.node[graph.nodes()[0]]['degree']]
for node in graph.nodes():
if graph.node[node]['degree'] < root_node[1]:
root_node = [node, graph.node[node]['degree']]
break
data = json_graph.tree_data(graph, root=root_node[0])
with open(filename, 'w') as f_handle:
f_handle.write(json.dumps(data))
return
def createTree():
g=nx.DiGraph()
g.add_node("Kaushik",size=2000)
g.add_node("Apoorva",size=1000)
g.add_node("Chaitanya",size=1000)
g.add_node("Dilip",size=1000)
g.add_edge("Kaushik","Apoorva")
g.add_edge("Kaushik","Dilip")
g.add_edge("Kaushik","Chaitanya")
t=json_graph.tree_data(g,root="Kaushik")
x=str(t)
x=re.sub(x,'id','name')
print x
nx.draw(g)
pylab.show()
def hierarchical_clustering(): # does hierarchical clustering
g=nx.DiGraph()
list_of_nodes = os.listdir(os.getcwd() + "/Clusters")
no_of_hier_nodes = 0
score = 0
g.add_node("Entire Dataset",size=2000)
for items in list_of_nodes :
g.add_node(str(no_of_hier_nodes),size=2000)
g.add_edge("Entire Dataset",str(no_of_hier_nodes))
node_sizes["Entire Dataset"] = 2000
if items not in g.nodes() :
g.add_node(cluster_names[items],size=2000)
node_sizes[cluster_names[items]] = 2000
g.add_edge(str(no_of_hier_nodes),cluster_names[items])
for node in list_of_nodes :
if node != items and node not in g.nodes():
score = e2.eqn2(cluster_features[items],cluster_features[node])
if score > 0.3 :
g.add_node(cluster_names[node],size=2000)
node_sizes[cluster_names[node]] = 2000
g.add_edge(str(no_of_hier_nodes),cluster_names[node])
list_of_nodes.remove(node)
list_of_nodes.remove(items)
no_of_hier_nodes = no_of_hier_nodes + 1
T=nx.bfs_tree(g,source="Entire Dataset")
nx.draw(g)
pylab.show()
g=T
nx.set_node_attributes(g,"size",node_sizes)
t=json_graph.tree_data(T,root="Entire Dataset")
x=str(t)
x=re.sub("\'id\'","\'name\'",x)
x=re.sub("\'","\"",x)
#x=re.sub("\'","\"",x)
#x=re.sub("\"name\"[\:] \"(.)*\"","\"name\"[:] \"(.)*\" \"size\": 2000",x)
fl = open("graph.json","w")
fl.write(x)
fl.close()
print x
def makeNestedJson(leaf) :
leaf=json.loads(leaf)
#A tree is a directed graph - create one with a dummy root
DG=nx.DiGraph()
DG.add_node('root')
#Construct the tree as a directed graph and annotate the nodes with attributes
#Edges go from parent to child
for e in leaf:
DG.add_node(e['id'],label=e['label'])
#If there's a parent, use it...
if 'parent' in e: DG.add_edge(e['parent'],e['id'])
#else create a dummy parent from the dummy root
else: DG.add_edge('root',e['id'])
#Get the tree as JSON
data = json_graph.tree_data(DG,root='root')
#and dump the data from the dummy root's children down...
return json.dumps(data['children'])
def make_directed_json_graph(gmt_fn, d_id_name, d_id_category, d_category_color, outfn=None):
# perform HC and make a directed graph and write to json
# for pack visualization
d_gmt = read_gmt(gmt_fn)
d_gmt_filt = {}
for term, genes in d_gmt.items():
if len(genes) >= 5:
d_gmt_filt[term] = genes
d_gmt = d_gmt_filt
print 'number of terms:', len(d_gmt)
umls_ids_kept = d_gmt.keys()
adj_matrix = jaccard_matrix(d_gmt)
hc = AgglomerativeClustering(n_clusters=10)
hc.fit(adj_matrix)
m = adj_matrix > 0.2
adj_matrix = adj_matrix * m.astype(int)
Gu = nx.from_numpy_matrix(adj_matrix) # undirected Graph, to get size
G = nx.DiGraph()
print adj_matrix.shape, len(umls_ids_kept)
for i in range(adj_matrix.shape[0]):
cluster_label = hc.labels_[i]
umls_id = umls_ids_kept[i]
name = d_id_name[umls_id]
G.add_edge('root', cluster_label)
G.add_edge(cluster_label, umls_id)
G.node[umls_id]['size'] = Gu.degree(i)
G.node[umls_id]['label'] = name
category = d_id_category[umls_id]
color = d_category_color[category]
G.node[umls_id]['color'] = color
print G.number_of_nodes(), G.number_of_edges()
graph_data = json_graph.tree_data(G,root='root')
json.dump(graph_data, open(outfn, 'wb'))
return
def buildHierarchiesGraph(fileName):
global cnt
data = []
roots = getSetOfName(fileName,"REG1")
graphs = []
g = nx.DiGraph()
g.add_node(1,label="root")
for rs in roots:
trav=[]
#print(rs,cnt)
g.add_edge(1,cnt, label = "child of root")
g.add_node(cnt,label=rs)
trav.append(''+ g.node[cnt]['label'])
#trav.append(lS[1])
cnt = cnt + 1
graphLevel(g, fileName, cnt-1, trav)
data = json_graph.tree_data(g,root=1)
with io.open('tree'+g.node[1]['label']+'.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(data, ensure_ascii=False)))
def main():
beam_data = np.load(ARGS.data)
# Optionally load vocabulary data
vocab = None
if ARGS.vocab:
with open(ARGS.vocab) as file:
vocab = file.readlines()
vocab = [_.strip() for _ in vocab]
vocab += ["UNK", "SEQUENCE_START", "SEQUENCE_END"]
if not os.path.exists(ARGS.output_dir):
os.makedirs(ARGS.output_dir)
# Copy required files
shutil.copy2("./bin/tools/beam_search_viz/tree.css", ARGS.output_dir)
shutil.copy2("./bin/tools/beam_search_viz/tree.js", ARGS.output_dir)
for idx in range(len(beam_data["predicted_ids"])):
predicted_ids = beam_data["predicted_ids"][idx]
parent_ids = beam_data["beam_parent_ids"][idx]
scores = beam_data["scores"][idx]
graph = create_graph(
predicted_ids=predicted_ids,
parent_ids=parent_ids,
scores=scores,
vocab=vocab)
json_str = json.dumps(
json_graph.tree_data(graph, (0, 0)),
ensure_ascii=False)
html_str = HTML_TEMPLATE.substitute(DATA=json_str)
output_path = os.path.join(ARGS.output_dir, "{:06d}.html".format(idx))
with open(output_path, "w") as file:
file.write(html_str)
print(output_path)
def save_profile(Tree, args):
'''
Save the profile, after aggregation, to files. One for white and one for black.
'''
for c in range(len(Tree)):
setattr(args, 'colour', args.colours[c])
if args.v:
print('The',args.colour,'tree has', Tree[c].number_of_nodes(), 'nodes.')
print('The',args.colour,'tree has', Tree[c].number_of_edges(), 'edges.')
if args.tree:
# for a tree format and the graph needs to be created with DiGraph
data = json_graph.tree_data(Tree[c], root=args.root)
else:
Tree[c] = nx.convert_node_labels_to_integers(Tree[c], first_label=0, label_attribute='fen')
# name is changed to: "(Carlsen (black))_with_int_labels"
Tree[c].name = ''.join([args.player, ' (', args.colour, ')']) # above convert also converts the name
# for a Force directed graph
data = json_graph.node_link_data(Tree[c])
s = json.dumps(data)
js = open(profile_file_name(args), 'w')
js.write(s)
js.close()
def get_prereq_graph(course_id, format=None):
"""
Generate a graph of prerequisites within a course. If format is not
requested, simply return a NetworkX graph object.
couse_id: the ID of the requested course
format: what format to return in (optional)
node: json formatted as node-link style
adjacency: json formatted as adjacency style
tree: json formatted as tree style
"""
from trajectory.models import Department, Course
from trajectory.models.meta import session
from trajectory.utils.common import row2dict
from networkx.readwrite import json_graph
import networkx as nx
import json
if format not in [None, "node", "adjacency", "tree"]:
raise RuntimeError("Unknown requested data format %s" % format)
# Initialize a new NetworkX graph.
G = nx.DiGraph()
# Attempt to look up the requested course.
course = session.query(Course).get(course_id)
if course is None:
return None
# Recursively add course ids in a subtree to the graph.
def add_tree(G, tree, parent=None):
cid = tree[0] # unpack information
prereqs = tree[1] # unpack information
course = session.query(Course).get(cid)
# Insert all known data, including department abbreviation.
node_data = row2dict(course)
node_data['dept'] = course.department.abbreviation
# Identify the primary course in the graph (the requested).
if str(cid) == str(course_id):
node_data['prime'] = True
else:
node_data['prime'] = False
# If the course has already been added, generate a unique ID for it
# based on its parent, and add it anyway. But don't recurse into
# its list of prereqs.
seen = False
if cid in G.nodes():
cid = str(parent) + "-" + str(cid)
seen = True
# Add course and an edge from its parent, if relevant.
G.add_node(cid, node_data)
if parent is not None:
G.add_edge(parent, cid)
# Recurse through the prerequisite tree and add in subtrees.
if not seen:
for prereq in prereqs:
add_tree(G, prereq, cid)
# Navigate the prerequisite tree and add the course ids as nodes, and
# prerequisite relationships as unweighted edges.
prereq_tree = get_prereq_tree(course_id)
add_tree(G, prereq_tree)
if G is None:
return G
# Calculate and apply a basic layout.
pos = nx.spring_layout(G)
for node in G.nodes():
G.node[node]["viz"] = {
'position': {
'x': pos[node][0],
'y': pos[node][1]
}
}
# Apply any requested data output formatting.
if format == "node":
return json.dumps(json_graph.node_link_data(G))
elif format == "adjacency":
return json.dumps(json_graph.adjacency_data(G))
elif format == "tree":
return json.dumps(json_graph.tree_data(G, int(course_id)))
else:
return G
def infer(self, save_to_db):
propagators = self.dqa.propagators
# bisect 不支持 key, 故只能再记录一个时间序列
# 排除本答案的回答/点赞者
times = [action.time for action in propagators if action.aid != self.aid]
upvoters_added = [self.root] # 记录已经加入图中的点赞者
# 按时间顺序一起处理
pq = PriorityQueue()
i1 = i2 = i3 = 0
if self.upvoters:
pq.put(self.upvoters[0])
i1 += 1
if self.commenters:
pq.put(self.commenters[0])
i2 += 1
if self.collectors:
pq.put(self.collectors[0])
i3 += 1
l1, l2, l3 = len(self.upvoters), len(self.commenters), len(self.collectors)
while not pq.empty():
action = pq.get()
if self.graph.has_node(action.uid):
# 融合 uid 相同的点
self.graph.node[action.uid]['acttype'] = \
action.acttype | self.graph.node[action.uid]['acttype']
else:
relation = self._infer_node(action, propagators, times, upvoters_added)
self.add_node(relation.tail)
self.add_edge(*relation)
if action.acttype == UPVOTE_ANSWER and i1 < l1:
pq.put(self.upvoters[i1])
upvoters_added.append(action)
i1 += 1
elif action.acttype == COMMENT_ANSWER and i2 < l2:
pq.put(self.commenters[i2])
i2 += 1
elif action.acttype == COLLECT_ANSWER and i3 < l3:
pq.put(self.collectors[i3])
i3 += 1
for node in self.graph.nodes():
self.graph.node[node]['acttype'] = acttype2str(self.graph.node[node]['acttype'])
tree_data = json_graph.tree_data(self.graph, root=self.root.uid)
node_links = json_graph.node_link_data(self.graph)
links = [
{
'source': node_links['nodes'][link['source']]['id'],
'target': node_links['nodes'][link['target']]['id'],
'reltype': link['reltype']
}
for link in node_links['links']
]
tree_data['links'] = links
if save_to_db:
self.dynamic_collection.replace_one({'aid': self.aid},
transform_incoming(tree_data),
upsert=True)
else:
with open('data/dump.json', 'w') as f:
json.dump(tree_data, f, cls=MyEncoder, indent='\t')