def test_job_set_accepted(self):
# Arrange
job_id = "1"
job_name = "new job"
g = nx.Graph()
expected_result = {'name': job_name, 'graph': json_graph.node_link_data(g), 'status': "Accepted", 'id': job_id}
class TestJobManager(object):
def __init__(self):
self.job = None
def get_job(self, job_id):
return self.job
def insert_job(self, job_name, graph):
self.job = {'name': job_name, 'graph': graph, 'status': "Received", "id": job_id}
return job_id
def update_job(self, job_id, job):
self.job['status'] = job['status']
api.config['REPOSITORY'] = JobManagerRepository(TestJobManager())
self.app = api.test_client()
# Act
data = {'job_name': job_name, 'graph': json_graph.node_link_data(g)}
rv = self.app.post('/jobs', data=json.dumps(data))
rv2 = self.app.put('/jobs/' + job_id + '/status', data=json.dumps({'status': 'Accepted'}))
rv3 = self.app.get('/jobs/' + job_id)
# Assert
final_job = json.loads(rv3.data.decode(rv.charset))
self.assertEqual(expected_result, final_job)
def build_network(seed_network,utility_params,network_length):
now_start = datetime.datetime.now()
start_node = len(seed_network)
for i in range(len(seed_network),network_length):
print ("sim network length = " + str(len(seed_network)+1))
seed_network.expansion_utilities(utility_params)
if seed_network.expansion_city_ghs:
new_nodes = {node:data for node,data in seed_network.expansion_cache.items() if node in seed_network.expansion_city_ghs}
best_node = max(new_nodes, key= lambda x: new_nodes[x][-1])#last value in array
print (best_node)
else:
print ("no expansion cities found at node " + str(i))
file_name = "simulated_networks/" + str(start_node) + "_" + str(len(seed_network)) + "_" + "_".join([str(param) for param in utility_params.tolist()]) + ".pickle"
with open(file_name, 'wb') as outfile:
pickle.dump(seed_network, outfile)
file_name = "simulated_networks/" + str(start_node) + "_" + str(len(seed_network)) + "_" + "_".join([str(param) for param in utility_params.tolist()]) + ".json"
with open(file_name, 'w') as outfile:
json.dump(json_graph.node_link_data(seed_network), outfile)
return seed_network
seed_network.add_SC(best_node)
if len(seed_network[best_node].keys()) == 0:
print ("added node has no conenctions")
print (seed_network.expansion_cache[best_node])
file_name = "simulated_networks/" + str(start_node) + "_" + str(len(seed_network)) + "_" + "_".join([str(param) for param in utility_params.tolist()]) + ".pickle"
with open(file_name, 'wb') as outfile:
pickle.dump(seed_network, outfile)
file_name = "simulated_networks/" + str(start_node) + "_" + str(len(seed_network)) + "_" + "_".join([str(param) for param in utility_params.tolist()]) + ".json"
with open(file_name, 'w') as outfile:
json.dump(json_graph.node_link_data(seed_network), outfile)
now_finish = datetime.datetime.now()
elapsedTime = now_finish- now_start
print(elapsedTime / timedelta(minutes=1))
return seed_network
def send_network(self):
if self.mininet:
d = json_graph.node_link_data(self.mininet_to_nx_graph())
else:
d = json_graph.node_link_data(self.net.to_nx_graph())
d['message_type'] = 'network'
d['mininet'] = bool(self.mininet)
self.send_to_ws(json.dumps(d))
def computebwmod(graph,path,protlist,path_lenght,alone):
import itertools
fixed=[]
combination=[]
betweennees=nx.betweenness_centrality(graph,normalized=True)
print "------Starting Graph------"
print nx.info(graph)
d = json_graph.node_link_data(graph) # node-link format to serialize
# write json
json.dump(d, open('mcn.json','w'))
count={}
for i in graph.nodes():
if i in protlist:
continue
else:
count[i]=0
combination=list(itertools.combinations(path,2))
pa={}
for i in path:
pa[i]=[]
for j in path[i]:
if path.has_key(i):
pa[i].extend(j[1:len(j)-1])
else:
pa[i].extend(j[1:len(j)-1])
for i in path:
pa[i]=list(set(sum(path[i],[])))
for i in pa:
for j in list(set(pa[i])):
if j in protlist:
continue
else:
count[j]=count[j]+1
countsort = sorted(count, key=count.get)
removable=set(countsort).difference(set(protlist))
print len(protlist)
graphred=check(graph,path_lenght,removable,protlist,path)
for i in graphred.nodes():
if i in protlist:
graphred.node[i]["group"]=5
else:
graphred.node[i]["group"]=10
f1=open("bwmodproteins.txt","w")
for i in graphred.nodes():
f1.write(i+"\n")
d = json_graph.node_link_data(graphred) # node-link format to serialize
# write json
json.dump(d, open('filteredgraph.json','w'))
nx.write_gpickle(graphred,"bwmodfiltgraph.gpickle")
def main():
Motifset = Motifsets()
patterns2 = enumerate2()
output_file1 = "/net/data/graph-models/sim-graphs/approx3-json"
with open(output_file1, 'w') as fout:
for item in patterns2:
string_item = json.dumps(json_graph.node_link_data(item))
fout.write(string_item + "\n")
subprocess.check_call("hdfs dfs -put /net/data/graph-models/sim-graphs/approx3-json approx3-json", shell=True)
approx3Motifs = sc.textFile("hdfs://scrapper/user/xiaofeng/approx3-json", 192)
#.number of partitions
collapsed_patterns = approx3Motifs.flatMap(lambda line: worker_all_collapse(Motifset, line))
subprocess.check_call("hdfs dfs -rm -r patterns_queue", shell=True)
collapsed_patterns.saveAsTextFile("hdfs://scrapper/user/xiaofeng/patterns_queue")
#save to HDFS, as a text file, and keep using that RDD
collapsed_patterns.persist()
non_iso_set = set()
# while not collapsed_patterns.isEmpty(): #or use count() != 0 as an alternative
# povet = collapsed_patterns.take(1)[0]#BROADCAST
# povet_broad = sc.broadcast(povet)
# print type(povet)
# non_iso_set.add(povet)
# collapsed_patterns = collapsed_patterns.filter(lambda x: not nx.is_isomorphic(x, povet_broad.value))
#
###########write to hard disk the queue of elements waiting to be processed
while True:
collapsed_patterns = sc.textFile("hdfs://scrapper/user/xiaofeng/patterns_queue")
if collapsed_patterns.count() == 0:
break
else:
povet = collapsed_patterns.take(1)[0]#BROADCAST
povet_broad = sc.broadcast(povet)
non_iso_set.add(povet)
collapsed_patterns_new = collapsed_patterns.filter(lambda x: not nx.is_isomorphic(x, povet_broad.value))
subprocess.check_call("hdfs dfs -rm -r patterns_queue", shell=True)
collapsed_patterns_new.saveAsTextFile("hdfs://scrapper/user/xiaofeng/patterns_queue")
print collapsed_patterns.count()
output_file2 = "/net/data/graph-models/sim-graphs/approx5-json"
with open(output_file2, 'w') as fout:
for item in non_iso_set:
string_item = json.dumps(json_graph.node_link_data(item))
fout.write(string_item + '\n')
def store_network_as_jason(up_path_edges,down_path_edges,upband=2000,downband=2000):
'''
Description:
This function export the node and link information to the frontend lib, to visiualize the
topo and the calculated path between hosts.
'''
graph1=nx.DiGraph()
graph2=nx.DiGraph()
for edge in edges:
if edge['localnode'] not in graph1.nodes():
group=1
if edge['localaddr'] in ipmap.keys():
group=0
graph1.add_node(edge['localnode'],{'localinterface':edge['localifname'],'IPAdd':edge['localaddr'],'group':group})
if edge['remotenode'] not in graph1.nodes():
group=1
if edge['remoteaddr'] in ipmap.keys():
group=0
graph1.add_node(edge['remotenode'],{'remoteinterface':edge['remoteifname'],'IPAdd':edge['remoteaddr'],'group':group})
graph1.add_edge(edge['localnode'],edge['remotenode'],{'AvailableBandwidth':edge['available_bandwidth'],'value':edge['available_bandwidth']})
for edge in up_path_edges:
if edge['localnode'] not in graph2.nodes():
group=1
if edge['localaddr'] in ipmap.keys():
group=0
graph2.add_node(edge['localnode'],{'localinterface':edge['localifname'],'IPAdd':edge['localaddr'],'group':group})
if edge['remotenode'] not in graph2.nodes():
group=1
if edge['remoteaddr'] in ipmap.keys():
group=0
graph2.add_node(edge['remotenode'],{'remoteinterface':edge['remoteifname'],'IPAdd':edge['remoteaddr'],'group':group})
graph2.add_edge(edge['localnode'],edge['remotenode'],{'UpBandwidth':upband,'value':upband})
for edge in down_path_edges:
if edge['localnode'] not in graph2.nodes():
group=1
if edge['localaddr'] in ipmap.keys():
group=0
graph2.add_node(edge['localnode'],{'localinterface':edge['localifname'],'IPAdd':edge['localaddr'],'group':group})
if edge['remotenode'] not in graph2.nodes():
group=1
if edge['remoteaddr'] in ipmap.keys():
group=0
graph2.add_node(edge['remotenode'],{'remoteinterface':edge['remoteifname'],'IPAdd':edge['remoteaddr'],'group':group})
graph2.add_edge(edge['localnode'],edge['remotenode'],{'DownBandwidth':downband,'value':downband})
for node in host.keys():
graph1.add_node(host[node],{'IPAdd':node,'group':0})
graph2.add_node(host[node],{'IPAdd':node,'group':0})
graph1.add_edge(host[node],ipmap[node])
graph2.add_edge(host[node],ipmap[node])
d1=json_graph.node_link_data(graph1)
d2=json_graph.node_link_data(graph2)
json.dump(d1,open('/Users/eric/Desktop/topo/1.json','w'))
json.dump(d2,open('/Users/eric/Desktop/topo/2.json','w'))
def vis_hierarchy(sieve, column_label, max_edges=200, prefix=''):
"""Visualize a hierarchy of representations."""
import textwrap
column_label = map(lambda q: '\n'.join(textwrap.wrap(q, width=20)), column_label)
def f(j):
if j < sieve.nv:
return j
else:
return (1, j - sieve.nv)
# Construct non-tree graph
g = nx.DiGraph()
max_node_weight = np.max(sieve.tcs)
for i, c in enumerate(column_label):
if i < sieve.nv:
g.add_node(i)
g.node[i]['weight'] = 1
g.node[i]['label'] = c
g.node[i]['name'] = c # JSON uses this field
else:
g.add_node(f(i))
g.node[f(i)]['weight'] = 0.33 * np.clip(sieve.tcs[i - sieve.nv] / max_node_weight, 0.33, 1)
if i >= sieve.nv:
g.add_weighted_edges_from([(f(j), (1, i - sieve.nv), sieve.mi_j(i - sieve.nv)[j]) for j in range(i)])
# Display pruned version
h = g.copy() # trim(g.copy(), max_parents=max_parents, max_children=max_children)
h.remove_edges_from(sorted(h.edges(data=True), key=lambda q: q[2]['weight'])[:-max_edges])
edge2pdf(h, prefix + '/graphs/graph_%d' % max_edges, labels='label', directed=True, makepdf=True)
# Display tree version
tree = g.copy()
tree = trim(tree, max_parents=1, max_children=False)
edge2pdf(tree, prefix + '/graphs/tree', labels='label', directed=True, makepdf=True)
# Output JSON files
try:
import os
print os.path.dirname(os.path.realpath(__file__))
copyfile(os.path.dirname(os.path.realpath(__file__)) + '/tests/d3_files/force.html', prefix + '/graphs/force.html')
except:
print "Couldn't find 'force.html' file for visualizing d3 output"
import json
from networkx.readwrite import json_graph
mapping = dict([(n, tree.node[n].get('label', str(n))) for n in tree.nodes()])
tree = nx.relabel_nodes(tree, mapping)
json.dump(json_graph.node_link_data(tree), safe_open(prefix + '/graphs/force.json', 'w+'))
json.dump(json_graph.node_link_data(h), safe_open(prefix + '/graphs/force_nontree.json', 'w+'))
return g
def NetworkAnalysis(jsonGraph):
"""gets graph defined by for export json and computes the top 3
most connected subgraphs"""
G = json_graph.node_link_graph(jsonGraph)
graphs = sorted(nx.connected_component_subgraphs(G), key = len, reverse=True)
(GC1, GC2, GC3, GC4, GC5) = graphs[0:5]
top5 = nx.compose_all([GC1,GC2,GC3,GC4,GC5])
deg = top5.degree()
nx.set_node_attributes(top5, "degree", deg)
take = {
"nodes": json_graph.node_link_data(top5)["nodes"],
"links": json_graph.node_link_data(top5)["links"]
}
return take
def vis_hierarchy(corexes, column_label=None, max_edges=100, prefix='topics', n_anchors=0):
"""Visualize a hierarchy of representations."""
if column_label is None:
column_label = map(str, range(corexes[0].alpha.shape[1]))
# make l1 label
alpha = corexes[0].alpha
mis = corexes[0].mis
l1_labels = []
annotate = lambda q, s: q if s > 0 else '~' + q
for j in range(corexes[0].n_hidden):
# inds = np.where(alpha[j] * mis[j] > 0)[0]
inds = np.where(alpha[j] >= 1.)[0]
inds = inds[np.argsort(-alpha[j, inds] * mis[j, inds])]
group_number = u"red_" + unicode(j) if j < n_anchors else unicode(j)
label = group_number + u':' + u' '.join([annotate(column_label[ind], corexes[0].sign[j,ind]) for ind in inds[:6]])
label = textwrap.fill(label, width=25)
l1_labels.append(label)
# Construct non-tree graph
weights = [corex.alpha.clip(0, 1) * corex.mis for corex in corexes[1:]]
node_weights = [corex.tcs for corex in corexes[1:]]
g = make_graph(weights, node_weights, l1_labels, max_edges=max_edges)
# Display pruned version
h = g.copy() # trim(g.copy(), max_parents=max_parents, max_children=max_children)
edge2pdf(h, prefix + '/graphs/graph_prune_' + str(max_edges), labels='label', directed=True, makepdf=True)
# Display tree version
tree = g.copy()
tree = trim(tree, max_parents=1, max_children=False)
edge2pdf(tree, prefix + '/graphs/tree', labels='label', directed=True, makepdf=True)
# Output JSON files
try:
import os
copyfile(os.path.dirname(os.path.realpath(__file__)) + '/tests/d3_files/force.html', prefix + '/graphs/force.html')
except:
print "Couldn't find 'force.html' file for visualizing d3 output"
import json
from networkx.readwrite import json_graph
mapping = dict([(n, tree.node[n].get('label', str(n))) for n in tree.nodes()])
tree = nx.relabel_nodes(tree, mapping)
json.dump(json_graph.node_link_data(tree), safe_open(prefix + '/graphs/force.json', 'w+'))
json.dump(json_graph.node_link_data(h), safe_open(prefix + '/graphs/force_nontree.json', 'w+'))
return g
def nonSocialConnect(USER,REPO):
filename = "./static/bootstrap/data/" +USER +REPO+ ".json"
if os.path.exists(filename):
return 1
else:
username,password=readcfg()
client = Github(login_or_token=username,password=password, per_page=100)
user = client.get_user(USER)
repo = user.get_repo(REPO)
print 'chgithub.SocialConnect->get start'
stargazers = [ s for s in repo.get_stargazers() ] #获得关注者，通常这个人数比较多
contributors = [ s for s in repo.get_contributors() ] #获得贡献者
g = nx.DiGraph()
g.add_node(repo.name + '(r)', type='repo', lang=repo.language, owner=user.login)
for sg in stargazers:
g.add_node(sg.login + '(u)', type='user')
g.add_edge(sg.login + '(u)', repo.name + '(r)', type='gazes')
print 'chgithub.SocialConnect->finish add stargazers'
for sg in contributors:
g.add_node(sg.login + '(u)', type='user')
g.add_edge(sg.login + '(u)', repo.name + '(r)', type='conbs')
print 'chgithub.SocialConnect->finish add contributors'
d = json_graph.node_link_data(g)
json.dump(d, open(filename, 'w'))
print 'chgithub.SocialConnect->DONE'
return 1
def create_filtered_graph(gfilter):
try:
gfilter = json.loads(gfilter)
print_json(gfilter)
except:
warnings.warn('Ошибка при обработке json-массива gfilter', UserWarning)
raise
# Создаем граф из данных "семантической кучи";
# производим фильтрацию узлов графа по переданному массиву типов сущностей taxonomy;
SG = SGraph()
G = SG.create(int(gfilter.get('stopper')), gfilter.get('taxonomy'))
# Исключаем из графа узлы с нулевым весом (без связей)
G = GFilterZero(G, gfilter['options'].get('removeZero'))
#G = GFilterAttributes(G, gfilter.get('attributes')) # Фильтрация узлов графа по переданным в ассоциативном массивe attributes атрибутам узлов;
data = json_graph.node_link_data(G) # Средствами бибилиотеки NetworkX, экспортируем граф в виде подходящeм для json-сериализации
graph = StorageGraph() # Создаём экземпляр класса Graph, для хранения структуры графа в базе данных
numberOfNodes = G.number_of_nodes() # Получаем кол-во узлов графа
numberOfEdges = G.number_of_edges() # Получаем кол-во дуг графа
graph.title = "Проекция: узлов " + str(numberOfNodes) + "; дуг " + str(numberOfEdges) # Определяем заголовок графа
graph.body = json.dumps(data, ensure_ascii=False) # Преобразуем данные в json-формат
#graph.layout_spring = to_main_graph(graph.body) # получаем массив компоновки по-умолчанию (типа spring)
graph.save() # Сохраняем граф в собственную базу данных
#jsonContent = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '), ensure_ascii=False); print(jsonContent) # отладочная информация
pdev('yзлов %i, дуг %i' % (numberOfNodes, numberOfEdges)) # отладка: выводим кол-во узлов и дуг
return graph.body
def create_graph(self):
name = self.name
groups = self.groups
try:
groups = open(groups, "r")
load_groups = json.load(groups)
G = nx.Graph()
file_list = Functions.open_folder(name,'C:/Users/MOYIN/Desktop/Flask/WebSc/result/' + name + '/')
print(file_list)
for file in file_list:
load = open(file, "r")
result_json = json.load(load)
G.add_node("Emotion", x=500, y=400, fixed=True)
for wd in result_json['emotions']:
for word in wd:
G.add_node(word, group=load_groups[word])
G.add_edge("Emotion", word, value=wd[word])
d = json_graph.node_link_data(G)
# file = open("result\\" + self + "\\Force_layout\\" + os.path.basename(file), 'w')
filex = open("C:/Users/MOYIN/Desktop/Flask/static/Companies/" + name + "/"+os.path.basename(file), 'w')
json.dump(d, filex)
print("Graph files created: ", filex.name)
return True
except BaseException as e:
print("Graph creation error : ", e)
def save_to_jsonfile(filename, graph): ''' Save graph object to filename ''' g = graph g_json = json_graph.node_link_data(g) # node-link format to serialize json.dump(g_json, open(filename,'w'))
def main():
"""
Populate a graph using an iterator over our Hbase table.
"""
G = nx.Graph()
for tweet_id, tweet_dict in getHbaseIterator():
# Parse the tweet out
'''
Information contained in tweet dict:
- text
- in_reply_to_status_id
- entities
- in_reply_to_screen_name
- id_str
- retweet_count
- in_reply_to_user_id
- created_at
'''
text = tweet_dict['text']
in_reply_to_status_id = tweet_dict['in_reply_to_status_id']
in_reply_to_user_id = tweet_dict['in_reply_to_user_id']
retweet_count = tweet_dict['retweet_count']
url = "http://www.twitter.com/{}/status/{}".format(username, tweet_id)
# Add it to graph as a node
G.add_node(node_name, retweet_count=retweet_count, url=url, text=text)
# Add its edges
G.add_edge(tweet_id, in_reply_to_status_id)
# Dump to JSON
data = json_graph.node_link_data(G)
with open("hbase-link.json", 'w') as f:
json.dump(data, f)
def _install_one_route(self, chain_graph, res_graph, s, t, backroute=False):
self._debug('Install route between %s - %s (backroute=%s)' %
(s, t, backroute))
route_id = self.next_route_id()
self.routes[route_id] = { 'chain': [],
'res': [],
'status': RouteChanged.PENDING,
'res_graph': res_graph,
}
if backroute:
#route_search = DefaultRouteAlgorithm()
#route_search.graph(json_graph.node_link_data(res_graph))
# send backward traffic directly to the source:
chain_hops = [(s, t)]
else:
route_search = self.chain_route_search
route_search.graph(json_graph.node_link_data(chain_graph))
chain_hops = route_search.chain_hops(s, t)
self.routes[route_id]['chain'] = chain_hops
self._fire_route_state_change(None, route_id)
self.install_pending_routes(res_graph)
def generate_weak_links_map(self):
weak_nodes = self.detect_weak_nodes(-5,25)
active_weak_nodes = [node[0] for node in weak_nodes if max([l[1] for l in node[1]]) > 10]
ap_nodes = [node for node in self.g.nodes() if self.g.in_degree(node) > 0]
edges = self.g.edges(active_weak_nodes)
snr_g = nx.DiGraph()
snr_g.add_nodes_from(active_weak_nodes + ap_nodes)
snr_g.add_edges_from(edges)
for node in active_weak_nodes:
snr_g.node[node]['type'] = 'sta'
for node in ap_nodes:
snr_g.node[node]['type'] = 'ap'
nx.write_gpickle(snr_g,'graph_pickle_connectivity_%d.pkl' % time.time())
#nx.draw(snr_g,with_labels=False)
#pylab.savefig("connectivity-graph-%d.png" % (int(time.time())))
d = json_graph.node_link_data(snr_g) # node-link format to serialize
# write json
json.dump(d, open('force/force.json','w'))
print ap_nodes
def _measurement(self):
while True:
print 'ActiveFlows: ', self.active_flows
print 'FlowRate: ', self.flow_rate
print 'Graph: ', json.dumps(json_graph.node_link_data(self.graph))
self._send_measure_request()
hub.sleep(1)
def simpleDisplay(ipaddress = "localhost",port = "9999"):
'''
利用每次处理后保存的图来进行恢复展示
:return:
'''
# client,repo,stargazers,user = getRespond()
# g = addTOGraph(repo,stargazers,user)
# addEdge(stargazers,client,g)
# getPopular(g)
# savaGraph1(g)
# top10(g)
g = nx.read_gpickle("data/github.1")
print nx.info(g)
print
mtsw_users = [n for n in g if g.node[n]['type'] == 'user']
h = g.subgraph(mtsw_users)
print nx.info(h)
print
d = json_graph.node_link_data(h)
json.dump(d, open('data/githubRec.json', 'w'))
cmdstr = "python3 -m http.server %s" % port
webbrowser.open_new_tab("http://%s:%s/%s.html"%(ipaddress,port, "display_githubRec"))
os.system(cmdstr)
def plot_graph(graph, scope=None, parent=None,
excludes=(), d3page='fixedforce.html', minimal=False):
"""Open up a display of the graph in a browser window."""
tmpdir = tempfile.mkdtemp()
fdir = os.path.dirname(os.path.abspath(__file__))
shutil.copy(os.path.join(fdir, 'd3.js'), tmpdir)
shutil.copy(os.path.join(fdir, d3page), tmpdir)
graph = _clean_graph(graph, excludes=excludes,
scope=scope, parent=parent, minimal=minimal)
data = node_link_data(graph)
tmp = data.get('graph', [])
data['graph'] = [dict(tmp)]
startdir = os.getcwd()
os.chdir(tmpdir)
try:
# write out the json as a javascript var
# so we we're not forced to start our own webserver
# to avoid cross-site issues
with open('__graph.js', 'w') as f:
f.write("__mygraph__json = ")
json.dump(data, f)
f.write(";\n")
# open URL in web browser
wb = webbrowser.get()
wb.open('file://'+os.path.join(tmpdir, d3page))
except Exception as err:
print str(err)
finally:
os.chdir(startdir)
print "remember to remove temp directory '%s'" % tmpdir
def build_pi_graph(criterion_uri):
G = nx.DiGraph()
q_up = render_template('pi_to_pi_upwards.q', criterion_uri = criterion_uri)
q_down = render_template('pi_to_pi_downwards.q', criterion_uri = criterion_uri)
q_trial = render_template('pi_originating_trial.q', criterion_uri = criterion_uri)
G = build_graph(G, criterion_uri, source="child", target="trial", query=q_up, intermediate = "parent")
G = build_graph(G, criterion_uri, source="parent", target="trial", query=q_down, intermediate = "child")
G = build_graph(G, criterion_uri, source="child", target="trial", query=q_trial)
types = {}
names = {}
for n, nd in G.nodes(data=True):
print nd['type']
if nd['type'] != 'trial' :
label = nd['label'][:-3]
types[n] = 'criterion_cg'
names[n] = label
else :
label = nd['label'].replace('Trial','').upper()
types[n] = 'trial_cg'
names[n] = label
nx.set_node_attributes(G,'type', types)
nx.set_node_attributes(G,'label', names)
g_json = json_graph.node_link_data(G) # node-link format to serialize
return g_json
def draw_to_browser(agents):
data1 = {}
data2= []
for i in range(0,250): #param
for j in range(20):
#print graphStrings[i]
G = create_graph_of_agent(agents[str(i)][str(j)])
d = json_graph.node_link_data(G)
d["directed"] = 1
d["multigraph"] = 1
text= "s " +str(i + 1)+" | " + str(j + 1)
data1[text] = d
data2.append({"text": text, "value": text})
data = {"0": data1,
"codes": data2}
write_html(data)
json.dump(data, open('IPD_output/data.json', 'w'))
print('Wrote node-link JSON data to temp.json')
# open URL in running web browser
http_server.load_url('IPD_output/overall.html')
print('Or copy all files to webserver and load graph.html')
def _with_networkx(documents, threshold=1):
G = nx.Graph()
G.add_nodes_from(documents)
nodes = G.nodes()
for i, node in enumerate(nodes):
for other in nodes[i+1:]:
a = set(node.keywords)
b = set(other.keywords)
intersection = a.intersection(b)
if len(intersection) > threshold:
G.add_edge(node, other)
G[node][other]['weight'] = len(intersection)
# remove any isolated vertices before we perform community detection
orphans = []
for node in G.nodes():
if not G.neighbors(node):
G.remove_node(node)
orphans.append(node)
partition_lookup = community.best_partition(G).iteritems()
G.add_nodes_from(orphans)
partitions = {node.r_id: value for node, value in partition_lookup}
as_json = json_graph.node_link_data(G)
frontend_compatable = {}
frontend_compatable['nodes'] = [node['id'] for node in as_json['nodes']]
for node in frontend_compatable['nodes']:
if G.neighbors(node):
node.partition = partitions[node.r_id]
frontend_compatable['nodes'] = [json.loads(node.to_json()) for node in frontend_compatable['nodes']]
for node in frontend_compatable['nodes']:
if node['_id'] in partitions:
node['partition'] = partitions[node['_id']]
frontend_compatable['edges'] = as_json['links']
return frontend_compatable
def build_trial_to_criterion_graph(trial_uri, trial_id):
G = nx.Graph()
q = render_template('specific_trial_to_criterion.q', trial_uri = trial_uri)
G = build_graph(G, trial_uri, "trial", "concept", q, intermediate = "criterion")
origin_node_id = "Trial{}".format(trial_id.lower())
G.node[origin_node_id]['type'] = 'origin'
names = {}
for n, nd in G.nodes(data=True):
if nd['type'] == 'trial' or nd['type'] == 'origin':
label = nd['label'].replace('Trial','').upper()
names[n] = label
else :
names[n] = nd['label']
nx.set_node_attributes(G,'label', names)
deg = nx.degree(G)
nx.set_node_attributes(G,'degree',deg)
g_json = json_graph.node_link_data(G) # node-link format to serialize
return g_json
def main(name, divide):
"""
old_g = pickle.load(open("/net/data/facebook/facebook-ucsb/Facebook_2008/"+name +"/original_pickles/"+name +".pickle", 'r'))
new_g = networkx.Graph()
for node, friends in old_g.adj.iteritems():
if node not in new_g.nodes():
new_g.add_node(node)
for friend in friends.iterkeys():
new_g.add_node(friend)
new_g.add_edge(node, friend)
"""
# serialize the networkx graph as text files of edgelist
# into a text file for workers to read
# networkx.write_edgelist(new_g, "edgelist/"+name, data=False)
# subprocess.check_call("hdfs dfs -put edgelist/"+name+ " edgelist/", shell=True)
new_g = networkx.read_adjlist(name + "_list.txt") # Egypt_list is an edge list
sc = SparkContext(appName="Sorted_removal")
dataG = json_graph.node_link_data(new_g)
stringG = json.dumps(dataG)
originalG = sc.broadcast(stringG)
edges = sc.textFile("hdfs://scrapper/user/xiaofeng/edgelist/" + name, 192 * 4 * int(divide))
costs = edges.map(lambda line: line.split(" ")).map(lambda edge: edge_to_cost(edge, originalG.value))
costs.saveAsTextFile("hdfs://scrapper/user/xiaofeng/costs_" + name)
sc.stop()
subprocess.check_call("hdfs dfs -get costs_" + name + " /home/xiaofeng/facebook/FacebookProject/costs/", shell=True)
Reformat("/home/xiaofeng/facebook/FacebookProject/costs/costs_" + name + "/", name)
def to_json(net, gformat):
if "list" in gformat:
jsn = jsg.node_link_data(net)
else:
jsn = jsg.adjacency_data(net)
jsn = json.dumps(jsn, indent=2, sort_keys=True)
return jsn
def default(self, obj):
if isinstance(obj, set):
return str(obj)
if isinstance(obj, netaddr.IPAddress):
return str(obj)
if isinstance(obj, netaddr.IPNetwork):
return str(obj)
if isinstance(obj, autonetkit.nidb.DmNode):
#TODO: need to unserialize nidb nodes...
return str(obj)
if isinstance(obj, autonetkit.anm.NmEdge):
log.warning("%s is anm overlay_edge. Use attribute rather than object in compiler." % obj)
return str(obj)
if isinstance(obj, autonetkit.nidb.ConfigStanza):
retval = obj.to_json()
return retval
if isinstance(obj, autonetkit.nidb.DmInterface):
#TODO: check this is consistent with deserialization
return str(obj)
if isinstance(obj, nx.classes.Graph):
#TODO: remove now?
return json_graph.node_link_data(obj)
if isinstance(obj, logging.LoggerAdapter):
#TODO: filter this out in the to_json methods
return ""
return json.JSONEncoder.default(self, obj)
def mk_mean_digraph(self, output):
Hnx = nx.DiGraph()
for node in self.digraph.vs:
Hnx.add_node(node["name"])
for node in self.digraph.vs:
for innode in node.predecessors():
idEdge = self.digraph.get_eid(innode.index, node.index)
edge = self.digraph.es[idEdge]
mean = np.mean(sum(edge.attributes().values(), []))
Hnx.add_edge(innode["name"], node["name"], value = mean, type = "arrow_in")
for outnode in node.successors():
idEdge = self.digraph.get_eid(node.index, outnode.index)
edge = self.digraph.es[idEdge]
mean = np.mean(sum(edge.attributes().values(), []))
Hnx.add_edge(node["name"], outnode["name"], value = mean, type = "arrow_out")
#check for self loops and null
for edge in Hnx.edges():
if Hnx.edge[edge[0]][edge[1]]["value"] < 1.:
print edge
#del Hnx.edge[edge[0]][edge[1]]
elif edge[0] == edge[1]:
del Hnx.edge[edge[0]][edge[1]]
self.Hnx = nx.relabel_nodes(Hnx, names)
self.data = json_graph.node_link_data(self.Hnx)
formated = json.dumps(self.data).replace("id", "name").replace("},", "},\n")
out = open(output, "w")
out.write(formated)
out.close()
def graphmltojson(graphfile, outfile): """ Converts GraphML file to json while adding communities/modularity groups using python-louvain. JSON output is usable with D3 force layout. Usage: >>> python convert.py -i mygraph.graphml -o outfile.json """ G = nx.read_graphml(graphfile) G = nx.Graph(G) #G = nx.DiGraph.to_undirected(G) #karate = Nexus.get(G) #cl = karate.community_fastgreedy() #k = 57 #cl.as_clustering(k).membership #finds best community using louvain partition = community.best_partition(G) #adds partition/community number as attribute named 'modularitygroup' for n,d in G.nodes_iter(data=True): d['group'] = partition[n] node_link = json_graph.node_link_data(G) json = json_graph.dumps(node_link) # Write to file fo = open(outfile, "w") fo.write(json); fo.close()
def build_topic_graph(documents):
G = nx.Graph()
# used to keep track of and increment the number of connections that
# exist between any two keywords
edge_weights = collections.Counter()
keyword_frequencies = collections.Counter(
[keyword['text'] for document in documents for keyword in document.keywords])
for document in documents:
for keyword in document.keywords:
G.add_node(keyword['text'], frequency=keyword_frequencies[keyword['text']])
# Its alright to just add the keywords without checking if it is
# already in the graph. NetworkX doesn't modify the graph if that
# is the case.
document_keywords = map(lambda x: x['text'], document.keywords)
G.add_nodes_from(document_keywords)
edges = itertools.combinations(document_keywords, 2)
for edge in edges:
# unlike sets frozensets are hashable
_set = frozenset(edge)
edge_weights[_set] += 1
x, y = edge
if edge_weights[_set] > 1:
G.add_edge(x, y, weight=edge_weights[_set])
for node in G.nodes():
if keyword_frequencies[node] < 2:
G.remove_node(node)
return json_graph.node_link_data(G)
def dump(self, fname):
data = json_graph.node_link_data(self.G)
# save nodes as simple objects
nodes = []
for node in data["nodes"]:
cell = node["id"]
if type(cell.value) == Range:
range = cell.value
value = [(range.cells, range.nb_rows, range.nb_cols), range.values()]
else:
value = cell.value
nodes += [{
"address": cell.address(),
"formula": cell.formula,
"value": value,
"python_expression": cell.python_expression,
"is_named_range": cell.is_named_range,
"always_eval": cell.always_eval
}]
data["nodes"] = nodes
# save ranges as simple objects
ranges = {k: [(r.cells, r.nb_rows, r.nb_cols), r.values()] for k,r in self.ranges.items()}
data["ranges"] = ranges
data["named_ranges"] = self.named_ranges
with gzip.GzipFile(fname, 'w') as outfile:
outfile.write(json.dumps(data))
def main(dataset):
# Load in kgat data
data_dir = "/people/hamc649/recommendation/kgat_data"
#data_dir = '/Users/d3m432/git/GraphSAGE/sutanay_graphsage/kgat_data'
#dataset = 'last-fm'
dataset_dir = f"{data_dir}/{dataset}"
output_dir = "/people/hamc649/recommendation/GraphSAGE/example_data"
#output_dir = '/Users/d3m432/git/GraphSAGE/sutanay_graphsage/GraphSAGE/example_data'
train_path = f"{dataset_dir}/train.txt"
valid_path = f"{dataset_dir}/valid.txt" # exclude false edges
kg_path = f"{dataset_dir}/kg_final.txt"
node_type_vocab_path = f"{dataset_dir}/node_type_vocab.txt"
# read in edges
edges = list()
train_edges = read_edges(train_path, 'train')
valid_edges = read_edges(valid_path, 'valid')
kg_edges = read_edges(kg_path, 'train')
edges.extend(train_edges)
edges.extend(valid_edges)
edges.extend(kg_edges)
# Get all node ids from edges
node_ids = set()
for edge in edges:
n1 = edge[0]
n2 = edge[1]
node_ids.add(n1)
node_ids.add(n2)
max_node_id = max(node_ids)
min_node_id = min(node_ids)
# Read in maps
with open(node_type_vocab_path, 'r') as f:
node_type_vocab = json.load(f)
# convert map to one hot
node_type_vocab_oh = dict()
for k in node_type_vocab:
v = node_type_vocab[k]
if v == 'item':
nv = [1, 0, 0]
elif v == 'entity':
nv = [0, 1, 0]
else:
# user
nv = [0, 0, 1]
node_type_vocab_oh[k] = nv
# Transform kgat_data into networkx graph
G = nx.from_edgelist(edges)
# We don't need to hardcode max_node_id
all_node_ids = {i for i in range(max_node_id + 1)}
remaining_node_ids = all_node_ids.difference(node_ids)
#import pdb;pdb.set_trace()
#for i in range(max_node_id + 1):
# G.add_node(i)
for node_id in remaining_node_ids:
G.add_node(node_id)
#import pdb;pdb.set_trace()
nx.set_node_attributes(G, 'val', False)
nx.set_node_attributes(G, 'test', False)
#import pdb;pdb.set_trace()
# Dump necessary networkx files
# 1. Dump graph
output_path = f"{output_dir}/{dataset}-G.json"
data = json_graph.node_link_data(G)
#import pdb;pdb.set_trace()
with open(output_path, 'w') as f:
json.dump(data, f)
# 2. dump class map
# for each node, add class to dictionary
class_map = dict()
nodes = G.nodes()
print(f"Num nodes: {G.number_of_nodes()}, Num edges: {G.number_of_edges()}")
for node in nodes:
#class_map[str(node)] = [1, 0, 0]
class_map[str(node)] = node_type_vocab_oh[str(node)]
class_map_path = f"{output_dir}/{dataset}-class_map.json"
with open(class_map_path, 'w') as f:
json.dump(class_map, f)
# 3. dump id map
id_map = {str(i): i for i in range(len(nodes))}
id_map_path = f"{output_dir}/{dataset}-id_map.json"
with open(id_map_path, 'w') as f:
json.dump(id_map, f)
def dump(G):
"""
Convert nx graph G as a JSON dump
"""
data = json_graph.node_link_data(G)
return data
def test_file_to_nx(graph_tools):
g = graph_tools.file_to_nx("test_graph.json")
data = json_graph.node_link_data(g)
print(f"test_file_to_nx {data['nodes'][0]['attr_dict']}")
assert data['nodes'][0]['attr_dict']['id'] == 'DOID:9352'
def start(login, depth=2):
nodes = multiprocessing.Manager().dict()
tasks = multiprocessing.Queue()
links = multiprocessing.Manager().list()
lock = multiprocessing.Lock()
indices = multiprocessing.Value('i', 0)
AMOUNT_OF_PROCESS = multiprocessing.cpu_count() * 6
node = (login, 0)
nodes[node] = indices.value
indices.value += 1
requester = session.get_session()
if is_valid(login, requester):
following = extract_info(FOLLOWING_URL % login, requester)
followers = extract_info(FOLLOWERS_URL % login, requester)
intersection = set(following).intersection(followers)
for user in intersection:
tmpu = (user, 1)
nodes[tmpu] = indices.value
indices.value += 1
tasks.put(tmpu)
links.append({"source": nodes[node], "target": nodes[tmpu]})
else:
sys.exit(0)
requests = [session.get_session() for i in xrange(AMOUNT_OF_PROCESS)]
process = [None for i in xrange(AMOUNT_OF_PROCESS)]
for i in xrange(AMOUNT_OF_PROCESS):
process[i] = multiprocessing.Process(target=worker,
args=(login, depth, requests[i],
nodes, links, tasks, lock,
indices))
process[i].start()
for i in xrange(AMOUNT_OF_PROCESS):
process[i].join()
print "generate graph ..."
sorted_nodes = sorted(dict(nodes).items(), key=operator.itemgetter(1))
nodes = [{
"name": node[0][0],
"group": node[0][1]
} for node in sorted_nodes]
links = [link for link in links]
for link in links[:]:
if {"source": link["target"], "target": link["source"]} not in links:
links.remove(link)
data = {"nodes": nodes, "links": links}
graph = json_graph.node_link_graph(data, directed=False, multigraph=False)
graphs = list(networkx.connected_component_subgraphs(graph))
for graph in graphs:
if 0 in graph.nodes():
nodes = [
node["name"] for node in nodes
if nodes.index(node) in graph.nodes()
]
labels = {}
for index, node in zip(graph.nodes(), nodes):
labels[index] = node
graph = networkx.relabel_nodes(graph, labels)
data = json_graph.node_link_data(graph)
# with open("/var/www/html/msif/" + login + "_twitter.json", 'w') as outfile:
# with open("/var/www/html/msif/twitter.json", 'w') as outfile:
with open("./static/data/" + login + "_twitter.json",
'w') as outfile:
json.dump(data, outfile)
matrix = networkx.to_numpy_matrix(graph)
return matrix, nodes
def write_json_graph(path, graph, indent=None):
data = json_graph.node_link_data(graph)
with open(path, 'w') as output:
json.dump(data, output, cls=EntityEncoder, indent=indent)
#!/usr/bin/python
import json
import networkx as nx
from networkx.readwrite import json_graph
#open input directory
f = open('databefore.json')
nodes = json.load(f)
f.close()
#link the nodes from input
graph = json_graph.node_link_graph(nodes)
#call library
while nx.number_connected_components(graph) < 2:
edges = nx.edge_betweenness_centrality(graph)
Max = 0
for edge in edges:
if edges[edge] > Max:
#iterate over it
Max_edge = edge
Max = edges[edge]
graph.remove_edge(Max_edge[0], Max_edge[1])
output = json_graph.node_link_data(graph)
g = open('dataAfter.json', 'w')
g.write(json.dumps(output))
g.close()
print("Name:", tb[0], "| Betweenness Centrality:", tb[1], "| Degree:", degree)
communities = community.greedy_modularity_communities(G)
modularity_dict = {} # Create a blank dictionary
for i,c in enumerate(communities): # Loop through the list of communities, keeping track of the number for the community
for name in c: # Loop through each person in a community
modularity_dict[name] = i # Create an entry in the dictionary for the person, where the value is which group they belong to.
# Now you can add modularity information like we did the other metrics
nx.set_node_attributes(G, modularity_dict, 'modularity')
nx.write_gexf(G, 'quaker_network.gexf')
from networkx.readwrite import json_graph
import json
data1 = json_graph.node_link_data(G)
# with open('personal.json', 'w') as json_file:
# json.dump(data1, json_file)
json_dump = json.dumps(data1)
print(json_dump)
filename_out = 'pcap_export.json'
json_out = open(filename_out,'w')
json_out.write(json_dump)
json_out.close()
def draw(self):
G = networkx.convert_node_labels_to_integers(self.G,
label_attribute='url')
d = json_graph.node_link_data(G)
return json.dumps(d)
def for_json(self):
data = {PYANNOTE_JSON: self.__class__.__name__}
data[PYANNOTE_JSON_CONTENT] = node_link_data(self)
return data
def genNewGraph(graph_id, total_nodes, frac_primal, num_k):
dir_name = "./GraphSAGE-master/graph_data/" + "graph" + str(graph_id) + "/"
if not os.path.exists(os.path.dirname(dir_name)):
os.makedirs(dir_name)
graph_name = dir_name + "graph" + str(graph_id)
# UG = nx.gnm_random_graph(total_nodes, edges_per_node)
UG = gen_setcover_inst(total_nodes,frac_primal)
# page_rank = nx.pagerank(UG)
# div_rank = divrank(UG)
degree_of_nodes = UG.degree()
all_nodes = nx.nodes(UG)
features = []
sum_degree = 0
for node_i in all_nodes:
features.append([])
sum_degree = sum_degree + degree_of_nodes[node_i]
for node_i in all_nodes:
# features[node_i].append(page_rank[node_i])
# features[node_i].append(div_rank[node_i])
norm_value = degree_of_nodes[node_i]*1.0/sum_degree
features[node_i].append(norm_value)
validation_set = (0.95*total_nodes)
test_set = (0.98*total_nodes)
random_list = [i for i in range(total_nodes)]
shuffle(random_list)
for node in range(0,total_nodes):
if node<validation_set:
UG.node[random_list[node]]['val'] = False
UG.node[random_list[node]]['test'] = False
elif node<test_set:
UG.node[random_list[node]]['val'] = True
UG.node[random_list[node]]['test'] = False
else:
UG.node[random_list[node]]['val'] = False
UG.node[random_list[node]]['test'] = True
nx.set_edge_attributes(UG, 'test_removed', False)
nx.set_edge_attributes(UG, 'train_removed', False)
json_graph_name = graph_name + "-G.json"
json_id_map_name = graph_name + "-id_map.json"
feats_file_name = graph_name + "-feats.npy"
np.save(feats_file_name,features)
data = json_graph.node_link_data(UG)
graphjson = json.dumps(data)
f1=open(json_graph_name, 'w')
f1.write(graphjson)
f1.close()
id_map = {}
for node in range(0,total_nodes):
id_map[str(node)] = node
iddata = json.dumps(id_map)
f2=open(json_id_map_name, 'w')
f2.write(iddata)
f2.close()
nodes = [n for n in UG.nodes() if not UG.node[n]["val"] and not UG.node[n]["test"]]
G = UG.subgraph(nodes)
pairs = run_random_walks(G, nodes)
out_file = graph_name + "-walks.txt"
with open(out_file, "w") as fp:
fp.write("\n".join([str(p[0]) + "\t" + str(p[1]) for p in pairs]))
class_map_file = graph_name + "-class_map.json"
class_map = {}
os.chdir("./greedy_baseline")
graph_file_name = "." + json_graph_name
command = "sh ./find_greedy.sh " + graph_file_name + " " + str(num_k)
os.system(command)
solution_file_name = graph_file_name + ".greedySol"
# solution_file_name = "./greedy_baseline/solution_greedy.txt"
solution_file = open(solution_file_name,"r")
os.chdir("../")
greedy_nodes = solution_file.readlines()
temp_selected_nodes = greedy_nodes[0].strip().split(' ')
os.chdir("./random_baseline")
graph_file_name = "." + json_graph_name
command = "sh ./find_random.sh " + graph_file_name + " " + str(num_k)
os.system(command)
os.chdir("../")
os.chdir("./top-k_baseline")
graph_file_name = "." + json_graph_name
command = "sh ./find_top-k.sh " + graph_file_name + " " + str(num_k)
os.system(command)
os.chdir("../")
for node in range(0,total_nodes):
class_map[str(node)] = [float(temp_selected_nodes[node])]
classdata = json.dumps(class_map)
f2=open(class_map_file, 'w')
f2.write(classdata)
f2.close()
def main():
"""
Begin Main
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
graphs_collection = mongo_connector.get_graphs_connection()
graphs_data_collection = mongo_connector.get_graphs_data_connection()
graphs_links_collection = mongo_connector.get_graphs_links_connection()
graphs_docs_collection = mongo_connector.get_graphs_docs_connection()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_graphs2')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
for zone in zones:
groups = []
graph = nx.Graph()
add_to_list(zone, groups)
graph.add_node(zone,
data_type="tld",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>")
find_all_dns_by_zone(graph, zone, groups, dns_manager, ip_manager)
find_srdns_by_zone(graph, zone, groups, mongo_connector, ip_manager)
data = json_graph.node_link_data(graph)
reformat_data(data, zone, groups)
new_data = {}
new_data['directed'] = data['directed']
new_data['graph'] = data['graph']
new_data['multigraph'] = data['multigraph']
new_data['errs'] = []
config = {}
config['title'] = zone + " Network Map"
config['graph'] = {}
config['graph']['linkDistance'] = 150
config['graph']['charge'] = -400
config['graph']['height'] = 800
config['graph']['numColors'] = len(groups)
config['graph']['labelPadding'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['labelMargin'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['ticksWithoutCollisions'] = 50
config['graph_type'] = "tracked_domain"
config['types'] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
group = tgroup.replace(REPLACE_CHAR, ".")
data_type = "tpd"
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config['types'][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type
}
config['constraints'] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config['constraints'].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos]
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config['jsonUrl'] = "/api/v1.0/graphs/" + zone
new_data['config'] = config
new_data['created'] = datetime.now()
new_data['zone'] = zone
new_docs_data = {}
new_docs_data['docs'] = {}
new_docs_data['zone'] = zone
new_docs_data['created'] = datetime.now()
new_graph_data = {}
new_graph_data['data'] = {}
for i in range(0, len(data['nodes'])):
new_graph_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]
new_docs_data['docs'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]['docs']
del new_graph_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)]['docs']
new_graph_data['created'] = datetime.now()
new_graph_data['zone'] = zone
new_graph_data['directed'] = data['directed']
new_graph_data['multigraph'] = data['multigraph']
new_graph_data['errs'] = []
new_links_data = {}
new_links_data['links'] = data['links']
new_links_data['created'] = datetime.now()
new_links_data['zone'] = zone
new_links_data['directed'] = data['directed']
new_links_data['multigraph'] = data['multigraph']
new_links_data['errs'] = []
try:
graphs_collection.remove({'zone': zone})
graphs_collection.insert_one(new_data)
graphs_data_collection.remove({'zone': zone})
graphs_data_collection.insert_one(new_graph_data)
graphs_links_collection.remove({'zone': zone})
graphs_links_collection.insert_one(new_links_data)
graphs_docs_collection.remove({'zone': zone})
graphs_docs_collection.insert_one(new_docs_data)
except:
logger.error("ERROR: Can't insert: " + zone)
time.sleep(1)
# Remove last week's old entries
# In theory, shouldn't do anything but being complete
lastweek = datetime.now() - timedelta(days=7)
graphs_collection.remove({'created': {"$lt": lastweek}})
graphs_data_collection.remove({'created': {"$lt": lastweek}})
graphs_links_collection.remove({'created': {"$lt": lastweek}})
graphs_docs_collection.remove({'created': {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete")
def saveJSON(graph, exportFilename):
data = json_graph.node_link_data(graph)
with open(exportFilename, 'w') as out:
json.dump(data, out, indent=4)
def descriptor_viewer():
"""
This function just responds to the browser Url
:return: the rendered template 'descriptor.html'
"""
if request.method == 'POST':
try:
already_onboarded_in_so = False
# retrieving the IFA descriptor
# print(request.form, request.files)
if 'convert_text' in request.form:
ifa_json = json.loads(request.form['convert_text'])
elif 'file_to_convert' in request.files:
f = request.files['file_to_convert']
response = f.read()
ifa_json = json.loads(response.decode('utf-8'))
elif 'show_json' in request.form:
ifa_json = eval(request.form['show_json'])
already_onboarded_in_so = True
elif 'onboard_json' in request.form:
ifa_json = eval(request.form['onboard_json'])
record = {}
if 'nsd' in ifa_json:
# nsd case
if 'vnfdId' in ifa_json['nsd']:
record = {"nsdId": ifa_json["nsd"]["nsdIdentifier"],
"nsdCloudifyId": {},
"version": ifa_json["nsd"]["version"],
"nsdName": ifa_json["nsd"]["nsdName"],
"nsdJson": ifa_json,
"shareable": True,
"domain": "local"}
if nsd_db.get_nsd_json(nsdId=record['nsdId']) is None:
nsd_db.insert_nsd(record)
message = {"Success": 'nsdId : {} onboarded on SO with success!'.format(record['nsdId'])}
else:
log_queue.put(["DEBUG", 'nsdId already in the SO DB'])
raise ValueError('nsdId already in the SO DB')
# nsd-composite case
else:
record = {"nsdId": ifa_json["nsd"]["nsdIdentifier"],
"nsdCloudifyId": {},
"version": ifa_json["nsd"]["version"],
"nsdName": ifa_json["nsd"]["nsdName"],
"nsdJson": ifa_json,
"shareable": False,
"domain": "Composite"}
if nsd_db.get_nsd_json(nsdId=record['nsdId']) is None:
nsd_db.insert_nsd(record)
message = {"Success": 'nsdId : {} onboarded on SO with success!'.format(record['nsdId'])}
else:
log_queue.put(["DEBUG", 'nsdId already in the SO DB'])
raise ValueError('nsdId already in the SO DB')
# vnfd case
else:
record = {"vnfdId": ifa_json["vnfdId"],
"vnfdVersion": ifa_json["vnfdVersion"],
"vnfdName": ifa_json["vnfProductName"],
"vnfdJson": ifa_json}
if vnfd_db.get_vnfd_json(vnfdId=ifa_json["vnfdId"]) is None:
vnfd_db.insert_vnfd(record)
message = {'Success': 'vnfdId : {} onboarded on SO with success!'.format(record['vnfdId'])}
else:
log_queue.put(["DEBUG", 'vnfdId already in the SO DB'])
raise ValueError('vnfdId already in the SO DB')
log_queue.put(["INFO", message["Success"]])
flash(message['Success'], 'success')
already_onboarded_in_so = True
else:
raise ValueError('No text/file valid')
if 'nsd' in ifa_json:
if 'vnfdId' in ifa_json['nsd']:
# convert a NSD
list_osm_json, default_index = gui_utils.ifa014_conversion(ifa_json)
default_osm_json = list_osm_json[default_index]
osm_json_network = []
for level in list_osm_json:
osm_json_network.append(json_graph.node_link_data(gui_utils.json_network_nsd(level)))
descriptor_type = 'nsd'
else:
# convert a composite NSD
list_osm_json, default_index = gui_utils.composite_desc_conversion(ifa_json)
default_osm_json = list_osm_json[default_index]
osm_json_network = []
for level in list_osm_json:
osm_json_network.append(json_graph.node_link_data(gui_utils.json_network_composite_nsd(level)))
descriptor_type = 'nsd-composite'
else:
# convert a VNFD
list_osm_json = [gui_utils.ifa011_conversion(ifa_json)]
default_osm_json = list_osm_json[0] # done in case of possible list of ifa vnfd conversion
osm_json_network = [json_graph.node_link_data(gui_utils.json_network_vnfd(default_osm_json))]
descriptor_type = 'vnfd'
yaml_descriptor_list = []
for osm_json in list_osm_json:
yaml_descriptor_list.append(yaml.safe_dump(osm_json, default_flow_style=False))
yaml_ifa_descriptor = yaml.safe_dump(ifa_json, default_flow_style=False)
return render_template('descriptor.html', html_title='Descriptor Viewer',
descriptor_type=descriptor_type,
yaml_network=osm_json_network,
list_osm_json=list_osm_json,
yaml_osm_descriptor=yaml_descriptor_list,
yaml_ifa_descriptor=yaml_ifa_descriptor,
ifa_json=ifa_json,
already_onboarded_in_so=already_onboarded_in_so)
except (TypeError, KeyError, ValueError) as error:
message = {'Error': 'Error: {}'.format(error)}
log_queue.put(["ERROR", message['Error']])
flash(message['Error'], 'danger')
return redirect(url_for('home'))
def data():
#read requirement file
try:
namafile = 'config.ini'
config = configparser.ConfigParser()
config.read_file(open(namafile))
rurl = config.get('redis', 'REDIS_URL')
rport = config.get('redis', 'REDIS_PORT')
rpass = config.get('redis', 'REDIS_PASS')
except KeyError:
sys.stderr.write("Tidak Bisa Membuka File" + namafile + "\n")
sys.exit(1)
#Networkx graph configuration
G = nx.Graph()
#Redis graph configuration
r = redis.Redis(host=rurl, port=rport, db=0, password=rpass)
#list redis keys
for k in r.keys('*'):
#get value from redis keys
value = str(r.get(k))
#delete the header format (b') from value
panjang = len(value)
value = value[2:(panjang - 1)]
#split the value
arrvalue = value.split(',')
#change data type to string
root = str(k)
#delete the header format (b') from root
panjangkey = len(root)
root = root[2:(panjangkey - 1)]
for follower in arrvalue:
# create edges list from key and value
G.add_edge(root, follower)
# Initialize graph, add nodes and edges, calculate modularity and centrality.
groups = community.best_partition(G)
degree = cn.degree_centrality(G)
# Add node attributes for name, modularity, and three types of centrality.
nx.set_node_attributes(G, groups, 'group')
nx.set_node_attributes(G, degree, 'degree')
# create json dictionary format for networkx edges
data1 = json_graph.node_link_data(G)
#output json file
with open('static/data.json', 'w') as output:
json.dump(data1,
output,
sort_keys=True,
indent=4,
separators=(',', ':'))
return data1
def make_topo(self, f_g, _color):
#
l_tp = self.bez_to_list(f_g["beziers"])
#
g_coord_flip = self.flipCoordPath(l_tp,False,True)
#
if len(f_g["graph_json"]):
#
_g = json_graph.node_link_graph(f_g["graph_json"])
f_g["graph"]
_g.clear()
#
else:
#if hasattr(self,'G'):
if f_g["graph"] != None:
#
_g = f_g["graph"]#self.G
_g.clear()
#
else:
#
f_g["graph"] = nx.Graph()
_g = f_g["graph"]
#
#
x_mm = self.value_mid(g_coord_flip, 0)
y_mm = self.value_mid(g_coord_flip, 1)
#
g_coord_flip_simp = g_coord_flip.copy()
#
g_coord_flip.insert(0,[x_mm,y_mm])
#
node_color_map = []
edge_width_map = []
node_width_map = []
edge_color_map = []
edge_label_map = {}
node_label_map = {}
node_order_map = OrderedDict()
#
for x in range(len(g_coord_flip)):
#
_g.add_edge(0,x)
#
_g.add_node(x,pos=g_coord_flip[x])
#
#
for x in range(len(g_coord_flip)):
#
node_color_map.append(_color)
edge_color_map.append(_color)
edge_width_map.append(0.3)
node_width_map.append(70)
#
#
pos=nx.get_node_attributes(_g,'pos')
#
edge_lengths = self.get_edge_lengths(_g, pos)
#
sorted_length = sorted(edge_lengths.items(), key=lambda k: k[1]['len'], reverse=True)
#
sort_by_length = OrderedDict(sorted_length)
#
#
i = 0
d = 0
#
sorted_nodes = []
#
for n in _g.nodes():
#
i = 0
#
for k, v in sort_by_length.items():
#
node_inx = g_coord_flip.index(pos[n])
#
#
if i == n:
#
node_order_map[n] = node_inx
#
if k != (0,0):
#
#
edge_label_map[k] = str(v["len"])
node_label_map[v["node"]] = str(n)
#
#
i = i + 1
#
else:
#
node_label_map[v["node"]] = 'c'
#
#
sorted_nodes.append(v["node"])
#
#
#
i = i + 1
#
#
#
for k,v in sort_by_length.items():
#
sort_by_length[k]["order"] = node_order_map[k[1]]
#
#
f_g["graph_data"] = {
#"pos":pos,
"node_color_map":node_color_map,
"edge_width_map":edge_width_map,
"node_width_map":node_width_map,
"edge_color_map":edge_color_map,
"edge_label_map":edge_label_map,
"node_label_map":node_label_map,
"node_order_map":node_order_map,
"g_coord_flip_simp":g_coord_flip_simp,
"sort_by_length":sort_by_length
}
#
f_g["graph_json"] = json_graph.node_link_data(_g)
#
#
return f_g
def main():
"""
The main thread for this program.
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
jobs_manager = JobsManager.JobsManager(mongo_connector,
"create_tpd_graphs")
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
tpds = get_tpds(mongo_connector)
# For third-party-domain in the list of third-party-domains
for tpd in tpds:
groups = []
graph = nx.DiGraph()
add_to_list(tpd, groups)
# A space is added because sometimes the tpd is the same as the end target node
graph.add_node(
tpd + " ",
data_type="tld",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>",
)
# Get the zones associated with the tpd
find_zones_by_tld(graph, tpd, groups, mongo_connector)
data = json_graph.node_link_data(graph)
reformat_data(data, tpd, groups)
new_data = {}
new_data["directed"] = data["directed"]
new_data["graph"] = data["graph"]
new_data["multigraph"] = data["multigraph"]
new_data["errs"] = []
new_data["links"] = data["links"]
new_data["data"] = {}
for i in range(0, len(data["nodes"])):
new_data["data"][data["nodes"][i]["id"].replace(
".", REPLACE_CHAR)] = data["nodes"][i]
for entry in new_data["data"]:
for dep in new_data["data"][entry]["depends"]:
if (new_data["data"][entry]["name"]
not in new_data["data"][dep.replace(
".", REPLACE_CHAR)]["dependedOnBy"]):
new_data["data"][dep.replace(
".", REPLACE_CHAR)]["dependedOnBy"].append(
new_data["data"][entry]["name"])
for dep in new_data["data"][entry]["dependedOnBy"]:
if (new_data["data"][entry]["name"]
not in new_data["data"][dep.replace(
".", REPLACE_CHAR)]["depends"]):
new_data["data"][dep.replace(
".", REPLACE_CHAR)]["depends"].append(
new_data["data"][entry]["name"])
for entry in new_data["data"]:
new_data["data"][entry]["docs"] = build_docs(
new_data["data"][entry], tpd, groups)
config = {}
config["title"] = tpd + " Network Map"
config["graph"] = {}
config["graph"]["linkDistance"] = 150
config["graph"]["charge"] = -400
config["graph"]["height"] = 800
config["graph"]["numColors"] = len(groups)
config["graph"]["labelPadding"] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config["graph"]["labelMargin"] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config["graph"]["ticksWithoutCollisions"] = 50
config["graph_type"] = "tpd"
config["types"] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
data_type = "tpd"
group = tgroup.replace(REPLACE_CHAR, ".")
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config["types"][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type,
}
config["constraints"] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config["constraints"].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos],
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config["jsonUrl"] = "/api/v1.0/tpd_graphs/" + tpd
new_data["config"] = config
new_data["created"] = datetime.now()
new_data["zone"] = tpd
tpd_graphs_collection = mongo_connector.get_tpd_graphs_connection()
tpd_graphs_collection.delete_one({"zone": tpd})
try:
tpd_graphs_collection.insert_one(new_data)
except:
logger.error("ERROR: Could not insert " + tpd)
time.sleep(1)
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
tpd_graphs_collection.delete_many({"created": {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def publication_coauthorships(request, max_position=None, within_group=False):
G = nx.Graph()
if max_position and int(max_position) > 1:
pub_authors = PublicationAuthor.objects.exclude(position__gt=max_position)
else:
pub_authors = PublicationAuthor.objects.all()
pub_authors = pub_authors.values("publication_id", "author_id", "author_id__full_name")
# 'publication_id': [ pub_author1, pub_author2, pub_author3 ]
authors_per_publication = defaultdict(list)
names = set()
if within_group:
people = Person.objects.all().values("is_active", "id")
active_by_id = {}
for person in people:
active_by_id[person['id']] = person['is_active']
for pub_author in pub_authors:
check_coauthorship = True
if within_group and not active_by_id[pub_author['author_id']]:
check_coauthorship = False
if check_coauthorship:
entry = (pub_author['author_id'], pub_author['author_id__full_name'])
authors_per_publication[pub_author['publication_id']].append(entry)
for relations in authors_per_publication.values():
for (author_id1, name1), (author_id2, name2) in combinations(relations, 2):
G.add_edge(author_id1, author_id2)
try:
G[author_id1][author_id2]['weight'] += 1
except:
G[author_id1][author_id2]['weight'] = 1
G.node[author_id1]['name'] = name1
G.node[author_id2]['name'] = name2
names.add(name1)
names.add(name2)
try:
G = nx_graph.analyze(G)
except:
# Avoid networkx's "power iteration failed to converge" error
pass
data = json_graph.node_link_data(G)
return_dict = {
'data': json.dumps(data),
'names': list(names),
'web_title': u'Publications co-authorship',
'within_group': within_group,
}
return render(request, "charts/publications/co_authorship.html", return_dict)
def main():
parser = OptionParser()
parser.add_option('-o', dest='filename', default='graph.gexf',
help='output filename')
parser.add_option('--names', dest='names', default='data/names.json',
help='names filename')
parser.add_option('--clusters', dest='clusters',
default='data/clusters.json',
help='clusters filename')
parser.add_option('--start', dest='start', default='20110301',
help='start date (default=20110301)')
parser.add_option('--stop', dest='stop', default='20130401',
help='stop date (default=20130401)')
parser.add_option('--sampling', dest='sampling', type='float', default=1.0,
help='tweet sampling rate (default=1.0)')
parser.add_option('--influencers', dest='n', type='int', default=None,
help='maximum number of influencers (default=inf)')
parser.add_option('--mincount', dest='mincount', type='int', default=1,
help='minimum number of retweet (default=1)')
parser.add_option('--mindegree', dest='mindegree', type='int', default=0,
help='minimum acceptable degree of nodes (default=0)')
parser.add_option('--group', dest='group', default='month',
help='month or week (default=month)')
options, args = parser.parse_args()
names = load_names(options.names)
clusters = load_clusters(options.clusters)
start_date = ymd_to_datetime(options.start)
stop_date = ymd_to_datetime(options.stop)
tweets = load_tweet(*args)
tweets = filter_by_datetime(tweets, start_date, stop_date)
random.seed(0)
tweets = sampling(tweets, options.sampling)
if options.group == 'month':
groups = [t.strftime('t%Y%m%d') for t
in months_between(start_date, stop_date)]
else:
groups = [t.strftime('t%Y%m%d') for t
in weeks_between(start_date, stop_date)]
graph = nx.DiGraph()
graph.graph['groups'] = groups
for ruid, tss in group_by_influencer(tweets, options.n, options.mincount):
for uid, ts in group_by_user(tss):
ts = list(ts)
labels = {d: False for d in groups}
for t in ts:
d = to_datetime(t['created_at']).strftime('t%Y%m%d')
labels[groups[bisect_right(groups, d) - 1]] = True
graph.add_edge(ruid, uid, weight=len(ts), **labels)
for node in graph.nodes():
if graph.degree(node, 'weight') < options.mindegree:
graph.remove_node(node)
continue
graph.node[node]['label'] = names.get(node, '')
graph.node[node]['cluster'] = clusters.get(node, -1)
if graph.out_degree(node) == 0:
cluster_count = {}
for ruid in graph.predecessors(node):
c = clusters.get(ruid, -1)
if c not in cluster_count:
cluster_count[c] = 0
cluster_count[c] += 1
graph.node[node]['cluster'] = max(cluster_count.items(),
key=lambda r: r[1])[0]
for d in groups:
graph.node[node][d] = False
for d in groups:
for u, v in graph.edges():
if graph[u][v][d]:
graph.node[u][d] = True
graph.node[v][d] = True
out_format = options.filename.split('.')[-1]
if out_format == 'gexf':
nx.write_gexf(graph, options.filename)
else:
obj = json_graph.node_link_data(graph)
json.dump(obj, open(options.filename, 'w'))
print('(|V|, |E|) = ({}, {})'.format(graph.number_of_nodes(),
graph.number_of_edges()))
def write_html(g, output):
graph_data, graph = to_json(g)
graph_data = json.dumps(graph_data, indent=2)
html = """<!DOCTYPE html>
<meta charset="utf-8">
<script src="https://platform.twitter.com/widgets.js"></script>
<script src="https://d3js.org/d3.v4.min.js"></script>
<script src="https://code.jquery.com/jquery-3.1.1.min.js"></script>
<style>
.links line {
stroke: #999;
stroke-opacity: 0.6;
}
.nodes circle {
stroke: #fff;
stroke-width: 1.5px;
}
text {
font-family: sans-serif;
font-size: 10px;
}
#graph {
width: 99vw;
height: 99vh;
}
</style>
<svg id="graph"></svg>
<script>
var width = $(window).width();
var height = $(window).height();
var svg = d3.select("svg")
.attr("height", height)
.attr("width", width);
function normalize(val, max=15, min=5){ return min + (max-min) *val}
var color = d3.scaleOrdinal(d3.schemeCategory20);
var simulation = d3.forceSimulation()
.force("link", d3.forceLink().id(function(d) {
return d.id; }))
.force('charge',
d3.forceManyBody().strength(-200)
)
.force("center", d3.forceCenter(width / 2, height / 2))
.force('y', d3.forceY(0))
.force('x', d3.forceX(0));
var graph = %s;
var link = svg.append("g")
.attr("class", "links")
.selectAll("line")
.data(graph.links)
.enter().append("line")
.attr("stroke-width", 2);;
var node = svg.append("g")
.attr("class", "nodes")
.selectAll("g")
.data(graph.nodes)
.enter().append("g")
var circles = node.append("circle")
.attr("r", function(d){return normalize(d.value)})
.attr("fill", '#aec7e8')
.call(d3.drag()
.on("start", dragstarted)
.on("drag", dragged)
.on("end", dragended));
var lables = node.append("text")
.text(function(d) {
return d.value >0 ? (d.authors ? d.title + ' - ' + d.authors : d.title) : '';
})
.attr('x', 6)
.attr('y', 3);
node.append("title")
.text(function(d) { return d.value >0 ? (d.authors ? d.title + ' - ' + d.authors : d.title) : ''; });
simulation
.nodes(graph.nodes)
.on("tick", ticked);
simulation.force("link")
.links(graph.links);
function ticked() {
link
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
node
.attr("transform", function(d) {
return "translate(" + d.x + "," + d.y + ")";
});
}
function dragstarted(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function dragended(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
</script>
""" % graph_data
open(output, 'w').write(html)
data = json_graph.node_link_data(graph)
with open('output.json', 'w') as outfile:
json.dump(data, outfile)
def networkDict(self):
return json_graph.node_link_data(self.G)
"""
Test script for synComGraph.py. Saves the output communication graph topology
as a JSON in the same format as the input feeder topology.
"""
import synComGraph as scg
import json
from networkx.readwrite import json_graph
feedername = 'R5-12.47-5' #Name of JSON for feeder
perRew = 0.2 #percent of the links to be rewired in communication graph
distThresh = 1000 #maximum Euclidean distance between linked nodes allowed in rewiring
comG = scg.synFeederComGraph(feedername,
perRew=perRew,
distThresh=distThresh,
plotGraphs=True)
#save the output NetworkX communication graph as a JSON file
with open('commNet_' + feedername + '.json', 'w') as outfile1:
outfile1.write(json.dumps(json_graph.node_link_data(comG)))
def to_json(self):
"""Dump map data to a Node Link JSON output"""
return dumps(node_link_data(self))
def write_graph(G, out_file):
data = json_graph.node_link_data(G)
with open(out_file, 'w') as f:
json.dump(data, f, indent=4, separators=(',', ': '), sort_keys=True)
def main(params):
os.makedirs(params['dest_dir'], exist_ok=True)
nodes_fn = os.path.join(params['dest_dir'], 'nodes.json')
graph_fn = os.path.join(params['dest_dir'], 'graph.json')
nodelist_fn = os.path.join(params['dest_dir'], 'nodelist.json')
now = datetime.utcnow().replace(microsecond=0)
# parse mesh param and instantiate Alfred/Batman instances
alfred_instances = []
batman_instances = []
for value in params['mesh']:
# (1) only batman-adv if, no alfred sock
if ':' not in value:
if len(params['mesh']) > 1:
raise ValueError('Multiple mesh interfaces require the use of '
'alfred socket paths.')
alfred_instances.append(Alfred(unix_sockpath=None))
batman_instances.append(Batman(mesh_interface=value))
else:
# (2) batman-adv if + alfred socket
try:
batif, alfredsock = value.split(':')
alfred_instances.append(Alfred(unix_sockpath=alfredsock))
batman_instances.append(
Batman(mesh_interface=batif, alfred_sockpath=alfredsock))
except ValueError:
raise ValueError(
'Unparseable value "{0}" in --mesh parameter.'.format(
value))
# read nodedb state from node.json
try:
with open(nodes_fn, 'r') as nodedb_handle:
nodedb = json.load(nodedb_handle)
except IOError:
nodedb = {'nodes': dict()}
# flush nodedb if it uses the old format
if 'links' in nodedb:
nodedb = {'nodes': dict()}
# set version we're going to output
nodedb['version'] = NODES_VERSION
# update timestamp and assume all nodes are offline
nodedb['timestamp'] = now.isoformat()
for node_id, node in nodedb['nodes'].items():
node['flags']['online'] = False
# integrate alfred nodeinfo
for alfred in alfred_instances:
nodeinfo = validate_nodeinfos(alfred.nodeinfo())
nodes.import_nodeinfo(nodedb['nodes'],
nodeinfo,
now,
assume_online=True)
# integrate static aliases data
for aliases in params['aliases']:
with open(aliases, 'r') as f:
nodeinfo = validate_nodeinfos(json.load(f))
nodes.import_nodeinfo(nodedb['nodes'],
nodeinfo,
now,
assume_online=False)
nodes.reset_statistics(nodedb['nodes'])
for alfred in alfred_instances:
nodes.import_statistics(nodedb['nodes'], alfred.statistics())
# acquire gwl and visdata for each batman instance
mesh_info = []
for batman in batman_instances:
vd = batman.vis_data()
gwl = batman.gateway_list()
mesh_info.append((vd, gwl))
# update nodedb from batman-adv data
for vd, gwl in mesh_info:
nodes.import_mesh_ifs_vis_data(nodedb['nodes'], vd)
nodes.import_vis_clientcount(nodedb['nodes'], vd)
nodes.mark_vis_data_online(nodedb['nodes'], vd, now)
nodes.mark_gateways(nodedb['nodes'], gwl)
# clear the nodedb from nodes that have not been online in $prune days
if params['prune']:
nodes.prune_nodes(nodedb['nodes'], now, params['prune'])
# build nxnetworks graph from nodedb and visdata
batadv_graph = nx.DiGraph()
for vd, gwl in mesh_info:
graph.import_vis_data(batadv_graph, nodedb['nodes'], vd)
# force mac addresses to be vpn-link only (like gateways for example)
if params['vpn']:
graph.mark_vpn(batadv_graph, frozenset(params['vpn']))
def extract_tunnel(nodes):
macs = set()
for id, node in nodes.items():
try:
for mac in node["nodeinfo"]["network"]["mesh"]["bat0"][
"interfaces"]["tunnel"]:
macs.add(mac)
except KeyError:
pass
return macs
graph.mark_vpn(batadv_graph, extract_tunnel(nodedb['nodes']))
batadv_graph = graph.merge_nodes(batadv_graph)
batadv_graph = graph.to_undirected(batadv_graph)
# write processed data to dest dir
with open(nodes_fn, 'w') as f:
json.dump(nodedb, f)
graph_out = {
'batadv': json_graph.node_link_data(batadv_graph),
'version': GRAPH_VERSION
}
with open(graph_fn, 'w') as f:
json.dump(graph_out, f)
with open(nodelist_fn, 'w') as f:
json.dump(export_nodelist(now, nodedb), f)
# optional rrd graphs (trigger with --rrd)
if params['rrd']:
script_directory = os.path.dirname(os.path.realpath(__file__))
rrd = RRD(os.path.join(script_directory, 'nodedb'),
os.path.join(params['dest_dir'], 'nodes'))
rrd.update_database(nodedb['nodes'])
rrd.update_images()
- https://github.com/networkx/networkx/tree/master/examples/javascript/force
"""
import json
import flask
import networkx as nx
from networkx.readwrite import json_graph
G = nx.barbell_graph(6, 3)
# this d3 example uses the name attribute for the mouse-hover value,
# so add a name to each node
for n in G:
G.nodes[n]["name"] = n
# write json formatted data
d = json_graph.node_link_data(G) # node-link format to serialize
# write json
json.dump(d, open("force/force.json", "w"))
print("Wrote node-link JSON data to force/force.json")
# Serve the file over http to allow for cross origin requests
app = flask.Flask(__name__, static_folder="force")
@app.route("/")
def static_proxy():
return app.send_static_file("force.html")
print("\nGo to http://localhost:8000 to see the example\n")
app.run(port=8000)
def vis_hierarchy(corexes,
column_label=None,
max_edges=100,
prefix='topics',
n_anchors=0):
"""Visualize a hierarchy of representations."""
if column_label is None:
column_label = list(map(str, range(corexes[0].alpha.shape[1])))
# make l1 label
alpha = corexes[0].alpha
mis = corexes[0].mis
l1_labels = []
annotate = lambda q, s: q if s > 0 else '~' + q
for j in range(corexes[0].n_hidden):
# inds = np.where(alpha[j] * mis[j] > 0)[0]
inds = np.where(alpha[j] >= 1.)[0]
inds = inds[np.argsort(-alpha[j, inds] * mis[j, inds])]
group_number = str('red_') + str(j) if j < n_anchors else str(j)
label = group_number + ':' + ' '.join([
annotate(column_label[ind], corexes[0].sign[j, ind])
for ind in inds[:6]
])
label = textwrap.fill(label, width=25)
l1_labels.append(label)
# Construct non-tree graph
weights = [corex.alpha.clip(0, 1) * corex.mis for corex in corexes[1:]]
node_weights = [corex.tcs for corex in corexes[1:]]
g = make_graph(weights, node_weights, l1_labels, max_edges=max_edges)
# Display pruned version
h = g.copy(
) # trim(g.copy(), max_parents=max_parents, max_children=max_children)
edge2pdf(h,
prefix + '/graphs/graph_prune_' + str(max_edges),
labels='label',
directed=True,
makepdf=True)
# Display tree version
tree = g.copy()
tree = trim(tree, max_parents=1, max_children=False)
edge2pdf(tree,
prefix + '/graphs/tree',
labels='label',
directed=True,
makepdf=True)
# Output JSON files
try:
import os
#copyfile(os.path.dirname(os.path.realpath(__file__)) + '/tests/d3_files/force.html', prefix + '/graphs/force.html')
copyfile(
os.path.dirname(os.path.realpath('tests')) +
'/tests/d3_files/force.html', prefix + '/graphs/force.html')
except:
print("Couldn't find 'force.html' file for visualizing d3 output")
import json
from networkx.readwrite import json_graph
mapping = dict([(n, tree.nodes[n].get('label', str(n)))
for n in tree.nodes()])
tree = nx.relabel_nodes(tree, mapping)
json.dump(json_graph.node_link_data(tree),
safe_open(prefix + '/graphs/force.json', 'w+'))
json.dump(json_graph.node_link_data(h),
safe_open(prefix + '/graphs/force_nontree.json', 'w+'))
return g
def nx_to_d3_json(self, g):
return json_graph.node_link_data(g)
def filter_with_ground_truth(self,
in_arcs,
out_arcs,
gt_in_arcs,
gt_out_arcs,
label_scheme='train',
val_count=20,
val_in_arcs=None,
val_out_arcs=None):
if not gt_in_arcs:
gt_in_arcs = in_arcs
gt_out_arcs = out_arcs
selected_indices = set()
background_indices = set()
gt_pos_indices = set()
gt_neg_indices = set()
val_pos_indices = set()
val_neg_indices = set()
for arc in in_arcs:
selected_indices.add(arc)
for arc in out_arcs:
background_indices.add(arc)
cvt_indices = in_arcs.union(out_arcs)
for arc in gt_in_arcs:
gt_pos_indices.add(arc)
for arc in gt_out_arcs:
gt_neg_indices.add(arc)
for arc in val_in_arcs:
val_pos_indices.add(arc)
for arc in val_out_arcs:
val_neg_indices.add(arc)
val_set = val_pos_indices.union(val_neg_indices)
compiled_data = self.compile_features()
node_map = {}
G = self.construct_dual_graph()
current_arc_id = 0
node_ids = {}
node_labels = {}
i_val = 0 ##!!!! need to set size for val set
for arc, features in zip(self.arcs, compiled_data):
index = tuple(arc.node_ids) + (len(arc.line), )
for node_id in arc.node_ids:
#
# !! need to fix here to accomadate only ascending ridges
#
if node_id not in node_map:
node_map[node_id] = []
node_map[node_id].append(current_arc_id)
label = [
int(index in gt_neg_indices),
int(index in gt_pos_indices),
]
node = G.node[current_arc_id]
node["index"] = arc.node_ids
node["size"] = len(arc.line)
node["features"] = features.tolist()
# labeled nodes assigned as train, test, or val
if bool(np.sum(label)):
modified = 0
if index in cvt_indices: # and i_val < val_count:
modified = 1
node["train"] = True
node["test"] = False
node["val"] = False
if index in val_set:
modified = 1
node["train"] = False
node["test"] = False
node["val"] = True
elif not bool(modified):
node["test"] = True
node["val"] = False
node["train"] = False
"""Label all non-selected arcs as test"""
#if not bool(np.sum(label)):
#node["test"] = True
if bool(np.sum(label)):
node["label"] = label
node["prediction"] = None
#G.node[current_arc_id] = node
#current_arc_id += 1
node_ids[current_arc_id] = current_arc_id
node_labels[current_arc_id] = label
current_arc_id += 1
for arc_id, arc in list(G.nodes_iter(data=True)): #G.nodes.items():
for node_id in arc["index"]:
for connected_arc_id in node_map[node_id]:
G.add_edge(arc_id, connected_arc_id)
data1 = json_graph.node_link_data(G)
s1 = json.dumps(data1) # input graph
s2 = json.dumps(node_ids) # dict: nodes to ints
s3 = json.dumps(node_labels) # dict: node_id to class
from networkx.readwrite import json_graph
from nets2 import visualize
from itertools import count
distance_matrix_file = '/home/liaoth/data2/16s/171027_16s/16s_pipeliens/XK/final_result/analysis/beta/unifrac.txt'
dm = pd.read_csv(distance_matrix_file, sep='\t', header=0, index_col=0)
assert list(dm.columns) == list(dm.index)
# MST part
nodes_n = dm.shape[0]
G = nx.from_numpy_matrix(dm.values)
nx.set_node_attributes(G, 'samples',
dict(zip(range(nodes_n), [[it] for it in dm.index])))
names = dict(zip(range(nodes_n), dm.index))
G = nx.relabel_nodes(G, names)
mst = nx.minimum_spanning_tree(G)
mst_json = json_graph.node_link_data(mst)
for each in mst_json['links']:
each['type'] = 'mst'
#mst_json['links'] = []
# if you uncomment upper line, this will also display MST lines or it will only display KNN line.
# KNN part
# KNN config
nearest_num = 1
###
M = knc(weights='distance', metric='precomputed')
M.fit(dm.values.tolist(), list(dm.index))
query_dict = {}
for _idx, name in enumerate(mst_json['nodes']):
g=networkx.Graph() for article,art_refs in itertools.groupby(references_by_articles_filtered,key=lambda e:e[0]): #one link between ref cited by same article for r1,r2 in itertools.combinations((r for a,r in art_refs),2): g.add_node(r1,type="references",occurence_count=references_occs[r1]) g.add_node(r2,type="references",occurence_count=references_occs[r2]) add_edge_weight(g,r1,r2) print "remove edges with weight < %s"%CONFIG["spans"][span]["references"]["weight"] g.remove_edges_from((r1,r2) for (r1,r2,d) in g.edges(data=True) if d['weight'] <CONFIG["spans"][span]["references"]["weight"]) print "remove nodes with degree = 0" g.remove_nodes_from(r for (r,d) in g.degree_iter() if d <1) networkx.set_node_attributes(g, 'type', "reference") if CONFIG["export_ref_format"] =="gexf": print "write gexf export" networkx.write_gexf(g,os.path.join(CONFIG["parsed_data"],span,"%s.gexf"%span),encoding="UTF-8") elif CONFIG["export_ref_format"] == "edgelist": print "write csv export" networkx.write_weighted_edgelist(g,os.path.join(CONFIG["parsed_data"],span,"%s.csv"%span),delimiter="\t") elif CONFIG["export_ref_format"] == "pajek": print "write pajek export" networkx.write_pajek(g,os.path.join(CONFIG["parsed_data"],span,"%s.net"%span),encoding='UTF-8') elif CONFIG["export_ref_format"] == "json": print "write json export" data = json_graph.node_link_data(g) json.dump(data,open(os.path.join(CONFIG["parsed_data"],span,"%s.json"%span),"w"),encoding='UTF-8') else: print "no export compatible export format specified"
