def main(args):
# Initializing Parser
model_path = config.OPENIE['local_path']
model = Model(model_path)
# Reading input and process the pages
pages = read_file(args['filepath'])
# Process each page
for page in tqdm(pages):
page_bbox = page.cropbox # bounding box of full page
text_blocks = page.get_text("blocks")
# Text block containing the "Abstract"
abstract_block = get_block_containing_abstract(text_blocks)
x0, y0, x1, y1, text, abstract_block_no, block_type = abstract_block
# Running OPENIE prediction on abstract text
result = model.predict(sentence=text)
# Extracting SVO triplets
ARGS=get_ARGS(result)
svo_triplets=get_svo_triplets(ARGS)
# Initializing graph
net = Network(height=1500, width=1500)
net=build_graph(svo_triplets, net)
net.show('../output/graph.html')
# net.save_graph('../output/graph.html')
break
def to_pyvis(graph, layout=True):
"""
This method takes an exisitng Networkx graph and translates
it to a PyVis graph format that can be accepted by the VisJs
API in the Jinja2 template.
Parameters
----------
graph : nx.DiGraph
NetworkX directed graph.
layout : bool
Use hierarchical layout if this is set.
Returns
-------
net : Network
PyVis Network
"""
def add_node(node_id):
attr = nodes[node_id]
net.add_node(node_id, label=attr['label'], level=attr['level'], color=attr.get('color', None),
title=attr['title'])
edges = graph.edges.data()
nodes = graph.nodes
net = Network(height="960px", width="1280px", directed=True, layout=layout)
for v, u, edge_attr in edges:
add_node(v)
add_node(u)
net.add_edge(v, u, title=edge_attr['label'], color=edge_attr['color'])
return net
def getCourseText():
"""
Visualize courses and their assigned texts.
"""
coursesAndTexts = g.query("""
select distinct ?courseName ?textTitle ?authorLast where {
?a a ccso:Course .
?a ccso:csName ?courseName .
?a ccso:hasLM ?t .
?t res:resource ?doc .
?doc dcterms:title ?textTitle .
?doc dcterms:creator ?author .
?author foaf:surname ?authorLast .
} limit 50""")
for line in coursesAndTexts:
print(line)
net = Network(height='750px', width='100%')
for courseName, textTitle, authorLast in coursesAndTexts:
net.add_node(courseName, shape='square', title=str(courseName))
net.add_node(textTitle,
shape='circle',
label=str(authorLast),
title=str(textTitle))
net.add_edge(courseName, textTitle, title="hasLM")
net.save_graph('../website/graph-viz.html')
def vis(G):
gvis = Network(height='700px', width='700px')
gvis.from_nx(G)
gvis.toggle_physics(True)
gvis.show_buttons()
gvis.show("Visualization of INFECTTIOUS daily graph.html")
def reset(self, manual_vis: bool = False):
self.G = DiGraph()
self.network = Network(height=1000, width=1000, directed=True)
if manual_vis:
self.network.show_buttons()
else:
self.set_fixed_nodes_option()
def to_interactive_html(self, destination: str):
from pyvis.network import Network # type: ignore
G = Network(height='100%',
width='100%',
directed=True,
bgcolor='#000000')
G.toggle_stabilization(False)
all_nodes: Set[Node] = set()
nodes_to_explore: Set[Node] = set([self])
while len(nodes_to_explore) > 0:
all_nodes.update(nodes_to_explore)
new_nodes_to_explore: Set[Node] = set()
for node_to_explore in nodes_to_explore:
connected_nodes = node_to_explore.query.through_edge().results
connected_nodes = cast(Set[Node], connected_nodes)
new_nodes_to_explore.update([
node for node in connected_nodes if node not in all_nodes
])
nodes_to_explore = new_nodes_to_explore
for node in all_nodes:
G.add_node(node.graph_id, label=str(node))
for node in all_nodes:
for out in node.outbound:
G.add_edge(node.graph_id, out.destination.graph_id)
# links.append((node, out.destination, {'label': out.label}))
G.show(destination)
def on_save(self):
"""Method called when the user presses the save button in the editor configuration. It applies the changes in the temporary network and updates the network graph of the canvas.
Parameters:
- self: current instance of the class;
"""
G = Network()
print(self.routers)
print(self.switches)
print(self.hosts)
print(self.hosts)
network_core.dictionary_to_nodes(self.routers, G)
network_core.dictionary_to_nodes(self.switches, G)
network_core.dictionary_to_nodes(self.hosts, G)
network_core.dictionary_to_nodes(self.others, G)
network_core.dictionary_to_edges(self.temporary_network.edges, G)
self.main_window_object.current_network = G
G.save_graph("./NetworkGraphs/Temp_Network/temp_network.html")
network_core.html_fix(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html"))
self.main_window_object.update_canvas_html(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html"))
self.main_window_object.ssh_window = ssh_connection(self.main_window_object)
self.main_window_object.edge_window = edge_editors(self.main_window_object)
self.close()
def open_network(network_path):
"""Function that imports a network from an html file.
Parameters:
- network_path: path of the network to be imported
Returns:
- G: network object
"""
G = Network()
html = open(network_path, "r")
lines_html = html.readlines()
nodes = ""
edges = ""
for line in lines_html:
if "nodes = new" in line:
nodes = json.loads(line.split('(')[1].split(')')[0])
if "edges = new" in line:
edges = json.loads(line.split('(')[1].split(')')[0])
dictionary_to_nodes(nodes, G)
dictionary_to_edges(edges, G)
G.save_graph("./NetworkGraphs/Temp_Network/temp_network.html")
html_fix(os.path.abspath("./NetworkGraphs/Temp_Network/temp_network.html"))
return G
def __init__(self):
self.file_name_html = "books_connection_viewer.html"
self.net = Network(height="100%",
width="100%",
bgcolor="black",
font_color="white")
self.net.barnes_hut()
self.threshold = 0
def __init__(self, net, LinGen, name):
self.net = net
self.LinGen = LinGen
self.Grafo = Network(height="600px", width="1000px")
self.name = name
self.Grafo.toggle_physics(False)
self.Grafo.inherit_edge_colors_from(False)
def interactiveGraphExtended(G):
# plot interactive graph using pyvis, with degree, no of node labeled, size depends on nodes' degree
nt = Network(height="750px", width="100%", bgcolor="#222222", font_color="white")
nt.barnes_hut(spring_strength=0.006)
for node in G:
nt.add_node(node, label=G.degree(node), title="I am node "+str(node), value=G.degree(node))
for edge in G.edges:
nt.add_edge(int(edge[0]), int(edge[1]), color='white')
nt.show("nx.html")
def visualization():
G = Network(height='1000px',
width='1000px',
# directed=True,
heading='Social graph of friends',
)
createNodesAndEdges(G)
G.show("G.html")
def add_metadata(self): self.nt = Network(height=DISPLAY_PAGE_HEIGHT, width=DISPLAY_PAGE_WIDTH, directed=True, bgcolor="#090909", heading="BI Apps data lineage for :"+self.filter_raw) self.nt.from_nx(self.H) for node in self.nt.nodes: node["consumed_by_count"] = len(self.H.out_edges(node["id"])) node["depends_on_count"] = len(self.H.in_edges(node["id"])) for node in self.nt.nodes: color_shader(node)
def __init__(self, layout_type, output_suffix=""):
"""
Initialize the browser based on width, height and other properties defined in json files.
:param layout_type: The type of layout that needs to be applied on the output browser.
:param output_suffix: The name of the HTML file.
"""
self.graph = Network(height="750px", width="100%", directed=True)
with open(os.path.join(os.path.dirname(__file__), 'layouts', '{}_layout.json'.format(layout_type))) as f:
self.options = json.load(f)
self.output_suffix = output_suffix
def generate_graph_from_dataframe(self, v: pd.DataFrame, e: pd.DataFrame, directed=False) -> None:
'''
从dataframe中初始化网络，并获取网络各类属性值
:param v: 节点表
:param e: 边表
:param directed: 是否是有向图
'''
# 加锁
self.lock.acquire()
try:
# self.vertex_df = v
# 生成网络图对象
self.g = ig.Graph.DataFrame(edges=e, vertices=v, directed=directed) # igraph图对象
self.vg = Network(height='100%', width='100%', directed=directed) # 可视化图对象
self.vg.add_nodes([i for i in range(self.g.vcount())])
self.vg.add_edges(self.g.get_edgelist())
self.vg.show_buttons(filter_=['physics'])
# 获取网络属性
self.vertex_count = self.g.vcount() # 节点数
self.edge_count = self.g.ecount() # 边数
self.avg_degree = np.average(self.g.degree()) # 平均度
self.max_degree = self.g.maxdegree() # 最大度
self.degree_list = self.g.degree() # 节点度列表
self.diameter = self.g.diameter() # 网络直径
self.avg_path_len = self.g.average_path_length() # 平均路经长
self.clustering_coefficient = self.g.transitivity_undirected() # 聚集系数
self.density = self.g.density() # 网络密度
self.clique_number = self.g.clique_number() # 团数量
self.g.vs.set_attribute_values(attrname='_度_', values=self.degree_list)
# self.betweenness = self.g.betweenness() # 介数列表
self.betweenness = [-1 if np.isnan(v) else v for v in self.g.betweenness()]
self.g.vs.set_attribute_values(attrname='_介数_', values=self.betweenness)
# self.closeness = self.g.closeness() # 紧密中心性列表
self.closeness = [-1 if np.isnan(v) else v for v in self.g.closeness()]
self.g.vs.set_attribute_values(attrname='_紧密中心性_', values=self.closeness)
self.page_rank = [-1 if np.isnan(v) else v for v in self.g.pagerank()]
self.g.vs.set_attribute_values(attrname='_PageRank_', values=self.page_rank)
self.community_detect_algorithm_dict = {
'EdgeBetweenness': self.g.community_edge_betweenness, # 速度慢
'FastGreedy': self.g.community_fastgreedy, # 不能有重复边
'InfoMap': self.g.community_infomap, # 返回VertexClustering
'LabelPropagation': self.g.community_label_propagation,
'LeadingEigenvector': self.g.community_leading_eigenvector, # 速度较慢
# 'LeadingEigenvectorNaive': self.g.community_leading_eigenvector_naive,
'Leiden': self.g.community_leiden,
'MultiLevel': self.g.community_multilevel,
'SpinGlass': self.g.community_spinglass, # 速度慢
'WalkTrap': self.g.community_walktrap
}
finally:
self.lock.release()
def show(self, file="state_graph.html", physics=True):
from pyvis.network import Network
net = Network()
for (i, j), weight in self.edges.items():
net.add_node(i)
net.add_node(j)
net.add_edge(i, j, value=weight)
net.toggle_physics(physics)
net.show(file)
def visualize(edges, error_codes, resouce_pages, args):
G = nx.DiGraph()
G.add_edges_from(edges)
if args.save_txt is not None or args.save_npz is not None:
nodes = list(G.nodes())
adj_matrix = nx.to_numpy_matrix(G, nodelist=nodes, dtype=int)
if args.save_npz is not None:
base_fname = args.save_npz.replace('.npz', '')
scipy.sparse.save_npz(args.save_npz,
scipy.sparse.coo_matrix(adj_matrix))
else:
base_fname = args.save_txt.replace('.txt', '')
np.savetxt(args.save_txt, adj_matrix, fmt='%d')
node_info = get_node_info(nodes, error_codes, resource_pages, args)
with open(base_fname + '_nodes.txt', 'w') as f:
f.write('\n'.join(
[nodes[i] + '\t' + node_info[i] for i in range(len(nodes))]))
net = Network(width=args.width, height=args.height, directed=True)
net.from_nx(G)
if args.show_buttons:
net.show_buttons()
elif args.options is not None:
try:
with open(args.options, 'r') as f:
net.set_options(f.read())
except FileNotFoundError as e:
print('Error: options file', args.options, 'not found.')
except Exception as e:
print('Error applying options:', e)
for node in net.nodes:
node['size'] = 15
node['label'] = ''
if node['id'].startswith(args.site_url):
node['color'] = INTERNAL_COLOR
if node['id'] in resouce_pages:
node['color'] = RESOURCE_COLOR
else:
node['color'] = EXTERNAL_COLOR
if node['id'] in error_codes:
node[
'title'] = f'{error_codes[node["id"]]} Error: <a href="{node["id"]}">{node["id"]}</a>'
node['color'] = ERROR_COLOR
else:
node['title'] = f'<a href="{node["id"]}">{node["id"]}</a>'
net.save_graph(args.vis_file)
def main():
net = Network(height=600, width=900)
net.barnes_hut()
dbName = "pcapDb"
if (not os.path.exists(dbName)):
conn = sqlite3.connect(dbName)
conn.execute(
"""create table packetdata (src text,dst text,len integer);""")
else:
conn = sqlite3.connect(dbName)
cursor = conn.cursor()
header_struct = struct.Struct('!I')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("localhost", 9888))
sock.listen(1)
q = queue.Queue()
reciever, _ = sock.accept()
t = threading.Thread(target=display, args=(q, net))
# t.start()
while (True):
data_len = recvall(reciever, header_struct.size)
(data_len, ) = header_struct.unpack(data_len)
data = recvall(reciever, data_len)
data = pickle.loads(data)
# print(data)
nodes = set()
for i in data:
if (i is None):
continue
src, dst, length = i
# print(type(i))
cursor.execute(
"insert into packetdata (src,dst,len) values(?,?,?)",
(src, dst, length))
nodes.add(src)
nodes.add(dst)
weights = []
# print("Here")
for i in nodes:
la = "IP: " + i
net.add_node(i, label=i, title=la)
for i in data:
if (i is None):
continue
src, dst, weight = i
net.add_edge(src, dst, value=weight)
net.write_html("index.html")
time.sleep(4)
def visualize_content(root):
net = Network(width="100%", height="100%")
def recurse(node):
net.add_node(node.id, node.name)
for content in node.contents:
net.add_node(content.id, content.name)
net.add_edge(node.id, content.id)
recurse(content)
recurse(root)
net.show("content.html")
def build_pyvis_graph(nx_graph,
link_title_dict,
link_content_dict,
node_shape_dict=None):
graph = Network(notebook=True, directed=True, width=1000, height=1000)
graph.from_nx(nx_graph)
for node in graph.nodes:
node['label'] = link_title_dict[node['id']]
if node['id'] in link_content_dict:
node['title'] = link_content_dict[node['id']]
if node_shape_dict:
node['value'] = node_shape_dict[node['id']]
return graph
def corr_graph(Source=list(windows_df_filtered_tb.Table.unique()), Destination=list(windows_df_filtered_tb.Table.unique())):
sp_graph, sp_list, sp_colors, sp_sizes = shortest_path(windows_df_filtered_tb,Source,Destination)
# Check if the sp_list returned is not NULL which means there is no path and if sp_list == 1 this means that the Table name in Source and Destination is the same
if sp_list is not None and len(sp_list) > 1:
sp_gr=Network(notebook=True, bgcolor="#222222", font_color="white")
sp_gr.add_nodes(sp_list, value=sp_sizes, title=sp_list, color=sp_colors)
sp_gr.barnes_hut()
sp_gr.from_nx(sp_graph)
return(sp_gr.show("graphs/shortest_path_graph.html"))
def Plot_Non_Spatial(Graph, border, center, tag, dt_string, h):
net = Network()
for n in Graph.nodes():
if n in border[0]:
color = "#FF0000"
else:
color = "#A0CBE2"
net.add_node(n, label=n, color=color)
print(n)
for e in Graph.edges():
net.add_edge(int(e[0]), int(e[1]))
net.show_buttons()
net.show('img/Border_Plot_'+ tag + '_' + dt_string + '_h=' + str(h) + '.html')
def __init__(self, net_graph: nx.DiGraph, help_net_graph: nx.DiGraph,
filename: str, path: str) -> None:
# Configuration data
themes_dir_path = self.find_path()
vis_theme_option = self.parse_theme_option(themes_dir_path)
vis_theme_data = self.parse_theme(themes_dir_path, vis_theme_option)
self.vis_graph = Network(height='100%', width='100%')
self.set_icons(vis_theme_data)
self.net_graph = net_graph
self.vis_graph.from_nx(help_net_graph)
self.filename = filename
self.path = path
def create_viz_html(graph, filename):
filename = ensure_html_filename(filename)
g_vis = Network()
g_vis.barnes_hut()
g_vis.from_nx(graph)
g_vis.show(filename)
class show_path:
def __init__(self):
self.graph = Network(height="750px", width="100%", directed=True)
with open('layouts\path_layout.json') as f:
self.composite_options = json.load(f)
def show_graph(self):
dirOutput = "output"
if not os.path.exists(dirOutput):
os.makedirs("output")
self.graph.show("output\path_graph.html")
def add_path(self, input, amount):
input_edge = zip(input[0:len(input) - 1], input[1:len(input)], amount)
for i in input_edge:
input = i[0]
output = i[1]
weight = i[2]
self.graph.add_node(input)
self.graph.add_node(output)
self.graph.add_edge(input, output, value=weight, title=weight)
self.graph.options = self.composite_options
def show(self):
self.show_graph()
class transaction_graph:
def __init__(self):
self.graph = Network(height="750px", width="100%", directed=True)
with open('layouts\\transaction_graph_layout.json') as f:
self.options = json.load(f)
def show_graph(self):
dirOutput = "output"
if not os.path.exists(dirOutput):
os.makedirs("output")
self.graph.show("output\\transaction_gragh.html")
def add_transaction(self, input, output, in_time, out_time, amount):
input_edge = zip(input, in_time, output, out_time, amount)
for i in input_edge:
input = i[0]
label = i[0]
time_in = i[1]
output = i[2]
time_out = i[3]
weight = i[4]
self.graph.add_node(input, level=time_in, shape="square")
self.graph.add_node(output, level=time_out, shape="square")
self.graph.add_edge(input, output, value=weight, title=weight)
self.graph.options = self.options
def getUniCourseGraph():
"""
Get university-course graph.
"""
results = g.query("""
select distinct ?courseID ?courseName ?instructorFN ?instructorGN ?university where {
?courseID a ccso:Course .
?courseID ccso:csName ?courseName .
?courseID ccso:hasInstructor ?inst .
?inst foaf:familyName ?instructorFN .
?inst foaf:givenName ?instructorGN .
?courseID ccso:offeredBy ?dept .
?dept ccso:belongsTo ?uni .
?uni ccso:legalName ?university .
}""")
visGraph = Network(height='750px',
width='100%') # PyVis-Network, for visualization.
nxGraph = nx.Graph() # NetworkX, for analyses
for courseID, courseName, instLast, instFirst, uni in results:
instName = f"{instFirst} {instLast}"
nxGraph.add_node(uni)
visGraph.add_node(uni, shape='circle', title=uni, mass=2)
nxGraph.add_node(courseID)
visGraph.add_node(courseID, shape='box', label=courseName)
visGraph.add_edge(uni, courseID)
nxGraph.add_edge(uni, courseID)
return nxGraph, visGraph
def show_network(G):
nt = Network("700px", "700px", directed=True)
nt.from_nx(G)
#UNCOMMENT BELOW TO PLAY WITH PHYSICS OF SHOWN GRAPH
#nt.show_buttons(filter_=["physics"])
#nt.show_buttons(filter_=['nodes', 'interaction', 'selection'])
nt.toggle_physics(status=True)
nt.force_atlas_2based(gravity=-500,
central_gravity=0.02,
damping=0.2,
overlap=0.5)
nt.show("Most_Related_Artists.html")
def draw_net(package):
sample, data = package
net = Network()
nodes = targeted_isotpye
colors = list(map(lambda x: isotype_color[x], nodes))
# nodes =
# sizes = data.groupby("isotype")["cluster"].c ount()[targeted_isotpye][nodes].to_list()
# net.add_nodes(nodes, color = colors, size = sizes)
net.add_nodes(nodes, color=colors)
tmp = deepcopy(nodes)
edges = []
sample_num = len(data)
max = 0
while len(tmp) > 0:
element_from = tmp.pop(0)
for element_to in tmp:
if set([element_from, element_to]) == set(["IGHA1", "IGHA2"]):
continue
set_x = set(data[data["isotype"] == element_from]["cluster"])
set_y = set(data[data["isotype"] == element_to]["cluster"])
x_and_y = len(set_x.intersection(set_y))
x_and_Ny = len(set_x - set_y)
Nx_and_y = len(set_y - set_x)
Ny_and_Nx = sample_num - len(set_x.union(set_y))
odd, p = fisher_exact([[x_and_y, Nx_and_y], [x_and_Ny, Ny_and_Nx]])
# p = -log10(p)
# p = 2 if p < -log10(0.01) else p #卡阈值为0.01
width = x_and_y
# width = 1 if width > 0 and width < 1 else width
edges.append(
[element_from,
len(set_x), element_to,
len(set_y), width, p])
edges = pd.DataFrame(
edges, columns=["from", "from_size", "to", "to_size", "weight", "p"])
edges["qValue"] = qvalue(edges["p"])[1]
edges["Sig"] = ["*" if y < 0.05 else "" for y in edges["qValue"]]
edges["sample"] = sample
edges = edges.set_index("sample", append=True)
# edges.to_csv("./data/{}.csv".format(sample))
# edges = edges.set_index(["index",sample])
return edges
def create_network_kmeans(text,
kmeans=[],
tfidf=None,
key_terms=None,
freq_df=[],
k_list=[2],
n=20):
'''
create html files with network graphs, where for each cluster it is stored the most frequent terms
'''
if len(kmeans) == 0:
if tfidf is None or key_terms is None:
#determine tfidf, key_terms
tfidf, key_terms = determine_tfidf(text)
#determine kmeans
for k in k_list:
kmeans.append(k_means(tfidf, k))
if len(freq_df) == 0:
#determine frequency for each key term
freq_df = find_frequent_terms(key_terms, text)
for c in range(len(k_list)):
#number of clusters
k = k_list[c]
#clusters lables
labels = pd.Series(kmeans[c].labels_.tolist())
#start network
G = Network(height="750px", width="100%", font_color="black")
for i in labels.unique():
#filter text items within the cluster
filter_text = text.loc[labels == i].reset_index(drop=True)
#add node with the cluster name, have the size as the number of items
G.add_nodes(['Cluster ' + str(i)], value=[len(filter_text)])
#determine frequency of the key within cluster only
new_freq_df = find_frequent_terms(key_terms, filter_text)
#for the most frequent terms within the cluster
for j in range(n):
#get term
term = new_freq_df['terms'][j]
#add node with the term with the respective frequency overall as size
G.add_node(str(term),
value=freq_df.loc[freq_df['terms'] == term, ]
['count'].tolist()[0])
#add edge between cluster label and term
G.add_edge('Cluster ' + str(i), str(term))
#create and html file with network graph
G.show('_' + str(k) + "clusters.html")
