def alignment_graph(lengths=[], pairings=[], alignments=[]):
#print('making graph')
g = Graph(directed=False)
seq_index = g.new_vertex_property("int")
time = g.new_vertex_property("int")
#add vertices
g.add_vertex(sum(lengths))
seq_index.a = np.concatenate([np.repeat(i,l) for i,l in enumerate(lengths)])
time.a = np.concatenate([np.arange(l) for l in lengths])
#add edges (alignments)
alignment_index = g.new_edge_property("int")
segment_index = g.new_edge_property("int")
for i,a in enumerate(alignments):
if len(a) > 0:
j, k = pairings[i]
pairs = np.concatenate(a, axis=0)
indicesJ = (np.arange(lengths[j]) + sum(lengths[:j]))[pairs.T[0]]
indicesK = (np.arange(lengths[k]) + sum(lengths[:k]))[pairs.T[1]]
seg_indices = np.concatenate([np.repeat(i, len(a))
for i,a in enumerate(a)])
g.add_edge_list(np.vstack([indicesJ, indicesK,
np.repeat(i, len(pairs)), seg_indices]).T,
eprops=[alignment_index, segment_index])
#g.add_edge_list([(b, a) for (a, b) in g.edges()])
#print('created alignment graph', g)
#g = prune_isolated_vertices(g)
#print('pruned alignment graph', g)
#g = transitive_closure(g)
#graph_draw(g, output_size=(1000, 1000), output="results/casey_jones_bars.pdf")
return g, seq_index, time, alignment_index, segment_index
class StackGraph(object):
def __init__(self):
self.g = None
def load(self, filename):
# Initialize the graph
self.g = Graph()
# Each node will store a FunctionWrapper() class instance.
self.g.vertex_properties["functions"] = self.g.new_vertex_property("object")
self.g.vertex_properties["display"] = self.g.new_vertex_property("string")
# Each edge will store a [ ..tbd.. ] .
self.g.edge_properties["calls"] = self.g.new_edge_property("object")
# Load the log file and build the graph
i = 0
f = open(filename, "rb")
for line in f:
i += 1
try:
# Skip any informational lines
if "*" in line: continue
# Extract a call stack snapshot
words = line.split()
time = words[0][2:]
depth = words[1][2:]
stack = [FunctionWrapper(instring=item) for item in words[2].split("->")]
# Add the top 2 functions to the graph, if necessary. Format: f1()->f2()
f1, f2 = stack[-2], stack[-1]
v1, v2 = None, None
# Search for the vertices
for v in self.g.vertices():
if self.g.vp.functions[v] == f1: v1 = v
if self.g.vp.functions[v] == f2: v2 = v
if v1 != None and v2 != None: break
# Add new vertices if needed
if v1 == None:
v1 = self.g.add_vertex()
self.g.vp.functions[v1] = f1
self.g.vp.display[v1] = f1.graphDisplayString()
if v2 == None:
v2 = self.g.add_vertex()
self.g.vp.functions[v2] = f2
self.g.vp.display[v2] = f2.graphDisplayString()
# Add the edge if necessary, and then add data to it
if not self.g.edge(v1, v2):
e = self.g.add_edge(v1, v2)
self.g.ep.calls[e] = CallList(v1, v2)
self.g.ep.calls[e].addCall(time, depth)
except Exception as e:
print "Exception on line", i, ":", e
print [str(x) for x in stack]
exit()
def __init_properties(g: gt.Graph):
# sets properties used to keep trace of the opinion of each vertex
opinion = g.new_vertex_property("int", 0)
g.vertex_properties["opinion"] = opinion
# used for graphic representation
opinion_color = g.new_vertex_property("vector<double>")
g.vertex_properties["opinion_color"] = opinion_color
for v in g.vertices():
g.vertex_properties["opinion_color"][v] = __color_map(0)
return g
def build_word_graph(model_fname, limiar=0.2):
"""
Constroi um grafo de walavras ponderado pela similaridade entre elas
de acordo com o modelo.
:param model_fname: Nome do arquivo com o modelo word2vec como foi salvo
:return: objeto grafo
"""
m = Word2Vec.load(model_fname)
g = Graph()
freq = g.new_vertex_property("int")
weight = g.new_edge_property("float")
i = 0
vdict = {}
for w1, w2 in combinations(m.vocab.keys(), 2):
if w1 == '' or w2 == '':
continue
# print(w1,w2)
v1 = g.add_vertex() if w1 not in vdict else vdict[w1]
vdict[w1] = v1
freq[v1] = m.vocab[w1].count
v2 = g.add_vertex() if w2 not in vdict else vdict[w2]
vdict[w2] = v2
freq[v2] = m.vocab[w2].count
sim = m.similarity(w1, w2)
if sim > 0.1:
e = g.add_edge(v1, v2)
weight[e] = sim
if i > 10000:
break
i += 1
g.vertex_properties['freq'] = freq
g.edge_properties['sim'] = weight
return g
def build_word_graph(model_fname, limiar=0.2):
"""
Constroi um grafo de walavras ponderado pela similaridade entre elas
de acordo com o modelo.
:param model_fname: Nome do arquivo com o modelo word2vec como foi salvo
:return: objeto grafo
"""
m = Word2Vec.load(model_fname)
g = Graph()
freq = g.new_vertex_property("int")
weight = g.new_edge_property("float")
i = 0
vdict = {}
for w1, w2 in combinations(m.vocab.keys(), 2):
if w1 == '' or w2 == '':
continue
# print(w1,w2)
v1 = g.add_vertex() if w1 not in vdict else vdict[w1]
vdict[w1] = v1
freq[v1] = m.vocab[w1].count
v2 = g.add_vertex() if w2 not in vdict else vdict[w2]
vdict[w2] = v2
freq[v2] = m.vocab[w2].count
sim = m.similarity(w1, w2)
if sim > 0.1:
e = g.add_edge(v1, v2)
weight[e] = sim
if i > 10000:
break
i += 1
g.vertex_properties['freq'] = freq
g.edge_properties['sim'] = weight
return g
def simulate_cascade(g, p, source=None, return_tree=False):
"""
graph_tool version of simulating cascade
return np.ndarray on vertices as the infection time in cascade
uninfected node has dist -1
"""
gv = sample_graph_by_p(g, p)
if source is None:
# consider the largest cc
infected_nodes = np.nonzero(label_largest_component(gv).a)[0]
source = np.random.choice(infected_nodes)
times = get_infection_time(gv, source)
if return_tree:
# get the tree edges
_, pred_map = shortest_distance(gv, source=source, pred_map=True)
edges = [(pred_map[i], i) for i in infected_nodes if i != source]
# create tree
tree = Graph(directed=True)
tree.add_vertex(g.num_vertices())
for u, v in edges:
tree.add_edge(int(u), int(v))
vfilt = tree.new_vertex_property('bool')
vfilt.a = False
for v in set(itertools.chain(*edges)):
vfilt[v] = True
tree.set_vertex_filter(vfilt)
if return_tree:
return source, times, tree
else:
return source, times
def session_draw_bis_melty(sessions_id, weblog, weblog_columns_dict):
"""
Draw the graph of sessions with sessions_id given in entry
"""
from graph_tool.all import Graph
from graph_tool.all import graph_draw
session = weblog[weblog.session_id == sessions_id]
session = session.rename(index=str,columns = {weblog_columns_dict['requested_page_column']:'requested_page',\
weblog_columns_dict['referrer_page_column']:'referrer_page'})
s_pages = session[['requested_page', 'requested_external']]
s_pages_ref = session[['referrer_page', 'referrer_external']]
s_pages_ref = s_pages_ref.rename(index=str,
columns={
'referrer_page': 'requested_page',
'referrer_external':
'requested_external'
})
s_pages = s_pages.append(s_pages_ref)
s_pages.drop_duplicates(subset='requested_page', inplace=True)
g = Graph()
v = {}
halo = g.new_vertex_property("bool")
for row in s_pages.itertuples():
v[row.requested_page] = g.add_vertex()
if row.requested_external:
halo[v[row.requested_page]] = True
else:
halo[v[row.requested_page]] = False
session.apply(
lambda x: g.add_edge(v[x.referrer_page], v[x.requested_page]), axis=1)
graph_draw(g,
vertex_halo=halo,
output="./_session" + str(sessions_id) + ".png")
return
def graph_from_pdb(pdb_str):
g = Graph(directed=False)
vertex_types = g.new_vertex_property("string")
g.vertex_properties['type'] = vertex_types
vertices = []
def get_connect_list():
connect_ids_list = [
[int(str_id) for str_id in line.split()[1:]]
for line in pdb_str.splitlines()
if is_pdb_connect_line(line)
]
return reduce(
lambda acc, e: acc + e,
[
[
(i, j)
for (i, j) in map(
lambda i_j: (i_j[0] - 1, i_j[1] - 1),
zip(cycle(connect_ids[0:1]), connect_ids[1:]),
)
if i < j
]
for connect_ids in connect_ids_list
],
[],
)
connects = get_connect_list()
def get_valence(atom_id):
return sum([1 for connect in connects if atom_id in connect])
atom_lines = [
line for line in pdb_str.splitlines()
if is_pdb_atom_line(line)
]
for (atom_id, line) in enumerate(atom_lines):
fields = pdb_fields(line)
v = g.add_vertex()
vertex_types[v] = type_identifier_for(
fields[11].strip().upper(),
get_valence(atom_id),
)
vertices.append(v)
for (i, j) in connects:
g.add_edge(vertices[i], vertices[j])
return g
def vytvořím_graph_tool_graf():
from graph_tool.all import Graph
graf = Graph()
u1 = graf.add_vertex()
u2 = graf.add_vertex()
graf.add_edge(u1, u2)
vprop_double = graf.new_vertex_property("double") # Double-precision floating point
vprop_double[graf.vertex(1)] = 3.1416
vprop_vint = graf.new_vertex_property("vector<int>") # Vector of ints
vprop_vint[graf.vertex(0)] = [1, 3, 42, 54]
eprop_dict = graf.new_edge_property("object") # Arbitrary python object. In this case, a dictionary.
eprop_dict[graf.edges().next()] = {"foo": "bar", "gnu": 42}
gprop_bool = graf.new_graph_property("bool") # Boolean
gprop_bool[graf] = True
graf.save('./data/graph_tool.graphml', fmt='xml')
def rysuj_graf_wejsciowy(g, output=None, size=(600, 600), bez_napisow=False):
gx = Graph(g)
gx.vertex_properties['wyswietlany_tekst'] = gx.new_vertex_property('string')
for v in gx.vertices():
gx.vertex_properties['wyswietlany_tekst'][v] = v.__str__() + ': ' + \
str(gx.vertex_properties['liczba_kolorow'][v])
if bez_napisow:
graph_draw(gx
# , vertex_text=gx.vertex_properties['wyswietlany_tekst']
, bg_color=[255., 255., 255., 1]
, output_size=size
, output=output
)
else:
graph_draw(gx
, vertex_text=gx.vertex_properties['wyswietlany_tekst']
, bg_color=[255., 255., 255., 1]
, output_size=size
, output=output
)
def rysuj_graf_wejsciowy(g, output=None, size=(600, 600), bez_napisow=False):
gx = Graph(g)
gx.vertex_properties['wyswietlany_tekst'] = gx.new_vertex_property(
'string')
for v in gx.vertices():
gx.vertex_properties['wyswietlany_tekst'][v] = v.__str__() + ': ' + \
str(gx.vertex_properties['liczba_kolorow'][v])
if bez_napisow:
graph_draw(
gx
# , vertex_text=gx.vertex_properties['wyswietlany_tekst']
,
bg_color=[255., 255., 255., 1],
output_size=size,
output=output)
else:
graph_draw(gx,
vertex_text=gx.vertex_properties['wyswietlany_tekst'],
bg_color=[255., 255., 255., 1],
output_size=size,
output=output)
def build_graph(m_codes, m_list):
n_models, n_attributes = m_codes.shape
g = Graph()
v_map = {}
names = g.new_vertex_property("object")
v_atts = g.add_vertex(n_attributes)
v_mods = g.add_vertex(n_models)
v_imps = g.add_vertex(n_attributes)
for v_idx, v in enumerate(v_atts):
v_n = v_name(v_idx, kind="data")
v_map[v_n] = int(v)
names[v] = v_n
for v_idx, v in enumerate(v_mods):
v_n = v_name(v_idx, kind="model")
v_map[v_n] = int(v)
names[v] = v_n
in_edges = ((d, v) for d in m_list[v_idx].desc_ids)
out_edges = ((v, t) for t in m_list[v_idx].targ_ids)
g.add_edge_list(in_edges)
g.add_edge_list(out_edges)
for v_idx, v in enumerate(v_imps):
v_n = v_name(v_idx, kind="imputation")
v_map[v_n] = int(v)
names[v] = v_n
g.vp.names = names
g.v_map = v_map
return g
def gen_graph((repo, events)):
graph = Graph()
repo_on_graph = graph.new_graph_property('string')
repo_on_graph[graph] = repo
graph.graph_properties['repo_on_graph'] = repo_on_graph
language_on_graph = graph.new_graph_property('string')
language_on_graph[graph] = events[0]['language']
graph.graph_properties['language_on_graph'] = language_on_graph
events_on_vertices = graph.new_vertex_property('object')
graph.vertex_properties['events_on_vertices'] = events_on_vertices
actors_on_vertices = graph.new_vertex_property('string')
graph.vertex_properties['actors_on_vertices'] = actors_on_vertices
weights_on_edges = graph.new_edge_property('long double')
graph.edge_properties['weights_on_edges'] = weights_on_edges
# pre_vertices = []
pre_events_map = {}
pre_vertices_map = {}
# owner_vertex = graph.add_vertex()
# owner = repo.split('/')[0]
# actors_on_vertices[owner_vertex] = owner
# pre_vertices_map[owner] = owner_vertex
events = sorted(events, key=lambda x: x['created_at'])
for event in events:
actor = event['actor']
if actor in pre_events_map:
continue
created_at = event['created_at']
vertex = graph.add_vertex()
events_on_vertices[vertex] = event
actors_on_vertices[vertex] = actor
if 'actor-following' not in event:
continue
following = set(event['actor-following'])
commons = following.intersection(pre_vertices_map.keys())
# pre_vertices.append(vertex)
# if len(commons) == 0:
# edge = graph.add_edge(vertex, owner_vertex)
# weights_on_edges[edge] = 1.0
for pre_actor in commons:
edge = graph.add_edge(vertex, pre_vertices_map[pre_actor])
interval =\
(created_at - pre_events_map[pre_actor]['created_at']).days
weight = 1.0 / fib(interval + 2)
weights_on_edges[edge] = weight
pre_events_map[actor] = event
pre_vertices_map[actor] = vertex
return graph
filename = '../pairparser/results/p_pairs7.txt'
print(filename)
coefficient = 3
word_dict = {}
add_dict = {}
f = open('bad.txt', 'r', encoding="utf-8")
for s in f:
# print(s.split(' ')[0])
add_dict[s.split(' ')[0]] = 1
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("int")
ver_names = pairs_graph.new_vertex_property("string")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 in add_dict or w2 in add_dict:
continue
def graph(data,
vertex_text=None,
decorate_graph=True,
use_random_colors: bool = True,
shuffle_equivalence_classes: bool = True):
try:
from graph_tool.all import Graph
except ImportError as e:
if RAISE_IF_MISSING_GRAPH_TOOL:
raise
else:
from sys import stderr
stderr.write(
'Package graph_tool could not be imported. Error was: {0}'.
format(e))
return None
g = Graph(directed=False)
vertex_types = g.new_vertex_property("string")
g.vertex_properties['type'] = vertex_types
if decorate_graph:
vertex_equivalence_classes = g.new_vertex_property("vector<double>")
g.vertex_properties['equivalence_class'] = vertex_equivalence_classes
if USE_DIFFERENT_ELEMENT_SHAPES:
vertex_shapes = g.new_vertex_property("int")
g.vertex_properties['shape'] = vertex_shapes
equivalence_classes = [
atom['equivalenceGroup'] for atom in list(data.atoms.values())
]
unique_equivalence_classes = {}
assigned_equivalence_classes = {}
next_equivalence_class = 0
for (atom_index, atom) in sorted(list(data.atoms.items()),
key=itemgetter(0)):
if atom['equivalenceGroup'] == -1:
unique_equivalence_classes[atom_index] = next_equivalence_class
next_equivalence_class += 1
else:
if atom['equivalenceGroup'] not in assigned_equivalence_classes:
assigned_equivalence_classes[
atom['equivalenceGroup']] = next_equivalence_class
next_equivalence_class += 1
unique_equivalence_classes[
atom_index] = assigned_equivalence_classes[
atom['equivalenceGroup']]
range_unique_equivalence_classes = list(
range(len(set(unique_equivalence_classes.values()))))
equivalence_class_permutation = dict(
list(
zip(
range_unique_equivalence_classes,
(random.permutation(range_unique_equivalence_classes)
if shuffle_equivalence_classes else
range_unique_equivalence_classes), # pylint: disable=no-member
)), )
unique_elements = set(
[atom['type'] for atom in list(data.atoms.values())])
symbol_index_for_element = dict(
list(
map(
lambda n_element: (n_element[1], n_element[
0] % 15), # There are only 15 different symbols
enumerate(unique_elements),
)), )
hues = linspace( # pylint: disable=no-member
0.0,
1.0,
len(set(list(unique_equivalence_classes.values()), ), ) // 2 + 1,
).tolist()
colours = list(
map(
lambda c: hsv_to_rgb(*c),
[(H, S, 0.5) for H in hues for S in (0.3, 0.9)],
))
# Shuffle the colours around the graph
colours = ([
# Numpy cast all tuples to lists; needs to be reverted
tuple(colour) for colour in random.permutation(colours).tolist()
] if use_random_colors else colours)
if vertex_text == 'element_valence':
vertex_text_fct = lambda atom_index, atom: '{element}{valence}'.format(
element=atom['type'],
valence=len(atom['conn']) if USE_NEIGHBOUR_VALENCES else '',
)
elif vertex_text == 'element_equivalence':
vertex_text_fct = lambda atom_index, atom: '{element}{equivalence}'.format(
element=atom['type'],
equivalence=unique_equivalence_classes[atom_index],
)
elif vertex_text == 'name_equivalence':
vertex_format_str = '{symbol} ({equivalence})'
vertex_text_fct = lambda atom_index, atom: vertex_format_str.format(
symbol=atom['symbol'],
equivalence=unique_equivalence_classes[atom_index],
)
else:
raise Exception('Unvalid vertex_text')
vertices = {}
for (atom_index, atom) in sorted(list(data.atoms.items()),
key=itemgetter(0)):
v = g.add_vertex()
vertex_types[v] = vertex_text_fct(atom_index, atom)
vertices[atom_index] = v
if decorate_graph:
vertex_equivalence_classes[v] = colours[
unique_equivalence_classes[atom_index]] + (OPAQUE, )
if USE_DIFFERENT_ELEMENT_SHAPES:
#vertex_shapes[v] = symbol_index_for_element[atom['type']]
pass
for (i, j) in [bond['atoms'] for bond in data.bonds]:
g.add_edge(vertices[i], vertices[j])
return g
def steiner_tree_mst(g, root, infection_times, source, terminals,
closure_builder=build_closure,
strictly_smaller=True,
return_closure=False,
k=-1,
debug=False,
verbose=True):
gc, eweight, r2pred = closure_builder(g, root, terminals,
infection_times,
strictly_smaller=strictly_smaller,
k=k,
debug=debug,
verbose=verbose)
# get the minimum spanning arborescence
# graph_tool does not provide minimum_spanning_arborescence
if verbose:
print('getting mst')
gx = gt2nx(gc, root, terminals, edge_attrs={'weight': eweight})
try:
nx_tree = nx.minimum_spanning_arborescence(gx, 'weight')
except nx.exception.NetworkXException:
if debug:
print('fail to find mst')
if return_closure:
return None, gc, None
else:
return None
if verbose:
print('returning tree')
mst_tree = Graph(directed=True)
for _ in range(g.num_vertices()):
mst_tree.add_vertex()
for u, v in nx_tree.edges():
mst_tree.add_edge(u, v)
if verbose:
print('extract edges from original graph')
# extract the edges from the original graph
# sort observations by time
# and also topological order
topological_index = {}
for i, e in enumerate(bfs_iterator(mst_tree, source=root)):
topological_index[int(e.target())] = i
sorted_obs = sorted(
set(terminals) - {root},
key=lambda o: (infection_times[o], topological_index[o]))
tree_nodes = {root}
tree_edges = set()
# print('root', root)
for u in sorted_obs:
if u in tree_nodes:
if debug:
print('{} covered already'.format(u))
continue
# print(u)
v, u = map(int, next(mst_tree.vertex(u).in_edges())) # v is ancestor
tree_nodes.add(v)
late_nodes = [n for n in terminals if infection_times[n] > infection_times[u]]
vis = init_visitor(g, u)
# from child to any tree node, including v
cpbfs_search(g, source=u, terminals=list(tree_nodes),
forbidden_nodes=late_nodes,
visitor=vis,
count_threshold=1)
# dist, pred = shortest_distance(g, source=u, pred_map=True)
node_set = {v for v, d in vis.dist.items() if d > 0}
reachable_tree_nodes = node_set.intersection(tree_nodes)
ancestor = min(reachable_tree_nodes, key=vis.dist.__getitem__)
edges = extract_edges_from_pred(g, u, ancestor, vis.pred)
edges = {(j, i) for i, j in edges} # need to reverse it
if debug:
print('tree_nodes', tree_nodes)
print('connecting {} to {}'.format(v, u))
print('using ancestor {}'.format(ancestor))
print('adding edges {}'.format(edges))
tree_nodes |= {u for e in edges for u in e}
tree_edges |= edges
t = Graph(directed=True)
for _ in range(g.num_vertices()):
t.add_vertex()
for u, v in tree_edges:
t.add_edge(t.vertex(u), t.vertex(v))
tree_nodes = {u for e in tree_edges for u in e}
vfilt = t.new_vertex_property('bool')
vfilt.a = False
for v in tree_nodes:
vfilt[t.vertex(v)] = True
t.set_vertex_filter(vfilt)
if return_closure:
return t, gc, mst_tree
else:
return t
def main():
network = json.load(open('../in_use_network.json', 'r'))
num_switches = len(network['switches'])
max_port_num = 4
#we need num_switches + num_switches * max_port_num bits
bitstring_len = int(
math.pow(2, math.ceil(math.log(num_switches * (max_port_num + 1), 2))))
#Generate graph representation of network
g = Graph(directed=False)
name_to_vertex = {}
vprop_name = g.new_vertex_property("string")
vprop_dev = g.new_vertex_property("int")
vprop_bp = g.new_vertex_property("int")
bp = 1
for switch in network['switches']:
v_switch = g.add_vertex()
name_to_vertex[switch['name']] = v_switch
vprop_name[v_switch] = switch['name']
vprop_dev[v_switch] = 1
vprop_bp[v_switch] = bp
bp += 1
for host in switch['hosts']:
v_host = g.add_vertex()
g.add_edge(v_switch, v_host)
name_to_vertex[host['name']] = v_host
vprop_name[v_host] = host['name']
vprop_dev[v_host] = 2
for link in network['switch_links']:
v_node1 = name_to_vertex[link['node1']['name']]
v_node2 = name_to_vertex[link['node2']['name']]
g.add_edge(v_node1, v_node2)
#remove old files
for switch in network['switches']:
silent_rm(file_templ.format(switch['name']))
switches_dict = {}
switches_list = []
for switch in network['switches']:
switches_dict[switch['name']] = switch
switches_list.append(switch)
#thrift_server = switch['control_network_ip']
#Default drop packet
#append_entry_file("table_set_default bift _drop", switch['name'])
for i in range(0, num_switches):
bits_of_interest = ['0'] * bitstring_len
bits_of_interest[bitstring_len - i - 1] = '1'
append_entry_file("table_add bift local_decap {0} =>".format(i + 1),
switches_list[i]['name'])
#Erste X ports sind hosts
port_offset = len(switch['hosts']) + 1
for j in range(0, max_port_num):
bit_pos = (i * max_port_num) + j + num_switches
bits_of_interest[bitstring_len - bit_pos - 1] = '1'
append_entry_file(
"table_add bift forward_connected {0} => {1}".format(
bit_pos + 1, port_offset + j), switches_list[i]['name'])
append_entry_file(
"table_add get_bits_of_interest save_bits_of_interest 0/0 => 0b" +
"".join(bits_of_interest), switches_list[i]['name'])
#NNHs
v_switch = name_to_vertex[switches_list[i]['name']]
print("NNHs of {0}".format(switches_list[i]['name']))
nnhs = ['0'] * bitstring_len
for nh in v_switch.out_neighbours():
for nnh in nh.out_neighbours():
if nnh != v_switch and vprop_dev[nnh] == 1:
nnhs[bitstring_len - vprop_bp[nnh]] = '1'
print("".join(nnhs))
plt.plot(range(b + 1, len(ver_attr)), ver_attr[(b + 1):], 'ro')
print("-------------------------------------------------")
filename = '../pairparser/results/en_pairs(7).txt'
ftag = '7_3imp'
coefficient = 3
word_dict = {} # dict with indexes of nodes by word
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("double")
ver_names = pairs_graph.new_vertex_property("string")
ver_id = pairs_graph.new_vertex_property("int")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 not in word_dict:
class Network:
def __init__(self, nodes_info=None, links_info=None, file_name=None):
self.g = Graph()
if nodes_info and links_info:
self.nodes_info = nodes_info
self.links_info = links_info
self.g.vertex_properties["name"] = self.g.new_vertex_property(
'string')
self.g.vertex_properties["id"] = self.g.new_vertex_property(
'int32_t')
self.g.edge_properties["weight"] = self.g.new_edge_property(
'int32_t')
self.create_network()
self.g.vertex_properties["pagerank"] = pagerank(
self.g, weight=self.g.edge_properties["weight"])
self.g.vertex_properties[
"degree_centrality"] = self.degree_centrality()
elif file_name:
self.load_network(file_name)
def create_network(self):
# Add Nodes
for node in self.nodes_info:
self.add_n(node)
# Add Links
for link in self.links_info:
n_loser = 0
n_winner = 0
loser = link['loser']
winner = link['winner']
weight = link['rounds']
for team_id in self.g.vertex_properties.id:
if loser == team_id:
break
n_loser += 1
for team_id in self.g.vertex_properties.id:
if winner == team_id:
break
n_winner += 1
self.add_l(n_loser, n_winner, 16 / weight * 100)
def load_network(self, file_name):
new_file_name = '..' + sep + '..' + sep + 'network-graphs' + sep + file_name
self.g.load(new_file_name, fmt="gt")
def get_normalized_pagerank(self):
max_pgr = 0
for pgr in self.g.vertex_properties.pagerank:
if pgr > max_pgr:
max_pgr = pgr
return [
self.g.vertex_properties.pagerank[v] / max_pgr
for v in self.g.vertices()
]
def add_n(self, node_info):
n = self.g.add_vertex()
self.g.vertex_properties.id[n] = node_info['id']
self.g.vertex_properties.name[n] = node_info['Team_Name']
def add_l(self, loser, winner, weight):
n1 = self.g.vertex(loser)
n2 = self.g.vertex(winner)
l = self.g.add_edge(n1, n2)
self.g.edge_properties.weight[l] = weight
def draw(self, output_file, fmt):
graph_draw(self.g,
vertex_text=self.g.vertex_index,
output=output_file,
fmt=fmt)
def save_network(self, file_name):
try:
new_file_name = '..' + sep + '..' + sep + 'network-graphs' + sep + file_name
self.g.save(new_file_name, fmt="gt")
except:
return False
return True
def vp_pagerank(self):
return self.g.vertex_properties.pagerank
def vp_degree_cent(self):
return self.g.vertex_properties.degree_centrality
def vp_name(self):
return self.g.vertex_properties.name
def vp_id(self):
return self.g.vertex_properties.id
def ep_weight(self):
return self.g.edge_properties.weight
# Calcula as características básicas da rede
def get_basic_info(self):
info = {}
try:
n_vertices = self.g.num_vertices()
n_edges = self.g.num_edges()
density = n_edges / ((n_vertices * (n_vertices - 1)) / 2)
mean_degree = (2 * n_edges) / n_vertices
# Cálculo do coeficiente de clusterização "na mão", usando a média dos
# coeficientes locais calculados pela Graph Tools
local_cc = local_clustering(self.g)
clustering_coef = fsum(
[local_cc[x] for x in self.g.vertices() if local_cc[x] != 0.0])
clustering_coef /= n_vertices
info["Número de times"] = n_vertices
info["Número de confrontos"] = n_edges
info["Densidade"] = density
info["Grau médio"] = mean_degree
info["Coeficiente de Clusterização"] = clustering_coef
except:
info.clear()
return info
def degree_centrality(self):
degree_centrality = self.g.new_vertex_property('float')
for v in self.g.vertices():
degree_centrality[v] = v.in_degree() / (self.g.num_vertices() - 1)
return degree_centrality
# Calcula a distribuição de graus da rede
def degree_distribution(self):
degree_dist = {}
try:
for v in self.g.vertices():
if v.in_degree() not in degree_dist.keys():
degree_dist[v.in_degree()] = 1
else:
degree_dist[v.in_degree()] += 1
for k in degree_dist.keys():
degree_dist[k] /= self.g.num_vertices()
except:
degree_dist.clear()
return degree_dist
class ResourceGraph(object):
v3_color = '#1C366B'
v2_color = '#1DACE8'
special_color = '#C4CFD0'
def __init__(self):
self.graph = Graph()
self.v_names = self.graph.new_vertex_property("string")
self.v_colors = self.graph.new_vertex_property("string")
self.e_names = self.graph.new_edge_property("string")
self.vertices = {}
self.edges = {}
def add_resource(self, resource_name, is_v3=True, is_special=False):
vertex = self.graph.add_vertex()
self.vertices[resource_name] = vertex
self.v_names[vertex] = resource_name
self.v_colors[vertex] = self.get_color(is_v3, is_special)
def get_color(self, is_v3, is_special):
if is_special:
return self.__class__.special_color
elif is_v3:
return self.__class__.v3_color
else:
return self.__class__.v2_color
def has_resource(self, resource_name):
return resource_name in self.vertices
def add_link(self, source_name, destination_name, name):
source = self.vertices[source_name]
destination = self.vertices[destination_name]
edge = self.graph.add_edge(source, destination)
edge_id = self.edge_id(source_name, destination_name, name)
self.edges[edge_id] = edge
self.e_names[edge] = name
def has_link(self, source_name, destination_name, name):
return self.edge_id(source_name, destination_name, name) in self.edges
def edge_id(self, source_name, destination_name, name):
return f'{name} -- {source_name} -- {destination_name}'
def draw(self):
graph_draw(
self.graph,
pos=radial_tree_layout(self.graph, self.graph.vertex(0)),
vertex_text=self.v_names,
vertex_fill_color=self.v_colors,
edge_text=self.e_names,
output_size=(2000, 1300),
fit_view=True,
vertex_font_size=10,
vertex_pen_width=1,
vertex_halo=False,
edge_pen_width=3,
)
def phylomemetic_graph(steps,
communities,
min_size=3,
max_size=50,
parent_limit=2,
workers='auto',
chunksize='auto',
method='fast',
min_backwards_containment=0,
min_forward_containment=0):
'''phylomemetic_graph
Parameters
----------
steps : :obj:`iter` of :obj:`int`
communities : :obj:`iter` of :obj:`iter` of :obj:`int`
min_size : :obj:`int`
max_size : :obj:`int`
parent_limit : :obj:`int`
workers : :obj:`int`
chunksize : :obj:`int`
method : :obj:`str`
min_backwards_containment : :obj:`float`
min_forward_containment : :obj:`float`
Returns
-------
g : :obj:`graph_too.Graph`
group_link_strength : :obj:`graph_tool.EdgePropertyMap`
single_link_strength : :obj:`graph_tool.EdgePropertyMap`
vertex_steps : :obj:`graph_tool.VertexPropertyMap`
element_vertex_map : :obj:`dict`
'''
if workers == 'auto':
workers = cpu_count() - 1
communities_filt = []
communities_lengths = []
element_community_mappings = []
for sequences in communities:
s_filt = list(filter_by_size(sequences, min_size, max_size))
communities_filt.append(s_filt)
communities_lengths.append(len(s_filt))
element_community_mappings.append(reverse_index(s_filt))
community_vertex_maps = []
communities_offsets = []
cumsum_lengths = np.cumsum(communities_lengths)
for length, count in zip(communities_lengths, cumsum_lengths):
start = count - length
end = count
communities_offsets.append((start, end))
community_vertex_maps.append(
{c: v
for c, v in zip(range(length), range(start, end))})
n_communities = np.sum(communities_lengths)
phylomemetic_links = []
for i, (cps, cfs) in enumerate(window(communities_filt, 2)):
n_cf = len(cfs)
logger.info(f'Processing {i+1} of {len(communities)-1} periods')
if chunksize == 'auto':
chunksize_i = int(np.ceil((1 / workers) * n_cf))
else:
chunksize_i = chunksize
with Pool(workers) as pool:
phylomemetic_links.append(
pool.map(
find_links,
zip(
cfs,
range(0, len(cfs)),
repeat(cps, n_cf),
repeat(communities_offsets[i], n_cf),
repeat(element_community_mappings[i], n_cf),
repeat(parent_limit, n_cf),
),
chunksize=chunksize_i,
))
pool.close()
pool.join()
g = Graph(directed=True)
g.add_vertex(n_communities)
group_link_strength = g.new_edge_property('float')
single_link_strength = g.new_edge_property('float')
phylomemetic_links = flatten(flatten(phylomemetic_links))
g.add_edge_list(phylomemetic_links,
eprops=[group_link_strength, single_link_strength])
element_vertex_map = reverse_index_communities(flatten(communities_filt))
vertex_steps = g.new_vertex_property('int')
for (start, end), step in zip(communities_offsets, steps):
vertex_steps.a[start:end] = step
return (g, group_link_strength, single_link_strength, vertex_steps,
element_vertex_map)
filename = 'buf.txt'
print(filename)
coefficient = 3
word_dict = {}
add_dict = {}
f = open('bad.txt', 'r', encoding="utf-8")
for s in f:
# print(s.split(' ')[0])
add_dict[s.split(' ')[0]] = 1
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("int")
ver_names = pairs_graph.new_vertex_property("string")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 in add_dict or w2 in add_dict:
continue
class BiblioNetwork():
"Bibliography network displayer"
def __init__(self, filepath):
self.filepath = filepath
self.db = None
self._auth_betw = None
self._auth_betw_computed_from = 0
self.layout_pos = None
self.graph = None
self.author_list = []
@staticmethod
def _split_authors(row):
"Split authors of the row"
auth = row['Authors'].split(", ")
auth = [", ".join(auth[2*i:2*i+2])
for i in range(int(len(auth)/2))]
return auth
def parse(self, nmb_to_import=None, delimiter=","):
"Parse the database csv file"
# import database
self.db = pd.read_csv(self.filepath, delimiter, index_col=False,
nrows=nmb_to_import, encoding="ISO8859",
error_bad_lines=False, warn_bad_lines=True)
self.db.reset_index()
# separate authors
self.db['Authors'] = self.db.apply(self._split_authors, axis=1)
# Replace missing values
self.db['Cited by'].fillna(0, inplace=True)
# Updat author list
self.update_author_list()
def clean(self, min_citations=10):
"Remove some entries"
len_bef = len(self.db)
self.db.drop(self.db[self.db["Cited by"] < min_citations].index,
inplace=True)
len_after = len(self.db)
print(" Removed {} articles, {} remaining".format(len_bef-len_after,
len_after))
self.update_author_list()
self._auth_betw = None
def remove_anterior(self, year):
"Remove some entries"
len_bef = len(self.db)
self.db.drop(self.db[self.db["Year"] <= year].index,
inplace=True)
len_after = len(self.db)
print(" Removed {} articles, {} remaining".format(len_bef-len_after,
len_after))
self.update_author_list()
self._auth_betw = None
def remove_posterior(self, year):
"Remove some entries"
len_bef = len(self.db)
self.db.drop(self.db[self.db["Year"] > year].index,
inplace=True)
len_after = len(self.db)
print(" Removed {} articles, {} remaining".format(len_bef-len_after,
len_after))
self.update_author_list()
self._auth_betw = None
def update_author_list(self):
"Update author list from database"
auths = list(set(np.concatenate(self.db['Authors'].values)))
self.author_list = np.sort(auths)
@property
def author_betweeness(self):
"Compute authors betweness"
# If already computed, just return it
if self._auth_betw is not None and \
self._auth_betw_computed_from == len(self.db):
return self._auth_betw
# else compute it
self._auth_betw_computed_from = len(self.db)
auth_betw = {auth: {}
for auth in self.author_list}
for auths in self.db['Authors']:
# skip if only one author
if len(auths) == 1:
continue
# Loop on authors couples
for i1, auth1 in enumerate(auths):
for auth2 in auths[i1+1::]:
keys = auth_betw.keys()
# create couple if necessary, or increment
if auth2 not in auth_betw[auth1].keys():
auth_betw[auth1][auth2] = 1
else:
auth_betw[auth1][auth2] += 1
if auth1 not in auth_betw[auth2].keys():
auth_betw[auth2][auth1] = 1
else:
auth_betw[auth2][auth1] += 1
self._auth_betw = auth_betw
return self._auth_betw
@author_betweeness.setter
def author_betweeness(self, val):
raise Exception("You cannot change that")
def get_total_citation(self):
""" Return total number of citations for each author"""
nmbcits = {}
for _, art in self.db.iterrows():
auths = art['Authors']
nmbcit = int(art['Cited by'])
for auth in auths:
if auth in nmbcits.keys():
nmbcits[auth] += nmbcit
else:
nmbcits[auth] = nmbcit
return nmbcits
def get_auth_nmb_of_art(self):
""" Return number of article for each author"""
nmbart = {}
for _, art in self.db.iterrows():
auths = art['Authors']
for auth in auths:
if auth in nmbart.keys():
nmbart[auth] += 1
else:
nmbart[auth] = 1
return nmbart
def _get_author_publication(self):
auth2pub = {}
for _, art in self.db.iterrows():
for auth in art['Authors']:
if auth in auth2pub.keys():
auth2pub[auth] += [art.name]
else:
auth2pub[auth] = [art.name]
return auth2pub
def write_author_list(self, filepath):
with open(filepath, "w") as f:
data = ['{}: {}\n'.format(i, auth)
for i, auth in enumerate(self.author_list)]
f.writelines(data)
def make_article_graph(self, layout="arf"):
"""Make an article graph"""
self.graph = Graph(directed=False)
# add vertex
self.graph.add_vertex(len(self.db))
# add properties
cb = self.graph.new_vertex_property("int", self.db['Cited by'].values)
self.graph.vertex_properties['nmb_citation'] = cb
# Add links
auths = list(self.author_betweeness.keys())
auth2ind = {auths[i]: i
for i in range(len(auths))}
auth2pub = self._get_author_publication()
for _, pubs in auth2pub.items():
if len(pubs) < 2:
continue
combis = itertools.combinations(pubs, 2)
self.graph.add_edge_list(list(combis))
# layout
if layout == "arf":
self.layout_pos = arf_layout(self.graph)
elif layout == "sfpd":
self.layout_pos = sfdp_layout(self.graph)
elif layout == "fr":
self.layout_pos = fruchterman_reingold_layout(self.graph)
elif layout == "radial":
self.layout_pos = radial_tree_layout(self.graph,
auth2ind['Logan, B.E.'])
else:
raise ValueError()
def make_author_graph(self, layout="arf"):
"""Make an author graph"""
self.graph = Graph(directed=False)
# add vertex
auths = self.author_list
self.graph.add_vertex(len(auths))
# add links
auth2ind = {auths[i]: i
for i in range(len(auths))}
abet = []
authbet = copy.deepcopy(self.author_betweeness)
for auth in auths:
for col, weight in authbet[auth].items():
if col == auth:
continue
self.graph.add_edge(auth2ind[auth], auth2ind[col])
del authbet[col][auth] # ensure that edges are not doubled
abet.append(weight)
# add properties
cb = self.graph.new_edge_property("int", abet)
self.graph.edge_properties['weight'] = cb
# layout
if layout == "arf":
self.layout_pos = arf_layout(self.graph,
weight=self.graph.ep.weight,
pos=self.layout_pos,
max_iter=10000)
elif layout == "sfpd":
self.layout_pos = sfdp_layout(self.graph,
eweight=self.graph.ep.weight,
pos=self.layout_pos)
elif layout == "fr":
self.layout_pos = fruchterman_reingold_layout(self.graph,
weight=self.graph.ep.weight,
circular=True,
pos=self.layout_pos)
elif layout == "radial":
nc = self.get_total_citation()
main_auth_ind = np.argmax(list(nc.values()))
main_auth = list(nc.keys())[main_auth_ind]
self.layout_pos = radial_tree_layout(self.graph,
auth2ind[main_auth])
elif layout == "planar":
self.layout_pos = planar_layout(self.graph)
else:
raise ValueError()
def display_article_graph(self, out="graph.pdf", min_size=1,
max_size=10, indice=False):
"""Display an article graph
One point per article.
Size and color corespond to the number of citation.
"""
cb = np.log(np.array(self.graph.vp.nmb_citation.a)+2)
ms = cb/max(cb)*(max_size - min_size) + min_size
ms = self.graph.new_vertex_property('float', ms)
graph_draw(self.graph, pos=self.layout_pos, output=out,
vertex_size=ms,
vertex_fill_color=self.graph.vp.nmb_citation,
vcmap=plt.cm.viridis)
def display_author_graph(self, out="graph.pdf", min_size=1, max_size=10,
indice=False):
"""Display an author graph """
auths = self.author_list
nc = self.get_total_citation()
nc = [int(nc[auth]) for auth in auths]
na = self.get_auth_nmb_of_art()
na = [int(na[auth]) for auth in auths]
# normalize citation number
nc = np.array(nc, dtype=float)
nc /= np.max(nc)
nc *= (max_size - min_size)
nc += min_size
# normalize edge width
weight = np.array(self.graph.ep.weight.a, dtype=float)
weight /= np.max(weight)
weight *= (1 - 0.1)
weight += 0.1
# Get vertex display order
vorder = np.argsort(nc)
# Get index
if indice:
text = range(len(vorder))
textg = self.graph.new_vertex_property('string', text)
else:
textg = None
# plot
ncg = self.graph.new_vertex_property('float', nc)
nag = self.graph.new_vertex_property('int', na)
vorderg = self.graph.new_vertex_property('int', vorder)
weightg = self.graph.new_edge_property('float', weight)
self.graph.vp['nmb_citation'] = ncg
graph_draw(self.graph, pos=self.layout_pos, output=out,
vertex_fill_color=nag, vertex_size=ncg,
edge_pen_width=weightg, vertex_text=textg,
vorder=vorderg,
vertex_text_position=0,
vcmap=plt.cm.PuBu)
class Network:
def __init__(self):
self.g = Graph(directed=True)
self.player_id_to_vertex = {}
self.pairs = {} # player pair: edge
# property maps for additional information
self.g.vertex_properties['player_id'] = self.g.new_vertex_property(
"string")
self.g.vertex_properties['player_coords'] = self.g.new_vertex_property(
"vector<float>")
self.g.vertex_properties[
'average_player_coords'] = self.g.new_vertex_property(
"vector<float>")
self.g.vertex_properties[
'player_n_coords'] = self.g.new_vertex_property("int")
self.g.edge_properties['weight'] = self.g.new_edge_property("float")
@property
def edge_weights(self):
return self.g.edge_properties['weight']
@property
def player_id_pmap(self):
return self.g.vertex_properties['player_id']
@property
def player_coords_pmap(self):
return self.g.vertex_properties['player_coords']
@property
def player_n_coords_pmap(self):
return self.g.vertex_properties['player_n_coords']
@property
def average_player_coords_pmap(self):
# lazy evaluation of means
for v in self.g.vertices():
self.g.vertex_properties['average_player_coords'][v] = np.asarray(
self.player_coords_pmap[v]) / self.player_n_coords_pmap[v]
return self.g.vertex_properties['average_player_coords']
def add_players(self, pids: List[str]):
n = len(pids)
vs = list(self.g.add_vertex(n))
self.player_id_to_vertex.update({pids[i]: vs[i] for i in range(n)})
for i in range(n):
self.player_id_pmap[vs[i]] = pids[i]
return vs
def add_passes(self,
id_pairs: List[Tuple],
coords_pairs: List[Tuple],
pass_scores=None):
pairs = [(self.player_id_to_vertex[i1], self.player_id_to_vertex[i2])
for i1, i2 in id_pairs]
# append player coordinates
n = len(coords_pairs)
if pass_scores is None:
pass_scores = [1 for _ in range(n)]
for i in range(n):
# remember orig and dest location
# orig player
coords = self.player_coords_pmap[pairs[i][0]]
if len(coords) == 0:
coords = np.asarray([coords_pairs[i][0], coords_pairs[i][1]])
else:
# accumulate
coords += np.asarray([coords_pairs[i][0], coords_pairs[i][1]])
self.player_coords_pmap[pairs[i][0]] = coords
self.player_n_coords_pmap[pairs[i][0]] += 1
# dest player
coords = self.player_coords_pmap[pairs[i][1]]
if len(coords) == 0:
coords = np.asarray([coords_pairs[i][2], coords_pairs[i][3]])
else:
# accumulate
coords += np.asarray([coords_pairs[i][2], coords_pairs[i][3]])
self.player_coords_pmap[pairs[i][1]] = coords
self.player_n_coords_pmap[pairs[i][1]] += 1
# if the edge exists, increment its weight instead of creating a new edge
e = self.pairs.get(pairs[i])
if e is not None:
self.edge_weights[e] += pass_scores[i]
else:
e = self.g.add_edge(*pairs[i])
self.pairs[pairs[i]] = e
self.edge_weights[e] = pass_scores[i]
def cleanup(self):
"""remove isolated vertices"""
to_remove = []
for v in self.g.vertices():
if v.in_degree() + v.out_degree() == 0:
to_remove.append(v)
n = len(to_remove)
self.g.remove_vertex(to_remove, fast=True)
print("Removed {0} isolated vertices".format(n))
def save(self, file: str):
self.g.save(file, fmt='graphml')
class SentenceGraph():
def __init__(self, sentence, directed=False, graph=None):
# Create a SentenceGraph from an existing graph tool graph
if graph is not None:
self.sentence_graph = graph
return
# Create a new SentenceGraph from scratch
self.sentence_graph = Graph(directed=directed)
# Graph properties
sentence_property = self.sentence_graph.new_graph_property("string", sentence)
self.sentence_graph.graph_properties[SENTENCE_KEY] = sentence_property
# Vertex properties
word_property = self.sentence_graph.new_vertex_property("string")
part_of_speech_property = self.sentence_graph.new_vertex_property("string")
vertex_color_property = self.sentence_graph.new_vertex_property("vector<double>")
self.sentence_graph.vertex_properties[WORD_KEY] = word_property
self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY] = part_of_speech_property
self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY] = vertex_color_property
# Edge properties
sentence_edge_property = self.sentence_graph.new_edge_property("string")
definition_edge_property = self.sentence_graph.new_edge_property("string")
parsed_dependencies_edge_property = self.sentence_graph.new_edge_property("string")
inter_sentence_edge_property = self.sentence_graph.new_edge_property("string")
edge_color_property = self.sentence_graph.new_edge_property("vector<double>")
dependency_edge_property = self.sentence_graph.new_edge_property("string")
self.sentence_graph.edge_properties[SENTENCE_EDGE_KEY] = sentence_edge_property
self.sentence_graph.edge_properties[DEFINITION_EDGE_KEY] = definition_edge_property
self.sentence_graph.edge_properties[PARSED_DEPENDENCIES_EDGE_KEY] = parsed_dependencies_edge_property
self.sentence_graph.edge_properties[INTER_SENTENCE_EDGE_KEY] = inter_sentence_edge_property
self.sentence_graph.edge_properties[EDGE_COLOR_KEY] = edge_color_property
self.sentence_graph.edge_properties[PARSE_TREE_DEPENDENCY_VALUE_KEY] = dependency_edge_property
# Edge filter properties
definition_edge_filter_property = self.sentence_graph.new_edge_property("bool")
inter_sentence_edge_filter_property = self.sentence_graph.new_edge_property("bool")
parsed_dependencies_edge_filter_property = self.sentence_graph.new_edge_property("bool")
sentence_edge_filter_property = self.sentence_graph.new_edge_property("bool")
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY] = definition_edge_filter_property
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY] = inter_sentence_edge_filter_property
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY] = parsed_dependencies_edge_filter_property
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY] = sentence_edge_filter_property
def get_sentence(self):
return self.sentence_graph.graph_properties[SENTENCE_KEY]
def add_vertex(self, word, pos):
word_pos_tuple = (word, pos)
# Create vertex, set properties
word_vertex = self.sentence_graph.add_vertex()
self.sentence_graph.vertex_properties[WORD_KEY][word_vertex] = word
self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][word_vertex] = pos
self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY][word_vertex] = [0, 0, 1, 1]
return word_vertex
def set_vertex_color_from_word(self, word, pos, color=[1, 0, 0, 1]):
word_vertex = self.get_vertex(word, pos)
return self.set_vertex_color(word_vertex, color)
def set_vertex_color(self, vertex, color=[1, 0, 0, 1]):
self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY][vertex] = color
def set_vertices_color(self, vertices, color=[1, 0, 0, 1]):
for vertex in vertices:
self.set_vertex_color(vertex, color)
def add_sentence_edge_from_words(self, word1, pos1, word2, pos2):
return self.add_sentence_edge(self.get_vertex(word1, pos1), self.get_vertex(word2, pos2))
def add_sentence_edge(self, word_vertex1, word_vertex2):
sentence_edge = self.sentence_graph.add_edge(word_vertex1, word_vertex2, add_missing=False)
self.sentence_graph.edge_properties[SENTENCE_EDGE_KEY][sentence_edge] = sentence_edge
# Green
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][sentence_edge] = [0.2, 1, 0.2, 1]
self._set_edge_to_zero_in_all_filters(sentence_edge)
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY][sentence_edge] = True
return sentence_edge
def add_sentence_edges(self, sentence_vertices):
for i in range(1, len(sentence_vertices)):
self.add_sentence_edge(sentence_vertices[i - 1], sentence_vertices[i])
def add_parsed_dependency_edge(self, word_vertex1, word_vertex2, dependency_relationship):
parsed_dependency_edge = self.sentence_graph.add_edge(word_vertex1, word_vertex2, add_missing=False)
self.sentence_graph.edge_properties[PARSED_DEPENDENCIES_EDGE_KEY][parsed_dependency_edge] = parsed_dependency_edge
self.sentence_graph.edge_properties[PARSE_TREE_DEPENDENCY_VALUE_KEY][parsed_dependency_edge] = dependency_relationship
# Blue
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][parsed_dependency_edge] = [0, 0, 1, 1]
self._set_edge_to_zero_in_all_filters(parsed_dependency_edge)
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY][parsed_dependency_edge] = True
return parsed_dependency_edge
def add_parsed_dependency_edge_from_words(self, word1, pos1, word2, pos2, dependency_relationship):
return self.add_parsed_dependency_edge(
self.get_vertex(word1, pos1),
self.get_vertex(word2, pos2),
dependency_relationship)
def add_definition_edge_from_words(self, word, pos, definition_word, definition_pos):
return self.add_definition_edge(
self.get_vertex(word, pos),
self.get_vertex(definition_word, definition_pos))
def _set_edge_to_zero_in_all_filters(self, edge):
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY][edge] = False
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY][edge] = False
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY][edge] = False
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY][edge] = False
def add_definition_edge(self, word_vertex, definition_word_vertex):
definition_edge = self.sentence_graph.add_edge(word_vertex, definition_word_vertex, add_missing=False)
self.sentence_graph.edge_properties[DEFINITION_EDGE_KEY][definition_edge] = definition_edge
# Red
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][definition_edge] = [1, 0.1, 0.1, 1]
self._set_edge_to_zero_in_all_filters(definition_edge)
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY][definition_edge] = True
return definition_edge
def add_definition_edges(self, word_vertex, definition_word_vertices):
# Add edges from the word_vertex to all definition vertices and set
# the definition edge property on each edge
for definition_word_vertex in definition_word_vertices:
self.add_definition_edge(word_vertex, definition_word_vertex)
return self
def add_inter_sentence_edge(self, sentence1_word_vertex, sentence2_word_vertex):
inter_sentence_edge = self.sentence_graph.add_edge(sentence1_word_vertex, sentence2_word_vertex, add_missing=False)
self.sentence_graph.edge_properties[INTER_SENTENCE_EDGE_KEY][inter_sentence_edge] = inter_sentence_edge
# Pink
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][inter_sentence_edge] = [1, 0.05, 1, 1]
self._set_edge_to_zero_in_all_filters(inter_sentence_edge)
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY][inter_sentence_edge] = True
return inter_sentence_edge
def add_inter_sentence_edge_from_words(self, word1, pos1, word2, pos2):
return self.add_inter_sentence_edge(
self.get_vertex(word1, pos1),
self.get_vertex(word2, pos2))
def remove_vertex_by_word(self, word, pos):
self.remove_vertex(self.get_vertex(word, pos))
def remove_vertex(self, vertex):
word = self.sentence_graph.vertex_properties[WORD_KEY][vertex]
pos = self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][vertex]
self.sentence_graph.remove_vertex(vertex)
def remove_edge(self, word1, pos1, word2, pos2):
self.sentence_graph.remove_edge(self.get_edge(word1, pos1, word2, pos2))
def contains(self, word, pos):
return self.get_vertex(word, pos) is not None
def get_vertex(self, word, pos):
for vertex in self.sentence_graph.vertices():
try:
vertex_word = self.sentence_graph.vertex_properties[WORD_KEY][vertex]
vertex_pos = self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][vertex]
if vertex_word == word and vertex_pos == pos:
return vertex
except:
pass
return None
def get_word_pos_tuple(self, vertex):
return self.sentence_graph.vertex_properties[WORD_KEY][vertex],\
self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][vertex]
def get_word_pos_tuple_by_index(self, index):
return self.get_word_pos_tuple(self.get_vertex_by_index(index))
def get_vertex_by_index(self, index):
return self.sentence_graph.vertex(index)
def get_vertices_iterator(self):
return self.sentence_graph.vertices()
def get_vertices(self):
return [x for x in self.sentence_graph.vertices()]
def get_vertex_out_neighbor_word_pos_tuples(self, vertex):
return [self.get_word_pos_tuple(neighbor_vertex)
for neighbor_vertex in self.get_vertex_out_neighbors(vertex)]
def get_vertex_in_neighbor_word_pos_tuples(self, vertex):
return [self.get_word_pos_tuple(neighbor_vertex)
for neighbor_vertex in self.get_vertex_in_neighbors(vertex)]
def get_vertex_out_neighbors(self, vertex):
return [neighbor_vertex for neighbor_vertex in vertex.out_neighbours()]
def get_vertex_in_neighbors(self, vertex):
return [neighbor_vertex for neighbor_vertex in vertex.in_neighbours()]
def get_word_pos_tuples(self):
return [self.get_word_pos_tuple(v) for v in self.sentence_graph.vertices()]
def get_num_vertices(self):
return self.sentence_graph.num_vertices()
def get_num_edges(self):
return self.sentence_graph.num_edges()
def get_edge(self, word1, pos1, word2, pos2):
vertex_1 = self.get_vertex(word1, pos1)
vertex_2 = self.get_vertex(word2, pos2)
return None\
if vertex_1 is None or vertex_2 is None\
else self.sentence_graph.edge(vertex_1, vertex_2)
def get_edges_iterator(self):
return self.sentence_graph.edges()
def get_edges(self):
return [x for x in self.sentence_graph.edges()]
def set_definition_edge_filter(self, inverted=False):
self.sentence_graph.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY],
inverted=inverted)
def set_inter_sentence_edge_filter(self, inverted=False):
self.sentence_graph.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY],
inverted=inverted)
def set_parsed_dependency_edge_filter(self, inverted=False):
self.sentence_edge.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY],
inverted=inverted)
def set_sentence_edge_filter(self, inverted=False):
self.sentence_graph.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY],
inverted=inverted)
def clear_filters(self):
self.sentence_graph.clear_filters()
def get_definition_edges(self):
return filter(lambda x: x in self.get_definition_edge_properties(), self.get_edges())
def get_word_vertex_properties(self):
return self.sentence_graph.vertex_properties[WORD_KEY]
def get_pos_vertex_properties(self):
return self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY]
def get_color_vertex_properties(self):
return self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY]
def get_sentence_edge_properties(self):
return self.sentence_graph.edge_properties[SENTENCE_EDGE_KEY]
def get_definition_edge_properties(self):
return self.sentence_graph.edge_properties[DEFINITION_EDGE_KEY]
def get_inter_sentence_edge_properties(self):
return self.sentence_graph.edge_properties[INTER_SENTENCE_EDGE_KEY]
def get_color_edge_properties(self):
return self.sentence_graph.edge_properties[EDGE_COLOR_KEY]
def get_vertex_index(self, vertex):
return self.sentence_graph.vertex_index[vertex]
def get_degree_properties(self, degree_type):
return self.sentence_graph.degree_property_map(degree_type)
def get_graph(self):
return self.sentence_graph
def copy(self):
return SentenceGraph(
sentence=self.sentence_graph.graph_properties[SENTENCE_KEY],
graph=self.sentence_graph.copy())
def plot_networks(stationary_state,
free_energies,
transition_matrix,
cvs,
save=True,
outputpath="/home/oliverfl/Pictures/network-%s.svg",
description="Transition Network",
simu_id=None):
"""
Do imports only when necessarry to avoid errors with GTK symbols backends
See the error message I get on https://stackoverflow.com/questions/19773190/graph-tool-pyside-gtk-2-x-and-gtk-3-x
"""
# import gtk
import graph_tool as gt
from graph_tool.all import graph_draw, Graph
# from pylab import * # for plotting
if len(stationary_state.shape) == 2:
stationary_state = np.mean(stationary_state, axis=0)
free_energy, boxplot_data = _fix_field_and_data(free_energies, 1000)
# Create graph and labels
graph = Graph(directed=True)
vertices = []
vertex_sizes = graph.new_vertex_property("float")
vertex_labels = graph.new_vertex_property("string")
edge_sizes = graph.new_edge_property("float")
edge_labels = graph.new_edge_property("string")
cluster_to_label = {
0: "M", # "#"Intermediate",
1: "I", # "Inactive",
2: "A" # "Active"
}
#####CREATE VERTICES###########
for i, rho in enumerate(stationary_state):
# print(rho)
v = graph.add_vertex()
vertices.append(v)
vsize = (1 + 1 * rho / max(stationary_state)
) * 50 # np.log(1 + rho / max(stationary_state)) * 100
vertex_sizes[v] = vsize
# Beware that long tables can make the nodes expand to fit the text (and thus override vsize)
vertex_labels[v] = cluster_to_label.get(i) + " ({0:0.01f})".format(
free_energy[i])
####CREATE EDGES##########
max_transition_value = (
transition_matrix - np.diag(transition_matrix.diagonal())
).max() # Max value of matrix excluding diagonal elements
for i, row in enumerate(transition_matrix):
total_traj_count = sum(row)
for j, rhoij in enumerate(row):
if rhoij > 0 and i != j:
e = graph.add_edge(
vertices[i], vertices[j]
) # , weight=rhoij, label="{}->{}".format(i, j))
edge_labels[e] = "{}/{}".format(int(rhoij),
int(total_traj_count))
# The edge width is proportional to the relative number of transition from this starting state
size = (1 + 10 * rhoij / total_traj_count) * 3
edge_sizes[e] = size
# Using matplotlib for rendering if save==False
plt.figure(figsize=(10, 10))
graph_draw(
graph,
pos=gt.draw.sfdp_layout(graph),
# output_size=(400, 400),
output=outputpath % simu_id if save else None,
# inline=True,
mplfig=plt.gcf() if not save else None,
vertex_text=vertex_labels,
vertex_font_size=10,
vertex_size=vertex_sizes,
edge_text=edge_labels,
edge_pen_width=edge_sizes)
if not save:
plt.title(description)
plt.xticks([], [])
plt.yticks([], [])
plt.show()
class Workflow:
def __init__(self, edges, weights):
self.edges = edges
self.graph = Graph()
self.size = len(edges['target'])
self.graph.add_vertex(self.size)
self.weights = weights
# init weights part
self.graph.vp.weights = self.graph.new_vertex_property('int16_t')
for index in range(0, self.size):
self.graph.vp.weights[index] = weights[index]
for source in self.edges['source'].keys():
for target in self.edges['source'][source]:
self._add_edge(source, target)
self.depth_per_node = {x: 0 for x in range(0, self.size)}
self.accum_weights = {x: 0 for x in range(0, self.size)}
self.find_depth()
self.find_accum_weights(self.size - 1)
self.depth = {x: [] for x in set(self.depth_per_node.values())}
for node, depth in self.depth_per_node.items():
self.depth[depth].append(node)
self.routes_t = {}
self.find_routes(self.size - 1, 0, self.routes_t)
self.routes = []
self.transpose_routes(self.size - 1, self.routes_t[self.size - 1])
def _add_edge(self, source, target):
self.graph.add_edge(self.graph.vertex(source),
self.graph.vertex(target))
def show(self, size=1500):
return graph_draw(self.graph,
vertex_text=self.graph.vertex_index,
vertex_font_size=18,
output_size=(size, size),
output="graph.png")
def find_accum_weights(self, actual_node, accum_weight=0):
already_accum_weight = self.accum_weights[actual_node]
self.accum_weights[actual_node] = max(
already_accum_weight, accum_weight + self.weights[actual_node])
for fathers in self.edges['target'][actual_node]:
self.find_accum_weights(fathers, self.accum_weights[actual_node])
def find_depth(self, actual_node=0, actual_depth=0):
self.depth_per_node[actual_node] = max(
self.depth_per_node[actual_node], actual_depth)
for next_node in self.edges['source'][actual_node]:
self.find_depth(next_node, actual_depth + 1)
def find_routes(self, actual_node, weight=0, routes={}):
weight += self.weights[actual_node]
if actual_node != 0:
routes[actual_node] = {}
for fathers in self.edges['target'][actual_node]:
self.find_routes(fathers, weight, routes[actual_node])
else:
routes[actual_node] = weight
def transpose_routes(self, actual_node, routes, path=[]):
if actual_node != 0:
path = path.copy()
path.append(actual_node)
for child in routes.keys():
self.transpose_routes(child, routes[child], path)
else:
self.routes.append({'path': path, 'weight': routes})
def find_cycles(self):
visited = [False for _ in range(0, self.size)]
return self.find_cycles_helper(0, [])
def find_cycles_helper(self, actual_node, rec_list):
if actual_node in rec_list:
print(rec_list, actual_node)
return True
call_this = []
for child in self.edges['source'][actual_node]:
new_rec_list = rec_list.copy()
new_rec_list.append(actual_node)
call_this.append([child, new_rec_list])
return any([self.find_cycles_helper(x[0], x[1]) for x in call_this])
def gen_graph((repo, events)):
graph = Graph()
repo_on_graph = graph.new_graph_property('string')
repo_on_graph[graph] = repo
graph.graph_properties['repo_on_graph'] = repo_on_graph
language_on_graph = graph.new_graph_property('string')
language_on_graph[graph] = events[0]['language']
graph.graph_properties['language_on_graph'] = language_on_graph
events_on_vertices = graph.new_vertex_property('object')
graph.vertex_properties['events_on_vertices'] = events_on_vertices
actors_on_vertices = graph.new_vertex_property('string')
graph.vertex_properties['actors_on_vertices'] = actors_on_vertices
weights_on_edges = graph.new_edge_property('long double')
graph.edge_properties['weights_on_edges'] = weights_on_edges
# pre_vertices = []
pre_events_map = {}
pre_vertices_map = {}
owner_vertex = graph.add_vertex()
owner = repo.split('/')[0]
dummy_event = {'created_at': events[0]['repo-created_at']}
actors_on_vertices[owner_vertex] = owner
events_on_vertices[owner_vertex] = dummy_event
pre_vertices_map[owner] = owner_vertex
pre_events_map[owner] = dummy_event
events = sorted(events, key=lambda x: x['created_at'])
for event in events:
actor = event['actor']
if actor in pre_events_map:
continue
created_at = event['created_at']
vertex = graph.add_vertex()
events_on_vertices[vertex] = event
actors_on_vertices[vertex] = actor
# if 'actor-following' not in event:
# continue
following = set(event['actor-following'])
commons = following.intersection(pre_vertices_map.keys())
# pre_vertices.append(vertex)
# if len(commons) == 0:
# edge = graph.add_edge(vertex, owner_vertex)
# weights_on_edges[edge] = 1.0
for pre_actor in commons:
interval =\
(created_at - pre_events_map[pre_actor]['created_at']).days
if interval < 0:
continue
edge = graph.add_edge(vertex, pre_vertices_map[pre_actor])
if pre_actor == owner:
weight = 1.0
else:
weight = 1.0 / fib(interval + 2)
weights_on_edges[edge] = weight
pre_events_map[actor] = event
pre_vertices_map[actor] = vertex
return graph
from graph_tool.all import Graph, graph_draw, fruchterman_reingold_layout
import json
network = json.load(open('../FRRNetwork.json', 'r'))
g = Graph(directed=False)
vprop_text = g.new_vertex_property("string")
vprop_color = g.new_vertex_property("int")
vprop_size = g.new_vertex_property("int")
vprop_shape = g.new_vertex_property("string")
name_to_vertex = {}
for switch in network['switches']:
v_switch = g.add_vertex()
name_to_vertex[switch['name']] = v_switch
vprop_text[v_switch] = switch['name']
vprop_color[v_switch] = 1
vprop_size[v_switch] = 50
vprop_shape[v_switch] = "hexagon"
for host in switch['hosts']:
v_host = g.add_vertex()
e_link = g.add_edge(v_switch, v_host)
vprop_text[v_host] = host['name']
vprop_color[v_host] = 100
vprop_size[v_host] = 40
vprop_shape[v_host] = "circle"
name_to_vertex[host['name']] = v_host
for link in network['switch_links']:
v_node1 = name_to_vertex[link['node1']['name']]
v_node2 = name_to_vertex[link['node2']['name']]
coefficient = 5.278
word_dict = {}
add_dict = {}
f = open('bad.txt', 'r', encoding="utf-8")
for s in f:
# print(s.split(' ')[0])
add_dict[s.split(' ')[0]] = 1
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("int")
pos_weights = pairs_graph.new_edge_property("int")
neg_weights = pairs_graph.new_edge_property("int")
ver_names = pairs_graph.new_vertex_property("string")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 in add_dict or w2 in add_dict:
continue
def main():
options = parse_arguments()
clusters_enabled = options["mapping_location"] is not None
if options["should_merge"] and not clusters_enabled:
raise "You need to provide a mapping to use merged view.`"
# Get the string containing the input matrix form a file/pipe
if options["matrix_location"] is not None:
with open(options["matrix_location"], 'r') as file:
matrix_string = file.read().strip().split("\n")
else:
matrix_string = sys.stdin
# Parse the input matrix string
height, width, matrix = parse_matrix(matrix_string, options["has_size"])
# Get the string containing the mapping if specified
if clusters_enabled:
with open(options["mapping_location"], 'r') as file:
mapping_string = file.read().strip()
mapping = parse_mapping(mapping_string, options["has_size"])
else:
mapping = None
if options["should_merge"]:
height, width, matrix, mapping = merge_clusters(matrix, mapping)
graph = Graph()
graph.add_vertex(height + width)
shape = graph.new_vertex_property("string")
color = graph.new_vertex_property("string")
index = graph.new_vertex_property("string")
for i in range(height):
v = graph.vertex(i)
shape[v] = "square"
color[v] = "red"
index[v] = str(i)
for i in range(width):
v = graph.vertex(height + i)
shape[v] = "circle"
if clusters_enabled:
color[v] = COLORS[mapping[i] % len(COLORS)]
else:
color[v] = COLORS[0]
index[v] = str(i)
for i in range(height):
for j in range(width):
if abs(matrix[i][j]) < EPSILON:
continue
graph.add_edge(graph.vertex(i), graph.vertex(height + j))
graph.set_directed(False)
if clusters_enabled:
groups = graph.new_vertex_property("int")
for i in range(width):
v = graph.vertex(height + i)
groups[v] = mapping[i]
position = sfdp_layout(graph, groups=groups)
else:
position = None
graph_draw(graph,
pos=position,
vertex_text=index,
vertex_shape=shape,
vertex_fill_color=color,
vertex_pen_width=1.2,
vertex_color="black",
edge_pen_width=3.4,
fit_view=True,
bg_color=(255, 255, 255, 1),
output=options["output_file"])
filename = 'buf.txt'
print(filename)
coefficient = 3
word_dict = {}
add_dict = {}
f = open('bad.txt', 'r', encoding="utf-8")
for s in f:
# print(s.split(' ')[0])
add_dict[s.split(' ')[0]] = 1
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("int")
ver_names = pairs_graph.new_vertex_property("string")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 in add_dict or w2 in add_dict:
continue
class TTC(AbstractMatchingAlgorithm):
"""This class searches for cycles where each student gets his best option.
This takes a list of students, a list of schools and a ruleset
(which is used to calculate priorities).
This works by generating a directed graph, where each student points
at at his best option, and each school points at the student (or students)
with the highest priority.
"""
EDGE_WIDTH_SIZE_FACTOR = 700
"""Size factor (in the image) of each edge that is not part of the main cycle."""
EDGE_WIDTH_CYCLE_SIZE = 10
"""Size factor (in the image) of each edge that takes part of the main cycle."""
def __init__(self,
generate_images=False,
images_folder="TTC_images",
use_longest_cycle=True):
"""Initializes the algorithm.
:param generate_images: If the process generates images or not.
:type generate_images: bool
:param images_folder: Where images are saved.
:type images_folder: str
:param use_longest_cycle: If the algorithm applies the longest cycle available, or the first one encountered.
:type use_longest_cycle: bool
"""
self.generate_images = generate_images
self.images_folder = images_folder
self.use_longest_cycle = use_longest_cycle
self.__graph = None
self.__vertices_by_school_id = None
self.__vertices_by_student_id = None
self.__students_by_id = None
self.__schools_by_id = None
self.__entity_id = None
self.__entity_type = None
def reset_variables(self):
"""Resets all variables."""
self.__graph = Graph()
self.__vertices_by_school_id = {}
self.__vertices_by_student_id = {}
self.__students_by_id = {}
self.__schools_by_id = {}
self.__entity_id = self.__graph.new_vertex_property("int")
self.__graph.vertex_properties["entity_id"] = self.__entity_id
self.__entity_type = self.__graph.new_vertex_property("string")
self.__graph.vertex_properties["entity_type"] = self.__entity_type
def run(self, students, schools, ruleset):
"""Runs the algorithm.
First it creates the graph, then it lists all the cycles available,
after that it selects one cycle, and applies it. Finally, it starts
the process again.
:param students: List of students.
:type students: list
:param schools: List of school.
:type schools: list
:param ruleset: Set of rules used.
:type ruleset: Ruleset
"""
self.reset_variables()
can_improve = True
iteration_counter = 1
while can_improve:
self.structure_graph(students, schools)
cycles = [c for c in all_circuits(self.__graph, unique=True)]
# print("CYCLES", cycles, "iteration", iteration_counter)
cycle_edges = []
if cycles:
for cycle in cycles: # ToDo: Possible optimisation: apply all disjoint cycles at once
for current_v_index in range(len(cycle)):
next_v_index = (current_v_index + 1) % len(cycle)
from_v = self.__graph.vertex(cycle[current_v_index])
target_v = self.__graph.vertex(cycle[next_v_index])
edge = self.__graph.edge(from_v, target_v)
cycle_edges.append(edge)
if self.__entity_type[from_v] == "st":
sel_student = self.__students_by_id[
self.__entity_id[from_v]]
sel_school = self.__schools_by_id[
self.__entity_id[target_v]]
sel_student.assigned_school = sel_school
sel_school.assignation.append(sel_student)
# vertex_school_target_id = self.__entity_id[target_v]
# vertex_school_target = self.__schools_by_id[vertex_school_target_id]
# print("CYCLE: Student", sel_student.id, "School", sel_school.id)
# print("VVV: School {} -> School {} (Student {}) ".format(self.__entity_id[from_v], self.__entity_id[target_v], self.__entity_id[self.__graph.edge(from_v, target_v)]))
if self.generate_images:
self.generate_image(cycle_edges,
iteration_n=iteration_counter)
else:
can_improve = False
self.__graph.clear()
iteration_counter += 1
def structure_graph(self, students, schools):
"""Creates a graph where students points to schools, and schools points to students.
In the graph, each student points at at his best option, and each school points
at the student (or students) with the highest priority.
:param students: List of students.
:type students: list
:param schools:
:type schools: list
"""
if not self.__students_by_id and not self.__schools_by_id:
for student in students:
self.__students_by_id[student.id] = student
for school in schools:
self.__schools_by_id[school.id] = school
for school in schools:
setattr(school, 'preferences',
StudentQueue(school, preference_mode=True))
remaining_students = [
student for student in students if not student.assigned_school
]
for student in remaining_students:
for pref_school in student.preferences:
pref_school.preferences.append(student)
for student in remaining_students:
v_source_student = self.create_vertex_student(student)
pref_school = next(
(school for school in student.preferences if
len(school.assignation.get_all_students()) < school.capacity),
None)
if pref_school:
v_target_school = self.create_vertex_school(pref_school)
self.create_edge(v_source_student, v_target_school)
for school in schools:
if len(school.assignation.get_all_students()) < school.capacity:
v_source_school = self.create_vertex_school(school)
pref_student = next(
iter(school.preferences.get_all_students()), None)
if pref_student:
v_target_student = self.create_vertex_student(pref_student)
self.create_edge(v_source_school, v_target_student)
# graph_draw(self.__graph,
# vertex_text=self.__entity_id, vertex_shape="circle",
# output_size=(1000, 1000), bg_color=[1., 1., 1., 1], output="graph.png")
def create_vertex_student(self, student):
"""Defines a new student as a vertex in the graph (if it did not existed before)."""
if student.id in self.__vertices_by_student_id:
vertex = self.__vertices_by_student_id[student.id]
else:
vertex = self.__graph.add_vertex()
self.__vertices_by_student_id[student.id] = vertex
self.__entity_id[vertex] = student.id
self.__entity_type[
vertex] = "st" # ToDo: There may be other ways to do this.
return vertex
def create_vertex_school(self, school):
"""Defines a new school as a vertex in the graph (if it did not existed before)."""
if school.id in self.__vertices_by_school_id:
vertex = self.__vertices_by_school_id[school.id]
else:
vertex = self.__graph.add_vertex()
self.__vertices_by_school_id[school.id] = vertex
self.__entity_id[vertex] = school.id
self.__entity_type[vertex] = "sc"
return vertex
def create_edge(self, source_v, target_v):
"""Creates a directed edge between two vertices."""
self.__graph.add_edge(source_v, target_v)
def generate_image(self, cycle_edges, iteration_n=0):
"""Generates an image of a graph.
:param cycle_edges: Edges which are part of the main cycle (they will be highlighted in red).
:type cycle_edges: list
:param iteration_n: Number of iteration of the algorithm (this is added in the filename of the image).
:type iteration_n: int
.. DANGER::
This is an experimental feature.
"""
edge_color = self.__graph.new_edge_property("vector<float>")
edge_width = self.__graph.new_edge_property("int")
for edge in self.__graph.edges():
if edge in cycle_edges:
edge_color[edge] = [1., 0.2, 0.2, 0.999]
edge_width[edge] = 7
else:
edge_color[edge] = [0., 0., 0., 0.3]
edge_width[edge] = 4
vertex_shape = self.__graph.new_vertex_property("string")
vertex_size = self.__graph.new_vertex_property("int")
for vertex in self.__graph.vertices():
if self.__entity_type[vertex] == "st":
vertex_shape[vertex] = "circle"
vertex_size[vertex] = 1
else:
vertex_shape[vertex] = "double_circle"
vertex_size[vertex] = 100
# pos = sfdp_layout(self.__graph, C=10, p=5, theta=2, gamma=1)
pos = arf_layout(self.__graph, d=0.2, a=3)
graph_draw(
self.__graph,
pos=pos,
vertex_text=self.__entity_id,
vertex_font_size=
1, # ToDo: Move image related code outside the class.
vertex_fill_color=[0.97, 0.97, 0.97, 1],
vertex_color=[0.05, 0.05, 0.05, 0.95],
vertex_shape=vertex_shape,
edge_color=edge_color,
edge_pen_width=edge_width,
output_size=(1000, 1000),
bg_color=[1., 1., 1., 1],
output=self.generate_filename(iteration_n))
def generate_filename(self, iteration_n): # ToDo: Move this to utils
"""Returns a filename (which is used to generate the images)."""
filename = "Graph (iteration {})".format(
iteration_n) if iteration_n > 0 else "Graph"
output_file = gen_filepath(self.images_folder,
filename=filename,
extension="png")
return output_file
def steiner_tree_mst(g,
root,
infection_times,
source,
terminals,
closure_builder=build_closure,
strictly_smaller=True,
return_closure=False,
k=-1,
debug=False,
verbose=True):
gc, eweight, r2pred = closure_builder(g,
root,
terminals,
infection_times,
strictly_smaller=strictly_smaller,
k=k,
debug=debug,
verbose=verbose)
# get the minimum spanning arborescence
# graph_tool does not provide minimum_spanning_arborescence
if verbose:
print('getting mst')
gx = gt2nx(gc, root, terminals, edge_attrs={'weight': eweight})
try:
nx_tree = nx.minimum_spanning_arborescence(gx, 'weight')
except nx.exception.NetworkXException:
if debug:
print('fail to find mst')
if return_closure:
return None, gc, None
else:
return None
if verbose:
print('returning tree')
mst_tree = Graph(directed=True)
for _ in range(g.num_vertices()):
mst_tree.add_vertex()
for u, v in nx_tree.edges():
mst_tree.add_edge(u, v)
if verbose:
print('extract edges from original graph')
# extract the edges from the original graph
# sort observations by time
# and also topological order
topological_index = {}
for i, e in enumerate(bfs_iterator(mst_tree, source=root)):
topological_index[int(e.target())] = i
sorted_obs = sorted(set(terminals) - {root},
key=lambda o:
(infection_times[o], topological_index[o]))
tree_nodes = {root}
tree_edges = set()
# print('root', root)
for u in sorted_obs:
if u in tree_nodes:
if debug:
print('{} covered already'.format(u))
continue
# print(u)
v, u = map(int, next(mst_tree.vertex(u).in_edges())) # v is ancestor
tree_nodes.add(v)
late_nodes = [
n for n in terminals if infection_times[n] > infection_times[u]
]
vis = init_visitor(g, u)
# from child to any tree node, including v
cpbfs_search(g,
source=u,
terminals=list(tree_nodes),
forbidden_nodes=late_nodes,
visitor=vis,
count_threshold=1)
# dist, pred = shortest_distance(g, source=u, pred_map=True)
node_set = {v for v, d in vis.dist.items() if d > 0}
reachable_tree_nodes = node_set.intersection(tree_nodes)
ancestor = min(reachable_tree_nodes, key=vis.dist.__getitem__)
edges = extract_edges_from_pred(g, u, ancestor, vis.pred)
edges = {(j, i) for i, j in edges} # need to reverse it
if debug:
print('tree_nodes', tree_nodes)
print('connecting {} to {}'.format(v, u))
print('using ancestor {}'.format(ancestor))
print('adding edges {}'.format(edges))
tree_nodes |= {u for e in edges for u in e}
tree_edges |= edges
t = Graph(directed=True)
for _ in range(g.num_vertices()):
t.add_vertex()
for u, v in tree_edges:
t.add_edge(t.vertex(u), t.vertex(v))
tree_nodes = {u for e in tree_edges for u in e}
vfilt = t.new_vertex_property('bool')
vfilt.a = False
for v in tree_nodes:
vfilt[t.vertex(v)] = True
t.set_vertex_filter(vfilt)
if return_closure:
return t, gc, mst_tree
else:
return t
plt.plot(range(a + 1, b + 1), ver_attr[(a + 1):(b + 1)], 'bo')
plt.plot(range(b + 1, len(ver_attr)), ver_attr[(b + 1):], 'ro')
print("-------------------------------------------------")
filename = '../pairparser/results/en_pairs(7).txt'
ftag = '7_3imp'
coefficient = 3
word_dict = {} # dict with indexes of nodes by word
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("double")
ver_names = pairs_graph.new_vertex_property("string")
ver_id = pairs_graph.new_vertex_property("int")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 not in word_dict:
filename = '../pairparser/results/p_pairs7.txt'
print(filename)
coefficient = 3
word_dict = {}
add_dict = {}
f = open('bad.txt', 'r', encoding="utf-8")
for s in f:
# print(s.split(' ')[0])
add_dict[s.split(' ')[0]] = 1
f = open(filename, 'r', encoding="utf-8")
pairs_graph = Graph(directed=False)
edge_weights = pairs_graph.new_edge_property("int")
ver_names = pairs_graph.new_vertex_property("string")
for line in f:
spl_line = line.split(' ')
if len(spl_line) == 1:
continue
pos = int(spl_line[0])
neg = int(spl_line[1])
cur_weight = pos + coefficient * neg
w1 = spl_line[2].strip(' \n\uefef')
w2 = spl_line[3].strip(' \n\uefef')
if w1 in add_dict or w2 in add_dict:
continue
from notion.client import NotionClient
from notion.collection import CollectionRowBlock
from graph_tool.all import Graph, graph_draw
client = NotionClient(token_v2='[token]')
#page = client.get_block('https://www.notion.so/Knowledge-a078d9ed592c459e93e8f7a1e433bcd2')
data = client.get_collection_view('[database page link]')
graph = Graph(directed=False)
# Dictionary from entry id to vertex id to keep track of vertices that have been added
entries = {}
vertex_names = graph.new_vertex_property('string')
# For every table entry
for row in data.collection.get_rows():
# If this entry is not in the graph, add it
if row.id not in entries:
entries[row.id] = graph.add_vertex()
vertex_names[entries[row.id]] = row.title
# Add the links of the current entry to the graph
for child in row.children:
if isinstance(child, CollectionRowBlock):
if child.id not in entries:
entries[child.id] = graph.add_vertex()
class PointerProvenancePlot(Plot):
"""
Base class for plots using the pointer provenance graph.
"""
def __init__(self, *args, **kwargs):
super(PointerProvenancePlot, self).__init__(*args, **kwargs)
self._cached_dataset_valid = False
"""Tells whether we need to rebuild the dataset when caching."""
def init_parser(self, dataset, tracefile):
if self.caching and os.path.exists(self._get_cache_file()):
# if caching we will nevere use this
return None
return PointerProvenanceParser(dataset, tracefile)
def init_dataset(self):
logger.debug("Init provenance graph for %s", self.tracefile)
self.dataset = Graph(directed=True)
vdata = self.dataset.new_vertex_property("object")
self.dataset.vp["data"] = vdata
return self.dataset
def _get_cache_file(self):
return self.tracefile + "_provenance_plot.gt"
def build_dataset(self):
"""
Build the provenance tree
"""
if self.caching:
try:
logger.debug("Load cached provenance graph")
self.dataset = load_graph(self._get_cache_file())
except IOError:
self.parser.parse()
self.dataset.save(self._get_cache_file())
else:
self.parser.parse()
num_nodes = self.dataset.num_vertices()
logger.debug("Total nodes %d", num_nodes)
vertex_mask = self.dataset.new_vertex_property("bool")
progress = ProgressPrinter(num_nodes, desc="Search kernel nodes")
for node in self.dataset.vertices():
# remove null capabilities
# remove operations in kernel mode
vertex_data = self.dataset.vp.data
node_data = vertex_data[node]
if ((node_data.pc != 0 and node_data.is_kernel) or
(node_data.cap.length == 0 and node_data.cap.base == 0)):
vertex_mask[node] = True
progress.advance()
progress.finish()
self.dataset.set_vertex_filter(vertex_mask, inverted=True)
vertex_mask = self.dataset.copy_property(vertex_mask)
num_nodes = self.dataset.num_vertices()
logger.debug("Filtered kernel nodes, remaining %d", num_nodes)
progress = ProgressPrinter(
num_nodes, desc="Merge (cfromptr + csetbounds) sequences")
for node in self.dataset.vertices():
progress.advance()
# merge cfromptr -> csetbounds subtrees
num_parents = node.in_degree()
if num_parents == 0:
# root node
continue
elif num_parents > 1:
logger.error("Found node with more than a single parent %s", node)
raise RuntimeError("Too many parents for a node")
parent = next(node.in_neighbours())
parent_data = self.dataset.vp.data[parent]
node_data = self.dataset.vp.data[node]
if (parent_data.origin == CheriNodeOrigin.FROMPTR and
node_data.origin == CheriNodeOrigin.SETBOUNDS):
# the child must be unique to avoid complex logic
# when merging, it may be desirable to do so with
# more complex traces
node_data.origin = CheriNodeOrigin.PTR_SETBOUNDS
if parent.in_degree() == 1:
next_parent = next(parent.in_neighbours())
vertex_mask[parent] = True
self.dataset.add_edge(next_parent, node)
elif parent.in_degree() == 0:
vertex_mask[parent] = True
else:
logger.error("Found node with more than a single parent %s",
parent)
raise RuntimeError("Too many parents for a node")
progress.finish()
self.dataset.set_vertex_filter(vertex_mask, inverted=True)
vertex_mask = self.dataset.copy_property(vertex_mask)
num_nodes = self.dataset.num_vertices()
logger.debug("Merged (cfromptr + csetbounds), remaining %d", num_nodes)
progress = ProgressPrinter(num_nodes, desc="Find short-lived cfromptr")
for node in self.dataset.vertices():
progress.advance()
node_data = self.dataset.vp.data[node]
if node_data.origin == CheriNodeOrigin.FROMPTR:
vertex_mask[node] = True
# if (node_data.origin == CheriNodeOrigin.FROMPTR and
# len(node_data.address) == 0 and
# len(node_data.deref["load"]) == 0 and
# len(node_data.deref["load"]) == 0):
# # remove cfromptr that are never stored or used in
# # a dereference
# remove_list.append(node)
progress.finish()
self.dataset.set_vertex_filter(vertex_mask, inverted=True)