def init_graph(graphml_path):
global g
g = Graph(directed=True)
t0 = time()
g.load(graphml_path)
t1 = time()
print "Loaded from GraphML in", t1-t0
print "Loaded", g.num_vertices(), "nodes"
print "Loaded", g.num_edges(), "edges"
class Network:
def __init__(self, nodes_info=None, links_info=None, file_name=None):
self.g = Graph()
if nodes_info and links_info:
self.nodes_info = nodes_info
self.links_info = links_info
self.g.vertex_properties["name"] = self.g.new_vertex_property(
'string')
self.g.vertex_properties["id"] = self.g.new_vertex_property(
'int32_t')
self.g.edge_properties["weight"] = self.g.new_edge_property(
'int32_t')
self.create_network()
self.g.vertex_properties["pagerank"] = pagerank(
self.g, weight=self.g.edge_properties["weight"])
self.g.vertex_properties[
"degree_centrality"] = self.degree_centrality()
elif file_name:
self.load_network(file_name)
def create_network(self):
# Add Nodes
for node in self.nodes_info:
self.add_n(node)
# Add Links
for link in self.links_info:
n_loser = 0
n_winner = 0
loser = link['loser']
winner = link['winner']
weight = link['rounds']
for team_id in self.g.vertex_properties.id:
if loser == team_id:
break
n_loser += 1
for team_id in self.g.vertex_properties.id:
if winner == team_id:
break
n_winner += 1
self.add_l(n_loser, n_winner, 16 / weight * 100)
def load_network(self, file_name):
new_file_name = '..' + sep + '..' + sep + 'network-graphs' + sep + file_name
self.g.load(new_file_name, fmt="gt")
def get_normalized_pagerank(self):
max_pgr = 0
for pgr in self.g.vertex_properties.pagerank:
if pgr > max_pgr:
max_pgr = pgr
return [
self.g.vertex_properties.pagerank[v] / max_pgr
for v in self.g.vertices()
]
def add_n(self, node_info):
n = self.g.add_vertex()
self.g.vertex_properties.id[n] = node_info['id']
self.g.vertex_properties.name[n] = node_info['Team_Name']
def add_l(self, loser, winner, weight):
n1 = self.g.vertex(loser)
n2 = self.g.vertex(winner)
l = self.g.add_edge(n1, n2)
self.g.edge_properties.weight[l] = weight
def draw(self, output_file, fmt):
graph_draw(self.g,
vertex_text=self.g.vertex_index,
output=output_file,
fmt=fmt)
def save_network(self, file_name):
try:
new_file_name = '..' + sep + '..' + sep + 'network-graphs' + sep + file_name
self.g.save(new_file_name, fmt="gt")
except:
return False
return True
def vp_pagerank(self):
return self.g.vertex_properties.pagerank
def vp_degree_cent(self):
return self.g.vertex_properties.degree_centrality
def vp_name(self):
return self.g.vertex_properties.name
def vp_id(self):
return self.g.vertex_properties.id
def ep_weight(self):
return self.g.edge_properties.weight
# Calcula as características básicas da rede
def get_basic_info(self):
info = {}
try:
n_vertices = self.g.num_vertices()
n_edges = self.g.num_edges()
density = n_edges / ((n_vertices * (n_vertices - 1)) / 2)
mean_degree = (2 * n_edges) / n_vertices
# Cálculo do coeficiente de clusterização "na mão", usando a média dos
# coeficientes locais calculados pela Graph Tools
local_cc = local_clustering(self.g)
clustering_coef = fsum(
[local_cc[x] for x in self.g.vertices() if local_cc[x] != 0.0])
clustering_coef /= n_vertices
info["Número de times"] = n_vertices
info["Número de confrontos"] = n_edges
info["Densidade"] = density
info["Grau médio"] = mean_degree
info["Coeficiente de Clusterização"] = clustering_coef
except:
info.clear()
return info
def degree_centrality(self):
degree_centrality = self.g.new_vertex_property('float')
for v in self.g.vertices():
degree_centrality[v] = v.in_degree() / (self.g.num_vertices() - 1)
return degree_centrality
# Calcula a distribuição de graus da rede
def degree_distribution(self):
degree_dist = {}
try:
for v in self.g.vertices():
if v.in_degree() not in degree_dist.keys():
degree_dist[v.in_degree()] = 1
else:
degree_dist[v.in_degree()] += 1
for k in degree_dist.keys():
degree_dist[k] /= self.g.num_vertices()
except:
degree_dist.clear()
return degree_dist
class PointerProvenancePlot(Plot):
"""
Base class for plots using the pointer provenance graph.
"""
def __init__(self, *args, **kwargs):
super(PointerProvenancePlot, self).__init__(*args, **kwargs)
self._cached_dataset_valid = False
"""Tells whether we need to rebuild the dataset when caching."""
def init_parser(self, dataset, tracefile):
if self.caching and os.path.exists(self._get_cache_file()):
# if caching we will nevere use this
return None
return PointerProvenanceParser(dataset, tracefile)
def init_dataset(self):
logger.debug("Init provenance graph for %s", self.tracefile)
self.dataset = Graph(directed=True)
vdata = self.dataset.new_vertex_property("object")
self.dataset.vp["data"] = vdata
return self.dataset
def _get_cache_file(self):
return self.tracefile + "_provenance_plot.gt"
def build_dataset(self):
"""
Build the provenance tree
"""
if self.caching:
try:
logger.debug("Load cached provenance graph")
self.dataset = load_graph(self._get_cache_file())
except IOError:
self.parser.parse()
self.dataset.save(self._get_cache_file())
else:
self.parser.parse()
num_nodes = self.dataset.num_vertices()
logger.debug("Total nodes %d", num_nodes)
vertex_mask = self.dataset.new_vertex_property("bool")
progress = ProgressPrinter(num_nodes, desc="Search kernel nodes")
for node in self.dataset.vertices():
# remove null capabilities
# remove operations in kernel mode
vertex_data = self.dataset.vp.data
node_data = vertex_data[node]
if ((node_data.pc != 0 and node_data.is_kernel) or
(node_data.cap.length == 0 and node_data.cap.base == 0)):
vertex_mask[node] = True
progress.advance()
progress.finish()
self.dataset.set_vertex_filter(vertex_mask, inverted=True)
vertex_mask = self.dataset.copy_property(vertex_mask)
num_nodes = self.dataset.num_vertices()
logger.debug("Filtered kernel nodes, remaining %d", num_nodes)
progress = ProgressPrinter(
num_nodes, desc="Merge (cfromptr + csetbounds) sequences")
for node in self.dataset.vertices():
progress.advance()
# merge cfromptr -> csetbounds subtrees
num_parents = node.in_degree()
if num_parents == 0:
# root node
continue
elif num_parents > 1:
logger.error("Found node with more than a single parent %s", node)
raise RuntimeError("Too many parents for a node")
parent = next(node.in_neighbours())
parent_data = self.dataset.vp.data[parent]
node_data = self.dataset.vp.data[node]
if (parent_data.origin == CheriNodeOrigin.FROMPTR and
node_data.origin == CheriNodeOrigin.SETBOUNDS):
# the child must be unique to avoid complex logic
# when merging, it may be desirable to do so with
# more complex traces
node_data.origin = CheriNodeOrigin.PTR_SETBOUNDS
if parent.in_degree() == 1:
next_parent = next(parent.in_neighbours())
vertex_mask[parent] = True
self.dataset.add_edge(next_parent, node)
elif parent.in_degree() == 0:
vertex_mask[parent] = True
else:
logger.error("Found node with more than a single parent %s",
parent)
raise RuntimeError("Too many parents for a node")
progress.finish()
self.dataset.set_vertex_filter(vertex_mask, inverted=True)
vertex_mask = self.dataset.copy_property(vertex_mask)
num_nodes = self.dataset.num_vertices()
logger.debug("Merged (cfromptr + csetbounds), remaining %d", num_nodes)
progress = ProgressPrinter(num_nodes, desc="Find short-lived cfromptr")
for node in self.dataset.vertices():
progress.advance()
node_data = self.dataset.vp.data[node]
if node_data.origin == CheriNodeOrigin.FROMPTR:
vertex_mask[node] = True
# if (node_data.origin == CheriNodeOrigin.FROMPTR and
# len(node_data.address) == 0 and
# len(node_data.deref["load"]) == 0 and
# len(node_data.deref["load"]) == 0):
# # remove cfromptr that are never stored or used in
# # a dereference
# remove_list.append(node)
progress.finish()
self.dataset.set_vertex_filter(vertex_mask, inverted=True)
def florians_procedure(g: gt.Graph, use_simplicial):
n = g.num_vertices()
if not use_simplicial:
s = simplicial_vertices(g)
a = s[0]
while a in s:
a = np.random.randint(0, n)
b = a
while a == b or b in s:
b = np.random.randint(0, n)
else:
a = np.random.randint(0, n)
b = a
while a == b:
b = np.random.randint(0, n)
A = np.zeros(n, dtype=np.bool)
A[a] = True
B = np.zeros(n, dtype=np.bool)
B[b] = True
F = set(range(n)).difference(np.where(A | B == True)[0])
i = 0
while len(F) > 0:
e = F.pop()
if i % 2 == 0:
A[e] = True
A_new = (g, np.where(A == True)[0])
if not np.any(B & A_new):
A = A_new
F = F.difference(set(np.where(A == True)[0]))
else:
A[e] = False
B[e] = True
B_new = compute_hull(g, np.where(B == True)[0])
if not np.any(A & B_new):
B = B_new
F = F.difference(set(np.where(A == True)[0]))
else:
B[e] = False
else:
B[e] = True
B_new = compute_hull(g, np.where(B == True)[0])
if not np.any(A & B_new):
B = B_new
F = F.difference(set(np.where(A == True)[0]))
else:
B[e] = False
A[e] = True
A_new = compute_hull(g, np.where(A == True)[0])
if not np.any(B & A_new):
A = A_new
F = F.difference(set(np.where(A == True)[0]))
i += 1
print(len(F))
return A, B
class SentenceGraph():
def __init__(self, sentence, directed=False, graph=None):
# Create a SentenceGraph from an existing graph tool graph
if graph is not None:
self.sentence_graph = graph
return
# Create a new SentenceGraph from scratch
self.sentence_graph = Graph(directed=directed)
# Graph properties
sentence_property = self.sentence_graph.new_graph_property("string", sentence)
self.sentence_graph.graph_properties[SENTENCE_KEY] = sentence_property
# Vertex properties
word_property = self.sentence_graph.new_vertex_property("string")
part_of_speech_property = self.sentence_graph.new_vertex_property("string")
vertex_color_property = self.sentence_graph.new_vertex_property("vector<double>")
self.sentence_graph.vertex_properties[WORD_KEY] = word_property
self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY] = part_of_speech_property
self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY] = vertex_color_property
# Edge properties
sentence_edge_property = self.sentence_graph.new_edge_property("string")
definition_edge_property = self.sentence_graph.new_edge_property("string")
parsed_dependencies_edge_property = self.sentence_graph.new_edge_property("string")
inter_sentence_edge_property = self.sentence_graph.new_edge_property("string")
edge_color_property = self.sentence_graph.new_edge_property("vector<double>")
dependency_edge_property = self.sentence_graph.new_edge_property("string")
self.sentence_graph.edge_properties[SENTENCE_EDGE_KEY] = sentence_edge_property
self.sentence_graph.edge_properties[DEFINITION_EDGE_KEY] = definition_edge_property
self.sentence_graph.edge_properties[PARSED_DEPENDENCIES_EDGE_KEY] = parsed_dependencies_edge_property
self.sentence_graph.edge_properties[INTER_SENTENCE_EDGE_KEY] = inter_sentence_edge_property
self.sentence_graph.edge_properties[EDGE_COLOR_KEY] = edge_color_property
self.sentence_graph.edge_properties[PARSE_TREE_DEPENDENCY_VALUE_KEY] = dependency_edge_property
# Edge filter properties
definition_edge_filter_property = self.sentence_graph.new_edge_property("bool")
inter_sentence_edge_filter_property = self.sentence_graph.new_edge_property("bool")
parsed_dependencies_edge_filter_property = self.sentence_graph.new_edge_property("bool")
sentence_edge_filter_property = self.sentence_graph.new_edge_property("bool")
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY] = definition_edge_filter_property
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY] = inter_sentence_edge_filter_property
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY] = parsed_dependencies_edge_filter_property
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY] = sentence_edge_filter_property
def get_sentence(self):
return self.sentence_graph.graph_properties[SENTENCE_KEY]
def add_vertex(self, word, pos):
word_pos_tuple = (word, pos)
# Create vertex, set properties
word_vertex = self.sentence_graph.add_vertex()
self.sentence_graph.vertex_properties[WORD_KEY][word_vertex] = word
self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][word_vertex] = pos
self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY][word_vertex] = [0, 0, 1, 1]
return word_vertex
def set_vertex_color_from_word(self, word, pos, color=[1, 0, 0, 1]):
word_vertex = self.get_vertex(word, pos)
return self.set_vertex_color(word_vertex, color)
def set_vertex_color(self, vertex, color=[1, 0, 0, 1]):
self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY][vertex] = color
def set_vertices_color(self, vertices, color=[1, 0, 0, 1]):
for vertex in vertices:
self.set_vertex_color(vertex, color)
def add_sentence_edge_from_words(self, word1, pos1, word2, pos2):
return self.add_sentence_edge(self.get_vertex(word1, pos1), self.get_vertex(word2, pos2))
def add_sentence_edge(self, word_vertex1, word_vertex2):
sentence_edge = self.sentence_graph.add_edge(word_vertex1, word_vertex2, add_missing=False)
self.sentence_graph.edge_properties[SENTENCE_EDGE_KEY][sentence_edge] = sentence_edge
# Green
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][sentence_edge] = [0.2, 1, 0.2, 1]
self._set_edge_to_zero_in_all_filters(sentence_edge)
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY][sentence_edge] = True
return sentence_edge
def add_sentence_edges(self, sentence_vertices):
for i in range(1, len(sentence_vertices)):
self.add_sentence_edge(sentence_vertices[i - 1], sentence_vertices[i])
def add_parsed_dependency_edge(self, word_vertex1, word_vertex2, dependency_relationship):
parsed_dependency_edge = self.sentence_graph.add_edge(word_vertex1, word_vertex2, add_missing=False)
self.sentence_graph.edge_properties[PARSED_DEPENDENCIES_EDGE_KEY][parsed_dependency_edge] = parsed_dependency_edge
self.sentence_graph.edge_properties[PARSE_TREE_DEPENDENCY_VALUE_KEY][parsed_dependency_edge] = dependency_relationship
# Blue
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][parsed_dependency_edge] = [0, 0, 1, 1]
self._set_edge_to_zero_in_all_filters(parsed_dependency_edge)
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY][parsed_dependency_edge] = True
return parsed_dependency_edge
def add_parsed_dependency_edge_from_words(self, word1, pos1, word2, pos2, dependency_relationship):
return self.add_parsed_dependency_edge(
self.get_vertex(word1, pos1),
self.get_vertex(word2, pos2),
dependency_relationship)
def add_definition_edge_from_words(self, word, pos, definition_word, definition_pos):
return self.add_definition_edge(
self.get_vertex(word, pos),
self.get_vertex(definition_word, definition_pos))
def _set_edge_to_zero_in_all_filters(self, edge):
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY][edge] = False
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY][edge] = False
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY][edge] = False
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY][edge] = False
def add_definition_edge(self, word_vertex, definition_word_vertex):
definition_edge = self.sentence_graph.add_edge(word_vertex, definition_word_vertex, add_missing=False)
self.sentence_graph.edge_properties[DEFINITION_EDGE_KEY][definition_edge] = definition_edge
# Red
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][definition_edge] = [1, 0.1, 0.1, 1]
self._set_edge_to_zero_in_all_filters(definition_edge)
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY][definition_edge] = True
return definition_edge
def add_definition_edges(self, word_vertex, definition_word_vertices):
# Add edges from the word_vertex to all definition vertices and set
# the definition edge property on each edge
for definition_word_vertex in definition_word_vertices:
self.add_definition_edge(word_vertex, definition_word_vertex)
return self
def add_inter_sentence_edge(self, sentence1_word_vertex, sentence2_word_vertex):
inter_sentence_edge = self.sentence_graph.add_edge(sentence1_word_vertex, sentence2_word_vertex, add_missing=False)
self.sentence_graph.edge_properties[INTER_SENTENCE_EDGE_KEY][inter_sentence_edge] = inter_sentence_edge
# Pink
self.sentence_graph.edge_properties[EDGE_COLOR_KEY][inter_sentence_edge] = [1, 0.05, 1, 1]
self._set_edge_to_zero_in_all_filters(inter_sentence_edge)
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY][inter_sentence_edge] = True
return inter_sentence_edge
def add_inter_sentence_edge_from_words(self, word1, pos1, word2, pos2):
return self.add_inter_sentence_edge(
self.get_vertex(word1, pos1),
self.get_vertex(word2, pos2))
def remove_vertex_by_word(self, word, pos):
self.remove_vertex(self.get_vertex(word, pos))
def remove_vertex(self, vertex):
word = self.sentence_graph.vertex_properties[WORD_KEY][vertex]
pos = self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][vertex]
self.sentence_graph.remove_vertex(vertex)
def remove_edge(self, word1, pos1, word2, pos2):
self.sentence_graph.remove_edge(self.get_edge(word1, pos1, word2, pos2))
def contains(self, word, pos):
return self.get_vertex(word, pos) is not None
def get_vertex(self, word, pos):
for vertex in self.sentence_graph.vertices():
try:
vertex_word = self.sentence_graph.vertex_properties[WORD_KEY][vertex]
vertex_pos = self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][vertex]
if vertex_word == word and vertex_pos == pos:
return vertex
except:
pass
return None
def get_word_pos_tuple(self, vertex):
return self.sentence_graph.vertex_properties[WORD_KEY][vertex],\
self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY][vertex]
def get_word_pos_tuple_by_index(self, index):
return self.get_word_pos_tuple(self.get_vertex_by_index(index))
def get_vertex_by_index(self, index):
return self.sentence_graph.vertex(index)
def get_vertices_iterator(self):
return self.sentence_graph.vertices()
def get_vertices(self):
return [x for x in self.sentence_graph.vertices()]
def get_vertex_out_neighbor_word_pos_tuples(self, vertex):
return [self.get_word_pos_tuple(neighbor_vertex)
for neighbor_vertex in self.get_vertex_out_neighbors(vertex)]
def get_vertex_in_neighbor_word_pos_tuples(self, vertex):
return [self.get_word_pos_tuple(neighbor_vertex)
for neighbor_vertex in self.get_vertex_in_neighbors(vertex)]
def get_vertex_out_neighbors(self, vertex):
return [neighbor_vertex for neighbor_vertex in vertex.out_neighbours()]
def get_vertex_in_neighbors(self, vertex):
return [neighbor_vertex for neighbor_vertex in vertex.in_neighbours()]
def get_word_pos_tuples(self):
return [self.get_word_pos_tuple(v) for v in self.sentence_graph.vertices()]
def get_num_vertices(self):
return self.sentence_graph.num_vertices()
def get_num_edges(self):
return self.sentence_graph.num_edges()
def get_edge(self, word1, pos1, word2, pos2):
vertex_1 = self.get_vertex(word1, pos1)
vertex_2 = self.get_vertex(word2, pos2)
return None\
if vertex_1 is None or vertex_2 is None\
else self.sentence_graph.edge(vertex_1, vertex_2)
def get_edges_iterator(self):
return self.sentence_graph.edges()
def get_edges(self):
return [x for x in self.sentence_graph.edges()]
def set_definition_edge_filter(self, inverted=False):
self.sentence_graph.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_DEFINITION_EDGE_KEY],
inverted=inverted)
def set_inter_sentence_edge_filter(self, inverted=False):
self.sentence_graph.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_INTER_SENTENCE_EDGE_KEY],
inverted=inverted)
def set_parsed_dependency_edge_filter(self, inverted=False):
self.sentence_edge.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_PARSED_DEPENDENCIES_EDGE_KEY],
inverted=inverted)
def set_sentence_edge_filter(self, inverted=False):
self.sentence_graph.set_edge_filter(
self.sentence_graph.edge_properties[FILTER_SENTENCE_EDGE_KEY],
inverted=inverted)
def clear_filters(self):
self.sentence_graph.clear_filters()
def get_definition_edges(self):
return filter(lambda x: x in self.get_definition_edge_properties(), self.get_edges())
def get_word_vertex_properties(self):
return self.sentence_graph.vertex_properties[WORD_KEY]
def get_pos_vertex_properties(self):
return self.sentence_graph.vertex_properties[PART_OF_SPEECH_KEY]
def get_color_vertex_properties(self):
return self.sentence_graph.vertex_properties[VERTEX_COLOR_KEY]
def get_sentence_edge_properties(self):
return self.sentence_graph.edge_properties[SENTENCE_EDGE_KEY]
def get_definition_edge_properties(self):
return self.sentence_graph.edge_properties[DEFINITION_EDGE_KEY]
def get_inter_sentence_edge_properties(self):
return self.sentence_graph.edge_properties[INTER_SENTENCE_EDGE_KEY]
def get_color_edge_properties(self):
return self.sentence_graph.edge_properties[EDGE_COLOR_KEY]
def get_vertex_index(self, vertex):
return self.sentence_graph.vertex_index[vertex]
def get_degree_properties(self, degree_type):
return self.sentence_graph.degree_property_map(degree_type)
def get_graph(self):
return self.sentence_graph
def copy(self):
return SentenceGraph(
sentence=self.sentence_graph.graph_properties[SENTENCE_KEY],
graph=self.sentence_graph.copy())
