def _filter_short_branch(self, filter=False, short=30):
"""
filter out very short branches: do this maybe not right for some models, for models with flat part, it is right
I will test how this effect the final matching results
need to delete nodes, switch with the last one then delete last
"""
if filter == False:
self.verts = self.verts_init
self.edges = self.edges_init
else:
init_graph = Graph(directed=False)
init_graph.add_vertex(len(self.verts_init))
for edge in self.edges_init:
init_graph.add_edge(init_graph.vertex(edge[0]), init_graph.vertex(edge[1]))
terminal_node = []
for v in init_graph.vertices():
if v.out_degree() == 1:
terminal_node.append(v)
visitor = DepthVisitor()
short_nodes = []
for tn in terminal_node:
search.dfs_search(init_graph, tn, visitor)
tmp_node = visitor.get_short_branch(min_length=short)
visitor.reset()
for n in tmp_node:
short_nodes.append(n)
## get edges on the short paths
short_nodes = list(set(short_nodes))
short_edges = []
temp_verts = self.verts_init[:]
v_num = len(self.verts_init)
if len(short_nodes):
for v in reversed(sorted(short_nodes)):
for ve in init_graph.vertex(v).out_edges():
short_edges.append(ve)
## delete edges first, then vertex
short_edges = list(set(short_edges))
for e in short_edges:
init_graph.remove_edge(e)
print 'deleting vertex',
for v in reversed(sorted(short_nodes)):
print v,
temp_verts[int(v)] = temp_verts[v_num-1]
init_graph.remove_vertex(v, fast=True)
v_num -= 1
print '\ndeleting related edges' # already done above, just info user
else:
print 'no short branches'
######## new vertices and edges ########
self.verts = temp_verts[:v_num]
self.edges = []
for e in init_graph.edges():
self.edges.append([int(e.source()), int(e.target())])
def attack(g: GT.Graph,
attack_strategy,
protection_strategy=lambda g, nodes, n_protected: set()):
nodes = attack_strategy(g)
protected = protection_strategy(g, nodes, Sim.n_protected)
giant_component_history = [giant_component_size(g)]
while giant_component_history[-1] > 1:
try:
g.remove_vertex(Sim.get_best(nodes, protected), fast=True)
giant_component_history.append(giant_component_size(g))
except NodeProtected: # the node is protected, the giant component size remains the same
giant_component_history.append(giant_component_history[-1])
except IndexError:
break # no more removable vertex
print(f"{g.num_vertices()} nodes remain ", end='\r')
print('')
return giant_component_history
class ob_viz(QWidget):
def __init__(self, bg_color):
QWidget.__init__(self)
self.background_color = bg_color
self.c = 0
# K = 0.5
# how many iterations the realignment is supposed to take
self.step = 15
self.rwr_c = 0
# dumper([qt_coords])
dumper(['obv viz init'])
# self.show()
# with open("/tmp/eaf3.csv", "a") as fo:
# wr = csv.writer(fo)
# wr.writerow([self.c, "runs4"])
# dumper([self.c, "runs4"])
# self.node_names [g_id[i] for i in g.vertices()]
def init2(self, emacs_var_dict):
self.emacs_var_dict = emacs_var_dict
self.link_str = self.emacs_var_dict['links']
self.g = Graph()
self.label_ep = self.g.new_edge_property("string")
self.links = self.link_str.split(";")
link_tpls = [i.split(" -- ") for i in self.links]
dumper([str(i) for i in link_tpls])
self.g_id = self.g.add_edge_list(link_tpls,
hashed=True,
string_vals=True,
eprops=[self.label_ep])
self.adj = np.array([(int(i.source()), int(i.target()))
for i in self.g.edges()])
self.node_names = [self.g_id[i] for i in self.g.vertices()]
self.vd = {}
for i in self.g.vertices():
self.vd[self.g_id[i]] = int(i)
# self.pos_vp = sfdp_layout(self.g, K=0.5)
self.pos_vp = fruchterman_reingold_layout(self.g)
self.base_pos_ar = self.pos_vp.get_2d_array((0, 1)).T
self.qt_coords = self.nolz_pos_ar(self.base_pos_ar)
dumper([str(self.qt_coords)])
# dumper([link_str])
def update_graph(self, emacs_var_dict):
"""set new links and nodes"""
new_link_str = emacs_var_dict['links']
new_links = new_link_str.split(";")
new_link_tpls = [i.split(" -- ") for i in new_links]
links_to_add = list(set(new_links) - set(self.links))
links_to_del = list(set(self.links) - set(new_links))
# setting new stuff
self.links = new_links
new_nodes = []
for tpl in new_link_tpls:
new_nodes.append(tpl[0])
new_nodes.append(tpl[1])
new_nodes_unique = list(set(new_nodes))
nodes_to_del = list(set(self.node_names) - set(new_nodes_unique))
nodes_to_add = list(set(new_nodes_unique) - set(self.node_names))
dumper([
"nodes_to_add: ", nodes_to_add, "nodes_to_del: ", nodes_to_del,
"links_to_add: ", links_to_add, "links_to_del: ", links_to_del
])
# first add nodes + index them, but not there yet (first links)
for n in nodes_to_add:
dumper(['adding node'])
v = self.g.add_vertex()
# how to new nodes pos to parents? separate loop afterwards
self.vd[n] = int(v)
self.g_id[v] = n
del_node_ids = [self.vd[i] for i in nodes_to_del]
self.g.remove_vertex(del_node_ids)
# have to reindex after deletion
self.vd = {}
for i in self.g.vertices():
self.vd[self.g_id[i]] = int(i)
dumper(['node deleted'])
# nodes_to_del_id =
# dumper(['old nodes deleted, add new links'])
for l in links_to_add:
tpl = l.split(" -- ")
n0, n1 = tpl[0], tpl[1]
self.g.add_edge(self.vd[n0], self.vd[n1])
# dumper(['new links added, delete old links'])
for l in links_to_del:
tpl = l.split(" -- ")
n0 = tpl[0]
n1 = tpl[1]
dumper([list(self.vd.keys())])
# only remove edge when neither of nodes removed
if n0 in self.vd.keys() and n1 in self.vd.keys():
self.g.remove_edge(self.g.edge(self.vd[n0], self.vd[n1]))
# dumper(['graph modifications done'])
# set positions of new nodes to parent nodes
for n in nodes_to_add:
v = self.g.vertex(self.vd[n])
v_prnt = list(v.all_neighbors())[0]
self.pos_vp[v] = self.pos_vp[v_prnt]
# dumper(['node positions adjusted'])
self.adj = np.array([(int(i.source()), int(i.target()))
for i in self.g.edges()])
self.node_names = [self.g_id[i] for i in self.g.vertices()]
# dumper(['storage objects updated'])
# dumper(["nbr_edges new: ", str(len([i for i in self.g.edges()]))])
# dumper(['nodes_to_add'] + nodes_to_add)
# seems to work
dumper(['to here'])
self.recalculate_layout()
dumper(['to here2'])
def recalculate_layout(self):
"""calculate new change_array, set rwr_c counter"""
dumper(['recalculating starting'])
self.base_pos_ar = self.pos_vp.get_2d_array((0, 1)).T
# set_dict = {'p': 2, 'max_level': 20, 'adaptive_cooling': False,
# 'gamma': 1, 'theta': 1, 'cooling_step': 0.3, 'C': 0.6, 'mu_p': 1.2}
# self.goal_vp = sfdp_layout(self.g, K=0.5, pos=self.pos_vp, **set_dict)
self.goal_vp = fruchterman_reingold_layout(self.g, pos=self.pos_vp)
goal_ar = self.goal_vp.get_2d_array([0, 1]).T
self.chng_ar = (goal_ar - self.base_pos_ar) / self.step
self.rwr_c = self.step
dumper(["base_pos_ar: ", self.base_pos_ar])
dumper(["goal_ar: ", goal_ar])
dumper(["chng_ar: ", self.chng_ar])
dumper(['recalculating done'])
def redraw_layout(self):
"""actually do the drawing, run multiple (step (rwr_c)) times"""
self.cur_pos_ar = np.round(
self.base_pos_ar + self.chng_ar * (self.step - self.rwr_c), 3)
self.qt_coords = self.nolz_pos_ar(self.cur_pos_ar)
self.rwr_c -= 1
self.update()
# dumper(['redrawing'])
# def draw_arrow(qp, p1x, p1y, p2x, p2y):
def draw_arrow(self, qp, p1x, p1y, p2x, p2y, node_width):
"""draw arrow from p1 to rad units before p2"""
# get arrow angle, counterclockwise from center -> east line
# dumper(['painting time'])
angle = degrees(atan2((p1y - p2y), (p1x - p2x)))
# calculate attach point
arw_goal_x = p2x + node_width * cos(radians(angle))
arw_goal_y = p2y + node_width * sin(radians(angle))
# calculate start point: idk how trig works but does
start_px = p1x - node_width * cos(radians(angle))
start_py = p1y - node_width * sin(radians(angle))
# arrow stuff: +/- 30 deg
ar1 = angle + 25
ar2 = angle - 25
arw_len = 10
# need to focus on vector from p2 to p1
ar1_x = arw_goal_x + arw_len * cos(radians(ar1))
ar1_y = arw_goal_y + arw_len * sin(radians(ar1))
ar2_x = arw_goal_x + arw_len * cos(radians(ar2))
ar2_y = arw_goal_y + arw_len * sin(radians(ar2))
# qp.drawLine(p1x, p1y, p2x, p2y)
# qp.drawLine(p1x, p1y, arw_goal_x, arw_goal_y)
qp.drawLine(start_px, start_py, arw_goal_x, arw_goal_y)
qp.drawLine(ar1_x, ar1_y, arw_goal_x, arw_goal_y)
qp.drawLine(ar2_x, ar2_y, arw_goal_x, arw_goal_y)
def paintEvent(self, event):
# dumper(['start painting'])
node_width = 10
qp = QPainter(self)
edges = [(self.qt_coords[i[0]], self.qt_coords[i[1]])
for i in self.adj]
# dumper([str(i) for i in edges])
qp.setPen(QPen(Qt.green, 2, Qt.SolidLine))
# [qp.drawLine(e[0][0], e[0][1], e[1][0], e[1][1]) for e in edges]
[
self.draw_arrow(qp, e[0][0], e[0][1], e[1][0], e[1][1],
(node_width / 2) + 5) for e in edges
]
qp.setPen(QColor(168, 34, 3))
# qp.setPen(Qt.green)
qp.setFont(QFont('Decorative', 10))
[
qp.drawText(t[0][0] + node_width, t[0][1], t[1])
for t in zip(self.qt_coords, self.node_names)
]
# dumper(['done painting'])
qp.setPen(QPen(Qt.black, 3, Qt.SolidLine))
# qp.setBrush(QBrush(Qt.green, Qt.SolidPattern))
dumper(['painting nodes'])
for i in zip(self.qt_coords, self.node_names):
if self.emacs_var_dict['cur_node'] == i[1]:
qp.setPen(QPen(Qt.black, 4, Qt.SolidLine))
qp.drawEllipse(i[0][0] - (node_width / 2),
i[0][1] - (node_width / 2), node_width,
node_width)
qp.setPen(QPen(Qt.black, 3, Qt.SolidLine))
else:
qp.drawEllipse(i[0][0] - (node_width / 2),
i[0][1] - (node_width / 2), node_width,
node_width)
# qp.drawEllipse(self.c, self.c, 7, 7)
# qp.end()
def nolz_pos_ar(self, pos_ar_org):
"""normalize pos ar to window limits"""
# pos_ar_org = goal_ar
size = self.size()
limits = [[20, size.width() - 50], [20, size.height() - 20]]
x_max = max(pos_ar_org[:, 0])
x_min = min(pos_ar_org[:, 0])
y_max = max(pos_ar_org[:, 1])
y_min = min(pos_ar_org[:, 1])
# need linear maping function again
pos_ar2 = pos_ar_org
pos_ar2[:, 0] = (((pos_ar2[:, 0] - x_min) / (x_max - x_min)) *
(limits[0][1] - limits[0][0])) + limits[0][0]
pos_ar2[:, 1] = (((pos_ar2[:, 1] - y_min) / (y_max - y_min)) *
(limits[1][1] - limits[1][0])) + limits[1][0]
return (pos_ar2)
class GraphDataset:
"""
Class for managing datasets with graph data
"""
def __init__(self, name, edges, object_ids, weights, hidden_graph=None):
"""
Params:
name (str): unique string to name this dataset (for pickling and
unpickling)
edges (numpy.ndarray): numpy array of shape [num_edges, 2]
containing the indices of nodes in all edges
objects (List[str]): string object ids for all nodes
weights (numpy.ndarray): numpy array of shape [num_edges]
containing edge weights
hidden_graph (GraphDataset): Graph data that should be excluded
but not considered as negative edges. (i.e. train
edges should not be in eval dataset but they shouldn't be
counted as negatives either)
"""
self.name = name
self.edges = edges
self.object_ids = np.asarray(object_ids)
self.weights = weights
self.hidden_graph = hidden_graph
self.graph = Graph(directed=False)
self.graph.add_vertex(len(object_ids))
edge_weights = [[edge[0], edge[1], weight]
for edge, weight in zip(self.edges, self.weights)]
self.weight_property = self.graph.new_edge_property("float")
eprops = [self.weight_property]
self.graph.add_edge_list(edge_weights, eprops=eprops)
self.manifold_nns = None
def gen_neighbor_data(self, verbose=True) -> Dict:
"""
Generates the graph data needed to run the cython iterator
Returns a dict with the neighbor data which will have values
- 'non_empty_vertices' the indices of vertices which have edges
emanating from them
- 'all_graph_neighbors' a list of lists of ints such that the list of
edges emanating from the vertex with index non_empty_vertices[i] is
stored in all_graph_neighbors[i]
- 'all_graph_weights' a list of lists of ints such that
all_graph_weights[i][j] represents the weight of the connection in
all_graph_neighbors[i][j]
- 'N' number of nodes in the graph
Parameters:
verbose (bool): should graph loading be printed out
"""
all_graph_neighbors = []
all_graph_weights = []
non_empty_vertices = []
empty_vertices = []
if verbose:
iterator = tqdm(range(self.n_nodes()),
desc="Generating Neighbor Data",
dynamic_ncols=True)
else:
iterator = range(self.n_nodes())
for i in iterator:
in_edges = self.graph.get_in_edges(i, [self.weight_property])
out_edges = self.graph.get_out_edges(i, [self.weight_property])
if in_edges.size + out_edges.size > 0:
non_empty_vertices.append(i)
if in_edges.size == 0:
all_graph_neighbors.append(out_edges[:,
1].astype(np.int64))
all_graph_weights.append(out_edges[:,
2].astype(np.float32))
elif out_edges.size == 0:
all_graph_neighbors.append(in_edges[:, 1].astype(np.int64))
all_graph_weights.append(in_edges[:, 2].astype(np.float32))
else:
all_graph_neighbors.append(
np.concatenate([in_edges[:, 0],
out_edges[:, 1]]).astype(np.int64))
all_graph_weights.append(
np.concatenate([in_edges[:, 2],
out_edges[:, 2]]).astype(np.float32))
else:
empty_vertices.append(i)
# graph_neighbors = np.concatenate(all_graph_neighbors)
# graph_neighbor_weights = np.concatenate(all_graph_weights)
non_empty_vertices = np.array(non_empty_vertices, dtype=np.int64)
empty_vertices = np.array(empty_vertices, dtype=np.int64)
return {
"all_graph_neighbors": all_graph_neighbors,
"all_graph_weights": all_graph_weights,
"non_empty_vertices": non_empty_vertices,
"empty_vertices": empty_vertices,
"N": self.n_nodes()
}
def add_manifold_nns(self, graph_embedder: GraphEmbedder):
manifold = graph_embedder.get_manifold()
data_points = graph_embedder.retrieve_nodes(self.n_nodes())
self.manifold_nns = ManifoldNNS(data_points, manifold)
def n_nodes(self) -> int:
"""
Returns the number of nodes in the graph
"""
return len(self.object_ids)
def collapse_nodes(self, node_ids):
all_new_edges = []
for node_id in tqdm(node_ids,
desc="Collapsing Nodes",
dynamic_ncols=True):
in_edges = self.graph.get_in_edges(node_id, [self.weight_property])
out_edges = self.graph.get_out_edges(node_id,
[self.weight_property])
neighbors = np.concatenate([out_edges[:, 1:3], in_edges[:, 0:3:2]])
if neighbors.shape[0] > 1:
neighbor_combos = \
neighbors[comb_index(neighbors.shape[0], 2)]
neighbor_combos = \
neighbor_combos.reshape(neighbor_combos.shape[0], 4)
new_edges = np.zeros((neighbor_combos.shape[0], 3))
new_edges[:, :2] += neighbor_combos[:, 0:3:2]
new_edges[:,2] += (neighbor_combos[:,1] + \
neighbor_combos[:,3])/4
all_new_edges.append(new_edges)
self.graph.add_edge_list(np.concatenate(all_new_edges),
eprops=[self.weight_property])
self.object_ids = np.delete(self.object_ids, np.array(node_ids))
self.graph.remove_vertex(node_ids)
edges_weights = self.graph.get_edges(eprops=[self.weight_property])
edges = edges_weights[:, 0:2]
weights = edges_weights[:, 2]
self.edges = edges
self.weights = weights
def get_neighbor_iterator(
self,
graph_sampling_config: GraphSamplingConfig,
data_fraction: float = 1,
) -> Iterator[GraphDataBatch]:
"""
Gets an efficient iterator of edge batches
"""
neighbor_data = load_or_gen(f"GraphDataset.{self.name}",
self.gen_neighbor_data)
if self.hidden_graph is None:
# GraphDataBatchIterator is defined in cython with these arguments.
# noinspection PyArgumentList
iterator = GraphDataBatchIterator(neighbor_data,
graph_sampling_config)
iterator.data_fraction = data_fraction
else:
hidden_neighbor_data = load_or_gen(
f"GraphDataset.{self.hidden_graph.name}",
self.hidden_graph.gen_neighbor_data)
# GraphDataBatchIterator is defined in cython with these arguments.
# noinspection PyArgumentList
iterator = GraphDataBatchIterator(neighbor_data,
graph_sampling_config,
hidden_neighbor_data)
iterator.data_fraction = data_fraction
if self.manifold_nns is not None:
sampling_config = get_config().sampling
_, nns = \
self.manifold_nns.knn_query_all(sampling_config.manifold_nn_k)
all_manifold_neighbors = [
nns[i][1:].astype(np.int64) for i in range(self.n_nodes())
]
iterator.refresh_manifold_nn(all_manifold_neighbors)
return iterator
@classmethod
def make_train_eval_split(cls, name, edges, object_ids, weights):
"""
Returns a tuple of a train eval split of the graph as defined in the
data config.
"""
data_config = get_config().data
np.random.seed(data_config.split_seed)
if data_config.split_by_edges:
# TODO Doesn't save to file in this mode
shuffle_order = np.arange(edges.shape[0])
np.random.shuffle(shuffle_order)
num_eval = floor(edges.shape[0] * data_config.split_size)
eval_indices = shuffle_order[:num_eval]
train_indices = shuffle_order[num_eval:]
train_edges = edges[train_indices]
train_weights = weights[train_indices]
eval_edges = edges[eval_indices]
eval_weights = weights[eval_indices]
else:
shuffle_order = np.arange(len(object_ids))
np.random.shuffle(shuffle_order)
num_eval = floor(len(object_ids) * data_config.split_size)
eval_indices = shuffle_order[:num_eval]
test_set = data_config.generate_test_set
if test_set:
test_indices = shuffle_order[num_eval:2 * num_eval]
train_indices = shuffle_order[2 * num_eval:] if test_set else \
shuffle_order[num_eval:]
train_edges = []
eval_edges = []
train_weights = []
eval_weights = []
if test_set:
test_edges = []
test_weights = []
for edge, weight in zip(edges, weights):
if test_set and (edge[0] in test_indices
or edge[1] in test_indices):
test_edges.append(edge)
test_weights.append(weight)
elif edge[0] in eval_indices or edge[1] in eval_indices:
eval_edges.append(edge)
eval_weights.append(weight)
else:
train_edges.append(edge)
train_weights.append(weight)
if test_set:
save_graph_data(test_edges, test_weights, object_ids,
data_config.test_path)
save_graph_data(train_edges, train_weights, object_ids,
data_config.train_path)
save_graph_data(eval_edges, eval_weights, object_ids,
data_config.eval_path)
train_edges = np.array(train_edges)
eval_edges = np.array(eval_edges)
train_weights = np.array(train_weights)
eval_weights = np.array(eval_weights)
train_data = GraphDataset(f"{name}_train", train_edges, object_ids,
train_weights)
eval_data = GraphDataset(f"{name}_eval",
eval_edges,
object_ids,
eval_weights,
hidden_graph=train_data)
return train_data, eval_data
class BaseGraph(object):
"""
Class representing a graph. We do not use pure graph_tool.Graph for we want
to be able to easily change this library. Neither we use inheritance
as graph_tool has inconvenient licence.
"""
def __init__(self):
self._g = None
self._node_dict = {}
self._syn_to_vertex_map = None
self._lemma_to_nodes_dict = None
self._lu_on_vertex_dict = None
def use_graph_tool(self):
"""
Returns underlying graph_tool.Graph. It should be avoided at all costs.
"""
return self._g
def get_node_for_synset_id(self, syn_id):
"""
Lazy function to makes the map of synset identifiers to nodes into
the graph. The building of map is made only on the first funcion call.
The first and the next calls of this function will return the built map.
"""
if not self._syn_to_vertex_map:
self._syn_to_vertex_map = {}
for node in self.all_nodes():
if node.synset:
synset_id = node.synset.synset_id
self._syn_to_vertex_map[synset_id] = node
return self._syn_to_vertex_map.get(syn_id, None)
def pickle(self, filename):
self._g.save(filename)
def unpickle(self, filename):
self._g = load_graph(filename)
def init_graph(self, drctd=False):
self._g = Graph(directed=drctd)
def copy_graph_from(self, g):
self._g = g._g.copy()
def set_directed(self, drctd):
self._g.set_directed(drctd)
def is_directed(self):
return self._g.is_directed()
def merge_graphs(self, g1, g2):
self._g = graph_union(g1._g, g2._g, internal_props=True)
# Node operations:
def all_nodes(self):
for node in self._g.vertices():
yield BaseNode(self._g, node)
def create_node_attribute(self, name, kind, value=None):
if not self.has_node_attribute(name):
node_attr = self._g.new_vertex_property(kind, value)
self._g.vertex_properties[name] = node_attr
def create_node_attributes(self, node_attributes_list):
for attr in node_attributes_list:
if not self.has_node_attribute(attr[0]):
node_attr = self._g.new_vertex_property(attr[1])
self._g.vertex_properties[attr[0]] = node_attr
def has_node_attribute(self, name):
""" Checks if a node attribute already exists """
return name in self._g.vertex_properties
def delete_node_attribute(self, name):
""" Delete node attribute """
del self._g.vertex_properties[name]
def add_node(self, name, node_attributes_list=None):
if node_attributes_list is None:
node_attributes_list = []
if name not in self._node_dict:
new_node = self._g.add_vertex()
self._node_dict[name] = BaseNode(self._g, new_node)
for attr in node_attributes_list:
self._g.vertex_properties[attr[0]][new_node] = attr[1]
return self._node_dict[name]
def get_node(self, name):
return self._node_dict[name]
def remove_node(self, name):
self._g.remove_vertex(self._node_dict[name]._node)
del self._node_dict[name]
def nodes_filter(self,
nodes_to_filter_set,
inverted=False,
replace=False,
soft=False):
"""
Filters out nodes from set
Args:
nodes_to_filter_set (Iterable): Nodes which fill be filtered out.
inverted (bool): If True, nodes NOT in set will be filtered out.
Defaults to False.
replace (bool): Replace current filter instead of combining the two.
Defaults to False.
soft (bool): Hide nodes without removing them so they can be restored
with reset_nodes_filter. Defaults to False.
"""
predicate = lambda node: node not in nodes_to_filter_set
self.nodes_filter_conditional(predicate, inverted, replace, soft)
def nodes_filter_conditional(self,
predicate,
inverted=False,
replace=False,
soft=False):
"""
Filters node based on a predicate
Args:
predicate (Callable): Predicate returning False for nodes that should be
filtered out.
inverted (bool): Invert condition. Defaults to False.
replace (bool): Replace current filter instead of combining the two.
Defaults to False.
soft (bool): Hide nodes without removing them so they can be restored
with reset_nodes_filter. Defaults to False.
"""
(old_filter, old_inverted) = self._g.get_vertex_filter()
new_filter = self._g.new_vertex_property("bool")
for node in self.all_nodes():
kept = predicate(node) != inverted
if not replace and old_filter:
old_kept = bool(old_filter[node._node]) != old_inverted
kept = kept and old_kept
new_filter[node._node] = kept
self._g.set_vertex_filter(new_filter, False)
if not soft:
self.apply_nodes_filter()
def apply_nodes_filter(self):
""" Removes nodes that are currently filtered out """
self._g.purge_vertices()
def reset_nodes_filter(self):
""" Clears node filter """
self._g.set_vertex_filter(None)
# Edge operations:
def num_edges(self):
return self._g.num_edges()
def all_edges(self):
for e in self._g.edges():
yield BaseEdge(self._g, e)
def get_edges_between(self, source, target):
"""
Return all edges between source and target. Source and target can be either
BaseNode or integer.
"""
if isinstance(source, BaseNode):
source = source._node
if isinstance(target, BaseNode):
target = target._node
for e in self._g.edge(source, target, all_edges=True):
yield BaseEdge(self._g, e)
def get_edge(self, source, target, add_missing=False):
"""
Return some edge between source and target. Source and target can be either
BaseNode or integer.
"""
if isinstance(source, BaseNode):
source = source._node
if isinstance(target, BaseNode):
target = target._node
e = self._g.edge(source, target, add_missing)
if e is not None:
return BaseEdge(self._g, e)
else:
return None
def create_edge_attribute(self, name, kind, value=None):
if not self.has_edge_attribute(name):
edge_attr = self._g.new_edge_property(kind, value)
self._g.edge_properties[name] = edge_attr
def alias_edge_attribute(self, name, alias):
self._g.edge_properties[alias] = self._g.edge_properties[name]
def create_edge_attributes(self, edge_attributes_list):
for attr in edge_attributes_list:
if not self.has_edge_attribute(attr[0]):
edge_attr = self._g.new_edge_property(attr[1])
self._g.edge_properties[attr[0]] = edge_attr
def has_edge_attribute(self, name):
""" Checks if an edge attribute already existst """
return name in self._g.edge_properties
def delete_edge_attribute(self, name):
""" Delete edge attribute """
del self._g.edge_properties[name]
def add_edge(self, parent, child, edge_attributes_list=None):
if edge_attributes_list is None:
edge_attributes_list = []
new_edge = self._g.add_edge(parent._node, child._node)
for attr in edge_attributes_list:
self._g.edge_properties[attr[0]][new_edge] = attr[1]
return BaseEdge(self._g, new_edge)
def edges_filter(self, edges_to_filter_set):
edge_filter = self._g.new_edge_property("bool")
for e in self.all_edges():
if e in edges_to_filter_set:
edge_filter[e._edge] = False
else:
edge_filter[e._edge] = True
self._g.set_edge_filter(edge_filter)
self._g.purge_edges()
def ungraph_tool(self, thingy, lemma_on_only_synset_node_dict):
"""
Converts given data structure so that it no longer have any graph_tool dependencies.
"""
logger = logging.getLogger(__name__)
if type(thingy) == dict:
return {
self.ungraph_tool(k, lemma_on_only_synset_node_dict):
self.ungraph_tool(thingy[k], lemma_on_only_synset_node_dict)
for k in thingy
}
nodes_to_translate = set()
for vset in lemma_on_only_synset_node_dict.values():
for v in vset:
nodes_to_translate.add(v)
if type(thingy) == gt.PropertyMap:
dct = {}
if thingy.key_type() == 'v':
for node in nodes_to_translate:
dct[node] = thingy[node.use_graph_tool()]
elif thingy.key_type() == 'e':
for edge in self.all_edges():
dct[edge] = thingy[edge.use_graph_tool()]
else:
logger.error('Unknown property type %s', thingy.key_type())
raise NotImplemented
return dct
def generate_lemma_to_nodes_dict_synsets(self):
"""
This method generates a utility dictionary, which maps lemmas to
corresponding node objects. It is expensive in menas of time
needed to generate the dictionary. It should therefore be executed
at the beginning of the runtime and later its results should be reused
as many times as needed without re-executing the function.
"""
lemma_to_nodes_dict = defaultdict(set)
for node in self.all_nodes():
try:
lu_set = node.synset.lu_set
except KeyError:
continue
for lu in lu_set:
lemma = lu.lemma.lower()
lemma_to_nodes_dict[lemma].add(node)
self._lemma_to_nodes_dict = lemma_to_nodes_dict
def generate_lemma_to_nodes_dict_lexical_units(self):
"""
This method generates a utility dictionary, which maps lemmas to
corresponding node objects. It is expensive in menas of time
needed to generate the dictionary. It should therefore be executed
at the beginning of the runtime and later its results should be reused
as many times as needed without re-executing the function.
"""
lemma_to_nodes_dict = defaultdict(set)
for node in self.all_nodes():
try:
lemma = node.lu.lemma.lower()
lemma_to_nodes_dict[lemma].add(node)
except:
continue
self._lemma_to_nodes_dict = lemma_to_nodes_dict
@property
def lemma_to_nodes_dict(self):
return self._lemma_to_nodes_dict
def _make_lu_on_v_dict(self):
"""
Makes dictionary lu on vertex
"""
lu_on_vertex_dict = defaultdict(set)
for node in self.all_nodes():
try:
nl = node.lu
except Exception:
continue
if nl:
lu_on_vertex_dict[node.lu.lu_id] = node
self._lu_on_vertex_dict = lu_on_vertex_dict
def gen_fs(dicProperties):
np.random.seed()
graphFS = Graph()
# on définit la fraction des arcs à utiliser la réciprocité
f = dicProperties["Reciprocity"]
rFracRecip = f/(2.0-f)
# on définit toutes les grandeurs de base
rInDeg = dicProperties["InDeg"]
rOutDeg = dicProperties["OutDeg"]
nNodes = 0
nEdges = 0
rDens = 0.0
if "Nodes" in dicProperties.keys():
nNodes = dicProperties["Nodes"]
graphFS.add_vertex(nNodes)
if "Edges" in dicProperties.keys():
nEdges = dicProperties["Edges"]
rDens = nEdges / float(nNodes**2)
dicProperties["Density"] = rDens
else:
rDens = dicProperties["Density"]
nEdges = int(np.floor(rDens*nNodes**2))
dicProperties["Edges"] = nEdges
else:
nEdges = dicProperties["Edges"]
rDens = dicProperties["Density"]
nNodes = int(np.floor(np.sqrt(nEdges/rDens)))
graphFS.add_vertex(nNodes)
dicProperties["Nodes"] = nNodes
# on définit le nombre d'arcs à créer
nArcs = int(np.floor(rDens*nNodes**2)/(1+rFracRecip))
# on définit les paramètres fonctions de probabilité associées F(x) = A x^{-tau}
Ai = nArcs*(rInDeg-1)/(nNodes)
Ao = nArcs*(rOutDeg-1)/(nNodes)
# on définit les moyennes des distributions de pareto 2 = lomax
rMi = 1/(rInDeg-2.)
rMo = 1/(rOutDeg-2.)
# on définit les trois listes contenant les degrés sortant/entrant/bidirectionnels associés aux noeuds i in range(nNodes)
lstInDeg = np.random.pareto(rInDeg,nNodes)+1
lstOutDeg = np.random.pareto(rOutDeg,nNodes)+1
lstInDeg = np.floor(np.multiply(Ai/np.mean(lstInDeg), lstInDeg)).astype(int)
lstOutDeg = np.floor(np.multiply(Ao/np.mean(lstOutDeg), lstOutDeg)).astype(int)
# on génère les stubs qui vont être nécessaires et on les compte
nInStubs = int(np.sum(lstInDeg))
nOutStubs = int(np.sum(lstOutDeg))
lstInStubs = np.zeros(np.sum(lstInDeg))
lstOutStubs = np.zeros(np.sum(lstOutDeg))
nStartIn = 0
nStartOut = 0
for vert in range(nNodes):
nInDegVert = lstInDeg[vert]
nOutDegVert = lstOutDeg[vert]
for j in range(np.max([nInDegVert,nOutDegVert])):
if j < nInDegVert:
lstInStubs[nStartIn+j] += vert
if j < nOutDegVert:
lstOutStubs[nStartOut+j] += vert
nStartOut+=nOutDegVert
nStartIn+=nInDegVert
# on vérifie qu'on a à peu près le nombre voulu d'edges
while nInStubs*(1+rFracRecip)/float(nArcs) < 0.95 :
vert = np.random.randint(0,nNodes)
nAddInStubs = int(np.floor(Ai/rMi*(np.random.pareto(rInDeg)+1)))
lstInStubs = np.append(lstInStubs,np.repeat(vert,nAddInStubs)).astype(int)
nInStubs+=nAddInStubs
while nOutStubs*(1+rFracRecip)/float(nArcs) < 0.95 :
nAddOutStubs = int(np.floor(Ao/rMo*(np.random.pareto(rOutDeg)+1)))
lstOutStubs = np.append(lstOutStubs,np.repeat(vert,nAddOutStubs)).astype(int)
nOutStubs+=nAddOutStubs
# on s'assure d'avoir le même nombre de in et out stubs (1.13 is an experimental correction)
nMaxStubs = int(1.13*(2.0*nArcs)/(2*(1+rFracRecip)))
if nInStubs > nMaxStubs and nOutStubs > nMaxStubs:
np.random.shuffle(lstInStubs)
np.random.shuffle(lstOutStubs)
lstOutStubs.resize(nMaxStubs)
lstInStubs.resize(nMaxStubs)
nOutStubs = nInStubs = nMaxStubs
elif nInStubs < nOutStubs:
np.random.shuffle(lstOutStubs)
lstOutStubs.resize(nInStubs)
nOutStubs = nInStubs
else:
np.random.shuffle(lstInStubs)
lstInStubs.resize(nOutStubs)
nInStubs = nOutStubs
# on crée le graphe, les noeuds et les stubs
nRecip = int(np.floor(nInStubs*rFracRecip))
nEdges = nInStubs + nRecip +1
# les stubs réciproques
np.random.shuffle(lstInStubs)
np.random.shuffle(lstOutStubs)
lstInRecip = lstInStubs[0:nRecip]
lstOutRecip = lstOutStubs[0:nRecip]
lstEdges = np.array([np.concatenate((lstOutStubs,lstInRecip)),np.concatenate((lstInStubs,lstOutRecip))]).astype(int)
# add edges
graphFS.add_edge_list(np.transpose(lstEdges))
remove_self_loops(graphFS)
remove_parallel_edges(graphFS)
lstIsolatedVert = find_vertex(graphFS, graphFS.degree_property_map("total"), 0)
graphFS.remove_vertex(lstIsolatedVert)
graphFS.reindex_edges()
nNodes = graphFS.num_vertices()
nEdges = graphFS.num_edges()
rDens = nEdges / float(nNodes**2)
# generate types
rInhibFrac = dicProperties["InhibFrac"]
lstTypesGen = np.random.uniform(0,1,nEdges)
lstTypeLimit = np.full(nEdges,rInhibFrac)
lstIsExcitatory = np.greater(lstTypesGen,lstTypeLimit)
nExc = np.count_nonzero(lstIsExcitatory)
epropType = graphFS.new_edge_property("int",np.multiply(2,lstIsExcitatory)-np.repeat(1,nEdges)) # excitatory (True) or inhibitory (False)
graphFS.edge_properties["type"] = epropType
# and weights
if dicProperties["Weighted"]:
lstWeights = dicGenWeights[dicProperties["Distribution"]](graphFS,dicProperties,nEdges,nExc) # generate the weights
epropW = graphFS.new_edge_property("double",lstWeights) # crée la propriété pour stocker les poids
graphFS.edge_properties["weight"] = epropW
return graphFS
