def add_edges(keys: List[str], w2v: Word2Vec, nearest_neighbor_count: int,
min_count: int, graph: Graph) -> None:
"""Adds edges to an existing graph based on a list of keys and constraints to their similarity and frequency."""
edge_dict: Dict[str, Set[str]] = {}
for key in keys:
sims: List[str] = [
x[0] for x in w2v.wv.most_similar(key, topn=nearest_neighbor_count)
]
for i in range(len(sims)):
if w2v.wv.vocab[sims[i]].count >= min_count:
graph.add_edge(key, sims[i])
sub_sims: List[str] = [
x[0] for x in w2v.wv.most_similar(sims[i],
topn=nearest_neighbor_count)
]
sub_sims = [
x for x in sub_sims if w2v.wv.vocab[x].count >= min_count
]
for sub_sim in sub_sims:
edge_dict[sims[i]] = edge_dict.get(sims[i],
set()).union({sub_sim})
edge_dict[sub_sim] = edge_dict.get(sub_sim,
set()).union({sims[i]})
for edge_source in edge_dict:
if graph.has_node(edge_source):
for edge_target in edge_dict[edge_source]:
if graph.has_node(edge_target):
graph.add_edge(edge_source, edge_target)
def to_networkx(self, graph: nx.Graph = None) -> nx.Graph:
"""
Return networkx graph of entities.
Parameters
----------
graph : nx.Graph, optional
Graph to add entities to. If not supplied the function
creates and returns a new graph.
By default None
Returns
-------
nx.Graph
Graph with entity and any connected entities.
"""
graph = graph or nx.Graph()
if not graph.has_node(self):
graph.add_node(self, **self.node_properties)
for edge in self.edges:
if graph.has_edge(edge.source, edge.target):
continue
graph.add_edge(edge.source, edge.target, **edge.attrs)
for node in (edge.source, edge.target):
# If this node has edges that are not in our graph
# call to_networkx recursively on that node.
if any(edge for edge in node.edges
if not graph.has_edge(edge.source, edge.target)):
ent_node = typing.cast(Entity, node)
ent_node.to_networkx(graph)
return graph
def upsert_node(slug: str, g: nx.Graph):
if g.has_node(slug):
return g.nodes[slug]
g.add_node(slug, slug=slug, size=0)
return g.nodes[slug]
def plot_link_prediction_graph(G: nx.Graph,
pred_edges: [],
pred_acc=None,
idx2node=None):
G = G.copy()
if pred_acc is None:
pred_acc = []
pos = nx.spring_layout(G, seed=6) # positions for all nodes
nx.draw_networkx_nodes(G, pos)
# edges
nx.draw_networkx_edges(G, pos, width=1.0, alpha=0.5)
pred_edges_ = []
edges_labels = {}
for i, (u, v) in enumerate(pred_edges):
if G.has_node(u) and G.has_node(v):
G.add_edge(u, v)
pred_edges_.append((u, v))
if pred_acc:
edges_labels[(u, v)] = round(pred_acc[i], 2)
nx.draw_networkx_edges(
G,
pos,
edgelist=pred_edges_,
# width=1,
# alpha=0.5,
edge_color='r')
if pred_acc:
nx.draw_networkx_edge_labels(G,
pos,
edge_labels=edges_labels,
font_color='red')
if idx2node is not None:
labels = {}
for u in G.nodes:
labels[u] = str(idx2node[u])
nx.draw_networkx_labels(G, pos, labels=labels)
else:
nx.draw_networkx_labels(G, pos=pos)
plt.axis('off')
plt.show()
def parse():
print 'parsing ngrams...'
phrases = defaultdict(set)
for row in table:
parse_row(phrases, row)
# report_phrases(phrases)
print 'loading mapping...'
res = requests.get(CSV)
mapping = {}
for row in unicodecsv.DictReader(res.iter_lines()):
phrase = row.pop('phrase')
mapping[phrase] = row
print 'generating graph nodes....'
G = Graph()
for phrase, mapped in mapping.items():
map_to = mapped.get('map_to') or None
group = mapped.get('group') or None
if not map_to and not group:
continue
name = map_to or phrase
if not G.has_node(name):
G.add_node(name, label=name, cases=[], type='phrase')
if group:
G.node[name]['theme'] = group
for (p, i), cases in phrases.items():
if p == phrase:
for case in cases:
if case not in G.node[name]['cases']:
G.node[name]['cases'].append(case)
G.add_node('setting', type='theme')
G.add_node('incident', type='theme')
G.add_node('reaction', type='theme')
print 'generating graph edges....'
for outer in G.nodes():
if G.node[outer]['type'] != 'phrase':
continue
otheme = G.node[outer].get('theme')
if otheme is None:
continue
ocases = set(G.node[outer].get('cases', []))
G.add_edge(outer, otheme, type='theme')
for inner in G.nodes():
if inner >= outer or G.node[inner]['type'] != 'phrase':
continue
icases = set(G.node[inner].get('cases', []))
ixcases = len(ocases.intersection(icases))
if ixcases:
G.add_edge(outer, inner, type='common', weight=ixcases)
data = json_graph.node_link_data(G)
with open('web/graph.json', 'w') as fh:
json.dump(data, fh, indent=2)
def add_contacts_edge(G: nx.Graph, interaction_type: str) -> nx.Graph:
"""
Adds specific interaction types to the protein graph.
:param G: networkx protein graph
:type G: nx.Graph
:param interaction_type: interaction type to be added
:type interaction_type: str
:return G: nx.Graph with specified interaction-based edges added.
:rtype: nx.Graph
"""
log.debug(f"Adding {interaction_type} edges to graph")
if "contacts_df" not in G.graph:
log.info("No 'contacts_df' found in G.graph. Running GetContacts.")
G.graph["contacts_df"] = get_contacts_df(
G.graph["config"].get_contacts_config, G.graph["pdb_id"]
)
contacts = G.graph["contacts_df"]
# Select specific interaction type
interactions = contacts.loc[
contacts["interaction_type"] == interaction_type
]
for label, [res1, res2, interaction_type] in interactions.iterrows():
# Check residues are actually in graph
if not (G.has_node(res1) and G.has_node(res2)):
continue
if G.has_edge(res1, res2):
G.edges[res1, res2]["kind"].add(interaction_type)
else:
G.add_edge(res1, res2, kind={interaction_type})
return G
def minimal_startnodes_for_node(spg: nx.Graph,
targetNode: Set[type]) -> Set[Set]:
if spg.has_node(targetNode):
# The targetNode is already part of the spg.
# (because it has been added in one of the update
# steps as an argument set of a computer)
# in which case we can simply return the startnodes
# of the paths leading to it.
path_dict = nx.shortest_path(spg, target=targetNode)
possible_startnodes = frozenset(path_dict.keys())
else:
# Although we do not find the node itself
# we can find the nodes for the single element sets
# of the mvars in the node, since we have built the graph
# starting wiht them. E.g if node {A,B,C} is not part of
# the graph we know at least that the nodes {A}, {B} amd {C}
# are in the graph.
# For each of them we can compute the subgraph of
# spg that leads to it. If we compute the product of these
# subgraphs it will contain the desired node.
# fixme: mm 02-26-2020
# We could make this more efficient by looking for all the
# disjoint unions of the targetNode Mvars and compute
# the product of the graphs leading to the subsets.
# If the subsets are not one element sets we need fewer
# multiplications.
# prod_g=product_graph(*[target_subgraph(spg,frozenset({v})) for v in targetNode])
prod_g = product_graph(*[
minimal_target_subgraph_for_single_var(spg, v) for v in targetNode
])
prod_path_dict = nx.shortest_path(prod_g, target=targetNode)
possible_startnodes = frozenset(prod_path_dict.keys())
def filter_func(n):
# remove every set that contains one of the variables we are looking for ...
return not (any([(v in n) for v in targetNode]))
minimal_startnodes = frozenset(
[n for n in filter(filter_func, possible_startnodes)])
return minimal_startnodes
def _connect_subgraph(G: nx.Graph,
a_nodes: List[int],
b_nodes: List[int],
same=False):
"""Helper function. Connects graph."""
# G must contain all nodes in a_nodes and b_nodes
assert all([G.has_node(node) for node in a_nodes + b_nodes])
for idx in tqdm(range(len(a_nodes)), "Discovering edges"):
n_name = a_nodes[idx]
n_data = G.nodes()[n_name]
this = n_data.get('shape')
has_connection = False
# if a_nodes == b_nodes, don't need to compare anything in b_nodes[:i] to a_nodes[i]
for o_name in b_nodes[idx:] if same else b_nodes:
o_data = G.nodes()[o_name]
other = o_data.get('shape')
if this is not other and this.touches(other):
has_connection = True
border = this.intersection(other)
if border.length == 0.0:
continue
G.add_edge(n_name, o_name, border=border.length)
if same and not has_connection:
# if this node is marooned, connect it to the closest object
sequence = [node for node in a_nodes if node != n_name]
if not sequence:
continue
closest = min([node for node in a_nodes if node != n_name],
key=lambda o_name, t=this: t.centroid.distance(
G.nodes()[o_name]['shape'].centroid))
G.add_edge(n_name, closest, border=0.0)
class Network(HasTraits):
""" The implementation of the Connectome Networks """
implements(INetwork)
# Network ID, from parsed GraphML the graphid
networkid = ''
# Network name
networkname = Str
# network name as seen in the TreeView
name = Str
# Is it an hierarchical network?
hierarchical = CBool(False)
# TODO: later, also Hypergraph?!
# see: http://www.ploscompbiol.org/article/info%3Adoi%2F10.1371%2Fjournal.pcbi.1000385
hypergraph = CBool(False)
# Directionality of the Network, {True: 'directed', False: 'undirected'}
directed = CBool(False)
# metadata for the network
metadata = Dict
# NodeKeys from the parsed GraphML
# These are Dict of Dict, all having strings
nodekeys = {}
# Edgekeys, from parsed GraphML
edgekeys = {}
# A NetworkX AttrGraph containing all the information
graph = Any
# Surface containers
surfaces = List(ISurfaceContainer)
# Surface containers loaded
surfaces_loaded = List(ISurfaceContainer)
# Volume data
volumes = List(IVolume)
# Track data
tracks = List(ITrackfile)
# is this network active, and thus a render manager displayed?
active = Bool
# the render manager of this network
rendermanager = Instance(RenderManager)
# DatasourceManager Instance of this network
datasourcemanager = Instance(DatasourceManager)
# private traits
###########
# parent cfile this networks belongs to
_parentcfile = Any
# filezip of cfile
_filezip = DelegatesTo('_parentcfile')
# edge parameters for visualization
_edge_para = Instance(EdgeParameters)
# View
traits_view = View(
Item('networkname', style = 'readonly'),
Item('hierarchical', style = 'simple'),
Item('hypergraph', style = 'simple'),
Item('directed', style = 'simple'),
Item('active', style = 'simple'),
title = 'A network',
)
def __init__(self, name, src = None, directed = '0', pickled_graph = None, \
hierarchical ='0', hypergraph = '0', graph = None):
""" Initializes the network and sets the traits.
Parameters
----------
name : string
the name of the network
src : file handle or StringIO object
the source text of the network to parse
pickled_graph : NetworkX graph
reference to a graph object, src should be None
directed : bool
Is the network directed?
hierarchical : bool
Is the network hierarchical? (default: '0') Not implemented yet.
hypergraph : bool
Is the network a hypergraph? (default: '0') Not implemented yet.
"""
# initialize the traits
self.networkname = name
self.directed = int(directed)
self.hierarchical = int(hierarchical)
self.hypergraph = int(hypergraph)
if src is None and not pickled_graph is None:
self.load_pickled_graphml(pickled_graph)
else:
if not src is None:
# generates NetworkX Graph
self.graph = self.parse_network_graphml(src)
elif not graph is None:
self.graph = graph
else:
if self.directed:
from networkx import DiGraph
self.graph = DiGraph()
logger.info("Initialize with empty directed Graph")
else:
from networkx import Graph
self.graph = Graph()
logger.info("Initialize with empty undirected Graph")
# initializes the weight key of the graph
# with the first edgekey
if len(self.edgekeys) > 0:
edgk = self.edgekeys.keys()
if not 'weight' in edgk:
self.set_weight_key(edgk[0])
else:
# try grabbing first edge from the graph
if self.graph.number_of_edges() > 0:
it = self.graph.edges_iter(data=True)
edg = it.next()
if len(edg[2]) > 0:
# if it has a weigth key, just leave it
edgk = edg[2].keys()
if not 'weight' in edgk:
self.set_weight_key(edgk[0])
else:
pass
# logger.error('Cannot set weight key for network : ' + self.networkname)
def _name_default(self):
return self.networkname
def _active_default(self):
return False
def _active_changed(self , value):
if value:
n = self.name
if ' [Active]' not in n:
self.name = "%s [Active]" % n
# XXX: do refactor with threaded loading of surfaces
# and default spring force layout for graph rendering!
# see also TraitsUI Demos: Multi thread demo
# load the surface containers data
# make a deep copy of the already loaded surface containers
import copy
self.surfaces = copy.deepcopy(self.surfaces_loaded)
for surfcont in self.surfaces:
surfcont.load_surface_container()
if self.rendermanager is None:
self._create_datasourcemanager()
self._create_renderer()
# if there are no surfaces, initialize
# network rendering, but only if dn_positions are given
if len(self.surfaces) == 0:
logger.debug('No surfaces found. Try to render graph view with dn_position information.')
self.rendermanager.datasourcemanager._compute_3DLayout(-1, -1)
self.rendermanager.visualize_graph()
else:
logger.debug('SurfaceContainer found. Try to render 3D View using %s.' % self.surfaces[0].name)
if len(self.surfaces[0].surfaces) == 0:
logger.debug('Rendering not possible because SurfaceContainer contains no surfaces.')
else:
logger.debug('Using first surface for rendering.')
self.surfaces[0].surfaces[0]._layout_3DView()
if not self._parentcfile._workbenchwin is None:
#from enthought.pyface.timer.api import do_later
from enthought.pyface.api import GUI
GUI.invoke_later(self._parentcfile._workbenchwin.status_bar_manager.set, message = '')
else:
self.name = self.name.replace(' [Active]', '')
logger.debug('Close RenderManager scenes')
self.rendermanager.close_scenes()
logger.debug('All scenes closed.')
# FIXME: what is happening in the following?
# e.g. for instances. e.g. reset traits?
# XXX: this is somehow not correct. do i need to use del
# or remove/reset traits?
self.rendermanager = None
self.datasourcemanager = None
self.surfaces = []
def _de_activate(self):
""" Toggles the internal state of the activation """
if self.active:
self.active = False
else:
self._parentcfile._workbenchwin.status_bar_manager.message = 'Activating network ...'
self.active = True
def _edge_parameters(self):
""" Dialog to change edge attribute and thresholding """
if self._edge_para is None:
self._edge_para = EdgeParameters(self, self.rendermanager.attract.point_scalars_name)
self._edge_para.configure_traits()
def _create_renderer(self):
""" Creates the renderer instance if not yet available
and opens the scenes in mayavi """
if self.active:
if self.rendermanager is None:
logger.debug('Create a RenderManager instance')
self.rendermanager = RenderManager(network=self)
else:
logger.debug('RenderManager instance already running. This is an error.')
def _create_datasourcemanager(self):
""" Creates the datasource manager instance if not yet available """
if self.active:
if self.datasourcemanager is None:
logger.debug('Create a DatasourceManager instance')
self.datasourcemanager = DatasourceManager(network=self)
else:
logger.debug('DatasourceManager instance already running. This is an error.')
def _render_matrix(self):
""" Invokes the connectivity matrix viewer """
# assume the network is activated (i.e. data source generated)
# we need the edge parameter instance initialized
if self._edge_para is None:
self._edge_para = EdgeParameters(self, self.rendermanager.attract.point_scalars_name)
logger.debug('Invoke Matrix Viewer...')
self.rendermanager.invoke_matrix_viewer()
def _trackvis_launch(self):
""" Generates scene file and launch Trackvis on the selected nodes """
import tempfile
logger.debug('Starting TrackVis ...')
# extract selected subgraph
selectionlist = self.get_selectiongraph_list()
if len(selectionlist) == 0:
# message
from enthought.traits.ui.message import message
message(message = 'No nodes selected for ROI creation!', title = 'Infomessage', buttons = [ 'OK' ], parent = None)
tmpgraph = self.graph.subgraph(selectionlist)
# extract trackfile temporarily
if len(self.tracks) == 0:
logger.info('No trackfile found to invoke Trackvis.')
return
else:
# load the first trackfile
trackfname = self.tracks[0].load_trackfile_to_file()
# find the first valid segmentation volume in the self.volumes list
for vol in self.volumes:
if vol.segmentation:
logger.debug('Found a segmentation volume file. Assume labels are corresponding.')
volumefname = vol.load_volume_to_file()
break
# generate the scene file in the temporary folder
tmpscenefile=tempfile.mkstemp(prefix='tmp', suffix='.scene')
# generate trackfile
generate_scene_file(scenefname=tmpscenefile[1], \
trackfname = trackfname, \
volumefname = volumefname, \
selectiongraph = tmpgraph)
# execute trackvis in a thread
pref = preference_manager.preferences
action = ThreadedTrackvis(tvpath = pref.get('cviewer.plugins.ui.trackvispath'), \
fname = tmpscenefile[1], \
trkfname = trackfname,\
volfname = volumefname)
action.start()
def add_surface_container(self, surfacecontainer):
""" Add a surface container to the loaded list
Parameters
----------
surfacecontainer : `ISurfaceContainer` instance
a surface container object
"""
surfacecontainer._networkref = self
self.surfaces_loaded.append(surfacecontainer)
def add_volume(self, volume):
""" Adds a volume to the volumes list
Parameters
----------
volume : `IVolume` instance
a volume object
"""
self.volumes.append(volume)
def add_trackfile(self, trackfile):
""" Adds a trackfile to the tracks list
Parameters
----------
trackfile : `ITrackfile` instance
a trackfile of type ITrackfile
"""
self.tracks.append(trackfile)
def unselect_all(self):
""" Unselects every node in the current network """
if self.datasourcemanager is None:
raise Exception('No DatasourceManager. You have to first activate the network and render it.')
from numpy import array
# get all the nodes
graphnodes = self.datasourcemanager._srcobj.relabled_graph.nodes()
# and unselect all nodes
self.rendermanager._select_nodes(selection_node_array = array(graphnodes))
def select_all(self):
""" Selects all nodes in the current network """
if self.datasourcemanager is None:
raise Exception('No DatasourceManager. You have to first activate the network and render it.')
from numpy import array
# get all the nodes
graphnodes = self.datasourcemanager._srcobj.relabled_graph.nodes()
# and select all nodes
self.rendermanager._select_nodes(selection_node_array = array(graphnodes), activate = True)
def set_selectiongraph(self, sellist, activate = False):
""" Sets the selected nodes in the network to active.
Parameters
----------
sellist : array_like
a list of nodeids conforming to the NetworkX node id
activate : boolean
set the selectionlist nodes to activated?
"""
from numpy import array, int16
graphnodes = self.graph.nodes(data=False)
if self.rendermanager is None:
raise Exception('No RenderManager. You have to first activate the network and render it.')
if len(sellist) == 0:
self.unselect_all()
return
from numpy import array, append
tmparr = array([])
for node in sellist:
# check if it is a valid graph node id
if node in graphnodes:
# get the node id as integer
j = int(node.lstrip('n'))-1
# extend empty array with node id
tmparr = append(tmparr, j)
self.rendermanager._select_nodes(selection_node_array = array(tmparr, dtype = int16), activate = activate)
def get_selectiongraph_list(self):
""" Returns a list of the node ids that were selected in
the rendered scene.
"""
if self.datasourcemanager is None:
raise Exception('No DatasourceManager. You have to first activate the network and render it.')
import numpy as np
sel_list = []
if not self.active:
return sel_list
selnodesarray = self.datasourcemanager._srcobj.selected_nodes
# array with indices where the nodes are selected (==1)
idx = np.where(selnodesarray == 1)[0]
for i in idx:
sel_list.append('n' + str(i + 1))
return sel_list
def set_weight_key(self, weight_key = None):
""" Sets the weight key in the graph representation of the network.
Parameters
----------
weight_key : Str
Must be a possible existing edge key
"""
if not weight_key is None:
for u,v,d in self.graph.edges(data=True):
self.graph[u][v]['weight']=d[weight_key]
return True
else:
return False
def get_matrix(self, weight_key = None):
""" Returns the connectivity matrix of the network with the nodes
ordered according to their id in the GraphML file.
Parameters
----------
weight_key : Str
Possible key value of the edges
Returns
-------
matrix : `Numpy.array` instance
The connectivity matrix
"""
nr_nodes = len(self.graph.nodes())
if not weight_key is None:
#FIXME: sanity check if weight_key exists
# thanks to Aric Hagberg
for u,v,d in self.graph.edges(data=True):
self.graph[u][v]['weight']=d[weight_key]
nodes = [(lambda nmod:'n'+str(nmod))(node) for node in range(1,nr_nodes + 1)]
from networkx import to_numpy_matrix
return to_numpy_matrix(self.graph, nodelist = nodes)
def toggle_surface(self):
""" Toggle the surface for the selected network nodes """
if self.rendermanager is None:
raise Exception('No RenderManager. You have to first activate the network and render it.')
self.rendermanager._toggle_surface()
def show_surface(self):
""" Shows the surface for the selected network nodes """
if self.rendermanager is None:
raise Exception('No RenderManager. You have to first activate the network and render it.')
self.rendermanager._show_surface()
def load_pickled_graphml(self, graph):
""" Loads a pickled GraphML file
Parameters
----------
graph : NetworkX Graph instance
A graph instance
"""
# setting the graph
self.graph = graph
if self.graph.has_node('n0'):
if self.graph.node['n0'].has_key('nodekeys'):
# extracting the node keys from the first node
self.nodekeys = self.graph.node['n0']['nodekeys']
# extracting the edge keys from the first edge (without explanation)
if self.graph.node['n0'].has_key('edgekeys'):
self.edgekeys = self.graph.node['n0']['edgekeys']
if self.graph.node['n0'].has_key('graphid'):
self.networkid = self.graph.node['n0']['graphid']
# remove node
self.graph.remove_node('n0')
def _return_default_edgevalue(self, edgekeys, key):
""" Looks up if there is a default value defined, otherwise
return zero """
if edgekeys[key].has_key('default'):
return float(edgekeys[key]['default'])
else:
return 0.0
def parse_network_graphml(self, path):
""" Read network in GraphML format from a path.
Parameters
----------
path : string
path the the GraphML file
Returns
-------
graph : NetworkX `Graph`
"""
import networkx as nx
from networkx.utils import _get_fh
from lxml import etree
# Return a file handle for given path.
# Path can be a string or a file handle.
# Attempt to uncompress/compress files ending in .gz and .bz2.
fh=_get_fh(path,mode='r')
tree = etree.parse(fh)
# get the root node from parsed lxml
root = tree.getroot()
# Schema Validation
# http://codespeak.net/lxml/validation.html#xmlschema
# define the namespace prefixes
nsprefix = "{%s}" % root.nsmap[None]
nsxlink = "{%s}" % root.nsmap['xlink']
nodekeys = {}
edgekeys = {}
defaultDirected = [True]
# Parse the KEYs
for child in root.iterchildren():
if child.tag == (nsprefix+'key'):
attribs = child.attrib
ddkeys = {}
for mchildren in child:
if mchildren.tag == (nsprefix+'default'):
ddkeys['default'] = mchildren.text
elif mchildren.tag == (nsprefix+'desc'):
ddkeys['desc'] = mchildren.text
if child.attrib['for'] == 'node':
# Parse all the node keys
# Read in the description and the default (if existing)
# dict of dicts for nodes: key1: the id; key2: rest: attr.name, attr.type, desc, default
nodekeys[attribs['id']] = {'attr.name' : attribs['attr.name'], \
'attr.type' : attribs['attr.type']}
# add default/desc keys if existing
nodekeys[attribs['id']] = ddkeys
elif child.attrib['for'] == 'edge':
# Parse all the edge keys
# Read in the description and the default (if existing)
# dict of dicts for edges: key1: the id; key2: rest: attr.name, attr.type, desc, default
edgekeys[attribs['id']] = {'attr.name' : attribs['attr.name'], \
'attr.type' : attribs['attr.type']}
# add default/desc keys if existing
edgekeys[attribs['id']] = ddkeys
else:
logger.error("The 'for' attribute of key-tag not known, must be either node or edge")
elif child.tag == (nsprefix+'graph'):
# start parsing the graph into networkx data structure
# create graph depending on (either AttrGraph or AttrDiGraph)
# directionality: undirected/directed
# version of networkx:
# contains self-loops
# edges have dicts
# data per graph/node/edge
for attr, value in child.items():
if attr == 'edgedefault' and value == 'undirected':
defaultDirected[0] = False
elif attr == 'id':
graphid = value
if defaultDirected[0]:
G = nx.DiGraph()
else:
G = nx.Graph()
# add id, nodekeys and edkeys as traits
self.networkid = graphid
self.nodekeys = nodekeys
self.edgekeys = edgekeys
# iterate over all nodes and edges
for children in child.iterchildren():
if children.tag == (nsprefix+'node'):
# parse the node
for attr, value in children.items():
if attr == 'id':
# add the node with corresponding id
G.add_node(value)
# keep node id to store attributes
nodeid = value
elif attr == (nsxlink+'href'):
# add xlink to node dictionary
G.node[nodeid]['xlink'] = value
else:
# node attribute not known
logger.warning('The following node attribute is not known and thus discarded:'+ attr + ':' + value)
# parse node data, add to node dict
for data in children.iterchildren():
# read the keylabel, i.e. the data attribute name
keylabel = data.attrib['key']
# is the keylabel in the list of allowed keys
if nodekeys.has_key(keylabel):
if not data.text == '':
# add data to the node's dict
G.node[nodeid][keylabel] = data.text
else:
# no data available, check if default value exists
if nodekeys[keylabel].has_key('default'):
# add default data to the node's dict
G.node[nodeid][keylabel] = nodekeys[keylabel]['default']
logger.debug('Added default value '+ keylabel + ':' + nodekeys[keylabel]['default'])
else:
logger.warning('Nor data nor default value defined for ' + keylabel)
# TODO: Work with exceptions!
else:
logger.warning("Data entry with key " + keylabel + " not defined.")
elif children.tag == (nsprefix+'edge'):
# parse the edge
# parse its attributes
for attr, value in children.items():
if attr == 'id':
# no usage of edge id
# add the edge with corresponding id
src = children.attrib['source']
tar = children.attrib['target']
G.add_edge(src, tar)
# keep dest and tar id to store attributes
srcid = src
tarid = tar
elif attr == (nsxlink+'href'):
# add xlink to edge dictionary
G.edge[srcid][tarid]['xlink'] = value
# parse data, and add to the edge dict
for data in children.iterchildren():
# read the keylabel, i.e. the data attribute name
keylabel = data.attrib['key']
# is the keylabel in the list of allowed keys
if self.edgekeys.has_key(keylabel):
if not data.text == '':
# add data to the edge's dict, assume float!!
G.edge[srcid][tarid][keylabel] = float(data.text)
else:
# no data available, check if default value exists
G.edge[srcid][tarid][keylabel] = self._return_default_edgevalue(self.edgekeys, keylabel)
data_keys = G.edge[srcid][tarid].keys()
# check if we missed some edge keys that are available in the header
for k, v in self.edgekeys.items():
if not k in data_keys:
G.edge[srcid][tarid][k] = self._return_default_edgevalue(self.edgekeys, k)
# return the generated network graph
return G
def _settlement_location_available(board: Graph, position: PointCoordinate):
return board.has_node(position) \
and board.nodes[position]["building"] == BuildingTypes.Empty \
and _on_land(board, position)
def clique_merge(graph: nx.Graph, report=False) -> nx.Graph:
"""
Builds up cliques using the `same_as` attribute of each node. Uses those
cliques to build up a mapping for relabelling nodes. Chooses labels so as
to preserve the original nodes, rather than taking xrefs that don't appear
as nodes in the graph.
This method will also expand the `same_as` attribute of the nodes to
include the discovered clique.
"""
original_size = len(graph)
print('original graph has {} nodes'.format(original_size))
cliqueGraph = nx.Graph()
with click.progressbar(
graph.nodes(data=True),
label='building cliques from same_as node property') as bar:
for n, attr_dict in bar:
if 'same_as' in attr_dict:
for m in attr_dict['same_as']:
cliqueGraph.add_edge(n, m)
with click.progressbar(graph.edges(data=True),
label='building cliques from same_as edges') as bar:
for u, v, attr_dict in bar:
if 'edge_label' in attr_dict and attr_dict[
'edge_label'] == 'same_as':
cliqueGraph.add_edge(u, v)
edges = []
with click.progressbar(cliqueGraph.edges(),
label='Breaking invalid cliques') as bar:
for u, v in bar:
try:
u_categories = graph.node[u].get('category', [])
v_categories = graph.node[v].get('category', [])
except:
continue
l = len(edges)
for a in u_categories:
if len(edges) > l:
break
if get_toolkit().get_element(a) is None:
continue
for b in v_categories:
if get_toolkit().get_element(b) is None:
continue
a_ancestors = get_toolkit().ancestors(a)
b_ancestors = get_toolkit().ancestors(b)
if a_ancestors == b_ancestors == []:
continue
elif a not in b_ancestors and b not in a_ancestors:
edges.append((u, v))
break
print('breaking {} many edges'.format(len(edges)))
cliqueGraph.remove_edges_from(edges)
mapping = {}
connected_components = list(nx.connected_components(cliqueGraph))
print('Discovered {} cliques'.format(len(connected_components)))
with click.progressbar(connected_components,
label='building mapping') as bar:
for nodes in bar:
nodes = list(nodes)
categories = set()
for n in nodes:
if not graph.has_node(n):
continue
attr_dict = graph.node[n]
attr_dict['same_as'] = nodes
if 'category' in attr_dict:
categories.update(listify(attr_dict['category']))
if 'categories' in attr_dict:
categories.update(listify(attr_dict['categories']))
list_of_prefixes = []
for category in categories:
try:
list_of_prefixes.append(
get_toolkit().get_element(category).id_prefixes)
except:
pass
nodes.sort()
nodes.sort(key=build_sort_key(list_of_prefixes))
for n in nodes:
if n != nodes[0]:
mapping[n] = nodes[0]
g = relabel_nodes(graph, mapping)
edges = []
for u, v, key, data in g.edges(keys=True, data=True):
if data.get('edge_label') == 'same_as':
edges.append((u, v, key))
g.remove_edges_from(edges)
for n, data in g.nodes(data=True):
data['iri'] = expand_uri(n)
if 'id' in data and data['id'] != n:
data['id'] = n
if 'same_as' in data and n in data['same_as']:
data['same_as'].remove(n)
if data['same_as'] == []:
del data['same_as']
final_size = len(g)
print('Resulting graph has {} nodes'.format(final_size))
print('Eliminated {} nodes'.format(original_size - final_size))
return g
class WaypointServer:
def __init__(self):
self.server = InteractiveMarkerServer("waypoint_server")
self.waypoint_graph = Graph()
self.next_waypoint_id = 0
self.next_edge_id = 0
self.state = STATE_REGULAR
self.connect_first_marker = ""
self.edge_line_publisher = rospy.Publisher("~edges",
MarkerArray,
queue_size=10)
self.marker_frame = rospy.get_param("~marker_frame", "map")
self.uuid_name_map = {}
self.removeService = rospy.Service('~remove_edge', RemoveEdge,
self.remove_edge_service_call)
self.loadService = rospy.Service('~load_waypoints', LoadWaypoints,
self.load_waypoints_service)
self.saveService = rospy.Service('~save_waypoints', SaveWaypoints,
self.save_waypoints_service)
self.getWaypointGraphService = rospy.Service(
'~get_waypoint_graph', GetWaypointGraph,
self.get_waypoint_graph_service_call)
rospy.Subscriber("/clicked_point", PointStamped,
self.insert_marker_callback)
rospy.on_shutdown(self.clear_all_markers)
rospy.logwarn(
"The waypoint server is waiting for RViz run and to subscribe to {0}."
.format(rospy.resolve_name("~edges")))
while self.edge_line_publisher.get_num_connections() == 0:
rospy.sleep(1.0)
self.clear_all_markers()
load_file = rospy.get_param("~waypoint_file",
"") # yaml file with waypoints to load
if len(load_file) != 0:
rospy.loginfo(
"Waypoint_Server is loading initial waypoint file {0}.".format(
load_file))
server.load_waypoints_from_file(load_file)
def insert_marker_callback(self, pos):
rospy.logdebug(
"Inserting new waypoint at position ({0},{1},{2}).".format(
pos.point.x, pos.point.y, pos.point.z))
self.insert_marker(pos.point)
def clear_all_markers(self):
edges = MarkerArray()
marker = Marker()
marker.header.stamp = rospy.Time.now()
marker.header.frame_id = self.marker_frame
marker.ns = "waypoint_edges"
marker.id = 0
marker.action = Marker.DELETEALL
edges.markers.append(marker)
self.edge_line_publisher.publish(edges)
self.waypoint_graph.clear()
def _make_marker(self, msg):
marker = Marker()
marker.type = Marker.SPHERE
marker.scale.x = msg.scale * 0.45
marker.scale.y = msg.scale * 0.45
marker.scale.z = msg.scale * 0.45
marker.color.r = 0
marker.color.g = 1
marker.color.b = 0
marker.color.a = 1.0
return marker
def _make_edge(self, scale, begin, end):
marker = Marker()
marker.header.frame_id = self.marker_frame
marker.header.stamp = rospy.Time.now()
marker.ns = "waypoint_edges"
marker.id = self.next_edge_id
self.next_edge_id += 1
marker.type = Marker.LINE_LIST
marker.action = Marker.ADD
marker.scale.x = scale * 0.45
marker.scale.y = scale * 0.45
marker.scale.z = scale * 0.45
marker.color.r = 0
marker.color.g = 0
marker.color.b = 1
marker.color.a = 1.0
marker.points.append(begin)
marker.points.append(end)
return marker
def _remove_marker(self, name):
self._clear_marker(name)
self.waypoint_graph.remove_node(name)
del self.uuid_name_map[name]
self.server.erase(name)
self.server.applyChanges()
def _clear_marker(self, name):
# remove all edges to a waypoint
edges = MarkerArray()
to_remove = []
for u, v, marker in self.waypoint_graph.edges(name, data='marker'):
marker.action = Marker.DELETE
to_remove.append((u, v, marker))
# update knowledge database to remove all edges
for u, v, marker in to_remove:
edges.markers.append(marker)
self.waypoint_graph.remove_edge(u, v)
self.edge_line_publisher.publish(edges) # publish deletion
def _connect_markers(self, u, v, cost=0.0):
if self.waypoint_graph.has_edge(u, v):
# remove edge
edges = MarkerArray()
marker = self.waypoint_graph.get_edge_data(u, v)["marker"]
marker.action = Marker.DELETE
edges.markers.append(marker)
self.waypoint_graph.remove_edge(u, v)
self.edge_line_publisher.publish(edges) # publish deletion
else:
name_u = self.uuid_name_map[u]
name_v = self.uuid_name_map[v]
u_pos = self.server.get(u).pose.position
v_pos = self.server.get(v).pose.position
# insert edge
marker = self._make_edge(0.2, u_pos, v_pos)
marker.text = str(cost) if cost is not 0 else ""
# insert edge into graph
self.waypoint_graph.add_edge(u,
v,
u=u,
v=v,
cost=cost,
marker=marker)
self.update_edges()
def update_edges(self):
edges = MarkerArray()
for u, v, marker in self.waypoint_graph.edges(data='marker'):
edges.markers.append(marker)
self.edge_line_publisher.publish(edges)
def process_feedback(self, feedback):
if feedback.event_type == InteractiveMarkerFeedback.MENU_SELECT:
handle = feedback.menu_entry_id
if handle == MENU_CONNECT:
self.state = STATE_CONNECT
self.connect_first_marker = feedback.marker_name
elif handle == MENU_CLEAR:
self._clear_marker(feedback.marker_name)
elif handle == MENU_REMOVE:
self._remove_marker(feedback.marker_name)
elif feedback.event_type == InteractiveMarkerFeedback.MOUSE_UP:
if self.state == STATE_CONNECT:
self.state = STATE_NONE
self._connect_markers(self.connect_first_marker,
feedback.marker_name)
elif self.state == STATE_NONE:
pass # ignore
else:
pos = feedback.pose.position
rospy.logdebug(
"Updateing pose of marker {3} to ({0},{1},{2})".format(
pos.x, pos.y, pos.z, feedback.marker_name))
# update database
# push to scene database
pstamped = PoseStamped()
pstamped.header.frame_id = self.marker_frame
pstamped.pose = feedback.pose
elif feedback.event_type == InteractiveMarkerFeedback.MOUSE_DOWN:
if self.state == STATE_NONE:
self.state = STATE_REGULAR
self.server.applyChanges()
def move_feedback(self, feedback):
pose = feedback.pose
self.server.setPose(feedback.marker_name, pose)
# correct all edges
for u, v, data in self.waypoint_graph.edges([feedback.marker_name],
data=True):
if feedback.marker_name == data["u"]:
data["marker"].points[0] = pose.position
else:
data["marker"].points[1] = pose.position
self.update_edges()
self.server.applyChanges()
def insert_marker(self, position, name=None, uuid=None, frame_id=None):
if frame_id is None:
frame_id = self.marker_frame
if uuid is None:
uuid = unique_id.fromRandom()
if name is None:
name = "wp{0}".format(self.next_waypoint_id)
self.next_waypoint_id += 1
self.uuid_name_map[str(uuid)] = name
# insert waypoint
int_marker = InteractiveMarker()
int_marker.header.frame_id = frame_id
int_marker.pose.position = position
int_marker.pose.orientation.w = 1
int_marker.scale = 0.5
int_marker.name = str(uuid)
int_marker.description = name
# make markers moveable in the plane
control = InteractiveMarkerControl()
control.orientation.w = 1
control.orientation.x = 0
control.orientation.y = 1
control.orientation.z = 0
control.interaction_mode = InteractiveMarkerControl.MOVE_PLANE
# interactive menu for each marker
menu_handler = MenuHandler()
menu_handler.insert("Connect/Disconnect",
callback=self.process_feedback)
menu_handler.insert("Clear", callback=self.process_feedback)
menu_handler.insert("Remove", callback=self.process_feedback)
# make a box which also moves in the plane
control.markers.append(self._make_marker(int_marker))
control.always_visible = True
int_marker.controls.append(control)
# we want to use our special callback function
self.server.insert(int_marker, self.process_feedback)
menu_handler.apply(self.server, int_marker.name)
# set different callback for POSE_UPDATE feedback
pose_update = InteractiveMarkerFeedback.POSE_UPDATE
self.server.setCallback(int_marker.name, self.move_feedback,
pose_update)
self.server.applyChanges()
# insert into graph
self.waypoint_graph.add_node(str(uuid))
def save_waypoints_service(self, request):
filename = request.file_name
self.save_waypoints_to_file(filename)
rospy.loginfo("Saved waypoints to {0}".format(filename))
return SaveWaypointsResponse()
def load_waypoints_service(self, request):
filename = request.file_name
self.load_waypoints_from_file(filename)
rospy.loginfo("Loaded waypoints from {0}".format(filename))
return LoadWaypointsResponse()
def save_waypoints_to_file(self, filename):
data = {"waypoints": {}, "edges": []}
for uuid in self.waypoint_graph.nodes():
name = self.uuid_name_map[uuid]
pos = self.server.get(uuid).pose.position
data["waypoints"].update(
{uuid: {
"name": name,
"x": pos.x,
"y": pos.y,
"z": pos.z
}})
for u, v, cost in self.waypoint_graph.edges(data='cost'):
data["edges"].append({'u': u, 'v': v, 'cost': cost})
with open(filename, 'w') as f:
yaml.dump(data, f, default_flow_style=False)
def load_waypoints_from_file(self, filename):
self.clear_all_markers()
self.server.clear()
with open(filename, 'r') as f:
data = yaml.load(f)
for uuid, wp in data["waypoints"].items():
point = Point(wp["x"], wp["y"], wp["z"])
self.insert_marker(position=point, uuid=uuid, name=wp['name'])
for edge in data["edges"]:
self._connect_markers(edge['u'], edge['v'], edge['cost'])
def remove_edge_service_call(self, request):
if self.waypoint_graph.has_edge(request.u, request.v):
self._connect_markers(request.u, request.v)
return RemoveConnectionResponse()
def get_waypoint_graph_service_call(self, request):
ret = GetWaypointGraphResponse()
for r in request.waypoints:
if not self.waypoint_graph.has_node(str(r)):
raise rospy.ServiceException("invalid waypoint {:}".format(r))
if len(request.waypoints) == 0:
graph = self.waypoint_graph
else:
graph = self.waypoint_graph.subgraph(request.waypoints)
for node in graph.nodes():
pose = self.server.get(node).pose
ret.names.append(node)
ret.positions.append(pose)
for u, v in graph.edges():
e = Edge()
e.source = u
e.target = v
ret.edges.append(e)
return ret
def neighbourhood(graph : nx.Graph, vs):
if graph.has_node(vs):
return set(graph.neighbors(vs))
return union([set(graph.neighbors(v)) for v in vs] + [set()])
def handle(self, *args, **kwargs):
print(datetime.today(), "Building dashboard...")
pages = {os.path.basename(x).rstrip('.md'): open(x, 'r').read() for x in glob.glob(Filepaths.BACKUP_DIR.value) if os.path.isfile(x)}
graph = Graph()
flashcards: List[AnkiFlashcard] = []
full_deck_names = make_anki_request('deckNames')
# Create nodes
for page_title, page in pages.items():
lines = squeeze(page.split('\n'))
if not len(lines):
continue
for page_type in [Tags.QUESTION.value, Tags.REFERENCE.value, Tags.NOTE.value, Tags.POST.value]:
matches = [x for x in lines if f'#{page_type}' in x]
content, metadata = partition(lambda x: '::' in x, matches)
for line in content:
# This node is a single line.
graph.add_node(line.strip().lstrip('- '), type=page_type, word_count=len(line.split(' ')))
if len(metadata):
# This node is a page.
graph.add_node(page_title, lines=lines, type=page_type)
# Find flashcards
bulleted_lines = get_bulleted_lines(lines)
num_bulleted_lines = len(bulleted_lines)
for i, line in enumerate(bulleted_lines):
if f"#{Tags.FLASHCARD.value}" in line:
well_formatted_flashcard = re.match(FLASHCARD_FRONT_REGEX, line)
if not well_formatted_flashcard:
post_slack(f"Flashcard front improperly formatted: {line}")
continue
front_1, flashcard_uuid, front_2 = well_formatted_flashcard.groups()
front = front_1.strip() + " " + front_2.strip()
# Handle Cloze-style flashcards
is_cloze = re.search(CLOZE_REGEX, front)
if is_cloze:
front = re.sub(CLOZE_REGEX, lambda x: "{{c1::" + x.group(1) + "}}", front)
front = AnkiFlashcard.maybe_download_media_files_and_convert(front) + f"<hr/><div>{page_title}</div>"
maybe_deck_name, line = extract_tags(ANKI_DECK_TAG_REGEX, line)
deck_name = get_full_deck_name(full_deck_names, maybe_deck_name)
flashcards.append(ClozeFlashcard(uuid=flashcard_uuid, front=front, deck=deck_name))
continue
is_num_lines_long_enough = i + 1 < num_bulleted_lines
if not is_num_lines_long_enough:
post_slack(f"Flashcard is missing a back: {line}")
continue
# Handle two-sided flashcards
next_line = bulleted_lines[i + 1]
next_line_is_indented = re.match(BULLET_REGEX, line).span()[1] < re.match(BULLET_REGEX, next_line).span()[1]
if next_line_is_indented:
back = re.match(SANS_BULLET_REGEX, next_line).group(1).strip()
front = AnkiFlashcard.maybe_download_media_files_and_convert(front)
back = AnkiFlashcard.maybe_download_media_files_and_convert(back) + f"<hr/><div>{page_title}</div>"
maybe_deck_name, line = extract_tags(ANKI_DECK_TAG_REGEX, line)
deck_name = get_full_deck_name(full_deck_names, maybe_deck_name)
flashcards.append(TwoSidedFlashcard(uuid=flashcard_uuid, front=front, back=back, deck=deck_name))
continue
post_slack(f"Flashcard improperly formatted (next line not indented): {line}\n{next_line}")
try:
make_anki_request('sync')
except Exception:
print("Waiting on sync...")
time.sleep(5)
try:
make_anki_request('sync')
except Exception:
print("Waiting on sync...")
time.sleep(5)
make_anki_request('sync')
# POST flashcards to AnkiConnect
for flashcard in flashcards:
query = f"deck:{flashcard.deck} ID:{flashcard.uuid}"
anki_ids = make_anki_request('findNotes', query=query)
if len(anki_ids) == 0:
make_anki_request('addNote', note=flashcard.to_add_note_json())
elif len(anki_ids) == 1:
make_anki_request('updateNoteFields', note=flashcard.to_update_note_fields_json(anki_ids[0]))
else:
raise AssertionError(f"{len(anki_ids)} number of cards returned for query {query}")
make_anki_request('sync')
num_edges = 0
word_count = 0
num_shares = 0
# Add information to nodes
for title, data in graph.nodes(data=True):
if 'lines' not in data:
continue
# Edges
links = flatten([parse_links(x) for x in data['lines']])
for link in [x for x in links if graph.has_node(x)]:
graph.add_edge(link, title)
num_edges += 1
# Word count
word_count += sum([len(x.split(' ')) for x in data['lines']]) # TODO FIXME
# Shares
num_shares += len(flatten([parse_shares(x) for x in data['lines'] if f"{Tags.SHARES.value}::" in x]))
# Save num_edges
with open(Filepaths.NUM_EDGES_FILEPATH.value, "r+") as f:
line = f"\n{datetime.today()} {num_edges}"
if line not in f.read():
f.write(line)
# Save word count
with open(Filepaths.WORD_COUNT_FILEPATH.value, "r+") as f:
line = f"\n{datetime.today()} {word_count}"
if line not in f.read():
f.write(line)
# Save shares
with open(Filepaths.NUM_SHARES_FILEPATH.value, "r+") as f:
line = f"\n{datetime.today()} {num_shares}"
if line not in f.read():
f.write(line)
# Count edges
for title, data in graph.nodes(data=True):
data['num_edges'] = len(graph[title])
# Build index.html
nodes = list(graph.nodes(data=True))
questions = [(title, data) for title, data in nodes if data['type'] == Tags.QUESTION.value]
references = [(title, data) for title, data in nodes if data['type'] == Tags.REFERENCE.value]
notes = [(title, data) for title, data in nodes if data['type'] == Tags.NOTE.value]
posts = [(title, data) for title, data in nodes if data['type'] == Tags.POST.value]
template = Template(open(Filepaths.DASHBOARD_HTML.value).read())
last_updated = datetime.fromtimestamp(os.path.getmtime(glob.glob(Filepaths.BACKUP_DIR.value)[0]))
with open(Filepaths.INDEX_HTML.value, "w") as f:
f.write(template.render(
questions=questions,
references=references,
notes=notes,
posts=posts,
words_metric=get_words_metric(),
connections_metric=get_connections_metric(),
shares_metric=get_shares_metric(),
last_updated=last_updated,
last_built=open(Filepaths.LAST_BUILT.value).read()
))
print(datetime.today(), "Done building dashboard!")
class Network(object):
def __init__(self):
# All known IP:port addresses
self._all_addresses = {}
# All verified Peer objects (Peer.address must be in _all_addresses)
self.verified_peers = []
# The networkx graph containing the addresses and peers
self.graph = Graph()
self.graph_lock = RLock()
# Peers we should not add to the network
# For example, bootstrap peers
self.blacklist = []
# Excluded mids
self.blacklist_mids = []
# Map of advertised services (set) per peer
self.services_per_peer = {}
# Map of service identifiers to local overlays
self.service_overlays = {}
def discover_address(self, peer, address):
"""
A peer has introduced us to another IP address.
:param peer: the peer that performed the introduction
:param address: the introduced address
"""
if address in self.blacklist:
self.add_verified_peer(peer)
return
self.graph_lock.acquire()
if (address not in self._all_addresses) or (not self.graph.has_node(
self._all_addresses[address])):
# This is a new address, or our previous parent has been removed
self._all_addresses[address] = b64encode(peer.mid)
if not self.get_verified_by_address(
address) and not self.graph.has_edge(
self._all_addresses[address], address):
# Don't remap already verified peers and don't add an edge which already exists.
if address in self.graph.node and address not in self.graph.adj:
del self.graph.node[address]
self.graph.add_edge(b64encode(peer.mid), address, color='orange')
self.graph_lock.release()
self.add_verified_peer(peer)
def discover_services(self, peer, services):
"""
A peer has advertised some services he can use.
:param peer: the peer to update the services for
:param services: the list of services to register
"""
self.graph_lock.acquire()
if peer.public_key.key_to_bin() not in self.services_per_peer:
self.services_per_peer[peer.public_key.key_to_bin()] = set(
services)
else:
self.services_per_peer[peer.public_key.key_to_bin()] |= set(
services)
self.graph_lock.release()
def add_verified_peer(self, peer):
"""
The holepunching layer has a new peer for us.
:param peer: the new peer
"""
if peer.mid in self.blacklist_mids:
return
self.graph_lock.acquire()
# This may just be an address update
for known in self.verified_peers:
if known.mid == peer.mid:
known.address = peer.address
self.graph_lock.release()
return
if peer.address in self._all_addresses and self.graph.has_node(
peer.address):
introducer = self._all_addresses[peer.address]
self.graph.remove_node(peer.address)
self.graph.add_node(b64encode(peer.mid))
self.graph.add_edge(introducer, b64encode(peer.mid), color='green')
if peer not in self.verified_peers:
# This should always happen, unless someone edits the verified_peers dict directly.
# This would be a programmer 'error', but we will allow it.
self.verified_peers.append(peer)
elif (peer.address not in self.blacklist):
if peer.address not in self._all_addresses:
self._all_addresses[peer.address] = ''
if not self.graph.has_node(b64encode(peer.mid)):
self.graph.add_node(b64encode(peer.mid))
if peer not in self.verified_peers:
self.verified_peers.append(peer)
self.graph_lock.release()
def register_service_provider(self, service_id, overlay):
"""
Register an overlay to provide a certain service id.
:param service_id: the name/id of the service
:param overlay: the actual service
"""
self.graph_lock.acquire()
self.service_overlays[service_id] = overlay
self.graph_lock.release()
def get_peers_for_service(self, service_id):
"""
Get peers which support a certain service.
:param service_id: the service name/id to fetch peers for
"""
out = []
with self.graph_lock:
for peer in self.verified_peers:
key_bin = peer.public_key.key_to_bin()
if key_bin in self.services_per_peer:
if service_id in self.services_per_peer[key_bin]:
out.append(peer)
return out
def get_services_for_peer(self, peer):
"""
Get the known services supported by a peer.
:param peer: the peer to check services for
"""
with self.graph_lock:
return self.services_per_peer.get(peer.public_key.key_to_bin(),
set())
def get_walkable_addresses(self, service_id=None):
"""
Get all addresses ready to be walked to.
:param service_id: the service_id to filter on
"""
with self.graph_lock:
verified = [peer.address for peer in self.verified_peers]
out = list(set(self._all_addresses.keys()) - set(verified))
if service_id:
new_out = []
for address in out:
b64mid_intro = self._all_addresses[address]
encoded_services_per_peer = {
b64encode(sha1(k).digest()): v
for k, v in self.services_per_peer.iteritems()
}
services = encoded_services_per_peer.get(b64mid_intro, [])
if service_id in services:
new_out.append(address)
out = new_out
return out
def get_verified_by_address(self, address):
"""
Get a verified Peer by its IP address.
:param address: the (IP, port) tuple to search for
:return: the Peer object for this address or None
"""
self.graph_lock.acquire()
for i in range(len(self.verified_peers)):
if self.verified_peers[i].address == address:
out = self.verified_peers[i]
self.graph_lock.release()
return out
self.graph_lock.release()
def get_verified_by_public_key_bin(self, public_key_bin):
"""
Get a verified Peer by its public key bin.
:param public_key_bin: the string representation of the public key
:return: the Peer object for this public_key_bin or None
"""
self.graph_lock.acquire()
for i in range(len(self.verified_peers)):
if self.verified_peers[i].public_key.key_to_bin(
) == public_key_bin:
out = self.verified_peers[i]
self.graph_lock.release()
return out
self.graph_lock.release()
def get_introductions_from(self, peer):
"""
Get the addresses introduced to us by a certain peer.
:param peer: the peer to get the introductions for
:return: a list of the introduced addresses (ip, port)
"""
with self.graph_lock:
return [
k for k, v in self._all_addresses.iteritems()
if v == b64encode(peer.mid)
]
def remove_by_address(self, address):
"""
Remove all walkable addresses and verified peers using a certain IP address.
:param address: the (ip, port) address to remove
"""
self.graph_lock.acquire()
if address in self._all_addresses:
del self._all_addresses[address]
to_remove = []
for i in range(len(self.verified_peers)):
if self.verified_peers[i].address == address:
to_remove.insert(0, i)
graph_node = b64encode(self.verified_peers[i].mid)
if self.graph.has_node(graph_node):
self.graph.remove_node(graph_node)
key_bin = self.verified_peers[i].public_key.key_to_bin()
if key_bin in self.services_per_peer:
del self.services_per_peer[key_bin]
for index in to_remove:
self.verified_peers.pop(index)
if self.graph.has_node(address):
self.graph.remove_node(address)
self.graph_lock.release()
def remove_peer(self, peer):
"""
Remove a verified peer.
:param peer: the Peer to remove
"""
self.graph_lock.acquire()
if peer.address in self._all_addresses:
del self._all_addresses[peer.address]
if peer in self.verified_peers:
self.verified_peers.remove(peer)
graph_node = b64encode(peer.mid)
if self.graph.has_node(graph_node):
self.graph.remove_node(graph_node)
if self.graph.has_node(peer.address):
self.graph.remove_node(peer.address)
key_bin = peer.public_key.key_to_bin()
if key_bin in self.services_per_peer:
del self.services_per_peer[key_bin]
self.graph_lock.release()
def snapshot(self):
"""
Get a snapshot of all verified peers.
:return: the serialization (str) of all verified peers
"""
with self.graph_lock:
out = ""
for peer in self.verified_peers:
if peer.address and peer.address != ('0.0.0.0', 0):
out += inet_aton(peer.address[0]) + pack(
">H", peer.address[1])
return out
def load_snapshot(self, snapshot):
"""
Load a snapshot into the walkable addresses.
:param snapshot: the snapshot (created by snapshot())
"""
snaplen = len(snapshot)
if (snaplen % 6) != 0:
import logging
logging.error(
"Snapshot has invalid length! Aborting snapshot load.")
return
with self.graph_lock:
for i in xrange(0, snaplen, 6):
sub = snapshot[i:i + 6]
ip = inet_ntoa(sub[0:4])
port = unpack(">H", sub[4:])[0]
self._all_addresses[(ip, port)] = ''
def draw(self, filename="network_view.png"):
"""
Draw this graph to a file, for debugging.
"""
import matplotlib.pyplot as plt
plt.clf()
pos = circular_layout(self.graph)
draw(self.graph,
pos,
with_labels=False,
arrows=False,
hold=False,
edge_color=[
self.graph[u][v]['color'] for u, v in self.graph.edges()
],
node_color=[
'orange' if v in self._all_addresses else 'green'
for v in self.graph.nodes()
])
plt.savefig(filename)
class LangGraph(object):
"""
A graph of all the relationships in a document and/or sentence
"""
def __init__(self, directed=False):
"""
Builds a graph out of the given document
"""
self.isDirected = directed
#a graph that is meant to be full of class Instance
if self.isDirected:
self.graph = DiGraph()
else:
self.graph = Graph()
self.start = None #an Instance
#keep the graph also according to temporal, redundant probably needs
#refactoring
self.temporal = None
self.temporalMap = None
def setStart(self, start):
"""
Sets the starting instance, also builds the temporal ordering
of the graph
"""
self.start = start
self.temporal = self.narrativeOrder()
self.temporalMap = self.narrativeMapping()
def indexToInst(self, index):
"""
Returns the instance corresponding to the given index
"""
result = index
#if the index is an int, lookup the instance associated with it
if type(index) == int:
result = self.temporal[index]
return result
def instToIndex(self, instance):
"""
Return the index associated with the instance
"""
return self.temporalMap[instance]
def narrativeOrder(self):
"""
Returns the instances in narrative order
"""
results = []
node = self.start
prev = None
#while there are more nodes, keep adding them
while node is not None:
#record the current node
results.append(node)
#get the connected nodes
fringe = [n for n in self.adj(node, WORD_EDGE) if n != prev]
nextNode = fringe[0] if fringe else None
#advance to the next node
prev = node
node = nextNode
return results
def narrativeMapping(self):
"""
Makes the mapping from instances to their narrative index
"""
return {inst:i for i,inst in enumerate(self.temporal)}
def addNode(self, node):
"""
Adds a node to the graph
"""
self.graph.add_node(node)
def addEdge(self, start, end, type):
"""
Adds an edge between the two instances
"""
#if the edge exists, just add the type
if self.graph.has_edge(start, end):
self.addType(start, end, type)
else:
self.graph.add_edge(start, end, TYPES=set([type]))
def removeEdge(self, start, end, edgeType):
"""
Removes an edge with a given type from the edge type
"""
#remove the type
self.removeType(start, end, edgeType)
#if there are no types, remove the edge itself
types = self.edgeTypes(start, end)
#remove the edge
if not len(types) and self.graph.has_edge(start, end):
self.graph.remove_edge(start, end)
def addType(self, start, end, type):
"""
Adds a type between the edges
"""
#look for existing types
types = self.graph[start][end].get(TYPES, set())
#add the new type
types.add(type)
self.graph[start][end][TYPES] = types
def removeType(self, start, end, edgeType):
"""
Removes the type on the edge
"""
for prefix in [PARENT, CHILD]:
edgeType = removePrefix(prefix, edgeType)
types = self.graph[start][end][TYPES]
#if the types contains the edge, remove
if edgeType in types:
types.remove(edgeType)
def hasType(self, start, end, type):
"""
Returns true if the edge between the two nodes has the given
type
"""
return type in self.edgeTypes(start, end)
def singleEdgeTypes(self, start, end):
"""
Returns the types on the edge if any, or an empty set is returned
"""
#make sure we are using instances rather than indexes
start = self.indexToInst(start)
end = self.indexToInst(end)
data = self.graph.get_edge_data(start,end)
result = set()
#if there is data, get the types
if data is not None:
result = data.get(TYPES, set())
return result
def edgeTypes(self, start, end):
"""
Returns the types on the edge if any, or an empty set is returned
"""
if self.isDirected:
parent = addPrefixes(PARENT, self.singleEdgeTypes(end, start))
child = addPrefixes(CHILD, self.singleEdgeTypes(start, end))
types = parent.union(child)
else:
types = self.singleEdgeTypes(start, end)
return types
def allEdgeTypes(self):
"""
Returns all the edge types
"""
results = set()
#collect all the edges with all the types
for s,e,types in self.allEdges():
#look up the edge types to make sure everything is covered
for edgeType in types:
results.add(edgeType)
#add in the reverse types
for edgeType in self.edgeTypes(e,s):
results.add(edgeType)
return results
def allEdges(self):
"""
Yield all the edges in the graph
"""
for start, end in self.graph.edges():
yield start, end, self.edgeTypes(start, end)
def contains(self, instance):
"""
Returns true if the graph contains the instance
"""
return self.graph.has_node(instance)
def instances(self):
"""
Return all the instances in the graph
"""
return self.graph.nodes()
def edges(self, instance):
"""
Returns all the edges connected to this instance
"""
inst = self.indexToInst(instance)
#make get the directed edges
if self.isDirected:
results = [t for _, t in self.graph.out_edges(inst)] + [t for t, _ in self.graph.in_edges(inst)]
else:
results = self.graph.adj[inst]
return results
def docType(self):
"""
Returns the document type (String)
"""
return self.temporal[0].event.docType
def adj(self, instance, type=None):
"""
Returns the adjancent node with a given type
"""
return [other for other in self.edges(instance)
if self.hasType(instance, other, type) or type is None]
def nonNarrativeAdj(self, instance, returnIndex=False):
"""
Returns the nodes that are not adjancent to the given instance
"""
results = []
#add each node if it has a non-narrative (temporal) connection
for node in self.edges(instance):
#get the non narrative types
edgeTypes = nonNarrativeTypes(self.edgeTypes(instance, node))
#if there is a non-narrative edge, add it
if edgeTypes:
#lookup the index of the node
nodeMarker = self.instToIndex(node) if returnIndex else node
results.append((nodeMarker, edgeTypes))
return results
def words(self):
"""
Returns the words in narrative order
"""
return [t.word for t in self.tokens()]
def tokens(self):
"""
Returns the tokens in narrative order
"""
return [i.token for i in self.temporal]
def labels(self):
"""
Returns the sequence of labels for the instances
"""
return [i.event.type for i in self.temporal]
def removeAny(self, blackList):
"""
Removes any nodes/tokens/instances that match the words in the blacklist
"""
#if a token or its lemma match any of the words in the blacklist
#mark it for removal
toRemove = {inst.token for inst in self.temporal
if inst.token.word.lower() in blackList
or inst.token.lemma.lower() in blackList}
self.removeNodes(toRemove)
def removeNodes(self, tokens):
"""
Removes the token from the graph
"""
startLen = len(self)
#mark all the instances/indexes to remove
instances = {inst:i for inst,i in self.temporalMap.items()
if inst.token in tokens}
#determine the remaining nodes
remaining = sorted(list(set(range(startLen)) - {i for i in instances.values()}))
#add in all the bypasses
for startIndex, endIndex in iterPairs(remaining):
start = self.temporal[startIndex]
end = self.temporal[endIndex]
self.addEdge(start, end, WORD_EDGE)
#remove the edges
for inst in instances:
self.graph.remove_node(inst)
#if there are remaining nodes then reset the temporal mapping
if remaining:
startIndex = min(remaining)
self.start = self.temporal[startIndex]
#redo narrative order
self.temporal = self.narrativeOrder()
self.temporalMap = self.narrativeMapping()
else:
self.start = None
self.temporal = []
self.temporalMap = {}
def copy(self):
"""
Performs a shallow copy of the graph
"""
newGraph = LangGraph(self.isDirected, self.entEdges)
#create new instances
newInst = {i:me.Instance(copy(i.token), i.event) for i in self.temporal}
#add in all the edges
for start, end in self.graph.edges():
for eType in self.edgeTypes(start, end):
newGraph.addEdge(newInst[start], newInst[end], eType)
newGraph.setStart(newInst[self.start])
return newGraph
def graphString(self):
"""
Returns the graph as a string
"""
return " ".join([t.word for t in self.tokens()])
def __len__(self):
"""
Returns the number of nodes (tokens) in the graph
"""
return len(self.graph)
def __repr__(self):
"""
Returns a summary string of the graph
"""
return "LangGraph {} nodes, {} edges".format(len(self.graph.nodes()), len(self.graph.edges()))
def is_valid_city_position(board: Graph, position: Coordinate, player: int):
return board.has_node(("point", position)) \
and board.nodes[("point", position)]["owner"] == player \
and board.nodes[("point", position)]["building"] == BuildingTypes.Settlement
class Network:
# Initializes the object
def __init__(self, database_name):
self.initialize_network(database_name)
self.find_all_pairs_shortest_route()
# Initializes the network
def initialize_network(self, database_name):
self.network = Graph()
database_connection = connect(database_name)
self.add_stations(database_connection)
self.add_station_zone_assignments(database_connection)
self.add_connections(database_connection)
database_connection.close()
# Adds stations to the network from the database
def add_stations(self, database_connection):
database_cursor = database_connection.cursor()
select_from_station_table_string = \
"SELECT S.name " + \
"FROM station S"
database_cursor.execute(select_from_station_table_string)
database_result_set = database_cursor.fetchall()
for database_result in database_result_set:
station_name = database_result[0]
self.network.add_node(station_name, zones=set())
database_cursor.close()
# Adds station zone assignments to the network from the database
def add_station_zone_assignments(self, database_connection):
database_cursor = database_connection.cursor()
select_from_station_zone_assignment_table_string = \
"SELECT S.name, Z.id " + \
"FROM station_zone_assignment SZA, station S, zone Z " + \
"WHERE SZA.station = S.id AND SZA.zone = Z.id"
database_cursor.execute(
select_from_station_zone_assignment_table_string)
database_result_set = database_cursor.fetchall()
for database_result in database_result_set:
station_name = database_result[0]
station_zone = database_result[1]
self.network.node[station_name]["zones"].add(station_zone)
database_cursor.close()
# Adds connections to the network from the database
def add_connections(self, database_connection):
database_cursor = database_connection.cursor()
select_from_connection_table_string = \
"SELECT S1.name, S2.name, C.distance " + \
"FROM station S1, station S2, connection C " + \
"WHERE station_a = S1.id AND station_b = S2.id"
database_cursor.execute(select_from_connection_table_string)
database_result_set = database_cursor.fetchall()
for database_result in database_result_set:
station_a_name = database_result[0]
station_b_name = database_result[1]
distance_between_stations = database_result[2]
self.network.add_edge(station_a_name,
station_b_name,
weight=distance_between_stations)
database_cursor.close()
# Finds shortest routes between all pairs of stations in the network
def find_all_pairs_shortest_route(self):
self.all_pairs_shortest_route = all_pairs_dijkstra_path(self.network)
# Returns the shortest route from origin station to destination station
# in the network
def get_shortest_route(self, origin_station, destination_station):
return self.all_pairs_shortest_route[origin_station][
destination_station]
# Returns the set of zones visited by the route taken in the network
def get_zones_visited(self, route):
zones_visited = set()
for station in route:
station_zones = self.network.node[station]["zones"]
for zone in station_zones:
zones_visited.add(zone)
return zones_visited
# Checks if a station is in the network
def has_station(self, station):
return self.network.has_node(station)
def _mergeClusters(self, messageClusters, clusterStats,
alignedFieldClasses, matchingClusters,
matchingConditions):
import IPython
from tabulate import tabulate
from nemere.utils.evaluationHelpers import printClusterMergeConditions
from nemere.inference.templates import Template
remDue2gaps = [
clunuAB for clunuAB in matchingClusters if not len([
True for a in matchingConditions[clunuAB][1:]
if a[0] == True or a[1] == True
]) <= numpy.ceil(.4 * len(matchingConditions[clunuAB][1:]))
]
print("\nremove due to more than 40% gaps:")
print(
tabulate([(clupair,
len([
True for a in matchingConditions[clupair][1:]
if a[0] == True or a[1] == True
]), len(matchingConditions[clupair]) - 1)
for clupair in remDue2gaps],
headers=("clpa", "gaps", "fields")))
print()
remDue2gapsInARow = list()
for clunuAB in matchingClusters:
for flip in (0, 1):
rowOfGaps = [a[flip] for a in matchingConditions[clunuAB][1:]]
startOfGroups = [
i for i, g in enumerate(rowOfGaps)
if g and i > 1 and not rowOfGaps[i - 1]
]
endOfGroups = [
i for i, g in enumerate(rowOfGaps)
if g and i < len(rowOfGaps) - 1 and not rowOfGaps[i + 1]
]
if len(startOfGroups
) > 0 and startOfGroups[-1] == len(rowOfGaps) - 1:
endOfGroups.append(startOfGroups[-1])
if len(endOfGroups) > 0 and endOfGroups[0] == 0:
startOfGroups = [0] + startOfGroups
# field index before and after all gap groups longer than 2
groupOfLonger = [
(sog - 1, eog + 1)
for sog, eog in zip(startOfGroups, endOfGroups)
if sog < eog - 1
]
for beforeGroup, afterGroup in groupOfLonger:
if not ((beforeGroup < 0 or isinstance(
alignedFieldClasses[clunuAB][flip][beforeGroup],
MessageSegment)) or
(afterGroup >= len(rowOfGaps) or isinstance(
alignedFieldClasses[clunuAB][flip][afterGroup],
MessageSegment))):
remDue2gapsInARow.append(clunuAB)
break
if clunuAB in remDue2gapsInARow:
# already removed
break
print(
"\nremove due to more than 2 gaps in a row not surounded by STAs:")
print(remDue2gapsInARow)
print()
# remove pairs based on more then 25% gaps
matchingClusters = [
clunuAB for clunuAB in matchingClusters
if clunuAB not in remDue2gaps and clunuAB not in remDue2gapsInARow
]
# search in filteredMatches for STATIC - DYNAMIC - STATIC with different static values and remove from matchingClusters
# : the matches on grounds of the STA value in DYN condition, with the DYN role(s) in a set in the first element of each tuple
dynStaPairs = list()
for clunuPair in matchingClusters:
dynRole = [
clunuPair[0] if isinstance(
alignedFieldClasses[clunuPair][0][fieldNum], Template) else
clunuPair[1] for fieldNum, fieldCond in enumerate(
matchingConditions[clunuPair][1:])
if not any(fieldCond[:5]) and (
fieldCond[5] or fieldCond[6] or fieldCond[7])
]
if dynRole:
dynStaPairs.append((set(dynRole), clunuPair))
dynRoles = set(
chain.from_iterable(
[dynRole for dynRole, clunuPair in dynStaPairs]))
# List of STA roles for each DYN role
staRoles = {
dynRole: [
clunuPair[0] if clunuPair[1] in dr else clunuPair[1]
for dr, clunuPair in dynStaPairs if dynRole in dr
]
for dynRole in dynRoles
}
removeFromMatchingClusters = list()
# for each cluster that holds at least one DYN field class...
for dynRole, staRoleList in staRoles.items():
try:
# alt: use staMismatch and subsequent accesses to remove whole group of connected clusters at once
# staMismatch = False
staValues = dict()
clunuPairs = dict()
# match the STA values corresponding to the DYN fields...
for staRole in staRoleList:
clunuPair = (dynRole, staRole) if (dynRole, staRole) in matchingConditions else (staRole, dynRole) \
if (staRole, dynRole) in matchingConditions else None
if clunuPair is None:
# print("Skipping ({}, {})".format(staRole, dynRole))
continue
cluPairCond = matchingConditions[clunuPair]
fieldMatches = [
fieldNum
for fieldNum, fieldCond in enumerate(cluPairCond[1:])
if not any(fieldCond[:5]) and (
fieldCond[5] or fieldCond[6] or fieldCond[7])
]
dynTemplates = [
(alignedFieldClasses[clunuPair][0][fieldNum],
alignedFieldClasses[clunuPair][1][fieldNum])
if dynRole == clunuPair[0] else
(alignedFieldClasses[clunuPair][1][fieldNum],
alignedFieldClasses[clunuPair][0][fieldNum])
for fieldNum in fieldMatches
]
for dynT, custva in dynTemplates:
if not isinstance(dynT, Template) or not isinstance(
custva, MessageSegment):
continue
if dynT not in staValues:
# set the current static value to the STA-field values for the DYN template
staValues[dynT] = custva
clunuPairs[dynT] = clunuPair
elif staValues[
dynT].values != custva.values and self.dc.pairDistance(
staValues[dynT], custva) > 0.1:
#
# staMismatch = True
#
# if dc.pairDistance(dynT.medoid, staValues[dynT]) > dc.pairDistance(dynT.medoid, custva):
# removeFromMatchingClusters.append(clunuPairs[dynT])
# else:
# removeFromMatchingClusters.append(clunuPair)
#
# print("prevValue {} and {} currentValues".format(staValues[dynT], custva))
if staValues[dynT].values not in [
bsv.values for bsv in dynT.baseSegments
]:
# and dc.pairDistance(dynT.medoid, staValues[dynT]) > 0.15:
print("remove", clunuPairs[dynT], "because of",
staValues[dynT])
removeFromMatchingClusters.append(
clunuPairs[dynT])
if custva.values not in [
bsv.values for bsv in dynT.baseSegments
]:
# and dc.pairDistance(dynT.medoid, custva) > 0.15:
print("remove", clunuPair, "because of",
custva)
removeFromMatchingClusters.append(clunuPair)
#
# break
#
#
# if staMismatch:
# break
#
# if staMismatch:
# # mask to remove the clunuPairs of all combinations with this dynRole
# removeFromMatchingClusters.extend([
# (dynRole, staRole) if (dynRole, staRole) in matchingConditions else (staRole, dynRole)
# for staRole in staRoleList
# ])
except KeyError as e:
print("KeyError:", e)
IPython.embed()
raise e
print("remove for transitive STA mimatch:", removeFromMatchingClusters)
print()
# if a chain of matches would merge more than .66 of all clusters, remove that chain
dracula = Graph()
dracula.add_edges_from(
set(matchingClusters) - set(removeFromMatchingClusters))
connectedDracula = list(connected_components(dracula))
for clusterChain in connectedDracula:
if len(clusterChain) > .66 * len(
self.alignedClusters
): # TODO increase to .66 (nbns tshark)
for clunu in clusterChain:
for remainingPair in set(matchingClusters) - set(
removeFromMatchingClusters):
if clunu in remainingPair:
removeFromMatchingClusters.append(remainingPair)
remainingClusters = set(matchingClusters) - set(
removeFromMatchingClusters)
if len(remainingClusters) > 0:
print("Clusters could be merged:")
for clunuAB in remainingClusters:
printClusterMergeConditions(clunuAB, alignedFieldClasses,
matchingConditions, self.dc)
print("remove finally:", removeFromMatchingClusters)
print("remain:", remainingClusters)
chainedRemains = Graph()
chainedRemains.add_edges_from(remainingClusters)
connectedClusters = list(connected_components(chainedRemains))
# for statistics
if clusterStats is not None:
missedmerges = ClusterClusterer.printShouldMerge(
connectedClusters, clusterStats)
missedmergepairs = [
k for k in remainingClusters if any([
k[0] in mc and k[1] in mc
or k[0] in mc and k[1] in chain.from_iterable(
[cc for cc in connectedClusters
if k[0] in cc]) or k[0] in chain.from_iterable(
[cc for cc in connectedClusters
if k[1] in cc]) and k[1] in mc
for mc in missedmerges
])
]
singleClusters = {
ck: ml
for ck, ml in messageClusters.items()
if not chainedRemains.has_node(ck)
}
mergedClusters = {
str(mergelist): list(
chain.from_iterable(
[messageClusters[clunu] for clunu in mergelist]))
for mergelist in connectedClusters
}
mergedClusters.update(singleClusters)
return mergedClusters
def add_atomic_edges(G: nx.Graph) -> nx.Graph:
"""
Computes covalent edges based on atomic distances. Covalent radii add_atomic_edges assigned to each atom based on its bond assign_bond_states_to_dataframe
The distance matrix is then thresholded to entries less than this distance plus some tolerance to create and adjacency matrix.
This adjacency matrix is then parsed into an edge list and covalent edges added
:param G: Atomic graph (nodes correspond to atoms) to populate with atomic bonds as edges
:type G: nx.Graph
:return: Atomic graph with edges between bonded atoms added
:rtype: nx.Graph
"""
TOLERANCE = 0.56 # 0.4 0.45, 0.56 This is the distance tolerance
dist_mat = compute_distmat(G.graph["pdb_df"])
# We assign bond states to the dataframe, and then map these to covalent radii
G.graph["pdb_df"] = assign_bond_states_to_dataframe(G.graph["pdb_df"])
G.graph["pdb_df"] = assign_covalent_radii_to_dataframe(G.graph["pdb_df"])
# Create a covalent 'distance' matrix by adding the radius arrays with its transpose
covalent_radius_distance_matrix = np.add(
np.array(G.graph["pdb_df"]["covalent_radius"]).reshape(-1, 1),
np.array(G.graph["pdb_df"]["covalent_radius"]).reshape(1, -1),
)
# Add the tolerance
covalent_radius_distance_matrix = (covalent_radius_distance_matrix +
TOLERANCE)
# Threshold Distance Matrix to entries where the eucl distance is less than the covalent radius plus tolerance and larger than 0.4
dist_mat = dist_mat[dist_mat > 0.4]
t_distmat = dist_mat[dist_mat < covalent_radius_distance_matrix]
# Store atomic adjacency matrix in graph
G.graph["atomic_adj_mat"] = np.nan_to_num(t_distmat)
# Get node IDs from non NaN entries in the thresholded distance matrix and add the edge to the graph
inds = zip(*np.where(~np.isnan(t_distmat)))
for i in inds:
length = t_distmat[i[0]][i[1]]
node_1 = G.graph["pdb_df"]["node_id"][i[0]]
node_2 = G.graph["pdb_df"]["node_id"][i[1]]
chain_1 = G.graph["pdb_df"]["chain_id"][i[0]]
chain_2 = G.graph["pdb_df"]["chain_id"][i[1]]
# Check nodes are in graph
if not (G.has_node(node_1) and G.has_node(node_2)):
continue
# Check atoms are in the same chain
if not (chain_1 and chain_2):
continue
if G.has_edge(node_1, node_2):
G.edges[node_1, node_2]["kind"].add("covalent")
G.edges[node_1, node_2]["bond_length"] = length
else:
G.add_edge(node_1, node_2, kind={"covalent"}, bond_length=length)
# Todo checking degree against MAX_NEIGHBOURS
return G
