def build_graph(nodes: List[Node], edges: List[Edge],
**kwargs) -> MultiDiGraph:
force_rebuild = kwargs.get("force_rebuild", False)
save_checkpoint = kwargs.get("save_checkpoint", True)
if not force_rebuild:
if os.path.exists(GRAPH_CHECKPOINT):
return load_graph()
else:
log.info("Graph checkpoint does not exist, building graph.")
graph = MultiDiGraph()
graph.add_nodes_from(nodes)
ebunch = list(map(lambda x: (x.source, x.destination, x), edges))
graph.add_edges_from(ebunch)
assert graph.number_of_nodes() == len(nodes)
assert graph.number_of_edges() == len(edges)
if save_checkpoint:
log.info("Checkpointing graph...")
with open(GRAPH_CHECKPOINT, "wb") as file:
pickle.dump(graph, file)
return graph
def __add__(self, o):
if not isinstance(o, MossNet):
raise TypeError("unsupported operand type(s) for +: 'MossNet' and '%s'" % type(o).__name__)
g = MultiDiGraph()
g.add_edges_from(list(self.graph.edges(data=True)) + list(o.graph.edges(data=True)))
g.add_nodes_from(list(self.graph.nodes(data=True)) + list(o.graph.nodes(data=True)))
return MossNet(g)
def __init__(self, F, arg):
"""
If arg is a list:
Generate the core graph corresponding to the group generated by group_gens.
If arg is a graph:
Check for validity and do all folds.
"""
assert is_FreeGroup(F), "F must be a free group"
self.F = F
self.F_rank = F.rank()
self.letter_types = range(1, self.F_rank + 1)
self.letters = list(range(-self.F_rank, 0)) + list(range(1, self.F_rank + 1))
# -r, ..., -1, 1, ..., r
if isinstance(arg, list):
group_gens = arg
assert all([gen in F for gen in group_gens]), "The generators must be elements of F."
self.group_gens = group_gens
G = MultiDiGraph()
G.add_node((0,)) # the marked vertex (id)
for i, gen in enumerate(self.group_gens):
word = gen.Tietze()
word_len = len(word)
# new_nodes = [(i, j) for j in range(1, word_len)]
# G.add_nodes_from(new_nodes)
get_node = lambda j: (0,) if (j % word_len == 0) else (i, j)
for j in range(word_len):
G.add_edge(get_node(j), get_node(j + 1), label=word[j])
G.add_edge(get_node(j + 1), get_node(j), label=-word[j])
elif isinstance(arg, MultiDiGraph):
# We are going to copy the graph, add reverse edges when needed,
# and sort the edges.
# The reason we sort the edges is to get a "canonical" version
# of the object, so subgroup_gens would be the same in different
# objects with the same graph.
G = MultiDiGraph()
G.add_nodes_from(arg.nodes())
edges = arg.edges(data='label')
G_edges = [e for e in edges]
assert len(edges) == len(arg.edges()), "Every edge has to be labelled."
for src, dst, letter in edges:
assert letter in self.letters, \
f"The edge betwen {src} and {dst} has an invalid label"
if (dst, src, -letter) not in G.edges(data='label'):
G_edges.append((dst, src, -letter))
G.add_weighted_edges_from(sorted(G_edges), weight='label')
else:
raise ValueError("arg must be a list of words or a MultiDiGraph.")
self.G = do_all_folds(G)
# The subgraph of positive edges
G_pos = MultiDiGraph()
G_pos.add_edges_from([e for e in self.G.edges(data=True) if e[2]['label'] > 0])
self.G_pos = G_pos
self.subgroup_gens = tuple(sorted(self.get_subgroup()))
def combine_dots(dotlist,sciid):
g1=drawing.nx_agraph.read_dot(TMP+dotlist[-1])
g = MultiDiGraph()
for d in dotlist:
g2=drawing.nx_agraph.read_dot(TMP + d)
g1=compose(g1,g2)
g.add_nodes_from(g1.nodes(data=True))
g.add_edges_from(g1.edges(data=True))
g.to_directed()
g = nx_agraph.to_agraph(g)
g.write(TMP+'-'.join(sciid)+'.dot')
def combine_dots(dotlist, mathid):
g1 = drawing.nx_agraph.read_dot(TMP + dotlist[-1])
g = MultiDiGraph()
for d in dotlist:
g2 = drawing.nx_agraph.read_dot(TMP + d)
g1 = compose(g1, g2)
g.add_nodes_from(g1.nodes(data=True))
g.add_edges_from(g1.edges(data=True))
g.to_directed()
g = nx_agraph.to_agraph(g)
g.write(TMP + '-'.join(mathid) + '.dot')
def combine_dots(dotlist):
g1=drawing.nx_agraph.read_dot('dots/'+dotlist[-1])
g = MultiDiGraph()
for i in xrange(len(dotlist)):
g2=drawing.nx_agraph.read_dot('dots/'+dotlist[i])
g1=compose(g1,g2)
g.add_nodes_from(g1.nodes(data=True))
g.add_edges_from(g1.edges(data=True))
g.to_directed()
g = nx_agraph.to_agraph(g)
g.write('dots/combined.dot')
def cyclic_multi_graph(acyclic: bool) -> MultiDiGraph:
g = MultiDiGraph()
g.add_nodes_from([1, 2, 3])
g.add_edge(1, 2, EdgeKey(1, 2, "default"))
g.add_edge(1, 3, EdgeKey(1, 3, "default"))
g.add_edge(2, 3, EdgeKey(2, 3, "default"))
g.add_edge(2, 1, EdgeKey(2, 1, "delete"))
g.add_edge(3, 2, EdgeKey(3, 2, "delete"))
g.add_edge(3, 1, EdgeKey(3, 1, "delete"))
if not acyclic:
g.add_edge(3, 1, EdgeKey(3, 1, "default"))
g.add_edge(1, 3, EdgeKey(1, 3, "delete"))
return g
def to_directed(self):
"""Return a directed representation of the graph.
A new multidigraph is returned with the same name, same nodes and
with each edge (u,v,data) replaced by two directed edges
(u,v,data) and (v,u,data).
"""
from networkx import MultiDiGraph
G=MultiDiGraph()
G.add_nodes_from(self)
G.add_edges_from( ((u,v,data) for u,nbrs in self.adjacency_iter() \
for v,datalist in nbrs.iteritems() for data in datalist) )
return G
def add_to_graph(self, G: nx.MultiDiGraph) -> None:
"""
Adds nodes for the given relative dating to the graph.
Args:
G: the graph to work on
"""
G.add_nodes_from(self.items)
for source in self.sources:
G.add_edges_from(pairwise(self.items),
kind=self.kind,
source=source,
comments=self.comments,
dating=self,
xml=self.xmlsource,
ignore=self.ignore)
def to_networkx(network: Network) -> Any:
from networkx import Graph, DiGraph, MultiDiGraph, MultiGraph
if network.directed and network.multiedges:
G = MultiDiGraph()
elif not network.directed and network.multiedges:
G = MultiGraph()
elif network.directed and not network.multiedges:
G = DiGraph()
else:
G = Graph()
G.add_nodes_from([(v, network.nodes[v].attributes)
for v in network.nodes.uids])
G.add_edges_from([(network.edges[e].v.uid, network.edges[e].w.uid,
network.edges[e].attributes)
for e in network.edges.uids])
return G
def get_graph(model_spec, debug=False):
'''
Builds a networkx representation of the graph,
'''
graph = MultiDiGraph()
graph.add_nodes_from(
get_nodes(model_spec['edges']) + model_spec['legs']['in'].keys() +
model_spec['legs']['out'].keys())
graph.add_edges_from([(edge['source'], edge['target'], {
'operator': edge['operator']
}) for edge in model_spec['edges']])
if debug:
draw_graph(graph)
return graph
def add_nodes(mygraph: MultiDiGraph, mynodes: dict) -> MultiDiGraph:
for key, value in mynodes.items():
if key and value:
mygraph.add_nodes_from([(key, {"color": value})])
return mygraph
def load_parallel_component(file_descr, graph: nx.MultiDiGraph, prev_layer_id):
"""
Load ParallelComponent of the Kaldi model.
ParallelComponent contains parallel nested networks.
Slice is inserted before nested networks.
Outputs of nested networks concatenate with layer Concat.
:param file_descr: descriptor of the model file
:param graph: graph with the topology.
:param prev_layer_id: id of the input layers for parallel component layer
:return: id of the concat layer - last layer of the parallel component layers
"""
nnet_count = read_token_value(file_descr, b'<NestedNnetCount>')
log.debug(
'Model contains parallel component with {} nested networks'.format(
nnet_count))
slice_id = unique_id(graph, prefix='Slice')
graph.add_node(slice_id, parameters=None, op='slice', kind='op')
slice_node = Node(graph, slice_id)
graph.add_edge(prev_layer_id, slice_id,
**create_edge_attrs(prev_layer_id, slice_id))
slices_points = []
outputs = []
for i in range(nnet_count):
read_token_value(file_descr, b'<NestedNnet>')
collect_until_token(file_descr, b'<Nnet>')
g, shape = load_kalid_nnet1_model(file_descr,
'Nested_net_{}'.format(i))
input_nodes = [
n for n in graph.nodes(data=True) if n[1]['op'] == 'Input'
]
if i != nnet_count - 1:
slices_points.append(shape[1])
g.remove_node(input_nodes[0][0])
mapping = {
node: unique_id(graph, node)
for node in g.nodes(data=False) if node in graph
}
g = nx.relabel_nodes(g, mapping)
for val in mapping.values():
g.node[val]['name'] = val
graph.add_nodes_from(g.nodes(data=True))
graph.add_edges_from(g.edges(data=True))
sorted_nodes = tuple(nx.topological_sort(g))
edge_attrs = create_edge_attrs(slice_id, sorted_nodes[0])
edge_attrs['out'] = i
graph.add_edge(slice_id, sorted_nodes[0], **edge_attrs)
outputs.append(sorted_nodes[-1])
packed_sp = struct.pack("B", 4) + struct.pack("I", len(slices_points))
for i in slices_points:
packed_sp += struct.pack("I", i)
slice_node.parameters = io.BytesIO(packed_sp)
concat_id = unique_id(graph, prefix='Concat')
graph.add_node(concat_id, parameters=None, op='concat', kind='op')
for i, output in enumerate(outputs):
edge_attrs = create_edge_attrs(output, concat_id)
edge_attrs['in'] = i
graph.add_edge(output, concat_id, **edge_attrs)
return concat_id
def imbalancedCitationsPublicationsExample():
"""
Illustrative example of imbalanced citations / publications to verify ShapeSim is working correctly
"""
graph = MultiDiGraph()
authors = ['Alice', 'Bob', 'Carol', 'Dave', 'Ed', 'Frank']
conference = 'KDD'
# Citation & publication count configuration
citationsPublications = {
'Alice': (100, 10),
'Bob': (80, 10),
'Carol': (100, 100),
'Dave': (50, 10),
'Ed': (10, 10),
'Frank': (1000, 100)
}
actualCitationsPublications = defaultdict(lambda: (0, 0))
# Helper functions for repeatedly adding papers to the graph
addPapersToAuthor = lambda n, author: [addPublicationPaper(author) for _ in itertools.repeat(None, n)]
addCitationsToPaper = lambda n, paper, author: [addCitationPaper(paper, author) for _ in itertools.repeat(None, n)]
# Helper for getting the next id
def __getNextId():
global nextId
oldId = nextId
nextId += 1
return oldId
def addPublicationPaper(author):
"""
Helper method to add a 'publication' paper, connected to both an author and a conference
"""
paper = "%s's Paper %d" % (author, (__getNextId()))
graph.add_node(paper)
graph.add_edges_from([(author, paper), (paper, author), (paper, conference), (conference, paper)])
citationCount, publicationCount = actualCitationsPublications[author]
actualCitationsPublications[author] = (citationCount, publicationCount + 1)
return paper
def addCitationPaper(citedPaper, citedAuthor):
"""
Helper method to add a 'citation' paper, which is only connected to the conference and the paper it cites
"""
citingPaper = "Citing Paper %d" % __getNextId()
graph.add_node(citingPaper)
graph.add_edges_from([(conference, citingPaper), (citingPaper, conference), (citingPaper, citedPaper)])
citationCount, publicationCount = actualCitationsPublications[citedAuthor]
actualCitationsPublications[citedAuthor] = (citationCount + 1, publicationCount)
return citingPaper
allPapers = []
# Construct the graph
graph.add_nodes_from(authors + [conference])
for authorName in citationsPublications:
citationCount, publicationCount = citationsPublications[authorName]
# Add citations & publications to author
authorPapers = addPapersToAuthor(publicationCount, authorName)
allPapers.extend(authorPapers)
citationsPerPaper = citationCount / publicationCount
for paper in authorPapers:
citingPapers = addCitationsToPaper(citationsPerPaper, paper, authorName)
allPapers.extend(citingPapers)
nodeIndex = {
'paper': {i: allPapers[i] for i in xrange(0, len(allPapers))},
'conference': {0: 'KDD'},
'author': {0: 'Alice', 1: 'Bob', 2: 'Carol', 3: 'Dave', 4: 'Ed', 5: 'Frank'}
}
# Test PathSim / NeighborSim
cpaAdjMatrix, extraData = getMetaPathAdjacencyData(graph, nodeIndex, ['conference', 'paper', 'author'])
extraData['fromNodes'] = extraData['toNodes']
extraData['fromNodesIndex'] = extraData['toNodesIndex']
neighborSimMostSimilar, similarityScores = findMostSimilarNodes(
cpaAdjMatrix, 'Alice', extraData, method=getNeighborSimScore
)
# Test ShapeSim
cppaAdjTensor, extraData = getMetaPathAdjacencyTensorData(
graph, nodeIndex, ['conference', 'paper', 'paper', 'author']
)
extraData['fromNodes'] = extraData['toNodes']
extraData['fromNodesIndex'] = extraData['toNodesIndex']
shapeSimMostSimilar, similarityScores = findMostSimilarNodes(
cppaAdjTensor, 'Alice', extraData, method=getNumpyShapeSimScore, alpha=1.0
)
# Output similarity scores
for name, mostSimilar in [('NeighborSim', neighborSimMostSimilar), ('ShapeSim', shapeSimMostSimilar)]:
print('\n%s Most Similar to "%s":' % (name, 'Alice'))
mostSimilarTable = texttable.Texttable()
rows = [['Author', 'Score']]
rows += [[name, score] for name, score in mostSimilar]
mostSimilarTable.add_rows(rows)
print(mostSimilarTable.draw())
def generate_nodes(graph: nx.MultiDiGraph, stops_df: pd.DataFrame) -> None:
def node_generator():
for stop_id, stop_name, stop_lat, stop_lon, _, _ in stops_df.itertuples():
yield stop_id, {'stop_name': stop_name, 'stop_lat': stop_lat, 'stop_lon': stop_lon}
graph.add_nodes_from(node_generator())
