def __save_obj(data_path, sentence_path):
sentences = ["""我 想 去 北京""",
"""我 想 吃饭 了""",
"""北京 在 哪里""",
"""我 想要 办理 汉口 银行 的 信用卡""",
"""怎么 办理 信用卡""",
"""信用卡 怎么 办理""",
"""我 要 取号""",
"""我 想去 天安门""",]
# for i in [['我', '要', '取号'], ['我', '要', '吃饭']]:
with open(sentence_path, 'r', encoding='utf-8') as f:
for content in f.readlines():
sentences_list = get_participle(content)
sentences.extend(w.gen_sentences(sentences_list, subsystem_id='hm256f8c953411e8aea35254004210bf'))
# sentences = [w.gen_sentences(i, subsystem_id='hm256f8c953411e8aea35254004210bf') for i in [['我', '要', '取号'], ['我', '要', '吃饭']]]
# sentences.extend(sentences)
word_list = [item.split() for item in sentences]
# temp = [set(item) for item in word_list]
# word_set = reduce(lambda a, b: a | b, temp) # 保证词语不重复
# logger.debug(word_set)
begin1 = time.time()
# dg = Digraph(word_set)
# # 构建词序
# for words in word_list:
# l = len(words)
# for i in range(l):
# if i < l - 1:
# dg.addEdge(words[i], words[i + 1])
dg = Digraph()
for w_l in word_list:
dg.build(w_l)
logger.debug("构建耗时:{}秒".format(time.time() - begin1))
save_obj(dg, file_name=data_path)
return dg
def __init__(self, filename, sp):
self.st = {}
with open(filename, 'r') as f:
while True:
line = f.readline()
if line == '':
break
line = line.split('\n')[0]
for a in line.split(sp):
if a not in self.st:
self.st[a] = len(self.st)
self.keys = [0] * len(self.st)
for name in self.st.keys():
self.keys[self.st[name]] = name
self.g = Digraph(len(self.st))
with open(filename, 'r') as f:
while True:
line = f.readline()
if line == '':
break
line = line.split('\n')[0]
a = line.split(sp)
v = self.st[a[0]]
for t in a[1:]:
self.g.addEdge(v, self.st[t])
def __init__(self):
self.box_dl = Digraph() # box node -> deadlocks
self._boxes_to_deadlock = defaultdict(list)
self._box_to_size_to_nodeA = defaultdict(dict)
self._nbox_to_size_to_nodeA = defaultdict(dict)
self._last_node = -1
def kosaraju(graph: Digraph) -> list:
connected_components = []
vertex_stack = [] # S
visited_vertex_map = {v: False for v in graph.vertices}
while len(vertex_stack) < len(graph.vertices):
# dfs_vertex = 3
# dfs_vertex = 0
dfs_vertex = choice(list(graph.vertices - set(vertex_stack)))
dfs_kosaraju(graph, dfs_vertex, vertex_stack=vertex_stack,
visited_vertex_map=visited_vertex_map)
#vertex_stack = list(reversed(vertex_stack))
transposed_graph = graph.transpose()
visited_vertex_map_t = {v: False for v in transposed_graph.vertices}
while vertex_stack:
top_stack_vertex = vertex_stack.pop()
connected_component = list()
dfs_kosaraju(transposed_graph, top_stack_vertex, connected_component,
visited_vertex_map_t)
connected_component = set(connected_component)
connected_components.append(connected_component)
remaining_edges = [edge for edge in transposed_graph.edges
if edge[0] and edge[1] not in connected_component]
transposed_graph = Digraph(edges=remaining_edges,
vertices=transposed_graph.vertices)
for component_vertex in connected_component:
try:
vertex_stack.remove(component_vertex)
except ValueError:
continue
return connected_components
def graph_from_adj_matrix(matrix):
""" Devuelve el Digrafo generado a partir de la matriz de ady. """
graph = Digraph(len(matrix))
for i in range(len(matrix)):
for j in range(len(matrix[i])):
if matrix[i][j] != 0:
graph.add_edge(i, j, matrix[i][j])
return graph
def test_add_node(self):
"""
test add_node
"""
dgraph = Digraph()
dgraph.add_node('one')
self.assertIn('one', dgraph.nodes)
def test_has_node(self):
"""
test has_node
"""
dgraph = Digraph()
dgraph.add_node('one')
self.assertTrue(dgraph.has_node('one'))
def save_dot(self, dot_file):
assert self.root is not None
g = Digraph(len(self.nodes))
for n in self.nodes:
g.set_vertex_label(n.idx, n.op)
if n.pred is not None:
g.add_edge(n.pred.idx, n.idx)
g.save_dot(dot_file)
def save_dot(self, dot_file):
g = Digraph(len(self.obj.infos))
for infos in self.obj.infos:
n = infos.node
g.set_vertex_label(n.idx, get_label(infos))
if n.pred is not None:
g.add_edge(n.pred.idx, n.idx)
g.save_dot(dot_file)
def test_del_node(self):
"""
test del_node
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
dgraph.del_node('one')
self.assertNotIn('one', dgraph.nodes)
def test4():
dag = Digraph()
dag.add_edge("v1", "v2")
dag.add_edge("v2", "v3")
bn = BayesianNetwork()
bn.set_dag(dag)
bn.set_pd("v1", [], {1 : 0.8, 0 : 0.2})
bn.set_pd("v2", ["v1"], {1 : {"1" : 0.4, "0" : 0.9}, 0 : {"1" : 0.6, "0" : 0.1}})
bn.set_pd("v3", ["v2"], {1 : {"1" : 0.7, "0" : 0.5}, 0 : {"1" : 0.3, "0" : 0.5}})
evidence = {"v3" : {1 : 1, 0 : 0}}
bn.compute_posterior(evidence)
def test_add_edge(self):
"""
test add_edge
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
dgraph.add_edge(('one', 'two'))
self.assertIn('two', dgraph.nodes['one'])
def test6():
dag = Digraph()
dag.add_edge("v1", "v2")
dag.add_edge("v2", "v3")
bn = BayesianNetwork()
bn.set_dag(dag)
bn.set_pd("v1", [], {"v1_true" : 0.8, "v1_false" : 0.2})
bn.set_pd("v2", ["v1"], {"v2_true" : {"v1_true" : 0.4, "v1_false" : 0.9}, "v2_false" : {"v1_true" : 0.6, "v1_false" : 0.1}})
bn.set_pd("v3", ["v2"], {"v3_true" : {"v2_true" : 0.7, "v2_false" : 0.5}, "v3_false" : {"v2_true" : 0.3, "v2_false" : 0.5}})
evidence = {"v3" : {"v3_true" : 1, "v3_false" : 0}}
bn.compute_posterior(evidence)
def test_get_node_order(self):
"""
Test get_node_order
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
if n != 'one':
dgraph.add_edge(('one', n))
self.assertEquals(dgraph.get_node_order('one'), 2)
def test_transitivity(self):
g = Digraph(edges=[(1, 2), (1, 3), (2, 3), (3, 3)], vertices=[4])
g.add_edge(5, 3)
self.assertTrue(g.is_transitive())
g.add_edge(3, 4)
self.assertFalse(g.is_transitive())
g.add_edge(5, 4)
self.assertFalse(g.is_transitive())
g.add_edge(2, 4)
self.assertFalse(g.is_transitive())
g.add_edge(1, 4)
self.assertTrue(g.is_transitive())
def test_del_edge(self):
"""
Test del_edge
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
if n != 'one':
dgraph.add_edge(('one', n))
dgraph.del_edge(('one', 'two'))
self.assertNotIn('two', dgraph.nodes['one'])
def test_has_edge(self):
"""
test has_edge
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
if n != 'one':
dgraph.add_edge(('one', n))
self.assertTrue(dgraph.has_edge(('one', 'two')))
self.assertTrue(dgraph.has_edge(('one', 'three')))
def test_get_nodes(self):
"""
test get_nodes
We will add a bunch of nodes and make sure it returns all of them. We're
using a set comparison, because
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
self.assertEquals(set(dgraph.get_nodes()), set(node_list))
def test2():
dag = Digraph()
dag.add_edge("v1", "v2")
dag.add_edge("v2", "v3")
bn = BayesianNetwork()
bn.set_dag(dag)
bn.set_pd("v1", [[0.2, 0.8]])
bn.set_pd("v2", [[0.9, 0.1], [0.6, 0.4]])
bn.set_pd("v3", [[0.5, 0.5], [0.3, 0.7]])
evidence = {"v3" : {0 : 0.0, 1 : 1.0}}
bn.compute_posterior(evidence)
def save_dot(self, dot_file):
g = Digraph(len(self.obj.infos))
for infos in self.obj.infos:
n = infos.node
if n.pred is not None:
g.add_edge(n.pred.idx, n.idx)
self.obj.apply(self)
for (u, label) in enumerate(self.labels):
g.set_vertex_label(u, label)
g.save_dot(dot_file)
def test_get_neighbors(self):
"""
test get_neighbors
Add a few nodes to a graph, and then add a few neighbors to that node,
and check whether get_neighbors returns the expected result
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
if n != 'one':
dgraph.add_edge(('one', n))
self.assertEquals(dgraph.get_neighbors('one'), ['two', 'three'])
def __enter__(self):
"""
Startup commands for the context manager.
"""
engine = create_engine(self.db)
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
self.session = Session()
edges = self.session.query(Edge)
self._graph = Digraph(len(edges.all()))
for edge in edges:
if not self._graph.has_edge(edge.stop_from_id, edge.stop_to_id):
self._graph.add_edge(DirectedEdge(
edge.stop_from_id, edge.stop_to_id, edge.length
))
return self
def test_get_edges(self):
"""
test get_edges
Add a few nodes to a graph, and then add a few edges to the graph. Check
whether get_edges returns the expected edges
"""
dgraph = Digraph()
node_list = ['one', 'two', 'three']
for n in node_list:
dgraph.add_node(n)
if n != 'one':
dgraph.add_edge(('one', n))
edges = [('one', 'two'), ('one', 'three')]
self.assertEquals(dgraph.get_edges(), edges)
def test3():
dag = Digraph()
dag.add_edge("v2", "v1")
dag.add_edge("v3", "v1")
bn = BayesianNetwork()
bn.set_dag(dag)
bn.set_pd("v1", ["v2", "v3"], {0 : {"11" : 0.8, "01" : 0.9, "10" : 0.5, "00" : 0.6}, 0 : {"11" : 0.2, "01" : 0.1, "10" : 0.5, "00" : 0.4}})
bn.set_pd("v2", [], {1 : 0.1, 0 : 0.9})
bn.set_pd("v3", [], {1 : 0.4, 0 : 0.6})
print bn
evidence = {}
#evidence = {"v1" : {1 : 1, 0 : 0}}
bn.compute_posterior(evidence)
def test5():
dag = Digraph()
dag.add_edge("v1", "v2")
dag.add_edge("v2", "v3")
dag.add_edge("v2", "v4")
bn = BayesianNetwork()
bn.set_dag(dag)
bn.set_pd("v1", [], {0 : 0.5, 1 : 0.5})
bn.set_pd("v2", ["v1"], {0 : {"0" : 0.5, "1" : 0.5}, 1 : {"0" : 0.5, "1" : 0.5}})
bn.set_pd("v3", ["v2"], {0 : {"0" : 0.5, "1" : 0.5}, 1 : {"0" : 0.5, "1" : 0.5}})
bn.set_pd("v4", ["v2"], {0 : {"0" : 0.5, "1" : 0.5}, 1 : {"0" : 0.5, "1" : 0.5}})
print bn
evidence = {"v3" : {0 : 0.0, 1 : 1.0}, "v4" : {0 : 0.9, 1 : 0.1}}
bn.compute_posterior(evidence)
def build_dag(T, use_tree=True, blast=True, gtg=True):
parents, children = findParentsAndChildren(T, ROOT_NODE)
dag = Digraph()
observed_species = set()
ancestors = set()
if use_tree:
for u in children:
ancestors.add(u)
for v in children[u]:
dag.add_edge(u, v)
for u in T:
if u not in children:
observed_species.add(u)
if blast:
dag.add_edge(u, get_blast_node_name(u))
if gtg:
dag.add_edge(u, get_gtg_node_name(u))
return dag, observed_species, ancestors
def __init__(self, dl_set=None, fname=None, sample_state=None):
self.fname = fname
self.dependencies = Digraph() # deadlock -> descendants
if dl_set is None: dl_set = DeadlockSet()
self.dl_set = dl_set
self._last_full_index = -1
self.debug_data = []
self.debug_fname = "bug.log"
if fname is not None:
assert sample_state is not None
if os.path.exists(fname):
print("loading deadlocks...")
try:
blocks = deadlocks_from_file(fname, sample_state)
except:
blocks = None
def backup_fnames_gen():
base_fname = fname + "_backup"
yield base_fname
i = 0
while True:
yield base_fname + str(i)
backup_fnames = backup_fnames_gen()
while True:
backup_fname = next(backup_fnames)
if not os.path.exists(backup_fname): break
os.rename(fname, backup_fname)
print("deadlock file corrupted, renamed to '{}'".format(
backup_fname))
if blocks is not None:
for dl in chain.from_iterable(blocks):
self.debug_data.append(
"dummy_deadlocks[{}] = make_dummy_deadlock({})".
format(id(dl), dl.full_index))
self.dl_set.add(dl)
self._last_full_index = dl.full_index
print("loaded {} deadlocks".format(self._last_full_index +
1))
def test():
dag = Digraph()
dag.add_edge("v1", "v2")
dag.add_edge("v2", "v3")
dag.add_edge("v2", "v4")
bn = BayesianNetwork()
bn.set_dag(dag)
bn.set_pd("v1", [], {0 : 0.6, 1 : 0.4})
# bn.set_pd("v1", [], {0 : 0.99, 1 : 0.01})
bn.set_pd("v2", ["v1"], {0 : {"0" : 0.9, "1" : 0.1}, 1 : {"0" : 0.7, "1" : 0.3}})
bn.set_pd("v3", ["v2"], {0 : {"0" : 0.1, "1" : 0.9}, 1 : {"0" : 0.8, "1" : 0.2}})
bn.set_pd("v4", ["v2"], {0 : {"0" : 0.5, "1" : 0.5}, 1 : {"0" : 0.25, "1" : 0.75}})
print bn
# evidence = {"v3" : {0 : 0.0, 1 : 1.0}}
# evidence = {"v1" : {0 : 0.0, 1 : 1.0}}
evidence = {}
bn.compute_posterior(evidence)
def __init__(self, file_name, separator):
self.__symbols = {}
with open(file_name, 'r') as f:
for line in f:
tokens = line.strip().split(separator)
for token in tokens:
if token not in self.__symbols:
self.__symbols[token] = len(self.__symbols)
self.__keys = [0] * len(self.__symbols)
for k, v in self.__symbols.items():
self.__keys[v] = k
self.__g = Digraph(len(self.__symbols))
with open(file_name, 'r') as f:
for line in f:
tokens = line.strip().split(separator)
v = self.__symbols[tokens[0]]
for token in tokens[1:]:
self.__g.add_edge(v, self.__symbols[token])
def find_order(word_sorted_list):
"""
Given a list of words in an alien language in dictionary sorted order, return the alphabetical
order of the letters in the alien language.
The algorithm is as follows:
1) Find the number of letters in the alphabet, by going 1 pass over the input array.
This takes time O(wl). if there are w words and l letters on an avg per word.
Lets call the result, alpha
2) Create a digraph with num_vertices = alpha
3) For every pair of consecutive words (word1, word2) in the input list (there are w - 1 pairs
in a w-word list), compare letters of word1 & word2 until you find the 1st mismatching
character. Add an edge in the digraph from letter in word1 to letter in word2
4) Topologically sort the digraph. This takes time O(V + E) = O(alpha + alpha - 1)
Total time = O(wL) + O(1) + O[(w-1)(L)] + O(alpha)
= O(wL) where L = avg number of letters per word
:param word_sorted_list:
:return:
"""
alphabet = set()
for word in word_sorted_list:
for letter in word:
alphabet.add(letter)
num_vertices = len(alphabet)
dg = Digraph(num_vertices)
# graph takes vertex values as integers,
# so using 2 maps for letter -> vertex_id and inverse mapping
letter_to_vertex_id_dict = {}
vertex_id_to_letter = []
for i in xrange(len(word_sorted_list) - 1):
word1 = word_sorted_list[i]
word2 = word_sorted_list[i + 1]
if len(word1) < len(word2):
smaller_word = word_sorted_list[i]
else:
smaller_word = word_sorted_list[i + 1]
for word_index in xrange(len(smaller_word)):
if word1[word_index] == word2[word_index]:
continue
else:
letter1 = word1[word_index].lower()
letter2 = word2[word_index].lower()
vertex1 = _get_vertex_for_letter(letter1,
letter_to_vertex_id_dict,
vertex_id_to_letter)
vertex2 = _get_vertex_for_letter(letter2,
letter_to_vertex_id_dict,
vertex_id_to_letter)
dg.add_edge(vertex1, vertex2)
# no need to consider other characters in this word pair,
# break out of inner for-loop
break
# now just do a topological sort of the letters
ordered_vertex_ids = topological_sort(dg)
lst = [vertex_id_to_letter[vertex] for vertex in ordered_vertex_ids]
return ''.join([i for i in lst])
