def cfpq_tensor(graph: Graph, grammar: CFG):
rfa = RFA().from_cfg(grammar)
result = Graph()
result.size = graph.size
if graph.size == 0:
return Matrix.sparse(BOOL, graph.size, graph.size)
for label in graph.label_dictionary:
result.label_dictionary[label] = graph.label_dictionary[label].dup(
)
for label in rfa.label_dictionary:
if label not in result.label_dictionary:
result.label_dictionary[label] = Matrix.sparse(
BOOL, graph.size, graph.size)
for term in grammar.terminals:
if term.value not in result.label_dictionary:
result.label_dictionary[term.value] = Matrix.sparse(
BOOL, graph.size, graph.size)
for production in grammar.productions:
if not production.body:
for vertex in graph.vertices:
result.label_dictionary[production.head.value][vertex,
vertex] = 1
matrix_changing = True
closure = None
while matrix_changing:
matrix_changing = False
dictionary = {}
size = 0
for label in rfa.label_dictionary:
dictionary[label] = result.label_dictionary[label].kronecker(
rfa.label_dictionary[label])
if size == 0:
size = dictionary[label].ncols
intersection = Matrix.sparse(BOOL, size, size)
with semiring.LOR_LAND_BOOL:
for label in dictionary:
if dictionary[label].nrows < size:
dictionary[label].resize(size, size)
intersection += dictionary[label]
if closure is None:
previous = 0
else:
previous = closure.nvals
closure = transitive_closure(intersection)
for start, end in get_reachable(closure):
start_m = start // rfa.size
start_rfa = start % rfa.size
end_m = end // rfa.size
end_rfa = end % rfa.size
if start_rfa in rfa.start_states and end_rfa in rfa.final_states:
label = rfa.head_by_vertices[(start_rfa, end_rfa)]
result.label_dictionary[label][start_m, end_m] = 1
if previous != closure.nvals:
matrix_changing = True
return result.label_dictionary[grammar.start_symbol.value]
def cfpq_hellings(grammar: CFG, graph: Graph):
cfg = GrammarUtils.to_cnf(grammar)
graph_size = graph.size
start_sym = cfg.start_symbol
result = Graph()
result.size = graph_size
m = deque()
for variable in cfg.variables:
result.label_dictionary[variable] = Matrix.sparse(
BOOL, graph_size, graph_size)
if cfg.generate_epsilon():
for vertex in range(graph_size):
result.label_dictionary[start_sym][vertex, vertex] = 1
for label in graph.label_dictionary:
terminal = Terminal(label)
result.label_dictionary[terminal] = graph.label_dictionary[
label].dup()
for from_vertex, to_vertex in graph.get_edges(label):
for production in cfg.productions:
if len(production.body
) == 1 and production.body[0] == terminal:
head = production.head
result.label_dictionary[head][from_vertex,
to_vertex] = 1
for label in result.label_dictionary:
for i, j in result.get_edges(label):
m.append((label, i, j))
terminal_productions = set()
nonterminal_productions = set()
for production in cfg.productions:
if len(production.body) == 1:
terminal_productions.add(production)
elif len(production.body) >= 2:
nonterminal_productions.add(production)
while m:
var, v, u = m.popleft()
for var_left in result.label_dictionary:
for v_new, v_ in result.get_edges(var_left):
if v_ == v:
for production in nonterminal_productions:
if production.body[1] == var and production.body[
0] == var_left:
if (v_new, u) not in result.get_edges(
production.head):
result.label_dictionary[production.head][
v_new, u] = True
m.append((production.head, v_new, u))
for var_right in result.label_dictionary:
for u_, u_new in result.get_edges(var_right):
if u_ == u:
for production in nonterminal_productions:
if production.body[
1] == var_right and production.body[
0] == var:
if (v, u_new) not in result.get_edges(
production.head):
result.label_dictionary[production.head][
v, u_new] = True
m.append((production.head, v, u_new))
return result.label_dictionary[start_sym]
def prepare_for_exctract_paths(self):
sizeKron = self.graph.matrices_size * self.grammar.matrices_size
self.kron = Matrix.sparse(BOOL, sizeKron, sizeKron)
for label in self.grammar.labels:
self.kron += self.grammar[label].kronecker(self.graph[label])
def rpq(config: Config) -> Union[bool, Set[Tuple[Vertex, Vertex]]]:
data_base = config['data_base']
query = config['regular_query']
# tensor product of data base and query
intersection_matrices: Dict[Symbol, Matrix] = {
S: data_base.matrices[S].kronecker(query.matrices[S])
for S in data_base.symbols & query.symbols
}
# reachability matrix for one step
if intersection_matrices != dict():
reachability_matrix_for_one_step: Matrix = \
reduce(operator.add, intersection_matrices.values())
else:
reachability_matrix_for_one_step: Matrix = \
Matrix.sparse(
types.BOOL,
data_base.count_vertexes * query.count_vertexes,
data_base.count_vertexes * query.count_vertexes
)
# reachability matrix
prev_nvals = 0
reachability_matrix: Matrix = reachability_matrix_for_one_step
while reachability_matrix.nvals != prev_nvals:
prev_nvals = reachability_matrix.nvals
reachability_matrix += \
reachability_matrix @ reachability_matrix
# ... or ...
# reachability_matrix @ reachability_matrix_for_one_step
# number of reachable pairs
if 'return_number_of_pairs' in config and config['return_number_of_pairs']:
return reachability_matrix.nvals
# input vertexes
if 'input_vertexes' not in config:
Vs_from = list(range(data_base.count_vertexes))
else:
Vs_from = config['input_vertexes']
tensor_product_input_vertexes, data_base_input_vertexes = ( \
list(map(
lambda x:
x * query.count_vertexes + query.start_vertex,
Vs_from)),
Vs_from
)
# output vertexes
if 'output_vertexes' not in config:
Vs_to = list(range(data_base.count_vertexes))
else:
Vs_to = config['output_vertexes']
tensor_product_output_vertexes, data_base_output_vertexes = \
list(map(list, zip(*[
(
data_base_V_to * query.count_vertexes + query_final_V,
data_base_V_to
)
for data_base_V_to in Vs_to
for query_final_V in query.final_vertexes
])))
# reachable pairs
return {(data_base_input_vertexes[i], data_base_output_vertexes[j])
for i, j, _ in reachability_matrix.extract_matrix(
tensor_product_input_vertexes, tensor_product_output_vertexes)}
def solve(self, sources: Iterable):
# Creating new index per solve call
# index = SingleSourceIndex(self.graph, self.grammar)
nonterminals = self.__initial_nonterminals.clone()
# Initialize sources and nonterms nnz
# nnz: (l, r1, r2) in complex rules -> (nnz(l), nnz(r1), nnz(r2))
nnz = {}
for l, r1, r2 in self.grammar.complex_rules:
nnz[(l, r1, r2)] = (0, nonterminals[r1].nvals,
nonterminals[r2].nvals)
# Initialize source matrices masks
m_src = Matrix.sparse(BOOL, self.graph.matrices_size,
self.graph.matrices_size)
for v in sources:
m_src[v, v] = True
self.sources[self.grammar.start_nonterm][v, v] = True
# Create temporary matrix
tmp = Matrix.sparse(BOOL, self.graph.matrices_size,
self.graph.matrices_size)
# Algo's body
iter = 0
changed = True
while changed:
iter += 1
changed = False
# Number of instances before operation
# old_nnz_nonterms = {nonterm: index.nonterms[nonterm].nvals for nonterm in index.grammar.nonterms}
# old_nnz_sources = {nonterm: index.sources[nonterm].nvals for nonterm in index.grammar.nonterms}
# Iterate through all complex rules
for l, r1, r2 in self.grammar.complex_rules:
new_nnz = self.sources[l].nvals, nonterminals[
r1].nvals, nonterminals[r2].nvals
if nnz[(l, r1, r2)] != new_nnz:
# 1) r1_src += {(j, j) : (i, j) \in l_src}
update_sources(self.sources[l], self.sources[r1])
# 2) tmp = l_src * r1
tmp = self.sources[l].mxm(nonterminals[r1],
semiring=BOOL.ANY_PAIR)
# 3) r2_src += {(j, j) : (i, j) \in tmp}
update_sources(tmp, self.sources[r2])
# 4) l += tmp * r2
nonterminals[l] += tmp.mxm(nonterminals[r2],
semiring=BOOL.ANY_PAIR)
# update nnz
nnz[(l, r1, r2)] = self.sources[l].nvals, nonterminals[
r1].nvals, nonterminals[r2].nvals
changed = True
return ResultAlgo(m_src.mxm(nonterminals[self.grammar.start_nonterm], semiring=BOOL.ANY_PAIR), iter), \
nonterminals[self.grammar.start_nonterm]
def solve(self):
restore_eps_paths(self.grammar.start_and_finish, self.graph)
sizeKron = self.graph.matrices_size * self.grammar.matrices_size
prev_kron = Matrix.sparse(BOOL, sizeKron, sizeKron)
iter = 0
block = LabelGraph(self.graph.matrices_size)
changed = True
first_iter = True
while changed:
changed = False
iter += 1
kron = Matrix.sparse(BOOL, sizeKron, sizeKron)
if first_iter:
for label in self.grammar.labels:
kron += self.grammar[label].kronecker(self.graph[label])
else:
for nonterminal in block.matrices:
kron += self.grammar[nonterminal].kronecker(block[nonterminal])
block[nonterminal] = Matrix.sparse(BOOL, self.graph.matrices_size, self.graph.matrices_size)
transitive_closure(kron)
if not first_iter:
part = prev_kron.mxm(kron, semiring=BOOL.ANY_PAIR)
with BOOL.ANY_PAIR:
kron += prev_kron + part @ prev_kron + part + kron @ prev_kron
prev_kron = kron
for nonterminal in self.grammar.nonterminals:
for element in self.grammar.states[nonterminal]:
i = element[0]
j = element[1]
start_i = i * self.graph.matrices_size
start_j = j * self.graph.matrices_size
control_sum = self.graph[nonterminal].nvals
if first_iter:
block[nonterminal] += kron[start_i:start_i + self.graph.matrices_size - 1,
start_j:start_j + self.graph.matrices_size - 1]
else:
new_edges = kron[start_i:start_i + self.graph.matrices_size - 1,
start_j:start_j + self.graph.matrices_size - 1]
part = new_edges - block[nonterminal]
block[nonterminal] += part.select('==', True)
self.graph[nonterminal] += block[nonterminal]
new_control_sum = self.graph[nonterminal].nvals
if new_control_sum != control_sum:
changed = True
first_iter = False
if self.grammar.nonterminals.isdisjoint(self.grammar.labels):
break
return ResultAlgo(self.graph[self.grammar.start_nonterm], iter)
def __getitem__(self, item: str) -> Matrix:
if item not in self.matrices:
self.matrices[item] = Matrix.sparse(SAVEMIDDLETYPE,
self.matrices_size,
self.matrices_size)
return self.matrices[item]
def solve(self, sources: Iterable):
new_sources = LabelGraph(self.graph.matrices_size)
# Initialize sources and nonterms nnz
# nnz: (l, r1, r2) in complex rules -> (nnz(new[l]), nnz(index[r1]), nnz(index[r2]))
nnz = {}
for l, r1, r2 in self.grammar.complex_rules:
nnz[(l, r1, r2)] = (0, self.nonterminals[r1].nvals,
self.nonterminals[r2].nvals)
# Initialize source matrices masks
m_src = Matrix.sparse(BOOL, self.graph.matrices_size,
self.graph.matrices_size)
for i in sources:
m_src[i, i] = True
if (i, i) not in self.sources[self.grammar.start_nonterm]:
new_sources[self.grammar.start_nonterm][i, i] = True
# Create temporary matrix
tmp = Matrix.sparse(BOOL, self.graph.matrices_size,
self.graph.matrices_size)
# Algo's body
changed = True
iter = 0
while changed:
iter += 1
changed = False
# Iterate through all complex rules
for l, r1, r2 in self.grammar.complex_rules:
# l -> r1 r2 ==> index[l] += (new[l_src] * index[r1]) * index[r2]
new_nnz = new_sources[l].nvals, self.nonterminals[
r1].nvals, self.nonterminals[r2].nvals
if nnz[(l, r1, r2)] != new_nnz:
# 1) new[r1_src] += {(j, j) : (j, j) in new[l_src] and not in index[r1_src]}
for i, _, _ in new_sources[l]:
if (i, i) not in self.sources[r1]:
new_sources[r1][i, i] = True
# 2) tmp = new[l_src] * index[r1]
new_sources[l].mxm(self.nonterminals[r1],
out=tmp,
semiring=BOOL.ANY_PAIR)
# 3) new[r2_src] += {(j, j) : (i, j) in tmp and not in index[r2_src]}
update_sources_opt(tmp, self.sources[r2], new_sources[r2])
# 4) index[l] += tmp * index[r2]
self.nonterminals[l] += tmp.mxm(self.nonterminals[r2],
semiring=BOOL.ANY_PAIR)
# update nnz
nnz[(l, r1, r2)] = new_sources[l].nvals, self.nonterminals[
r1].nvals, self.nonterminals[r2].nvals
changed = True
for n in self.grammar.nonterms:
self.sources[n] += new_sources[n]
return ResultAlgo(m_src.mxm(self.nonterminals[self.grammar.start_nonterm], semiring=BOOL.ANY_PAIR), iter), \
self.nonterminals[self.grammar.start_nonterm]
def __getitem__(self, item: str) -> Matrix:
if item not in self.matrices:
self.matrices[item] = Matrix.sparse(self.type, self.matrices_size,
self.matrices_size)
return self.matrices[item]
def maximal_matrix(T):
return Matrix.sparse(T, GxB_INDEX_MAX, GxB_INDEX_MAX)
def solve(self, sources: Iterable):
restore_eps_paths(self.grammar.start_and_finish, self.graph)
# Initialize source matrices masks
m_src = Matrix.sparse(BOOL, self.graph.matrices_size,
self.graph.matrices_size)
for v in sources:
m_src[v, v] = True
self.src_for_states[self.grammar.start_state[
self.grammar.start_nonterm]][v, v] = True
sizeKron = self.graph.matrices_size * self.grammar.matrices_size
kron = Matrix.sparse(BOOL, sizeKron, sizeKron)
changed = True
src_changed = True
iter = 0
while changed or src_changed:
iter += 1
changed = False
src_changed = False
for box in self.grammar.boxes:
for state in self.grammar.boxes[box]:
out_state = self.grammar.out_states.get(state, [])
for out in out_state:
if out[1] in self.grammar.nonterminals:
old_sum = self.src_for_states[
self.grammar.start_state[out[1]]].nvals
self.src_for_states[self.grammar.start_state[
out[1]]] += self.src_for_states[state]
if old_sum != self.src_for_states[
self.grammar.start_state[out[1]]].nvals:
src_changed = True
with BOOL.ANY_PAIR:
self.part_graph[
out[1]] += self.src_for_states[state].mxm(
self.graph[out[1]])
old_sum = self.src_for_states[out[0]].nvals
for elem in self.part_graph[out[1]].T.reduce_vector(
BOOL.ANY_MONOID):
self.src_for_states[out[0]][elem[0],
elem[0]] = True
if old_sum != self.src_for_states[out[0]].nvals:
src_changed = True
for label in self.grammar.labels:
kron += self.grammar[label].kronecker(self.part_graph[label])
transitive_closure(kron)
for start in self.grammar.nonterminals:
for element in self.grammar.states[start]:
i = element[0]
j = element[1]
start_i = i * self.graph.matrices_size
start_j = j * self.graph.matrices_size
control_sum = self.graph[start].nvals
block = kron[start_i:start_i + self.graph.matrices_size -
1, start_j:start_j +
self.graph.matrices_size - 1]
self.graph[start] += block
new_control_sum = self.graph[start].nvals
if new_control_sum != control_sum:
changed = True
return ResultAlgo(m_src.mxm(self.graph[self.grammar.start_nonterm], semiring=BOOL.ANY_PAIR), iter), \
self.graph[self.grammar.start_nonterm]
def __init__(self):
self.matrix = Matrix.sparse(BOOL)
def __init__(self):
self.matrix = Matrix.sparse(FP64)
