def test_matrix_gb_type():
v = Matrix.from_type(bool, 10)
assert v.gb_type == lib.GrB_BOOL
v = Matrix.from_type(int, 10)
assert v.gb_type == lib.GrB_INT64
v = Matrix.from_type(float, 10)
assert v.gb_type == lib.GrB_FP64
def execute_query(args, graph, query):
"""Finds reachable vertices in a graph for a query.
Args:
args: Parsed command line arguments that can contain fr: str (path to a file that contains many start vertices),
to: str (path to a file that contains many end vertices),
type: str (transitive closure type: Used 'adj' for multiplication by an adjacency matrix or
otherwise squaring is used
graph: src.classes.Graph containing vertices to search
query: src.classes.Graph containing path constraints
Returns:
A pair consisting of the intersection of a graph and a query and a matrix of reachable vertices.
"""
intersection = graph & query
intersection_matrix = Matrix.sparse(BOOL, intersection.vertices_amount, intersection.vertices_amount)
for _, matrix in intersection.label_matrices.items():
intersection_matrix += matrix
if args.type == 'adj':
print("adj closure:")
closure = transitive_closure_adj(intersection_matrix)
else:
print("sqr closure:")
closure = transitive_closure_sqr(intersection_matrix)
result = Matrix.sparse(BOOL, graph.vertices_amount, graph.vertices_amount)
for i, j, _ in zip(*closure.nonzero().to_lists()):
if (i in intersection.start_vertices) and (j in intersection.final_vertices):
result[i // query.vertices_amount, j // query.vertices_amount] = True
return intersection, filter_query_result(result, None if args.fr is None else read_vertices_set_from_file(args.fr),
None if args.to is None else read_vertices_set_from_file(args.to))
def from_cfg(self, cfg: CFG):
# Computing size as the sum of production sizes (1 for head + n in boody)
self.num_vert = sum(
1 + len(p.body) for p in cfg.productions
)
index_p = 0
for p in cfg.productions:
if (p.head.value not in self.graph_dict):
self.graph_dict[p.head.value] = Matrix.sparse(BOOL, self.num_vert, self.num_vert)
if p.body != []:
self.start_states.add(index_p)
self.var_by_vertices[(index_p, index_p + len(p.body))] = p.head.value
for body_sym in p.body:
if body_sym.value not in self.graph_dict:
self.graph_dict[body_sym.value] = Matrix.sparse(BOOL, self.num_vert, self.num_vert)
self.graph_dict[body_sym.value][index_p, index_p + 1] = True
self.vertices.add(index_p)
self.vertices.add(index_p + 1)
index_p += 1
self.final_states.add(index_p)
self.vertices.add(index_p)
index_p += 1
return self
def create_from_cfg(self, cfg):
self.vertices_num = sum([len(prod.body) + 1 for prod in cfg.productions])
curr_index = 0
for prod in cfg.productions:
if prod.head.value not in self.matrices.keys():
self.matrices[prod.head.value] = Matrix.sparse(types.BOOL, self.vertices_num, self.vertices_num)
start_state = curr_index
final_state = curr_index + len(prod.body)
self.start_states.add(start_state)
self.final_states.add(final_state)
self.production_heads[(start_state, final_state)] = prod.head.value
for body_part in prod.body:
v_from, label, v_to = curr_index, body_part.value, curr_index + 1
if label not in self.matrices.keys():
self.matrices[label] = Matrix.sparse(types.BOOL, self.vertices_num, self.vertices_num)
self.matrices[label][v_from, v_to] = True
curr_index += 1
curr_index += 1
def test_apply_lambda():
v = Matrix.from_lists([0, 1, 2], [0, 1, 2], [22, 33, 44])
w = v.apply(lambda x: mod(x, 10))
assert w == Matrix.from_lists([0, 1, 2], [0, 1, 2], [2, 3, 4])
w = v.apply(lambda x: mod(x, 7))
assert w == Matrix.from_lists([0, 1, 2], [0, 1, 2], [1, 5, 2])
def test_matrix_create_dup():
m = Matrix.from_type(int, 10, 10)
m[3, 3] = 3
n = Matrix.dup(m)
assert n.nrows == 10
assert n.ncols == 10
assert n.nvals == 1
assert n[3, 3] == 3
def test_kron():
n = Matrix.from_lists(list(range(3)), list(range(3)), list(range(3)))
m = Matrix.from_lists(list(range(3)), list(range(3)), list(range(3)))
o = n.kron(m)
assert o == Matrix.from_lists([0, 1, 2, 3, 4, 5, 6, 7, 8],
[0, 1, 2, 3, 4, 5, 6, 7, 8],
[0, 0, 0, 0, 1, 2, 0, 2, 4])
def test_mxm():
m = Matrix.from_lists([0, 1, 2], [0, 1, 2], [1, 2, 3])
n = Matrix.from_lists([0, 1, 2], [0, 1, 2], [1, 2, 3])
actual_res = m @ n
expected_res = Matrix.from_lists([0, 1, 2], [0, 1, 2], [1, 4, 9])
assert actual_res.iseq(expected_res), "Matrices are not equal"
def test_matrix_create_from_type():
m = Matrix.from_type(int)
assert m.nrows == 0
assert m.ncols == 0
assert m.nvals == 0
m = Matrix.from_type(int, 10, 10)
assert m.nrows == 10
assert m.ncols == 10
assert m.nvals == 0
def test_matrix_multiplication():
a = Matrix.from_lists([0, 0, 1, 1], [0, 1, 0, 1], [4, 1, 1, 2])
b = Matrix.from_lists([0, 0, 1, 1], [0, 1, 0, 1], [7, 1, 3, 1])
c = Matrix.from_lists([0, 0, 1, 1], [0, 1, 0, 1], [31, 5, 13, 3])
result = a @ b
assert result.iseq(c)
def test_apply():
v = Matrix.from_lists(
[0, 1, 2],
[0, 1, 2],
[2, 3, 4])
w = v.apply(unaryop.ainv_int64)
assert w == Matrix.from_lists(
[0, 1, 2],
[0, 1, 2],
[-2, -3, -4])
def load_images(neurons, dest):
fname = "{}/sparse-images-{}.{}"
binfile = fname.format(dest, neurons, "ssb")
if Path(binfile).exists():
return Matrix.from_binfile(binfile.encode("ascii"))
images = Path(fname.format(dest, neurons, "tsv"))
with images.open() as i:
m = Matrix.from_tsv(i, FP32, NFEATURES, neurons)
m.to_binfile(binfile.encode("ascii"))
return m
def test_matrix_eq():
v = Matrix.from_lists(list(range(10)), list(range(10)), list(range(10)))
w = Matrix.from_lists(list(range(10)), list(range(10)), list(range(10)))
x = Matrix.from_lists(list(range(1, 11)),
list(range(1, 11)),
list(range(1, 11)),
ncols=11,
nrows=11)
assert v == w
assert v != x
def load_layer(neurons, dest, i):
fname = "{}/neuron{}/n{}-l{}.{}"
binfile = fname.format(dest, neurons, neurons, str(i + 1), "ssb")
if Path(binfile).exists():
return Matrix.from_binfile(binfile.encode("ascii"))
l = Path(fname.format(dest, neurons, neurons, str(i + 1), "tsv"))
with l.open() as f:
m = Matrix.from_tsv(f, FP32, neurons, neurons)
m.to_binfile(binfile.encode("ascii"))
return m
def cfpq_tensor_base(graph: Graph, cfg_wrapper: CFGrammar, is_cnf):
if graph.vertices_num == 0:
return []
res = AdjMatrix(graph).M_bin
query = cfg_wrapper.cnf if is_cnf else cfg_wrapper.cfg
rfa = RFA(query)
if cfg_wrapper.eps and is_cnf:
res[query.start_symbol] = Matrix.sparse(types.BOOL, graph.vertices_num,
graph.vertices_num)
for vertice in graph.vertices:
res[query.start_symbol][vertice, vertice] = True
for production in query.productions:
label = production.head.value
if label not in res.keys():
res[label] = Matrix.sparse(types.BOOL, graph.vertices_num,
graph.vertices_num)
if not (is_cnf or production.body):
for vertice in graph.vertices:
res[label][vertice, vertice] = True
changing = True
while changing:
changing = False
num_v_intersection = rfa.vertices_num * graph.vertices_num
intersection = AdjMatrix.adj_kronecker(rfa.matrices, res)
full_m_intersection = AdjMatrix.merge_label_matrices(
intersection, num_v_intersection)
closure = AdjMatrix.transitive_closure(full_m_intersection)
for v_from, v_to in AdjMatrix.reachable_vertices(closure):
rfa_from, rfa_to = v_from // graph.vertices_num, v_to // graph.vertices_num
graph_from, graph_to = v_from % graph.vertices_num, v_to % graph.vertices_num
if rfa_from in rfa.start_states and rfa_to in rfa.final_states:
label = rfa.production_heads[(rfa_from, rfa_to)]
if not res[label].get(graph_from, graph_to, False):
changing = True
res[label][graph_from, graph_to] = True
result = [
(v_from, v_to)
for v_from, v_to, _ in zip(*res[query.start_symbol.value].to_lists())
]
return result
def test_matrix_transpose():
v = Matrix.from_lists(
list(range(2, -1, -1)),
list(range(3)),
list(range(3)))
w = v.transpose()
assert w == Matrix.from_lists(
[0, 1, 2],
[2, 1, 0],
[0, 1, 2]
)
def draw_vector(V, column=True, *args, **kwargs): # pragma: nocover
from pygraphblas import Matrix
if column:
A = Matrix.sparse(V.type, V.size, 1)
A[:, 0] = V
else:
A = Matrix.sparse(V.type, 1, V.size)
A[0, :] = V
return draw_matrix(A, *args, **kwargs)
def test_vector_ewise_mult():
v = Matrix.from_lists(list(range(10)), list(range(10)), list(range(10)))
w = Matrix.from_lists(list(range(10)), list(range(10)), list(range(10)))
x = v.ewise_mult(w)
assert x == Matrix.from_lists(list(range(10)), list(range(10)),
list(map(lambda x: x * x, list(range(10)))))
z = v * w
assert x == z
v *= w
assert v == z
def test_matrix_ewise_add():
v = Matrix.from_lists(list(range(10)), list(range(10)), list(range(10)))
w = Matrix.from_lists(list(range(10)), list(range(10)), list(range(10)))
x = v.ewise_add(w)
assert x == Matrix.from_lists(list(range(10)), list(range(10)),
list(range(0, 20, 2)))
z = v + w
assert x == z
v += w
assert v == z
def transitive_closure_adj(matrix: Matrix) -> Matrix:
adj = matrix.dup()
result = matrix.dup()
changed = True
while changed:
old_nvals = result.nvals
result += adj @ result
new_nvals = result.nvals
if old_nvals == new_nvals:
changed = False
return result
def test_check_mult():
row_inds = [0, 0, 1, 1]
col_inds = [0, 1, 0, 1]
matrix_1 = Matrix.from_lists(row_inds, col_inds, [1, 2, 0, 3])
matrix_2 = Matrix.from_lists(row_inds, col_inds, [0, 1, 3, 5])
matrix_res = matrix_1 @ matrix_2
expected = Matrix.from_lists(row_inds, col_inds, [6, 11, 9, 15])
assert expected.iseq(matrix_res), "Matrices not equal"
def test_select_ops():
I, J = tuple(map(list, zip(*product(range(3), repeat=2))))
V = list(range(9))
m = Matrix.from_lists(I, J, V, 3, 3)
assert m.tril() == Matrix.from_lists(
[0, 1, 1, 2, 2, 2],
[0, 0, 1, 0, 1, 2],
[0, 3, 4, 6, 7, 8])
assert m.triu() == Matrix.from_lists(
[0, 0, 0, 1, 1, 2],
[0, 1, 2, 1, 2, 2],
[0, 1, 2, 4, 5, 8])
assert m.diag() == Matrix.from_lists(
[0, 1, 2],
[0, 1, 2],
[0, 4, 8])
assert m.offdiag() == Matrix.from_lists(
[0, 0, 1, 1, 2, 2],
[1, 2, 0, 2, 0, 1],
[1, 2, 3, 5, 6, 7])
assert m.nonzero() == Matrix.from_lists(
[0, 0, 1, 1, 1, 2, 2, 2],
[1, 2, 0, 1, 2, 0, 1, 2],
[1, 2, 3, 4, 5, 6, 7, 8])
assert -m == Matrix.from_lists(
[0, 0, 0, 1, 1, 1, 2, 2, 2],
[0, 1, 2, 0, 1, 2, 0, 1, 2],
[0, -1, -2, -3, -4, -5, -6, -7, -8])
n = -m
assert abs(m) == Matrix.from_lists(
[0, 0, 0, 1, 1, 1, 2, 2, 2],
[0, 1, 2, 0, 1, 2, 0, 1, 2],
[0, 1, 2, 3, 4, 5, 6, 7, 8])
m = Matrix.from_lists(
[0, 1, 2],
[0, 1, 2],
[0.0, 1.0, 2.0], 3, 3)
n = ~m
assert n == Matrix.from_lists(
[0, 1, 2],
[0, 1, 2],
[float('inf'), 1.0, 0.5])
def cfpq_matrix(graph: Graph, grammar: CFG):
size = graph.size
if size == 0:
return Matrix.sparse(BOOL, size, size)
result = Graph()
start_symbol = grammar.start_symbol
result.size = size
for variable in grammar.variables:
result.label_dictionary[variable] = Matrix.sparse(BOOL, size, size)
for label in graph.label_dictionary:
terminal = Terminal(label)
result.label_dictionary[terminal] = graph.label_dictionary[
label].dup()
for from_, to in graph.get_edges(label):
for production in grammar.productions:
if len(production.body
) == 1 and production.body[0] == terminal:
head = production.head
result.label_dictionary[head][from_, to] = 1
if grammar.generate_epsilon():
for vertex in graph.vertices:
result.label_dictionary[start_symbol][vertex, vertex] = 1
terminal_productions = set()
nonterminal_productions = set()
for production in grammar.productions:
if len(production.body) == 1:
terminal_productions.add(production)
elif len(production.body) >= 2:
nonterminal_productions.add(production)
matrix_changing = True
with semiring.LOR_LAND_BOOL:
while matrix_changing:
matrix_changing = False
for production in nonterminal_productions:
head = production.head
body = production.body
if len(body) == 2:
previous = result.label_dictionary[head].nvals
current = result.label_dictionary[
body[0]] @ result.label_dictionary[body[1]]
result.label_dictionary[
head] = result.label_dictionary[head] + current
if previous != result.label_dictionary[head].nvals:
matrix_changing = True
return result.label_dictionary[start_symbol]
def test_matrix_prod():
A = Matrix.from_lists([0, 0, 0, 1, 1, 1, 2, 2, 2],
[0, 1, 2, 0, 1, 2, 0, 1, 2],
[1, 2, 1, 2, 1, 2, 1, 2, 1])
B = Matrix.from_lists([0, 0, 0, 1, 1, 1, 2, 2, 2],
[0, 1, 2, 0, 1, 2, 0, 1, 2],
[2, 1, 2, 1, 2, 1, 2, 1, 2])
A_times_B = Matrix.from_lists([0, 0, 0, 1, 1, 1, 2, 2, 2],
[0, 1, 2, 0, 1, 2, 0, 1, 2],
[6, 6, 6, 9, 6, 9, 6, 6, 6])
assert A_times_B.iseq(A @ B)
def test_mxm():
m = Matrix.from_lists([0, 1, 2], [1, 2, 0], [1, 2, 3])
n = Matrix.from_lists([0, 1, 2], [1, 2, 0], [2, 3, 4])
o = m.mxm(n)
assert o.nrows == 3
assert o.ncols == 3
assert o.nvals == 3
r = Matrix.from_lists([0, 1, 2], [2, 0, 1], [3, 8, 6])
assert o == r
assert r == m @ n
m @= n
assert r == m
o = m.mxm(n, semiring=semiring.lor_land_bool)
assert o == Matrix.from_lists([0, 1, 2], [0, 1, 2], [1, 1, 1])
def test_matrix_assign():
m = Matrix.from_lists(list(range(3)), list(range(3)), list(range(3)))
assert m.nvals == 3
m[2] = Vector.from_list(list(repeat(6, 3)))
assert m.nvals == 5
assert m == Matrix.from_lists([0, 1, 2, 2, 2], [0, 1, 0, 1, 2],
[0, 1, 6, 6, 6], 3, 3)
m = Matrix.from_lists(list(range(3)), list(range(3)), list(range(3)))
assert m.nvals == 3
m[2, :] = Vector.from_list(list(repeat(6, 3)))
assert m.nvals == 5
assert m == Matrix.from_lists([0, 1, 2, 2, 2], [0, 1, 0, 1, 2],
[0, 1, 6, 6, 6], 3, 3)
m = Matrix.from_lists(list(range(3)), list(range(3)), list(range(3)))
assert m.nvals == 3
m[:, 2] = Vector.from_list(list(repeat(6, 3)))
assert m.nvals == 5
assert m == Matrix.from_lists([0, 1, 0, 1, 2], [0, 1, 2, 2, 2],
[0, 1, 6, 6, 6], 3, 3)
m = Matrix.from_type(int, 3, 3)
assert m.nvals == 0
n = Matrix.from_lists([0, 1, 2], [0, 1, 2], [0, 1, 2])
m[:, :] = n
assert m == n
def cfpq_hellings(self, grammar: GrammarWrapper) -> Set[Tuple[int, int]]:
result: Dict[Variable, Matrix] = {}
working_queue = deque()
if grammar.generate_epsilon:
result[grammar.cfg.start_symbol] = Matrix.sparse(
types.BOOL, self.vertices_num, self.vertices_num)
for v in self.vertices:
result[grammar.cfg.start_symbol][v, v] = True
working_queue.append((v, v, grammar.cfg.start_symbol))
with semiring.LOR_LAND_BOOL:
for label, matrix in self.label_to_bool_matrix.items():
for prod in grammar.cnf.productions:
if len(prod.body) == 1 and Terminal(label) == prod.body[0]:
if prod.head in result:
result[prod.head] += matrix.dup()
else:
result[prod.head] = matrix.dup()
for i, j, _ in matrix:
working_queue.append((i, j, prod.head))
while len(working_queue) != 0:
node_from, node_to, var = working_queue.popleft()
update = []
for var_before, matrix in result.items():
for node_before, _ in matrix[:, node_from]:
for prod in grammar.cnf.productions:
if (len(prod.body) == 2 and prod.body[0] == var_before
and prod.body[1] == var and
(prod.head not in result or result[prod.head].get(
node_before, node_to) is None)):
update.append((node_before, node_to, prod.head))
for var_after, matrix in result.items():
for node_after, _ in matrix[node_to]:
for prod in grammar.cnf.productions:
if (len(prod.body) == 2 and prod.body[0] == var
and prod.body[1] == var_after and
(prod.head not in result or result[prod.head].get(
node_from, node_after) is None)):
update.append((node_from, node_after, prod.head))
for node_from, node_to, var in update:
working_queue.append((node_from, node_to, var))
if var in result:
result[var][node_from, node_to] = True
else:
empty_matrix = Matrix.sparse(types.BOOL, self.vertices_num,
self.vertices_num)
result[var] = empty_matrix
result[var][node_from, node_to] = True
return set([(i, j)
for i, j, _ in result.get(grammar.cfg.start_symbol, [])])
def context_free_path_querying_tensors(rec_automata: Graph,
epsilon_generate_set: AbstractSet,
graph: Graph):
m2 = graph.copy()
for n in epsilon_generate_set:
for j in range(m2.vertices_amount):
m2[n][j, j] = True
is_changed = True
while is_changed:
old_nvals = m2.nvals()
intersection = rec_automata & m2
m3 = Matrix.sparse(BOOL, intersection.vertices_amount,
intersection.vertices_amount)
for _, matrix in intersection.label_matrices.items():
m3 += matrix
tc3 = transitive_closure_sqr(m3)
for i, j, _ in zip(*tc3.nonzero().to_lists()):
if (i // graph.vertices_amount in rec_automata.start_vertices
) and (j // graph.vertices_amount
in rec_automata.final_vertices):
for label, matrix in m2.label_matrices.items():
if label.isupper():
m2[label][i % m2.vertices_amount,
j % m2.vertices_amount] = True
is_changed = m2.nvals() != old_nvals
return m2
def load_block_graph(self, suffix):
bhash = self.hash
prefix = self.chain.block_path / bhash[-2] / bhash[-1]
datafile = prefix / f"{self.number}_{bhash}_{suffix}.ssb"
if not datafile.exists():
return maximal_matrix(UINT64)
return Matrix.from_binfile(bytes(datafile))
def generate_bias():
for i in range(nneurons):
bias = Matrix.from_type(lib.GrB_FP32, nneurons, nneurons)
for i in range(nneurons):
bias[i, i] = 0.3
bias.nvals # causes async completion
yield bias
