def cc_push_topo(graph: PropertyGraph, property_name):
print("Executing Push algo\n")
num_nodes = graph.num_nodes()
timer = StatTimer("CC: Property Graph Numba: " + property_name)
timer.start()
# Stores the component id assignment
comp_current = np.empty((num_nodes,), dtype=np.uint32)
comp_old = np.empty((num_nodes,), dtype=np.uint32)
# Initialize
do_all(
range(num_nodes),
initialize_cc_push_operator(graph, comp_current, comp_old),
steal=True,
loop_name="initialize_cc_push",
)
# Execute while component ids are updated
changed = GReduceLogicalOr()
changed.update(True)
while changed.reduce():
changed.reset()
do_all(
range(num_nodes),
cc_push_topo_operator(graph, changed, comp_current, comp_old),
steal=True,
loop_name="cc_push_topo",
)
timer.stop()
# Add the component assignment as a new property to the property graph
graph.add_node_property(pyarrow.table({property_name: comp_current}))
def verify_bfs(graph: PropertyGraph, _source_i: int, property_id: int):
chunk_array = graph.get_node_property(property_id)
not_visited = GAccumulator[int](0)
max_dist = GReduceMax[int]()
do_all(
range(len(chunk_array)),
not_visited_operator(not_visited, chunk_array),
loop_name="not_visited_op",
)
if not_visited.reduce() > 0:
print(
not_visited.reduce(),
" unvisited nodes; this is an error if graph is strongly connected",
)
do_all(
range(len(chunk_array)),
max_dist_operator(max_dist, chunk_array),
steal=True,
loop_name="max_dist_operator",
)
print("BFS Max distance:", max_dist.reduce())
def bfs_sync_pg(graph: PropertyGraph, source, property_name):
next_level_number = 0
curr_level = InsertBag[np.uint64]()
next_level = InsertBag[np.uint64]()
timer = StatTimer("BFS Property Graph Numba: " + property_name)
timer.start()
distance = np.empty((len(graph), ), dtype=np.uint32)
initialize(graph, source, distance)
next_level.push(source)
while not next_level.empty():
curr_level.swap(next_level)
next_level.clear()
next_level_number += 1
do_all(
curr_level,
bfs_sync_operator_pg(graph, next_level, next_level_number,
distance),
steal=True,
loop_name="bfs_sync_pg",
)
timer.stop()
graph.add_node_property(pyarrow.table({property_name: distance}))
def test_do_all(modes):
@do_all_operator()
def f(out, i):
out[i] = i + 1
out = np.zeros(10, dtype=int)
do_all(range(10), f(out), **modes)
assert np.allclose(out, np.array(range(1, 11)))
def test_atomic_min_parallel(dtype, threads_many):
@do_all_operator()
def f(out, i):
atomic_min(out, 0, i)
out = np.array([500], dtype=dtype)
do_all(range(1000), f(out), steal=False)
assert out[0] == 0
def test_do_all_wrong_closure():
@for_each_operator()
def f(out, i, ctx):
out[i] = i + 1
out = np.zeros(10, dtype=int)
with pytest.raises(TypeError):
do_all(range(10), f(out))
def test_do_all_python(modes):
total = 0
def f(i):
nonlocal total
total += i
do_all(range(10), f, **modes)
assert total == 45
def verify_cc(graph: PropertyGraph, property_id: int):
chunk_array = graph.get_node_property(property_id)
num_components = GAccumulator[int](0)
do_all(
range(len(chunk_array)), verify_cc_operator(num_components, chunk_array), loop_name="num_components",
)
print("Number of components are : ", num_components.reduce())
def test_atomic_add_parallel_largearray(threads_many):
@do_all_operator()
def f(out, i):
atomic_add(out, 0, i)
out = LargeArray[int]()
out.allocateBlocked(1000)
do_all(range(1000), f(out.as_numpy()), steal=False)
assert out[0] == 499500
def test_GReduceLogicalAnd_parallel(threads_many):
T = GReduceLogicalAnd
acc = T()
@do_all_operator()
def f(acc, i):
acc.update(i % 3 == 0)
do_all(range(1000), f(acc), steal=False)
assert acc.reduce() == False
def test_GReduceMin_parallel(threads_many):
T = GReduceMin[float]
acc = T()
@do_all_operator()
def f(acc, i):
acc.update((i - 500) / 10)
do_all(range(1000), f(acc), steal=False)
assert acc.reduce() == -50.0
def test_GReduceMax_parallel(threads_many):
T = GReduceMax[int]
acc = T()
@do_all_operator()
def f(acc, i):
acc.update(abs(500 - i))
do_all(range(1000), f(acc), steal=False)
assert acc.reduce() == 500
def test_GAccumulator_parallel(threads_many):
T = GAccumulator[int]
acc = T()
@do_all_operator()
def f(acc, i):
acc.update(i)
do_all(range(1000), f(acc), steal=False)
assert acc.reduce() == 499500
def test_LargeArray_numpy_parallel(typ):
T = LargeArray[typ]
arr = T()
arr.allocateInterleaved(1000)
@do_all_operator()
def f(arr, i):
arr[i] = i
arr[i] += 1
do_all(range(1000), f(arr.as_numpy()), steal=False)
assert list(arr) == list(range(1, 1001))
def test_InsertBag_parallel(typ):
T = InsertBag[typ]
bag = T()
@do_all_operator()
def f(bag, i):
bag.push(i)
bag.push(i)
do_all(range(1000), f(bag), steal=False)
l = list(bag)
l.sort()
assert l == [v for i in range(1000) for v in [i, i]]
def test_LargeArray_parallel(typ):
T = LargeArray[typ]
arr = T()
arr.allocateInterleaved(1000)
@do_all_operator()
def f(arr, i):
# TODO: Use __setitem__
arr.set(i, i)
arr.set(i, arr.get(i) + 1)
do_all(range(1000), f(arr), steal=False)
assert list(arr) == list(range(1, 1001))
def verify_kcore(graph: PropertyGraph, property_name: str, k_core_num: int):
"""Check output sanity"""
chunk_array = graph.get_node_property(property_name)
alive_nodes = GAccumulator[float](0)
do_all(
range(len(chunk_array)),
sanity_check_operator(alive_nodes, chunk_array, k_core_num),
steal=True,
loop_name="sanity_check_operator",
)
print("Number of nodes in the", k_core_num, "-core is",
alive_nodes.reduce())
def test_InsertBag_parallel_opaque():
dt = np.dtype([
("x", np.float32),
("y", np.int16),
], align=True)
T = InsertBag[dt]
bag = T()
@do_all_operator()
def f(bag, i):
bag.push((i / 2.0, i))
do_all(range(1000), f(bag), steal=False)
for s in bag:
assert s.x == pytest.approx(s.y / 2.0)
def test_do_all_opaque(modes):
from galois.datastructures import InsertBag
@do_all_operator()
def f(out, s):
out[s.y] = s.x
dt = np.dtype([("x", np.float32), ("y", np.int8),], align=True)
input = InsertBag[dt]()
input.push((1.1, 0))
input.push((2.1, 1))
input.push((3.1, 3))
out = np.zeros(4, dtype=float)
do_all(input, f(out), **modes)
assert np.allclose(out, np.array([1.1, 2.1, 0, 3.1]))
def jaccard(g, key_node, property_name):
key_neighbors = np.zeros(len(g), dtype=bool)
output = np.empty(len(g), dtype=float)
for e in g.edges(key_node):
n = g.get_edge_dst(e)
key_neighbors[n] = True
do_all(
g,
jaccard_operator(g, key_neighbors, len(g.edges(key_node)), output),
steal=True,
loop_name="jaccard",
)
g.add_node_property(pyarrow.table({property_name: output}))
def test_do_all_specific_type(modes, typ):
from galois.datastructures import InsertBag
@do_all_operator()
def f(out, i):
out[int(i)] = i
input = InsertBag[typ]()
for i in range(1000):
input.push(i)
out = np.zeros(1000, dtype=typ)
do_all(input, f(out), **modes)
assert np.allclose(out, np.array(range(1000)))
# Check that the operator was actually compiled for the correct type
assert list(f.inspect_llvm().keys())[0][1][0] == from_dtype(np.dtype(typ))
def pagerank_pull_sync_residual(graph: PropertyGraph, maxIterations, tolerance, property_name):
num_nodes = graph.num_nodes()
rank = LargeArray[float](num_nodes, AllocationPolicy.INTERLEAVED)
nout = LargeArray[np.uint64](num_nodes, AllocationPolicy.INTERLEAVED)
delta = LargeArray[float](num_nodes, AllocationPolicy.INTERLEAVED)
residual = LargeArray[float](num_nodes, AllocationPolicy.INTERLEAVED)
# Initialize
do_all(
range(num_nodes),
initialize_residual_operator(rank.as_numpy(), nout.as_numpy(), delta.as_numpy(), residual.as_numpy(),),
steal=True,
loop_name="initialize_pagerank_pull_residual",
)
# Compute out-degree for each node
do_all(
range(num_nodes), compute_out_deg_operator(graph, nout.as_numpy()), steal=True, loop_name="Compute_out_degree",
)
print("Out-degree of 0: ", nout[0])
changed = GReduceLogicalOr(True)
iterations = 0
timer = StatTimer("Pagerank: Property Graph Numba: " + property_name)
timer.start()
while iterations < maxIterations and changed.reduce():
print("Iter: ", iterations, "\n")
changed.reset()
iterations += 1
do_all(
range(num_nodes),
compute_pagerank_pull_delta_operator(
rank.as_numpy(), nout.as_numpy(), delta.as_numpy(), residual.as_numpy(), tolerance, changed,
),
steal=True,
loop_name="pagerank_delta",
)
do_all(
range(num_nodes),
compute_pagerank_pull_residual_operator(graph, delta.as_numpy(), residual.as_numpy()),
steal=True,
loop_name="pagerank",
)
timer.stop()
# Add the ranks as a new property to the property graph
graph.add_node_property(pyarrow.table({property_name: rank}))
def test_LargeArray_numpy_parallel_opaque():
dt = np.dtype([
("x", np.float32),
("y", np.int16),
], align=True)
T = LargeArray[dt]
arr = T()
arr.allocateInterleaved(1000)
@do_all_operator()
def f(arr, i):
arr[i].x = i
arr[i].y = i
arr[i].x += 1.1
do_all(range(1000), f(arr.as_numpy()), steal=False)
for i, s in enumerate(arr):
assert s.x == pytest.approx(i + 1.1)
assert s.y == i
assert arr[i].x == pytest.approx(i + 1.1)
assert arr[i].y == i
def test_simple_algorithm(property_graph):
@do_all_operator()
def func_operator(g, prop, out, nid):
t = 0
for eid in g.edges(nid):
nid2 = g.get_edge_dst(eid)
if prop.is_valid(nid2):
t += prop[nid2]
out[nid] = t
g = property_graph
prop = g.get_node_property("length")
out = np.empty((g.num_nodes(), ), dtype=int)
do_all(g, func_operator(g, prop, out), "operator")
g.add_node_property(pyarrow.table(dict(referenced_total_length=out)))
oprop = g.get_node_property("referenced_total_length")
assert oprop[0].as_py() == 91
assert oprop[4].as_py() == 239
assert oprop[-1].as_py() == 0
def kcore_async(graph: PropertyGraph, k_core_num, property_name):
num_nodes = graph.num_nodes()
initial_worklist = InsertBag[np.uint64]()
current_degree = LargeArray[np.uint64](num_nodes,
AllocationPolicy.INTERLEAVED)
timer = StatTimer("Kcore: Property Graph Numba: " + property_name)
timer.start()
# Initialize
do_all(
range(num_nodes),
compute_degree_count_operator(graph, current_degree.as_numpy()),
steal=True,
)
# Setup initial worklist
do_all(
range(num_nodes),
setup_initial_worklist_operator(initial_worklist,
current_degree.as_numpy(), k_core_num),
steal=True,
)
# Compute k-core
for_each(
initial_worklist,
compute_async_kcore_operator(graph, current_degree.as_numpy(),
k_core_num),
steal=True,
disable_conflict_detection=True,
)
timer.stop()
# Add the ranks as a new property to the property graph
graph.add_node_property(pyarrow.table({property_name: current_degree}))
def verify_pr(graph: PropertyGraph, property_name: str, topn: int):
"""Check output sanity"""
chunk_array = graph.get_node_property(property_name)
sum_rank = GAccumulator[float](0)
max_rank = GReduceMax[float]()
min_rank = GReduceMin[float]()
do_all(
range(len(chunk_array)),
sanity_check_operator(sum_rank, max_rank, min_rank, chunk_array),
steal=True,
loop_name="sanity_check_operator",
)
print("Max rank is ", max_rank.reduce())
print("Min rank is ", min_rank.reduce())
print("rank sum is ", sum_rank.reduce())
# Print top N ranked nodes
if topn > 0:
np_array = np.array(chunk_array, dtype=np.float)
arr = np_array.argsort()[-topn:][::-1]
for i in arr:
print(np_array[i], " : ", i, "\n")
