def check_rpc_hetero_etype_sampling_shuffle(tmpdir, num_server):
generate_ip_config("rpc_ip_config.txt", num_server, num_server)
g = create_random_hetero(dense=True)
num_parts = num_server
num_hops = 1
partition_graph(g, 'test_sampling', num_parts, tmpdir,
num_hops=num_hops, part_method='metis', reshuffle=True)
pserver_list = []
ctx = mp.get_context('spawn')
for i in range(num_server):
p = ctx.Process(target=start_server, args=(i, tmpdir, num_server > 1, 'test_sampling'))
p.start()
time.sleep(1)
pserver_list.append(p)
fanout = 3
block, gpb = start_hetero_etype_sample_client(0, tmpdir, num_server > 1, fanout,
nodes={'n3': [0, 10, 99, 66, 124, 208]})
print("Done sampling")
for p in pserver_list:
p.join()
src, dst = block.edges(etype=('n1', 'r2', 'n3'))
assert len(src) == 18
src, dst = block.edges(etype=('n2', 'r3', 'n3'))
assert len(src) == 18
orig_nid_map = {ntype: F.zeros((g.number_of_nodes(ntype),), dtype=F.int64) for ntype in g.ntypes}
orig_eid_map = {etype: F.zeros((g.number_of_edges(etype),), dtype=F.int64) for etype in g.etypes}
for i in range(num_server):
part, _, _, _, _, _, _ = load_partition(tmpdir / 'test_sampling.json', i)
ntype_ids, type_nids = gpb.map_to_per_ntype(part.ndata[dgl.NID])
for ntype_id, ntype in enumerate(g.ntypes):
idx = ntype_ids == ntype_id
F.scatter_row_inplace(orig_nid_map[ntype], F.boolean_mask(type_nids, idx),
F.boolean_mask(part.ndata['orig_id'], idx))
etype_ids, type_eids = gpb.map_to_per_etype(part.edata[dgl.EID])
for etype_id, etype in enumerate(g.etypes):
idx = etype_ids == etype_id
F.scatter_row_inplace(orig_eid_map[etype], F.boolean_mask(type_eids, idx),
F.boolean_mask(part.edata['orig_id'], idx))
for src_type, etype, dst_type in block.canonical_etypes:
src, dst = block.edges(etype=etype)
# These are global Ids after shuffling.
shuffled_src = F.gather_row(block.srcnodes[src_type].data[dgl.NID], src)
shuffled_dst = F.gather_row(block.dstnodes[dst_type].data[dgl.NID], dst)
shuffled_eid = block.edges[etype].data[dgl.EID]
orig_src = F.asnumpy(F.gather_row(orig_nid_map[src_type], shuffled_src))
orig_dst = F.asnumpy(F.gather_row(orig_nid_map[dst_type], shuffled_dst))
orig_eid = F.asnumpy(F.gather_row(orig_eid_map[etype], shuffled_eid))
# Check the node Ids and edge Ids.
orig_src1, orig_dst1 = g.find_edges(orig_eid, etype=etype)
assert np.all(F.asnumpy(orig_src1) == orig_src)
assert np.all(F.asnumpy(orig_dst1) == orig_dst)
def test_sharing():
data = Frame(create_test_data())
f1 = FrameRef(data, index=toindex([0, 1, 2, 3]))
f2 = FrameRef(data, index=toindex([2, 3, 4, 5, 6]))
# test read
for k, v in f1.items():
assert F.allclose(F.narrow_row(data[k].data, 0, 4), v)
for k, v in f2.items():
assert F.allclose(F.narrow_row(data[k].data, 2, 7), v)
f2_a1 = f2['a1']
# test write
# update own ref should not been seen by the other.
f1[Index(F.tensor([0, 1]))] = {
'a1': F.zeros([2, D]),
'a2': F.zeros([2, D]),
'a3': F.zeros([2, D]),
}
assert F.allclose(f2['a1'], f2_a1)
# update shared space should been seen by the other.
f1[Index(F.tensor([2, 3]))] = {
'a1': F.ones([2, D]),
'a2': F.ones([2, D]),
'a3': F.ones([2, D]),
}
F.narrow_row_set(f2_a1, 0, 2, F.ones([2, D]))
assert F.allclose(f2['a1'], f2_a1)
def test_send_multigraph(idtype):
g = dgl.graph([(0, 1), (0, 1), (0, 1), (2, 1)],
idtype=idtype,
device=F.ctx())
def _message_a(edges):
return {'a': edges.data['a']}
def _message_b(edges):
return {'a': edges.data['a'] * 3}
def _reduce(nodes):
return {'a': F.max(nodes.mailbox['a'], 1)}
def answer(*args):
return F.max(F.stack(args, 0), 0)
assert g.is_multigraph
# send by eid
old_repr = F.randn((4, 5))
# send_and_recv_on
g.ndata['a'] = F.zeros((3, 5))
g.edata['a'] = old_repr
g.send_and_recv([0, 2, 3], message_func=_message_a, reduce_func=_reduce)
new_repr = g.ndata['a']
assert F.allclose(new_repr[1], answer(old_repr[0], old_repr[2],
old_repr[3]))
assert F.allclose(new_repr[[0, 2]], F.zeros((2, 5)))
def test_filter():
g = DGLGraph()
g.add_nodes(4)
g.add_edges([0,1,2,3], [1,2,3,0])
n_repr = F.zeros((4, 5))
e_repr = F.zeros((4, 5))
n_repr[[1, 3]] = 1
e_repr[[1, 3]] = 1
g.ndata['a'] = n_repr
g.edata['a'] = e_repr
def predicate(r):
return F.max(r.data['a'], 1) > 0
# full node filter
n_idx = g.filter_nodes(predicate)
assert set(F.zerocopy_to_numpy(n_idx)) == {1, 3}
# partial node filter
n_idx = g.filter_nodes(predicate, [0, 1])
assert set(F.zerocopy_to_numpy(n_idx)) == {1}
# full edge filter
e_idx = g.filter_edges(predicate)
assert set(F.zerocopy_to_numpy(e_idx)) == {1, 3}
# partial edge filter
e_idx = g.filter_edges(predicate, [0, 1])
assert set(F.zerocopy_to_numpy(e_idx)) == {1}
def build(self):
self.gamma = K.ones((self.num_lstm, ))
self.beta = K.zeros((self.num_lstm, ))
self.running_mean = K.zeros((self.num_lstm, ))
self.running_std = K.ones((self.num_lstm, ))
self.updates = [(self.running_mean, None), (self.running_std, None)]
def test_row1():
# test row getter/setter
data = create_test_data()
f = FrameRef(Frame(data))
# getter
# test non-duplicate keys
rowid = Index(F.tensor([0, 2]))
rows = f[rowid]
for k, v in rows.items():
assert tuple(F.shape(v)) == (len(rowid), D)
assert F.allclose(v, F.gather_row(data[k], F.tensor(rowid.tousertensor())))
# test duplicate keys
rowid = Index(F.tensor([8, 2, 2, 1]))
rows = f[rowid]
for k, v in rows.items():
assert tuple(F.shape(v)) == (len(rowid), D)
assert F.allclose(v, F.gather_row(data[k], F.tensor(rowid.tousertensor())))
# setter
rowid = Index(F.tensor([0, 2, 4]))
vals = {'a1' : F.zeros((len(rowid), D)),
'a2' : F.zeros((len(rowid), D)),
'a3' : F.zeros((len(rowid), D)),
}
f[rowid] = vals
for k, v in f[rowid].items():
assert F.allclose(v, F.zeros((len(rowid), D)))
# setting rows with new column should raise error with error initializer
f.set_initializer(lambda shape, dtype : assert_(False))
def failed_update_rows():
vals['a4'] = F.ones((len(rowid), D))
f[rowid] = vals
assert check_fail(failed_update_rows)
def __init__(self, num_lstm, dim_frame, output_dim):
self.num_lstm = num_lstm
self.dim_frame = dim_frame
self.output_dim = output_dim
self.W_in_to_ingate = f_init((self.dim_frame, self.num_lstm))
self.W_in_to_forgetgate = f_init((self.dim_frame, self.num_lstm))
self.W_in_to_cell = f_init((self.dim_frame, self.num_lstm))
self.W_in_to_outgate = f_init((self.dim_frame, self.num_lstm))
self.W_hid_to_ingate = f_inner_init((self.num_lstm, self.num_lstm))
self.W_hid_to_forgetgate = f_inner_init((self.num_lstm, self.num_lstm))
self.W_hid_to_cell = f_inner_init((self.num_lstm, self.num_lstm))
self.W_hid_to_outgate = f_inner_init((self.num_lstm, self.num_lstm))
self.b_ingate = K.zeros((self.num_lstm, ))
self.b_forgetgate = f_forget_bias_init((self.num_lstm, ))
self.b_cell = K.zeros((self.num_lstm, ))
self.b_outgate = K.zeros((self.num_lstm, ))
self.W_cell_to_ingate = f_init2((self.num_lstm, ))
self.W_cell_to_forgetgate = f_init2((self.num_lstm, ))
self.W_cell_to_outgate = f_init2((self.num_lstm, ))
self.W_output = f_init((self.num_lstm, self.output_dim))
self.b_output = K.zeros((self.output_dim, ))
def start_sample_client_shuffle(rank,
tmpdir,
disable_shared_mem,
g,
num_servers,
group_id=0):
os.environ['DGL_GROUP_ID'] = str(group_id)
gpb = None
if disable_shared_mem:
_, _, _, gpb, _, _, _ = load_partition(tmpdir / 'test_sampling.json',
rank)
dgl.distributed.initialize("rpc_ip_config.txt")
dist_graph = DistGraph("test_sampling", gpb=gpb)
sampled_graph = sample_neighbors(dist_graph, [0, 10, 99, 66, 1024, 2008],
3)
orig_nid = F.zeros((g.number_of_nodes(), ), dtype=F.int64, ctx=F.cpu())
orig_eid = F.zeros((g.number_of_edges(), ), dtype=F.int64, ctx=F.cpu())
for i in range(num_servers):
part, _, _, _, _, _, _ = load_partition(tmpdir / 'test_sampling.json',
i)
orig_nid[part.ndata[dgl.NID]] = part.ndata['orig_id']
orig_eid[part.edata[dgl.EID]] = part.edata['orig_id']
src, dst = sampled_graph.edges()
src = orig_nid[src]
dst = orig_nid[dst]
assert sampled_graph.number_of_nodes() == g.number_of_nodes()
assert np.all(F.asnumpy(g.has_edges_between(src, dst)))
eids = g.edge_ids(src, dst)
eids1 = orig_eid[sampled_graph.edata[dgl.EID]]
assert np.array_equal(F.asnumpy(eids1), F.asnumpy(eids))
def build(self):
self.gamma = K.ones((self.num_lstm,))
self.beta = K.zeros((self.num_lstm,))
self.running_mean = K.zeros((self.num_lstm,))
self.running_std = K.ones((self.num_lstm,))
self.updates = [(self.running_mean, None), (self.running_std, None)]
def start_node_dataloader(rank, tmpdir, num_server, num_workers):
import dgl
import torch as th
dgl.distributed.initialize("mp_ip_config.txt", 1, num_workers=num_workers)
gpb = None
disable_shared_mem = num_server > 1
if disable_shared_mem:
_, _, _, gpb, _, _, _ = load_partition(tmpdir / 'test_sampling.json',
rank)
num_nodes_to_sample = 202
batch_size = 32
train_nid = th.arange(num_nodes_to_sample)
dist_graph = DistGraph("test_mp",
gpb=gpb,
part_config=tmpdir / 'test_sampling.json')
orig_nid = F.zeros((dist_graph.number_of_nodes(), ), dtype=F.int64)
orig_eid = F.zeros((dist_graph.number_of_edges(), ), dtype=F.int64)
for i in range(num_server):
part, _, _, _, _, _, _ = load_partition(tmpdir / 'test_sampling.json',
i)
orig_nid[part.ndata[dgl.NID]] = part.ndata['orig_id']
orig_eid[part.edata[dgl.EID]] = part.edata['orig_id']
# Create sampler
sampler = dgl.dataloading.MultiLayerNeighborSampler([5, 10])
# We need to test creating DistDataLoader multiple times.
for i in range(2):
# Create DataLoader for constructing blocks
dataloader = dgl.dataloading.NodeDataLoader(dist_graph,
train_nid,
sampler,
batch_size=batch_size,
shuffle=True,
drop_last=False,
num_workers=num_workers)
groundtruth_g = CitationGraphDataset("cora")[0]
max_nid = []
for epoch in range(2):
for idx, (_, _,
blocks) in zip(range(0, num_nodes_to_sample, batch_size),
dataloader):
block = blocks[-1]
o_src, o_dst = block.edges()
src_nodes_id = block.srcdata[dgl.NID][o_src]
dst_nodes_id = block.dstdata[dgl.NID][o_dst]
src_nodes_id = orig_nid[src_nodes_id]
dst_nodes_id = orig_nid[dst_nodes_id]
has_edges = groundtruth_g.has_edges_between(
src_nodes_id, dst_nodes_id)
assert np.all(F.asnumpy(has_edges))
max_nid.append(np.max(F.asnumpy(dst_nodes_id)))
# assert np.all(np.unique(np.sort(F.asnumpy(dst_nodes_id))) == np.arange(idx, batch_size))
del dataloader
dgl.distributed.exit_client(
) # this is needed since there's two test here in one process
def test_repr(index_dtype):
G = dgl.graph([(0,1), (0,2), (1,2)], num_nodes=10, index_dtype=index_dtype)
repr_string = G.__repr__()
print(repr_string)
G.ndata['x'] = F.zeros((10, 5))
G.edata['y'] = F.zeros((3, 4))
repr_string = G.__repr__()
print(repr_string)
def test_repr():
G = dgl.graph([(0,1), (0,2), (1,2)], card=10)
repr_string = G.__repr__()
print(repr_string)
G.ndata['x'] = F.zeros((10, 5))
G.edata['y'] = F.zeros((3, 4))
repr_string = G.__repr__()
print(repr_string)
def test_simple_readout():
g1 = dgl.DGLGraph()
g1.add_nodes(3)
g2 = dgl.DGLGraph()
g2.add_nodes(4) # no edges
g1.add_edges([0, 1, 2], [2, 0, 1])
n1 = F.randn((3, 5))
n2 = F.randn((4, 5))
e1 = F.randn((3, 5))
s1 = F.sum(n1, 0) # node sums
s2 = F.sum(n2, 0)
se1 = F.sum(e1, 0) # edge sums
m1 = F.mean(n1, 0) # node means
m2 = F.mean(n2, 0)
me1 = F.mean(e1, 0) # edge means
w1 = F.randn((3, ))
w2 = F.randn((4, ))
max1 = F.max(n1, 0)
max2 = F.max(n2, 0)
maxe1 = F.max(e1, 0)
ws1 = F.sum(n1 * F.unsqueeze(w1, 1), 0)
ws2 = F.sum(n2 * F.unsqueeze(w2, 1), 0)
wm1 = F.sum(n1 * F.unsqueeze(w1, 1), 0) / F.sum(F.unsqueeze(w1, 1), 0)
wm2 = F.sum(n2 * F.unsqueeze(w2, 1), 0) / F.sum(F.unsqueeze(w2, 1), 0)
g1.ndata['x'] = n1
g2.ndata['x'] = n2
g1.ndata['w'] = w1
g2.ndata['w'] = w2
g1.edata['x'] = e1
assert F.allclose(dgl.sum_nodes(g1, 'x'), s1)
assert F.allclose(dgl.sum_nodes(g1, 'x', 'w'), ws1)
assert F.allclose(dgl.sum_edges(g1, 'x'), se1)
assert F.allclose(dgl.mean_nodes(g1, 'x'), m1)
assert F.allclose(dgl.mean_nodes(g1, 'x', 'w'), wm1)
assert F.allclose(dgl.mean_edges(g1, 'x'), me1)
assert F.allclose(dgl.max_nodes(g1, 'x'), max1)
assert F.allclose(dgl.max_edges(g1, 'x'), maxe1)
g = dgl.batch([g1, g2])
s = dgl.sum_nodes(g, 'x')
m = dgl.mean_nodes(g, 'x')
max_bg = dgl.max_nodes(g, 'x')
assert F.allclose(s, F.stack([s1, s2], 0))
assert F.allclose(m, F.stack([m1, m2], 0))
assert F.allclose(max_bg, F.stack([max1, max2], 0))
ws = dgl.sum_nodes(g, 'x', 'w')
wm = dgl.mean_nodes(g, 'x', 'w')
assert F.allclose(ws, F.stack([ws1, ws2], 0))
assert F.allclose(wm, F.stack([wm1, wm2], 0))
s = dgl.sum_edges(g, 'x')
m = dgl.mean_edges(g, 'x')
max_bg_e = dgl.max_edges(g, 'x')
assert F.allclose(s, F.stack([se1, F.zeros(5)], 0))
assert F.allclose(m, F.stack([me1, F.zeros(5)], 0))
assert F.allclose(max_bg_e, F.stack([maxe1, F.zeros(5)], 0))
def test_readonly():
g = dgl.DGLGraph()
g.add_nodes(5)
g.add_edges([0, 1, 2, 3], [1, 2, 3, 4])
g.ndata['x'] = F.zeros((5, 3))
g.edata['x'] = F.zeros((4, 4))
g.readonly(False)
assert g._graph.is_readonly() == False
assert g.number_of_nodes() == 5
assert g.number_of_edges() == 4
g.readonly()
assert g._graph.is_readonly() == True
assert g.number_of_nodes() == 5
assert g.number_of_edges() == 4
try:
g.add_nodes(5)
fail = False
except DGLError:
fail = True
finally:
assert fail
g.readonly()
assert g._graph.is_readonly() == True
assert g.number_of_nodes() == 5
assert g.number_of_edges() == 4
try:
g.add_nodes(5)
fail = False
except DGLError:
fail = True
finally:
assert fail
g.readonly(False)
assert g._graph.is_readonly() == False
assert g.number_of_nodes() == 5
assert g.number_of_edges() == 4
try:
g.add_nodes(10)
g.add_edges([4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
[5, 6, 7, 8, 9, 10, 11, 12, 13, 14])
fail = False
except DGLError:
fail = True
finally:
assert not fail
assert g.number_of_nodes() == 15
assert F.shape(g.ndata['x']) == (15, 3)
assert g.number_of_edges() == 14
assert F.shape(g.edata['x']) == (14, 4)
def check_rpc_in_subgraph_shuffle(tmpdir, num_server):
ip_config = open("rpc_ip_config.txt", "w")
for _ in range(num_server):
ip_config.write('{}\n'.format(get_local_usable_addr()))
ip_config.close()
g = CitationGraphDataset("cora")[0]
g.readonly()
num_parts = num_server
partition_graph(g,
'test_in_subgraph',
num_parts,
tmpdir,
num_hops=1,
part_method='metis',
reshuffle=True)
pserver_list = []
ctx = mp.get_context('spawn')
for i in range(num_server):
p = ctx.Process(target=start_server,
args=(i, tmpdir, num_server > 1, 'test_in_subgraph'))
p.start()
time.sleep(1)
pserver_list.append(p)
nodes = [0, 10, 99, 66, 1024, 2008]
time.sleep(3)
sampled_graph = start_in_subgraph_client(0, tmpdir, num_server > 1, nodes)
for p in pserver_list:
p.join()
orig_nid = F.zeros((g.number_of_nodes(), ), dtype=F.int64, ctx=F.cpu())
orig_eid = F.zeros((g.number_of_edges(), ), dtype=F.int64, ctx=F.cpu())
for i in range(num_server):
part, _, _, _, _, _, _ = load_partition(
tmpdir / 'test_in_subgraph.json', i)
orig_nid[part.ndata[dgl.NID]] = part.ndata['orig_id']
orig_eid[part.edata[dgl.EID]] = part.edata['orig_id']
src, dst = sampled_graph.edges()
src = orig_nid[src]
dst = orig_nid[dst]
assert sampled_graph.number_of_nodes() == g.number_of_nodes()
assert np.all(F.asnumpy(g.has_edges_between(src, dst)))
subg1 = dgl.in_subgraph(g, orig_nid[nodes])
src1, dst1 = subg1.edges()
assert np.all(np.sort(F.asnumpy(src)) == np.sort(F.asnumpy(src1)))
assert np.all(np.sort(F.asnumpy(dst)) == np.sort(F.asnumpy(dst1)))
eids = g.edge_ids(src, dst)
eids1 = orig_eid[sampled_graph.edata[dgl.EID]]
assert np.array_equal(F.asnumpy(eids1), F.asnumpy(eids))
def test_repr(idtype):
g = dgl.graph([(0, 1), (0, 2), (1, 2)],
num_nodes=10,
idtype=idtype,
device=F.ctx())
repr_string = g.__repr__()
print(repr_string)
g.ndata['x'] = F.zeros((10, 5))
g.edata['y'] = F.zeros((3, 4))
repr_string = g.__repr__()
print(repr_string)
def test_repr():
G = dgl.DGLGraph()
G.add_nodes(10)
G.add_edge(0, 1)
repr_string = G.__repr__()
print(repr_string)
G.ndata['x'] = F.zeros((10, 5))
G.add_edges([0, 1], 2)
G.edata['y'] = F.zeros((3, 4))
repr_string = G.__repr__()
print(repr_string)
def test_repr(idtype):
g = dgl.graph(([0, 0, 1], [1, 2, 2]),
num_nodes=10,
idtype=idtype,
device=F.ctx())
repr_string = g.__repr__()
print(repr_string)
g.ndata['x'] = F.zeros((10, 5))
g.edata['y'] = F.zeros((3, 4))
repr_string = g.__repr__()
print(repr_string)
def test_repr():
g = dgl.DGLGraph()
g = g.to(F.ctx())
g.add_nodes(10)
g.add_edge(0, 1)
repr_string = g.__repr__()
print(repr_string)
g.ndata['x'] = F.zeros((10, 5))
g.add_edges([0, 1], 2)
g.edata['y'] = F.zeros((3, 4))
repr_string = g.__repr__()
print(repr_string)
def test_v2v_snr_multi_fn(idtype):
u = F.tensor([0, 0, 0, 3, 4, 9], idtype)
v = F.tensor([1, 2, 3, 9, 9, 0], idtype)
def message_func(edges):
return {'m2': edges.src['f2']}
def message_func_edge(edges):
return {'m2': edges.src['f2'] * edges.data['e2']}
def reduce_func(nodes):
return {'v1': F.sum(nodes.mailbox['m2'], 1)}
g = generate_graph(idtype)
g.ndata.update({
'v1': F.zeros((10, D)),
'v2': F.zeros((10, D)),
'v3': F.zeros((10, D))
})
fld = 'f2'
g.send_and_recv((u, v), message_func, reduce_func)
v1 = g.ndata['v1']
# 1 message, 2 reduces
g.send_and_recv((u, v), fn.copy_src(src=fld, out='m'),
[fn.sum(msg='m', out='v2'),
fn.sum(msg='m', out='v3')], None)
v2 = g.ndata['v2']
v3 = g.ndata['v3']
assert F.allclose(v1, v2)
assert F.allclose(v1, v3)
# send and recv with edge weights, 2 message, 3 reduces
g.send_and_recv((u, v), [
fn.src_mul_edge(src=fld, edge='e1', out='m1'),
fn.src_mul_edge(src=fld, edge='e2', out='m2')
], [
fn.sum(msg='m1', out='v1'),
fn.sum(msg='m2', out='v2'),
fn.sum(msg='m1', out='v3')
], None)
v1 = g.ndata['v1']
v2 = g.ndata['v2']
v3 = g.ndata['v3']
assert F.allclose(v1, v2)
assert F.allclose(v1, v3)
# run UDF with single message and reduce
g.send_and_recv((u, v), message_func_edge, reduce_func, None)
v2 = g.ndata['v2']
assert F.allclose(v1, v2)
def test_add_rows():
data = Frame()
f1 = FrameRef(data)
f1.add_rows(4)
x = F.randn((1, 4))
f1[Index(F.tensor([0]))] = {'x': x}
ans = F.cat([x, F.zeros((3, 4))], 0)
assert F.allclose(f1['x'], ans)
f1.add_rows(4)
f1[toindex(slice(4, 8))] = {'x': F.ones((4, 4)), 'y': F.ones((4, 5))}
ans = F.cat([ans, F.ones((4, 4))], 0)
assert F.allclose(f1['x'], ans)
ans = F.cat([F.zeros((4, 5)), F.ones((4, 5))], 0)
assert F.allclose(f1['y'], ans)
def test_local_var():
g = DGLGraph(nx.path_graph(5))
g = g.to(F.ctx())
g.ndata['h'] = F.zeros((g.number_of_nodes(), 3))
g.edata['w'] = F.zeros((g.number_of_edges(), 4))
# test override
def foo(g):
g = g.local_var()
g.ndata['h'] = F.ones((g.number_of_nodes(), 3))
g.edata['w'] = F.ones((g.number_of_edges(), 4))
foo(g)
assert F.allclose(g.ndata['h'], F.zeros((g.number_of_nodes(), 3)))
assert F.allclose(g.edata['w'], F.zeros((g.number_of_edges(), 4)))
# test out-place update
def foo(g):
g = g.local_var()
g.nodes[[2, 3]].data['h'] = F.ones((2, 3))
g.edges[[2, 3]].data['w'] = F.ones((2, 4))
foo(g)
assert F.allclose(g.ndata['h'], F.zeros((g.number_of_nodes(), 3)))
assert F.allclose(g.edata['w'], F.zeros((g.number_of_edges(), 4)))
# test out-place update 2
def foo(g):
g = g.local_var()
g.apply_nodes(lambda nodes: {'h' : nodes.data['h'] + 10}, [2, 3])
g.apply_edges(lambda edges: {'w' : edges.data['w'] + 10}, [2, 3])
foo(g)
assert F.allclose(g.ndata['h'], F.zeros((g.number_of_nodes(), 3)))
assert F.allclose(g.edata['w'], F.zeros((g.number_of_edges(), 4)))
# test auto-pop
def foo(g):
g = g.local_var()
g.ndata['hh'] = F.ones((g.number_of_nodes(), 3))
g.edata['ww'] = F.ones((g.number_of_edges(), 4))
foo(g)
assert 'hh' not in g.ndata
assert 'ww' not in g.edata
# test initializer1
g = DGLGraph()
g = g.to(F.ctx())
g.add_nodes(2)
g.add_edges([0, 1], [1, 1])
g.set_n_initializer(dgl.init.zero_initializer)
def foo(g):
g = g.local_var()
g.nodes[0].data['h'] = F.ones((1, 1))
assert F.allclose(g.ndata['h'], F.tensor([[1.], [0.]]))
foo(g)
# test initializer2
def foo_e_initializer(shape, dtype, ctx, id_range):
return F.ones(shape)
g.set_e_initializer(foo_e_initializer, field='h')
def foo(g):
g = g.local_var()
g.edges[0, 1].data['h'] = F.ones((1, 1))
assert F.allclose(g.edata['h'], F.ones((2, 1)))
g.edges[0, 1].data['w'] = F.ones((1, 1))
assert F.allclose(g.edata['w'], F.tensor([[1.], [0.]]))
foo(g)
def test_local_scope():
g = DGLGraph(nx.path_graph(5))
g.ndata['h'] = F.zeros((g.number_of_nodes(), 3))
g.edata['w'] = F.zeros((g.number_of_edges(), 4))
# test override
def foo(g):
with g.local_scope():
g.ndata['h'] = F.ones((g.number_of_nodes(), 3))
g.edata['w'] = F.ones((g.number_of_edges(), 4))
foo(g)
assert F.allclose(g.ndata['h'], F.zeros((g.number_of_nodes(), 3)))
assert F.allclose(g.edata['w'], F.zeros((g.number_of_edges(), 4)))
# test out-place update
def foo(g):
with g.local_scope():
g.nodes[[2, 3]].data['h'] = F.ones((2, 3))
g.edges[[2, 3]].data['w'] = F.ones((2, 4))
foo(g)
assert F.allclose(g.ndata['h'], F.zeros((g.number_of_nodes(), 3)))
assert F.allclose(g.edata['w'], F.zeros((g.number_of_edges(), 4)))
# test out-place update 2
def foo(g):
with g.local_scope():
g.apply_nodes(lambda nodes: {'h': nodes.data['h'] + 10}, [2, 3])
g.apply_edges(lambda edges: {'w': edges.data['w'] + 10}, [2, 3])
foo(g)
assert F.allclose(g.ndata['h'], F.zeros((g.number_of_nodes(), 3)))
assert F.allclose(g.edata['w'], F.zeros((g.number_of_edges(), 4)))
# test auto-pop
def foo(g):
with g.local_scope():
g.ndata['hh'] = F.ones((g.number_of_nodes(), 3))
g.edata['ww'] = F.ones((g.number_of_edges(), 4))
foo(g)
assert 'hh' not in g.ndata
assert 'ww' not in g.edata
# test nested scope
def foo(g):
with g.local_scope():
g.ndata['hh'] = F.ones((g.number_of_nodes(), 3))
g.edata['ww'] = F.ones((g.number_of_edges(), 4))
with g.local_scope():
g.ndata['hhh'] = F.ones((g.number_of_nodes(), 3))
g.edata['www'] = F.ones((g.number_of_edges(), 4))
assert 'hhh' not in g.ndata
assert 'www' not in g.edata
foo(g)
assert 'hh' not in g.ndata
assert 'ww' not in g.edata
def check_rpc_sampling_shuffle(tmpdir):
num_server = 2
ip_config = open("rpc_sampling_ip_config.txt", "w")
for _ in range(num_server):
ip_config.write('{} 1\n'.format(get_local_usable_addr()))
ip_config.close()
g = CitationGraphDataset("cora")[0]
g.readonly()
num_parts = num_server
num_hops = 1
partition_graph(g,
'test_sampling',
num_parts,
tmpdir,
num_hops=num_hops,
part_method='metis',
reshuffle=True)
pserver_list = []
ctx = mp.get_context('spawn')
for i in range(num_server):
p = ctx.Process(target=start_server, args=(i, tmpdir))
p.start()
time.sleep(1)
pserver_list.append(p)
time.sleep(3)
sampled_graph = start_client(0, tmpdir)
print("Done sampling")
for p in pserver_list:
p.join()
orig_nid = F.zeros((g.number_of_nodes(), ), dtype=F.int64)
orig_eid = F.zeros((g.number_of_edges(), ), dtype=F.int64)
for i in range(num_server):
part, _, _, _ = load_partition(tmpdir / 'test_sampling.json', i)
orig_nid[part.ndata[dgl.NID]] = part.ndata['orig_id']
orig_eid[part.edata[dgl.EID]] = part.edata['orig_id']
src, dst = sampled_graph.edges()
src = orig_nid[src]
dst = orig_nid[dst]
assert sampled_graph.number_of_nodes() == g.number_of_nodes()
assert np.all(F.asnumpy(g.has_edges_between(src, dst)))
eids = g.edge_ids(src, dst)
eids1 = orig_eid[sampled_graph.edata[dgl.EID]]
assert np.array_equal(F.asnumpy(eids1), F.asnumpy(eids))
def test_mutation():
g = dgl.DGLGraph()
g = g.to(F.ctx())
# test add nodes with data
g.add_nodes(5)
g.add_nodes(5, {'h' : F.ones((5, 2))})
ans = F.cat([F.zeros((5, 2)), F.ones((5, 2))], 0)
assert F.allclose(ans, g.ndata['h'])
g.ndata['w'] = 2 * F.ones((10, 2))
assert F.allclose(2 * F.ones((10, 2)), g.ndata['w'])
# test add edges with data
g.add_edges([2, 3], [3, 4])
g.add_edges([0, 1], [1, 2], {'m' : F.ones((2, 2))})
ans = F.cat([F.zeros((2, 2)), F.ones((2, 2))], 0)
assert F.allclose(ans, g.edata['m'])
def test_column2():
# Test frameref column getter/setter
data = Frame(create_test_data())
f = FrameRef(data, toindex([3, 4, 5, 6, 7]))
assert f.num_rows == 5
assert len(f) == 3
assert F.allclose(f['a1'], F.narrow_row(data['a1'].data, 3, 8))
# set column should reflect on the referenced data
f['a1'] = F.zeros([5, D])
assert F.allclose(F.narrow_row(data['a1'].data, 3, 8), F.zeros([5, D]))
# add new partial column should fail with error initializer
f.set_initializer(lambda shape, dtype : assert_(False))
def failed_add_col():
f['a4'] = F.ones([5, D])
assert check_fail(failed_add_col)
def test_update_all_0deg(idtype):
# test#1
g = dgl.graph(([1, 2, 3, 4], [0, 0, 0, 0]), idtype=idtype, device=F.ctx())
def _message(edges):
return {'m': edges.src['h']}
def _reduce(nodes):
return {'x': nodes.data['h'] + F.sum(nodes.mailbox['m'], 1)}
def _apply(nodes):
return {'x': nodes.data['x'] * 2}
def _init2(shape, dtype, ctx, ids):
return 2 + F.zeros(shape, dtype, ctx)
g.set_n_initializer(_init2, 'x')
old_repr = F.randn((5, 5))
g.ndata['h'] = old_repr
g.update_all(_message, _reduce, _apply)
new_repr = g.ndata['x']
# the first row of the new_repr should be the sum of all the node
# features; while the 0-deg nodes should be initialized by the
# initializer and applied with UDF.
assert F.allclose(new_repr[1:], 2 * (2 + F.zeros((4, 5))))
assert F.allclose(new_repr[0], 2 * F.sum(old_repr, 0))
# test#2: graph with no edge
g = dgl.graph(([], []), num_nodes=5, idtype=idtype, device=F.ctx())
g.ndata['h'] = old_repr
g.update_all(_message, _reduce, lambda nodes: {'h': nodes.data['h'] * 2})
new_repr = g.ndata['h']
# should fallback to apply
assert F.allclose(new_repr, 2 * old_repr)
def test_update_all_0deg(index_dtype):
# test#1
g = dgl.graph([(1, 0), (2, 0), (3, 0), (4, 0)], index_dtype=index_dtype)
def _message(edges):
return {'m': edges.src['h']}
def _reduce(nodes):
return {'h': nodes.data['h'] + F.sum(nodes.mailbox['m'], 1)}
def _apply(nodes):
return {'h': nodes.data['h'] * 2}
def _init2(shape, dtype, ctx, ids):
return 2 + F.zeros(shape, dtype, ctx)
g.set_n_initializer(_init2, 'h')
old_repr = F.randn((5, 5))
g.ndata['h'] = old_repr
g.update_all(_message, _reduce, _apply)
new_repr = g.ndata['h']
# the first row of the new_repr should be the sum of all the node
# features; while the 0-deg nodes should be initialized by the
# initializer and applied with UDF.
assert F.allclose(new_repr[1:], 2 * (2 + F.zeros((4, 5))))
assert F.allclose(new_repr[0], 2 * F.sum(old_repr, 0))
# test#2:
g = dgl.graph([], num_nodes=5, index_dtype=index_dtype)
g.set_n_initializer(_init2, 'h')
g.ndata['h'] = old_repr
g.update_all(_message, _reduce, _apply)
new_repr = g.ndata['h']
# should fallback to apply
assert F.allclose(new_repr, 2 * old_repr)
def test_multi_send():
g = generate_graph()
def _fmsg(edges):
assert edges.src['h'].shape == (5, D)
return {'m': edges.src['h']}
g.register_message_func(_fmsg)
# many-many send
u = F.tensor([0, 0, 0, 0, 0])
v = F.tensor([1, 2, 3, 4, 5])
g.send((u, v))
# duplicate send
u = F.tensor([0])
v = F.tensor([1, 2, 3, 4, 5])
g.send((u, v))
# send more
u = F.tensor([1, 2, 3, 4, 5])
v = F.tensor([9])
g.send((u, v))
# check if message indicator is as expected
expected = F.copy_to(F.zeros((g.number_of_edges(), ), dtype=F.int64),
F.cpu())
eid = g.edge_ids([0, 0, 0, 0, 0, 1, 2, 3, 4, 5],
[1, 2, 3, 4, 5, 9, 9, 9, 9, 9])
expected = F.asnumpy(expected)
eid = F.asnumpy(eid)
expected[eid] = 1
assert np.array_equal(g._get_msg_index().tonumpy(), expected)
def test_mean_zero_degree(g, idtype):
g = g.astype(idtype).to(F.ctx())
g.ndata['h'] = F.ones((g.number_of_nodes(), 3))
g.update_all(fn.copy_u('h', 'm'), fn.mean('m', 'x'))
deg = F.asnumpy(g.in_degrees())
v = F.tensor(np.where(deg == 0)[0])
assert F.allclose(F.gather_row(g.ndata['x'], v), F.zeros((len(v), 3)))
def test_graph_serialize_with_labels():
num_graphs = 100
g_list = [generate_rand_graph(30) for _ in range(num_graphs)]
labels = {"label": F.zeros((num_graphs, 1))}
# create a temporary file and immediately release it so DGL can open it.
f = tempfile.NamedTemporaryFile(delete=False)
path = f.name
f.close()
save_graphs(path, g_list, labels)
idx_list = np.random.permutation(np.arange(num_graphs)).tolist()
loadg_list, l_labels0 = load_graphs(path, idx_list)
l_labels = load_labels(path)
assert F.allclose(l_labels['label'], labels['label'])
assert F.allclose(l_labels0['label'], labels['label'])
idx = idx_list[0]
load_g = loadg_list[0]
assert F.allclose(load_g.nodes(), g_list[idx].nodes())
load_edges = load_g.all_edges('uv', 'eid')
g_edges = g_list[idx].all_edges('uv', 'eid')
assert F.allclose(load_edges[0], g_edges[0])
assert F.allclose(load_edges[1], g_edges[1])
os.unlink(path)
def __init__(self, num_lstm, dim_frame):
self.num_lstm = num_lstm
self.dim_frame = dim_frame
self.W_hid_to_ingate = f_inner_init((self.num_lstm, self.num_lstm))
self.W_hid_to_forgetgate = f_inner_init((self.num_lstm, self.num_lstm))
self.W_hid_to_cell = f_inner_init((self.num_lstm, self.num_lstm))
self.W_hid_to_outgate = f_inner_init((self.num_lstm, self.num_lstm))
self.b_ingate = K.zeros((self.num_lstm,))
self.b_forgetgate = f_forget_bias_init((self.num_lstm,))
self.b_cell = K.zeros((self.num_lstm,))
self.b_outgate = K.zeros((self.num_lstm,))
self.W_cell_to_ingate = f_init2((self.num_lstm,))
self.W_cell_to_forgetgate = f_init2((self.num_lstm,))
self.W_cell_to_outgate = f_init2((self.num_lstm,))
self.W_output = f_init((self.num_lstm, self.dim_frame))
self.b_output = K.zeros((self.dim_frame,))
def zero(shape, name=None):
return K.zeros(shape, name=name)
