def train_hetu(num_epoch):
ctx = ndarray.gpu(0)
x_ = ad.Variable(name="x_")
y_ = ad.Variable(name="y_")
gcn1 = GCN(num_features, hidden_layer_size)
gcn2 = GCN(hidden_layer_size, num_classes)
mp_val = mp_matrix(graph, ctx, use_original_gcn_norm=True)
feed_dict = {
gcn1.mp:
mp_val,
gcn2.mp:
mp_val,
x_:
ndarray.array(graph.x, ctx=ctx),
y_:
ndarray.array(convert_to_one_hot(graph.y, max_val=num_classes),
ctx=ctx)
}
x = gcn1(x_)
x = ad.relu_op(x)
y = gcn2(x)
loss = ad.softmaxcrossentropy_op(y, y_)
opt = optimizer.AdamOptimizer(0.01)
train_op = opt.minimize(loss)
executor = ad.Executor([loss, y, train_op], ctx=ctx)
start_time = time.time()
for i in range(num_epoch):
loss_val, y_predicted, _ = executor.run(feed_dict=feed_dict)
y_predicted = y_predicted.asnumpy().argmax(axis=1)
acc = (y_predicted == graph.y).sum()
if i == 0:
start_time = time.time()
print("Train loss :", loss_val.asnumpy().mean())
print("Train accuracy:", acc / len(y_predicted))
print("Hetu time:", i, time.time() - start_time)
print("Hetu time:", time.time() - start_time)
mp_val = mp_matrix(graph_full, ctx)
feed_dict = {
gcn1.mp: mp_val,
gcn2.mp: mp_val,
x_: ndarray.array(graph_full.x, ctx=ctx),
}
executor_eval = ad.Executor([y], ctx=ctx)
y_predicted, = executor_eval.run(feed_dict=feed_dict)
y_predicted = y_predicted.asnumpy().argmax(axis=1)
acc = (y_predicted == graph_full.y)[train_split:].sum()
print("Test accuracy:", acc / len(y_predicted[train_split:]))
def test_Div():
X = ad.Variable(name="X")
B = ad.Variable(name="B")
y = ad.div_op(X, B)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(2, 2)).astype(np.float32)
B_val = rand.normal(scale=0.1, size=(2, 2)).astype(np.float32)
res = executor.run(feed_dict={X: X_val, B: B_val})
Check(executor, res, [X, B], [y], [X_val, B_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_Concat():
A = ad.Variable(name="A")
B = ad.Variable(name="B")
y = ad.concat_op(A, B, axis=1)
executor = ad.Executor([y], ctx=ctx)
A_val = rand.normal(scale=0.1, size=(2, 3)).astype(np.float32)
B_val = rand.normal(scale=0.1, size=(2, 3)).astype(np.float32)
res = executor.run(feed_dict={A: A_val, B: B_val})
Check(executor, res, [A, B], [y], [A_val, B_val])
print(sys._getframe().f_code.co_name, 'pass!')
def __init__(self, in_features, out_features, activation=None, dropout=0,
name="GCN", custom_init=None, mp_val=None):
if custom_init is not None:
self.weight = ad.Variable(value=custom_init[0], name=name+"_Weight")
self.bias = ad.Variable(value=custom_init[1], name=name+"_Bias")
else:
self.weight = initializers.xavier_uniform(shape=(in_features,out_features), name=name+"_Weight")
self.bias = initializers.zeros(shape=(out_features,), name=name+"_Bias")
#self.mp is a sparse matrix and should appear in feed_dict later
self.mp = ad.Variable("message_passing", trainable=False, value=mp_val)
self.activation = activation
self.dropout = dropout
def train_hetu(args):
with open(os.path.join(args.path, "meta.yml"), 'rb') as f:
meta = yaml.load(f.read(), Loader=yaml.FullLoader)
hidden_layer_size = args.hidden_size
num_epoch = args.num_epoch
rank = int(os.environ["WORKER_ID"])
nrank = int(os.environ["DMLC_NUM_WORKER"])
hosts, ports = load_ip_config(args.ip_config)
ctx = ndarray.gpu(rank)
distributed.grpc_init(hosts=hosts, ports=ports, rank=rank, nrank=nrank)
x_ = ad.Variable(name="x_")
y_ = ad.Variable(name="y_")
gcn1 = GCN(meta["feature"], hidden_layer_size, activation="relu")
gcn2 = GCN(hidden_layer_size, meta["class"])
x = gcn1(x_)
y = gcn2(x)
loss = ad.softmaxcrossentropy_op(y, y_)
loss = ad.reduce_mean_op(loss, [0])
opt = optimizer.SGDOptimizer(0.1)
train_op = opt.minimize(loss)
executor = ad.Executor([loss, y, train_op], ctx=ctx, comm_mode='PS')
def transform(graph):
mp_val = mp_matrix(graph, ndarray.gpu(rank))
return graph, mp_val
with DistributedSubgraphSampler(args.path, 4000, 2, rank=rank, nrank=nrank ,transformer=transform, backend="grpc") as sampler:
epoch = 0
nnodes = 0
start = time.time()
while True:
g_sample, mp_val = sampler.sample()
feed_dict = {
gcn1.mp : mp_val,
gcn2.mp : mp_val,
x_ : ndarray.array(g_sample.x, ctx=ctx),
y_ : ndarray.array(convert_to_one_hot(g_sample.y, max_val=g_sample.num_classes), ctx=ctx)
}
loss_val, y_predicted, _ = executor.run(feed_dict = feed_dict)
y_predicted = y_predicted.asnumpy().argmax(axis=1)
acc = (y_predicted == g_sample.y).sum()
distributed.ps_get_worker_communicator().BarrierWorker()
nnodes += g_sample.num_nodes
if nnodes > meta["partition"]["nodes"][rank]:
nnodes = 0
epoch += 1
print("Epoch :", epoch, time.time() - start)
print("Train accuracy:", acc/len(y_predicted))
start = time.time()
if epoch >= num_epoch:
break
def test_Where():
cond = ad.Variable(name="Cond", dtype=np.bool)
A = ad.Variable(name="A")
B = ad.Variable(name="B")
y = ad.where_op(cond, A, B)
executor = ad.Executor([y], ctx=ctx)
shape = [2, 2, 3]
Cond_val = rand.randint(0, 2, size=shape, dtype=np.bool)
A_val = rand.normal(scale=0.1, size=shape).astype(np.float32)
B_val = rand.normal(scale=0.1, size=shape).astype(np.float32)
res = executor.run(feed_dict={cond: Cond_val, A: A_val, B: B_val})
Check(executor, res, [cond, A, B], [y], [Cond_val, A_val, B_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_Transpose():
X = ad.Variable(name="X")
y = ad.transpose_op(X, [2, 0, 1])
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(3, 2, 5)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_Softmax():
X = ad.Variable(name="X")
y = ad.softmax_op(X)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(128, 150)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def rnn(x, y_):
'''
RNN model, for MNIST dataset.
Parameters:
x: Variable(hetu.gpu_ops.Node.Node), shape (N, dims)
y_: Variable(hetu.gpu_ops.Node.Node), shape (N, num_classes)
Return:
loss: Variable(hetu.gpu_ops.Node.Node), shape (1,)
y: Variable(hetu.gpu_ops.Node.Node), shape (N, num_classes)
'''
print("Building RNN model...")
diminput = 28
dimhidden = 128
dimoutput = 10
nsteps = 28
weight1 = init.random_normal(shape=(diminput, dimhidden),
stddev=0.1,
name='rnn_weight1')
bias1 = init.random_normal(shape=(dimhidden, ),
stddev=0.1,
name='rnn_bias1')
weight2 = init.random_normal(shape=(dimhidden + dimhidden, dimhidden),
stddev=0.1,
name='rnn_weight2')
bias2 = init.random_normal(shape=(dimhidden, ),
stddev=0.1,
name='rnn_bias2')
weight3 = init.random_normal(shape=(dimhidden, dimoutput),
stddev=0.1,
name='rnn_weight3')
bias3 = init.random_normal(shape=(dimoutput, ),
stddev=0.1,
name='rnn_bias3')
last_state = ad.Variable(value=np.zeros((1, )).astype(np.float32),
name='initial_state',
trainable=False)
for i in range(nsteps):
cur_x = ad.slice_op(x, (0, i * diminput), (-1, diminput))
h = ad.matmul_op(cur_x, weight1)
h = h + ad.broadcastto_op(bias1, h)
if i == 0:
last_state = ad.broadcastto_op(last_state, h)
s = ad.concat_op(h, last_state, axis=1)
s = ad.matmul_op(s, weight2)
s = s + ad.broadcastto_op(bias2, s)
last_state = ad.relu_op(s)
final_state = last_state
x = ad.matmul_op(final_state, weight3)
y = x + ad.broadcastto_op(bias3, x)
loss = ad.softmaxcrossentropy_op(y, y_)
loss = ad.reduce_mean_op(loss, [0])
return loss, y
def test_ReduceSum():
X = ad.Variable(name="X")
y = ad.reduce_sum_op(X, 0, keepdims=False)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(2, 23, 5)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_Reshape():
X = ad.Variable(name="X")
y = ad.array_reshape_op(X, [-1, 10 * 10 * 10])
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1,
size=(batch_size, 10, 10, 10)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_DivConst():
X = ad.Variable(name="X")
const = 5.5
y = ad.div_const_op(const, X)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(2, 2)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_AddConst():
X = ad.Variable(name="X")
val = 3.3
y = X + val
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(batch_size, 10)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_add_lazy(shape1=(1, 4, 1), shape2=(2, 3, 4, 5), ctx=ndarray.gpu(1)):
x = np.random.random(shape1).astype(np.float32)
z = np.random.random(shape2).astype(np.float32)
ath_x = ad.Variable(name='x', value=x)
ath_z = ad.Variable(name='z', value=z)
ath_y = ad.add_op(ad.broadcast_shape_op(ath_x, shape2), ath_z)
executor = ad.Executor([ath_y], ctx=ctx)
ath_results = [var.asnumpy() for var in executor.run()]
import tensorflow as tf
tf_x = tf.convert_to_tensor(x)
tf_z = tf.convert_to_tensor(z)
tf_y = tf_x + tf_z
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
tf_results = sess.run([tf_y])
np.testing.assert_allclose(ath_results[0], tf_results[0])
print('Passed add op test with shape ', shape1, shape2)
def test_Conv2d():
X = ad.Variable(name="X")
W1 = init.random_normal((32, 1, 5, 5), stddev=0.1, name='W1')
y = ad.conv2d_op(X, W1, padding=2, stride=1)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1,
size=(batch_size, 1, 28, 28)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_MaxPool():
X = ad.Variable(name="X")
y = ad.max_pool2d_op(X, kernel_H=2, kernel_W=2, padding=0, stride=2)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1,
size=(batch_size, 10, 10, 10)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_dense():
npw = np.random.random((5, 10)).astype(np.float32)
npx = np.random.random((7, 5)).astype(np.float32)
cpuctx = ndarray.cpu(0)
gpuctx = ndarray.gpu(0)
X = ad.Variable(name="x")
mid = X + 3
W = ad.Variable(name='w', value=npw, ctx=cpuctx)
y = ad.matmul_op(mid, W)
opt = optimizer.SGDOptimizer(learning_rate=0.1)
train_op = opt.minimize(y)
executor = ad.Executor([y, train_op], ctx=gpuctx)
pred_y, _ = executor.run(feed_dict={X: npx}, convert_to_numpy_ret_vals=True)
nppred_y = np.matmul((npx + 3), npw)
np.testing.assert_allclose(pred_y, nppred_y, rtol=1e-6)
new_npw = npw - 0.1 * np.matmul((npx+3).T, np.ones(nppred_y.shape).astype(np.float32))
np.testing.assert_allclose(W.tensor_value.asnumpy(), new_npw, rtol=1e-10)
def test_AddElewise():
X = ad.Variable(name="X")
b3 = init.random_normal((10, ), stddev=0.1, name='b3')
y = X + b3
executor = ad.Executor([y], ctx=ctx, enable_lazy=False)
X_val = rand.normal(scale=0.1, size=(batch_size, 10)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def positional_encoding(inputs, inputs_shape, maxlen, masking=True):
N, T, E = tuple(inputs_shape)
position_enc = np.array(
[[pos / np.power(10000, (i & -2) / E) for i in range(E)]
for pos in range(maxlen)])
position_enc[:, 0::2] = np.sin(position_enc[:, 0::2]) # dim 2i
position_enc[:, 1::2] = np.cos(position_enc[:, 1::2]) # dim 2i+1
position_enc = position_enc[:T, :]
outputs = ad.Variable(name='position_enc',
value=np.tile(position_enc, [N, 1, 1]),
trainable=False)
zeros = ad.Variable(name='zeros',
value=np.zeros(inputs_shape),
trainable=False)
if masking:
outputs = ad.where_op(inputs, outputs, zeros)
return outputs
def test_Pad():
X = ad.Variable(name="X")
paddings = [[1, 1], [1, 1], [2, 1], [1, 3]]
y = ad.pad_op(X, paddings, constant_values=0)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1, size=(1, 1, 1, 1)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_broadcast(shape1=(3, 1), shape2=(2, 3, 4)):
ctx = ndarray.gpu(1)
x = np.random.random(shape1).astype(np.float32)
y = np.random.random(shape2).astype(np.float32)
ath_x = ad.Variable(name='x', value=x)
ath_z = ad.Variable(name='y', value=y)
ath_y = ad.broadcastto_op(ath_x, ath_z)
ath_grad = ad.gradients(ath_y, [ath_x])[0]
executor = ad.Executor([ath_y, ath_grad], ctx=ctx, enable_lazy=False)
ath_results = [var.asnumpy() for var in executor.run()]
import tensorflow as tf
tf_x = tf.convert_to_tensor(x)
tf_y = tf.broadcast_to(tf_x, shape2)
tf_grad = tf.gradients(tf_y, tf_x)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
tf_results = sess.run([tf_y, tf_grad])
np.testing.assert_allclose(ath_results[0], tf_results[0])
np.testing.assert_allclose(ath_results[1], np.reshape(tf_results[1], ath_results[1].shape))
print('Passed broadcast shape op test with shape ', shape1, shape2)
def _onnx_graph_2_hetu(cls,graph_def,opset,):
handlers = cls._get_handlers(opset)
if graph_def.initializer:
initialized = {
init.name: onnx.numpy_helper.to_array(init)
for init in graph_def.initializer
}
input_dict_items = cls._onnx_initializer_to_input_dict_items(
graph_def.initializer,
initialized,
)
else:
input_dict_items = []
initialized = {}
for node in graph_def.node:
node = OnnxNode(node)
#todo:should check.
if node.op_type == 'Constant':
initialized[node.output_tensor_names[0]] = numpy_helper.to_array(
node.attrs["value"]
)
# creating placeholders for currently unknown inputs
for value_info in graph_def.input:
if value_info.name in initialized.keys():
continue
shape = list(
d.dim_value if (d.dim_value > 0 and d.dim_param == "") else None
for d in value_info.type.tensor_type.shape.dim
)
#todo,check here ,shape not use
input_dict_items.append((value_info.name,
ad.Variable(name=value_info.name),
))
tensor_dict = dict(input_dict_items)
for node in graph_def.node:
onnx_node = OnnxNode(node)
print(onnx_node.name,onnx_node.op_type)
output_ops = cls._onnx_node_to_hetu_op(
onnx_node,
tensor_dict,
initialized,
handlers,
opset = opset,
)
return tensor_dict
def test_batch_matmul(shape1=(7, 4, 6), shape2=(7, 6, 5), transA=False, transB=False):
executor_ctx = ndarray.gpu(1)
if transA:
shape1 = tuple(list(shape1)[:-2] + [shape1[-1], shape1[-2]])
if transB:
shape2 = tuple(list(shape2)[:-2] + [shape2[-1], shape2[-2]])
data = np.random.normal(0.0, 0.2, shape1).astype(np.float32)
weights = np.random.normal(0.0, 0.1, shape2).astype(np.float32)
ath_data = ad.Variable(name='data')
ath_weights = ad.Variable(name='weights')
ath_output = ad.batch_matmul_op(ath_data, ath_weights, trans_A=transA, trans_B=transB)
ath_grads = ad.gradients(ath_output, [ath_data, ath_weights])
executor = ad.Executor(
[ath_output] + ath_grads,
ctx=executor_ctx
)
ath_results = executor.run(feed_dict={ath_data: data, ath_weights: weights})
ath_results = [res.asnumpy() for res in ath_results]
import tensorflow as tf
tf_data = tf.placeholder(name='data', dtype=tf.float32)
tf_weights = tf.placeholder(name='weights', dtype=tf.float32)
tf_output = tf.matmul(tf_data, tf_weights, transpose_a=transA, transpose_b=transB)
tf_grads = tf.gradients(tf_output, [tf_data, tf_weights])
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
tf_results = sess.run([tf_output] + tf_grads, feed_dict={tf_data: data, tf_weights: weights})
np.testing.assert_allclose(ath_results[0], tf_results[0], atol=1e-6)
np.testing.assert_allclose(ath_results[1], tf_results[1], atol=1e-6)
np.testing.assert_allclose(ath_results[2], tf_results[2], atol=1e-6)
print('Pass batch matmul op test with shape ', shape1, shape2)
def test_Onehot():
X = ad.Variable(name="X")
classes = 10
y = ad.one_hot_op(X, classes)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.randint(
0,
10,
20,
).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X], [y], [X_val])
print(sys._getframe().f_code.co_name, 'pass!')
def test_csrmv_op(executor_ctx):
X = ad.Variable(name="X")
W = ad.Variable(name="W")
Y = ad.csrmv_op(X, W)
Y_ = ad.Variable(name="Y_")
temp = Y + (-1) * Y_
loss = temp * temp
grads = ad.gradients(loss, [W, Y])
executor = ad.Executor(
[loss, grads[0], grads[1]], ctx=executor_ctx)
rand = np.random.RandomState(seed=123)
W_val =rand.normal(scale=0.1, size=[70000, ])
if ndarray.is_gpu_ctx(executor_ctx):
W_val = ndarray.array(W_val, ctx=executor_ctx)
X_val = scipy.sparse.rand(500, 70000, density=1e-5,format='coo',dtype=np.float32)
Y_val = np.random.uniform(0, 10, size=(500, )).astype(np.float32)
loss_val = executor.run(feed_dict={X: X_val, Y_: Y_val, W: W_val})
if ndarray.is_gpu_ctx(executor_ctx):
W_val = W_val.asnumpy()
loss_val = [val.asnumpy() for val in loss_val]
y_groundtruth = X_val.dot(W_val)
loss_groundtruth = (y_groundtruth - Y_val) ** 2
Y_grad_groundtruth = 2 * (y_groundtruth - Y_val) * np.ones(loss_groundtruth.shape)
W_grad_groundtruth = X_val.T.dot(Y_grad_groundtruth)
np.testing.assert_allclose(loss_val[0], loss_groundtruth, rtol=1e-4)
np.testing.assert_allclose(loss_val[1], W_grad_groundtruth, rtol=1e-4)
np.testing.assert_allclose(loss_val[2], Y_grad_groundtruth, rtol=1e-4)
def __init__(self,
in_features: int,
out_features: int,
npart: int,
name="GCN",
custom_init=None):
if custom_init is not None:
self.weight = ad.Variable(value=custom_init[0],
name=name + "_Weight")
self.bias = ad.Variable(value=custom_init[1], name=name + "_Bias")
else:
self.weight = initializers.xavier_uniform(shape=(in_features,
out_features),
name=name + "_Weight")
self.bias = initializers.zeros(shape=(out_features, ),
name=name + "_Bias")
self.in_features = in_features
self.out_features = out_features
self.npart = npart
#npart * npart message_passing matrix, either dense or sparse
self.mp = [[
ad.Variable("message_passing", trainable=False)
for j in range(npart)
] for i in range(npart)]
def test_sparse():
npemb = np.random.random((100, 20)).astype(np.float32)
npind = np.array(np.random.randint(100, size=(10,)))
npw = np.random.random((20, 30)).astype(np.float32)
cpuctx = ndarray.cpu(0)
gpuctx = ndarray.gpu(0)
embedding = ad.Variable('embeddingtable', value=npemb, ctx=cpuctx)
index = ad.Variable(name="index", ctx=cpuctx)
W = ad.Variable(name="w", value=npw)
y = ad.embedding_lookup_op(embedding, index) # (10, 20)
y = ad.matmul_op(y, W)
opt = optimizer.SGDOptimizer(0.1)
train_op = opt.minimize(y)
executor = ad.Executor([y, train_op],ctx=gpuctx)
out, _ = executor.run(feed_dict={index: npind.astype(np.float32)}, convert_to_numpy_ret_vals=True)
np_out = np.matmul(npemb[npind], npw)
np.testing.assert_allclose(out, np_out, rtol=1e-6)
tmp_grad = np.matmul(np.ones(np_out.shape).astype(np.float32), npw.T)
for i, localid in enumerate(npind):
npemb[localid] -= 0.1 * tmp_grad[i]
np.testing.assert_allclose(embedding.tensor_value.asnumpy(), npemb, rtol=1e-6)
def test_csrmm_op(executor_ctx):
X = ad.Variable(name="X")
W = ad.Variable(name="W")
Y = ad.csrmm_op(X, W)
Y_ = ad.Variable(name="Y_")
loss = ad.softmaxcrossentropy_op(Y, Y_)
loss = ad.reduce_mean_op(loss, [0])
grads = ad.gradients(loss, [W, Y])
executor = ad.Executor(
[loss, grads[0], grads[1]], ctx=executor_ctx)
rand = np.random.RandomState(seed=123)
W_val = rand.normal(scale=0.1, size=[70000, 2]).astype(np.float32)
if ndarray.is_gpu_ctx(executor_ctx):
W_val = ndarray.array(W_val, ctx=executor_ctx)
X_val = scipy.sparse.rand(500, 70000, density=1e-5,format='coo',dtype=np.float32)
Y_val = np.random.uniform(0, 10, size=(500, 2)).astype(np.float32)
loss_val = executor.run(feed_dict={X: X_val, Y_: Y_val, W: W_val})
if ndarray.is_gpu_ctx(executor_ctx):
W_val = W_val.asnumpy()
loss_val = [val.asnumpy() for val in loss_val]
y_groundtruth = X_val.dot(W_val)
loss_groundtruth = np.mean(
-np.sum(Y_val * np.log(softmax_func(y_groundtruth)), axis=1), keepdims=True)
Y_grad_groundtruth = (softmax_func(y_groundtruth) + -1 * Y_val) * np.ones(loss_groundtruth.shape) / 500
W_grad_groundtruth = X_val.T.dot(Y_grad_groundtruth)
np.testing.assert_allclose(loss_val[0], loss_groundtruth, rtol=1e-4)
np.testing.assert_allclose(loss_val[1], W_grad_groundtruth, rtol=1e-4)
np.testing.assert_allclose(loss_val[2], Y_grad_groundtruth, rtol=1e-4)
def _onnx_initializer_to_input_dict_items(cls, initializer,
initialized,):
def get_flow_shape(shape):
if len(shape) == 0:
return (1,)
return shape
return [
(
init.name,
ad.Variable(name=init.name,value=initialized[init.name] ),
)
for init in initializer
]
def test_BatchNorm():
X = ad.Variable(name="X")
bn_scale = init.random_normal((64, ), stddev=0.1, name='bn_scale')
bn_bias = init.random_normal((64, ), stddev=0.1, name='bn_bias')
y = ad.batch_normalization_op(X, bn_scale, bn_bias)
executor = ad.Executor([y], ctx=ctx)
X_val = rand.normal(scale=0.1,
size=(batch_size, 64, 28, 28)).astype(np.float32)
res = executor.run(feed_dict={X: X_val})
Check(executor, res, [X, bn_scale, bn_bias], [y],
[X_val, bn_scale.tensor_value, bn_bias.tensor_value])
print(sys._getframe().f_code.co_name, 'pass!')
