def train_hetu(num_epoch): ctx = ndarray.gpu(0) x_ = ad.Variable(name="x_") y_ = ad.Variable(name="y_") gcn1 = GCN(num_features, hidden_layer_size) gcn2 = GCN(hidden_layer_size, num_classes) mp_val = mp_matrix(graph, ctx, use_original_gcn_norm=True) feed_dict = { gcn1.mp: mp_val, gcn2.mp: mp_val, x_: ndarray.array(graph.x, ctx=ctx), y_: ndarray.array(convert_to_one_hot(graph.y, max_val=num_classes), ctx=ctx) } x = gcn1(x_) x = ad.relu_op(x) y = gcn2(x) loss = ad.softmaxcrossentropy_op(y, y_) opt = optimizer.AdamOptimizer(0.01) train_op = opt.minimize(loss) executor = ad.Executor([loss, y, train_op], ctx=ctx) start_time = time.time() for i in range(num_epoch): loss_val, y_predicted, _ = executor.run(feed_dict=feed_dict) y_predicted = y_predicted.asnumpy().argmax(axis=1) acc = (y_predicted == graph.y).sum() if i == 0: start_time = time.time() print("Train loss :", loss_val.asnumpy().mean()) print("Train accuracy:", acc / len(y_predicted)) print("Hetu time:", i, time.time() - start_time) print("Hetu time:", time.time() - start_time) mp_val = mp_matrix(graph_full, ctx) feed_dict = { gcn1.mp: mp_val, gcn2.mp: mp_val, x_: ndarray.array(graph_full.x, ctx=ctx), } executor_eval = ad.Executor([y], ctx=ctx) y_predicted, = executor_eval.run(feed_dict=feed_dict) y_predicted = y_predicted.asnumpy().argmax(axis=1) acc = (y_predicted == graph_full.y)[train_split:].sum() print("Test accuracy:", acc / len(y_predicted[train_split:]))
def test_Div(): X = ad.Variable(name="X") B = ad.Variable(name="B") y = ad.div_op(X, B) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(2, 2)).astype(np.float32) B_val = rand.normal(scale=0.1, size=(2, 2)).astype(np.float32) res = executor.run(feed_dict={X: X_val, B: B_val}) Check(executor, res, [X, B], [y], [X_val, B_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_Concat(): A = ad.Variable(name="A") B = ad.Variable(name="B") y = ad.concat_op(A, B, axis=1) executor = ad.Executor([y], ctx=ctx) A_val = rand.normal(scale=0.1, size=(2, 3)).astype(np.float32) B_val = rand.normal(scale=0.1, size=(2, 3)).astype(np.float32) res = executor.run(feed_dict={A: A_val, B: B_val}) Check(executor, res, [A, B], [y], [A_val, B_val]) print(sys._getframe().f_code.co_name, 'pass!')
def __init__(self, in_features, out_features, activation=None, dropout=0, name="GCN", custom_init=None, mp_val=None): if custom_init is not None: self.weight = ad.Variable(value=custom_init[0], name=name+"_Weight") self.bias = ad.Variable(value=custom_init[1], name=name+"_Bias") else: self.weight = initializers.xavier_uniform(shape=(in_features,out_features), name=name+"_Weight") self.bias = initializers.zeros(shape=(out_features,), name=name+"_Bias") #self.mp is a sparse matrix and should appear in feed_dict later self.mp = ad.Variable("message_passing", trainable=False, value=mp_val) self.activation = activation self.dropout = dropout
def train_hetu(args): with open(os.path.join(args.path, "meta.yml"), 'rb') as f: meta = yaml.load(f.read(), Loader=yaml.FullLoader) hidden_layer_size = args.hidden_size num_epoch = args.num_epoch rank = int(os.environ["WORKER_ID"]) nrank = int(os.environ["DMLC_NUM_WORKER"]) hosts, ports = load_ip_config(args.ip_config) ctx = ndarray.gpu(rank) distributed.grpc_init(hosts=hosts, ports=ports, rank=rank, nrank=nrank) x_ = ad.Variable(name="x_") y_ = ad.Variable(name="y_") gcn1 = GCN(meta["feature"], hidden_layer_size, activation="relu") gcn2 = GCN(hidden_layer_size, meta["class"]) x = gcn1(x_) y = gcn2(x) loss = ad.softmaxcrossentropy_op(y, y_) loss = ad.reduce_mean_op(loss, [0]) opt = optimizer.SGDOptimizer(0.1) train_op = opt.minimize(loss) executor = ad.Executor([loss, y, train_op], ctx=ctx, comm_mode='PS') def transform(graph): mp_val = mp_matrix(graph, ndarray.gpu(rank)) return graph, mp_val with DistributedSubgraphSampler(args.path, 4000, 2, rank=rank, nrank=nrank ,transformer=transform, backend="grpc") as sampler: epoch = 0 nnodes = 0 start = time.time() while True: g_sample, mp_val = sampler.sample() feed_dict = { gcn1.mp : mp_val, gcn2.mp : mp_val, x_ : ndarray.array(g_sample.x, ctx=ctx), y_ : ndarray.array(convert_to_one_hot(g_sample.y, max_val=g_sample.num_classes), ctx=ctx) } loss_val, y_predicted, _ = executor.run(feed_dict = feed_dict) y_predicted = y_predicted.asnumpy().argmax(axis=1) acc = (y_predicted == g_sample.y).sum() distributed.ps_get_worker_communicator().BarrierWorker() nnodes += g_sample.num_nodes if nnodes > meta["partition"]["nodes"][rank]: nnodes = 0 epoch += 1 print("Epoch :", epoch, time.time() - start) print("Train accuracy:", acc/len(y_predicted)) start = time.time() if epoch >= num_epoch: break
def test_Where(): cond = ad.Variable(name="Cond", dtype=np.bool) A = ad.Variable(name="A") B = ad.Variable(name="B") y = ad.where_op(cond, A, B) executor = ad.Executor([y], ctx=ctx) shape = [2, 2, 3] Cond_val = rand.randint(0, 2, size=shape, dtype=np.bool) A_val = rand.normal(scale=0.1, size=shape).astype(np.float32) B_val = rand.normal(scale=0.1, size=shape).astype(np.float32) res = executor.run(feed_dict={cond: Cond_val, A: A_val, B: B_val}) Check(executor, res, [cond, A, B], [y], [Cond_val, A_val, B_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_Transpose(): X = ad.Variable(name="X") y = ad.transpose_op(X, [2, 0, 1]) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(3, 2, 5)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_Softmax(): X = ad.Variable(name="X") y = ad.softmax_op(X) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(128, 150)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def rnn(x, y_): ''' RNN model, for MNIST dataset. Parameters: x: Variable(hetu.gpu_ops.Node.Node), shape (N, dims) y_: Variable(hetu.gpu_ops.Node.Node), shape (N, num_classes) Return: loss: Variable(hetu.gpu_ops.Node.Node), shape (1,) y: Variable(hetu.gpu_ops.Node.Node), shape (N, num_classes) ''' print("Building RNN model...") diminput = 28 dimhidden = 128 dimoutput = 10 nsteps = 28 weight1 = init.random_normal(shape=(diminput, dimhidden), stddev=0.1, name='rnn_weight1') bias1 = init.random_normal(shape=(dimhidden, ), stddev=0.1, name='rnn_bias1') weight2 = init.random_normal(shape=(dimhidden + dimhidden, dimhidden), stddev=0.1, name='rnn_weight2') bias2 = init.random_normal(shape=(dimhidden, ), stddev=0.1, name='rnn_bias2') weight3 = init.random_normal(shape=(dimhidden, dimoutput), stddev=0.1, name='rnn_weight3') bias3 = init.random_normal(shape=(dimoutput, ), stddev=0.1, name='rnn_bias3') last_state = ad.Variable(value=np.zeros((1, )).astype(np.float32), name='initial_state', trainable=False) for i in range(nsteps): cur_x = ad.slice_op(x, (0, i * diminput), (-1, diminput)) h = ad.matmul_op(cur_x, weight1) h = h + ad.broadcastto_op(bias1, h) if i == 0: last_state = ad.broadcastto_op(last_state, h) s = ad.concat_op(h, last_state, axis=1) s = ad.matmul_op(s, weight2) s = s + ad.broadcastto_op(bias2, s) last_state = ad.relu_op(s) final_state = last_state x = ad.matmul_op(final_state, weight3) y = x + ad.broadcastto_op(bias3, x) loss = ad.softmaxcrossentropy_op(y, y_) loss = ad.reduce_mean_op(loss, [0]) return loss, y
def test_ReduceSum(): X = ad.Variable(name="X") y = ad.reduce_sum_op(X, 0, keepdims=False) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(2, 23, 5)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_Reshape(): X = ad.Variable(name="X") y = ad.array_reshape_op(X, [-1, 10 * 10 * 10]) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(batch_size, 10, 10, 10)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_DivConst(): X = ad.Variable(name="X") const = 5.5 y = ad.div_const_op(const, X) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(2, 2)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_AddConst(): X = ad.Variable(name="X") val = 3.3 y = X + val executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(batch_size, 10)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_add_lazy(shape1=(1, 4, 1), shape2=(2, 3, 4, 5), ctx=ndarray.gpu(1)): x = np.random.random(shape1).astype(np.float32) z = np.random.random(shape2).astype(np.float32) ath_x = ad.Variable(name='x', value=x) ath_z = ad.Variable(name='z', value=z) ath_y = ad.add_op(ad.broadcast_shape_op(ath_x, shape2), ath_z) executor = ad.Executor([ath_y], ctx=ctx) ath_results = [var.asnumpy() for var in executor.run()] import tensorflow as tf tf_x = tf.convert_to_tensor(x) tf_z = tf.convert_to_tensor(z) tf_y = tf_x + tf_z with tf.Session() as sess: sess.run(tf.global_variables_initializer()) tf_results = sess.run([tf_y]) np.testing.assert_allclose(ath_results[0], tf_results[0]) print('Passed add op test with shape ', shape1, shape2)
def test_Conv2d(): X = ad.Variable(name="X") W1 = init.random_normal((32, 1, 5, 5), stddev=0.1, name='W1') y = ad.conv2d_op(X, W1, padding=2, stride=1) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(batch_size, 1, 28, 28)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_MaxPool(): X = ad.Variable(name="X") y = ad.max_pool2d_op(X, kernel_H=2, kernel_W=2, padding=0, stride=2) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(batch_size, 10, 10, 10)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_dense(): npw = np.random.random((5, 10)).astype(np.float32) npx = np.random.random((7, 5)).astype(np.float32) cpuctx = ndarray.cpu(0) gpuctx = ndarray.gpu(0) X = ad.Variable(name="x") mid = X + 3 W = ad.Variable(name='w', value=npw, ctx=cpuctx) y = ad.matmul_op(mid, W) opt = optimizer.SGDOptimizer(learning_rate=0.1) train_op = opt.minimize(y) executor = ad.Executor([y, train_op], ctx=gpuctx) pred_y, _ = executor.run(feed_dict={X: npx}, convert_to_numpy_ret_vals=True) nppred_y = np.matmul((npx + 3), npw) np.testing.assert_allclose(pred_y, nppred_y, rtol=1e-6) new_npw = npw - 0.1 * np.matmul((npx+3).T, np.ones(nppred_y.shape).astype(np.float32)) np.testing.assert_allclose(W.tensor_value.asnumpy(), new_npw, rtol=1e-10)
def test_AddElewise(): X = ad.Variable(name="X") b3 = init.random_normal((10, ), stddev=0.1, name='b3') y = X + b3 executor = ad.Executor([y], ctx=ctx, enable_lazy=False) X_val = rand.normal(scale=0.1, size=(batch_size, 10)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def positional_encoding(inputs, inputs_shape, maxlen, masking=True): N, T, E = tuple(inputs_shape) position_enc = np.array( [[pos / np.power(10000, (i & -2) / E) for i in range(E)] for pos in range(maxlen)]) position_enc[:, 0::2] = np.sin(position_enc[:, 0::2]) # dim 2i position_enc[:, 1::2] = np.cos(position_enc[:, 1::2]) # dim 2i+1 position_enc = position_enc[:T, :] outputs = ad.Variable(name='position_enc', value=np.tile(position_enc, [N, 1, 1]), trainable=False) zeros = ad.Variable(name='zeros', value=np.zeros(inputs_shape), trainable=False) if masking: outputs = ad.where_op(inputs, outputs, zeros) return outputs
def test_Pad(): X = ad.Variable(name="X") paddings = [[1, 1], [1, 1], [2, 1], [1, 3]] y = ad.pad_op(X, paddings, constant_values=0) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(1, 1, 1, 1)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_broadcast(shape1=(3, 1), shape2=(2, 3, 4)): ctx = ndarray.gpu(1) x = np.random.random(shape1).astype(np.float32) y = np.random.random(shape2).astype(np.float32) ath_x = ad.Variable(name='x', value=x) ath_z = ad.Variable(name='y', value=y) ath_y = ad.broadcastto_op(ath_x, ath_z) ath_grad = ad.gradients(ath_y, [ath_x])[0] executor = ad.Executor([ath_y, ath_grad], ctx=ctx, enable_lazy=False) ath_results = [var.asnumpy() for var in executor.run()] import tensorflow as tf tf_x = tf.convert_to_tensor(x) tf_y = tf.broadcast_to(tf_x, shape2) tf_grad = tf.gradients(tf_y, tf_x) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) tf_results = sess.run([tf_y, tf_grad]) np.testing.assert_allclose(ath_results[0], tf_results[0]) np.testing.assert_allclose(ath_results[1], np.reshape(tf_results[1], ath_results[1].shape)) print('Passed broadcast shape op test with shape ', shape1, shape2)
def _onnx_graph_2_hetu(cls,graph_def,opset,): handlers = cls._get_handlers(opset) if graph_def.initializer: initialized = { init.name: onnx.numpy_helper.to_array(init) for init in graph_def.initializer } input_dict_items = cls._onnx_initializer_to_input_dict_items( graph_def.initializer, initialized, ) else: input_dict_items = [] initialized = {} for node in graph_def.node: node = OnnxNode(node) #todo:should check. if node.op_type == 'Constant': initialized[node.output_tensor_names[0]] = numpy_helper.to_array( node.attrs["value"] ) # creating placeholders for currently unknown inputs for value_info in graph_def.input: if value_info.name in initialized.keys(): continue shape = list( d.dim_value if (d.dim_value > 0 and d.dim_param == "") else None for d in value_info.type.tensor_type.shape.dim ) #todo,check here ,shape not use input_dict_items.append((value_info.name, ad.Variable(name=value_info.name), )) tensor_dict = dict(input_dict_items) for node in graph_def.node: onnx_node = OnnxNode(node) print(onnx_node.name,onnx_node.op_type) output_ops = cls._onnx_node_to_hetu_op( onnx_node, tensor_dict, initialized, handlers, opset = opset, ) return tensor_dict
def test_batch_matmul(shape1=(7, 4, 6), shape2=(7, 6, 5), transA=False, transB=False): executor_ctx = ndarray.gpu(1) if transA: shape1 = tuple(list(shape1)[:-2] + [shape1[-1], shape1[-2]]) if transB: shape2 = tuple(list(shape2)[:-2] + [shape2[-1], shape2[-2]]) data = np.random.normal(0.0, 0.2, shape1).astype(np.float32) weights = np.random.normal(0.0, 0.1, shape2).astype(np.float32) ath_data = ad.Variable(name='data') ath_weights = ad.Variable(name='weights') ath_output = ad.batch_matmul_op(ath_data, ath_weights, trans_A=transA, trans_B=transB) ath_grads = ad.gradients(ath_output, [ath_data, ath_weights]) executor = ad.Executor( [ath_output] + ath_grads, ctx=executor_ctx ) ath_results = executor.run(feed_dict={ath_data: data, ath_weights: weights}) ath_results = [res.asnumpy() for res in ath_results] import tensorflow as tf tf_data = tf.placeholder(name='data', dtype=tf.float32) tf_weights = tf.placeholder(name='weights', dtype=tf.float32) tf_output = tf.matmul(tf_data, tf_weights, transpose_a=transA, transpose_b=transB) tf_grads = tf.gradients(tf_output, [tf_data, tf_weights]) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) tf_results = sess.run([tf_output] + tf_grads, feed_dict={tf_data: data, tf_weights: weights}) np.testing.assert_allclose(ath_results[0], tf_results[0], atol=1e-6) np.testing.assert_allclose(ath_results[1], tf_results[1], atol=1e-6) np.testing.assert_allclose(ath_results[2], tf_results[2], atol=1e-6) print('Pass batch matmul op test with shape ', shape1, shape2)
def test_Onehot(): X = ad.Variable(name="X") classes = 10 y = ad.one_hot_op(X, classes) executor = ad.Executor([y], ctx=ctx) X_val = rand.randint( 0, 10, 20, ).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X], [y], [X_val]) print(sys._getframe().f_code.co_name, 'pass!')
def test_csrmv_op(executor_ctx): X = ad.Variable(name="X") W = ad.Variable(name="W") Y = ad.csrmv_op(X, W) Y_ = ad.Variable(name="Y_") temp = Y + (-1) * Y_ loss = temp * temp grads = ad.gradients(loss, [W, Y]) executor = ad.Executor( [loss, grads[0], grads[1]], ctx=executor_ctx) rand = np.random.RandomState(seed=123) W_val =rand.normal(scale=0.1, size=[70000, ]) if ndarray.is_gpu_ctx(executor_ctx): W_val = ndarray.array(W_val, ctx=executor_ctx) X_val = scipy.sparse.rand(500, 70000, density=1e-5,format='coo',dtype=np.float32) Y_val = np.random.uniform(0, 10, size=(500, )).astype(np.float32) loss_val = executor.run(feed_dict={X: X_val, Y_: Y_val, W: W_val}) if ndarray.is_gpu_ctx(executor_ctx): W_val = W_val.asnumpy() loss_val = [val.asnumpy() for val in loss_val] y_groundtruth = X_val.dot(W_val) loss_groundtruth = (y_groundtruth - Y_val) ** 2 Y_grad_groundtruth = 2 * (y_groundtruth - Y_val) * np.ones(loss_groundtruth.shape) W_grad_groundtruth = X_val.T.dot(Y_grad_groundtruth) np.testing.assert_allclose(loss_val[0], loss_groundtruth, rtol=1e-4) np.testing.assert_allclose(loss_val[1], W_grad_groundtruth, rtol=1e-4) np.testing.assert_allclose(loss_val[2], Y_grad_groundtruth, rtol=1e-4)
def __init__(self, in_features: int, out_features: int, npart: int, name="GCN", custom_init=None): if custom_init is not None: self.weight = ad.Variable(value=custom_init[0], name=name + "_Weight") self.bias = ad.Variable(value=custom_init[1], name=name + "_Bias") else: self.weight = initializers.xavier_uniform(shape=(in_features, out_features), name=name + "_Weight") self.bias = initializers.zeros(shape=(out_features, ), name=name + "_Bias") self.in_features = in_features self.out_features = out_features self.npart = npart #npart * npart message_passing matrix, either dense or sparse self.mp = [[ ad.Variable("message_passing", trainable=False) for j in range(npart) ] for i in range(npart)]
def test_sparse(): npemb = np.random.random((100, 20)).astype(np.float32) npind = np.array(np.random.randint(100, size=(10,))) npw = np.random.random((20, 30)).astype(np.float32) cpuctx = ndarray.cpu(0) gpuctx = ndarray.gpu(0) embedding = ad.Variable('embeddingtable', value=npemb, ctx=cpuctx) index = ad.Variable(name="index", ctx=cpuctx) W = ad.Variable(name="w", value=npw) y = ad.embedding_lookup_op(embedding, index) # (10, 20) y = ad.matmul_op(y, W) opt = optimizer.SGDOptimizer(0.1) train_op = opt.minimize(y) executor = ad.Executor([y, train_op],ctx=gpuctx) out, _ = executor.run(feed_dict={index: npind.astype(np.float32)}, convert_to_numpy_ret_vals=True) np_out = np.matmul(npemb[npind], npw) np.testing.assert_allclose(out, np_out, rtol=1e-6) tmp_grad = np.matmul(np.ones(np_out.shape).astype(np.float32), npw.T) for i, localid in enumerate(npind): npemb[localid] -= 0.1 * tmp_grad[i] np.testing.assert_allclose(embedding.tensor_value.asnumpy(), npemb, rtol=1e-6)
def test_csrmm_op(executor_ctx): X = ad.Variable(name="X") W = ad.Variable(name="W") Y = ad.csrmm_op(X, W) Y_ = ad.Variable(name="Y_") loss = ad.softmaxcrossentropy_op(Y, Y_) loss = ad.reduce_mean_op(loss, [0]) grads = ad.gradients(loss, [W, Y]) executor = ad.Executor( [loss, grads[0], grads[1]], ctx=executor_ctx) rand = np.random.RandomState(seed=123) W_val = rand.normal(scale=0.1, size=[70000, 2]).astype(np.float32) if ndarray.is_gpu_ctx(executor_ctx): W_val = ndarray.array(W_val, ctx=executor_ctx) X_val = scipy.sparse.rand(500, 70000, density=1e-5,format='coo',dtype=np.float32) Y_val = np.random.uniform(0, 10, size=(500, 2)).astype(np.float32) loss_val = executor.run(feed_dict={X: X_val, Y_: Y_val, W: W_val}) if ndarray.is_gpu_ctx(executor_ctx): W_val = W_val.asnumpy() loss_val = [val.asnumpy() for val in loss_val] y_groundtruth = X_val.dot(W_val) loss_groundtruth = np.mean( -np.sum(Y_val * np.log(softmax_func(y_groundtruth)), axis=1), keepdims=True) Y_grad_groundtruth = (softmax_func(y_groundtruth) + -1 * Y_val) * np.ones(loss_groundtruth.shape) / 500 W_grad_groundtruth = X_val.T.dot(Y_grad_groundtruth) np.testing.assert_allclose(loss_val[0], loss_groundtruth, rtol=1e-4) np.testing.assert_allclose(loss_val[1], W_grad_groundtruth, rtol=1e-4) np.testing.assert_allclose(loss_val[2], Y_grad_groundtruth, rtol=1e-4)
def _onnx_initializer_to_input_dict_items(cls, initializer, initialized,): def get_flow_shape(shape): if len(shape) == 0: return (1,) return shape return [ ( init.name, ad.Variable(name=init.name,value=initialized[init.name] ), ) for init in initializer ]
def test_BatchNorm(): X = ad.Variable(name="X") bn_scale = init.random_normal((64, ), stddev=0.1, name='bn_scale') bn_bias = init.random_normal((64, ), stddev=0.1, name='bn_bias') y = ad.batch_normalization_op(X, bn_scale, bn_bias) executor = ad.Executor([y], ctx=ctx) X_val = rand.normal(scale=0.1, size=(batch_size, 64, 28, 28)).astype(np.float32) res = executor.run(feed_dict={X: X_val}) Check(executor, res, [X, bn_scale, bn_bias], [y], [X_val, bn_scale.tensor_value, bn_bias.tensor_value]) print(sys._getframe().f_code.co_name, 'pass!')