def test_conv2d(): x2 = ad.Variable(name='x2') w2 = ad.Variable(name='w2') b2 = ad.Variable(name='b2') y = au.nn.conv2d(input=x2, filter=w2, bias=b2) grad_x2, grad_w2, grad_b2 = ad.gradients(y, [x2, w2, b2]) executor = ad.Executor([y, grad_x2, grad_w2, grad_b2]) x2_val = np.random.randn(1, 2, 4, 4) w2_val = np.random.randn(2, 2, 3, 3) b2_val = np.random.randn(2, ) y_val, grad_x2_val, grad_w2_val, grad_b2_val = executor.run(feed_shapes={ x2: x2_val, w2: w2_val, b2: b2_val }) numerical_grad_w2 = ad.eval_numerical_grad(y, feed_dict={ x2: x2_val, w2: w2_val, b2: b2_val }, wrt=w2) numerical_grad_x2 = ad.eval_numerical_grad(y, feed_dict={ x2: x2_val, w2: w2_val, b2: b2_val }, wrt=x2) numerical_grad_b2 = ad.eval_numerical_grad(y, feed_dict={ x2: x2_val, w2: w2_val, b2: b2_val }, wrt=b2) assert isinstance(y, ad.Node) npt.assert_array_almost_equal(numerical_grad_w2, grad_w2_val) npt.assert_array_almost_equal(numerical_grad_x2, grad_x2_val) npt.assert_array_almost_equal(numerical_grad_b2, grad_b2_val) x2 = ad.Variable(name='x2') w2 = ad.Parameter(name='w2', init=w2_val) b2 = ad.Parameter(name='b2', init=b2_val) y = au.nn.conv2d(x2, w2, b2) grad_x2, grad_w2, grad_b2 = ad.gradients(y, [x2, w2, b2]) executor = ad.Executor([y, grad_x2, grad_w2, grad_b2]) y_val, grad_x2_val, grad_w2_val, grad_b2_val = executor.run( feed_shapes={x2: x2_val}) assert isinstance(y, ad.Node) npt.assert_array_almost_equal(numerical_grad_w2, grad_w2_val) npt.assert_array_almost_equal(numerical_grad_b2, grad_b2_val) npt.assert_array_almost_equal(numerical_grad_x2, grad_x2_val)
def test_matmul_two_vars(): x2 = ad.Variable(name='x2') x3 = ad.Variable(name='x3') y = ad.matmul(x2, x3) grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = np.array([[1, 2], [3, 4], [5, 6]]) # 3x2 x3_val = np.array([[7, 8, 9], [10, 11, 12]]) # 2x3 y_val, grad_x2_val, grad_x3_val = executor.run(feed_shapes={ x2: x2_val, x3: x3_val }) expected_yval = np.matmul(x2_val, x3_val) expected_grad_x2_val = np.matmul(np.ones_like(expected_yval), np.transpose(x3_val)) expected_grad_x3_val = np.matmul(np.transpose(x2_val), np.ones_like(expected_yval)) assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val)
def test_softmax(): shape = (2, 2) x_val = np.random.uniform(-5, 5, shape).astype(np.float32) x2 = ad.Variable(name='x2') prob = au.nn.softmax(x2) executor = ad.Executor([prob], use_gpu=True) y, = executor.run(feed_shapes={x2: x_val}) y = y.asnumpy() np.testing.assert_allclose(au.nn.softmax_func(x_val), y, rtol=1e-5)
def __init__(self, cost, params, lr=0.1, use_gpu=False): self.cost = cost # if use_gpu == True, create matrices in GPU self.params = self._copy_to_gpu(params) if use_gpu else params self.lr = lr grads = ad.gradients(cost, params) grads.insert(0, cost) self.use_gpu = use_gpu self.executor = ad.Executor(grads, use_gpu=use_gpu)
def measure_accuracy(activation, data, use_gpu=False): X_val, y_val = data executor = ad.Executor([activation], use_gpu=use_gpu) prob_val, = executor.run(feed_shapes={X: X_val}) if use_gpu: prob_val = prob_val.asnumpy() correct = np.sum(np.equal(y_val, np.argmax(prob_val, axis=1))) percentage = (correct / (y_val.shape[0])) * 100.00 return percentage
def test_reshape(): x2 = ad.Variable(name='x2') y = ad.reshape(x2, newshape=(1, 4)) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = np.random.randn(2, 2) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) assert isinstance(y, ad.Node) assert y_val.shape == (1, 4) npt.assert_array_equal(grad_x2_val, np.ones((2, 2)))
def test_relu(): x2 = ad.Variable(name='x2') y = au.nn.relu(x2) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = np.array([[-1, 2, 3], [1, -2, 0]]) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) expected_y_val = np.array([[0, 2, 3], [1, 0, 0]]) expected_x2_grad = np.array([[0, 1, 1], [1, 0, 0]]) assert np.array_equal(y_val, expected_y_val) assert np.array_equal(grad_x2_val, expected_x2_grad)
def test_sub_by_const(): x2 = ad.Variable(name='x2') y = x2 - 3 grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 2 * np.ones(3) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) # asserts assert isinstance(y, ad.Node) assert np.array_equal(y_val, -1 * np.ones(3)) assert np.array_equal(grad_x2_val, np.ones_like(x2_val))
def test_reduce_sum(): x2 = ad.Variable(name='x2') y = ad.reduce_sum(x2) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = np.array([[1, 2, 3], [4, 5, 6]]) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) # asserts assert isinstance(y, ad.Node) assert np.array_equal(y_val, np.array([5, 7, 9])) assert np.array_equal(grad_x2_val, np.array([1, 1, 1]))
def test_add_by_const(): x2 = ad.Variable(name="x2") y = 5 + x2 grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 2 * np.ones(3) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x2_val + 5) assert np.array_equal(grad_x2_val, np.ones_like(x2_val))
def test_sigmoid_activation(): x2 = ad.Variable(name='x2') y = au.nn.sigmoid(x2) x2_val = np.array([-100, 0, 100]) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) npt.assert_array_almost_equal(np.array([0.000, 0.500, 1.0]), y_val) npt.assert_array_almost_equal(np.array([0, 0.25, 0]), grad_x2_val) # testing with extreme values for numerical stability. x2_val = np.array([-9.9e10, 9.9e10]).astype(np.float32) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) npt.assert_array_almost_equal(np.array([0.0, 1.0]), y_val) npt.assert_array_almost_equal(np.array([0.0, 0.0]), grad_x2_val)
def test_identity(): x2 = ad.Variable(name='x2') y = x2 grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2], use_gpu=True) x2_val = 2 * np.ones(3) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) y_val_np = y_val.asnumpy() grad_x2_val_np = grad_x2_val.asnumpy() assert isinstance(y, ad.Node) assert np.array_equal(y_val_np, x2_val) assert np.array_equal(grad_x2_val_np, np.ones_like(x2_val))
def test_cross_entropy(): x2_pred = ad.Variable(name='x2_pred') x2_actu = ad.Variable(name='x2_actu') y = au.nn.softmax_cross_entropy_with_logits(x2_pred, x2_actu) x2_pred_grad, x2_actu_grad = ad.gradients(y, [x2_pred, x2_actu]) x2_pred_val = np.array([[0.8, 0.01, 0.5], [0.8, 0.01, 0.5]]) x2_actu_val = np.array([[1.0, 1.0, 0], [1.0, 1.0, 0]]) executor = ad.Executor([y, x2_pred_grad, x2_actu_grad]) y_val, x2_pred_grad_val, x2_actu_grad_val = executor.run( feed_shapes={ x2_pred: x2_pred_val, x2_actu: x2_actu_val }) # print(x2_actu_grad_val) assert True
def test_broadcast_to(): x2 = ad.Variable(name='x2') x3 = ad.Variable(name='x3') y = ad.broadcast_to(x2, x3) grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = np.array([[1, 2, 3]]) x3_val = np.zeros((3, 3)) y_val, grad_x2_val, grad_x3_val = executor.run(feed_shapes={ x2: x2_val, x3: x3_val }) # asserts assert isinstance(y, ad.Node) assert np.array_equal(y_val, np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]])) assert np.array_equal(grad_x2_val, np.array([3, 3, 3]))
def test_max_pooling(): x2 = ad.Variable(name='x2') y = au.nn.maxPool(x2, filter=(2, 2), strides=(2, 2)) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = np.random.randn(1, 1, 4, 4) y_val, grad_x2_val = executor.run(feed_shapes={x2: x2_val}) numerical_grad_x2 = ad.eval_numerical_grad(y, feed_dict={x2: x2_val}, wrt=x2, h=1e-5) assert isinstance(y, ad.Node) # TODO: (upul) looks like a bug in my eval_numerical_grad implementation # Hence I'm using one decimal points npt.assert_array_almost_equal(grad_x2_val, numerical_grad_x2, decimal=2)
def test_div_two_var(): x2 = ad.Variable(name='x2') x3 = ad.Variable(name='x3') y = x2 / x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = 4 * np.ones(3) x3_val = 2 * np.ones(3) y_val, grad_x2_val, grad_x3_val = executor.run(feed_shapes={ x2: x2_val, x3: x3_val }) # asserts assert isinstance(y, ad.Node) assert np.array_equal(grad_x2_val, 1.0 / x3_val) assert np.array_equal(grad_x3_val, -1.0 * x2_val / (x3_val * x3_val))
def test_sub_two_vars(): x2 = ad.Variable(name='x2') x3 = ad.Variable(name='x3') y = x2 - x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = 4 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val = executor.run(feed_shapes={ x2: x2_val, x3: x3_val }) # asserts assert isinstance(y, ad.Node) assert np.array_equal(y_val, 1 * np.ones(3)) assert np.array_equal(grad_x2_val, np.ones(3)) assert np.array_equal(grad_x3_val, -1 * np.ones(3))
def test_matmul_var_and_param(): x2 = ad.Variable(name="x2") w2_val = np.array([[7, 8, 9], [10, 11, 12]]) # 2x3 w2 = ad.Parameter(name="w2", init=w2_val) y = ad.matmul(x2, w2) grad_x2, grad_w2 = ad.gradients(y, [x2, w2]) executor = ad.Executor([y, grad_x2, grad_w2]) x2_val = np.array([[1, 2], [3, 4], [5, 6]]) # 3x2 y_val, grad_x2_val, grad_w2_val = executor.run(feed_shapes={x2: x2_val}) expected_yval = np.matmul(x2_val, w2_val) expected_grad_x2_val = np.matmul(np.ones_like(expected_yval), np.transpose(w2_val)) expected_grad_x3_val = np.matmul(np.transpose(x2_val), np.ones_like(expected_yval)) assert isinstance(y, ad.Node)
def measure_accuracy(activation, data, batch_size=32, use_gpu=False): X_val, y_val = data executor = ad.Executor([activation], use_gpu=use_gpu) max_val = len(X_val) - len(X_val) % batch_size y_val = y_val[0:max_val] prediction = np.zeros(max_val) for i in range(0, max_val, batch_size): start = i end = i + batch_size X_batch, y_batch = X_val[start:end], y_val[start:end] prob_val, = executor.run(feed_shapes={images: X_batch}) if use_gpu: prob_val = prob_val.asnumpy() prediction[start:end] = np.argmax(prob_val, axis=1) correct = np.sum(np.equal(y_val, prediction)) percentage = (correct / len(prediction)) * 100.00 return percentage