def test_grad_of_grad(): x2 = ad.Variable(name = "x2") x3 = ad.Variable(name = "x3") y = x2 * x2 + x2 * x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) grad_x2_x2, grad_x2_x3 = ad.gradients(grad_x2, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3, grad_x2_x2, grad_x2_x3]) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val, grad_x2_x2_val, grad_x2_x3_val = executor.run(feed_dict = {x2: x2_val, x3: x3_val}) expected_yval = x2_val * x2_val + x2_val * x3_val expected_grad_x2_val = 2 * x2_val + x3_val expected_grad_x3_val = x2_val expected_grad_x2_x2_val = 2 * np.ones_like(x2_val) expected_grad_x2_x3_val = 1 * np.ones_like(x2_val) assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val) assert np.array_equal(grad_x2_x2_val, expected_grad_x2_x2_val) assert np.array_equal(grad_x2_x3_val, expected_grad_x2_x3_val)
def test_eq_node(): x2 = ad.Variable(name = "x2") x3 = ad.Variable(name = "x3") y = (x2==x3) executor = ad.Executor([y]) x2_val = 2 * np.ones(3) x3_val = 2 * np.ones(3) y_val = executor.run(feed_dict = {x2 : x2_val, x3 : x3_val}) assert np.array_equal(y_val[0], x2_val==x3_val)
def test_mul_two_vars(): x2 = ad.Variable(name = "x2") x3 = ad.Variable(name = "x3") y = x2 * x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val = executor.run(feed_dict = {x2: x2_val, x3: x3_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x2_val * x3_val) assert np.array_equal(grad_x2_val, x3_val) assert np.array_equal(grad_x3_val, x2_val)
def test_add_mul_mix_1(): x1 = ad.Variable(name = "x1") x2 = ad.Variable(name = "x2") x3 = ad.Variable(name = "x3") y = x1 + x2 * x3 * x1 grad_x1, grad_x2, grad_x3 = ad.gradients(y, [x1, x2, x3]) executor = ad.Executor([y, grad_x1, grad_x2, grad_x3]) x1_val = 1 * np.ones(3) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x1_val, grad_x2_val, grad_x3_val = executor.run(feed_dict = {x1 : x1_val, x2: x2_val, x3 : x3_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x1_val + x2_val * x3_val) assert np.array_equal(grad_x1_val, np.ones_like(x1_val) + x2_val * x3_val) assert np.array_equal(grad_x2_val, x3_val * x1_val) assert np.array_equal(grad_x3_val, x2_val * x1_val)
def test_ne_const_node(): x2 = ad.Variable(name = "x2") y = (x2!=2) executor = ad.Executor([y]) x2_val = 2 * np.ones(3) y_val = executor.run(feed_dict = {x2 : x2_val}) assert np.array_equal(y_val[0], x2_val!=2)
def test_matmul_two_vars(): x2 = ad.Variable(name = "x2") x3 = ad.Variable(name = "x3") y = ad.matmul_op(x2, x3) grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = np.array([[1, 2], [3, 4], [5, 6]]) # 3x2 x3_val = np.array([[7, 8, 9], [10, 11, 12]]) # 2x3 y_val, grad_x2_val, grad_x3_val = executor.run(feed_dict = {x2: x2_val, x3: x3_val}) expected_yval = np.matmul(x2_val, x3_val) expected_grad_x2_val = np.matmul(np.ones_like(expected_yval), np.transpose(x3_val)) expected_grad_x3_val = np.matmul(np.transpose(x2_val), np.ones_like(expected_yval)) assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val)
def test_add_mul_mix_3(): x2 = ad.Variable(name = "x2") x3 = ad.Variable(name = "x3") z = x2 * x2 + x2 + x3 + 3 y = z * z + x3 grad_x2, grad_x3 = ad.gradients(y, [x2, x3]) executor = ad.Executor([y, grad_x2, grad_x3]) x2_val = 2 * np.ones(3) x3_val = 3 * np.ones(3) y_val, grad_x2_val, grad_x3_val = executor.run(feed_dict = {x2: x2_val, x3: x3_val}) z_val = x2_val * x2_val + x2_val + x3_val + 3 expected_yval = z_val * z_val + x3_val expected_grad_x2_val = 2 * (x2_val * x2_val + x2_val + x3_val + 3) * (2 * x2_val + 1) expected_grad_x3_val = 2 * (x2_val * x2_val + x2_val + x3_val + 3) + 1 assert isinstance(y, ad.Node) assert np.array_equal(y_val, expected_yval) assert np.array_equal(grad_x2_val, expected_grad_x2_val) assert np.array_equal(grad_x3_val, expected_grad_x3_val)
def test_mul_by_const(): x2 = ad.Variable(name = "x2") y = 5 * x2 grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 2 * np.ones(3) y_val, grad_x2_val= executor.run(feed_dict = {x2 : x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, x2_val * 5) assert np.array_equal(grad_x2_val, np.ones_like(x2_val) * 5)
def test_log(): x2 = ad.Variable(name = "x2") y = ad.log_op(x2, 5) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 0.5 * np.ones(3) y_val, grad_x2_val= executor.run(feed_dict = {x2 : x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, np.log(x2_val)/np.log(5)) assert (abs(grad_x2_val - 1./(x2_val*np.log(5)))<1e-7 ).all()
def test_tanh(): # this can test tan, cos, sin and power at the same time x2 = ad.Variable(name = "x2") y = ad.tanh_op(x2) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 2 * np.ones(3) y_val, grad_x2_val= executor.run(feed_dict = {x2 : x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, np.tanh(x2_val)) assert np.array_equal(grad_x2_val, (np.cosh(x2_val)**2 - np.sinh(x2_val)**2)/np.cosh(x2_val)**2)
def test_logistic(): x2 = ad.Variable(name = "x2") y = ad.logistic_op(x2) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 0.5 * np.ones(3) y_val, grad_x2_val= executor.run(feed_dict = {x2 : x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, 1./(1+np.exp(-x2_val))) print(grad_x2_val - 2*(1./(1+np.exp(-x2_val)))*(1-1./(1+np.exp(-x2_val)))) assert (abs(grad_x2_val - (1./(1+np.exp(-x2_val)))*(1-1./(1+np.exp(-x2_val))))<1e-7 ).all()
def test_arccos(): # this can test tan, cos, sin and power at the same time x2 = ad.Variable(name = "x2") y = ad.arccos_op(x2) grad_x2, = ad.gradients(y, [x2]) executor = ad.Executor([y, grad_x2]) x2_val = 0.5 * np.ones(3) y_val, grad_x2_val= executor.run(feed_dict = {x2 : x2_val}) assert isinstance(y, ad.Node) assert np.array_equal(y_val, np.arccos(x2_val)) assert (abs(grad_x2_val + 1./(1-x2_val**2)**0.5)<1e-7 ).all()