def test_double_variable_power_throws():
x, y = ad.Variable(), ad.Variable()
f = x**(y + 5.0)
g = x**y
with pytest.raises(NotImplementedError):
g.hessian({x: 1, y: 1})
with pytest.raises(NotImplementedError):
f.hessian({x: 1, y: 1})
def test_arithmetic_multivar_hessian():
x, y, z = ad.Variable(), ad.Variable(), ad.Variable()
f = 3 * x * x * y + (z - 1.0 / x)**5
h = f.hessian({x: 1, y: 2, z: 3})
assert (equals(h[x][x], 12.0))
assert (equals(h[y][y], 0))
assert (equals(h[z][z], 160))
assert (equals(h[x][y], 6))
assert (equals(h[y][z], 0))
assert (equals(h[x][z], 160))
def test_trig_multivar_hessian_1():
x, y, z = ad.Variable(), ad.Variable(), ad.Variable()
f = ad.Sin(x * y) + z * ad.Cos(z * ad.Tan(1 / z))
h = f.hessian({x: 1, y: 3, z: 5})
assert (equals(h[x][x], -1.2700800725))
assert (equals(h[y][y], -0.14112000805))
assert (equals(h[z][z], -0.0050593837))
assert (equals(h[x][y], -1.41335252))
assert (equals(h[y][z], 0))
assert (equals(h[x][z], 0))
def test_trig_multivar_hessian_2():
x, y, z = ad.Variable(), ad.Variable(), ad.Variable()
f = ad.Sinh(x * y) + (x + y) * (z**2) * ad.Cosh(z * ad.Tanh(1 / z))
h = f.hessian({x: 1, y: 2, z: 3})
assert (equals(h[x][x], 14.50744163138))
assert (equals(h[y][y], 3.6268604078))
assert (equals(h[z][z], 9.3022279093))
assert (equals(h[x][y], 11.015916506))
assert (equals(h[y][z], 9.2426242912))
assert (equals(h[x][z], 9.2426242912))
def test_logexp_multivar_hessian_2():
x, y, z = ad.Variable(), ad.Variable(), ad.Variable()
f = ad.Sinh(ad.Exp(x - 3.0) * y) + ad.Log(y + x**2) * z * ad.Sin(x)
h = f.hessian({x: 1, y: 3, z: 5})
assert (equals(h[x][x], -1.5705707143))
assert (equals(h[y][y], -0.2553174360))
assert (equals(h[z][z], 0))
assert (equals(h[x][y], 0.31902899408))
assert (equals(h[y][z], 0.2103677462))
assert (equals(h[x][z], 1.1697535323))
def test_variable_inheritance():
x = ad.Variable()
y = ad.Variable()
f = x * y
assert (x in f.dep_vars)
assert (y in f.dep_vars)
assert (len(f.dep_vars) == 2)
assert (y not in x.dep_vars)
assert (x not in y.dep_vars)
def test_arithmetic_multivar_hessian_2():
x, y, z = ad.Variable(), ad.Variable(), ad.Variable()
f = -x * (x - y + 1.0)**(-4) + (z / x)**2
h = f.hessian({x: 1, y: 3, z: 5})
assert (equals(h[x][x], 122))
assert (equals(h[y][y], -20))
assert (equals(h[z][z], 2))
assert (equals(h[x][y], 24))
assert (equals(h[y][z], 0))
assert (equals(h[x][z], -20))
def softmaxtest():
x1 = ad.Variable("x1")
x2 = ad.Variable("x2")
d = ad.Variable("d")
d_val = np.array([0, 1, 0]).reshape(3, 1)
x1_val = 3 * np.ones((3, 1))
x2_val = 5 * np.ones((3, 1))
J = ad.softmax_crossent(x1 * x2, d)
ex = ad.Executor([J] + ad.gradients(J, [x1, x2]))
res = ex.run({x1: x1_val, x2: x2_val, d: d_val})
pass
def multiconnection():
x1 = ad.Variable("x1")
x2 = ad.Variable("x2")
x3 = ad.Variable("x3")
s = x1 + x2
y = (s + x3) + (5 * s)
grads = ad.gradients(y, [x1, x2, x3])
executor = ad.Executor([y] + grads)
feed_dict = {x1: 1, x2: 2, x3: 3}
res = executor.run(feed_dict)
return
def subtest():
x1 = ad.Variable("x1")
x2 = ad.Variable("x2")
x1_val = 9
x2_val = 6
y = x1 - x2
z = x1 - 4
u = 8 - x2
Ex_y = ad.Executor([y] + ad.gradients(y, [x1, x2]))
Ex_z = ad.Executor([z] + ad.gradients(z, [x1]))
Ex_u = ad.Executor([u] + ad.gradients(u, [x2]))
print(Ex_y.run(feed_dict={x1: x1_val, x2: x2_val}))
print(Ex_z.run(feed_dict={x1: x1_val}))
print(Ex_u.run(feed_dict={x2: x2_val}))
return
def test_binop():
x = ad.Variable("x")
y = x**3
assert np.isclose(y.d_n(n=0, val=2.0), 8.0)
assert np.isclose(y.d_n(n=1, val=2.0), 12.0)
assert np.isclose(y.d_n(n=3, val=2.0), 6.0)
assert np.isclose(y.d_n(n=5, val=2.0), 0.0)
y = x + x**3
assert np.isclose(y.d_n(n=0, val=2.0), 10.0)
assert np.isclose(y.d_n(n=1, val=2.0), 13.0)
assert np.isclose(y.d_n(n=3, val=2.0), 6.0)
assert np.isclose(y.d_n(n=5, val=2.0), 0.0)
y = x - x**3
assert np.isclose(y.d_n(n=0, val=2.0), -6.0)
assert np.isclose(y.d_n(n=1, val=2.0), -11.0)
assert np.isclose(y.d_n(n=3, val=2.0), -6.0)
y = x * (x**2)
assert np.isclose(y.d_n(n=0, val=2.0), 8.0)
assert np.isclose(y.d_n(n=1, val=2.0), 12.0)
assert np.isclose(y.d_n(n=3, val=2.0), 6.0)
assert np.isclose(y.d_n(n=5, val=2.0), 0.0)
y = (x**5) / (x**2)
assert np.isclose(y.d_n(n=0, val=2.0), 8.0)
assert np.isclose(y.d_n(n=1, val=2.0), 12.0)
assert np.isclose(y.d_n(n=3, val=2.0), 6.0)
assert np.isclose(y.d_n(n=5, val=2.0), 0.0)
def test_single_variable_trig_cos():
"""analytic hessian is -25 cos (5 * x + 3)"""
x = ad.Variable()
f = ad.Cos(5 * x + 3)
assert (equals(f.hessian({x: 1}), -25 * np.cos(5 * 1 + 3)))
assert (equals(f.hessian({x: 2}), -25 * np.cos(5 * 2 + 3)))
assert (equals(f.hessian({x: 3}), -25 * np.cos(5 * 3 + 3)))
def test_single_variable_trig_tan():
"""analytic hessian is -25 cos (5 * x + 3)"""
x = ad.Variable()
f = ad.Cos(5 * x + 3) * ad.Tan(x * x - 5)
assert (equals(f.hessian({x: 1}), -48.05115800))
assert (equals(f.hessian({x: 2}), -170.9403025))
assert (equals(f.hessian({x: 3}), 218.2792716))
def test_single_variable_trig_hyperbolic_2():
x = ad.Variable()
# x^2 Cosh[Sin[x] + Tanh[Exp[3 * x] + Log[x]]]
g = ad.Sin(x) + ad.Tanh(ad.Exp(3 * x) + ad.Log(x))
f = x * x * ad.Cosh(g)
assert (equals(f.hessian({x: 1}), 11.464317742))
assert (equals(f.hessian({x: 2}), -13.704377252))
def test_cosine():
x = ad.Variable("x")
y = ad.Cos(x)
yd1 = y.d_expr()
assert np.isclose(yd1.eval({x: pi / 2}), -1.0)
assert np.isclose(yd1.eval({x: 0.0}), 0.0)
assert np.isclose(yd1.d({x: 0.0}), -1.0)
def test_constant_division():
a = ad.Constant(5)
a2 = ad.Constant(-5)
b = ad.Variable('b')
c = a / b
d = b / a2
assert c.eval({b: 5}) == 1
assert d.eval({b: 5}) == -1
def test_constant_subtraction():
a = ad.Constant(5)
a2 = ad.Constant(-5)
b = ad.Variable('b')
c = a - b
d = b - a2
assert c.eval({b: 5}) == 0
assert d.eval({b: 5}) == 10
def test_constant_multiplication():
a = ad.Constant(5)
a2 = ad.Constant(-5)
b = ad.Variable('b')
c = a * b
d = b * a2
assert c.eval({b: 5}) == 25
assert d.eval({b: 5}) == -25
def test_inverse_trig():
a, b = ad.Variable('a'), ad.Variable('b')
c = ad.Arcsin(a / b)
assert np.isclose(np.arcsin(0.5), c.eval({a: 1.0, b: 2.0}))
d = ad.Arcsin(a)
assert np.isclose(1.0 / np.sqrt(1.0 - 0.5**2), d.d({a: 0.5}))
c = ad.Arccos(a * b)
assert np.isclose(np.arccos(0.25), c.eval({a: 0.5, b: 0.5}))
d = ad.Arccos(a)
assert np.isclose(-1.0 / np.sqrt(1.0 - 0.5**2), d.d({a: 0.5}))
c = ad.Arctan(a / b)
assert np.isclose(np.arctan(0.5), c.eval({a: 1.0, b: 2.0}))
d = ad.Arctan(a)
assert np.isclose(1.0 / (1 + 0.5**2), d.d({a: 0.5}))
def test_variable_inheritance_three():
x = ad.Variable()
y = ad.Variable()
z = ad.Variable()
f = ad.Cos(x) * y
g = ad.Sin(f) + z * z * ad.Log(z) + 1
assert (x in f.dep_vars)
assert (y in f.dep_vars)
assert (len(f.dep_vars) == 2)
assert (y not in x.dep_vars)
assert (x not in y.dep_vars)
assert (len(g.dep_vars) == 3)
assert (x in g.dep_vars)
assert (y in g.dep_vars)
assert (z in g.dep_vars)
assert (z not in f.dep_vars)
def test_substration_and_negation():
x = ad.Variable("x")
y = ad.Exp(2 * x) - x
yd1 = y.d_expr()
assert np.isclose(yd1.eval({x: 0.0}), 1.0)
y = -ad.Exp(3 * x)
yd1 = y.d_expr()
assert np.isclose(yd1.eval({x: 0.0}), -3.0)
def test_high_order():
x = ad.Variable("x")
y = x**4
yd5 = y.d_expr(5)
yd9 = y.d_expr(9)
assert np.isclose(yd5.eval({x: 4.0}), 0.0)
assert np.isclose(yd5.eval({x: 10.0}), 0.0)
assert np.isclose(yd9.eval({x: 4.0}), 0.0)
assert np.isclose(yd9.eval({x: 10.0}), 0.0)
def test_inverse_trig_exceptions():
x = ad.Variable('x')
y = ad.Variable('y')
f5 = ad.Arccos(x)
f6 = ad.Arcsin(x)
f7 = ad.Arctan(x)
f8 = ad.Log(x)
f9 = ad.Sin(x)
assert f5._d_expr(y).eval({x:0}) == 0
assert np.isclose(f5._d_expr(x).eval({x:0.5}), f5.d({x: 0.5}))
assert f6._d_expr(y).eval({x:0}) == 0
assert np.isclose(f6._d_expr(x).eval({x:0}), f6.d({x: 0}))
assert f7._d_expr(y).eval({x:0}) == 0
assert np.isclose(f7._d_expr(x).eval({x:0}), f7.d({x: 0}))
assert f8._d_expr(y).eval({x:1}) == 0
assert np.isclose(f8._d_expr(x).eval({x:2}), f8.d({x: 2}))
assert f9._d_expr(y).eval({x:1}) == 0
assert np.isclose(f9._d_expr(x).eval({x:2}), f9.d({x: 2}))
def test_power():
x = ad.Variable("x")
y = x**4
yd1 = y.d_expr()
yd2 = yd1.d_expr()
yd3 = yd2.d_expr()
yd4 = yd3.d_expr()
assert np.isclose(yd1.eval({x: 2.0}), 32.0)
assert np.isclose(yd2.eval({x: 2.0}), 48.0)
assert np.isclose(yd3.eval({x: 2.0}), 48.0)
assert np.isclose(yd4.eval({x: 2.0}), 24.0)
assert np.isclose(yd4.eval({x: 3.0}), 24.0)
def test_hyperbolic_expressions():
x = ad.Variable('x')
y = ad.Variable('y')
f1 = ad.Tan(x)
f2 = ad.Sinh(x)
f3 = ad.Cosh(x)
f4 = ad.Tanh(x)
f5 = ad.Exp(x)
f6 = ad.Cos(x)
assert f1._d_expr(y).eval({x:1}) == 0
assert np.isclose(f1._d_expr(x).eval({x:1}), f1.d({x: 1}))
assert f2._d_expr(y).eval({x:1}) == 0
assert np.isclose(f2._d_expr(x).eval({x:1}), f2.d({x: 1}))
assert f3._d_expr(y).eval({x:1}) == 0
assert np.isclose(f3._d_expr(x).eval({x:1}), f3.d({x: 1}))
assert f4._d_expr(y).eval({x:1}) == 0
assert np.isclose(f4._d_expr(x).eval({x:1}), f4.d({x: 1}))
assert f5._d_expr(y).eval({x:1}) == 0
assert np.isclose(f5._d_expr(x).eval({x:1}), f5.d({x: 1}))
assert f6._d_expr(y).eval({x:1}) == 0
assert np.isclose(f6._d_expr(x).eval({x:1}), f6.d({x: 1}))
def test_power_base0():
a = ad.Variable('a')
c = a**2
assert np.isclose(0, c.d_n(2, 0))
assert np.isclose(0, c.d_n(4, 0))
d = a**(-1)
with pytest.raises(ZeroDivisionError):
d.d_n(2, 0)
e = a**(3.5)
with pytest.raises(ZeroDivisionError):
e.d_n(4, 0)
def test_division():
x = ad.Variable("x")
y = 1 / x
yd1 = y.d_expr()
assert np.isclose(yd1.eval({x: 2.0}), -0.25)
y = x / 2
yd1 = y.d_expr()
assert np.isclose(yd1.eval({x: 1.0}), 0.5)
y = x / x
yd1 = y.d_expr()
assert np.isclose(yd1.eval({x: 100.0}), 0.0)
def test1():
x1 = ad.Variable(name="x1")
x2 = ad.Variable(name="x2")
x3 = ad.Variable(name="x3")
y = x1 / x2 - x3
executor = ad.Executor([x1, x2, x3, y])
print(executor.run({x1: 0, x2: 2, x3: 3}))
print(y)
grad_x1, grad_x2, grad_x3 = ad.gradients(y, [x1, x2, x3])
executor_backward = ad.Executor([y, grad_x1, grad_x2, grad_x3])
x1_val = 5 * np.ones(3)
x2_val = 2 * np.ones(3)
x3_val = 3 * np.ones(3)
y_val, grad_x1_val, grad_x2_val, grad_x3_val = executor_backward.run(
feed_dict={
x1: x1_val,
x2: x2_val,
x3: x3_val
})
print(y_val, grad_x1_val, grad_x2_val, grad_x3_val)
def test_expression_exceptions():
x = ad.Variable()
c = ad.Expression()
with pytest.raises(NotImplementedError):
c.eval({})
with pytest.raises(NotImplementedError):
c.d({})
with pytest.raises(NotImplementedError):
c.hessian({})
with pytest.raises(NotImplementedError):
c._d_expr(x)
with pytest.raises(NotImplementedError):
c._d_n(1, {}, {}, {})
def test_exp_exceptions():
x = ad.Variable('x')
y = ad.Exp(x)
z = ad.Cos(x)
a = ad.Log(x)
b = ad.Sin(x)
f1 = y*2 + y
f2 = z*2 + z
f3 = a*2 + a
f4 = b*2 + b
assert np.isclose(f1.d_n(1, 1), 8.154845485377136)
assert np.isclose(f2.d_n(1, 1), -2.5244129544236893)
assert np.isclose(f3.d_n(1, 1), 3.0)
assert np.isclose(f4.d_n(1, 1), 1.6209069176044193)
