def _RNN():
ran = lambda x: np.random.random(x) - 0.5
phrase_expr = lambda W_mat, word_left, word_right, bias: VSF(
'tanh', Add(Dot(W_mat, Concat(word_left, word_right)), bias))
W = declareAndCache(u'W')
W.val = ran((200, 400))
bias = declareAndCache(u'b', )
bias.val = ran((200, ))
u_score = declareAndCache(u'u_score', )
u_score.val = ran((200, ))
assert Var(u'W') is W
phrase_w1w2 = lambda w1, w2: phrase_expr(W, w1, w2, bias)
w1 = declareAndCache(u'w1')
w2 = declareAndCache(u'w2')
w1.val, w2.val = ran(200), ran(200)
score = lambda phrase: Dot(u_score, phrase)
#score(phrase_w1w2(w1,w2)).diff(W)
print unicode(phrase_w1w2(w1, w2).diff(W))
def test_GradientNumericalChecks(reset_NodeDict):
ran = lambda x: np.random.random(x) - 0.5
x, y, a, b = Var('x'), Var('y'), Var('a'), Var('b')
D, C, B, A = Var('D'), Var('C'), Var('B'), Var('A')
#g:= x⋅C⋅sin(B⋅sin(A⋅y+a)+b)
g = Dot(Dot(x, C), VSF('sin', Add(Dot(B, VSF('sin', Add(Dot(A, y), a))),
b)))
g.cache()
#0.01 is may not small enough for g=x⋅C⋅sin(B⋅sin(A⋅y+a)+b).
scale = 0.001
for var in [B, A, y, a]:
p = []
p_ran = []
for i in range(3):
x.val, y.val, a.val, b.val = ran((1, 4)), ran((4, 1)), ran(
(5, 1)), ran((6, 1))
#D,C,B,A = Var('D', ran((4,4))), Var('C', ran((4,6))), Var('B', ran((6,5))), Var('A', ran((5,4)))
D.val, C.val, B.val, A.val = ran((4, 4)), ran((4, 6)), ran(
(6, 5)), ran((5, 4))
gradient = Differentiation(g, var)
gradient.cache()
var0 = var.val
delta = NormalizedMatrix(ran(var.val.shape), scale)
rand_grad = gradient.val.copy()
np.random.shuffle(rand_grad)
rand_grad = NormalizedMatrix(ran(gradient.val.shape),
gradient.val.sum())
dg_ran = np.sum(delta * rand_grad)
dg_grad = np.sum(delta * gradient.val)
g0 = g.val
var.val = var0 + delta
g1 = g.val
dg = g1 - g0
p.append(dg / dg_grad)
p_ran.append(dg / dg_ran)
p = np.array(p)
p_ran = np.array(p_ran)
precision = np.abs(np.mean(p) - 1)
precision_ran = np.abs(np.mean(p_ran) - 1)
assert precision < 10 * scale
assert precision < precision_ran
def test_Evaluation(reset_NodeDict):
vx = np.array([1.0, 2.0, 3.0]).reshape(1, 3)
vy = np.array([2.0, 3.0, 4.0]).reshape(3, 1)
vz = np.array([3.0, 5.0, 7.0]).reshape(1, 3)
x = Var('x')
x.val = vx
y = Var('y', vy)
z = Var('z', vz)
with pytest.raises(ValueError):
Dot(x, Var('t', vy.T)).val
xy = Mul(x, y)
assert (unicode(xy) == 'x*y')
assert_all(xy.val == vx.dot(vy))
x_plus_z = Add(x, z)
assert (unicode(x_plus_z) == 'x+z')
assert_all(x_plus_z.val == vx + vz)
assert_all(CTimes(xy, z).val == CTimes(z, xy).val)
assert_all(CTimes(xy, z).val == vx.dot(vy) * vz)
s0 = 1.57
s = Var('s', s0)
fs = VSF('cos', s, np.cos)
assert (unicode(fs) == 'cos(s)')
assert (fs.val == np.cos(s0))
def test_expressions_code_generation():
ran=lambda x : np.random.random(x)-0.5
x=Var('x',ran((1,4)))
y=Var('y',ran((4,1)))
a=Var('a',ran((5,1)))
b=Var('b',ran((6,1)))
D,C,B,A = Var('D', ran((4,4))), Var('C', ran((4,6))), Var('B', ran((6,5))), Var('A', ran((5,4)))
ns={}
f = Dot(Dot(x,C),Dot(B,VSF('sin',Add(Dot(A,y),a))))
assert f.code()=='np.dot(np.dot(x,C),np.dot(B,np.sin(np.add(np.dot(A,y),a))))'
ns=Node.Compile('f',f, ns)
assert ns['f'](A=A.val,B=B.val,C=C.val, a=a.val,x=x.val,y=y.val)==f.val
g = Dot(Dot(x,C),VSF('sin',Add(Dot(B,VSF('sin',Add(Dot(A,y),a))),b)))
assert g.code()=='np.dot(np.dot(x,C),np.sin(np.add(np.dot(B,np.sin(np.add(np.dot(A,y),a))),b)))'
ns=Node.Compile('g',g, ns)
assert ns['g'](A=A.val,B=B.val,C=C.val, a=a.val,b=b.val,x=x.val,y=y.val)==g.val
h,w,b,u,w12 =Var('h'),Var('W'),Var('b'),Var('u'), Var('word12')
phrase=VSF('tanh', Add(Dot(w, h), b))
score=Dot(u,phrase)
assert phrase.code()=='np.tanh(np.add(np.dot(W,h),b))'
assert score.code()=='np.dot(u,np.tanh(np.add(np.dot(W,h),b)))'
def _AccumulateSideExpression(left, right):
if IsMatrix(left) or IsMatrix(right):
return Dot(left, right).simplify()
return CTimes(left, right).simplify()
def test_ReuseExistingExpressions(reset_NodeDict):
ran = lambda x: np.random.random(x) - 0.5
x, y, a, b = Var('x'), Var('y'), Var('a'), Var('b')
D, C, B, A = Var('D'), Var('C'), Var('B'), Var('A')
x.val, y.val, a.val, b.val = ran((1, 4)), ran((4, 1)), ran((5, 1)), ran(
(6, 1))
D.val, C.val, B.val, A.val = ran((4, 4)), ran((4, 6)), ran((6, 5)), ran(
(5, 4))
xDy = Dot(x, Dot(D, y))
xDy.cache()
assert Var('x') is xDy.x
assert Dot(D, y).y is Var('y')
assert Dot(D, y) is xDy.y
xDy2 = Dot(Dot(x, D), y)
tmp = Differentiation(xDy2, x)
tmp2 = Differentiation(xDy2, x)
assert tmp2 is not tmp
tmp.cache()
tmp2 = Differentiation(xDy2, x)
assert tmp2 is tmp
tmp3 = Differentiation(xDy, x)
assert tmp3 is tmp
f = Dot(Dot(x, C), Dot(B, VSF('sin', Add(Dot(A, y), a))))
f.cache()
dfdy = Differentiation(f, y)
dfdy2 = Differentiation(f, y)
assert dfdy2 is not dfdy
dfdy.cache()
dfdy2 = Differentiation(f, y)
assert dfdy2 is dfdy
dfda = Differentiation(f, a)
assert dfda.var is dfdy.var.x
g = Dot(Dot(x, C), VSF('sin', Add(Dot(B, VSF('sin', Add(Dot(A, y), a))),
b)))
g.cache()
assert g.y.var.x is f.y
dgdA = Differentiation(g, A)
dgdA.cache()
dgdB = Differentiation(g, B)
print unicode(dgdB)
assert dgdB.y.var is f.y.y
assert dgdA.x.var.x.x is dgdB.x.var
def test_VectorAndMatrixVariables(reset_NodeDict):
ran = lambda x: np.random.random(x) - 0.5
x = Var('x', ran((1, 4)))
y = Var('y', ran((4, 1)))
a = Var('a', ran((5, 1)))
b = Var('b', ran((6, 1)))
D, C, B, A = Var('D', ran((4, 4))), Var('C', ran(
(4, 6))), Var('B', ran((6, 5))), Var('A', ran((5, 4)))
assert (x.val.shape == (1, 4))
assert IsZero(Differentiation(Dot(x, y), D))
assert IsZero(Differentiation(Dot(x, y), a))
assert unicode(Differentiation(Dot(x, y), y)) == u'xᵀ'
assert unicode(Differentiation(Dot(x, y), x)) == u'yᵀ'
xDy = Dot(x, Dot(D, y))
assert unicode(xDy) == u'x⋅D⋅y'
assert unicode(Differentiation(xDy, y)) == u'[x⋅D]ᵀ'
assert unicode(Differentiation(xDy, x)) == u'[D⋅y]ᵀ'
xDy = Dot(Dot(x, D), y)
assert unicode(xDy) == u'x⋅D⋅y'
assert unicode(Differentiation(xDy, y)) == u'[x⋅D]ᵀ'
assert unicode(Differentiation(xDy, x)) == u'[D⋅y]ᵀ'
xCBAy = Dot(x, Dot(C, Dot(B, Dot(A, y))))
assert unicode(xCBAy) == u'x⋅C⋅B⋅A⋅y'
assert unicode(Differentiation(xCBAy, x)) == u'[C⋅B⋅A⋅y]ᵀ'
assert unicode(Differentiation(xCBAy, y)) == u'[x⋅C⋅B⋅A]ᵀ'
xCBAy = Dot(Dot(x, C), Dot(B, Dot(A, y)))
assert unicode(xCBAy) == u'x⋅C⋅B⋅A⋅y'
assert unicode(Differentiation(xCBAy, x)) == u'[C⋅B⋅A⋅y]ᵀ'
assert unicode(Differentiation(xCBAy, y)) == u'[x⋅C⋅B⋅A]ᵀ'
xfy = Dot(x, VSF('sin', y))
assert unicode(xfy) == u'x⋅sin(y)'
assert unicode(Differentiation(xfy, y)) == u'[x⨯[cos(y)]ᵀ]ᵀ'
f = Dot(Dot(x, C), Dot(B, VSF('sin', Add(Dot(A, y), a))))
assert unicode(f) == u'x⋅C⋅B⋅sin(A⋅y+a)'
assert unicode(Differentiation(f, y)) == u'[x⋅C⋅B⨯[cos(A⋅y+a)]ᵀ⋅A]ᵀ'
assert unicode(Differentiation(f, a)) == u'[x⋅C⋅B⨯[cos(A⋅y+a)]ᵀ]ᵀ'
g = Dot(Dot(x, C), VSF('sin', Add(Dot(B, VSF('sin', Add(Dot(A, y), a))),
b)))
assert unicode(g) == u'x⋅C⋅sin(B⋅sin(A⋅y+a)+b)'
assert unicode(Differentiation(g, x)) == u'[C⋅sin(B⋅sin(A⋅y+a)+b)]ᵀ'
assert unicode(Differentiation(g, b)) == u'[x⋅C⨯[cos(B⋅sin(A⋅y+a)+b)]ᵀ]ᵀ'
assert unicode(Differentiation(
g, a)) == u'[x⋅C⨯[cos(B⋅sin(A⋅y+a)+b)]ᵀ⋅B⨯[cos(A⋅y+a)]ᵀ]ᵀ'
assert unicode(Differentiation(
g, y)) == u'[x⋅C⨯[cos(B⋅sin(A⋅y+a)+b)]ᵀ⋅B⨯[cos(A⋅y+a)]ᵀ⋅A]ᵀ'
print unicode(Differentiation(g, A))
assert unicode(Differentiation(
g, A)) == u'[x⋅C⨯[cos(B⋅sin(A⋅y+a)+b)]ᵀ⋅B⨯[cos(A⋅y+a)]ᵀ]ᵀ⨯yᵀ'
assert unicode(Differentiation(
g, B)) == u'[x⋅C⨯[cos(B⋅sin(A⋅y+a)+b)]ᵀ]ᵀ⨯[sin(A⋅y+a)]ᵀ'
assert unicode(Differentiation(g, C)) == u'xᵀ⨯[sin(B⋅sin(A⋅y+a)+b)]ᵀ'
dgdx = Differentiation(g, x)
dgdb = Differentiation(g, b)
dgda = Differentiation(g, a)
dgdy = Differentiation(g, y)
delta = NormalizedMatrix(ran(x.val.shape), 0.01)
def test_Dot():
a = Var('A', ran((2, 3)))
b = Var('b', ran((3, )))
ab = Dot(a, b)
assert ab.val.shape == (2, )
assert ab.diff(b) == a