def _common_binary(self, shape, api, real, derive):
data = np.random.rand(*shape)
data2 = np.random.rand(*shape)
var = llo.variable(data, 'var')
var2 = llo.variable(data2, 'var2')
out = api(var, var2)
both = api(var, var)
fout = llo.evaluate(out, dtype=np.dtype(float))
fboth = llo.evaluate(both, dtype=np.dtype(float))
self._array_eq(real(data, data2), fout)
self._array_eq(real(data, data), fboth)
var3 = llo.variable(data, 'var3')
zero = llo.derive(out, var3)
ex = llo.derive(out, var)
ex2 = llo.derive(out, var2)
ex3 = llo.derive(both, var)
rej = llo.evaluate(zero)
der = llo.evaluate(ex)
der2 = llo.evaluate(ex2)
der3 = llo.evaluate(ex3)
data0 = np.zeros(shape, dtype=float)
exdata = derive(0, (data, data2))
exdata2 = derive(1, (data, data2))
exdata3 = derive(0, (data, data)) + derive(1, (data, data))
self._array_eq(data0, rej)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
self._array_close(exdata3, der3)
def _common_binary(self, shape, api, real, derive):
data = np.random.rand(*shape)
data2 = np.random.rand(*shape)
var = llo.variable(data, 'var')
var2 = llo.variable(data2, 'var2')
out = api(var, var2)
both = api(var, var)
fout = out.evaluate(dtype=np.dtype(float))
fboth = both.evaluate(dtype=np.dtype(float))
self._array_eq(real(data, data2), fout)
self._array_eq(real(data, data), fboth)
var3 = llo.variable(data, 'var3')
zero = llo.derive(out, var3)
ex = llo.derive(out, var)
ex2 = llo.derive(out, var2)
ex3 = llo.derive(both, var)
rej = zero.evaluate()
der = ex.evaluate()
der2 = ex2.evaluate()
der3 = ex3.evaluate()
data0 = np.zeros(shape, dtype=float)
exdata = derive(0, (data, data2))
exdata2 = derive(1, (data, data2))
exdata3 = derive(0, (data, data)) + derive(1, (data, data))
self._array_eq(data0, rej)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
self._array_close(exdata3, der3)
def test_flip(self):
shape = [3, 4, 5]
data = np.random.rand(*shape)
var = llo.variable(data, 'var')
tf_var = tf.Variable(data)
sess = tf.Session()
sess.run(tf_var.initializer)
out = age.flip(var, 1)
tf_out = tf.reverse(tf_var, [1])
fout = llo.evaluate(out, dtype=np.dtype(float))
tf_fout = sess.run(tf_out)
self._array_close(tf_fout, fout)
var2 = llo.variable(data, 'var2')
zero = llo.derive(out, var2)
ex = llo.derive(out, var)
rej = llo.evaluate(zero)
der = llo.evaluate(ex)
tf_grad = tf.gradients(tf_fout, [tf_var])[0]
self.assertEqual(None, tf_grad)
data0 = np.zeros(shape, dtype=float)
data1 = np.ones(shape, dtype=float)
self._array_eq(data0, rej)
self._array_eq(data1, der)
def test_flip(self):
shape = [3, 4, 5]
data = np.random.rand(*shape)
var = llo.variable(data, 'var')
tf_var = tf.Variable(data)
sess = tf.Session()
sess.run(tf_var.initializer)
out = age.flip(var, 1)
tf_out = tf.reverse(tf_var, [1])
fout = out.evaluate(dtype=np.dtype(float))
tf_fout = sess.run(tf_out)
self._array_close(tf_fout, fout)
var2 = llo.variable(data, 'var2')
zero = llo.derive(out, var2)
ex = llo.derive(out, var)
rej = zero.evaluate()
der = ex.evaluate()
tf_grad = tf.gradients(tf_fout, [tf_var])[0]
self.assertEqual(None, tf_grad)
data0 = np.zeros(shape, dtype=float)
data1 = np.ones(shape, dtype=float)
self._array_eq(data0, rej)
self._array_eq(data1, der)
def _common_reduce_1d(self, dim_reduce, tf_reduce):
shape = [3, 4, 5]
data = np.random.rand(*shape)
var = llo.variable(data, 'var')
tf_var = tf.Variable(data)
sess = tf.Session()
sess.run(tf_var.initializer)
out = dim_reduce(var, 1)
tf_out = tf_reduce(tf_var, [1])
fout = out.evaluate(dtype=np.dtype(float))
tf_fout = sess.run(tf_out)
self._array_close(tf_fout, fout)
var2 = llo.variable(data, 'var2')
ex = llo.derive(out, var)
zero = llo.derive(out, var2)
tf_grad = tf.gradients(tf_out, [tf_var])[0]
data0 = np.zeros(shape, dtype=float)
der = ex.evaluate()
rej = zero.evaluate()
exdata = sess.run(tf_grad)
self._array_close(exdata, der)
self._array_eq(data0, rej)
def test_convolution(self):
padding = "VALID"
batchsize = 2
inchannel = 3
outchannel = 4
dims = 5
shapes = [
([1, 3, 3, 1], [3, 3, 1, 1]),
([batchsize, dims, dims, inchannel], [3, 3, inchannel,
outchannel]),
]
for shape, kernelshape in shapes:
data = np.random.rand(*shape).astype(np.float32)
kernel = np.random.rand(*kernelshape).astype(np.float32)
var = llo.variable(data, 'var')
vkernel = llo.variable(kernel, 'vkernel')
tf_var = tf.Variable(data)
tf_kernel = tf.Variable(kernel)
sess = tf.Session()
sess.run(tf_var.initializer)
sess.run(tf_kernel.initializer)
out = age.convolution(var, vkernel)
tf_out = tf.nn.convolution(tf_var, tf_kernel, padding)
fout = llo.evaluate(out, dtype=np.dtype(float))
tf_fout = sess.run(tf_out)
self._array_close(tf_fout, fout)
var2 = llo.variable(data, 'var2')
zero = llo.derive(out, var2)
ex = llo.derive(out, var)
ex2 = llo.derive(out, vkernel)
rej = llo.evaluate(zero)
der = llo.evaluate(ex)
der2 = llo.evaluate(ex2)
data0 = np.zeros(shape, dtype=float)
tf_grad, tf_grad2 = tf.gradients(tf_out, [tf_var, tf_kernel])
exdata = sess.run(tf_grad)
exdata2 = sess.run(tf_grad2)
self._array_eq(data0, rej)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
def test_matmul(self):
shape = [5, 5]
data = np.random.rand(*shape)
data2 = np.random.rand(*shape)
var = llo.variable(data, 'var')
var2 = llo.variable(data2, 'var2')
tf_var = tf.Variable(data)
tf_var2 = tf.Variable(data2)
sess = tf.Session()
sess.run(tf_var.initializer)
sess.run(tf_var2.initializer)
out = age.matmul(var, var2)
both = age.matmul(var, var)
tf_out = tf.matmul(tf_var, tf_var2)
tf_both = tf.matmul(tf_var, tf_var)
fout = llo.evaluate(out, dtype=np.dtype(float))
fboth = llo.evaluate(both, dtype=np.dtype(float))
tf_fout = sess.run(tf_out)
tf_fboth = sess.run(tf_both)
self._array_close(tf_fout, fout)
self._array_close(tf_fboth, fboth)
var3 = llo.variable(data, 'var3')
zero = llo.derive(out, var3)
ex = llo.derive(out, var)
ex2 = llo.derive(out, var2)
ex3 = llo.derive(both, var)
rej = llo.evaluate(zero)
der = llo.evaluate(ex)
der2 = llo.evaluate(ex2)
der3 = llo.evaluate(ex3)
data0 = np.zeros(shape, dtype=float)
tf_grad, tf_grad2 = tf.gradients(tf_out, [tf_var, tf_var2])
tf_grad3 = tf.gradients(tf_both, [tf_var])[0]
exdata = sess.run(tf_grad)
exdata2 = sess.run(tf_grad2)
exdata3 = sess.run(tf_grad3)
self._array_eq(data0, rej)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
self._array_close(exdata3, der3)
def test_convolution(self):
padding = "VALID"
batchsize = 2
inchannel = 3
outchannel = 4
dims = 5
shapes = [
([1, 3, 3, 1], [3, 3, 1, 1]),
([batchsize, dims, dims, inchannel], [3, 3, inchannel, outchannel]),
]
for shape, kernelshape in shapes:
data = np.random.rand(*shape).astype(np.float32)
kernel = np.random.rand(*kernelshape).astype(np.float32)
var = llo.variable(data, 'var')
vkernel = llo.variable(kernel, 'vkernel')
tf_var = tf.Variable(data)
tf_kernel = tf.Variable(kernel)
sess = tf.Session()
sess.run(tf_var.initializer)
sess.run(tf_kernel.initializer)
out = age.convolution(var, vkernel)
tf_out = tf.nn.convolution(tf_var, tf_kernel, padding)
fout = out.evaluate(dtype=np.dtype(float))
tf_fout = sess.run(tf_out)
self._array_close(tf_fout, fout)
var2 = llo.variable(data, 'var2')
zero = llo.derive(out, var2)
ex = llo.derive(out, var)
ex2 = llo.derive(out, vkernel)
rej = zero.evaluate()
der = ex.evaluate()
der2 = ex2.evaluate()
data0 = np.zeros(shape, dtype=float)
tf_grad, tf_grad2 = tf.gradients(tf_out, [tf_var, tf_kernel])
exdata = sess.run(tf_grad)
exdata2 = sess.run(tf_grad2)
self._array_eq(data0, rej)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
def _common_unary(self, shape, api, real, derive):
data = np.random.rand(*shape) * 234
var = llo.variable(data, 'var')
out = api(var)
fout = llo.evaluate(out, dtype=np.dtype(float))
self._array_eq(real(data), fout)
var2 = llo.variable(data, 'var2')
ex = llo.derive(out, var)
zero = llo.derive(out, var2)
data0 = np.zeros(shape, dtype=float)
der = llo.evaluate(ex)
rej = llo.evaluate(zero)
exdata = derive(data)
self._array_close(exdata, der)
self._array_eq(data0, rej)
def _common_unary(self, shape, api, real, derive):
data = np.random.rand(*shape) * 234
var = llo.variable(data, 'var')
out = api(var)
fout = out.evaluate(dtype=np.dtype(float))
self._array_close(real(data), fout)
var2 = llo.variable(data, 'var2')
ex = llo.derive(out, var)
zero = llo.derive(out, var2)
data0 = np.zeros(shape, dtype=float)
der = ex.evaluate()
rej = zero.evaluate()
exdata = derive(data)
self._array_close(exdata, der)
self._array_eq(data0, rej)
def _common_reduce(self, all_reduce, dim_reduce, tf_reduce):
shape = [3, 4, 5]
data = np.random.rand(*shape)
var = llo.variable(data, 'var')
tf_var = tf.Variable(data)
sess = tf.Session()
sess.run(tf_var.initializer)
out = all_reduce(var)
out2 = dim_reduce(var, 1)
tf_out = tf_reduce(tf_var)
tf_out2 = tf_reduce(tf_var, [0, 1])
fout = llo.evaluate(out, dtype=np.dtype(float))
fout2 = llo.evaluate(out2, dtype=np.dtype(float))
tf_fout = np.array(sess.run(tf_out))
tf_fout2 = sess.run(tf_out2)
self._array_close(tf_fout, fout)
self._array_close(tf_fout2, fout2)
var2 = llo.variable(data, 'var2')
ex = llo.derive(out, var)
ex2 = llo.derive(out2, var)
zero = llo.derive(out, var2)
tf_grad = tf.gradients(tf_out, [tf_var])[0]
tf_grad2 = tf.gradients(tf_out2, [tf_var])[0]
data0 = np.zeros(shape, dtype=float)
der = llo.evaluate(ex)
der2 = llo.evaluate(ex2)
rej = llo.evaluate(zero)
exdata = sess.run(tf_grad)
exdata2 = sess.run(tf_grad2)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
self._array_eq(data0, rej)
def test_extend(self):
shape = [2]
data = np.random.rand(*shape) * 13
expected_out = np.array(list(data) * 3).reshape([3, 2])
var = llo.variable(data, 'var')
out = age.extend(var, 1, [3])
fout = llo.evaluate(out)
self._array_eq(expected_out, fout)
ex = llo.derive(out, var)
der = llo.evaluate(ex)
self._array_eq(np.array([3, 3]), der)
def test_extend(self):
shape = [2]
data = np.random.rand(*shape) * 13
expected_out = np.array(list(data) * 3).reshape([3, 2])
var = llo.variable(data, 'var')
out = age.extend(var, 1, [3])
fout = out.evaluate()
self._array_eq(expected_out, fout)
ex = llo.derive(out, var)
der = ex.evaluate()
self._array_eq(np.array([3, 3]), der)
def test_variable(self):
shape = [3, 4, 5]
data1 = np.ones(shape, dtype=float)
data0 = np.zeros(shape, dtype=float)
data = np.random.rand(3, 4, 5) * 234
var = llo.variable(data, 'var')
fout = llo.evaluate(var, dtype=np.dtype(float))
iout = llo.evaluate(var, dtype=np.dtype(int))
self.assertEqual(tuple(shape), fout.shape)
self.assertEqual(tuple(shape), iout.shape)
self._array_eq(data, fout)
self._array_eq(data.astype(int), iout)
var2 = llo.variable(data, 'var2')
one = llo.derive(var, var)
zero = llo.derive(var, var2)
out1 = llo.evaluate(one)
out0 = llo.evaluate(zero)
self.assertEqual(tuple(shape), out1.shape)
self.assertEqual(tuple(shape), out0.shape)
self._array_eq(data1, out1)
self._array_eq(data0, out0)
def test_variable(self):
shape = [3, 4, 5]
data1 = np.ones(shape, dtype=float)
data0 = np.zeros(shape, dtype=float)
data = np.random.rand(3, 4, 5) * 234
var = llo.variable(data, 'var')
fout = var.evaluate(dtype=np.dtype(float))
iout = var.evaluate(dtype=np.dtype(int))
self.assertEqual(tuple(shape), fout.shape)
self.assertEqual(tuple(shape), iout.shape)
self._array_eq(data, fout)
self._array_eq(data.astype(int), iout)
var2 = llo.variable(data, 'var2')
one = llo.derive(var, var)
zero = llo.derive(var, var2)
out1 = one.evaluate()
out0 = zero.evaluate()
self.assertEqual(tuple(shape), out1.shape)
self.assertEqual(tuple(shape), out0.shape)
self._array_eq(data1, out1)
self._array_eq(data0, out0)
def test_matmul(self):
shape = [5, 5]
data = np.random.rand(*shape)
data2 = np.random.rand(*shape)
var = llo.variable(data, 'var')
var2 = llo.variable(data2, 'var2')
tf_var = tf.Variable(data)
tf_var2 = tf.Variable(data2)
sess = tf.Session()
sess.run(tf_var.initializer)
sess.run(tf_var2.initializer)
# regular matmul
out = age.matmul(var, var2)
both = age.matmul(var, var)
# fast matmul
out_fast = age.fast_matmul(var, var2)
both_fast = age.fast_matmul(var, var)
# tensorflow matmul
tf_out = tf.matmul(tf_var, tf_var2)
tf_both = tf.matmul(tf_var, tf_var)
# evaluate regular matmul
fout = out.evaluate(dtype=np.dtype(float))
fboth = both.evaluate(dtype=np.dtype(float))
# evaluate fast matmul
fout_fast = out_fast.evaluate(dtype=np.dtype(float))
fboth_fast = both_fast.evaluate(dtype=np.dtype(float))
# evaluate tensorflow matmul
tf_fout = sess.run(tf_out)
tf_fboth = sess.run(tf_both)
# check regular matmul
self._array_close(tf_fout, fout)
self._array_close(tf_fboth, fboth)
# check fast matmul
self._array_close(tf_fout, fout_fast)
self._array_close(tf_fboth, fboth_fast)
var3 = llo.variable(data, 'var3')
zero = llo.derive(out, var3)
ex = llo.derive(out, var)
ex2 = llo.derive(out, var2)
ex3 = llo.derive(both, var)
zero_fast = llo.derive(out_fast, var3)
ex_fast = llo.derive(out_fast, var)
ex_fast2 = llo.derive(out_fast, var2)
ex_fast3 = llo.derive(both_fast, var)
# eval derivative of regular matmul
rej = zero.evaluate()
der = ex.evaluate()
der2 = ex2.evaluate()
der3 = ex3.evaluate()
# eval derivative of fast matmul
rej_fast = zero_fast.evaluate()
fast_der = ex_fast.evaluate()
fast_der2 = ex_fast2.evaluate()
fast_der3 = ex_fast3.evaluate()
# eval derivative of tensorflow matmul
data0 = np.zeros(shape, dtype=float)
tf_grad, tf_grad2 = tf.gradients(tf_out, [tf_var, tf_var2])
tf_grad3 = tf.gradients(tf_both, [tf_var])[0]
exdata = sess.run(tf_grad)
exdata2 = sess.run(tf_grad2)
exdata3 = sess.run(tf_grad3)
# check regular matmul
self._array_eq(data0, rej)
self._array_close(exdata, der)
self._array_close(exdata2, der2)
self._array_close(exdata3, der3)
# check fast matmul
self._array_eq(data0, rej_fast)
self._array_close(exdata, fast_der)
self._array_close(exdata2, fast_der2)
self._array_close(exdata3, fast_der3)