def test_qr_modes():
rng = np.random.RandomState(utt.fetch_seed())
A = tensor.matrix("A", dtype=theano.config.floatX)
a = rng.rand(4, 4).astype(theano.config.floatX)
f = function([A], qr(A))
t_qr = f(a)
n_qr = np.linalg.qr(a)
assert _allclose(n_qr, t_qr)
for mode in ["reduced", "r", "raw"]:
f = function([A], qr(A, mode))
t_qr = f(a)
n_qr = np.linalg.qr(a, mode)
if isinstance(n_qr, (list, tuple)):
assert _allclose(n_qr[0], t_qr[0])
assert _allclose(n_qr[1], t_qr[1])
else:
assert _allclose(n_qr, t_qr)
try:
n_qr = np.linalg.qr(a, "complete")
f = function([A], qr(A, "complete"))
t_qr = f(a)
assert _allclose(n_qr, t_qr)
except TypeError as e:
assert "name 'complete' is not defined" in str(e)
def test_qr_modes():
rng = np.random.RandomState(utt.fetch_seed())
A = tensor.matrix("A", dtype=theano.config.floatX)
a = rng.rand(4, 4).astype(theano.config.floatX)
f = function([A], qr(A))
t_qr = f(a)
n_qr = np.linalg.qr(a)
assert _allclose(n_qr, t_qr)
for mode in ["reduced", "r", "raw"]:
f = function([A], qr(A, mode))
t_qr = f(a)
n_qr = np.linalg.qr(a, mode)
if isinstance(n_qr, (list, tuple)):
assert _allclose(n_qr[0], t_qr[0])
assert _allclose(n_qr[1], t_qr[1])
else:
assert _allclose(n_qr, t_qr)
try:
n_qr = np.linalg.qr(a, "complete")
f = function([A], qr(A, "complete"))
t_qr = f(a)
assert _allclose(n_qr, t_qr)
except TypeError as e:
assert "name 'complete' is not defined" in str(e)
def test_rop_lop():
mx = tensor.matrix('mx')
mv = tensor.matrix('mv')
v = tensor.vector('v')
y = matrix_inverse(mx).sum(axis=0)
yv = tensor.Rop(y, mx, mv)
yv2 = tensor.Rop_via_Lop(y, mx, mv)
rop_f = function([mx, mv], [yv, yv2])
sy, _ = theano.scan(lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(),
sequences=tensor.arange(y.shape[0]),
non_sequences=[y, mx, mv])
scan_f = function([mx, mv], sy)
rng = np.random.RandomState(utt.fetch_seed())
vx = np.asarray(rng.randn(4, 4), theano.config.floatX)
vv = np.asarray(rng.randn(4, 4), theano.config.floatX)
v1 = scan_f(vx, vv)
v2, v3 = rop_f(vx, vv)
assert _allclose(v2, v1), ('Rop mismatch: %s %s' % (v2, v1))
assert _allclose(v3, v1), ('Rop_via_Lop mismatch: %s %s' % (v3, v1))
raised = False
try:
tensor.Rop(theano.clone(y, replace={mx: break_op(mx)}), mx, mv)
except ValueError:
raised = True
if not raised:
raise Exception(('Op did not raised an error even though the function'
' is not differentiable'))
try:
tensor.Rop_via_Lop(theano.clone(y, replace={mx: break_op(mx)}), mx, mv)
except theano.gradient.NullTypeGradError:
raised = True
except theano.gradient.DisconnectedInputError:
raised = True
if not raised:
raise Exception((
'Rop_via_Lop for Op did not raise an error even though the function'
' is not differentiable'))
vv = np.asarray(rng.uniform(size=(4, )), theano.config.floatX)
yv = tensor.Lop(y, mx, v)
lop_f = function([mx, v], yv)
sy = tensor.grad((v * y).sum(), mx)
scan_f = function([mx, v], sy)
v1 = lop_f(vx, vv)
v2 = scan_f(vx, vv)
assert _allclose(v1, v2), ('LOP mismatch: %s %s' % (v1, v2))
def test_svd():
rng = numpy.random.RandomState(utt.fetch_seed())
A = tensor.matrix("A", dtype=theano.config.floatX)
U, V, T = svd(A)
fn = function([A], [U, V, T])
a = rng.rand(4, 4).astype(theano.config.floatX)
n_u, n_v, n_t = numpy.linalg.svd(a)
t_u, t_v, t_t = fn(a)
assert _allclose(n_u, t_u)
assert _allclose(n_v, t_v)
assert _allclose(n_t, t_t)
def test_svd():
rng = np.random.RandomState(utt.fetch_seed())
A = tensor.matrix("A", dtype=theano.config.floatX)
U, V, T = svd(A)
fn = function([A], [U, V, T])
a = rng.rand(4, 4).astype(theano.config.floatX)
n_u, n_v, n_t = np.linalg.svd(a)
t_u, t_v, t_t = fn(a)
assert _allclose(n_u, t_u)
assert _allclose(n_v, t_v)
assert _allclose(n_t, t_t)
def test_svd(self):
A = tensor.matrix("A", dtype=self.dtype)
U, S, VT = svd(A)
fn = function([A], [U, S, VT])
a = self.rng.rand(4, 4).astype(self.dtype)
n_u, n_s, n_vt = np.linalg.svd(a)
t_u, t_s, t_vt = fn(a)
assert _allclose(n_u, t_u)
assert _allclose(n_s, t_s)
assert _allclose(n_vt, t_vt)
fn = function([A], svd(A, compute_uv=False))
t_s = fn(a)
assert _allclose(n_s, t_s)
def test_string_var(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = "raise"
x = T.matrix("x")
x.tag.test_value = np.random.rand(3, 4).astype(config.floatX)
y = T.matrix("y")
y.tag.test_value = np.random.rand(4, 5).astype(config.floatX)
z = theano.shared(np.random.rand(5, 6).astype(config.floatX))
# should work
out = T.dot(T.dot(x, y), z)
assert hasattr(out.tag, "test_value")
tf = theano.function([x, y], out)
assert _allclose(tf(x.tag.test_value, y.tag.test_value),
out.tag.test_value)
def f(x, y, z):
return T.dot(T.dot(x, y), z)
# this test should fail
z.set_value(np.random.rand(7, 6).astype(config.floatX))
with pytest.raises(ValueError):
f(x, y, z)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_inverse_correctness(self):
r = self.rng.randn(4, 4).astype(theano.config.floatX)
x = tensor.matrix()
xi = self.op(x)
ri = function([x], xi)(r)
assert ri.shape == r.shape
assert ri.dtype == r.dtype
rir = np.dot(ri, r)
rri = np.dot(r, ri)
assert _allclose(np.identity(4), rir), rir
assert _allclose(np.identity(4), rri), rri
def test_sparse_sparse(self):
for d1, d2 in [
("float32", "float32"),
("float32", "float64"),
("float64", "float32"),
("float64", "float64"),
("float32", "int16"),
("float32", "complex64"),
]:
for x_f, y_f in [("csc", "csc"), ("csc", "csr"), ("csr", "csc"), ("csr", "csr")]:
x = theano.sparse.SparseType(format=x_f, dtype=d1)("x")
y = theano.sparse.SparseType(format=x_f, dtype=d2)("x")
f_a = theano.function([x, y], theano.sparse.dot(x, y))
f_b = lambda x, y: x * y
vx = getattr(self, "x_" + x_f).astype(d1)
vy = getattr(self, "y_" + y_f).astype(d2)
assert _allclose(f_a(vx, vy), f_b(vx, vy).toarray())
# Test infer_shape
f_a = theano.function([x, y], theano.sparse.dot(x, y).shape)
f_b = lambda x, y: (x * y).shape
assert numpy.all(f_a(vx, vy) == f_b(vx, vy))
topo = f_a.maker.env.toposort()
if theano.config.mode != "FAST_COMPILE":
nb = 0
else:
nb = 1
assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo]) == nb
def test_string_var(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.matrix('x')
x.tag.test_value = numpy.random.rand(3,4).astype(config.floatX)
y = T.matrix('y')
y.tag.test_value = numpy.random.rand(4,5).astype(config.floatX)
z = theano.shared(numpy.random.rand(5,6).astype(config.floatX))
# should work
out = T.dot(T.dot(x,y), z)
assert hasattr(out.tag, 'test_value')
tf = theano.function([x,y], out)
assert _allclose(
tf(x.tag.test_value, y.tag.test_value),
out.tag.test_value)
def f(x,y,z):
return T.dot(T.dot(x,y),z)
# this test should fail
z.set_value(numpy.random.rand(7,6).astype(config.floatX))
self.assertRaises(ValueError, f, x, y, z)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_upcast(self):
typenames = ("float32", "int64", "int8", "int32", "int16", "float64", "complex64", "complex128")
for dense_dtype in typenames:
for sparse_dtype in typenames:
correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype)
a = SparseType("csc", dtype=sparse_dtype)()
b = tensor.matrix(dtype=dense_dtype)
d = structured_dot(a, b)
assert d.type.dtype == correct_dtype
# compile and run a function
f = theano.function([a, b], d)
M, N, K, nnz = (4, 3, 5, 3)
spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz))
# the following madness is necessary to workaround
# an intc vs. int32 bug.
# The lil makes an intc on my computer when sparse_dtype
# is int32.
spmat.dtype = numpy.dtype(sparse_dtype)
mat = numpy.asarray(numpy.random.randn(N, K) * 9, dtype=dense_dtype)
print "DTYPES", sparse_dtype, dense_dtype
print "sym types", a.type, b.type
print "dtype strings", spmat.dtype, mat.dtype
print "numpy dtype num", mat.dtype.num
print "scipy dtype num", spmat.data.dtype.num
theano_result = f(spmat, mat)
scipy_result = spmat * mat
assert theano_result.shape == scipy_result.shape
assert theano_result.dtype == scipy_result.dtype
assert _allclose(theano_result, scipy_result)
def test_string_var(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.matrix('x')
x.tag.test_value = numpy.random.rand(3, 4).astype(config.floatX)
y = T.matrix('y')
y.tag.test_value = numpy.random.rand(4, 5).astype(config.floatX)
z = theano.shared(numpy.random.rand(5, 6).astype(config.floatX))
# should work
out = T.dot(T.dot(x, y), z)
assert hasattr(out.tag, 'test_value')
tf = theano.function([x, y], out)
assert _allclose(tf(x.tag.test_value, y.tag.test_value),
out.tag.test_value)
def f(x, y, z):
return T.dot(T.dot(x, y), z)
# this test should fail
z.set_value(numpy.random.rand(7, 6).astype(config.floatX))
self.assertRaises(ValueError, f, x, y, z)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_inverse_correctness(self):
r = self.rng.randn(4, 4).astype(theano.config.floatX)
x = tensor.matrix()
xi = self.op(x)
ri = function([x], xi)(r)
assert ri.shape == r.shape
assert ri.dtype == r.dtype
rir = np.dot(ri, r)
rri = np.dot(r, ri)
assert _allclose(np.identity(4), rir), rir
assert _allclose(np.identity(4), rri), rri
def test_sparse_sparse(self):
for d1, d2 in [('float32', 'float32'),
('float32', 'float64'),
('float64', 'float32'),
('float64', 'float64'),
('float32', 'int16'),
('float32', 'complex64'),
]:
for x_f, y_f in [('csc', 'csc'),
('csc', 'csr'),
('csr', 'csc'),
('csr', 'csr'),
]:
x = theano.sparse.SparseType(format=x_f, dtype=d1)('x')
y = theano.sparse.SparseType(format=x_f, dtype=d2)('x')
f_a = theano.function([x, y], theano.sparse.dot(x, y))
f_b = lambda x, y: x * y
vx = getattr(self, 'x_' + x_f).astype(d1)
vy = getattr(self, 'y_' + y_f).astype(d2)
assert _allclose(f_a(vx, vy), f_b(vx, vy).toarray())
# Test infer_shape
f_a = theano.function([x, y], theano.sparse.dot(x, y).shape)
f_b = lambda x, y: (x * y).shape
assert numpy.all(f_a(vx, vy) == f_b(vx, vy))
topo = f_a.maker.env.toposort()
if theano.config.mode != 'FAST_COMPILE':
nb = 0
else:
nb = 1
assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense))
for node in topo]) == nb
def test_upcast(self):
typenames = ('float32', 'int64', 'int8', 'int32', 'int16', 'float64',
'complex64', 'complex128')
for dense_dtype in typenames:
for sparse_dtype in typenames:
correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype)
a = SparseType('csc', dtype=sparse_dtype)()
b = tensor.matrix(dtype=dense_dtype)
d = structured_dot(a, b)
assert d.type.dtype == correct_dtype
# compile and run a function
f = theano.function([a, b], d)
M, N, K, nnz = (4, 3, 5, 3)
spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz))
# the following madness is necessary to workaround
# an intc vs. int32 bug.
# The lil makes an intc on my computer when sparse_dtype
# is int32.
spmat.dtype = numpy.dtype(sparse_dtype)
mat = numpy.asarray(numpy.random.randn(N, K) * 9,
dtype=dense_dtype)
print 'DTYPES', sparse_dtype, dense_dtype
print 'sym types', a.type, b.type
print 'dtype strings', spmat.dtype, mat.dtype
print 'numpy dtype num', mat.dtype.num
print 'scipy dtype num', spmat.data.dtype.num
theano_result = f(spmat, mat)
scipy_result = spmat * mat
assert theano_result.shape == scipy_result.shape
assert theano_result.dtype == scipy_result.dtype
assert _allclose(theano_result, scipy_result)
def test_inverse_correctness():
rng = numpy.random.RandomState(utt.fetch_seed())
r = rng.randn(4, 4).astype(theano.config.floatX)
x = tensor.matrix()
xi = matrix_inverse(x)
ri = function([x], xi)(r)
assert ri.shape == r.shape
assert ri.dtype == r.dtype
rir = numpy.dot(ri, r)
rri = numpy.dot(r, ri)
assert _allclose(numpy.identity(4), rir), rir
assert _allclose(numpy.identity(4), rri), rri
def test_empty_elemwise(self):
x = theano.shared(np.random.rand(0, 6).astype(config.floatX), "x")
# should work
z = (x + 2) * 3
assert hasattr(z.tag, "test_value")
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
def test_rop_lop():
mx = tensor.matrix('mx')
mv = tensor.matrix('mv')
v = tensor.vector('v')
y = matrix_inverse(mx).sum(axis=0)
yv = tensor.Rop(y, mx, mv)
rop_f = function([mx, mv], yv)
sy, _ = theano.scan(lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(),
sequences=tensor.arange(y.shape[0]),
non_sequences=[y, mx, mv])
scan_f = function([mx, mv], sy)
rng = numpy.random.RandomState(utt.fetch_seed())
vx = numpy.asarray(rng.randn(4, 4), theano.config.floatX)
vv = numpy.asarray(rng.randn(4, 4), theano.config.floatX)
v1 = rop_f(vx, vv)
v2 = scan_f(vx, vv)
assert _allclose(v1, v2), ('ROP mismatch: %s %s' % (v1, v2))
raised = False
try:
tensor.Rop(
theano.clone(y, replace={mx: break_op(mx)}),
mx,
mv)
except ValueError:
raised = True
if not raised:
raise Exception((
'Op did not raised an error even though the function'
' is not differentiable'))
vv = numpy.asarray(rng.uniform(size=(4,)), theano.config.floatX)
yv = tensor.Lop(y, mx, v)
lop_f = function([mx, v], yv)
sy = tensor.grad((v * y).sum(), mx)
scan_f = function([mx, v], sy)
v1 = lop_f(vx, vv)
v2 = scan_f(vx, vv)
assert _allclose(v1, v2), ('LOP mismatch: %s %s' % (v1, v2))
def test_inverse_correctness():
rng = numpy.random.RandomState(utt.fetch_seed())
r = rng.randn(4, 4).astype(theano.config.floatX)
x = tensor.matrix()
xi = matrix_inverse(x)
ri = function([x], xi)(r)
assert ri.shape == r.shape
assert ri.dtype == r.dtype
rir = numpy.dot(ri, r)
rri = numpy.dot(r, ri)
assert _allclose(numpy.identity(4), rir), rir
assert _allclose(numpy.identity(4), rri), rri
def validate(self, image_shape, filter_shape, verify_grad=True):
image_dim = len(image_shape)
filter_dim = len(filter_shape)
input = T.TensorType("float64", [False] * image_dim)()
filters = T.TensorType("float64", [False] * filter_dim)()
bsize = image_shape[0]
if image_dim != 3:
bsize = 1
nkern = filter_shape[0]
if filter_dim != 3:
nkern = 1
############# THEANO IMPLEMENTATION ############
# we create a symbolic function so that verify_grad can work
def sym_conv2d(input, filters):
return conv.conv2d(input, filters)
output = sym_conv2d(input, filters)
theano_conv = theano.function([input, filters], output)
# initialize input and compute result
image_data = numpy.random.random(image_shape)
filter_data = numpy.random.random(filter_shape)
theano_output = theano_conv(image_data, filter_data)
############# REFERENCE IMPLEMENTATION ############
out_shape2d = numpy.array(image_shape[-2:]) - numpy.array(filter_shape[-2:]) + 1
ref_output = numpy.zeros(tuple(out_shape2d))
# reshape as 3D input tensors to make life easier
image_data3d = image_data.reshape((bsize,) + image_shape[-2:])
filter_data3d = filter_data.reshape((nkern,) + filter_shape[-2:])
# reshape theano output as 4D to make life easier
theano_output4d = theano_output.reshape((bsize, nkern) + theano_output.shape[-2:])
# loop over mini-batches (if required)
for b in range(bsize):
# loop over filters (if required)
for k in range(nkern):
image2d = image_data3d[b, :, :]
filter2d = filter_data3d[k, :, :]
output2d = numpy.zeros(ref_output.shape)
for row in range(ref_output.shape[0]):
for col in range(ref_output.shape[1]):
output2d[row, col] += (
image2d[row : row + filter2d.shape[0], col : col + filter2d.shape[1]] * filter2d[::-1, ::-1]
).sum()
self.assertTrue(_allclose(theano_output4d[b, k, :, :], output2d))
############# TEST GRADIENT ############
if verify_grad:
utt.verify_grad(sym_conv2d, [image_data, filter_data])
def test_rop_lop():
mx = tensor.matrix("mx")
mv = tensor.matrix("mv")
v = tensor.vector("v")
y = matrix_inverse(mx).sum(axis=0)
yv = tensor.Rop(y, mx, mv)
rop_f = function([mx, mv], yv)
sy, _ = theano.scan(
lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(),
sequences=tensor.arange(y.shape[0]),
non_sequences=[y, mx, mv],
)
scan_f = function([mx, mv], sy)
rng = np.random.RandomState(utt.fetch_seed())
vx = np.asarray(rng.randn(4, 4), theano.config.floatX)
vv = np.asarray(rng.randn(4, 4), theano.config.floatX)
v1 = rop_f(vx, vv)
v2 = scan_f(vx, vv)
assert _allclose(v1, v2), "ROP mismatch: %s %s" % (v1, v2)
raised = False
try:
tensor.Rop(theano.clone(y, replace={mx: break_op(mx)}), mx, mv)
except ValueError:
raised = True
if not raised:
raise Exception(("Op did not raised an error even though the function"
" is not differentiable"))
vv = np.asarray(rng.uniform(size=(4, )), theano.config.floatX)
yv = tensor.Lop(y, mx, v)
lop_f = function([mx, v], yv)
sy = tensor.grad((v * y).sum(), mx)
scan_f = function([mx, v], sy)
v1 = lop_f(vx, vv)
v2 = scan_f(vx, vv)
assert _allclose(v1, v2), "LOP mismatch: %s %s" % (v1, v2)
def test_tensorsolve():
rng = np.random.RandomState(utt.fetch_seed())
A = tensor.tensor4("A", dtype=theano.config.floatX)
B = tensor.matrix("B", dtype=theano.config.floatX)
X = tensorsolve(A, B)
fn = function([A, B], [X])
# slightly modified example from np.linalg.tensorsolve docstring
a = np.eye(2 * 3 * 4).astype(theano.config.floatX)
a.shape = (2 * 3, 4, 2, 3 * 4)
b = rng.rand(2 * 3, 4).astype(theano.config.floatX)
n_x = np.linalg.tensorsolve(a, b)
t_x = fn(a, b)
assert _allclose(n_x, t_x)
# check the type upcast now
C = tensor.tensor4("C", dtype='float32')
D = tensor.matrix("D", dtype='float64')
Y = tensorsolve(C, D)
fn = function([C, D], [Y])
c = np.eye(2 * 3 * 4, dtype='float32')
c.shape = (2 * 3, 4, 2, 3 * 4)
d = rng.rand(2 * 3, 4).astype('float64')
n_y = np.linalg.tensorsolve(c, d)
t_y = fn(c, d)
assert _allclose(n_y, t_y)
assert n_y.dtype == Y.dtype
# check the type upcast now
E = tensor.tensor4("E", dtype='int32')
F = tensor.matrix("F", dtype='float64')
Z = tensorsolve(E, F)
fn = function([E, F], [Z])
e = np.eye(2 * 3 * 4, dtype='int32')
e.shape = (2 * 3, 4, 2, 3 * 4)
f = rng.rand(2 * 3, 4).astype('float64')
n_z = np.linalg.tensorsolve(e, f)
t_z = fn(e, f)
assert _allclose(n_z, t_z)
assert n_z.dtype == Z.dtype
def test_tensorsolve():
rng = np.random.RandomState(utt.fetch_seed())
A = tensor.tensor4("A", dtype=theano.config.floatX)
B = tensor.matrix("B", dtype=theano.config.floatX)
X = tensorsolve(A, B)
fn = function([A, B], [X])
# slightly modified example from np.linalg.tensorsolve docstring
a = np.eye(2 * 3 * 4).astype(theano.config.floatX)
a.shape = (2 * 3, 4, 2, 3 * 4)
b = rng.rand(2 * 3, 4).astype(theano.config.floatX)
n_x = np.linalg.tensorsolve(a, b)
t_x = fn(a, b)
assert _allclose(n_x, t_x)
# check the type upcast now
C = tensor.tensor4("C", dtype='float32')
D = tensor.matrix("D", dtype='float64')
Y = tensorsolve(C, D)
fn = function([C, D], [Y])
c = np.eye(2 * 3 * 4, dtype='float32')
c.shape = (2 * 3, 4, 2, 3 * 4)
d = rng.rand(2 * 3, 4).astype('float64')
n_y = np.linalg.tensorsolve(c, d)
t_y = fn(c, d)
assert _allclose(n_y, t_y)
assert n_y.dtype == Y.dtype
# check the type upcast now
E = tensor.tensor4("E", dtype='int32')
F = tensor.matrix("F", dtype='float64')
Z = tensorsolve(E, F)
fn = function([E, F], [Z])
e = np.eye(2 * 3 * 4, dtype='int32')
e.shape = (2 * 3, 4, 2, 3 * 4)
f = rng.rand(2 * 3, 4).astype('float64')
n_z = np.linalg.tensorsolve(e, f)
t_z = fn(e, f)
assert _allclose(n_z, t_z)
assert n_z.dtype == Z.dtype
def validate(self, image_shape, filter_shape, verify_grad=True):
image_dim = len(image_shape)
filter_dim = len(filter_shape)
input = T.TensorType('float64', [False]*image_dim)()
filters = T.TensorType('float64', [False]*filter_dim)()
bsize = image_shape[0]
if image_dim!=3: bsize = 1
nkern = filter_shape[0]
if filter_dim!=3: nkern = 1
############# THEANO IMPLEMENTATION ############
# we create a symbolic function so that verify_grad can work
def sym_conv2d(input, filters):
return conv.conv2d(input, filters)
output = sym_conv2d(input, filters)
theano_conv = theano.function([input, filters], output)
# initialize input and compute result
image_data = numpy.random.random(image_shape)
filter_data = numpy.random.random(filter_shape)
theano_output = theano_conv(image_data, filter_data)
############# REFERENCE IMPLEMENTATION ############
out_shape2d = numpy.array(image_shape[-2:]) -\
numpy.array(filter_shape[-2:]) + 1
ref_output = numpy.zeros(tuple(out_shape2d))
# reshape as 3D input tensors to make life easier
image_data3d = image_data.reshape((bsize,)+image_shape[-2:])
filter_data3d = filter_data.reshape((nkern,)+filter_shape[-2:])
# reshape theano output as 4D to make life easier
theano_output4d = theano_output.reshape((bsize,nkern,)+theano_output.shape[-2:])
# loop over mini-batches (if required)
for b in range(bsize):
# loop over filters (if required)
for k in range(nkern):
image2d = image_data3d[b,:,:]
filter2d = filter_data3d[k,:,:]
output2d = numpy.zeros(ref_output.shape)
for row in range(ref_output.shape[0]):
for col in range(ref_output.shape[1]):
output2d[row,col] += (image2d[row:row+filter2d.shape[0],
col:col+filter2d.shape[1]]*filter2d[::-1,::-1]
).sum()
self.assertTrue(_allclose(theano_output4d[b,k,:,:], output2d))
############# TEST GRADIENT ############
if verify_grad:
utt.verify_grad(sym_conv2d, [image_data, filter_data])
def test_eval(self):
A = self.A
Ai = tensorinv(A)
n_ainv = np.linalg.tensorinv(self.a)
tf_a = function([A], [Ai])
t_ainv = tf_a(self.a)
assert _allclose(n_ainv, t_ainv)
B = self.B
Bi = tensorinv(B)
Bi1 = tensorinv(B, ind=1)
n_binv = np.linalg.tensorinv(self.b)
n_binv1 = np.linalg.tensorinv(self.b1, ind=1)
tf_b = function([B], [Bi])
tf_b1 = function([B], [Bi1])
t_binv = tf_b(self.b)
t_binv1 = tf_b1(self.b1)
assert _allclose(t_binv, n_binv)
assert _allclose(t_binv1, n_binv1)
def test_eval(self):
A = self.A
Ai = tensorinv(A)
n_ainv = np.linalg.tensorinv(self.a)
tf_a = function([A], [Ai])
t_ainv = tf_a(self.a)
assert _allclose(n_ainv, t_ainv)
B = self.B
Bi = tensorinv(B)
Bi1 = tensorinv(B, ind=1)
n_binv = np.linalg.tensorinv(self.b)
n_binv1 = np.linalg.tensorinv(self.b1, ind=1)
tf_b = function([B], [Bi])
tf_b1 = function([B], [Bi1])
t_binv = tf_b(self.b)
t_binv1 = tf_b1(self.b1)
assert _allclose(t_binv, n_binv)
assert _allclose(t_binv1, n_binv1)
def test_autoencoder_logistic_linear_tied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc="sigmoid", act_dec="linear", tied_weights=True)
w = ae.weights.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.dot(1.0 / (1 + np.exp(-hb - np.dot(data, w))), w.T) + vb
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_dot():
print >>sys.stdout, 'starting test_dot'
utt.seed_rng()
rng = numpy.random.RandomState(utt.fetch_seed())
a0 = theano._asarray(rng.randn(4, 7), dtype='float32')
a1 = theano._asarray(rng.randn(7, 6), dtype='float32')
b0 = cuda_ndarray.CudaNdarray(a0)
b1 = cuda_ndarray.CudaNdarray(a1)
assert _allclose(numpy.dot(a0, a1), cuda_ndarray.dot(b0, b1))
a1 = theano._asarray(rng.randn(6, 7), dtype='float32')
b1 = cuda_ndarray.CudaNdarray(a1)
numpy_version = numpy.dot(a0, a1.T)
transposed = cuda_ndarray.dimshuffle(b1,(1,0))
cuda_version = cuda_ndarray.dot(b0, transposed)
assert _allclose(numpy_version, cuda_version)
a1 = theano._asarray(rng.randn(7, 6), dtype='float32')
b1 = cuda_ndarray.CudaNdarray(a1)
a0 = theano._asarray(rng.randn(7, 4), dtype='float32')
b0 = cuda_ndarray.CudaNdarray(a0)
assert _allclose(numpy.dot(a0.T, a1),
cuda_ndarray.dot(cuda_ndarray.dimshuffle(b0,(1,0)), b1))
a1 = theano._asarray(rng.randn(6, 7), dtype='float32')
b1 = cuda_ndarray.CudaNdarray(a1)
assert _allclose(numpy.dot(a0.T, a1.T),
cuda_ndarray.dot(cuda_ndarray.dimshuffle(b0,(1,0)),
cuda_ndarray.dimshuffle(b1,(1,0))))
def test_autoencoder_tanh_cos_untied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc="tanh", act_dec="cos", tied_weights=False)
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_autoencoder_tanh_cos_untied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=False)
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_autoencoder_logistic_linear_tied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc='sigmoid', act_dec='linear',
tied_weights=True)
w = ae.weights.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.dot(1. / (1 + np.exp(-hb - np.dot(data, w))), w.T) + vb
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_constant(self):
x = tt.constant(np.random.rand(2, 3), dtype=config.floatX)
y = theano.shared(np.random.rand(3, 6).astype(config.floatX), "y")
# should work
z = tt.dot(x, y)
assert hasattr(z.tag, "test_value")
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
# this test should fail
x = tt.constant(np.random.rand(2, 4), dtype=config.floatX)
with pytest.raises(ValueError):
tt.dot(x, y)
def test_empty_elemwise(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = theano.shared(np.random.rand(0, 6).astype(config.floatX), 'x')
# should work
z = (x + 2) * 3
assert hasattr(z.tag, 'test_value')
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_shared(self):
x = tt.matrix("x")
x.tag.test_value = np.random.rand(3, 4).astype(config.floatX)
y = theano.shared(np.random.rand(4, 6).astype(config.floatX), "y")
# should work
z = tt.dot(x, y)
assert hasattr(z.tag, "test_value")
f = theano.function([x], z)
assert _allclose(f(x.tag.test_value), z.tag.test_value)
# this test should fail
y.set_value(np.random.rand(5, 6).astype(config.floatX))
with pytest.raises(ValueError):
tt.dot(x, y)
def test_pseudoinverse_correctness():
rng = np.random.RandomState(utt.fetch_seed())
d1 = rng.randint(4) + 2
d2 = rng.randint(4) + 2
r = rng.randn(d1, d2).astype(theano.config.floatX)
x = tensor.matrix()
xi = pinv(x)
ri = function([x], xi)(r)
assert ri.shape[0] == r.shape[1]
assert ri.shape[1] == r.shape[0]
assert ri.dtype == r.dtype
# Note that pseudoinverse can be quite unprecise so I prefer to compare
# the result with what np.linalg returns
assert _allclose(ri, np.linalg.pinv(r))
def test_pseudoinverse_correctness():
rng = np.random.RandomState(utt.fetch_seed())
d1 = rng.randint(4) + 2
d2 = rng.randint(4) + 2
r = rng.randn(d1, d2).astype(theano.config.floatX)
x = tensor.matrix()
xi = pinv(x)
ri = function([x], xi)(r)
assert ri.shape[0] == r.shape[1]
assert ri.shape[1] == r.shape[0]
assert ri.dtype == r.dtype
# Note that pseudoinverse can be quite unprecise so I prefer to compare
# the result with what np.linalg returns
assert _allclose(ri, np.linalg.pinv(r))
def test_empty_elemwise(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = theano.shared(numpy.random.rand(0, 6).astype(config.floatX),
'x')
# should work
z = (x + 2) * 3
assert hasattr(z.tag, 'test_value')
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_matrix_dot():
rng = np.random.RandomState(utt.fetch_seed())
n = rng.randint(4) + 2
rs = []
xs = []
for k in xrange(n):
rs += [rng.randn(4, 4).astype(theano.config.floatX)]
xs += [tensor.matrix()]
sol = matrix_dot(*xs)
theano_sol = function(xs, sol)(*rs)
numpy_sol = rs[0]
for r in rs[1:]:
numpy_sol = np.dot(numpy_sol, r)
assert _allclose(numpy_sol, theano_sol)
def test_matrix_dot():
rng = np.random.RandomState(utt.fetch_seed())
n = rng.randint(4) + 2
rs = []
xs = []
for k in xrange(n):
rs += [rng.randn(4, 4).astype(theano.config.floatX)]
xs += [tensor.matrix()]
sol = matrix_dot(*xs)
theano_sol = function(xs, sol)(*rs)
numpy_sol = rs[0]
for r in rs[1:]:
numpy_sol = np.dot(numpy_sol, r)
assert _allclose(numpy_sol, theano_sol)
def test_variable_only(self):
x = tt.matrix("x")
x.tag.test_value = np.random.rand(3, 4).astype(config.floatX)
y = tt.matrix("y")
y.tag.test_value = np.random.rand(4, 5).astype(config.floatX)
# should work
z = tt.dot(x, y)
assert hasattr(z.tag, "test_value")
f = theano.function([x, y], z)
assert _allclose(f(x.tag.test_value, y.tag.test_value), z.tag.test_value)
# this test should fail
y.tag.test_value = np.random.rand(6, 5).astype(config.floatX)
with pytest.raises(ValueError):
tt.dot(x, y)
def test_csc_dense(self):
x = theano.sparse.csc_matrix("x")
y = theano.tensor.matrix("y")
f_a = theano.function([x, y], theano.sparse.dot(x, y))
f_b = lambda x, y: x * y
assert _allclose(f_a(self.x_csc, self.y), f_b(self.x_csc, self.y))
# Test infer_shape
f_a = theano.function([x, y], theano.sparse.dot(x, y).shape)
f_b = lambda x, y: (x * y).shape
assert numpy.all(f_a(self.x_csc, self.y) == f_b(self.x_csc, self.y))
topo = f_a.maker.env.toposort()
if theano.config.mode != "FAST_COMPILE":
nb = 0
else:
nb = 1
assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo]) == nb
def test_numpy_compare(self):
rng = np.random.RandomState(utt.fetch_seed())
M = tensor.matrix("A", dtype=theano.config.floatX)
V = tensor.vector("V", dtype=theano.config.floatX)
a = rng.rand(4, 4).astype(theano.config.floatX)
b = rng.rand(4).astype(theano.config.floatX)
A = ( [None, 'fro', 'inf', '-inf', 1, -1, None, 'inf', '-inf', 0, 1, -1, 2, -2],
[M, M, M, M, M, M, V, V, V, V, V, V, V, V],
[a, a, a, a, a, a, b, b, b, b, b, b, b, b],
[None, 'fro', inf, -inf, 1, -1, None, inf, -inf, 0, 1, -1, 2, -2])
for i in range(0, 14):
f = function([A[1][i]], norm(A[1][i], A[0][i]))
t_n = f(A[2][i])
n_n = np.linalg.norm(A[2][i], A[3][i])
assert _allclose(n_n, t_n)
def test_constant(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.constant(np.random.rand(2, 3), dtype=config.floatX)
y = theano.shared(np.random.rand(3, 6).astype(config.floatX), 'y')
# should work
z = T.dot(x, y)
assert hasattr(z.tag, 'test_value')
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
# this test should fail
x = T.constant(np.random.rand(2, 4), dtype=config.floatX)
self.assertRaises(ValueError, T.dot, x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_constant(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.constant(numpy.random.rand(2,3), dtype=config.floatX)
y = theano.shared(numpy.random.rand(3,6).astype(config.floatX), 'y')
# should work
z = T.dot(x,y)
assert hasattr(z.tag, 'test_value')
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
# this test should fail
x = T.constant(numpy.random.rand(2,4), dtype=config.floatX)
self.assertRaises(ValueError, T.dot, x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_sdae():
"""
Tests that StackedDenoisingAutoencoder works correctly
"""
data = np.random.randn(10, 5).astype(config.floatX) * 100
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=False)
corruptor = BinomialCorruptor(corruption_level=0.5)
model = StackedDenoisingAutoencoder([ae], corruptor)
model._ensure_extensions()
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], model.reconstruct(d))
assert not _allclose(ff(data), result)
def test_constant(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = "raise"
x = T.constant(np.random.rand(2, 3), dtype=config.floatX)
y = theano.shared(np.random.rand(3, 6).astype(config.floatX), "y")
# should work
z = T.dot(x, y)
assert hasattr(z.tag, "test_value")
f = theano.function([], z)
assert _allclose(f(), z.tag.test_value)
# this test should fail
x = T.constant(np.random.rand(2, 4), dtype=config.floatX)
with pytest.raises(ValueError):
T.dot(x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_sdae():
"""
Tests that StackedDenoisingAutoencoder works correctly
"""
data = np.random.randn(10, 5).astype(config.floatX) * 100
ae = Autoencoder(5, 7, act_enc="tanh", act_dec="cos", tied_weights=False)
corruptor = BinomialCorruptor(corruption_level=0.5)
model = StackedDenoisingAutoencoder([ae], corruptor)
model._ensure_extensions()
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], model.reconstruct(d))
assert not _allclose(ff(data), result)
def test_shared(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.matrix('x')
x.tag.test_value = np.random.rand(3, 4).astype(config.floatX)
y = theano.shared(np.random.rand(4, 6).astype(config.floatX), 'y')
# should work
z = T.dot(x, y)
assert hasattr(z.tag, 'test_value')
f = theano.function([x], z)
assert _allclose(f(x.tag.test_value), z.tag.test_value)
# this test should fail
y.set_value(np.random.rand(5, 6).astype(config.floatX))
self.assertRaises(ValueError, T.dot, x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_shared(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.matrix('x')
x.tag.test_value = numpy.random.rand(3,4).astype(config.floatX)
y = theano.shared(numpy.random.rand(4,6).astype(config.floatX), 'y')
# should work
z = T.dot(x,y)
assert hasattr(z.tag, 'test_value')
f = theano.function([x], z)
assert _allclose(f(x.tag.test_value), z.tag.test_value)
# this test should fail
y.set_value(numpy.random.rand(5,6).astype(config.floatX))
self.assertRaises(ValueError, T.dot, x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_csc_dense(self):
x = theano.sparse.csc_matrix('x')
y = theano.tensor.matrix('y')
f_a = theano.function([x, y], theano.sparse.dot(x, y))
f_b = lambda x, y: x * y
assert _allclose(f_a(self.x_csc, self.y), f_b(self.x_csc, self.y))
# Test infer_shape
f_a = theano.function([x, y], theano.sparse.dot(x, y).shape)
f_b = lambda x, y: (x * y).shape
assert numpy.all(f_a(self.x_csc, self.y) == f_b(self.x_csc, self.y))
topo = f_a.maker.env.toposort()
if theano.config.mode != 'FAST_COMPILE':
nb = 0
else:
nb = 1
assert sum([
isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo
]) == nb
def test_variable_only(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.matrix('x')
x.tag.test_value = numpy.random.rand(3,4).astype(config.floatX)
y = T.matrix('y')
y.tag.test_value = numpy.random.rand(4,5).astype(config.floatX)
# should work
z = T.dot(x,y)
assert hasattr(z.tag, 'test_value')
f = theano.function([x,y], z)
assert _allclose(f(x.tag.test_value, y.tag.test_value),
z.tag.test_value)
# this test should fail
y.tag.test_value = numpy.random.rand(6,5).astype(config.floatX)
self.assertRaises(ValueError, T.dot, x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_variable_only(self):
orig_compute_test_value = theano.config.compute_test_value
try:
theano.config.compute_test_value = 'raise'
x = T.matrix('x')
x.tag.test_value = numpy.random.rand(3, 4).astype(config.floatX)
y = T.matrix('y')
y.tag.test_value = numpy.random.rand(4, 5).astype(config.floatX)
# should work
z = T.dot(x, y)
assert hasattr(z.tag, 'test_value')
f = theano.function([x, y], z)
assert _allclose(f(x.tag.test_value, y.tag.test_value),
z.tag.test_value)
# this test should fail
y.tag.test_value = numpy.random.rand(6, 5).astype(config.floatX)
self.assertRaises(ValueError, T.dot, x, y)
finally:
theano.config.compute_test_value = orig_compute_test_value
def test_sum():
"""
test sum pattern 1, 11, 10, 01, 001, 010, 100, 110, 011, 111,
0011, 0101, 0111, 1011, 1111
test sum pattern implemented with reshape:
1000, 0100, 0010, 0001, 11111
others implemented by reshape that are not tested
0011,0101,0110,1001,1010,1100
1110,1101,1011
TODO: test with broadcast
"""
for shape, pattern in [((100,3,1300),[1]),
((0,),[0]),((5,),[0]),
((0,0),[0,1]),((1,0),[0,1]),((5,4),[0,1]),((33,31),[0,1]),((5,4),[1]),((5,4),[0]),#need something bigger then 32 for some opt test.
((5,4,3),[0]),((5,4,3),[1]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[1,2]),((5,4,3),[0,1,2]),
((0,0,0,0),[0,1,2,3]),
((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]),
((5,4,3,10,11),[1,2]),
((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]),
#test shape bigger then 4096 on each dimension to make sure that we work correctly when we don't have enought thread/block in each dimensions
((4100,3),[0]),((3,4101),[0]),#10
((1024,33),[0]),((33,1024),[0]),#10
((1025,33),[0]),((33,1025),[0]),#10
((4100,3),[1]),((3,4101),[1]),#01
((1024,33),[1]),((33,1024),[1]),#01
((1025,33),[1]),((33,1025),[1]),#01
((4100,3),[0,1]),((3,4101),[0,1]),#11
((1024,33),[0,1]),((33,1024),[0,1]),#01
((1025,33),[0,1]),((33,1025),[0,1]),#01
((4100,4,3),[0]),((5,4100,3),[0]),((5,4,4100),[0]),#100
((4100,4,3),[1]),((5,4100,3),[1]),((5,4,4100),[1]),#010
((4100,4,3),[2]),((5,4100,3),[2]),((5,4,4100),[2]),#001
((4100,4,3),[0,1]),((5,4100,3),[0,1]),((5,4,4100),[0,1]),#110
((4100,4,3),[1,2]),((5,4100,3),[1,2]),((5,4,4100),[1,2]),#011
#((4100,4,3),[0,2]),((5,4100,3),[0,2]),((5,4,4100),[0,2]),#101 ##not implemented
((4100,4,3),[0,1,2]),((5,4100,3),[0,1,2]),((5,4,4100),[0,1,2]),#111
((4100,4,3,2),[2,3]),((4,4100,3,2),[2,3]),((4,3,4100,2),[2,3]),((4,3,2,4100),[2,3]),#0011
((4100,4,3,2),[1,3]),((4,4100,3,2),[1,3]),((4,3,4100,2),[1,3]),((4,3,2,4100),[1,3]),#0101
((4100,4,3,2),[0,2,3]),((4,4100,3,2),[0,2,3]),((4,3,4100,2),[0,2,3]),#((4,3,2,4100),[0,2,3]),#1011
((4100,4,3,2),[1,2,3]),((4,4100,3,2),[1,2,3]),((4,3,4100,2),[1,2,3]),((4,3,2,4100),[1,2,3]),#0111
((4100,2,3,4),[0,1,2,3]),((2,4100,3,4),[0,1,2,3]),((2,3,4100,4),[0,1,2,3]),((2,3,4,4100),[0,1,2,3]),#1111
#test pattern implemented by reshape
((4100,4,3,2),[0]),((4,4100,3,2),[0]),((4,3,4100,2),[0]),((4,3,2,4100),[0]),#1000
((4100,4,3,2),[1]),((4,4100,3,2),[1]),((4,3,4100,2),[1]),((4,3,2,4100),[1]),#0100
((4100,4,3,2),[2]),((4,4100,3,2),[2]),((4,3,4100,2),[2]),((4,3,2,4100),[2]),#0010
((4100,4,3,2),[3]),((4,4100,3,2),[3]),((4,3,4100,2),[3]),((4,3,2,4100),[3]),#0001
((1100,2,3,4,5),[0,1,2,3,4]),((2,1100,3,4,5),[0,1,2,3,4]),((2,3,1100,4,5),[0,1,2,3,4]),((2,3,4,1100,5),[0,1,2,3,4]),((2,3,4,5,1100),[0,1,2,3,4]),#11111
]:
a = tensor.TensorType('float32', (False,) * len(shape))()
b = T.Sum(pattern)(a)
val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
# val = numpy.ones(shape)
# val = numpy.arange(numpy.prod(shape)).reshape(shape)
val = theano._asarray(val, dtype='float32')
f = theano.function([a], b, mode=mode_with_gpu)
f2 = theano.function([a], b, mode=mode_without_gpu)
assert tcn.GpuSum in [x.op.__class__ for x in f.maker.env.toposort()]
assert T.Sum in [x.op.__class__ for x in f2.maker.env.toposort()]
if val.size == 0:
assert f2(val) == f(val), ('shape', shape, 'pattern', pattern)
else:
try:
#We raise the error threashold as we sum big matrix
#and this cause small rounding difference with some seed
#example in debug mode with unittests.rseed=9275
orig_rtol = theano.tensor.basic.float32_rtol
theano.tensor.basic.float32_rtol = 2e-5
assert _allclose(f2(val), f(val)), ('shape', shape,
'pattern', pattern,
sum([shape[i] for i in pattern]),
f2(val), f(val), val)
finally:
theano.tensor.basic.float32_rtol = orig_rtol
#test with dimshuffle
#we shuffle the 2 outer dims.
for shape, pattern in [#((5,),[0]),
((5,4),[0,1]),((5,4),[0]),
((5,4,3),[0]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[0,1,2]),
((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3])]:
a = tensor.TensorType('float32', (False,) * len(shape))()
dim_pattern = range(len(shape))
dim_pattern[0] = 1
dim_pattern[1] = 0
a = a.dimshuffle(dim_pattern)
b = T.Sum(pattern)(a)
val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
# val = numpy.ones(shape)
# val = numpy.arange(numpy.prod(shape)).reshape(shape)
val = theano._asarray(val, dtype='float32')
f = theano.function([a], b, mode=mode_with_gpu)
f2 = theano.function([a], b, mode=mode_without_gpu)
assert tcn.GpuSum in [x.op.__class__ for x in f.maker.env.toposort()]
assert T.Sum in [x.op.__class__ for x in f2.maker.env.toposort()]
assert _allclose(f2(val), f(val)), ('shape', shape,
'pattern', pattern,
sum([shape[i] for i in pattern]))
#test with broadcast
for shape, pattern in [((5,),[0]),
((5,4),[0,1]),((5,4),[0]),
((5,4,3),[0]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[0,1,2]),
((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3])]:
shape = numpy.asarray(shape) * 2
a = tensor.TensorType('float32', (False,) * len(shape))()
a2 = tcn.CudaNdarrayType((False,) * len(shape))()
b = T.Sum(pattern)(a)
b2 = T.Sum(pattern)(a2)
val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
# val = numpy.ones(shape)
# val = numpy.arange(numpy.prod(shape)).reshape(shape)
val = theano._asarray(val, dtype='float32')
val2 = cuda.CudaNdarray(val)
if len(shape) == 1:
val = val[::2]
val2 = val2[::2]
elif len(shape) == 2:
val = val[::2, ::2]
val2 = val2[::2, ::2]
elif len(shape) == 3:
val = val[::2, ::2, ::2]
val2 = val2[::2, ::2, ::2]
elif len(shape) == 4:
val = val[::2, ::2, ::2, ::2]
val2 = val2[::2, ::2, ::2, ::2]
f = theano.function([a], b, mode=mode_without_gpu)
f2 = theano.function([a2], b2, mode=mode_with_gpu)
assert tcn.GpuSum in [x.op.__class__ for x in f2.maker.env.toposort()]
assert T.Sum in [x.op.__class__ for x in f.maker.env.toposort()]
assert _allclose(f2(val2), f(val)), ('shape', shape,
'pattern', pattern,
sum([shape[i] for i in pattern]))
def validate(self, image_shape, filter_shape,
border_mode='valid', subsample=(1, 1),
N_image_shape=None, N_filter_shape=None,
input=None, filters=None,
unroll_batch=None, unroll_kern=None, unroll_patch=None,
verify_grad=True, should_raise=False):
"""
:param image_shape: The constant shape info passed to conv2d.
:param filter_shape: The constant shape info passed to conv2d.
:param N_image_shape: None(default to image_shape) or tuple of
4 elements with the shape of the input image
:param N_filter_shape: None(default to filter_shape) or tuple
of 4 elements with the shape of the
input filter
"""
if N_image_shape is None:
N_image_shape = [T.get_scalar_constant_value(T.
as_tensor_variable(x)) for x in image_shape]
if N_filter_shape is None:
N_filter_shape = [T.get_scalar_constant_value(T.
as_tensor_variable(x)) for x in filter_shape]
if input is None:
input = self.input
if not filters:
filters = self.filters
############# THEANO IMPLEMENTATION ############
# we create a symbolic function so that verify_grad can work
def sym_conv2d(input, filters):
# define theano graph and function
input.name = 'input'
filters.name = 'filters'
rval = conv.conv2d(input, filters, image_shape, filter_shape,
border_mode, subsample, unroll_batch=unroll_batch,
unroll_kern=unroll_kern, unroll_patch=unroll_patch)
rval.name = 'conv_output'
return rval
output = sym_conv2d(input, filters)
output.name = 'conv2d(%s,%s)' % (input.name, filters.name)
theano_conv = theano.function([input, filters], output)
# initialize input and compute result
image_data = numpy.random.random(N_image_shape)
filter_data = numpy.random.random(N_filter_shape)
try:
theano_output = theano_conv(image_data, filter_data)
except ValueError:
if not should_raise:
raise
return
else:
if should_raise:
raise Exception(
"ConvOp should have generated an error")
############# REFERENCE IMPLEMENTATION ############
s = 1.
orig_image_data = image_data
if border_mode is not 'full':
s = -1.
out_shape2d = numpy.array(N_image_shape[-2:]) +\
s * numpy.array(N_filter_shape[-2:]) - s
out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample))
out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d)
ref_output = numpy.zeros(out_shape)
# loop over output feature maps
ref_output.fill(0)
if border_mode == 'full':
image_data2 = numpy.zeros((N_image_shape[0], N_image_shape[1],
N_image_shape[2] + 2 * N_filter_shape[2] - 2,
N_image_shape[3] + 2 * N_filter_shape[3] - 2))
image_data2[:, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 + N_image_shape[2],
N_filter_shape[3] - 1:N_filter_shape[3] - 1 + N_image_shape[3]] = image_data
image_data = image_data2
N_image_shape = image_data.shape
for bb in range(N_image_shape[0]):
for nn in range(N_filter_shape[0]):
for im0 in range(N_image_shape[1]):
filter2d = filter_data[nn, im0, :, :]
image2d = image_data[bb, im0, :, :]
for row in range(ref_output.shape[2]):
irow = row * subsample[0] # image row
for col in range(ref_output.shape[3]):
icol = col * subsample[1] # image col
ref_output[bb, nn, row, col] += (image2d[
irow:irow + N_filter_shape[2],
icol:icol + N_filter_shape[3]] * filter2d[::-1,::-1]
).sum()
self.assertTrue(_allclose(theano_output, ref_output))
############# TEST GRADIENT ############
if verify_grad:
utt.verify_grad(sym_conv2d, [orig_image_data, filter_data])
def test_careduce():
"""
test sum pattern 1, 11, 10, 01, 001, 010, 100, 110, 011, 111,
0011, 0101, 0111, 1011, 1111
test sum pattern implemented with reshape:
1000, 0100, 0010, 0001, 11111
others implemented by reshape that are not tested
0011,0101,0110,1001,1010,1100
1110,1101,1011
TODO: test with broadcast
"""
for scalar_op in [theano.scalar.add, theano.scalar.maximum]:
for shape, pattern in [((1,1),(1,)),
((1,0),(1,)),
((0,1),(1,)),
((0,0),(1,)),
((0,0,0),(1,2)),
((0,0,0,0),(1,2,3)),
((2,1),(1,)),
((1,2),(1,)),
((100,3,1300),[1]),
((0,),[0]),((5,),[0]),
((0,0),[0,1]),((1,0),[0,1]),((5,4),[0,1]),((33,31),[0,1]),((5,4),[1]),((5,4),[0]),#need something bigger then 32 for some opt test.
((5,4,3),[0]),((5,4,3),[1]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[1,2]),((5,4,3),[0,1,2]),
((0,0,0,0),[0,1,2,3]),
((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]),
((5,4,3,10,11),[1,2]),
((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]),
#test shape bigger then 4096 on each dimension to make sure that we work correctly when we don't have enough thread/block in each dimensions
((4100,3),[0]),((3,4101),[0]),#10
((1024,33),[0]),((33,1024),[0]),#10
((1025,33),[0]),((33,1025),[0]),#10
((4100,3),[1]),((3,4101),[1]),#01
((1024,33),[1]),((33,1024),[1]),#01
((1025,33),[1]),((33,1025),[1]),#01
((4100,3),[0,1]),((3,4101),[0,1]),#11
((1024,33),[0,1]),((33,1024),[0,1]),#01
((1025,33),[0,1]),((33,1025),[0,1]),#01
((4100,4,3),[0]),((5,4100,3),[0]),((5,4,4100),[0]), ((3,65536,1), [0]),#100
((4100,4,3),[1]),((5,4100,3),[1]),((5,4,4100),[1]),#010
((4100,4,3),[2]),((5,4100,3),[2]),((5,4,4100),[2]),#001
((4100,4,3),[0,1]),((5,4100,3),[0,1]),((5,4,4100),[0,1]),#110
((4100,4,3),[1,2]),((5,4100,3),[1,2]),((5,4,4100),[1,2]),#011
#((4100,4,3),[0,2]),((5,4100,3),[0,2]),((5,4,4100),[0,2]),#101 ##not implemented
((4100,4,3),[0,1,2]),((5,4100,3),[0,1,2]),((5,4,4100),[0,1,2]),#111
((4100,4,3,2),[2,3]),((4,4100,3,2),[2,3]),((4,3,4100,2),[2,3]),((4,3,2,4100),[2,3]),#0011
((4100,4,3,2),[1,3]),((4,4100,3,2),[1,3]),((4,3,4100,2),[1,3]),((4,3,2,4100),[1,3]),#0101
((4100,4,3,2),[0,2,3]),((4,4100,3,2),[0,2,3]),((4,3,4100,2),[0,2,3]),#((4,3,2,4100),[0,2,3]),#1011
((4100,4,3,2),[1,2,3]),((4,4100,3,2),[1,2,3]),((4,3,4100,2),[1,2,3]),((4,3,2,4100),[1,2,3]),#0111
((4100,2,3,4),[0,1,2,3]),((2,4100,3,4),[0,1,2,3]),((2,3,4100,4),[0,1,2,3]),((2,3,4,4100),[0,1,2,3]),#1111
#test pattern implemented by reshape
((4100,4,3,2),[0]),((4,4100,3,2),[0]),((4,3,4100,2),[0]),((4,3,2,4100),[0]),#1000
((4100,4,3,2),[1]),((4,4100,3,2),[1]),((4,3,4100,2),[1]),((4,3,2,4100),[1]),#0100
((4100,4,3,2),[2]),((4,4100,3,2),[2]),((4,3,4100,2),[2]),((4,3,2,4100),[2]),#0010
((4100,4,3,2),[3]),((4,4100,3,2),[3]),((4,3,4100,2),[3]),((4,3,2,4100),[3]),#0001
((1100,2,3,4,5),[0,1,2,3,4]),((2,1100,3,4,5),[0,1,2,3,4]),((2,3,1100,4,5),[0,1,2,3,4]),((2,3,4,1100,5),[0,1,2,3,4]),((2,3,4,5,1100),[0,1,2,3,4]),#11111
]:
op = tensor.CAReduce(scalar_op, axis=pattern)
pat = tensor_pattern_to_gpu_pattern(shape, pattern)
#GpuCAReduce{maximum} support only those patterns
if scalar_op is theano.scalar.maximum and pat not in [
(0, 1), (0, 1, 1), (0, 1, 1)]:
continue
a = tensor.TensorType('float32', (False,) * len(shape))()
b = op(a)
val = numpy.random.rand(numpy.prod(shape)).reshape(shape)
# val = numpy.ones(shape)
# val = numpy.arange(numpy.prod(shape)).reshape(shape)
val = theano._asarray(val, dtype='float32')
f = theano.function([a], b, mode=mode_with_gpu)
f2 = theano.function([a], b, mode=mode_without_gpu)
assert tcn.GpuCAReduce in [x.op.__class__
for x in f.maker.fgraph.toposort()]
assert op.__class__ in [x.op.__class__
for x in f2.maker.fgraph.toposort()]
f_caused_value_error = False
try:
f_out = f(val)
except ValueError, e:
exc = e
f_caused_value_error = True
f2_caused_value_error = False
try:
f2_out = f2(val)
except ValueError, e:
exc2 = e
f2_caused_value_error = True
if f_caused_value_error != f2_caused_value_error:
if f_caused_value_error:
print 'f caused this value error:'
print exc
else:
print 'f did not raise a value error, but should have'
if f2_caused_value_error:
print 'f2 caused this value error:'
print exc2
else:
print 'f should not have raised a value error'
print 'shape was: ', shape
print 'pattern was: ', pattern
assert False
try:
#We raise the error threashold as we sum big matrix
#and this cause small rounding difference with some seed
#example in debug mode with unittests.rseed=9275
orig_rtol = theano.tensor.basic.float32_rtol
theano.tensor.basic.float32_rtol = 2e-5
assert _allclose(f_out, f2_out), ('shape', shape,
'pattern', pattern,
sum([shape[i] for i in pattern]),
f2(val), f(val), val)
finally:
theano.tensor.basic.float32_rtol = orig_rtol
def test(self):
def mat(format, name, dtype):
if format == "dense":
return theano.tensor.matrix(name, dtype=dtype)
else:
return theano.sparse.matrix(format, name, dtype=dtype)
params = product(*([["float32", "float64", "int16", "complex64"]] * 4 + [["dense", "csc", "csr"]] * 2))
# All test are too slow, so we randomly take 100 of them.
# The buildbot change the seed, so we will finish by running them all.
# As of this writing they where all passing.
# params = self.rng.permutation(list(params))[:500]
for dtype1, dtype2, dtype3, dtype4, format1, format2 in params:
if format1 == "dense" and format2 == "dense":
# Usmm won't be used!
continue
x = mat(format1, "x", dtype1)
y = mat(format2, "y", dtype2)
a = theano.tensor.scalar("a", dtype=dtype3)
z = theano.shared(numpy.asarray(self.z, dtype=dtype4).copy())
f_b = lambda z, a, x, y: z - a * (x * y)
x_data = numpy.asarray(self.x, dtype=dtype1)
if format1 != "dense":
x_data = as_sparse_format(x_data, format1)
y_data = numpy.asarray(self.y, dtype=dtype2)
if format2 != "dense":
y_data = as_sparse_format(y_data, format2)
a_data = numpy.asarray(1.5, dtype=dtype3)
z_data = numpy.asarray(self.z, dtype=dtype4)
f_b_out = f_b(z_data, a_data, x_data, y_data)
# Can it work inplace?
inplace = dtype4 == theano.scalar.upcast(dtype1, dtype2, dtype3)
# To make it easier to check the toposort
mode = theano.compile.mode.get_default_mode().excluding("fusion")
if inplace:
updates = {z: z - a * theano.sparse.dot(x, y)}
f_a = theano.function([a, x, y], [], updates=updates, mode=mode)
f_a(a_data, x_data, y_data)
f_a_out = z.get_value(borrow=True)
else:
f_a = theano.function([a, x, y], z - a * theano.sparse.dot(x, y), mode=mode)
# In DebugMode there is a strange difference with complex
# So we raise a little the threshold a little.
try:
orig_atol = theano.tensor.basic.float64_atol
orig_rtol = theano.tensor.basic.float64_rtol
theano.tensor.basic.float64_atol = 1e-7
theano.tensor.basic.float64_rtol = 1e-6
f_a_out = f_a(a_data, x_data, y_data)
finally:
theano.tensor.basic.float64_atol = orig_atol
theano.tensor.basic.float64_rtol = orig_rtol
assert _allclose(f_a_out, f_b_out, rtol=1e-5)
topo = f_a.maker.env.toposort()
up = theano.scalar.upcast(dtype1, dtype2, dtype3, dtype4)
fast_compile = theano.config.mode == "FAST_COMPILE"
if (y.type.dtype == up and format1 == "csc" and format2 == "dense" and not fast_compile) and up in (
"float32",
"float64",
):
# The op UsmmCscDense should be inserted
assert (
sum(
[
isinstance(node.op, tensor.Elemwise)
and isinstance(node.op.scalar_op, theano.scalar.basic.Cast)
for node in topo
]
)
== len(topo) - 5
)
new_topo = []
for node in topo:
if not (
isinstance(node.op, tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Cast)
):
new_topo.append(node)
topo = new_topo
assert len(topo) == 5, topo
# Usmm is tested at the same time in debugmode
# Check if the optimization local_usmm and local_usmm_csx is
# applied
assert isinstance(topo[0].op, theano.sparse.basic.CSMProperties)
assert isinstance(topo[1].op, theano.tensor.DimShuffle)
assert isinstance(topo[2].op, theano.tensor.Subtensor)
assert topo[3].op == theano.tensor.neg
assert isinstance(topo[4].op, theano.sparse.UsmmCscDense)
if inplace:
assert topo[4].op.inplace
elif not fast_compile:
# The op Usmm should be inserted
assert len(topo) == 3, topo
assert isinstance(topo[0].op, theano.tensor.DimShuffle)
assert topo[1].op == theano.tensor.neg
assert isinstance(topo[2].op, theano.sparse.Usmm)
def validate(self, image_shape, filter_shape,
border_mode='valid', subsample=(1, 1),
input=None, filters=None, verify_grad=True,
non_contiguous=False, filter_dilation=(1, 1)):
"""
:param image_shape: The constant shape info passed to corrMM.
:param filter_shape: The constant shape info passed to corrMM.
"""
N_image_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x))
for x in image_shape]
N_filter_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x))
for x in filter_shape]
if input is None:
input = self.input
if filters is None:
filters = self.filters
# THEANO IMPLEMENTATION
# we create a symbolic function so that verify_grad can work
def sym_CorrMM(input, filters):
# define theano graph and function
input.name = 'input'
filters.name = 'filters'
rval = corr.CorrMM(border_mode, subsample,
filter_dilation)(input, filters)
rval.name = 'corr_output'
return rval
output = sym_CorrMM(input, filters)
output.name = 'CorrMM()(%s,%s)' % (input.name, filters.name)
theano_corr = theano.function([input, filters], output, mode=self.mode)
# initialize input and compute result
image_data = numpy.random.random(N_image_shape).astype(self.dtype)
filter_data = numpy.random.random(N_filter_shape).astype(self.dtype)
if non_contiguous:
image_data = numpy.transpose(image_data, axes=(0, 1, 3, 2))
image_data = image_data.copy()
image_data = numpy.transpose(image_data, axes=(0, 1, 3, 2))
filter_data = numpy.transpose(filter_data, axes=(0, 1, 3, 2))
filter_data = filter_data.copy()
filter_data = numpy.transpose(filter_data, axes=(0, 1, 3, 2))
assert not image_data.flags['CONTIGUOUS']
assert not filter_data.flags['CONTIGUOUS']
theano_output = theano_corr(image_data, filter_data)
# REFERENCE IMPLEMENTATION
# Testing correlation, not convolution. Reverse filters.
filter_data_corr = numpy.array(filter_data[:, :, ::-1, ::-1],
copy=True,
order='C')
orig_image_data = image_data
img_shape2d = numpy.array(N_image_shape[-2:])
fil_shape2d = numpy.array(N_filter_shape[-2:])
dil_shape2d = numpy.array(filter_dilation)
dil_fil_shape2d = (fil_shape2d - 1) * dil_shape2d + 1
subsample2d = numpy.array(subsample)
if border_mode == 'full':
padHW = (dil_fil_shape2d - 1)
elif border_mode == 'valid':
padHW = numpy.array([0, 0])
elif border_mode == 'half':
padHW = numpy.floor(dil_fil_shape2d / 2).astype('int32')
elif isinstance(border_mode, tuple):
padHW = numpy.array(border_mode)
elif isinstance(border_mode, integer_types):
padHW = numpy.array([border_mode, border_mode])
else:
raise NotImplementedError('Unsupported border_mode {}'.format(border_mode))
out_shape2d = numpy.floor((img_shape2d + 2 * (padHW) - dil_fil_shape2d) / subsample2d) + 1
# avoid numpy deprecation
out_shape2d = out_shape2d.astype('int32')
out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d)
ref_output = numpy.zeros(out_shape)
# loop over output feature maps
ref_output.fill(0)
image_data2 = numpy.zeros((N_image_shape[0], N_image_shape[1],
N_image_shape[2] + 2 * padHW[0],
N_image_shape[3] + 2 * padHW[1]))
image_data2[:, :, padHW[0]:padHW[0] + N_image_shape[2],
padHW[1]:padHW[1] + N_image_shape[3]] = image_data
image_data = image_data2
N_image_shape = image_data.shape
for bb in range(N_image_shape[0]):
for nn in range(N_filter_shape[0]):
for im0 in range(N_image_shape[1]):
filter2d = filter_data_corr[nn, im0, :, :]
image2d = image_data[bb, im0, :, :]
for row in range(ref_output.shape[2]):
irow = row * subsample[0] # image row
for col in range(ref_output.shape[3]):
icol = col * subsample[1] # image col
ref_output[bb, nn, row, col] += (image2d[
irow:irow + dil_fil_shape2d[0]:filter_dilation[0],
icol:icol + dil_fil_shape2d[1]:filter_dilation[1]] * filter2d[::-1, ::-1]
).sum()
self.assertTrue(_allclose(theano_output, ref_output))
# TEST GRADIENT
if verify_grad:
utt.verify_grad(sym_CorrMM, [orig_image_data, filter_data],
mode=self.mode)
