def test_qr_modes(): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) a = rng.rand(4, 4).astype(theano.config.floatX) f = function([A], qr(A)) t_qr = f(a) n_qr = np.linalg.qr(a) assert _allclose(n_qr, t_qr) for mode in ["reduced", "r", "raw"]: f = function([A], qr(A, mode)) t_qr = f(a) n_qr = np.linalg.qr(a, mode) if isinstance(n_qr, (list, tuple)): assert _allclose(n_qr[0], t_qr[0]) assert _allclose(n_qr[1], t_qr[1]) else: assert _allclose(n_qr, t_qr) try: n_qr = np.linalg.qr(a, "complete") f = function([A], qr(A, "complete")) t_qr = f(a) assert _allclose(n_qr, t_qr) except TypeError as e: assert "name 'complete' is not defined" in str(e)
def test_rop_lop(): mx = tensor.matrix('mx') mv = tensor.matrix('mv') v = tensor.vector('v') y = matrix_inverse(mx).sum(axis=0) yv = tensor.Rop(y, mx, mv) yv2 = tensor.Rop_via_Lop(y, mx, mv) rop_f = function([mx, mv], [yv, yv2]) sy, _ = theano.scan(lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(), sequences=tensor.arange(y.shape[0]), non_sequences=[y, mx, mv]) scan_f = function([mx, mv], sy) rng = np.random.RandomState(utt.fetch_seed()) vx = np.asarray(rng.randn(4, 4), theano.config.floatX) vv = np.asarray(rng.randn(4, 4), theano.config.floatX) v1 = scan_f(vx, vv) v2, v3 = rop_f(vx, vv) assert _allclose(v2, v1), ('Rop mismatch: %s %s' % (v2, v1)) assert _allclose(v3, v1), ('Rop_via_Lop mismatch: %s %s' % (v3, v1)) raised = False try: tensor.Rop(theano.clone(y, replace={mx: break_op(mx)}), mx, mv) except ValueError: raised = True if not raised: raise Exception(('Op did not raised an error even though the function' ' is not differentiable')) try: tensor.Rop_via_Lop(theano.clone(y, replace={mx: break_op(mx)}), mx, mv) except theano.gradient.NullTypeGradError: raised = True except theano.gradient.DisconnectedInputError: raised = True if not raised: raise Exception(( 'Rop_via_Lop for Op did not raise an error even though the function' ' is not differentiable')) vv = np.asarray(rng.uniform(size=(4, )), theano.config.floatX) yv = tensor.Lop(y, mx, v) lop_f = function([mx, v], yv) sy = tensor.grad((v * y).sum(), mx) scan_f = function([mx, v], sy) v1 = lop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), ('LOP mismatch: %s %s' % (v1, v2))
def test_svd(): rng = numpy.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) U, V, T = svd(A) fn = function([A], [U, V, T]) a = rng.rand(4, 4).astype(theano.config.floatX) n_u, n_v, n_t = numpy.linalg.svd(a) t_u, t_v, t_t = fn(a) assert _allclose(n_u, t_u) assert _allclose(n_v, t_v) assert _allclose(n_t, t_t)
def test_svd(): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.matrix("A", dtype=theano.config.floatX) U, V, T = svd(A) fn = function([A], [U, V, T]) a = rng.rand(4, 4).astype(theano.config.floatX) n_u, n_v, n_t = np.linalg.svd(a) t_u, t_v, t_t = fn(a) assert _allclose(n_u, t_u) assert _allclose(n_v, t_v) assert _allclose(n_t, t_t)
def test_svd(self): A = tensor.matrix("A", dtype=self.dtype) U, S, VT = svd(A) fn = function([A], [U, S, VT]) a = self.rng.rand(4, 4).astype(self.dtype) n_u, n_s, n_vt = np.linalg.svd(a) t_u, t_s, t_vt = fn(a) assert _allclose(n_u, t_u) assert _allclose(n_s, t_s) assert _allclose(n_vt, t_vt) fn = function([A], svd(A, compute_uv=False)) t_s = fn(a) assert _allclose(n_s, t_s)
def test_string_var(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = "raise" x = T.matrix("x") x.tag.test_value = np.random.rand(3, 4).astype(config.floatX) y = T.matrix("y") y.tag.test_value = np.random.rand(4, 5).astype(config.floatX) z = theano.shared(np.random.rand(5, 6).astype(config.floatX)) # should work out = T.dot(T.dot(x, y), z) assert hasattr(out.tag, "test_value") tf = theano.function([x, y], out) assert _allclose(tf(x.tag.test_value, y.tag.test_value), out.tag.test_value) def f(x, y, z): return T.dot(T.dot(x, y), z) # this test should fail z.set_value(np.random.rand(7, 6).astype(config.floatX)) with pytest.raises(ValueError): f(x, y, z) finally: theano.config.compute_test_value = orig_compute_test_value
def test_inverse_correctness(self): r = self.rng.randn(4, 4).astype(theano.config.floatX) x = tensor.matrix() xi = self.op(x) ri = function([x], xi)(r) assert ri.shape == r.shape assert ri.dtype == r.dtype rir = np.dot(ri, r) rri = np.dot(r, ri) assert _allclose(np.identity(4), rir), rir assert _allclose(np.identity(4), rri), rri
def test_sparse_sparse(self): for d1, d2 in [ ("float32", "float32"), ("float32", "float64"), ("float64", "float32"), ("float64", "float64"), ("float32", "int16"), ("float32", "complex64"), ]: for x_f, y_f in [("csc", "csc"), ("csc", "csr"), ("csr", "csc"), ("csr", "csr")]: x = theano.sparse.SparseType(format=x_f, dtype=d1)("x") y = theano.sparse.SparseType(format=x_f, dtype=d2)("x") f_a = theano.function([x, y], theano.sparse.dot(x, y)) f_b = lambda x, y: x * y vx = getattr(self, "x_" + x_f).astype(d1) vy = getattr(self, "y_" + y_f).astype(d2) assert _allclose(f_a(vx, vy), f_b(vx, vy).toarray()) # Test infer_shape f_a = theano.function([x, y], theano.sparse.dot(x, y).shape) f_b = lambda x, y: (x * y).shape assert numpy.all(f_a(vx, vy) == f_b(vx, vy)) topo = f_a.maker.env.toposort() if theano.config.mode != "FAST_COMPILE": nb = 0 else: nb = 1 assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo]) == nb
def test_string_var(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.matrix('x') x.tag.test_value = numpy.random.rand(3,4).astype(config.floatX) y = T.matrix('y') y.tag.test_value = numpy.random.rand(4,5).astype(config.floatX) z = theano.shared(numpy.random.rand(5,6).astype(config.floatX)) # should work out = T.dot(T.dot(x,y), z) assert hasattr(out.tag, 'test_value') tf = theano.function([x,y], out) assert _allclose( tf(x.tag.test_value, y.tag.test_value), out.tag.test_value) def f(x,y,z): return T.dot(T.dot(x,y),z) # this test should fail z.set_value(numpy.random.rand(7,6).astype(config.floatX)) self.assertRaises(ValueError, f, x, y, z) finally: theano.config.compute_test_value = orig_compute_test_value
def test_upcast(self): typenames = ("float32", "int64", "int8", "int32", "int16", "float64", "complex64", "complex128") for dense_dtype in typenames: for sparse_dtype in typenames: correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype) a = SparseType("csc", dtype=sparse_dtype)() b = tensor.matrix(dtype=dense_dtype) d = structured_dot(a, b) assert d.type.dtype == correct_dtype # compile and run a function f = theano.function([a, b], d) M, N, K, nnz = (4, 3, 5, 3) spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz)) # the following madness is necessary to workaround # an intc vs. int32 bug. # The lil makes an intc on my computer when sparse_dtype # is int32. spmat.dtype = numpy.dtype(sparse_dtype) mat = numpy.asarray(numpy.random.randn(N, K) * 9, dtype=dense_dtype) print "DTYPES", sparse_dtype, dense_dtype print "sym types", a.type, b.type print "dtype strings", spmat.dtype, mat.dtype print "numpy dtype num", mat.dtype.num print "scipy dtype num", spmat.data.dtype.num theano_result = f(spmat, mat) scipy_result = spmat * mat assert theano_result.shape == scipy_result.shape assert theano_result.dtype == scipy_result.dtype assert _allclose(theano_result, scipy_result)
def test_string_var(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.matrix('x') x.tag.test_value = numpy.random.rand(3, 4).astype(config.floatX) y = T.matrix('y') y.tag.test_value = numpy.random.rand(4, 5).astype(config.floatX) z = theano.shared(numpy.random.rand(5, 6).astype(config.floatX)) # should work out = T.dot(T.dot(x, y), z) assert hasattr(out.tag, 'test_value') tf = theano.function([x, y], out) assert _allclose(tf(x.tag.test_value, y.tag.test_value), out.tag.test_value) def f(x, y, z): return T.dot(T.dot(x, y), z) # this test should fail z.set_value(numpy.random.rand(7, 6).astype(config.floatX)) self.assertRaises(ValueError, f, x, y, z) finally: theano.config.compute_test_value = orig_compute_test_value
def test_sparse_sparse(self): for d1, d2 in [('float32', 'float32'), ('float32', 'float64'), ('float64', 'float32'), ('float64', 'float64'), ('float32', 'int16'), ('float32', 'complex64'), ]: for x_f, y_f in [('csc', 'csc'), ('csc', 'csr'), ('csr', 'csc'), ('csr', 'csr'), ]: x = theano.sparse.SparseType(format=x_f, dtype=d1)('x') y = theano.sparse.SparseType(format=x_f, dtype=d2)('x') f_a = theano.function([x, y], theano.sparse.dot(x, y)) f_b = lambda x, y: x * y vx = getattr(self, 'x_' + x_f).astype(d1) vy = getattr(self, 'y_' + y_f).astype(d2) assert _allclose(f_a(vx, vy), f_b(vx, vy).toarray()) # Test infer_shape f_a = theano.function([x, y], theano.sparse.dot(x, y).shape) f_b = lambda x, y: (x * y).shape assert numpy.all(f_a(vx, vy) == f_b(vx, vy)) topo = f_a.maker.env.toposort() if theano.config.mode != 'FAST_COMPILE': nb = 0 else: nb = 1 assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo]) == nb
def test_upcast(self): typenames = ('float32', 'int64', 'int8', 'int32', 'int16', 'float64', 'complex64', 'complex128') for dense_dtype in typenames: for sparse_dtype in typenames: correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype) a = SparseType('csc', dtype=sparse_dtype)() b = tensor.matrix(dtype=dense_dtype) d = structured_dot(a, b) assert d.type.dtype == correct_dtype # compile and run a function f = theano.function([a, b], d) M, N, K, nnz = (4, 3, 5, 3) spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz)) # the following madness is necessary to workaround # an intc vs. int32 bug. # The lil makes an intc on my computer when sparse_dtype # is int32. spmat.dtype = numpy.dtype(sparse_dtype) mat = numpy.asarray(numpy.random.randn(N, K) * 9, dtype=dense_dtype) print 'DTYPES', sparse_dtype, dense_dtype print 'sym types', a.type, b.type print 'dtype strings', spmat.dtype, mat.dtype print 'numpy dtype num', mat.dtype.num print 'scipy dtype num', spmat.data.dtype.num theano_result = f(spmat, mat) scipy_result = spmat * mat assert theano_result.shape == scipy_result.shape assert theano_result.dtype == scipy_result.dtype assert _allclose(theano_result, scipy_result)
def test_inverse_correctness(): rng = numpy.random.RandomState(utt.fetch_seed()) r = rng.randn(4, 4).astype(theano.config.floatX) x = tensor.matrix() xi = matrix_inverse(x) ri = function([x], xi)(r) assert ri.shape == r.shape assert ri.dtype == r.dtype rir = numpy.dot(ri, r) rri = numpy.dot(r, ri) assert _allclose(numpy.identity(4), rir), rir assert _allclose(numpy.identity(4), rri), rri
def test_empty_elemwise(self): x = theano.shared(np.random.rand(0, 6).astype(config.floatX), "x") # should work z = (x + 2) * 3 assert hasattr(z.tag, "test_value") f = theano.function([], z) assert _allclose(f(), z.tag.test_value)
def test_rop_lop(): mx = tensor.matrix('mx') mv = tensor.matrix('mv') v = tensor.vector('v') y = matrix_inverse(mx).sum(axis=0) yv = tensor.Rop(y, mx, mv) rop_f = function([mx, mv], yv) sy, _ = theano.scan(lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(), sequences=tensor.arange(y.shape[0]), non_sequences=[y, mx, mv]) scan_f = function([mx, mv], sy) rng = numpy.random.RandomState(utt.fetch_seed()) vx = numpy.asarray(rng.randn(4, 4), theano.config.floatX) vv = numpy.asarray(rng.randn(4, 4), theano.config.floatX) v1 = rop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), ('ROP mismatch: %s %s' % (v1, v2)) raised = False try: tensor.Rop( theano.clone(y, replace={mx: break_op(mx)}), mx, mv) except ValueError: raised = True if not raised: raise Exception(( 'Op did not raised an error even though the function' ' is not differentiable')) vv = numpy.asarray(rng.uniform(size=(4,)), theano.config.floatX) yv = tensor.Lop(y, mx, v) lop_f = function([mx, v], yv) sy = tensor.grad((v * y).sum(), mx) scan_f = function([mx, v], sy) v1 = lop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), ('LOP mismatch: %s %s' % (v1, v2))
def validate(self, image_shape, filter_shape, verify_grad=True): image_dim = len(image_shape) filter_dim = len(filter_shape) input = T.TensorType("float64", [False] * image_dim)() filters = T.TensorType("float64", [False] * filter_dim)() bsize = image_shape[0] if image_dim != 3: bsize = 1 nkern = filter_shape[0] if filter_dim != 3: nkern = 1 ############# THEANO IMPLEMENTATION ############ # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): return conv.conv2d(input, filters) output = sym_conv2d(input, filters) theano_conv = theano.function([input, filters], output) # initialize input and compute result image_data = numpy.random.random(image_shape) filter_data = numpy.random.random(filter_shape) theano_output = theano_conv(image_data, filter_data) ############# REFERENCE IMPLEMENTATION ############ out_shape2d = numpy.array(image_shape[-2:]) - numpy.array(filter_shape[-2:]) + 1 ref_output = numpy.zeros(tuple(out_shape2d)) # reshape as 3D input tensors to make life easier image_data3d = image_data.reshape((bsize,) + image_shape[-2:]) filter_data3d = filter_data.reshape((nkern,) + filter_shape[-2:]) # reshape theano output as 4D to make life easier theano_output4d = theano_output.reshape((bsize, nkern) + theano_output.shape[-2:]) # loop over mini-batches (if required) for b in range(bsize): # loop over filters (if required) for k in range(nkern): image2d = image_data3d[b, :, :] filter2d = filter_data3d[k, :, :] output2d = numpy.zeros(ref_output.shape) for row in range(ref_output.shape[0]): for col in range(ref_output.shape[1]): output2d[row, col] += ( image2d[row : row + filter2d.shape[0], col : col + filter2d.shape[1]] * filter2d[::-1, ::-1] ).sum() self.assertTrue(_allclose(theano_output4d[b, k, :, :], output2d)) ############# TEST GRADIENT ############ if verify_grad: utt.verify_grad(sym_conv2d, [image_data, filter_data])
def test_rop_lop(): mx = tensor.matrix("mx") mv = tensor.matrix("mv") v = tensor.vector("v") y = matrix_inverse(mx).sum(axis=0) yv = tensor.Rop(y, mx, mv) rop_f = function([mx, mv], yv) sy, _ = theano.scan( lambda i, y, x, v: (tensor.grad(y[i], x) * v).sum(), sequences=tensor.arange(y.shape[0]), non_sequences=[y, mx, mv], ) scan_f = function([mx, mv], sy) rng = np.random.RandomState(utt.fetch_seed()) vx = np.asarray(rng.randn(4, 4), theano.config.floatX) vv = np.asarray(rng.randn(4, 4), theano.config.floatX) v1 = rop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), "ROP mismatch: %s %s" % (v1, v2) raised = False try: tensor.Rop(theano.clone(y, replace={mx: break_op(mx)}), mx, mv) except ValueError: raised = True if not raised: raise Exception(("Op did not raised an error even though the function" " is not differentiable")) vv = np.asarray(rng.uniform(size=(4, )), theano.config.floatX) yv = tensor.Lop(y, mx, v) lop_f = function([mx, v], yv) sy = tensor.grad((v * y).sum(), mx) scan_f = function([mx, v], sy) v1 = lop_f(vx, vv) v2 = scan_f(vx, vv) assert _allclose(v1, v2), "LOP mismatch: %s %s" % (v1, v2)
def test_tensorsolve(): rng = np.random.RandomState(utt.fetch_seed()) A = tensor.tensor4("A", dtype=theano.config.floatX) B = tensor.matrix("B", dtype=theano.config.floatX) X = tensorsolve(A, B) fn = function([A, B], [X]) # slightly modified example from np.linalg.tensorsolve docstring a = np.eye(2 * 3 * 4).astype(theano.config.floatX) a.shape = (2 * 3, 4, 2, 3 * 4) b = rng.rand(2 * 3, 4).astype(theano.config.floatX) n_x = np.linalg.tensorsolve(a, b) t_x = fn(a, b) assert _allclose(n_x, t_x) # check the type upcast now C = tensor.tensor4("C", dtype='float32') D = tensor.matrix("D", dtype='float64') Y = tensorsolve(C, D) fn = function([C, D], [Y]) c = np.eye(2 * 3 * 4, dtype='float32') c.shape = (2 * 3, 4, 2, 3 * 4) d = rng.rand(2 * 3, 4).astype('float64') n_y = np.linalg.tensorsolve(c, d) t_y = fn(c, d) assert _allclose(n_y, t_y) assert n_y.dtype == Y.dtype # check the type upcast now E = tensor.tensor4("E", dtype='int32') F = tensor.matrix("F", dtype='float64') Z = tensorsolve(E, F) fn = function([E, F], [Z]) e = np.eye(2 * 3 * 4, dtype='int32') e.shape = (2 * 3, 4, 2, 3 * 4) f = rng.rand(2 * 3, 4).astype('float64') n_z = np.linalg.tensorsolve(e, f) t_z = fn(e, f) assert _allclose(n_z, t_z) assert n_z.dtype == Z.dtype
def validate(self, image_shape, filter_shape, verify_grad=True): image_dim = len(image_shape) filter_dim = len(filter_shape) input = T.TensorType('float64', [False]*image_dim)() filters = T.TensorType('float64', [False]*filter_dim)() bsize = image_shape[0] if image_dim!=3: bsize = 1 nkern = filter_shape[0] if filter_dim!=3: nkern = 1 ############# THEANO IMPLEMENTATION ############ # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): return conv.conv2d(input, filters) output = sym_conv2d(input, filters) theano_conv = theano.function([input, filters], output) # initialize input and compute result image_data = numpy.random.random(image_shape) filter_data = numpy.random.random(filter_shape) theano_output = theano_conv(image_data, filter_data) ############# REFERENCE IMPLEMENTATION ############ out_shape2d = numpy.array(image_shape[-2:]) -\ numpy.array(filter_shape[-2:]) + 1 ref_output = numpy.zeros(tuple(out_shape2d)) # reshape as 3D input tensors to make life easier image_data3d = image_data.reshape((bsize,)+image_shape[-2:]) filter_data3d = filter_data.reshape((nkern,)+filter_shape[-2:]) # reshape theano output as 4D to make life easier theano_output4d = theano_output.reshape((bsize,nkern,)+theano_output.shape[-2:]) # loop over mini-batches (if required) for b in range(bsize): # loop over filters (if required) for k in range(nkern): image2d = image_data3d[b,:,:] filter2d = filter_data3d[k,:,:] output2d = numpy.zeros(ref_output.shape) for row in range(ref_output.shape[0]): for col in range(ref_output.shape[1]): output2d[row,col] += (image2d[row:row+filter2d.shape[0], col:col+filter2d.shape[1]]*filter2d[::-1,::-1] ).sum() self.assertTrue(_allclose(theano_output4d[b,k,:,:], output2d)) ############# TEST GRADIENT ############ if verify_grad: utt.verify_grad(sym_conv2d, [image_data, filter_data])
def test_eval(self): A = self.A Ai = tensorinv(A) n_ainv = np.linalg.tensorinv(self.a) tf_a = function([A], [Ai]) t_ainv = tf_a(self.a) assert _allclose(n_ainv, t_ainv) B = self.B Bi = tensorinv(B) Bi1 = tensorinv(B, ind=1) n_binv = np.linalg.tensorinv(self.b) n_binv1 = np.linalg.tensorinv(self.b1, ind=1) tf_b = function([B], [Bi]) tf_b1 = function([B], [Bi1]) t_binv = tf_b(self.b) t_binv1 = tf_b1(self.b1) assert _allclose(t_binv, n_binv) assert _allclose(t_binv1, n_binv1)
def test_autoencoder_logistic_linear_tied(): data = np.random.randn(10, 5).astype(config.floatX) ae = Autoencoder(5, 7, act_enc="sigmoid", act_dec="linear", tied_weights=True) w = ae.weights.get_value() ae.hidbias.set_value(np.random.randn(7).astype(config.floatX)) hb = ae.hidbias.get_value() ae.visbias.set_value(np.random.randn(5).astype(config.floatX)) vb = ae.visbias.get_value() d = tensor.matrix() result = np.dot(1.0 / (1 + np.exp(-hb - np.dot(data, w))), w.T) + vb ff = theano.function([d], ae.reconstruct(d)) assert _allclose(ff(data), result)
def test_dot(): print >>sys.stdout, 'starting test_dot' utt.seed_rng() rng = numpy.random.RandomState(utt.fetch_seed()) a0 = theano._asarray(rng.randn(4, 7), dtype='float32') a1 = theano._asarray(rng.randn(7, 6), dtype='float32') b0 = cuda_ndarray.CudaNdarray(a0) b1 = cuda_ndarray.CudaNdarray(a1) assert _allclose(numpy.dot(a0, a1), cuda_ndarray.dot(b0, b1)) a1 = theano._asarray(rng.randn(6, 7), dtype='float32') b1 = cuda_ndarray.CudaNdarray(a1) numpy_version = numpy.dot(a0, a1.T) transposed = cuda_ndarray.dimshuffle(b1,(1,0)) cuda_version = cuda_ndarray.dot(b0, transposed) assert _allclose(numpy_version, cuda_version) a1 = theano._asarray(rng.randn(7, 6), dtype='float32') b1 = cuda_ndarray.CudaNdarray(a1) a0 = theano._asarray(rng.randn(7, 4), dtype='float32') b0 = cuda_ndarray.CudaNdarray(a0) assert _allclose(numpy.dot(a0.T, a1), cuda_ndarray.dot(cuda_ndarray.dimshuffle(b0,(1,0)), b1)) a1 = theano._asarray(rng.randn(6, 7), dtype='float32') b1 = cuda_ndarray.CudaNdarray(a1) assert _allclose(numpy.dot(a0.T, a1.T), cuda_ndarray.dot(cuda_ndarray.dimshuffle(b0,(1,0)), cuda_ndarray.dimshuffle(b1,(1,0))))
def test_autoencoder_tanh_cos_untied(): data = np.random.randn(10, 5).astype(config.floatX) ae = Autoencoder(5, 7, act_enc="tanh", act_dec="cos", tied_weights=False) w = ae.weights.get_value() w_prime = ae.w_prime.get_value() ae.hidbias.set_value(np.random.randn(7).astype(config.floatX)) hb = ae.hidbias.get_value() ae.visbias.set_value(np.random.randn(5).astype(config.floatX)) vb = ae.visbias.get_value() d = tensor.matrix() result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb) ff = theano.function([d], ae.reconstruct(d)) assert _allclose(ff(data), result)
def test_autoencoder_tanh_cos_untied(): data = np.random.randn(10, 5).astype(config.floatX) ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=False) w = ae.weights.get_value() w_prime = ae.w_prime.get_value() ae.hidbias.set_value(np.random.randn(7).astype(config.floatX)) hb = ae.hidbias.get_value() ae.visbias.set_value(np.random.randn(5).astype(config.floatX)) vb = ae.visbias.get_value() d = tensor.matrix() result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb) ff = theano.function([d], ae.reconstruct(d)) assert _allclose(ff(data), result)
def test_autoencoder_logistic_linear_tied(): data = np.random.randn(10, 5).astype(config.floatX) ae = Autoencoder(5, 7, act_enc='sigmoid', act_dec='linear', tied_weights=True) w = ae.weights.get_value() ae.hidbias.set_value(np.random.randn(7).astype(config.floatX)) hb = ae.hidbias.get_value() ae.visbias.set_value(np.random.randn(5).astype(config.floatX)) vb = ae.visbias.get_value() d = tensor.matrix() result = np.dot(1. / (1 + np.exp(-hb - np.dot(data, w))), w.T) + vb ff = theano.function([d], ae.reconstruct(d)) assert _allclose(ff(data), result)
def test_constant(self): x = tt.constant(np.random.rand(2, 3), dtype=config.floatX) y = theano.shared(np.random.rand(3, 6).astype(config.floatX), "y") # should work z = tt.dot(x, y) assert hasattr(z.tag, "test_value") f = theano.function([], z) assert _allclose(f(), z.tag.test_value) # this test should fail x = tt.constant(np.random.rand(2, 4), dtype=config.floatX) with pytest.raises(ValueError): tt.dot(x, y)
def test_empty_elemwise(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = theano.shared(np.random.rand(0, 6).astype(config.floatX), 'x') # should work z = (x + 2) * 3 assert hasattr(z.tag, 'test_value') f = theano.function([], z) assert _allclose(f(), z.tag.test_value) finally: theano.config.compute_test_value = orig_compute_test_value
def test_shared(self): x = tt.matrix("x") x.tag.test_value = np.random.rand(3, 4).astype(config.floatX) y = theano.shared(np.random.rand(4, 6).astype(config.floatX), "y") # should work z = tt.dot(x, y) assert hasattr(z.tag, "test_value") f = theano.function([x], z) assert _allclose(f(x.tag.test_value), z.tag.test_value) # this test should fail y.set_value(np.random.rand(5, 6).astype(config.floatX)) with pytest.raises(ValueError): tt.dot(x, y)
def test_pseudoinverse_correctness(): rng = np.random.RandomState(utt.fetch_seed()) d1 = rng.randint(4) + 2 d2 = rng.randint(4) + 2 r = rng.randn(d1, d2).astype(theano.config.floatX) x = tensor.matrix() xi = pinv(x) ri = function([x], xi)(r) assert ri.shape[0] == r.shape[1] assert ri.shape[1] == r.shape[0] assert ri.dtype == r.dtype # Note that pseudoinverse can be quite unprecise so I prefer to compare # the result with what np.linalg returns assert _allclose(ri, np.linalg.pinv(r))
def test_empty_elemwise(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = theano.shared(numpy.random.rand(0, 6).astype(config.floatX), 'x') # should work z = (x + 2) * 3 assert hasattr(z.tag, 'test_value') f = theano.function([], z) assert _allclose(f(), z.tag.test_value) finally: theano.config.compute_test_value = orig_compute_test_value
def test_matrix_dot(): rng = np.random.RandomState(utt.fetch_seed()) n = rng.randint(4) + 2 rs = [] xs = [] for k in xrange(n): rs += [rng.randn(4, 4).astype(theano.config.floatX)] xs += [tensor.matrix()] sol = matrix_dot(*xs) theano_sol = function(xs, sol)(*rs) numpy_sol = rs[0] for r in rs[1:]: numpy_sol = np.dot(numpy_sol, r) assert _allclose(numpy_sol, theano_sol)
def test_variable_only(self): x = tt.matrix("x") x.tag.test_value = np.random.rand(3, 4).astype(config.floatX) y = tt.matrix("y") y.tag.test_value = np.random.rand(4, 5).astype(config.floatX) # should work z = tt.dot(x, y) assert hasattr(z.tag, "test_value") f = theano.function([x, y], z) assert _allclose(f(x.tag.test_value, y.tag.test_value), z.tag.test_value) # this test should fail y.tag.test_value = np.random.rand(6, 5).astype(config.floatX) with pytest.raises(ValueError): tt.dot(x, y)
def test_csc_dense(self): x = theano.sparse.csc_matrix("x") y = theano.tensor.matrix("y") f_a = theano.function([x, y], theano.sparse.dot(x, y)) f_b = lambda x, y: x * y assert _allclose(f_a(self.x_csc, self.y), f_b(self.x_csc, self.y)) # Test infer_shape f_a = theano.function([x, y], theano.sparse.dot(x, y).shape) f_b = lambda x, y: (x * y).shape assert numpy.all(f_a(self.x_csc, self.y) == f_b(self.x_csc, self.y)) topo = f_a.maker.env.toposort() if theano.config.mode != "FAST_COMPILE": nb = 0 else: nb = 1 assert sum([isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo]) == nb
def test_numpy_compare(self): rng = np.random.RandomState(utt.fetch_seed()) M = tensor.matrix("A", dtype=theano.config.floatX) V = tensor.vector("V", dtype=theano.config.floatX) a = rng.rand(4, 4).astype(theano.config.floatX) b = rng.rand(4).astype(theano.config.floatX) A = ( [None, 'fro', 'inf', '-inf', 1, -1, None, 'inf', '-inf', 0, 1, -1, 2, -2], [M, M, M, M, M, M, V, V, V, V, V, V, V, V], [a, a, a, a, a, a, b, b, b, b, b, b, b, b], [None, 'fro', inf, -inf, 1, -1, None, inf, -inf, 0, 1, -1, 2, -2]) for i in range(0, 14): f = function([A[1][i]], norm(A[1][i], A[0][i])) t_n = f(A[2][i]) n_n = np.linalg.norm(A[2][i], A[3][i]) assert _allclose(n_n, t_n)
def test_constant(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.constant(np.random.rand(2, 3), dtype=config.floatX) y = theano.shared(np.random.rand(3, 6).astype(config.floatX), 'y') # should work z = T.dot(x, y) assert hasattr(z.tag, 'test_value') f = theano.function([], z) assert _allclose(f(), z.tag.test_value) # this test should fail x = T.constant(np.random.rand(2, 4), dtype=config.floatX) self.assertRaises(ValueError, T.dot, x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_constant(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.constant(numpy.random.rand(2,3), dtype=config.floatX) y = theano.shared(numpy.random.rand(3,6).astype(config.floatX), 'y') # should work z = T.dot(x,y) assert hasattr(z.tag, 'test_value') f = theano.function([], z) assert _allclose(f(), z.tag.test_value) # this test should fail x = T.constant(numpy.random.rand(2,4), dtype=config.floatX) self.assertRaises(ValueError, T.dot, x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_sdae(): """ Tests that StackedDenoisingAutoencoder works correctly """ data = np.random.randn(10, 5).astype(config.floatX) * 100 ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=False) corruptor = BinomialCorruptor(corruption_level=0.5) model = StackedDenoisingAutoencoder([ae], corruptor) model._ensure_extensions() w = ae.weights.get_value() w_prime = ae.w_prime.get_value() ae.hidbias.set_value(np.random.randn(7).astype(config.floatX)) hb = ae.hidbias.get_value() ae.visbias.set_value(np.random.randn(5).astype(config.floatX)) vb = ae.visbias.get_value() d = tensor.matrix() result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb) ff = theano.function([d], model.reconstruct(d)) assert not _allclose(ff(data), result)
def test_constant(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = "raise" x = T.constant(np.random.rand(2, 3), dtype=config.floatX) y = theano.shared(np.random.rand(3, 6).astype(config.floatX), "y") # should work z = T.dot(x, y) assert hasattr(z.tag, "test_value") f = theano.function([], z) assert _allclose(f(), z.tag.test_value) # this test should fail x = T.constant(np.random.rand(2, 4), dtype=config.floatX) with pytest.raises(ValueError): T.dot(x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_sdae(): """ Tests that StackedDenoisingAutoencoder works correctly """ data = np.random.randn(10, 5).astype(config.floatX) * 100 ae = Autoencoder(5, 7, act_enc="tanh", act_dec="cos", tied_weights=False) corruptor = BinomialCorruptor(corruption_level=0.5) model = StackedDenoisingAutoencoder([ae], corruptor) model._ensure_extensions() w = ae.weights.get_value() w_prime = ae.w_prime.get_value() ae.hidbias.set_value(np.random.randn(7).astype(config.floatX)) hb = ae.hidbias.get_value() ae.visbias.set_value(np.random.randn(5).astype(config.floatX)) vb = ae.visbias.get_value() d = tensor.matrix() result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb) ff = theano.function([d], model.reconstruct(d)) assert not _allclose(ff(data), result)
def test_shared(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.matrix('x') x.tag.test_value = np.random.rand(3, 4).astype(config.floatX) y = theano.shared(np.random.rand(4, 6).astype(config.floatX), 'y') # should work z = T.dot(x, y) assert hasattr(z.tag, 'test_value') f = theano.function([x], z) assert _allclose(f(x.tag.test_value), z.tag.test_value) # this test should fail y.set_value(np.random.rand(5, 6).astype(config.floatX)) self.assertRaises(ValueError, T.dot, x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_shared(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.matrix('x') x.tag.test_value = numpy.random.rand(3,4).astype(config.floatX) y = theano.shared(numpy.random.rand(4,6).astype(config.floatX), 'y') # should work z = T.dot(x,y) assert hasattr(z.tag, 'test_value') f = theano.function([x], z) assert _allclose(f(x.tag.test_value), z.tag.test_value) # this test should fail y.set_value(numpy.random.rand(5,6).astype(config.floatX)) self.assertRaises(ValueError, T.dot, x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_csc_dense(self): x = theano.sparse.csc_matrix('x') y = theano.tensor.matrix('y') f_a = theano.function([x, y], theano.sparse.dot(x, y)) f_b = lambda x, y: x * y assert _allclose(f_a(self.x_csc, self.y), f_b(self.x_csc, self.y)) # Test infer_shape f_a = theano.function([x, y], theano.sparse.dot(x, y).shape) f_b = lambda x, y: (x * y).shape assert numpy.all(f_a(self.x_csc, self.y) == f_b(self.x_csc, self.y)) topo = f_a.maker.env.toposort() if theano.config.mode != 'FAST_COMPILE': nb = 0 else: nb = 1 assert sum([ isinstance(node.op, (Dot, Usmm, UsmmCscDense)) for node in topo ]) == nb
def test_variable_only(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.matrix('x') x.tag.test_value = numpy.random.rand(3,4).astype(config.floatX) y = T.matrix('y') y.tag.test_value = numpy.random.rand(4,5).astype(config.floatX) # should work z = T.dot(x,y) assert hasattr(z.tag, 'test_value') f = theano.function([x,y], z) assert _allclose(f(x.tag.test_value, y.tag.test_value), z.tag.test_value) # this test should fail y.tag.test_value = numpy.random.rand(6,5).astype(config.floatX) self.assertRaises(ValueError, T.dot, x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_variable_only(self): orig_compute_test_value = theano.config.compute_test_value try: theano.config.compute_test_value = 'raise' x = T.matrix('x') x.tag.test_value = numpy.random.rand(3, 4).astype(config.floatX) y = T.matrix('y') y.tag.test_value = numpy.random.rand(4, 5).astype(config.floatX) # should work z = T.dot(x, y) assert hasattr(z.tag, 'test_value') f = theano.function([x, y], z) assert _allclose(f(x.tag.test_value, y.tag.test_value), z.tag.test_value) # this test should fail y.tag.test_value = numpy.random.rand(6, 5).astype(config.floatX) self.assertRaises(ValueError, T.dot, x, y) finally: theano.config.compute_test_value = orig_compute_test_value
def test_sum(): """ test sum pattern 1, 11, 10, 01, 001, 010, 100, 110, 011, 111, 0011, 0101, 0111, 1011, 1111 test sum pattern implemented with reshape: 1000, 0100, 0010, 0001, 11111 others implemented by reshape that are not tested 0011,0101,0110,1001,1010,1100 1110,1101,1011 TODO: test with broadcast """ for shape, pattern in [((100,3,1300),[1]), ((0,),[0]),((5,),[0]), ((0,0),[0,1]),((1,0),[0,1]),((5,4),[0,1]),((33,31),[0,1]),((5,4),[1]),((5,4),[0]),#need something bigger then 32 for some opt test. ((5,4,3),[0]),((5,4,3),[1]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[1,2]),((5,4,3),[0,1,2]), ((0,0,0,0),[0,1,2,3]), ((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]), ((5,4,3,10,11),[1,2]), ((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]), #test shape bigger then 4096 on each dimension to make sure that we work correctly when we don't have enought thread/block in each dimensions ((4100,3),[0]),((3,4101),[0]),#10 ((1024,33),[0]),((33,1024),[0]),#10 ((1025,33),[0]),((33,1025),[0]),#10 ((4100,3),[1]),((3,4101),[1]),#01 ((1024,33),[1]),((33,1024),[1]),#01 ((1025,33),[1]),((33,1025),[1]),#01 ((4100,3),[0,1]),((3,4101),[0,1]),#11 ((1024,33),[0,1]),((33,1024),[0,1]),#01 ((1025,33),[0,1]),((33,1025),[0,1]),#01 ((4100,4,3),[0]),((5,4100,3),[0]),((5,4,4100),[0]),#100 ((4100,4,3),[1]),((5,4100,3),[1]),((5,4,4100),[1]),#010 ((4100,4,3),[2]),((5,4100,3),[2]),((5,4,4100),[2]),#001 ((4100,4,3),[0,1]),((5,4100,3),[0,1]),((5,4,4100),[0,1]),#110 ((4100,4,3),[1,2]),((5,4100,3),[1,2]),((5,4,4100),[1,2]),#011 #((4100,4,3),[0,2]),((5,4100,3),[0,2]),((5,4,4100),[0,2]),#101 ##not implemented ((4100,4,3),[0,1,2]),((5,4100,3),[0,1,2]),((5,4,4100),[0,1,2]),#111 ((4100,4,3,2),[2,3]),((4,4100,3,2),[2,3]),((4,3,4100,2),[2,3]),((4,3,2,4100),[2,3]),#0011 ((4100,4,3,2),[1,3]),((4,4100,3,2),[1,3]),((4,3,4100,2),[1,3]),((4,3,2,4100),[1,3]),#0101 ((4100,4,3,2),[0,2,3]),((4,4100,3,2),[0,2,3]),((4,3,4100,2),[0,2,3]),#((4,3,2,4100),[0,2,3]),#1011 ((4100,4,3,2),[1,2,3]),((4,4100,3,2),[1,2,3]),((4,3,4100,2),[1,2,3]),((4,3,2,4100),[1,2,3]),#0111 ((4100,2,3,4),[0,1,2,3]),((2,4100,3,4),[0,1,2,3]),((2,3,4100,4),[0,1,2,3]),((2,3,4,4100),[0,1,2,3]),#1111 #test pattern implemented by reshape ((4100,4,3,2),[0]),((4,4100,3,2),[0]),((4,3,4100,2),[0]),((4,3,2,4100),[0]),#1000 ((4100,4,3,2),[1]),((4,4100,3,2),[1]),((4,3,4100,2),[1]),((4,3,2,4100),[1]),#0100 ((4100,4,3,2),[2]),((4,4100,3,2),[2]),((4,3,4100,2),[2]),((4,3,2,4100),[2]),#0010 ((4100,4,3,2),[3]),((4,4100,3,2),[3]),((4,3,4100,2),[3]),((4,3,2,4100),[3]),#0001 ((1100,2,3,4,5),[0,1,2,3,4]),((2,1100,3,4,5),[0,1,2,3,4]),((2,3,1100,4,5),[0,1,2,3,4]),((2,3,4,1100,5),[0,1,2,3,4]),((2,3,4,5,1100),[0,1,2,3,4]),#11111 ]: a = tensor.TensorType('float32', (False,) * len(shape))() b = T.Sum(pattern)(a) val = numpy.random.rand(numpy.prod(shape)).reshape(shape) # val = numpy.ones(shape) # val = numpy.arange(numpy.prod(shape)).reshape(shape) val = theano._asarray(val, dtype='float32') f = theano.function([a], b, mode=mode_with_gpu) f2 = theano.function([a], b, mode=mode_without_gpu) assert tcn.GpuSum in [x.op.__class__ for x in f.maker.env.toposort()] assert T.Sum in [x.op.__class__ for x in f2.maker.env.toposort()] if val.size == 0: assert f2(val) == f(val), ('shape', shape, 'pattern', pattern) else: try: #We raise the error threashold as we sum big matrix #and this cause small rounding difference with some seed #example in debug mode with unittests.rseed=9275 orig_rtol = theano.tensor.basic.float32_rtol theano.tensor.basic.float32_rtol = 2e-5 assert _allclose(f2(val), f(val)), ('shape', shape, 'pattern', pattern, sum([shape[i] for i in pattern]), f2(val), f(val), val) finally: theano.tensor.basic.float32_rtol = orig_rtol #test with dimshuffle #we shuffle the 2 outer dims. for shape, pattern in [#((5,),[0]), ((5,4),[0,1]),((5,4),[0]), ((5,4,3),[0]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[0,1,2]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3])]: a = tensor.TensorType('float32', (False,) * len(shape))() dim_pattern = range(len(shape)) dim_pattern[0] = 1 dim_pattern[1] = 0 a = a.dimshuffle(dim_pattern) b = T.Sum(pattern)(a) val = numpy.random.rand(numpy.prod(shape)).reshape(shape) # val = numpy.ones(shape) # val = numpy.arange(numpy.prod(shape)).reshape(shape) val = theano._asarray(val, dtype='float32') f = theano.function([a], b, mode=mode_with_gpu) f2 = theano.function([a], b, mode=mode_without_gpu) assert tcn.GpuSum in [x.op.__class__ for x in f.maker.env.toposort()] assert T.Sum in [x.op.__class__ for x in f2.maker.env.toposort()] assert _allclose(f2(val), f(val)), ('shape', shape, 'pattern', pattern, sum([shape[i] for i in pattern])) #test with broadcast for shape, pattern in [((5,),[0]), ((5,4),[0,1]),((5,4),[0]), ((5,4,3),[0]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[0,1,2]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3])]: shape = numpy.asarray(shape) * 2 a = tensor.TensorType('float32', (False,) * len(shape))() a2 = tcn.CudaNdarrayType((False,) * len(shape))() b = T.Sum(pattern)(a) b2 = T.Sum(pattern)(a2) val = numpy.random.rand(numpy.prod(shape)).reshape(shape) # val = numpy.ones(shape) # val = numpy.arange(numpy.prod(shape)).reshape(shape) val = theano._asarray(val, dtype='float32') val2 = cuda.CudaNdarray(val) if len(shape) == 1: val = val[::2] val2 = val2[::2] elif len(shape) == 2: val = val[::2, ::2] val2 = val2[::2, ::2] elif len(shape) == 3: val = val[::2, ::2, ::2] val2 = val2[::2, ::2, ::2] elif len(shape) == 4: val = val[::2, ::2, ::2, ::2] val2 = val2[::2, ::2, ::2, ::2] f = theano.function([a], b, mode=mode_without_gpu) f2 = theano.function([a2], b2, mode=mode_with_gpu) assert tcn.GpuSum in [x.op.__class__ for x in f2.maker.env.toposort()] assert T.Sum in [x.op.__class__ for x in f.maker.env.toposort()] assert _allclose(f2(val2), f(val)), ('shape', shape, 'pattern', pattern, sum([shape[i] for i in pattern]))
def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1), N_image_shape=None, N_filter_shape=None, input=None, filters=None, unroll_batch=None, unroll_kern=None, unroll_patch=None, verify_grad=True, should_raise=False): """ :param image_shape: The constant shape info passed to conv2d. :param filter_shape: The constant shape info passed to conv2d. :param N_image_shape: None(default to image_shape) or tuple of 4 elements with the shape of the input image :param N_filter_shape: None(default to filter_shape) or tuple of 4 elements with the shape of the input filter """ if N_image_shape is None: N_image_shape = [T.get_scalar_constant_value(T. as_tensor_variable(x)) for x in image_shape] if N_filter_shape is None: N_filter_shape = [T.get_scalar_constant_value(T. as_tensor_variable(x)) for x in filter_shape] if input is None: input = self.input if not filters: filters = self.filters ############# THEANO IMPLEMENTATION ############ # we create a symbolic function so that verify_grad can work def sym_conv2d(input, filters): # define theano graph and function input.name = 'input' filters.name = 'filters' rval = conv.conv2d(input, filters, image_shape, filter_shape, border_mode, subsample, unroll_batch=unroll_batch, unroll_kern=unroll_kern, unroll_patch=unroll_patch) rval.name = 'conv_output' return rval output = sym_conv2d(input, filters) output.name = 'conv2d(%s,%s)' % (input.name, filters.name) theano_conv = theano.function([input, filters], output) # initialize input and compute result image_data = numpy.random.random(N_image_shape) filter_data = numpy.random.random(N_filter_shape) try: theano_output = theano_conv(image_data, filter_data) except ValueError: if not should_raise: raise return else: if should_raise: raise Exception( "ConvOp should have generated an error") ############# REFERENCE IMPLEMENTATION ############ s = 1. orig_image_data = image_data if border_mode is not 'full': s = -1. out_shape2d = numpy.array(N_image_shape[-2:]) +\ s * numpy.array(N_filter_shape[-2:]) - s out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample)) out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d) ref_output = numpy.zeros(out_shape) # loop over output feature maps ref_output.fill(0) if border_mode == 'full': image_data2 = numpy.zeros((N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * N_filter_shape[2] - 2, N_image_shape[3] + 2 * N_filter_shape[3] - 2)) image_data2[:, :, N_filter_shape[2] - 1:N_filter_shape[2] - 1 + N_image_shape[2], N_filter_shape[3] - 1:N_filter_shape[3] - 1 + N_image_shape[3]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter2d = filter_data[nn, im0, :, :] image2d = image_data[bb, im0, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] # image row for col in range(ref_output.shape[3]): icol = col * subsample[1] # image col ref_output[bb, nn, row, col] += (image2d[ irow:irow + N_filter_shape[2], icol:icol + N_filter_shape[3]] * filter2d[::-1,::-1] ).sum() self.assertTrue(_allclose(theano_output, ref_output)) ############# TEST GRADIENT ############ if verify_grad: utt.verify_grad(sym_conv2d, [orig_image_data, filter_data])
def test_careduce(): """ test sum pattern 1, 11, 10, 01, 001, 010, 100, 110, 011, 111, 0011, 0101, 0111, 1011, 1111 test sum pattern implemented with reshape: 1000, 0100, 0010, 0001, 11111 others implemented by reshape that are not tested 0011,0101,0110,1001,1010,1100 1110,1101,1011 TODO: test with broadcast """ for scalar_op in [theano.scalar.add, theano.scalar.maximum]: for shape, pattern in [((1,1),(1,)), ((1,0),(1,)), ((0,1),(1,)), ((0,0),(1,)), ((0,0,0),(1,2)), ((0,0,0,0),(1,2,3)), ((2,1),(1,)), ((1,2),(1,)), ((100,3,1300),[1]), ((0,),[0]),((5,),[0]), ((0,0),[0,1]),((1,0),[0,1]),((5,4),[0,1]),((33,31),[0,1]),((5,4),[1]),((5,4),[0]),#need something bigger then 32 for some opt test. ((5,4,3),[0]),((5,4,3),[1]),((5,4,3),[0,1]),((5,4,3),[2]),((5,4,3),[1,2]),((5,4,3),[0,1,2]), ((0,0,0,0),[0,1,2,3]), ((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]), ((5,4,3,10,11),[1,2]), ((5,4,3,20),[2,3]), ((5,4,3,2),[0,1,2,3]), ((5,4,3,2),[0,2,3]),((5,4,3,2),[1,2,3]), #test shape bigger then 4096 on each dimension to make sure that we work correctly when we don't have enough thread/block in each dimensions ((4100,3),[0]),((3,4101),[0]),#10 ((1024,33),[0]),((33,1024),[0]),#10 ((1025,33),[0]),((33,1025),[0]),#10 ((4100,3),[1]),((3,4101),[1]),#01 ((1024,33),[1]),((33,1024),[1]),#01 ((1025,33),[1]),((33,1025),[1]),#01 ((4100,3),[0,1]),((3,4101),[0,1]),#11 ((1024,33),[0,1]),((33,1024),[0,1]),#01 ((1025,33),[0,1]),((33,1025),[0,1]),#01 ((4100,4,3),[0]),((5,4100,3),[0]),((5,4,4100),[0]), ((3,65536,1), [0]),#100 ((4100,4,3),[1]),((5,4100,3),[1]),((5,4,4100),[1]),#010 ((4100,4,3),[2]),((5,4100,3),[2]),((5,4,4100),[2]),#001 ((4100,4,3),[0,1]),((5,4100,3),[0,1]),((5,4,4100),[0,1]),#110 ((4100,4,3),[1,2]),((5,4100,3),[1,2]),((5,4,4100),[1,2]),#011 #((4100,4,3),[0,2]),((5,4100,3),[0,2]),((5,4,4100),[0,2]),#101 ##not implemented ((4100,4,3),[0,1,2]),((5,4100,3),[0,1,2]),((5,4,4100),[0,1,2]),#111 ((4100,4,3,2),[2,3]),((4,4100,3,2),[2,3]),((4,3,4100,2),[2,3]),((4,3,2,4100),[2,3]),#0011 ((4100,4,3,2),[1,3]),((4,4100,3,2),[1,3]),((4,3,4100,2),[1,3]),((4,3,2,4100),[1,3]),#0101 ((4100,4,3,2),[0,2,3]),((4,4100,3,2),[0,2,3]),((4,3,4100,2),[0,2,3]),#((4,3,2,4100),[0,2,3]),#1011 ((4100,4,3,2),[1,2,3]),((4,4100,3,2),[1,2,3]),((4,3,4100,2),[1,2,3]),((4,3,2,4100),[1,2,3]),#0111 ((4100,2,3,4),[0,1,2,3]),((2,4100,3,4),[0,1,2,3]),((2,3,4100,4),[0,1,2,3]),((2,3,4,4100),[0,1,2,3]),#1111 #test pattern implemented by reshape ((4100,4,3,2),[0]),((4,4100,3,2),[0]),((4,3,4100,2),[0]),((4,3,2,4100),[0]),#1000 ((4100,4,3,2),[1]),((4,4100,3,2),[1]),((4,3,4100,2),[1]),((4,3,2,4100),[1]),#0100 ((4100,4,3,2),[2]),((4,4100,3,2),[2]),((4,3,4100,2),[2]),((4,3,2,4100),[2]),#0010 ((4100,4,3,2),[3]),((4,4100,3,2),[3]),((4,3,4100,2),[3]),((4,3,2,4100),[3]),#0001 ((1100,2,3,4,5),[0,1,2,3,4]),((2,1100,3,4,5),[0,1,2,3,4]),((2,3,1100,4,5),[0,1,2,3,4]),((2,3,4,1100,5),[0,1,2,3,4]),((2,3,4,5,1100),[0,1,2,3,4]),#11111 ]: op = tensor.CAReduce(scalar_op, axis=pattern) pat = tensor_pattern_to_gpu_pattern(shape, pattern) #GpuCAReduce{maximum} support only those patterns if scalar_op is theano.scalar.maximum and pat not in [ (0, 1), (0, 1, 1), (0, 1, 1)]: continue a = tensor.TensorType('float32', (False,) * len(shape))() b = op(a) val = numpy.random.rand(numpy.prod(shape)).reshape(shape) # val = numpy.ones(shape) # val = numpy.arange(numpy.prod(shape)).reshape(shape) val = theano._asarray(val, dtype='float32') f = theano.function([a], b, mode=mode_with_gpu) f2 = theano.function([a], b, mode=mode_without_gpu) assert tcn.GpuCAReduce in [x.op.__class__ for x in f.maker.fgraph.toposort()] assert op.__class__ in [x.op.__class__ for x in f2.maker.fgraph.toposort()] f_caused_value_error = False try: f_out = f(val) except ValueError, e: exc = e f_caused_value_error = True f2_caused_value_error = False try: f2_out = f2(val) except ValueError, e: exc2 = e f2_caused_value_error = True if f_caused_value_error != f2_caused_value_error: if f_caused_value_error: print 'f caused this value error:' print exc else: print 'f did not raise a value error, but should have' if f2_caused_value_error: print 'f2 caused this value error:' print exc2 else: print 'f should not have raised a value error' print 'shape was: ', shape print 'pattern was: ', pattern assert False try: #We raise the error threashold as we sum big matrix #and this cause small rounding difference with some seed #example in debug mode with unittests.rseed=9275 orig_rtol = theano.tensor.basic.float32_rtol theano.tensor.basic.float32_rtol = 2e-5 assert _allclose(f_out, f2_out), ('shape', shape, 'pattern', pattern, sum([shape[i] for i in pattern]), f2(val), f(val), val) finally: theano.tensor.basic.float32_rtol = orig_rtol
def test(self): def mat(format, name, dtype): if format == "dense": return theano.tensor.matrix(name, dtype=dtype) else: return theano.sparse.matrix(format, name, dtype=dtype) params = product(*([["float32", "float64", "int16", "complex64"]] * 4 + [["dense", "csc", "csr"]] * 2)) # All test are too slow, so we randomly take 100 of them. # The buildbot change the seed, so we will finish by running them all. # As of this writing they where all passing. # params = self.rng.permutation(list(params))[:500] for dtype1, dtype2, dtype3, dtype4, format1, format2 in params: if format1 == "dense" and format2 == "dense": # Usmm won't be used! continue x = mat(format1, "x", dtype1) y = mat(format2, "y", dtype2) a = theano.tensor.scalar("a", dtype=dtype3) z = theano.shared(numpy.asarray(self.z, dtype=dtype4).copy()) f_b = lambda z, a, x, y: z - a * (x * y) x_data = numpy.asarray(self.x, dtype=dtype1) if format1 != "dense": x_data = as_sparse_format(x_data, format1) y_data = numpy.asarray(self.y, dtype=dtype2) if format2 != "dense": y_data = as_sparse_format(y_data, format2) a_data = numpy.asarray(1.5, dtype=dtype3) z_data = numpy.asarray(self.z, dtype=dtype4) f_b_out = f_b(z_data, a_data, x_data, y_data) # Can it work inplace? inplace = dtype4 == theano.scalar.upcast(dtype1, dtype2, dtype3) # To make it easier to check the toposort mode = theano.compile.mode.get_default_mode().excluding("fusion") if inplace: updates = {z: z - a * theano.sparse.dot(x, y)} f_a = theano.function([a, x, y], [], updates=updates, mode=mode) f_a(a_data, x_data, y_data) f_a_out = z.get_value(borrow=True) else: f_a = theano.function([a, x, y], z - a * theano.sparse.dot(x, y), mode=mode) # In DebugMode there is a strange difference with complex # So we raise a little the threshold a little. try: orig_atol = theano.tensor.basic.float64_atol orig_rtol = theano.tensor.basic.float64_rtol theano.tensor.basic.float64_atol = 1e-7 theano.tensor.basic.float64_rtol = 1e-6 f_a_out = f_a(a_data, x_data, y_data) finally: theano.tensor.basic.float64_atol = orig_atol theano.tensor.basic.float64_rtol = orig_rtol assert _allclose(f_a_out, f_b_out, rtol=1e-5) topo = f_a.maker.env.toposort() up = theano.scalar.upcast(dtype1, dtype2, dtype3, dtype4) fast_compile = theano.config.mode == "FAST_COMPILE" if (y.type.dtype == up and format1 == "csc" and format2 == "dense" and not fast_compile) and up in ( "float32", "float64", ): # The op UsmmCscDense should be inserted assert ( sum( [ isinstance(node.op, tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Cast) for node in topo ] ) == len(topo) - 5 ) new_topo = [] for node in topo: if not ( isinstance(node.op, tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Cast) ): new_topo.append(node) topo = new_topo assert len(topo) == 5, topo # Usmm is tested at the same time in debugmode # Check if the optimization local_usmm and local_usmm_csx is # applied assert isinstance(topo[0].op, theano.sparse.basic.CSMProperties) assert isinstance(topo[1].op, theano.tensor.DimShuffle) assert isinstance(topo[2].op, theano.tensor.Subtensor) assert topo[3].op == theano.tensor.neg assert isinstance(topo[4].op, theano.sparse.UsmmCscDense) if inplace: assert topo[4].op.inplace elif not fast_compile: # The op Usmm should be inserted assert len(topo) == 3, topo assert isinstance(topo[0].op, theano.tensor.DimShuffle) assert topo[1].op == theano.tensor.neg assert isinstance(topo[2].op, theano.sparse.Usmm)
def validate(self, image_shape, filter_shape, border_mode='valid', subsample=(1, 1), input=None, filters=None, verify_grad=True, non_contiguous=False, filter_dilation=(1, 1)): """ :param image_shape: The constant shape info passed to corrMM. :param filter_shape: The constant shape info passed to corrMM. """ N_image_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x)) for x in image_shape] N_filter_shape = [T.get_scalar_constant_value(T.as_tensor_variable(x)) for x in filter_shape] if input is None: input = self.input if filters is None: filters = self.filters # THEANO IMPLEMENTATION # we create a symbolic function so that verify_grad can work def sym_CorrMM(input, filters): # define theano graph and function input.name = 'input' filters.name = 'filters' rval = corr.CorrMM(border_mode, subsample, filter_dilation)(input, filters) rval.name = 'corr_output' return rval output = sym_CorrMM(input, filters) output.name = 'CorrMM()(%s,%s)' % (input.name, filters.name) theano_corr = theano.function([input, filters], output, mode=self.mode) # initialize input and compute result image_data = numpy.random.random(N_image_shape).astype(self.dtype) filter_data = numpy.random.random(N_filter_shape).astype(self.dtype) if non_contiguous: image_data = numpy.transpose(image_data, axes=(0, 1, 3, 2)) image_data = image_data.copy() image_data = numpy.transpose(image_data, axes=(0, 1, 3, 2)) filter_data = numpy.transpose(filter_data, axes=(0, 1, 3, 2)) filter_data = filter_data.copy() filter_data = numpy.transpose(filter_data, axes=(0, 1, 3, 2)) assert not image_data.flags['CONTIGUOUS'] assert not filter_data.flags['CONTIGUOUS'] theano_output = theano_corr(image_data, filter_data) # REFERENCE IMPLEMENTATION # Testing correlation, not convolution. Reverse filters. filter_data_corr = numpy.array(filter_data[:, :, ::-1, ::-1], copy=True, order='C') orig_image_data = image_data img_shape2d = numpy.array(N_image_shape[-2:]) fil_shape2d = numpy.array(N_filter_shape[-2:]) dil_shape2d = numpy.array(filter_dilation) dil_fil_shape2d = (fil_shape2d - 1) * dil_shape2d + 1 subsample2d = numpy.array(subsample) if border_mode == 'full': padHW = (dil_fil_shape2d - 1) elif border_mode == 'valid': padHW = numpy.array([0, 0]) elif border_mode == 'half': padHW = numpy.floor(dil_fil_shape2d / 2).astype('int32') elif isinstance(border_mode, tuple): padHW = numpy.array(border_mode) elif isinstance(border_mode, integer_types): padHW = numpy.array([border_mode, border_mode]) else: raise NotImplementedError('Unsupported border_mode {}'.format(border_mode)) out_shape2d = numpy.floor((img_shape2d + 2 * (padHW) - dil_fil_shape2d) / subsample2d) + 1 # avoid numpy deprecation out_shape2d = out_shape2d.astype('int32') out_shape = (N_image_shape[0], N_filter_shape[0]) + tuple(out_shape2d) ref_output = numpy.zeros(out_shape) # loop over output feature maps ref_output.fill(0) image_data2 = numpy.zeros((N_image_shape[0], N_image_shape[1], N_image_shape[2] + 2 * padHW[0], N_image_shape[3] + 2 * padHW[1])) image_data2[:, :, padHW[0]:padHW[0] + N_image_shape[2], padHW[1]:padHW[1] + N_image_shape[3]] = image_data image_data = image_data2 N_image_shape = image_data.shape for bb in range(N_image_shape[0]): for nn in range(N_filter_shape[0]): for im0 in range(N_image_shape[1]): filter2d = filter_data_corr[nn, im0, :, :] image2d = image_data[bb, im0, :, :] for row in range(ref_output.shape[2]): irow = row * subsample[0] # image row for col in range(ref_output.shape[3]): icol = col * subsample[1] # image col ref_output[bb, nn, row, col] += (image2d[ irow:irow + dil_fil_shape2d[0]:filter_dilation[0], icol:icol + dil_fil_shape2d[1]:filter_dilation[1]] * filter2d[::-1, ::-1] ).sum() self.assertTrue(_allclose(theano_output, ref_output)) # TEST GRADIENT if verify_grad: utt.verify_grad(sym_CorrMM, [orig_image_data, filter_data], mode=self.mode)