def test_sparse_shared_memory(): # Note : There are no inplace ops on sparse matrix yet. If one is # someday implemented, we could test it here. a = random_lil((3, 4), 'float32', 3).tocsr() m1 = random_lil((4, 4), 'float32', 3).tocsr() m2 = random_lil((4, 4), 'float32', 3).tocsr() x = SparseType('csr', dtype='float32')() y = SparseType('csr', dtype='float32')() sdot = theano.sparse.structured_dot z = sdot(x * 3, m1) + sdot(y * 2, m2) f = theano.function( [theano.In(x, mutable=True), theano.In(y, mutable=True)], z, mode='FAST_RUN') def f_(x, y, m1=m1, m2=m2): return numpy.dot(x * 3, m1) + numpy.dot(y * 2, m2) assert SparseType.may_share_memory(a, a) # This is trivial result = f(a, a) result_ = f_(a, a) assert (result_.todense() == result.todense()).all()
def test_may_share_memory(): a = scipy.sparse.csc_matrix(scipy.sparse.eye(5, 3)) b = scipy.sparse.csc_matrix(scipy.sparse.eye(4, 3)) as_ar = lambda a: theano._asarray(a, dtype="int32") for a_, b_, rep in [ (a, a, True), (b, b, True), (a, b, False), (a, a.data, True), (a, a.indptr, True), (a, a.indices, True), (a, as_ar(a.shape), False), (a.data, a, True), (a.indptr, a, True), (a.indices, a, True), (as_ar(a.shape), a, False), (b, b.data, True), (b, b.indptr, True), (b, b.indices, True), (b, as_ar(b.shape), False), (b.data, b, True), (b.indptr, b, True), (b.indices, b, True), (as_ar(b.shape), b, False), (b.data, a, False), (b.indptr, a, False), (b.indices, a, False), (as_ar(b.shape), a, False), ]: assert SparseType.may_share_memory(a_, b_) == rep
def test_graph_bprop_rand(self): for i in range(10): xorig = numpy.random.rand(3, 2) for mtype in _mtypes: x = tensor.matrix('x') w = SparseType(dtype='float64', format=_mtype_to_str[mtype]).make_variable() xw = dense_from_sparse(true_dot(w, x)) y = dense_from_sparse(true_dot(w.T, xw)) diff = x - y loss = tensor.sum(tensor.sqr(diff)) gw = tensor.grad(loss, w) trainfn = compile.function([x, w], [y, loss, gw]) x = xorig w = mtype((500, 3)) w[(10, 1)] = 1 w[(20, 2)] = 2 lr = 0.001 y, origloss, gw = trainfn(x, w) for epoch in xrange(50): y, loss, gw = trainfn(x, w) w = w - (lr * gw) self.assertTrue(origloss > loss)
def test_graph_bprop0(self): for mtype in _mtypes: x = tensor.matrix( 'x' ) #TensorType('float64', broadcastable=[False,False], name='x') w = SparseType(dtype='float64', format=_mtype_to_str[mtype]).make_variable() xw = dense_from_sparse(true_dot(w, x)) y = dense_from_sparse(true_dot(w.T, xw)) diff = x - y loss = tensor.sum(tensor.sqr(diff)) gw = tensor.grad(loss, w) trainfn = compile.function([x, w], [y, loss, gw]) x = numpy.asarray([[1., 2], [3, 4], [2, 1]]) w = mtype((500, 3)) w[(10, 1)] = 1 w[(20, 2)] = 2 lr = 0.001 y, origloss, gw = trainfn(x, w) for epoch in xrange(50): y, loss, gw = trainfn(x, w) w = w - (lr * gw) print loss self.assertTrue(origloss > loss) self.assertTrue('1.05191241115' == str(loss))
def function(self, name=None, repr_index=-1, sparse_input=False): """ Compile a function computing representations on given layers. Parameters ---------- name : string, optional name of the function repr_index : int, optional Index of the hidden representation to return. 0 means the input, -1 the last output. sparse_input : bool, optional WRITEME Returns ------- WRITEME """ if sparse_input: inputs = SparseType('csr', dtype=theano.config.floatX)() else: inputs = tensor.matrix() return theano.function([inputs], outputs=self(inputs)[repr_index], name=name)
def test_upcast(self): typenames = ('float32', 'int64', 'int8', 'int32', 'int16', 'float64', 'complex64', 'complex128') for dense_dtype in typenames: for sparse_dtype in typenames: correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype) a = SparseType('csc', dtype=sparse_dtype)() b = tensor.matrix(dtype=dense_dtype) d = structured_dot(a, b) assert d.type.dtype == correct_dtype # compile and run a function f = theano.function([a, b], d) M, N, K, nnz = (4, 3, 5, 3) spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz)) # the following madness is necessary to workaround # an intc vs. int32 bug. # The lil makes an intc on my computer when sparse_dtype # is int32. spmat.dtype = numpy.dtype(sparse_dtype) mat = numpy.asarray(numpy.random.randn(N, K) * 9, dtype=dense_dtype) print 'DTYPES', sparse_dtype, dense_dtype print 'sym types', a.type, b.type print 'dtype strings', spmat.dtype, mat.dtype print 'numpy dtype num', mat.dtype.num print 'scipy dtype num', spmat.data.dtype.num theano_result = f(spmat, mat) scipy_result = spmat * mat assert theano_result.shape == scipy_result.shape assert theano_result.dtype == scipy_result.dtype assert _allclose(theano_result, scipy_result)
def test_may_share_memory(): a = scipy.sparse.csc_matrix(scipy.sparse.eye(5, 3)) b = scipy.sparse.csc_matrix(scipy.sparse.eye(4, 3)) as_ar = lambda a: theano._asarray(a, dtype='int32') for a_, b_, rep in [ (a, a, True), (b, b, True), (a, b, False), (a, a.data, True), (a, a.indptr, True), (a, a.indices, True), (a, as_ar(a.shape), False), (a.data, a, True), (a.indptr, a, True), (a.indices, a, True), (as_ar(a.shape), a, False), (b, b.data, True), (b, b.indptr, True), (b, b.indices, True), (b, as_ar(b.shape), False), (b.data, b, True), (b.indptr, b, True), (b.indices, b, True), (as_ar(b.shape), b, False), (b.data, a, False), (b.indptr, a, False), (b.indices, a, False), (as_ar(b.shape), a, False), ]: assert SparseType.may_share_memory(a_, b_) == rep
def function(self, name=None): """ Returns a compiled theano function to compute a representation Parameters ---------- name : str WRITEME """ inputs = SparseType('csr', dtype=theano.config.floatX)() return theano.function([inputs], self(inputs), name=name)
def test_dot_sparse_sparse(self): #test dot for 2 input sparse matrix sparse_dtype = 'float64' sp_mat = {'csc': sp.csc_matrix, 'csr': sp.csr_matrix} for sparse_format_a in ['csc', 'csr']: for sparse_format_b in ['csc', 'csr']: a = SparseType(sparse_format_a, dtype=sparse_dtype)() b = SparseType(sparse_format_b, dtype=sparse_dtype)() d = theano.dot(a, b) f = theano.function([a, b], theano.Out(d, borrow=True)) topo = f.maker.env.toposort() for M, N, K, nnz in [ (4, 3, 2, 3), (40, 30, 20, 3), (40, 30, 20, 30), (400, 3000, 200, 6000), ]: a_val = sp_mat[sparse_format_a](random_lil( (M, N), sparse_dtype, nnz)) b_val = sp_mat[sparse_format_b](random_lil( (N, K), sparse_dtype, nnz)) f(a_val, b_val)
def test_shape(): # Test that getting the shape of a sparse variable # does not actually create a dense tensor in the process. sparse_dtype = 'float32' a = SparseType('csr', dtype=sparse_dtype)() f = theano.function([a], a.shape) assert numpy.all( f(sp.csr_matrix(random_lil((100, 10), sparse_dtype, 3))) == (100, 10)) if theano.config.mode != 'FAST_COMPILE': topo = f.maker.env.toposort() assert len(topo) == 3 assert isinstance(topo[0].op, tensor.opt.Shape_i) assert isinstance(topo[1].op, tensor.opt.Shape_i) assert isinstance(topo[2].op, tensor.opt.MakeVector)
def test_csc_correct_output_faster_than_scipy(self): sparse_dtype = 'float64' dense_dtype = 'float64' a = SparseType('csc', dtype=sparse_dtype)() b = tensor.matrix(dtype=dense_dtype) d = theano.dot(a, b) f = theano.function([a, b], theano.Out(d, borrow=True)) for M, N, K, nnz in [ (4, 3, 2, 3), (40, 30, 20, 3), (40, 30, 20, 30), (400, 3000, 200, 6000), ]: spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz)) mat = numpy.asarray(numpy.random.randn(N, K), dense_dtype) theano_times = [] scipy_times = [] for i in xrange(5): t0 = time.time() theano_result = f(spmat, mat) t1 = time.time() scipy_result = spmat * mat t2 = time.time() theano_times.append(t1 - t0) scipy_times.append(t2 - t1) theano_time = numpy.min(theano_times) scipy_time = numpy.min(scipy_times) speedup = scipy_time / theano_time print scipy_times print theano_times print( 'M=%(M)s N=%(N)s K=%(K)s nnz=%(nnz)s theano_time' '=%(theano_time)s speedup=%(speedup)s') % locals() # fail if Theano is slower than scipy by more than a certain amount overhead_tol = 0.003 # seconds overall overhead_rtol = 1.2 # times as long self.assertTrue(numpy.allclose(theano_result, scipy_result)) if not theano.config.mode in ["DebugMode", "DEBUG_MODE"]: self.assertFalse( theano_time > overhead_rtol * scipy_time + overhead_tol)
def test_sparse_shared_memory(): # Note : There are no inplace ops on sparse matrix yet. If one is someday implemented, we could test it here. a = random_lil((3,4), 'float32', 3).tocsr() m1 = random_lil((4,4), 'float32', 3).tocsr() m2 = random_lil((4,4), 'float32', 3).tocsr() x = SparseType('csr', dtype='float32')() y = SparseType('csr', dtype='float32')() sdot = theano.sparse.structured_dot z = sdot(x*3,m1) + sdot(y*2, m2) f = theano.function([theano.In(x,mutable=True),theano.In(y,mutable = True)],z, mode='FAST_RUN') def f_(x,y,m1=m1,m2=m2): return numpy.dot(x*3,m1) + numpy.dot(y*2,m2) assert SparseType.may_share_memory(a,a) #This is trivial result = f(a,a) result_ = f_(a,a) assert (result_.todense() == result.todense()).all()
def test_csr_correct_output_faster_than_scipy(self): #contrast with test_grad, we put csr in float32, csc in float64 sparse_dtype = 'float32' dense_dtype = 'float32' a = SparseType('csr', dtype=sparse_dtype)() b = tensor.matrix(dtype=dense_dtype) d = theano.dot(a, b) f = theano.function([a, b], d) for M, N, K, nnz in [ (4, 3, 2, 3), (40, 30, 20, 3), (40, 30, 20, 30), (400, 3000, 200, 6000), ]: spmat = sp.csr_matrix(random_lil((M, N), sparse_dtype, nnz)) mat = numpy.asarray(numpy.random.randn(N, K), dense_dtype) t0 = time.time() theano_result = f(spmat, mat) t1 = time.time() scipy_result = spmat * mat t2 = time.time() theano_time = t1 - t0 scipy_time = t2 - t1 #print theano_result #print scipy_result print 'theano took', theano_time, print 'scipy took', scipy_time overhead_tol = 0.002 # seconds overhead_rtol = 1.1 # times as long self.assertTrue(numpy.allclose(theano_result, scipy_result)) if not theano.config.mode in ["DebugMode", "DEBUG_MODE"]: self.assertFalse( theano_time > overhead_rtol * scipy_time + overhead_tol)
def make_node(self, x, y): """ :note: Because of trickiness of implementing, we assume that the left argument x is SparseVariable (not dense) """ if x.type.dtype != y.type.dtype: raise NotImplementedError() if not _is_sparse_variable(x): raise TypeError(x) # These are the conversions performed by scipy.sparse.dot if x.type.format == "csc" or x.type.format == "coo": myformat = "csc" elif x.type.format == "csr": myformat = "csr" else: raise NotImplementedError() inputs = [x, y] # Need to convert? e.g. assparse outputs = [ SparseType(dtype=x.type.dtype, format=myformat).make_variable() ] return gof.Apply(self, inputs, outputs)
def test_shape_i(): sparse_dtype = 'float32' a = SparseType('csr', dtype=sparse_dtype)() f = theano.function([a], a.shape[1]) assert f(sp.csr_matrix(random_lil((100, 10), sparse_dtype, 3))) == 10