def test_sparse_shared_memory():
# Note : There are no inplace ops on sparse matrix yet. If one is
# someday implemented, we could test it here.
a = random_lil((3, 4), 'float32', 3).tocsr()
m1 = random_lil((4, 4), 'float32', 3).tocsr()
m2 = random_lil((4, 4), 'float32', 3).tocsr()
x = SparseType('csr', dtype='float32')()
y = SparseType('csr', dtype='float32')()
sdot = theano.sparse.structured_dot
z = sdot(x * 3, m1) + sdot(y * 2, m2)
f = theano.function(
[theano.In(x, mutable=True),
theano.In(y, mutable=True)],
z,
mode='FAST_RUN')
def f_(x, y, m1=m1, m2=m2):
return numpy.dot(x * 3, m1) + numpy.dot(y * 2, m2)
assert SparseType.may_share_memory(a, a) # This is trivial
result = f(a, a)
result_ = f_(a, a)
assert (result_.todense() == result.todense()).all()
def test_may_share_memory():
a = scipy.sparse.csc_matrix(scipy.sparse.eye(5, 3))
b = scipy.sparse.csc_matrix(scipy.sparse.eye(4, 3))
as_ar = lambda a: theano._asarray(a, dtype="int32")
for a_, b_, rep in [
(a, a, True),
(b, b, True),
(a, b, False),
(a, a.data, True),
(a, a.indptr, True),
(a, a.indices, True),
(a, as_ar(a.shape), False),
(a.data, a, True),
(a.indptr, a, True),
(a.indices, a, True),
(as_ar(a.shape), a, False),
(b, b.data, True),
(b, b.indptr, True),
(b, b.indices, True),
(b, as_ar(b.shape), False),
(b.data, b, True),
(b.indptr, b, True),
(b.indices, b, True),
(as_ar(b.shape), b, False),
(b.data, a, False),
(b.indptr, a, False),
(b.indices, a, False),
(as_ar(b.shape), a, False),
]:
assert SparseType.may_share_memory(a_, b_) == rep
def test_graph_bprop_rand(self):
for i in range(10):
xorig = numpy.random.rand(3, 2)
for mtype in _mtypes:
x = tensor.matrix('x')
w = SparseType(dtype='float64',
format=_mtype_to_str[mtype]).make_variable()
xw = dense_from_sparse(true_dot(w, x))
y = dense_from_sparse(true_dot(w.T, xw))
diff = x - y
loss = tensor.sum(tensor.sqr(diff))
gw = tensor.grad(loss, w)
trainfn = compile.function([x, w], [y, loss, gw])
x = xorig
w = mtype((500, 3))
w[(10, 1)] = 1
w[(20, 2)] = 2
lr = 0.001
y, origloss, gw = trainfn(x, w)
for epoch in xrange(50):
y, loss, gw = trainfn(x, w)
w = w - (lr * gw)
self.assertTrue(origloss > loss)
def test_graph_bprop0(self):
for mtype in _mtypes:
x = tensor.matrix(
'x'
) #TensorType('float64', broadcastable=[False,False], name='x')
w = SparseType(dtype='float64',
format=_mtype_to_str[mtype]).make_variable()
xw = dense_from_sparse(true_dot(w, x))
y = dense_from_sparse(true_dot(w.T, xw))
diff = x - y
loss = tensor.sum(tensor.sqr(diff))
gw = tensor.grad(loss, w)
trainfn = compile.function([x, w], [y, loss, gw])
x = numpy.asarray([[1., 2], [3, 4], [2, 1]])
w = mtype((500, 3))
w[(10, 1)] = 1
w[(20, 2)] = 2
lr = 0.001
y, origloss, gw = trainfn(x, w)
for epoch in xrange(50):
y, loss, gw = trainfn(x, w)
w = w - (lr * gw)
print loss
self.assertTrue(origloss > loss)
self.assertTrue('1.05191241115' == str(loss))
def function(self, name=None, repr_index=-1, sparse_input=False):
"""
Compile a function computing representations on given layers.
Parameters
----------
name : string, optional
name of the function
repr_index : int, optional
Index of the hidden representation to return.
0 means the input, -1 the last output.
sparse_input : bool, optional
WRITEME
Returns
-------
WRITEME
"""
if sparse_input:
inputs = SparseType('csr', dtype=theano.config.floatX)()
else:
inputs = tensor.matrix()
return theano.function([inputs],
outputs=self(inputs)[repr_index],
name=name)
def test_upcast(self):
typenames = ('float32', 'int64', 'int8', 'int32', 'int16', 'float64',
'complex64', 'complex128')
for dense_dtype in typenames:
for sparse_dtype in typenames:
correct_dtype = theano.scalar.upcast(sparse_dtype, dense_dtype)
a = SparseType('csc', dtype=sparse_dtype)()
b = tensor.matrix(dtype=dense_dtype)
d = structured_dot(a, b)
assert d.type.dtype == correct_dtype
# compile and run a function
f = theano.function([a, b], d)
M, N, K, nnz = (4, 3, 5, 3)
spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz))
# the following madness is necessary to workaround
# an intc vs. int32 bug.
# The lil makes an intc on my computer when sparse_dtype
# is int32.
spmat.dtype = numpy.dtype(sparse_dtype)
mat = numpy.asarray(numpy.random.randn(N, K) * 9,
dtype=dense_dtype)
print 'DTYPES', sparse_dtype, dense_dtype
print 'sym types', a.type, b.type
print 'dtype strings', spmat.dtype, mat.dtype
print 'numpy dtype num', mat.dtype.num
print 'scipy dtype num', spmat.data.dtype.num
theano_result = f(spmat, mat)
scipy_result = spmat * mat
assert theano_result.shape == scipy_result.shape
assert theano_result.dtype == scipy_result.dtype
assert _allclose(theano_result, scipy_result)
def test_may_share_memory():
a = scipy.sparse.csc_matrix(scipy.sparse.eye(5, 3))
b = scipy.sparse.csc_matrix(scipy.sparse.eye(4, 3))
as_ar = lambda a: theano._asarray(a, dtype='int32')
for a_, b_, rep in [
(a, a, True),
(b, b, True),
(a, b, False),
(a, a.data, True),
(a, a.indptr, True),
(a, a.indices, True),
(a, as_ar(a.shape), False),
(a.data, a, True),
(a.indptr, a, True),
(a.indices, a, True),
(as_ar(a.shape), a, False),
(b, b.data, True),
(b, b.indptr, True),
(b, b.indices, True),
(b, as_ar(b.shape), False),
(b.data, b, True),
(b.indptr, b, True),
(b.indices, b, True),
(as_ar(b.shape), b, False),
(b.data, a, False),
(b.indptr, a, False),
(b.indices, a, False),
(as_ar(b.shape), a, False),
]:
assert SparseType.may_share_memory(a_, b_) == rep
def function(self, name=None):
"""
Returns a compiled theano function to compute a representation
Parameters
----------
name : str
WRITEME
"""
inputs = SparseType('csr', dtype=theano.config.floatX)()
return theano.function([inputs], self(inputs), name=name)
def test_dot_sparse_sparse(self):
#test dot for 2 input sparse matrix
sparse_dtype = 'float64'
sp_mat = {'csc': sp.csc_matrix, 'csr': sp.csr_matrix}
for sparse_format_a in ['csc', 'csr']:
for sparse_format_b in ['csc', 'csr']:
a = SparseType(sparse_format_a, dtype=sparse_dtype)()
b = SparseType(sparse_format_b, dtype=sparse_dtype)()
d = theano.dot(a, b)
f = theano.function([a, b], theano.Out(d, borrow=True))
topo = f.maker.env.toposort()
for M, N, K, nnz in [
(4, 3, 2, 3),
(40, 30, 20, 3),
(40, 30, 20, 30),
(400, 3000, 200, 6000),
]:
a_val = sp_mat[sparse_format_a](random_lil(
(M, N), sparse_dtype, nnz))
b_val = sp_mat[sparse_format_b](random_lil(
(N, K), sparse_dtype, nnz))
f(a_val, b_val)
def test_shape():
# Test that getting the shape of a sparse variable
# does not actually create a dense tensor in the process.
sparse_dtype = 'float32'
a = SparseType('csr', dtype=sparse_dtype)()
f = theano.function([a], a.shape)
assert numpy.all(
f(sp.csr_matrix(random_lil((100, 10), sparse_dtype, 3))) == (100, 10))
if theano.config.mode != 'FAST_COMPILE':
topo = f.maker.env.toposort()
assert len(topo) == 3
assert isinstance(topo[0].op, tensor.opt.Shape_i)
assert isinstance(topo[1].op, tensor.opt.Shape_i)
assert isinstance(topo[2].op, tensor.opt.MakeVector)
def test_csc_correct_output_faster_than_scipy(self):
sparse_dtype = 'float64'
dense_dtype = 'float64'
a = SparseType('csc', dtype=sparse_dtype)()
b = tensor.matrix(dtype=dense_dtype)
d = theano.dot(a, b)
f = theano.function([a, b], theano.Out(d, borrow=True))
for M, N, K, nnz in [
(4, 3, 2, 3),
(40, 30, 20, 3),
(40, 30, 20, 30),
(400, 3000, 200, 6000),
]:
spmat = sp.csc_matrix(random_lil((M, N), sparse_dtype, nnz))
mat = numpy.asarray(numpy.random.randn(N, K), dense_dtype)
theano_times = []
scipy_times = []
for i in xrange(5):
t0 = time.time()
theano_result = f(spmat, mat)
t1 = time.time()
scipy_result = spmat * mat
t2 = time.time()
theano_times.append(t1 - t0)
scipy_times.append(t2 - t1)
theano_time = numpy.min(theano_times)
scipy_time = numpy.min(scipy_times)
speedup = scipy_time / theano_time
print scipy_times
print theano_times
print(
'M=%(M)s N=%(N)s K=%(K)s nnz=%(nnz)s theano_time'
'=%(theano_time)s speedup=%(speedup)s') % locals()
# fail if Theano is slower than scipy by more than a certain amount
overhead_tol = 0.003 # seconds overall
overhead_rtol = 1.2 # times as long
self.assertTrue(numpy.allclose(theano_result, scipy_result))
if not theano.config.mode in ["DebugMode", "DEBUG_MODE"]:
self.assertFalse(
theano_time > overhead_rtol * scipy_time + overhead_tol)
def test_sparse_shared_memory():
# Note : There are no inplace ops on sparse matrix yet. If one is someday implemented, we could test it here.
a = random_lil((3,4), 'float32', 3).tocsr()
m1 = random_lil((4,4), 'float32', 3).tocsr()
m2 = random_lil((4,4), 'float32', 3).tocsr()
x = SparseType('csr', dtype='float32')()
y = SparseType('csr', dtype='float32')()
sdot = theano.sparse.structured_dot
z = sdot(x*3,m1) + sdot(y*2, m2)
f = theano.function([theano.In(x,mutable=True),theano.In(y,mutable = True)],z, mode='FAST_RUN')
def f_(x,y,m1=m1,m2=m2):
return numpy.dot(x*3,m1) + numpy.dot(y*2,m2)
assert SparseType.may_share_memory(a,a) #This is trivial
result = f(a,a)
result_ = f_(a,a)
assert (result_.todense() == result.todense()).all()
def test_csr_correct_output_faster_than_scipy(self):
#contrast with test_grad, we put csr in float32, csc in float64
sparse_dtype = 'float32'
dense_dtype = 'float32'
a = SparseType('csr', dtype=sparse_dtype)()
b = tensor.matrix(dtype=dense_dtype)
d = theano.dot(a, b)
f = theano.function([a, b], d)
for M, N, K, nnz in [
(4, 3, 2, 3),
(40, 30, 20, 3),
(40, 30, 20, 30),
(400, 3000, 200, 6000),
]:
spmat = sp.csr_matrix(random_lil((M, N), sparse_dtype, nnz))
mat = numpy.asarray(numpy.random.randn(N, K), dense_dtype)
t0 = time.time()
theano_result = f(spmat, mat)
t1 = time.time()
scipy_result = spmat * mat
t2 = time.time()
theano_time = t1 - t0
scipy_time = t2 - t1
#print theano_result
#print scipy_result
print 'theano took', theano_time,
print 'scipy took', scipy_time
overhead_tol = 0.002 # seconds
overhead_rtol = 1.1 # times as long
self.assertTrue(numpy.allclose(theano_result, scipy_result))
if not theano.config.mode in ["DebugMode", "DEBUG_MODE"]:
self.assertFalse(
theano_time > overhead_rtol * scipy_time + overhead_tol)
def make_node(self, x, y):
"""
:note: Because of trickiness of implementing, we assume that the left argument x is SparseVariable (not dense)
"""
if x.type.dtype != y.type.dtype:
raise NotImplementedError()
if not _is_sparse_variable(x):
raise TypeError(x)
# These are the conversions performed by scipy.sparse.dot
if x.type.format == "csc" or x.type.format == "coo":
myformat = "csc"
elif x.type.format == "csr":
myformat = "csr"
else:
raise NotImplementedError()
inputs = [x, y] # Need to convert? e.g. assparse
outputs = [
SparseType(dtype=x.type.dtype, format=myformat).make_variable()
]
return gof.Apply(self, inputs, outputs)
def test_shape_i():
sparse_dtype = 'float32'
a = SparseType('csr', dtype=sparse_dtype)()
f = theano.function([a], a.shape[1])
assert f(sp.csr_matrix(random_lil((100, 10), sparse_dtype, 3))) == 10
