def perform(self,node, xs, zs):
"""
.. todo::
WRITEME
"""
x = xs[0]
z = zs[0]
if x.format != 'csc':
raise TypeError('Remove0 only works on csc matrices')
M, N = x.shape
data = x.data
indices = x.indices
indptr = x.indptr
#TODO: try using ndarrays and then prune() on the result
new_data = []
new_indices = []
new_indptr = [0]
for j in xrange(0, N):
for i_idx in xrange(indptr[j], indptr[j+1]):
if data[i_idx] != 0:
new_data.append(data[i_idx])
new_indices.append(indices[i_idx])
new_indptr.append(len(new_indices))
z[0] = sparse.csc_matrix((new_data, new_indices, new_indptr), (M,N))
def __init__(self,
in_dim,
out_dim=None,
weighted=False,
sparse_input=False,
output_name='out'):
self.input = Loss.F_CONTAINERS[in_dim]('input')
if sparse_input is True or \
isinstance(sparse_input, str) and sparse_input.lower() == 'csr':
assert in_dim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csr_matrix('input')
if isinstance(sparse_input, str) and sparse_input.lower() == 'csc':
assert in_dim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csc_matrix('input')
self.variables = [self.input]
self.target = None
if out_dim:
self.target = Loss.F_CONTAINERS[out_dim]('target')
self.variables.append(self.target)
self.weight = None
if weighted:
self.weight = Loss.F_CONTAINERS[out_dim or in_dim]('weight')
self.variables.append(self.weight)
self.output_name = output_name
if ':' not in self.output_name:
self.output_name += ':out'
def perform(self, node, xs, zs):
"""
.. todo::
WRITEME
"""
x = xs[0]
z = zs[0]
if x.format != 'csc':
raise TypeError('Remove0 only works on csc matrices')
M, N = x.shape
data = x.data
indices = x.indices
indptr = x.indptr
#TODO: try using ndarrays and then prune() on the result
new_data = []
new_indices = []
new_indptr = [0]
for j in xrange(0, N):
for i_idx in xrange(indptr[j], indptr[j + 1]):
if data[i_idx] != 0:
new_data.append(data[i_idx])
new_indices.append(indices[i_idx])
new_indptr.append(len(new_indices))
z[0] = sparse.csc_matrix((new_data, new_indices, new_indptr), (M, N))
def act(idx, W):
M = W.shape[0]
N = W.shape[1]
K = idx.shape[0]
data = T.ones(K)
I = sparse.csc_matrix()
return sparse.true_dot(I, W[idx,:])
def act(idx, W):
M = W.shape[0]
N = W.shape[1]
K = idx.shape[0]
data = T.ones(K)
I = sparse.csc_matrix()
return sparse.true_dot(I, W[idx, :])
class Remove0(Op):
"""
Remove explicit zeros from a sparse matrix, and resort indices
"""
def make_node(self, x):
return gof.Apply(self, [x], [x.type()])
def perform(self, node, (x, ), (z, )):
if x.format != 'csc':
raise TypeError('Remove0 only works on csc matrices')
M, N = x.shape
data = x.data
indices = x.indices
indptr = x.indptr
#TODO: try using ndarrays and then prune() on the result
new_data = []
new_indices = []
new_indptr = [0]
for j in xrange(0, N):
for i_idx in xrange(indptr[j], indptr[j + 1]):
if data[i_idx] != 0:
new_data.append(data[i_idx])
new_indices.append(indices[i_idx])
new_indptr.append(len(new_indices))
z[0] = sparse.csc_matrix((new_data, new_indices, new_indptr), (M, N))
def test_sparsevariable(self):
## Re-init counter
Variable.__count__ = count(0)
r1 = sparse.csc_matrix(name='x', dtype='float32')
r2 = sparse.dense_from_sparse(r1)
r3 = sparse.csc_from_dense(r2)
assert r1.auto_name == "auto_0"
assert r2.auto_name == "auto_1"
assert r3.auto_name == "auto_2"
def test_sparsevariable(self):
## Re-init counter
Variable.__count__ = count(0)
r1 = sparse.csc_matrix(name='x', dtype='float32')
r2 = sparse.dense_from_sparse(r1)
r3 = sparse.csc_from_dense(r2)
assert r1.auto_name == "auto_0"
assert r2.auto_name == "auto_1"
assert r3.auto_name == "auto_2"
def __init__(self, size, name="in", ndim=2, sparse=False):
self.input = util.FLOAT_CONTAINERS[ndim](name)
if sparse is True or isinstance(sparse, str) and sparse.lower() == "csr":
assert ndim == 2, "Theano only supports sparse arrays with 2 dims"
self.input = SS.csr_matrix("input")
if isinstance(sparse, str) and sparse.lower() == "csc":
assert ndim == 2, "Theano only supports sparse arrays with 2 dims"
self.input = SS.csc_matrix("input")
super(Input, self).__init__(size=size, name=name, inputs=0, activation="linear", ndim=ndim, sparse=sparse)
def test_sparsevariable(self):
## Get counter value
autoname_id = next(Variable.__count__)
Variable.__count__ = count(autoname_id)
r1 = sparse.csc_matrix(name='x', dtype='float32')
r2 = sparse.dense_from_sparse(r1)
r3 = sparse.csc_from_dense(r2)
assert r1.auto_name == "auto_" + str(autoname_id)
assert r2.auto_name == "auto_" + str(autoname_id + 1)
assert r3.auto_name == "auto_" + str(autoname_id + 2)
def test_sparsevariable(self):
# Get counter value
autoname_id = next(Variable.__count__)
Variable.__count__ = count(autoname_id)
r1 = sparse.csc_matrix(name="x", dtype="float32")
r2 = sparse.dense_from_sparse(r1)
r3 = sparse.csc_from_dense(r2)
assert r1.auto_name == "auto_" + str(autoname_id)
assert r2.auto_name == "auto_" + str(autoname_id + 1)
assert r3.auto_name == "auto_" + str(autoname_id + 2)
def __init__(self, size, name='in', ndim=2, sparse=False, **kwargs):
self.input = util.FLOAT_CONTAINERS[ndim](name)
if sparse is True or \
isinstance(sparse, util.basestring) and sparse.lower() == 'csr':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csr_matrix('input')
if isinstance(sparse, util.basestring) and sparse.lower() == 'csc':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csc_matrix('input')
super(Input, self).__init__(
size=size, name=name, activation='linear', ndim=ndim, sparse=sparse)
def test_sparsevariable(self):
# Get counter value
if not sparse.enable_sparse:
raise SkipTest('Optional package SciPy not installed')
autoname_id = next(Variable.__count__)
Variable.__count__ = count(autoname_id)
r1 = sparse.csc_matrix(name='x', dtype='float32')
r2 = sparse.dense_from_sparse(r1)
r3 = sparse.csc_from_dense(r2)
assert r1.auto_name == "auto_" + str(autoname_id)
assert r2.auto_name == "auto_" + str(autoname_id + 1)
assert r3.auto_name == "auto_" + str(autoname_id + 2)
def test_sparsevariable(self):
# Get counter value
if not sparse.enable_sparse:
raise SkipTest('Optional package SciPy not installed')
autoname_id = next(Variable.__count__)
Variable.__count__ = count(autoname_id)
r1 = sparse.csc_matrix(name='x', dtype='float32')
r2 = sparse.dense_from_sparse(r1)
r3 = sparse.csc_from_dense(r2)
assert r1.auto_name == "auto_" + str(autoname_id)
assert r2.auto_name == "auto_" + str(autoname_id + 1)
assert r3.auto_name == "auto_" + str(autoname_id + 2)
def __init__(self, size, name='in', ndim=2, sparse=False, **kwargs):
self.input = util.FLOAT_CONTAINERS[ndim](name)
if sparse is True or \
isinstance(sparse, util.basestring) and sparse.lower() == 'csr':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csr_matrix('input')
if isinstance(sparse, util.basestring) and sparse.lower() == 'csc':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csc_matrix('input')
super(Input, self).__init__(size=size,
name=name,
activation='linear',
ndim=ndim,
sparse=sparse)
def __init__(self, in_dim, out_dim, weighted=False, sparse_input=False, output_name="out:out"):
self.input = Loss.F_CONTAINERS[in_dim]("input")
if sparse_input is True or isinstance(sparse_input, str) and sparse_input.lower() == "csr":
assert in_dim == 2, "Theano only supports sparse arrays with 2 dims"
self.input = SS.csr_matrix("input")
if isinstance(sparse_input, str) and sparse_input.lower() == "csc":
assert in_dim == 2, "Theano only supports sparse arrays with 2 dims"
self.input = SS.csc_matrix("input")
self.target = Loss.I_CONTAINERS[out_dim]("target")
self.variables = [self.input, self.target]
self.weight = None
if weighted:
self.weight = Loss.F_CONTAINERS[out_dim]("weight")
self.variables.append(self.weight)
self.output_name = output_name
def __init__(self, name='in', ndim=2, sparse=False, **kwargs):
shape = kwargs.get('shape')
if shape:
ndim = 1 + len(shape)
else:
kwargs['shape'] = (None, ) * (ndim - 2) + (kwargs.pop('size'), )
self.input = util.FLOAT_CONTAINERS[ndim](name)
if sparse is True or \
isinstance(sparse, util.basestring) and sparse.lower() == 'csr':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csr_matrix('input')
if isinstance(sparse, util.basestring) and sparse.lower() == 'csc':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csc_matrix('input')
kwargs['activation'] = 'linear'
super(Input, self).__init__(name=name, **kwargs)
def __init__(self, name='in', ndim=2, sparse=False, **kwargs):
shape = kwargs.get('shape')
if shape:
ndim = 1 + len(shape)
else:
kwargs['shape'] = (None, ) * (ndim - 2) + (kwargs.pop('size'), )
self.input = util.FLOAT_CONTAINERS[ndim](name)
if sparse is True or \
isinstance(sparse, util.basestring) and sparse.lower() == 'csr':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csr_matrix('input')
if isinstance(sparse, util.basestring) and sparse.lower() == 'csc':
assert ndim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csc_matrix('input')
kwargs.setdefault('activation', 'linear')
super(Input, self).__init__(name=name, **kwargs)
def __init__(self, in_dim, out_dim, weighted=False, sparse_input=False,
output_name='out:out'):
self.input = Loss.F_CONTAINERS[in_dim]('input')
if sparse_input is True or \
isinstance(sparse_input, str) and sparse_input.lower() == 'csr':
assert in_dim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csr_matrix('input')
if isinstance(sparse_input, str) and sparse_input.lower() == 'csc':
assert in_dim == 2, 'Theano only supports sparse arrays with 2 dims'
self.input = SS.csc_matrix('input')
self.target = Loss.I_CONTAINERS[out_dim]('target')
self.variables = [self.input, self.target]
self.weight = None
if weighted:
self.weight = Loss.F_CONTAINERS[out_dim]('weight')
self.variables.append(self.weight)
self.output_name = output_name
def train_autoEncoder(trainFile, testFile, recordParserCallback, n_visible,
n_hidden, learning_rates, callback, corruption_levels,
activation, training_epochs, legend, normalize,
generate_attr_vec_callback, total, layer_no, params=None,
modulo=1000, useMomentum=False, momentumRate=0.90):
if len(corruption_levels) != len(n_hidden):
raise Exception("corruption level not provided for each layer...will use default")
if len(learning_rates) != len(n_hidden):
raise Exception("learning rates not provided for each layer...will use default")
legend.append("SAE %d layers" % len(n_hidden))
sample = sparse.csc_matrix(name='s', dtype='float32')
label = sparse.csc_matrix(name='l', dtype='float32')
x = sparse.csc_matrix(name='x', dtype='float32')
y = sparse.csc_matrix(name='y', dtype='float32')
corruption_level = T.scalar('corruption') # % of corruption to use
learning_rate = T.scalar('lr') # learning rate to use
momentum_rate = T.scalar('momentum') # learning rate to use
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
#bhid_value = numpy.zeros(n_hidden, dtype=theano.config.floatX)
W = None
b = None
if params is not None:
W = params['W']
if len(params['b']) == len(params['W']):
b = params['b']
sda = SdA(numpy_rng=rng, theano_rng=theano_rng, n_ins=n_visible,
hidden_layers_sizes=n_hidden, input=x, label=y,
activation=activation, W=W, b=b, useMomentum=useMomentum)
#activation=activation, b=bhid_value)
pretraining_fns = sda.pretraining_functions(sample, label, learning_rate,
corruption_level, momentum_rate)
#sampleMatrix = generate_feature(trainData, total, generate_attr_vec_callback)
if not params:
train(recordParserCallback, generate_attr_vec_callback, pretraining_fns,
learning_rates, corruption_levels, momentumRate, training_epochs,
batch_size=1, modulo=modulo)
# plot without before feature learning
#plot_transformed_vectors(testMatrix.toarray(), testDataLabel, title="before feature learning")
print "about to test..."
test(testFile, recordParserCallback, generate_attr_vec_callback, sda)
#sampleMatrix = generate_feature(trainData, total, generate_attr_vec_callback)
#errorVector = sda.get_reconstruction_errors(sampleMatrix.tocsc())
#testMatrix = generate_feature(testData, total, generate_attr_vec_callback)
#errorVectorTest = sda.get_reconstruction_errors(testMatrix.tocsc())
'''
def find_avg_error(errorMatrix):
error = errorMatrix
sqrdErrorMatrix = numpy.dot(error, numpy.transpose(error))
return numpy.diag(sqrdErrorMatrix)
print "error train: " + str(math.sqrt(sum(find_avg_error(errorVector))))
print "error test: " + str(math.sqrt(sum(find_avg_error(errorVectorTest))))
'''
# look at individual errors:
callback(sda, errorVectors, labels, legend)
def plot_learning_curve(trainingFile, testFile, recordParserCallback,
generate_attr_vec_callback, corruption_levels,
learning_rates, momentumRate, training_epochs, n_visible, n_hidden,
activation, useMomentum=False, batch_size=1000, modulo=1000):
sample = sparse.csc_matrix(name='s', dtype='float32')
label = sparse.csc_matrix(name='l', dtype='float32')
x = sparse.csc_matrix(name='x', dtype='float32')
y = sparse.csc_matrix(name='y', dtype='float32')
corruption_level = T.scalar('corruption') # % of corruption to use
learning_rate = T.scalar('lr') # learning rate to use
momentum_rate = T.scalar('momentum') # learning rate to use
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
#bhid_value = numpy.zeros(n_hidden, dtype=theano.config.floatX)
W = None
b = None
if params is not None:
W = params['W']
if len(params['b']) == len(params['W']):
b = params['b']
sda = SdA(numpy_rng=rng, theano_rng=theano_rng, n_ins=n_visible,
hidden_layers_sizes=n_hidden, input=x, label=y,
activation=activation, W=W, b=b, useMomentum=useMomentum)
#activation=activation, b=bhid_value)
pretraining_fns = sda.pretraining_functions(sample, label, learning_rate,
corruption_level, momentum_rate)
pl.clf()
def plotCurve(x, y, label):
# Compute ROC curve and area the curve
pl.plot(x, y, label=label)
#pl.plot([0, 1], [0, 1], 'k--')
#pl.xlim([0.0, 1.0])
#pl.ylim([0.0, 1.0])
#pl.xlabel('False Positive Rate')
#pl.ylabel('True Positive Rate')
#pl.title('Receiver operating characteristic')
pl.legend(loc="lower right")
#pl.show()
#sampleMatrix = generate_feature(trainData, total, generate_attr_vec_callback)
x = []
y = []
y_cost = []
for iteration in range(1, 5):
cost = train(recordParserCallback, generate_attr_vec_callback, pretraining_fns,
learning_rates, corruption_levels, momentumRate, training_epochs,
batch_size=iteration*batch_size, modulo=modulo,
stopping_fn=stop_after_mini_batch)
x.append(iteration)
y.append(cost)
print "about to test..."
testCost = test(testFile, recordParserCallback, generate_attr_vec_callback, sda)
y_cost.append(testCost)
# plot without before feature learning
#plot_transformed_vectors(testMatrix.toarray(), testDataLabel, title="before feature learning")
#print "about to test..."
#test(testFile, recordParserCallback, generate_attr_vec_callback, sda)
plotCurve(x, y, "train")
plotCurve(x, y_cost, "test")
pl.show()
import theano
import theano.tensor as T
import theano.sparse as Tsp
import numpy as np
import scipy.sparse as sp
a = T.fmatrix('a')
b = Tsp.csc_matrix('b')
c = Tsp.basic.dot(a, b)
test = theano.function(inputs=[a, b], outputs=[c], on_unused_input='warn')
aa = [[-1, 1, 1]]
aa = np.array(aa).astype(np.float32)
row = [0, 1, 2]
col = [0, 1, 2]
data = [1, -1, -1]
bb = sp.csc_matrix((data, (row, col))).astype(np.float32)
x = test(aa, bb)
print x
tsize = T.matrix()
tV = T.matrix()
tI = T.ivector()
bla = sparse.construct_sparse_from_list(tsize, tV, tI)
v = ones(2).astype(float32)
i = arange(2)
j = arange(2)
m = sp.csc_matrix((v, (i, j)), shape=(4, 2))
tdata = T.vector()
tindices = T.ivector()
tindptr = T.ivector()
tshape = T.ivector()
x = sparse.csc_matrix()
a, b, c, d = sparse.csm_properties(x)
print a.eval({x: m})
print b.eval({x: m})
print c.eval({x: m})
print d.eval({x: m})
tm = sparse.CSC(tdata, tindices, tindptr, tshape)
shape = array([5, 3]).astype(int32)
indices = array([0, 2, 4]).astype(int32)
indptr = arange(4).astype(int32)
data = ones(3).astype(float32)
m2 = tm.eval({tdata: data, tindices: indices, tindptr: indptr, tshape: shape})
ty = T.ivector()
''' examples of sparse matrices''' import numpy as np import scipy.sparse as sp import theano from theano import sparse # pylint: disable = bad-whitespace, invalid-name, no-member, bad-continuation, assignment-from-no-return # if shape[0] > shape[1], use csr. Otherwise, use csc # but, not all ops are available for both yet # so use the one that has what you need # to and fro x = sparse.csc_matrix(name='x', dtype='float32') y = sparse.dense_from_sparse(x) z = sparse.csc_from_dense(y) # resconstruct a csc from a csr x = sparse.csc_matrix(name='x', dtype='int64') data, indices, indptr, shape = sparse.csm_properties(x) y = sparse.CSR(data, indices, indptr, shape) f = theano.function([x], y) a = sp.csc_matrix(np.asarray([[0, 1, 1], [0, 0, 0], [1, 0, 0]])) print a.toarray() print f(a).toarray() # "structured" operations # act only on (originally) nonzero elements
# Most of the ops in Theano depend on the format of the sparse matrix. # That is why there are two kinds of constructors of sparse variable: # csc_matrix and csr_matrix. These can be called with the usual name # and dtype parameters, but no broadcastable flags are allowed. This is # forbidden since the sparse package, as the SciPy sparse module, does not # provide any way to handle a number of dimensions different from two. The # set of all accepted dtype for the sparse matrices can be found in # sparse.all_dtypes. print sparse.all_dtypes # 2.1 To and Fro # To move back and forth from a dense matrix to a sparse matrix # representation, Theano provides the dense_from_sparse and csc_from_dense # functions. No additional detail must be provided. Here is an example # that performs a full cycle from sparse to sparse: x = sparse.csc_matrix(name='x', dtype='float32') y = sparse.dense_from_sparse(x) z = sparse.csc_from_dense(y) # 2.2 Properties and Construction # Although sparse variables do not allow direct to their properties, this # can be accomplished using the csm_properties function. This will return # a tuple of one-dimensional tensor variables that represents the internal # characteristics of the sparse matrix. # In order to reconstruct a sparse matrix from some properties, the # function CSC and CSR can be used. This will create the sparse matrix in # desired format. As an example, the following code reconstructs a csc # matrix into a csr one. x = sparse.csc_matrix(name='x', dtype='int64') data, indices, indptr, shape = sparse.csm_properties(x)
def test_sparse(self):
# print '\n\n*************************************************'
# print ' TEST SPARSE'
# print '*************************************************'
# fixed parameters
bsize = 10 # batch size
imshp = (28, 28)
kshp = (5, 5)
nkern = 1 # per output pixel
ssizes = ((1, 1), (2, 2))
convmodes = ("full", "valid")
# symbolic stuff
bias = tensor.dvector()
kerns = tensor.dvector()
input = tensor.dmatrix()
rng = numpy.random.RandomState(3423489)
import theano.gof as gof
# Mode(optimizer='fast_run', linker=gof.OpWiseCLinker(allow_gc=False)),):
for mode in ("FAST_COMPILE", "FAST_RUN"): # ,profmode):
ntot, ttot = 0, 0
for conv_mode in convmodes:
for ss in ssizes:
output, outshp = sp.applySparseFilter(
kerns, kshp, nkern, input, imshp, ss, bias=bias, mode=conv_mode
)
f = function([kerns, bias, input], output, mode=mode)
# build actual input images
img2d = numpy.arange(bsize * numpy.prod(imshp)).reshape((bsize,) + imshp)
img1d = img2d.reshape(bsize, -1)
zeropad_img = numpy.zeros(
(bsize, img2d.shape[1] + 2 * (kshp[0] - 1), img2d.shape[2] + 2 * (kshp[1] - 1))
)
zeropad_img[
:, kshp[0] - 1 : kshp[0] - 1 + img2d.shape[1], kshp[1] - 1 : kshp[1] - 1 + img2d.shape[2]
] = img2d
# build kernel matrix -- flatten it for theano stuff
filters = numpy.arange(numpy.prod(outshp) * numpy.prod(kshp)).reshape(
nkern, numpy.prod(outshp[1:]), numpy.prod(kshp)
)
spfilt = filters.flatten()
biasvals = numpy.arange(numpy.prod(outshp))
# compute output by hand
ntime1 = time.time()
refout = numpy.zeros((bsize, nkern, outshp[1], outshp[2]))
patch = numpy.zeros((kshp[0], kshp[1]))
for b in xrange(bsize):
for k in xrange(nkern):
pixi = 0 # pixel index in raster order
for j in xrange(outshp[1]):
for i in xrange(outshp[2]):
n = j * ss[0]
m = i * ss[1]
patch = zeropad_img[b, n : n + kshp[0], m : m + kshp[1]]
refout[b, k, j, i] = numpy.dot(filters[k, pixi, :], patch.flatten())
pixi += 1
refout = refout.reshape(bsize, -1) + biasvals
ntot += time.time() - ntime1
# need to flatten images
ttime1 = time.time()
out1 = f(spfilt, biasvals, img1d)
ttot += time.time() - ttime1
temp = refout - out1
assert (temp < 1e-10).all()
# test downward propagation
vis = tensor.grad(0.5 * tensor.sqr(output).sum(), input)
downprop = function([kerns, output], vis)
temp1 = time.time()
for zz in range(100):
visval = downprop(spfilt, out1)
indices, indptr, spmat_shape, sptype, outshp, kmap = sp.convolution_indices.sparse_eval(
imshp, kshp, nkern, ss, conv_mode
)
spmat = sparse.csc_matrix((spfilt[kmap], indices, indptr), spmat_shape)
visref = numpy.dot(out1, spmat.todense())
assert numpy.all(visref == visval), (visref, visval)
def test_sparse(self):
# print '\n\n*************************************************'
# print ' TEST SPARSE'
# print '*************************************************'
# fixed parameters
bsize = 10 # batch size
imshp = (8, 8)
kshp = (5, 5)
nkern = 1 # per output pixel
ssizes = ((1, 1), (2, 2))
convmodes = (
'full',
'valid',
)
# symbolic stuff
bias = tensor.dvector()
kerns = tensor.dvector()
input = tensor.dmatrix()
rng = numpy.random.RandomState(3423489)
import theano.gof as gof
for mode in (None, ):
ntot, ttot = 0, 0
for conv_mode in convmodes:
for ss in ssizes:
output, outshp = sp.applySparseFilter(kerns, kshp,\
nkern, input, imshp, ss, bias=bias, mode=conv_mode)
f = function([kerns, bias, input], output, mode=mode)
# build actual input images
img2d = numpy.arange(
bsize * numpy.prod(imshp)).reshape((bsize, ) + imshp)
img1d = img2d.reshape(bsize, -1)
zeropad_img = numpy.zeros((bsize,\
img2d.shape[1]+2*(kshp[0]-1),\
img2d.shape[2]+2*(kshp[1]-1)))
zeropad_img[:, kshp[0] - 1:kshp[0] - 1 + img2d.shape[1],
kshp[1] - 1:kshp[1] - 1 +
img2d.shape[2]] = img2d
# build kernel matrix -- flatten it for theano stuff
filters = numpy.arange(numpy.prod(outshp)*numpy.prod(kshp)).\
reshape(nkern, numpy.prod(outshp[1:]), numpy.prod(kshp))
spfilt = filters.flatten()
biasvals = numpy.arange(numpy.prod(outshp))
# compute output by hand
ntime1 = time.time()
refout = numpy.zeros((bsize, nkern, outshp[1], outshp[2]))
patch = numpy.zeros((kshp[0], kshp[1]))
for b in xrange(bsize):
for k in xrange(nkern):
pixi = 0 # pixel index in raster order
for j in xrange(outshp[1]):
for i in xrange(outshp[2]):
n = j * ss[0]
m = i * ss[1]
patch = zeropad_img[b, n:n + kshp[0],
m:m + kshp[1]]
refout[b, k, j, i] = numpy.dot(filters[k, pixi, :],\
patch.flatten())
pixi += 1
refout = refout.reshape(bsize, -1) + biasvals
ntot += time.time() - ntime1
# need to flatten images
ttime1 = time.time()
out1 = f(spfilt, biasvals, img1d)
ttot += time.time() - ttime1
temp = refout - out1
assert (temp < 1e-10).all()
# test downward propagation
vis = tensor.grad(0.5 * tensor.sqr(output).sum(), input)
downprop = function([kerns, output], vis)
temp1 = time.time()
for zz in range(100):
visval = downprop(spfilt, out1)
indices, indptr, spmat_shape, sptype, outshp, kmap = \
sp.convolution_indices.sparse_eval(imshp, kshp, nkern, ss, conv_mode)
spmat = sparse.csc_matrix((spfilt[kmap], indices, indptr),
spmat_shape)
visref = numpy.dot(out1, spmat.todense())
assert numpy.all(visref == visval), (visref, visval)
def test_rbm(dataset,
datasize,
record=True,
l1=0.,
l2=0.0,
mom=0.9,
learning_rate=0.1,
training_epochs=50,
batch_size=100,
n_chains=2,
PCD_k=True,
update_scheme=None,
n_hidden=10,
save_path='./',
W=None,
hbias=None,
vbias=None):
"""
Demonstrate how to train and afterwards sample from it using Theano.
This is demonstrated on MNIST.
:param learning_rate: learning rate used for training the RBM
:param training_epochs: number of epochs used for training
:param dataset: path the the pickled dataset
:param batch_size: size of a batch used to train the RBM
:param n_chains: number of parallel Gibbs chains to be used for sampling
:param n_samples: number of samples to plot for each chain
"""
data = dataset
print 'Training Set Shape: %s' % (data.shape, )
dataset = theano.shared(data)
train_set_x = dataset
# compute number of minibatches for training, validation and testing
n_train_batches = int(
train_set_x.get_value(borrow=True).shape[0] * datasize) / batch_size
print 'batch selected : %i' % (n_train_batches)
n_dim = train_set_x.get_value(borrow=True).shape[1]
# allocate symbolic variables for the data
index = T.iscalar('index') # index to a [mini]batch
x = sp.csc_matrix('x') # the data is presented as rasterized images
# initialize storage for the persistent chain (state = hidden
# layer of chain)
if PCD_k:
persistent_chain = theano.shared(
numpy.zeros((batch_size, n_hidden), dtype=theano.config.floatX))
else:
persistent_chain = None
print 'Building the model'
# construct the RBM class
# convert sparse matrix to dense matrix for operation
m = x.toarray()
rbm = RBM(input=m,
n_visible=n_dim,
n_hidden=n_hidden,
W=W,
vbias=vbias,
hbias=hbias)
# get the cost and the gradient corresponding to one step of CD-15
n_chains = T.lscalar('k')
# lr = T.fscalar('lr')
cost, updates = rbm.get_cost_updates(mom=mom,
lr=learning_rate,
l2=l2,
persistent=persistent_chain,
k=n_chains)
free_energy = rbm.free_energy(m)
predict_score = (T.ge(rbm.propdown(T.ge(rbm.propup(m)[1], 0.5)*1)[1], 0.5)).sum()/\
m.sum()
print 'Building the Computational Graph'
#################################
# Training the RBM #
#################################
train_rbm = theano.function(
[index, n_chains],
cost,
updates=updates,
givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]},
name='train_rbm')
score = theano.function(
[index],
predict_score,
givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]})
cal_free_energy = theano.function(
[index],
free_energy,
givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]})
Date = str(date.today())
# go through training epochs
file_name = save_path
print 'Start Training'
val_free = []
train_free = []
hist_cost = []
# hist_score = []
for epoch in xrange(training_epochs):
if update_scheme is None:
if 30 > epoch & epoch > 15:
PCD_k = 3
elif epoch == 0:
PCD_k = 2
elif 0 < epoch & epoch < 15:
PCD_k = 1
elif epoch >= 30:
PCD_k = 6
else:
PCD_k, learning_rate = update_scheme(epoch)
# go through the training set
mean_cost = []
for batch_index in xrange(n_train_batches):
mean_cost += [train_rbm(batch_index, PCD_k)]
# val_free += [numpy.mean(cal_free_energy(n_train_batches + 1))]
# hist_score += [score(n_train_batches)]
# train_free += [numpy.mean(cal_free_energy(n_train_batches))]
hist_cost += [numpy.mean(mean_cost)]
print 'Training epoch: %d puesdo likelihood: %f'\
% (epoch, numpy.mean(mean_cost)) # train_free[-1], val_free[-1],)
if record:
if epoch % 5 == 0:
# save parameters
# train_free = numpy.array(train_free)
# val_free = numpy.array(val_free)
hist_cost_ = numpy.array(hist_cost)
print 'Saved at' + os.getcwd()
file_flag = Date + '-' + repr(epoch)
numpy.save(file_name + 'hist_cost@' + file_flag, hist_cost_)
# numpy.save(file_name + 'Train_free@' + file_flag, train_free)
# numpy.save(file_name + 'Val_free@' + file_flag, val_free)
numpy.save(file_name + 'RBM_model@' + file_flag, rbm)
print 'Result Save @' + os.getcwd()
return train_free, val_free, hist_cost, rbm
def train_autoEncoder(trainData, testData, trainDataLabel, testDataLabel,
n_visible, n_hidden, learning_rates, callback,
corruption_levels, activation, training_epochs, legend,
normalize, generate_attr_vec_callback, total, layer_no,
params=None):
if len(corruption_levels) != len(n_hidden):
raise Exception("corruption level not provided for each layer...will use default")
if len(learning_rates) != len(n_hidden):
raise Exception("learning rates not provided for each layer...will use default")
legend.append("SAE %d layers" % len(n_hidden))
sample = sparse.csc_matrix(name='s', dtype='float32')
label = sparse.csc_matrix(name='l', dtype='float32')
x = sparse.csc_matrix(name='x', dtype='float32')
y = sparse.csc_matrix(name='y', dtype='float32')
corruption_level = T.scalar('corruption') # % of corruption to use
learning_rate = T.scalar('lr') # learning rate to use
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
#bhid_value = numpy.zeros(n_hidden, dtype=theano.config.floatX)
W = None
b = None
if params is not None:
W = params['W']
if len(params['b']) == len(params['W']):
b = params['b']
sda = SdA(numpy_rng=rng, theano_rng=theano_rng, n_ins=n_visible,
hidden_layers_sizes=n_hidden, input=x, label=y,
activation=activation, W=W, b=b)
#activation=activation, b=bhid_value)
'''cost, updates = sda.get_cost_updates(corruption_level=corruption_level,
learning_rate=learning_rate)
# this is the function that theano will call to optimize the cost
# function.
train_da = theano.function([sample, label], cost, updates=updates,
givens={x: sample,
y: label})
'''
pretraining_fns = sda.pretraining_functions(sample, label, learning_rate,
corruption_level)
if normalize:
def normalize_data(data):
# normalize data
inputArray = numpy.array(data)
minValue = inputArray.flatten().min()
maxValue = inputArray.flatten().max()
inputArray = (inputArray - float(minValue))/(float(maxValue - minValue))
return inputArray
trainData = normalize_data(trainData)
testData = normalize_data(testData)
#sampleMatrix = generate_feature(trainData, total, generate_attr_vec_callback)
for idx, fn in enumerate(pretraining_fns):
print "training layer #%s" % str(idx)
for i in range(0, training_epochs):
for sample in trainData:
sampleMatrix = generate_feature([sample], total, generate_attr_vec_callback)
error = fn(sampleMatrix, sampleMatrix,
corruption_levels[idx], learning_rates[idx])
print "error train cost: " + str(error)
sampleMatrix = generate_feature(trainData, total, generate_attr_vec_callback)
errorVector = sda.get_reconstruction_errors(sampleMatrix.tocsc())
testMatrix = generate_feature(testData, total, generate_attr_vec_callback)
errorVectorTest = sda.get_reconstruction_errors(testMatrix.tocsc())
# plot without before feature learning
plot_transformed_vectors(testMatrix.toarray(), testDataLabel, title="before feature learning")
def find_avg_error(errorMatrix):
error = errorMatrix
sqrdErrorMatrix = numpy.dot(error, numpy.transpose(error))
return numpy.diag(sqrdErrorMatrix)
print "error train: " + str(math.sqrt(sum(find_avg_error(errorVector))))
print "error test: " + str(math.sqrt(sum(find_avg_error(errorVectorTest))))
# look at individual errors:
callback(sda, trainData, trainDataLabel, testData,
testDataLabel, generate_attr_vec_callback, legend, total)
transformSample = sda.get_hidden_values(sampleMatrix.tocsc())
transformTest = sda.get_hidden_values(testMatrix.tocsc())
return transformSample.eval(), transformTest.eval()
def test_rbm(dataset, datasize, record=True, l1=0., l2=0.0, mom=0.9,
learning_rate=0.1, training_epochs=50,
batch_size=100,
n_chains=2, PCD_k=True, update_scheme=None,
n_hidden=10, save_path='./', W=None, hbias=None, vbias=None):
"""
Demonstrate how to train and afterwards sample from it using Theano.
This is demonstrated on MNIST.
:param learning_rate: learning rate used for training the RBM
:param training_epochs: number of epochs used for training
:param dataset: path the the pickled dataset
:param batch_size: size of a batch used to train the RBM
:param n_chains: number of parallel Gibbs chains to be used for sampling
:param n_samples: number of samples to plot for each chain
"""
data = dataset
print 'Training Set Shape: %s' % (data.shape, )
dataset = theano.shared(data)
train_set_x = dataset
# compute number of minibatches for training, validation and testing
n_train_batches = int(train_set_x.get_value(borrow=True).shape[0] * datasize) / batch_size
print 'batch selected : %i' % (n_train_batches)
n_dim = train_set_x.get_value(borrow=True).shape[1]
# allocate symbolic variables for the data
index = T.iscalar('index') # index to a [mini]batch
x = sp.csc_matrix('x') # the data is presented as rasterized images
# initialize storage for the persistent chain (state = hidden
# layer of chain)
if PCD_k:
persistent_chain = theano.shared(numpy.zeros((batch_size, n_hidden),
dtype=theano.config.floatX))
else:
persistent_chain = None
print 'Building the model'
# construct the RBM class
# convert sparse matrix to dense matrix for operation
m = x.toarray()
rbm = RBM(input=m, n_visible=n_dim, n_hidden=n_hidden,
W=W, vbias=vbias, hbias=hbias)
# get the cost and the gradient corresponding to one step of CD-15
n_chains = T.lscalar('k')
# lr = T.fscalar('lr')
cost, updates = rbm.get_cost_updates(mom=mom, lr=learning_rate, l2=l2,
persistent=persistent_chain, k=n_chains)
free_energy = rbm.free_energy(m)
predict_score = (T.ge(rbm.propdown(T.ge(rbm.propup(m)[1], 0.5)*1)[1], 0.5)).sum()/\
m.sum()
print 'Building the Computational Graph'
#################################
# Training the RBM #
#################################
train_rbm = theano.function(
[index, n_chains],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size]
},
name='train_rbm'
)
score = theano.function(
[index],
predict_score,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size]
}
)
cal_free_energy = theano.function(
[index],
free_energy,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size]
}
)
Date = str(date.today())
# go through training epochs
file_name = save_path
print 'Start Training'
val_free = []
train_free = []
hist_cost = []
# hist_score = []
for epoch in xrange(training_epochs):
if update_scheme is None:
if 30 > epoch & epoch > 15:
PCD_k = 3
elif epoch == 0:
PCD_k = 2
elif 0 < epoch & epoch < 15:
PCD_k = 1
elif epoch >= 30:
PCD_k = 6
else:
PCD_k, learning_rate = update_scheme(epoch)
# go through the training set
mean_cost = []
for batch_index in xrange(n_train_batches):
mean_cost += [train_rbm(batch_index, PCD_k)]
# val_free += [numpy.mean(cal_free_energy(n_train_batches + 1))]
# hist_score += [score(n_train_batches)]
# train_free += [numpy.mean(cal_free_energy(n_train_batches))]
hist_cost += [numpy.mean(mean_cost)]
print 'Training epoch: %d puesdo likelihood: %f'\
% (epoch, numpy.mean(mean_cost)) # train_free[-1], val_free[-1],)
if record:
if epoch % 5 == 0:
# save parameters
# train_free = numpy.array(train_free)
# val_free = numpy.array(val_free)
hist_cost_ = numpy.array(hist_cost)
print 'Saved at' + os.getcwd()
file_flag = Date + '-' + repr(epoch)
numpy.save(file_name + 'hist_cost@' + file_flag, hist_cost_)
# numpy.save(file_name + 'Train_free@' + file_flag, train_free)
# numpy.save(file_name + 'Val_free@' + file_flag, val_free)
numpy.save(file_name + 'RBM_model@' + file_flag, rbm)
print 'Result Save @' + os.getcwd()
return train_free, val_free, hist_cost, rbm
print 'Nan detacted'
raise OverflowError
print "Layer %i Traing Epoch %i pesudo_likehood: %.3f" % \
(key, i, np.mean(epoch_cost))
his_cost += [np.mean(epoch_cost)]
def save(self):
return (self.__class__, self.__dict__)
def load(self, attributes):
self.__dict__.update(attributes)
return self
if __name__ == '__main__':
train_set = pk.load(open('/home/tn00372136/nlp-lab/Rec/data_lazy_test/\
penguin.train.rbm.debug.csc.p', 'rb')).astype(theano.config.floatX)
fitted = pk.load(open('./PTDBM.layer.p', 'rb'))['model'][-1][0]
shu = pk.load(open('./index', 'rb'))
shu[:np.ceil(train_set.shape[0]*0.7)]
train_set = train_set[shu]
train_set = theano.shared(train_set, borrow=True)
n_vis = train_set.get_value(borrow=True).shape[1]
x = sp.csc_matrix('x')
dens_x = x.toarray()
# testing the graph building process
model = PTDBN(input=dens_x, n_layers=2, n_hidden=[200, 100],
n_visible=n_vis, batch_size=100)
model.training(x, train_set_x=train_set, gamma=0.9)
pk.dump(model.save(), open('./PTDBN.p', 'wb'), -1)
def save(self):
return (self.__class__, self.__dict__)
def load(self, attributes):
self.__dict__.update(attributes)
return self
if __name__ == '__main__':
train_set = pk.load(
open(
'/home/tn00372136/nlp-lab/Rec/data_lazy_test/\
penguin.train.rbm.debug.csc.p', 'rb')).astype(theano.config.floatX)
fitted = pk.load(open('./PTDBM.layer.p', 'rb'))['model'][-1][0]
shu = pk.load(open('./index', 'rb'))
shu[:np.ceil(train_set.shape[0] * 0.7)]
train_set = train_set[shu]
train_set = theano.shared(train_set, borrow=True)
n_vis = train_set.get_value(borrow=True).shape[1]
x = sp.csc_matrix('x')
dens_x = x.toarray()
# testing the graph building process
model = PTDBN(input=dens_x,
n_layers=2,
n_hidden=[200, 100],
n_visible=n_vis,
batch_size=100)
model.training(x, train_set_x=train_set, gamma=0.9)
pk.dump(model.save(), open('./PTDBN.p', 'wb'), -1)
J, updates = theano.scan(lambda i, y, x: T.grad(y[i], x),
sequences=T.arange(y.shape[0]),
non_sequences=[y, x])
jacobian_func = function([x], J, updates=updates)
print(jacobian_func([4, 4]))
# 海森阵(hessian)
# 计算输出相对于输入的二阶偏导数
x = T.dvector('x')
y = x**2
cost = y.sum()
gy = T.grad(cost, x)
H, updates = theano.scan(lambda i, gy, x: T.grad(gy[i], x),
sequences=T.arange(gy.shape[0]),
non_sequences=[gy, x])
hessian_func = function([x], H, updates=updates)
print(hessian_func([4, 4]))
# sparse 稀疏操作
# csc 压缩列 csr 压缩行
x = sparse.csc_matrix(name='x', dtype='int32')
data, indices, indptr, shape = sparse.csm_properties(x)
# structure operation
y = sparse.structured_add(x, 2)
cac_func = function([x], y)
# csc_matrix function
a = sp.csr_matrix(np.asarray([[0, 1, 2], [0, 1, 0], [1, 0, 0]]))
print(a.toarray())
# csr to csc
print(cac_func(a).toarray())
c1L0 = connection('L0_Wi_', input_layer.output,input_shape,filter_shape, [5, 5],k=1)
layer0.addConnections([c1L0])
#layer1 = HebbianAdaptiveLayer(layer0.output,filter_shape,sigma,s1,s2,s2,final_shape,Wi=Wi,Wr=Wr)
layers = [layer0]#, layer1]'''
out = [input_layer.output]#, layer1.output]
inp = T.matrix()
#propagate = theano.function([out],y)
#updates = [(param_i + LR*layer0.state*(x-y)) for param_i in zip(params)]
#update = [(param_i, param_i + LR)]
index=T.lscalar()
csc_mat = sparse.csc_matrix('cscMat', dtype='float32')
qq,ww,ee,rr = sparse.csm_properties(csc_mat)
csc_trans = sparse.CSR(qq,ww,ee,rr)
#trans = theano.function([csc_mat],csc_trans)
Wis = []
Wrs = []
states = []
outs=[]
a = sp.csc_matrix(np.asarray([[0, 1, 1], [0, 0, 0], [1, 0, 0]],dtype='float32'))
print sparse.transpose(a).toarray()
old_W = sparse.csc_matrix('old_W',dtype='float32') # Old weight matrix
pop_i = sparse.csc_matrix('pop_i',dtype='float32') # Input layer
pop_j = sparse.csc_matrix('pop_j',dtype='float32') # Output layer
def __init__(self,
feature_count,
transformer,
k=8,
stdev=0.1,
X_format="dense"):
# ************************************************************
# * Option Processing
# ************************************************************
self.X_format = str(X_format).lower()
if self.X_format not in _SUPPORTED_FORMATS:
raise ValueError("Unsupported format: {}").format(X_format)
d = feature_count
# ************************************************************
# * Symbolic Variables
# ************************************************************
# design matrix
if X_format == "dense":
self.X = T.matrix()
elif X_format == "csr":
self.X = S.csr_matrix()
elif X_format == "csc":
self.X = S.csc_matrix()
self.y = T.vector() # response
self.s = T.vector() # sample weights
self.e = T.scalar() # current epoch
# ************************************************************
# * Model Parameters
# ************************************************************
# bias term (intercept)
w0_init = np.zeros(1)
self.w0 = theano.shared(w0_init, allow_downcast=True)
# first order coefficients
w1_init = np.zeros(d)
self.w1 = theano.shared(w1_init, allow_downcast=True)
# interaction factors
v_init = stdev * np.random.randn(k, d)
self.v = theano.shared(v_init, allow_downcast=True)
# ************************************************************
# * The Model
# ************************************************************
dot = T.dot
mul = T.mul
if X_format in ("csc", "csr"):
dot = S.dot
mul = S.mul
# The formula for pairwise interactions is from the bottom left
# of page 997 of Rendle 2010, "Factorization Machines."
# This version scales linearly in k and d, as opposed to O(d^2).
interactions = 0.5 * T.sum((dot(self.X, T.transpose(self.v)) ** 2) \
- dot(mul(self.X, self.X), T.transpose(self.v ** 2)), axis=1)
self.y_hat = self.w0[0] + dot(self.X, self.w1) + interactions
self.y_hat = transformer.transform(self.y_hat)
# ************************************************************
# * Prediction
# ************************************************************
self.theano_predict = theano.function(inputs=[self.X],
outputs=self.y_hat,
allow_input_downcast=True)
def test_DGCN(learning_rate=0.01,
L1_reg=0.00,
L2_reg=0.00,
n_epochs=200,
dataset='cora',
dropout_rate=0.3,
hidden_size=32,
cons=1,
tper=0.1):
adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask = dp.load_graph_data(
dataset)
train_mask = np.reshape(train_mask, (-1, len(train_mask)))
val_mask = np.reshape(val_mask, (-1, len(val_mask)))
test_mask = np.reshape(test_mask, (-1, len(test_mask)))
#### obtain diffusion and ppmi matrices
diffusions = diffusion_fun_sparse(adj.tocsc())
ppmi = diffusion_fun_improved_ppmi_dynamic_sparsity(adj, path_len=2, k=1.0)
#### construct the classifier model ####
rng = np.random.RandomState(1234)
tX = sparse.csc_matrix(
name='X', dtype=theano.config.floatX) # sparse matrix features
tD = sparse.csc_matrix(
name='D', dtype=theano.config.floatX) # sparse matrix diffusion
tP = sparse.csc_matrix(name='PPMI',
dtype=theano.config.floatX) # sparse matrix ppmi
tRU = T.scalar(name='ramp-up', dtype=theano.config.floatX)
feature_size = features.shape[1]
label_size = y_train.shape[1]
layer_sizes = [(feature_size, hidden_size), (hidden_size, label_size)]
print "Convolution Layers:" + str(layer_sizes)
classifier = DGCN(rng=rng,
input=tX,
layer_sizes=layer_sizes,
diffusion=tD,
ppmi=tP,
dropout_rate=dropout_rate,
nell_dataset=False)
#### loss function ####
tY = T.matrix('Y', dtype=theano.config.floatX)
tMask = T.matrix('Mask', dtype=theano.config.floatX)
cost = (classifier.supervised_loss(tY, tMask) +
tRU * classifier.unsupervised_loss() + L1_reg * classifier.l1 +
L2_reg * classifier.l2)
#### update rules for NN params ####
gparams = [T.grad(cost, param) for param in classifier.params]
updates = [(param, param - learning_rate * gparam)
for param, gparam in zip(classifier.params, gparams)]
#### the train, validation and test models ####
train_model = theano.function(inputs=[tY, tMask, tRU],
outputs=cost,
updates=updates,
givens={
tX:
features.tocsc().astype(
theano.config.floatX),
tD:
diffusions.astype(theano.config.floatX),
tP:
ppmi.astype(theano.config.floatX)
},
on_unused_input='warn')
validate_model = theano.function(
inputs=[tY, tMask],
outputs=classifier.acc(tY, tMask),
givens={
tX: features.tocsc().astype(theano.config.floatX),
tD: diffusions.astype(theano.config.floatX),
tP: ppmi.astype(theano.config.floatX)
},
on_unused_input='warn')
validate_model_cost = theano.function(
inputs=[tY, tMask, tRU],
outputs=cost,
givens={
tX: features.tocsc().astype(theano.config.floatX),
tD: diffusions.astype(theano.config.floatX),
tP: ppmi.astype(theano.config.floatX)
},
on_unused_input='warn')
test_model = theano.function(inputs=[tY, tMask],
outputs=classifier.acc(tY, tMask),
givens={
tX:
features.tocsc().astype(
theano.config.floatX),
tD:
diffusions.astype(theano.config.floatX),
tP:
ppmi.astype(theano.config.floatX)
},
on_unused_input='warn')
#### train model ####
print "...training..."
num_labels = np.sum(train_mask)
X_train_shape = features.shape[0]
accuracys = []
for epoch in range(n_epochs):
t = time.time()
scaled_unsup_weight_max = get_scaled_unsup_weight_max(
num_labels, X_train_shape, unsup_weight_max=15.0)
ramp_up = rampup(epoch,
scaled_unsup_weight_max,
exp=5.0,
rampup_length=120)
ramp_up = np.asarray(ramp_up, dtype=theano.config.floatX)
_train_cost = train_model(y_train.astype(theano.config.floatX),
train_mask.astype(theano.config.floatX),
ramp_up)
_valid_acc = validate_model(y_val.astype(theano.config.floatX),
val_mask.astype(theano.config.floatX))
_valid_cost = validate_model_cost(
y_val.astype(theano.config.floatX),
val_mask.astype(theano.config.floatX), ramp_up)
_test_acc = test_model(y_test.astype(theano.config.floatX),
test_mask.astype(theano.config.floatX))
print("Epoch:", '%04d' % (epoch + 1), "train_loss=",
'%.5f' % (_train_cost), "val_acc=", '%.5f' % (_valid_acc),
"test_acc=", '%.5f' % (_test_acc), "val_cost=",
'%.5f' % (_valid_cost), "time=", '%.5f' % (time.time() - t))
# xs.append(epoch)
accuracys.append(_test_acc)
#### test the trained model ####
test_acc = test_model(y_test.astype(theano.config.floatX),
test_mask.astype(theano.config.floatX))
print("Test Acc:", "%.5f" % (test_acc))
print("Best Test Acc:", "%.5f" % (max(accuracys)))
return test_acc, max(accuracys)
def test_sparse():
print('\n\n*************************************************')
print(' TEST SPARSE')
print('*************************************************')
# fixed parameters
bsize = 10 # batch size
imshp = (28, 28)
kshp = (5, 5)
nkern = 1 # per output pixel
ssizes = ((1, 1), (2, 2))
convmodes = (
'full',
'valid',
)
# symbolic stuff
bias = T.dvector()
kerns = T.dvector()
input = T.dmatrix()
rng = N.random.RandomState(3423489)
import theano.gof as gof
#Mode(optimizer='fast_run', linker=gof.OpWiseCLinker(allow_gc=False)),):
ntot, ttot = 0, 0
for conv_mode in convmodes:
for ss in ssizes:
output, outshp = sp.applySparseFilter(kerns, kshp,\
nkern, input, imshp, ss, bias=bias, mode=conv_mode)
f = function([kerns, bias, input], output)
# build actual input images
img2d = N.arange(bsize * N.prod(imshp)).reshape((bsize, ) +
imshp)
img1d = img2d.reshape(bsize, -1)
zeropad_img = N.zeros((bsize,\
img2d.shape[1]+2*(kshp[0]-1),\
img2d.shape[2]+2*(kshp[1]-1)))
zeropad_img[:, kshp[0] - 1:kshp[0] - 1 + img2d.shape[1],
kshp[1] - 1:kshp[1] - 1 + img2d.shape[2]] = img2d
# build kernel matrix -- flatten it for theano stuff
filters = N.arange(N.prod(outshp)*N.prod(kshp)).\
reshape(nkern,N.prod(outshp[1:]),N.prod(kshp))
spfilt = filters.flatten()
biasvals = N.arange(N.prod(outshp))
# compute output by hand
ntime1 = time.time()
refout = N.zeros((bsize, nkern, outshp[1], outshp[2]))
patch = N.zeros((kshp[0], kshp[1]))
for b in xrange(bsize):
for k in xrange(nkern):
pixi = 0 # pixel index in raster order
for j in xrange(outshp[1]):
for i in xrange(outshp[2]):
n = j * ss[0]
m = i * ss[1]
patch = zeropad_img[b, n:n + kshp[0],
m:m + kshp[1]]
refout[b,k,j,i] = N.dot(filters[k,pixi,:],\
patch.flatten())
pixi += 1
refout = refout.reshape(bsize, -1) + biasvals
ntot += time.time() - ntime1
# need to flatten images
ttime1 = time.time()
out1 = f(spfilt, biasvals, img1d)
ttot += time.time() - ttime1
temp = refout - out1
assert (temp < 1e-10).all()
# test downward propagation
vis = T.grad(output, input, output)
downprop = function([kerns, output], vis)
temp1 = time.time()
for zz in range(100):
visval = downprop(spfilt, out1)
indices, indptr, spmat_shape, sptype, outshp, kmap = \
sp.convolution_indices.sparse_eval(imshp,kshp,nkern,ss,conv_mode)
spmat = sparse.csc_matrix((spfilt[kmap], indices, indptr),
spmat_shape)
visref = N.dot(out1, spmat.todense())
assert N.all(visref == visval)
print('**** Sparse Profiling Results ****')
print('Numpy processing time: ', ntot)
print('Theano processing time: ', ttot)
tV = T.matrix()
tI = T.ivector()
bla = sparse.construct_sparse_from_list(tsize, tV, tI)
v = ones(2).astype(float32)
i = arange(2)
j = arange(2)
m = sp.csc_matrix( (v, (i,j)), shape=(4,2))
tdata = T.vector()
tindices = T.ivector()
tindptr = T.ivector()
tshape = T.ivector()
x = sparse.csc_matrix()
a,b,c,d = sparse.csm_properties(x)
print a.eval({x:m})
print b.eval({x:m})
print c.eval({x:m})
print d.eval({x:m})
tm = sparse.CSC(tdata, tindices, tindptr, tshape)
shape = array([5,3]).astype(int32)
indices = array( [0,2,4] ).astype(int32)
indptr = arange(4).astype(int32)
data = ones(3).astype(float32)
m2 = tm.eval( {tdata:data, tindices:indices, tindptr:indptr, tshape:shape})
ty = T.ivector()
