def build(self, d_init, w_init):
# Symbolic variables
self.u = T.dtensor3("u") # (M, N, L)
self.i = T.dmatrix("i") # (M, N)
self.p = T.dtensor3("p") # (K, N, L)
self.d = self.build_d(d_init)
self.w = theano.shared(w_init, name="w") # (N)
# Compute distance scores
s = T.batched_dot(self.u.dimshuffle((1, 0, 2)), self.d) # (N, M, L)
q = T.batched_dot(s, self.p.dimshuffle(
(1, 2, 0))) # (N, M, K) (order?)
# s = T.nnet.sigmoid(s)
if self.with_importance:
s = (q * self.i.dimshuffle(1, 0, 'x')).sum(0) # (M, K)
elif self.with_weights:
s = T.tensordot(q, self.w, axes=[[0], [0]]) # (M, K)
# s = T.nnet.sigmoid
else:
s = T.tensordot(q, np.ones(self.N), axes=[[0], [0]]) # (M, K)
# Final outcome
self.s = T.nnet.softmax(s) # (M, K)
inputs = [self.u, self.p]
if self.with_importance:
inputs += [self.i]
self.predict = theano.function(inputs=inputs, outputs=self.s)
self.get_latent = theano.function(inputs=inputs, outputs=q)
def test_fail(self):
# Test that conv2d fails for dimensions other than 2 or 3.
with pytest.raises(Exception):
conv.conv2d(T.dtensor4(), T.dtensor3())
with pytest.raises(Exception):
conv.conv2d(T.dtensor3(), T.dvector())
def variables(self):
# Define parameters 'w'
w = {}
for i in ['wz', 'bz', 'logsd', 'wx', 'bx']:
w[i] = T.dmatrix(i)
# Define variables 'x' and 'z'
z = {'eps': T.dtensor3('eps')}
x = {'x': T.dtensor3('x')}
return w, x, z
def variables(self):
# Define parameters 'w'
w = {}
for i in ['wz','bz','logsd','wx','bx']:
w[i] = T.dmatrix(i)
# Define variables 'x' and 'z'
z = {'eps':T.dtensor3('eps')}
x = {'x':T.dtensor3('x')}
return w, x, z
def SimpleRNN_MKL():
global x, h_init, w_x, w_h, b
X = T.dtensor3('X')
H_init = T.dmatrix('H_init')
W_x = T.dmatrix('W_x')
W_h = T.dmatrix('W_h')
B = T.dtensor3('B')
o = SimpleRNN()(X, H_init, W_x, W_h, B)
f = theano.function([X, H_init, W_x, W_h, B], o)
o_mkl = f(x, h_init, w_x, w_h, b)
return o_mkl
def test_fail(self):
"""
Test that conv2d fails for dimensions other than 2 or 3.
"""
try:
conv.conv2d(T.dtensor4(), T.dtensor3())
self.fail()
except:
pass
try:
conv.conv2d(T.dtensor3(), T.dvector())
self.fail()
except:
pass
def test_fail(self):
"""
Test that conv2d fails for dimensions other than 2 or 3.
"""
try:
conv.conv2d(T.dtensor4(), T.dtensor3())
self.fail()
except:
pass
try:
conv.conv2d(T.dtensor3(), T.dvector())
self.fail()
except:
pass
def build_model(self, train_x, train_mask_x, train_mask_out, train_target,
test_x, test_mask_x, test_mask_out, test_target):
self.train_x = train_x
self.train_mask_x = train_mask_x
self.train_mask_out = train_mask_out
self.train_target = train_target
self.test_x = test_x
self.test_mask_x = test_mask_x
self.test_mask_out = test_mask_out
self.test_target = test_target
self.index = T.iscalar('index')
self.num_batch_test = T.iscalar('index')
self.b_slice = slice(self.index * self.num_batch, (self.index + 1) * self.num_batch)
sym_x = T.dtensor3()
sym_mask_x = T.dmatrix()
sym_target = T.dtensor3()
sym_mask_out = T.dtensor3()
# sym_mask_out = T.dtensor3() should not be useful since output is still zero
# TODO think about this if it is true
out = lasagne.layers.get_output(self.model, inputs={self.l_in: sym_x, self.mask_input: sym_mask_x})
out_out = self.get_output_y(out)
loss = T.mean(lasagne.objectives.squared_error(out_out, sym_target)) / self.num_batch
out_test = lasagne.layers.get_output(self.model, inputs={self.l_in: sym_x, self.mask_input: sym_mask_x})
out_out_test = self.get_output_y(out_test)
loss_test = T.mean(lasagne.objectives.squared_error(out_out_test, sym_target)) / self.num_batch_test
all_params = [self.W] + [self.b] +lasagne.layers.get_all_params(self.model)
all_grads_target = [T.clip(g, -3, 3) for g in T.grad(loss, all_params)]
all_grads_target = lasagne.updates.total_norm_constraint(all_grads_target, 3)
updates_target = adam(all_grads_target, all_params)
train_model = theano.function([self.index],
[loss, out_out],
givens={sym_x: self.train_x[self.b_slice],
sym_mask_x: self.train_mask_x[self.b_slice],
sym_target: self.train_target[self.b_slice],
},
updates=updates_target)
test_model = theano.function([self.num_batch_test],
[loss_test, out_out_test],
givens={sym_x: self.test_x,
sym_mask_x: self.test_mask_x,
sym_target: self.test_target,
})
return train_model, test_model
def format_algs_theano_bypart(hds, sms, total_parts=46, n_algs=9, max_hd=4):
x = tt.dtensor3('x')
y = tt.dtensor3('y')
ass = np.array([i / (65 * 4 * 9) for i in xrange(9 * 65 * 4 * 46)])
# for i in xrange(46):
# print i, list(ass).count(i)
sms = sms[ass].reshape((46, 9, 65, 4))
x = tt.pow(sms, hds)
x = tt.sum(x, axis=3)
return x
def build_model(self, train_x, train_mask_x, train_mask_out, train_target,
test_x, test_mask_x, test_mask_out, test_target):
self.train_x = train_x
self.train_mask_x = train_mask_x
self.train_mask_out = train_mask_out
self.train_target = train_target
self.test_x = test_x
self.test_mask_x = test_mask_x
self.test_mask_out = test_mask_out
self.test_target = test_target
self.index = T.iscalar('index')
self.num_batch_test = T.iscalar('index')
self.b_slice = slice(self.index * self.num_batch, (self.index + 1) * self.num_batch)
sym_x = T.dtensor3()
sym_mask_x = T.dmatrix()
sym_target = T.dtensor3()
# sym_mask_out = T.dtensor3() should not be useful since output is still zero
# TODO think about this if it is true
output = lasagne.layers.get_output(self.model, inputs={self.l_in: sym_x, self.mask_input: sym_mask_x})
theta = self.get_output_y(output)
log_px = self.get_log_x(sym_target, theta)
log_px_sum_time = log_px.sum(axis=1, dtype=theano.config.floatX) # sum over tx
loss = - T.sum(log_px_sum_time) / self.num_batch # average over batch
##
log_px_test = self.get_log_x(sym_target, theta)
log_px_sum_time_test = log_px_test.sum(axis=1, dtype=theano.config.floatX) # sum over time
loss_test = - T.sum(log_px_sum_time_test) / self.num_batch_test # average over batch
# loss = T.mean(lasagne.objectives.squared_error(mu, sym_target))
all_params = [self.W_y_theta] + [self.b_y_theta] + lasagne.layers.get_all_params(self.model)
all_grads_target = [T.clip(g, -3, 3) for g in T.grad(loss, all_params)]
all_grads_target = lasagne.updates.total_norm_constraint(all_grads_target, 3)
updates_target = adam(all_grads_target, all_params)
train_model = theano.function([self.index],
[loss, theta, log_px],
givens={sym_x: self.train_x[self.b_slice],
sym_mask_x: self.train_mask_x[self.b_slice],
sym_target: self.train_target[self.b_slice]},
updates=updates_target)
test_model = theano.function([self.num_batch_test],
[loss_test, theta],
givens={sym_x: self.test_x,
sym_mask_x: self.test_mask_x,
sym_target: self.test_target})
return train_model, test_model
def __init__(self,retina=None,config=None,name=None,input_dependency=[],func=None,func_5=None,**kwargs):
self.model = retina
self.retina = retina # legacy naming, will be removed at some point
self.config = config
if self.config is None:
self.config = {}
if name is None:
name = str(uuid.uuid4())
self.name = self.config.get('name',name)
self.input_dependency = input_dependency
self.accept_dimensions = [3,5]
self._updates = None
self.collected_inputs = []
self.node_type = 'Node'
self.node_description = lambda: '- no computation -'
self.parameter_variables = []
self.state_variables = []
self.inital_states = []
self.updated_state_variables = []
self.state = None
self.compute = None
self.update_variables = theano.updates.OrderedUpdates() ## TODO!
self.__dict__.update(kwargs)
if len(self.input_dependency) == 0:
self.input_variable = T.dtensor3(self.name+"_input")
else:
self.input_variable = self.input_dependency[-1].output_variable
self.output_variable = self.input_variable+0.0# define some computation here
self.output_variable.name = self.name+'_output'
if func is not None:
d = func(self.input_variable,model=self.model,name=self.name,config=self.config)
self.__dict__.update(d)
def UV12_input(V1=Th.dmatrix(),
STAs=Th.dmatrix(),
STCs=Th.dtensor3(),
N_spikes=Th.dvector(),
**other):
other.update(locals())
return named(**other)
def test_max_pool_2d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = [(1, 2)]
imval = rng.rand(2, 3, 4)
images = tensor.dtensor3()
for maxpoolshp, ignore_border, mode in product(maxpoolshps,
[True, False],
['max', 'sum',
'average_inc_pad',
'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp,
ignore_border,
mode)
output = max_pool_2d(images, maxpoolshp, ignore_border,
mode=mode)
output_val = function([images], output)(imval)
assert numpy.all(output_val == numpy_output_val), (
"output_val is %s, numpy_output_val is %s"
% (output_val, numpy_output_val))
c = tensor.sum(output)
c_val = function([images], c)(imval)
g = tensor.grad(c, images)
g_val = function([images],
[g.shape,
tensor.min(g, axis=(0, 1, 2)),
tensor.max(g, axis=(0, 1, 2))]
)(imval)
def test_simple_3d(self):
"""Increments or sets part of a tensor by a scalar using full slice and
a partial slice depending on a scalar.
"""
a = tt.dtensor3()
increment = tt.dscalar()
sl1 = slice(None)
sl2_end = tt.lscalar()
sl2 = slice(sl2_end)
sl3 = 2
for do_set in [True, False]:
print "Set", do_set
if do_set:
resut = tt.set_subtensor(a[sl1, sl3, sl2], increment)
else:
resut = tt.inc_subtensor(a[sl1, sl3, sl2], increment)
f = theano.function([a, increment, sl2_end], resut)
val_a = numpy.ones((5, 3, 4))
val_inc = 2.3
val_sl2_end = 2
expected_result = numpy.copy(val_a)
result = f(val_a, val_inc, val_sl2_end)
if do_set:
expected_result[:, sl3, :val_sl2_end] = val_inc
else:
expected_result[:, sl3, :val_sl2_end] += val_inc
utt.assert_allclose(result, expected_result)
def make_minimizer(Model):
L, y = T.ivector('L'), T.dvector('y')
mu, eps = T.dscalar('mu'), T.dscalar('eps')
R, eta = T.dtensor3('R'), T.dvector('eta')
model = Model(L, y, mu, R, eta, eps)
return theano.function([L, y, mu, R, eta, eps], model.minimize())
def testSlidingWindowL2MaxPooling(self):
self.assertTrue(self.max_seq_len - self.filter_width > self.n_filters)
self.setSeeds()
input_shape = (self.batch_size, self.n_filters, self.max_seq_len,
self.filter_width)
output_shape = (self.batch_size, self.n_filters, self.filter_width,
self.filter_width)
x = np.zeros(shape=input_shape)
expected = np.zeros(shape=output_shape)
max_input_shape = (self.batch_size, self.filter_width,
self.filter_width)
# For the i-th filter, make i the offset at which the maximum
# L2 norm occurs.
for i in np.arange(self.n_filters):
start = i
end = i + self.filter_width
values = i + np.arange(np.prod(max_input_shape))
values = values.reshape(max_input_shape)
x[:, i, start:end, :] = values
expected[:, i, :, :] = values
it = T.iscalar()
x3d = T.dtensor3('x3d')
x4d = T.dtensor4('x4d')
layer = SlidingWindowL2MaxPooling(self.batch_size, self.n_filters,
self.filter_width, self.max_seq_len)
'''
Use the first sample and first filter to test `filter_dimension`.
'''
yt_filter_dim = layer.filter_dimension(it, x3d)
f_filter_dim = theano.function(inputs=[it, x3d], outputs=yt_filter_dim)
y_filter_dim_out = f_filter_dim(0, x[0])
self.assertEquals((self.filter_width, self.filter_width),
y_filter_dim_out.shape)
self.assertTrue(np.all(expected[0, 0, :, :] == y_filter_dim_out))
'''
Use the first sample to test `filter_dimension`.
'''
yt_sample_dim = layer.sample_dimension(it, x4d)
f_sample_dim = theano.function(inputs=[it, x4d], outputs=yt_sample_dim)
y_sample_dim_out = f_sample_dim(0, x)
self.assertEquals(
(self.n_filters, self.filter_width, self.filter_width),
y_sample_dim_out.shape)
self.assertTrue(np.all(expected[0, :, :, :] == y_sample_dim_out))
'''
Use all of `x` to test `_get_output`.
'''
yt_output = layer._get_output(x4d)
f_output = theano.function(inputs=[x4d], outputs=yt_output)
yt_out = f_output(x)
self.assertEquals((self.batch_size, self.n_filters, self.filter_width,
self.filter_width), yt_out.shape)
self.assertTrue(np.all(expected == yt_out))
def test_max_pool_2d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = [(1,2)]
imval = rng.rand(2,3,4)
images = tensor.dtensor3()
for maxpoolshp in maxpoolshps:
for ignore_border in [True,False]:
#print 'maxpoolshp =', maxpoolshp
#print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp, ignore_border)
output = max_pool_2d(images, maxpoolshp, ignore_border)
output_val = function([images], output)(imval)
assert numpy.all(output_val == numpy_output_val)
c = tensor.sum(output)
c_val = function([images], c)(imval)
g = tensor.grad(c, images)
g_val = function([images],
[g.shape,
tensor.min(g, axis=(0,1,2)),
tensor.max(g, axis=(0,1,2))]
)(imval)
def __init__(self,retina=None,config=None,name=None,input_variable=None):
self.retina = retina
self.config = config
self.state = None
if name is None:
name = str(uuid.uuid4())
self.name = self.config.get('name',name)
# 3d version
self._I = T.dtensor3(self.name+"_I")
self._preceding_V = T.dmatrix(self.name+"_preceding_V") # initial condition for sequence
self._b_0 = T.dscalar(self.name+"_b_0")
self._a_0 = T.dscalar(self.name+"_a_0")
self._a_1 = T.dscalar(self.name+"_a_1")
self._k = T.iscalar(self.name+"_k_bip") # number of iteration steps
def bipolar_step(input_image,
preceding_V,b_0, a_0, a_1):
V = (input_image * b_0 - preceding_V * a_1) / a_0
return V
# The order in theano.scan has to match the order of arguments in the function bipolar_step
self._result, self._updates = theano.scan(fn=bipolar_step,
outputs_info=[self._preceding_V],
sequences = [self._I],
non_sequences=[self._b_0, self._a_0, self._a_1],
n_steps=self._k)
self.output_varaible = self._result[0]
# The order of arguments presented here is arbitrary (will be inferred by the symbols provided),
# but function calls to compute_V_bip have to match this order!
self.compute_V = theano.function(inputs=[self._I,self._preceding_V,
self._b_0, self._a_0, self._a_1,
self._k],
outputs=self._result,
updates=self._updates)
def test_max_pool_3d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1, 1), (3, 2, 1))
imval = rng.rand(4, 5, 6)
images = tensor.dtensor3()
for maxpoolshp, ignore_border, mode in product(maxpoolshps,
[True, False],
['max', 'sum',
'average_inc_pad',
'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_nd(imval, maxpoolshp,
ignore_border,
mode=mode)
output = pool_3d(images, maxpoolshp, ignore_border,
mode=mode)
output_val = function([images], output)(imval)
utt.assert_allclose(output_val, numpy_output_val)
def mp(input):
return pool_3d(input, maxpoolshp, ignore_border,
mode=mode)
utt.verify_grad(mp, [imval], rng=rng)
def linear_parameterization( T = Th.dtensor3() , u = Th.dvector() ,
**other ):
# b = Th.dvector() , ub = Th.dvector(), **other ):
# U = ( Th.sum( T*ub , axis=2 ).T * b ).T + Th.sum( T*u , axis=2 )
U = Th.sum( T*u , axis=2 ) # U = Th.tensordot(T,u,axes=0)
other.update(locals())
return named( **other )
def __init__(self):
#return
print "Hello from toponyms.__init__"
#word2vec models
self.eng_word2vec_model = word2vec.Word2Vec.load_word2vec_format(
'c:/Scanex/Operative/Data/TEMP/news/data/wiki-100.model',
binary=False)
self.rus_word2vec_model = word2vec.Word2Vec.load_word2vec_format(
'c:/Scanex/Operative/Data/TEMP/news/data/100-hs-sg-joint.model',
binary=False)
# trained models
self.eng_model = 'c:/Scanex/Operative/Data/TEMP/news/data/model-eng.npz'
self.rus_model = 'c:/Scanex/Operative/Data/TEMP/news/data/model-rus.npz'
self.input_var = T.dtensor3('inputs')
# building nets for russian and english
self.rus_network = data.build_mlp(((None, 3, 201), 300),
self.input_var)
self.eng_network = data.build_mlp(((None, 3, 101), 150),
self.input_var)
with np.load(self.eng_model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(self.eng_network, param_values)
with np.load(self.rus_model) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(self.rus_network, param_values)
def make_minimizer(Model):
L, y = T.ivector('L'), T.dvector('y')
mu, eps = T.dscalar('mu'), T.dscalar('eps')
R, eta = T.dtensor3('R'), T.dvector('eta')
model = Model(L, y, mu, R, eta, eps)
return theano.function([L, y, mu, R, eta, eps], model.minimize())
def __build_eigens_t3(self):
t3_covs = T.dtensor3('covs')
# Get eigenvectors
egmatrix_t3, _ = theano.scan(
fn=lambda covariance: T.nlinalg.eig(covariance)[1],
sequences=[t3_covs])
# Transpose is equal to the inverse
egmatrix_inv_t3, _ = theano.scan(fn=lambda egv: egv.dimshuffle(1, 0),
sequences=[egmatrix_t3])
# Get eigenvalues
egvalues_t2, _ = theano.scan(
fn=lambda covariance: T.nlinalg.eig(covariance)[0],
sequences=[t3_covs])
# We can't have zero when computing sqrt's because
# it'll messup theano
egv_diag, _ = theano.scan(
fn=lambda x: T.nlinalg.diag(T.maximum(x, .000001)),
sequences=[egvalues_t2])
egvalues_t3, _ = theano.scan(
fn=lambda x: T.nlinalg.matrix_inverse(T.sqrt(x)),
sequences=[egv_diag])
return function(inputs=[t3_covs],
outputs=[egmatrix_t3, egvalues_t3, egmatrix_t3])
def test_max_pool_2d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = [(1, 2)]
imval = rng.rand(2, 3, 4)
images = tensor.dtensor3()
for maxpoolshp in maxpoolshps:
for ignore_border in [True, False]:
print 'maxpoolshp =', maxpoolshp
print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(
imval, maxpoolshp, ignore_border)
output = max_pool_2d(images, maxpoolshp, ignore_border)
output_val = function([images], output)(imval)
assert numpy.all(output_val == numpy_output_val)
c = tensor.sum(output)
c_val = function([images], c)(imval)
g = tensor.grad(c, images)
g_val = function([images], [
g.shape,
tensor.min(g, axis=(0, 1, 2)),
tensor.max(g, axis=(0, 1, 2))
])(imval)
def test_wrong_input(self):
"""
Make sure errors are raised when image and kernel are not 4D tensors
"""
try:
self.validate((3, 2, 8, 8), (4, 2, 5, 5),
'valid',
input=T.dmatrix())
# should never reach here
self.fail()
except:
pass
try:
self.validate((3, 2, 8, 8), (4, 2, 5, 5),
'valid',
filters=T.dvector())
# should never reach here
self.fail()
except:
pass
try:
self.validate((3, 2, 8, 8), (4, 2, 5, 5),
'valid',
input=T.dtensor3())
# should never reach here
self.fail()
except:
pass
def test_max_pool_2d_3D(self):
rng = numpy.random.RandomState(utt.fetch_seed())
maxpoolshps = [(1, 2)]
imval = rng.rand(2, 3, 4)
images = tensor.dtensor3()
for maxpoolshp, ignore_border, mode in product(
maxpoolshps, [True, False],
['max', 'average_inc_pad', 'average_exc_pad']):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_2d(imval, maxpoolshp,
ignore_border, mode)
output = max_pool_2d(images, maxpoolshp, ignore_border, mode=mode)
output_val = function([images], output)(imval)
assert numpy.all(output_val == numpy_output_val), (
"output_val is %s, numpy_output_val is %s" %
(output_val, numpy_output_val))
c = tensor.sum(output)
c_val = function([images], c)(imval)
g = tensor.grad(c, images)
g_val = function([images], [
g.shape,
tensor.min(g, axis=(0, 1, 2)),
tensor.max(g, axis=(0, 1, 2))
])(imval)
def SimpleRNN_theano():
X = T.dtensor3('X')
W_x = T.dmatrix('W_x')
W_h = T.dmatrix('W_h')
B = T.dvector('B')
Hid = T.dmatrix('hid')
def step(x, h):
h = T.tanh(T.dot(x, W_x) + T.dot(h, W_h))
return h
result, updates = theano.scan(step,
sequences=[X],
outputs_info=Hid,
name="SimpleRNN_theano")
f = theano.function([X, W_x, W_h, Hid], result)
o_theano = f(x, w_x, w_h, h_init)
"""
tic = time.time()
for i in range(1000):
o_theano = f(x, w_x, w_h, h_init)
toc = time.time()
print('Theano time: %.8f' %((toc - tic) / 1000))
"""
return o_theano
def test_simple_3d(self):
"""Increments or sets part of a tensor by a scalar using full slice and
a partial slice depending on a scalar.
"""
a = tt.dtensor3()
increment = tt.dscalar()
sl1 = slice(None)
sl2_end = tt.lscalar()
sl2 = slice(sl2_end)
sl3 = 2
for do_set in [True, False]:
print "Set", do_set
if do_set:
resut = tt.set_subtensor(a[sl1, sl3, sl2], increment)
else:
resut = tt.inc_subtensor(a[sl1, sl3, sl2], increment)
f = theano.function([a, increment, sl2_end], resut)
val_a = numpy.ones((5, 3, 4))
val_inc = 2.3
val_sl2_end = 2
expected_result = numpy.copy(val_a)
result = f(val_a, val_inc, val_sl2_end)
if do_set:
expected_result[:, sl3, :val_sl2_end] = val_inc
else:
expected_result[:, sl3, :val_sl2_end] += val_inc
self.assertTrue(numpy.array_equal(result, expected_result))
def normalize_theano_tensor(loss_tensor, lvl_acceptance):
normalized_loss_tensor = (loss_tensor - np.min(loss_tensor, axis=0)) / \
np.max(loss_tensor - np.min(loss_tensor, axis=0), axis=0)
Mshape = (np.shape(normalized_loss_tensor)[1],
np.shape(normalized_loss_tensor)[2])
mean_values = np.mean(normalized_loss_tensor, axis=2)
cropx = np.sort(mean_values)[:, lvl_acceptance]
normalized_tensor = T.dtensor3(
'normalized_tensor'
) # theano.shared(name='normalized_tensor', value=normalized_loss_tensor.astype(theano.config.floatX))
crops = T.dvector(
'crops'
) #theano.shared(name='crops', value=crops.astype(theano.config.floatX))
def crop_normalized_tensor(last, normalized_matrix, crop):
# x,y = T.dmatrices('x','y')
# v = T.where(, x, y)
normalized_matrix[T.lt(normalized_matrix, crop)] = crop
return normalized_matrix / T.max(normalized_matrix)
n_l_t, updates = theano.scan(
crop_normalized_tensor,
sequences=[normalized_tensor, crops],
outputs_info=[None,
dict(initial=T.zeros_like(normalized_tensor))])
normalizexx_loss_tensor = theano.function([normalized_tensor, crops],
n_l_t)
normalized_loss_tensor = normalizexx_loss_tensor(normalized_loss_tensor,
cropx)
return np.nan_to_num(normalized_loss_tensor)
def trainer_tester(mapping,train_data,test_data):
data = theano.shared(train_data)
test_data = theano.shared(test_data)
init_weights = 0.1*np.random.randn(len(mapping),2,100)
W = theano.shared(init_weights)
matches = T.wmatrix('matches')
weights = T.dtensor3('weights')
t_matches = T.wmatrix('t_matches')
delta = theano.shared(np.zeros(init_weights.shape))
cost, accuracy = cost_fn(matches,weights)
log_loss_fn = log_loss(t_matches,weights)
grad = T.grad(cost,wrt=weights)
train = theano.function(
inputs = [],
outputs = cost,
givens = { matches: data, weights: W },
updates = [
(W, W - 0.1*( grad + 0.5 * delta )),
(delta, 0.1*( grad + 0.5 * delta ))
]
)
test = theano.function(
inputs = [],
outputs = [log_loss_fn],
givens = { t_matches: test_data, weights: W }
)
return train,test,W
def testPredictFunc():
"""
Test the network predict function
"""
network = LSTMP2H()
symPremise = T.dtensor3("inputPremise")
symHypothesis = T.dtensor3("inputHypothesis")
premiseSent = np.random.randn(1,1,2)
hypothesisSent = np.random.randn(1,1,2)
predictFunc = network.predictFunc(symPremise, symHypothesis)
labels = network.predict(premiseSent, hypothesisSent, predictFunc)
for l in labels:
print "Label: %s" %(l)
def test_max_pool_3d_3D_deprecated_interface(self):
rng = np.random.RandomState(utt.fetch_seed())
maxpoolshps = ((1, 1, 1), (3, 2, 1))
imval = rng.rand(4, 5, 6)
images = tensor.dtensor3()
for maxpoolshp, ignore_border, mode in product(
maxpoolshps,
[True, False],
["max", "sum", "average_inc_pad", "average_exc_pad"],
):
# print 'maxpoolshp =', maxpoolshp
# print 'ignore_border =', ignore_border
numpy_output_val = self.numpy_max_pool_nd(imval,
maxpoolshp,
ignore_border,
mode=mode)
output = pool_3d(
input=images,
ds=maxpoolshp,
ignore_border=ignore_border,
st=maxpoolshp,
padding=(0, 0, 0),
mode=mode,
)
output_val = function([images], output)(imval)
utt.assert_allclose(output_val, numpy_output_val)
def mp(input):
return pool_3d(input, maxpoolshp, ignore_border, mode=mode)
def course_grain(excitation_grid, cg_factor):
""" excitation_grid should be list of 2d arrays in time order where each 2d array
is the animation state of the system at time t. The excitation_grid
of a system can be obtained using b = animator.Visual('file_name'), selecting your
desired animation range and then exporting excitation_grid = b.animation_data.
cg_factor is the unitless factor corresponding to the number of small original cells
along each side of the new course grained cell.
e.g. If a 200x200 array is processed with cg_factor = 5, the new course grained array
will be shape 40x40 where each new cell corresponds to the net excitations from 5x5
sections of the original array."""
exc = np.array(excitation_grid).astype(
'float') #Asserts data type of imported excitation_grid
filt = np.ones((cg_factor, cg_factor), dtype='float'
) #Square matrix of ones in shape of course_grained cells.
norm = cg_factor**2 #Number of original cells in each course grained cell
a = T.dtensor3(
'a'
) #Theano requires us to specify data types. dtensor3 is a 3d tensor of float64's
b = T.dmatrix('b') #Matrix of float64's
z = conv2d(a, b, subsample=(
cg_factor, cg_factor)) / norm #This specifies the function to process.
# Convolution with subsample step length results in course grained matrices
f = function(
[a, b], z
) #Theano function definition where inputs ([a,b]) and outputs (z) are specified
return f(exc,
filt) #Returns function with excitation_grid and filter as output
def preprocess_state(self, state): #TODO: Display to cross check.
"""
Preprocess a sequence of frames that make up a state.
Args:
-----
state: A sequence of frames.
Returns:
--------
Preprocessed state
"""
N, m, n = self.agent_params['state_frames'], self.game_params['crop_hei'], self.game_params['crop_wid']
factor = self.game_params['factor']
maxed = np.zeros((N, m, n), dtype='float64')
# max pool and downsample
maxed[0] = state[0].reshape(m, n)
for i in xrange(1, len(state)):
maxed[i] = np.max(np.asarray(state[i - 1: i]), axis=0).reshape(m, n)
x = tn.dtensor3('x')
f = thn.function([x], downsample.max_pool_2d(x, factor))
downsampled = f(maxed)
if self.ale_params['display_state']:
s = downsampled[-1].reshape(m / factor[0], n / factor[1])
plt.figure(1)
plt.clf()
plt.imshow(s, 'gray')
plt.pause(0.005)
return downsampled.reshape(1, np.prod(downsampled.shape[0:])) #Stack
def test_norm(self):
x = tensor.dtensor3()
a = np.random.rand(3, 2, 4).astype(theano.config.floatX)
mode = theano.compile.Mode(optimizer="fast_compile", linker="py")
for axis in [
0, 1, 2, [0], [1], [2], None, [0, 1], [1, 2], [0, 1, 2], [-1],
[-2], [-3], [-1, -2], [-1, -2, -3], [0, -2, 2]
]:
f = function([x], [
x.norm(L=1, axis=axis, keepdims=True),
self.makeKeepDims_local(
x, x.norm(L=1, axis=axis, keepdims=False), axis)
],
mode=mode)
ans1, ans2 = f(a)
assert np.allclose(ans1, ans2)
assert ans1.shape == ans2.shape
g = function([x], [
x.norm(L=2, axis=axis, keepdims=True),
self.makeKeepDims_local(
x, x.norm(L=2, axis=axis, keepdims=False), axis)
],
mode=mode)
ans1, ans2 = g(a)
assert np.allclose(ans1, ans2)
assert ans1.shape == ans2.shape
def UV( lU = Th.dmatrix('lU') , lV1 = Th.dmatrix('lV1') , V2 = Th.dvector('V2') ,
STAs = Th.dmatrix('STAs'), STCs = Th.dtensor3('STCs'), **other):
U = Th.exp(lU + 1e-10)
V1 = Th.exp(lV1+ 1e-10)
return [{'theta': Th.dot( U.T , V1[i] ) ,
'M' : Th.dot( V1[i] * U.T , (V2 * U.T).T ),
'STA': STAs[i,:],
'STC': STCs[i,:,:]} for i in range(N)]
def make_theano_tensors(list_of_datasets):
theano_tensor = [TT.dscalar(), TT.dvector(), TT.dmatrix(), TT.dtensor3()]
res = []
for d_set in list_of_datasets:
dim = len(np.array(d_set).shape)
theano_t = deepcopy(theano_tensor[dim])
res.append(theano_t)
return res
def linear_reparameterization( T = Th.dtensor3('T') , u = Th.dvector('u') ,
# V1 = Th.dmatrix('V1') , V2 = Th.dvector('V2') ,
# STAs = Th.dmatrix('STAs'), STCs = Th.dtensor3('STCs'),
# N_spikes = Th.dvector('N_spikes'),
**other):
other['U'] = Th.sum( T*u , axis=2 )
# other[name] = Th.tensordot(T,u,axes=0)
return other
def make_theano_tensors(list_of_datasets):
theano_tensor = [TT.dscalar(), TT.dvector(), TT.dmatrix(), TT.dtensor3()]
res = []
for d_set in list_of_datasets:
dim = len(np.array(d_set).shape)
theano_t = deepcopy(theano_tensor[dim])
res.append(theano_t)
return res
def create_distance_matrix(displacement_matrix):
X = T.dtensor3("d")
dist_matrix = T.sqrt(T.sum(X * X, axis=-1))
dist_func = theano.function([X], dist_matrix)
distance_matrix = dist_func(displacement_matrix)
return distance_matrix
def __build_center(self):
# We only want to compile our theano functions once
imgv = T.dtensor3("imgv")
# Get the mean
u = T.mean(imgv, 0)
# Get the standard deviation
s = T.std(T.std(imgv, 0), 0)
# Subtract our mean
return function(inputs=[imgv], outputs=[(imgv - u) / s])
def __build_center(self):
#We only want to compile our theano functions once
imgv = T.dtensor3('imgv')
# Get the mean
u = T.mean(imgv, 0)
# Get the standard deviation
s = T.std(T.std(imgv, 0), 0)
# Subtract our mean
return function(inputs=[imgv], outputs=[(imgv - u) / s])
def test_wrong_input(self):
# Make sure errors are raised when image and kernel are not 4D tensors
with pytest.raises(Exception):
self.validate((3, 2, 8, 8), (4, 2, 5, 5), "valid", input=tt.dmatrix())
with pytest.raises(Exception):
self.validate((3, 2, 8, 8), (4, 2, 5, 5), "valid", filters=tt.dvector())
with pytest.raises(Exception):
self.validate((3, 2, 8, 8), (4, 2, 5, 5), "valid", input=tt.dtensor3())
def test_wrong_input(self):
# Make sure errors are raised when image and kernel are not 4D tensors
self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5),
'valid', input=T.dmatrix())
self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5),
'valid', filters=T.dvector())
self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5),
'valid', input=T.dtensor3())
def test_REN_Cell():
cell = Cell(31, 10) # emb_dim, num_slots
_input = np.random.randn(11, 100, 31) # T,N,D
x = T.dtensor3('x')
H_init = theano.shared(np.zeros((100, 10, 31)), name='Initial Hidden')
keys_init = theano.shared(np.zeros((100, 10, 31)), name='Initial keys')
y, _ = cell(x, H_init, keys_init)
f = theano.function(inputs=[x], outputs=[y])
f(_input)
def UV( U = Th.dmatrix('U') , V1 = Th.dmatrix('V1') , V2 = Th.dvector('V2') ,
STAs = Th.dmatrix('STAs'), STCs = Th.dtensor3('STCs'),
N_spikes = Th.dvector('N_spikes'), **other):
return [{'theta': Th.dot( U.T , V1[i,:] ) ,
'M' : Th.dot( V1[i,:] * U.T , (V2 * U.T).T ),
'STA': STAs[i,:],
'STC': STCs[i,:,:],
'N_spike': N_spikes[i]/(Th.sum(N_spikes)) ,
'U' : U,
'logprior': 0. } for i in range(N)]
def UVi(i , V1 = Th.dmatrix() , STAs = Th.dmatrix(), STCs = Th.dtensor3(),
N_spikes = Th.dvector(), **other):
'''
Reparameterize a list of N (theta,M) parameters as a function of a
common U,V2 and a matrix of N rows containing V1.
'''
return named( **{'v1' : V1[i,:] ,
'STA' : STAs[i,:],
'STC' : STCs[i,:,:],
'N_spike': N_spikes[i]/(Th.sum(N_spikes))} )
def u2c_parameterization( T = Th.dtensor3() , V2 = Th.dvector() ,
u = Th.dvector() , uc = Th.dvector() ,
c = Th.dvector() , **other ):
# Ub = Th.sum( T*ub , axis=2 )
# Uc = Th.sum( T*uc , axis=2 )
U = Th.sum( T*u , axis=2 ) + ( Th.sum( T*uc , axis=2 ).T * c ).T
# U = ( Th.sum( T*ub , axis=2 ).T * b ).T + Th.sum( T*u , axis=2 )
# + ( Th.sum( T*uc , axis=2 ).T * V2 ).T
other.update(locals())
return named( **other )
def test_wrong_input(self):
"""
Make sure errors are raised when image and kernel are not 4D tensors
"""
self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5),
'valid', input=T.dmatrix())
self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5),
'valid', filters=T.dvector())
self.assertRaises(Exception, self.validate, (3, 2, 8, 8), (4, 2, 5, 5),
'valid', input=T.dtensor3())
def GRU_MKL():
X = T.dtensor3('X')
W_x = T.dmatrix('W_x')
W_h = T.dmatrix('W_h')
B = T.dvector('b')
Hid = T.dmatrix('Hid_init')
Z = GRU(hid=1000, return_sequences=True, max_len=100)(X, W_x, W_h, Hid)
f = theano.function([X, W_x, W_h, Hid], Z)
return f
def GRU_MKL():
X = T.dtensor3('X')
W_x = T.dmatrix('W_x')
W_h = T.dmatrix('W_h')
B = T.dvector('b')
Hid = T.dmatrix('Hid_init')
Z = GRU(hid=1000, return_sequences=True, max_len=100)(X, W_x, W_h, Hid, B)
f = theano.function([X, W_x, W_h, Hid, B], Z)
# theano.printing.pydotprint(f, outfile='gru.png', var_with_name_simple=True)
return f
def test_infer_shape(self):
z = tensor.dtensor3()
x = tensor.dmatrix()
y = tensor.dscalar()
self._compile_and_check([x, y], [self.op(x, y)],
[numpy.random.rand(8, 5),
numpy.random.rand()], self.op_class)
self._compile_and_check(
[z, y], [self.op(z, y)],
[numpy.random.rand(8, 8, 8),
numpy.random.rand()], self.op_class)
def test_infer_shape(self):
z = tensor.dtensor3()
x = tensor.dmatrix()
y = tensor.dscalar()
self._compile_and_check([x, y], [self.op(x, y)],
[numpy.random.rand(8, 5),
numpy.random.rand()],
self.op_class)
self._compile_and_check([z, y], [self.op(z, y)],
[numpy.random.rand(8, 8, 8),
numpy.random.rand()],
self.op_class)
def main(argv):
def formatter(prog):
return argparse.HelpFormatter(prog, max_help_position=100, width=200)
# Training labels, similarity matrix and weight of the regularization term
f, R, mu, eps = T.dvector('f'), T.dtensor3('R'), T.dvector('mu'), T.dscalar('eps')
sigma2 = T.dscalar('sigma2')
# Indices of labeled examples
l = T.ivector('l')
f_star = propagate(f, l, R, mu, eps)
ll = likelihood(f, l, R, mu, eps, sigma2)
propagate_f = theano.function([f, l, R, mu, eps], f_star, on_unused_input='warn')
likelihood_function = theano.function([f, l, R, mu, eps, sigma2], ll, on_unused_input='warn')
ll_grad = T.grad(ll, [mu, eps, sigma2])
likelihood_gradient_function = theano.function([f, l, R, mu, eps, sigma2], ll_grad, on_unused_input='warn')
nb_nodes = 64
R = np.zeros((nb_nodes, nb_nodes, 1))
even_edges = [(i, i + 2) for i in range(0, nb_nodes, 2) if (i + 2) < nb_nodes]
odd_edges = [(i, i + 2) for i in range(1, nb_nodes, 2) if (i + 2) < nb_nodes]
for source, target in even_edges + odd_edges:
R[source, target, 0], R[target, source, 0] = 1.0, 1.0
mu = np.ones(1)
eps = 1e-2
sigma2 = 1e-6
f = np.array([+ 1.0, - 1.0] + ([.0] * (nb_nodes - 2)))
l = np.array(f != 0, dtype='int8')
print(propagate_f(f, l, R, mu, eps))
learning_rate = 1e-2
for i in range(1024):
ll_value = likelihood_function(f, l, R, mu, eps, sigma2)
print('LL [%d]: %s' % (i, ll_value))
grad_value = likelihood_gradient_function(f, l, R, mu, eps, sigma2)
mu += learning_rate * grad_value[0]
eps += max(1e-6, learning_rate * grad_value[1])
sigma2 += max(1e-6, learning_rate * grad_value[2])
print('Mu: %s' % str(mu))
print('Eps: %s' % str(eps))
print('Sigma^2: %s' % str(sigma2))
def make_loss(Model, l1=0., l2=0.):
L, y = T.ivector('L'), T.dvector('y')
mu, eps = T.dscalar('mu'), T.dscalar('eps')
R, eta = T.dtensor3('R'), T.dvector('eta')
loss = Model.loss_symbolic(L, y, mu, R, eta, eps)
L1 = abs(mu) + T.sum(abs(eta)) + abs(eps)
L2 = mu ** 2 + T.sum(eta ** 2) + eps ** 2
regularized_loss = loss + l1 * L1 + l2 * L2
return theano.function([L, y, mu, R, eta, eps], regularized_loss)
def pack_param_helper_maker():
loose_cov_var = T.dtensor3('loose_cov')
loose_rot_var = T.dtensor3('loose_rot')
loose_hyd_var = T.dtensor3('loose_hyd')
loose_hydpl_var = T.dmatrix('loose_hydpl')
loose_rotpos_var = T.dmatrix('loose_rotpos')
loose_rotscalar_var = T.dmatrix('loose_rotscalar')
discrep_expr = (T.sum((unpack_rot_expr - loose_rot_var)**2) + T.sum(
(unpack_cov_expr - loose_cov_var)**2) + T.sum(
(unpack_hyd_expr - loose_hyd_var)**2) + T.sum(
(unpack_hydpl_expr - loose_hydpl_var)**2) + T.sum(
(unpack_rotpos_expr - loose_rotpos_var)**2) + T.sum(
(unpack_rotscalar_expr - loose_rotscalar_var)**2))
v = [
lparam, loose_rot_var, loose_cov_var, loose_hyd_var, loose_hydpl_var,
loose_rotpos_var, loose_rotscalar_var
]
discrep = theano.function(v, discrep_expr)
d_discrep = theano.function(v, T.grad(discrep_expr, lparam))
return discrep, d_discrep
def UV( U = Th.dmatrix('U') , V1 = Th.dmatrix('V1') , V2 = Th.dvector('V2') ,
STAs = Th.dmatrix('STAs'), STCs = Th.dtensor3('STCs'),
centers= Th.dvector('centers'), indices = Th.dmatrix('indices'), lam=Th.dscalar('lam'),
lambdas= Th.dvector('lambdas') ,
N_spikes = Th.dvector('N_spikes'), Ncones = Th.dscalar('Ncones'), **other):
return [{'theta': Th.dot( U.T , V1[i,:] ) ,
'M' : Th.dot( V1[i,:] * U.T , (V2 * U.T).T ),
'STA': STAs[i,:],
'STC': STCs[i,:,:],
'N_spike': N_spikes[i]/(Th.sum(N_spikes)) ,
'U' : U,
'logprior': - Th.sum( Th.sqrt(Th.sum(V1**2.,axis=0) + 0.000001) * lambdas) } for i in range(N)]
def test_infer_shape(self):
z = tensor.dtensor3()
x = tensor.dmatrix()
y = tensor.dscalar()
self._compile_and_check([x, y], [self.op(x, y)], [numpy.random.rand(8, 5), numpy.random.rand()], self.op_class)
self._compile_and_check(
[z, y],
[self.op(z, y)],
# must be square when nd>2
[numpy.random.rand(8, 8, 8), numpy.random.rand()],
self.op_class,
warn=False,
)
def test_norm(self):
x = tensor.dtensor3()
a = numpy.random.rand(3, 2, 4).astype(theano.config.floatX)
mode = theano.compile.Mode(optimizer="fast_compile", linker="py")
for axis in [
0,
1,
2,
[0],
[1],
[2],
None,
[0, 1],
[1, 2],
[0, 1, 2],
[-1],
[-2],
[-3],
[-1, -2],
[-1, -2, -3],
[0, -2, 2],
]:
f = function(
[x],
[
x.norm(L=1, axis=axis, keepdims=True),
self.makeKeepDims_local(x, x.norm(L=1, axis=axis, keepdims=False), axis),
],
mode=mode,
)
ans1, ans2 = f(a)
assert numpy.allclose(ans1, ans2)
assert ans1.shape == ans2.shape
g = function(
[x],
[
x.norm(L=2, axis=axis, keepdims=True),
self.makeKeepDims_local(x, x.norm(L=2, axis=axis, keepdims=False), axis),
],
mode=mode,
)
ans1, ans2 = g(a)
assert numpy.allclose(ans1, ans2)
assert ans1.shape == ans2.shape
