def input_row_from_variables(ori_ip,dest_ip,ori_lat,ori_long,dest_lat,dest_long,ori_type,dest_type,dist):
'''Create an input row for the MLP from the inputs'''
input_row = tensor.zeros([input_size])
offset = 0
ips = [ori_ip,dest_ip]
for ip in ips:
for _ in range(4):
input_row = add_one_shot(input_row, offset, tensor.mod(ip,256))
ip = tensor.int_div(ip,256)
offset += 256
for lat_,long_ in [(ori_lat,ori_long),(dest_lat,dest_long)]:
translated_lat = tensor.iround((coordinate_size-1)*(lat_/180 + 0.5))
input_row = add_thermo(input_row, offset,translated_lat)
offset += coordinate_size
translated_long = tensor.iround((coordinate_size-1)*(long_/360 + 0.5))
input_row = add_thermo(input_row, offset,translated_long)
offset += coordinate_size
for type_ in [ori_type,dest_type]:
add_one_shot(input_row, offset, type_ +1)
offset += type_size
translated_dist = tensor.iround((dest_size-1)*(tensor.minimum(1,dist/max_earth_distance)))
input_row = add_thermo(input_row, offset,translated_dist)
#could be useful if we want to add something
offset +=dest_size
return input_row
def ShiftConv(w_t_g, s_t, N):
shift = 2.*s_t-1.
Z = T.mod(shift+N, N)
simj = 1 - (Z - T.floor(Z))
imj = T.mod(T.arange(N) + T.iround(T.floor(Z)),N)
w_t_g_roll_1 = T.roll(w_t_g, -T.iround(T.floor(Z)))
w_t_g_roll_2 = T.roll(w_t_g, -(T.iround(T.floor(Z))+1))
w_t_s = w_t_g_roll_1*simj + w_t_g_roll_2*(1-simj)
return w_t_s
def rot_filters(self, theta):
fsize = self.filter_size[0]
ind = T.as_tensor_variable(
np.indices((fsize, fsize)) - (fsize - 1.0) / 2.0)
rotate = T.stack(T.cos(theta), -T.sin(theta), T.sin(theta),
T.cos(theta)).reshape((2, 2))
ind_rot = T.tensordot(rotate, ind, axes=((0, 0))) + (fsize - 1.0) / 2.0
transy = T.clip(ind_rot[0], 0, fsize - 1 - .00001)
transx = T.clip(ind_rot[1], 0, fsize - 1 - .00001)
vert = T.iround(transy)
horz = T.iround(transx)
return self.W[:, :, vert, horz]
def __init__(self, srng, data, image_shape, train_flag):
"""
A layer that applies random transformation to the input image
:type srng: theano.sandbox.rng_mrg.MRG_RandomStreams
:param srng: symbolic random number generator
:type data: theano.tensor.dtensor4
:param data: symbolic image tensor, of shape image_shape
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type train_flag: symbolic boolean
:param train_flag: whether or not it's training
"""
if train_flag is False:
self.output = data
return
p = srng.uniform(size=(1, ), ndim=1)[0]
temp = ifelse(p > .5, data, data[:, :, :, ::-1])
pad_x = 2
pad_y = 2
temp_padded = theano.shared(numpy.zeros(
shape=(image_shape[0], image_shape[1], image_shape[2] + pad_y * 2,
image_shape[3] + pad_x * 2),
dtype=theano.config.floatX),
borrow=True)
# TODO: simplify this, only need 1 number
rand_x = T.iround(
srng.uniform(size=(1, ), low=-0.49, high=4.49, ndim=1))[0]
rand_y = T.iround(
srng.uniform(size=(1, ), low=-0.49, high=4.49, ndim=1))[0]
temp_padded = T.set_subtensor(
temp_padded[:, :, rand_y:rand_y + image_shape[2],
rand_x:rand_x + image_shape[3]], temp)
self.output = temp_padded[:, :, pad_y:pad_y + image_shape[2],
pad_x:pad_x + image_shape[3]]
self.params = []
def __init__(self, srng, data, image_shape, train_flag):
"""
A layer that applies random transformation to the input image
:type srng: theano.sandbox.rng_mrg.MRG_RandomStreams
:param srng: symbolic random number generator
:type data: theano.tensor.dtensor4
:param data: symbolic image tensor, of shape image_shape
:type image_shape: tuple or list of length 4
:param image_shape: (batch size, num input feature maps,
image height, image width)
:type train_flag: symbolic boolean
:param train_flag: whether or not it's training
"""
if train_flag is False:
self.output = data
return
p = srng.uniform(size=(1,), ndim=1)[0]
temp = ifelse(p > .5, data, data[:, :, :, ::-1])
pad_x = 2
pad_y = 2
temp_padded = theano.shared(
numpy.zeros(
shape=(image_shape[0], image_shape[1], image_shape[2] + pad_y * 2, image_shape[3] + pad_x * 2),
dtype=theano.config.floatX
),
borrow=True
)
# TODO: simplify this, only need 1 number
rand_x = T.iround(srng.uniform(size=(1,), low=-0.49, high=4.49, ndim=1))[0]
rand_y = T.iround(srng.uniform(size=(1,), low=-0.49, high=4.49, ndim=1))[0]
temp_padded = T.set_subtensor(
temp_padded[:, :, rand_y:rand_y+image_shape[2], rand_x:rand_x+image_shape[3]],
temp
)
self.output = temp_padded[:, :, pad_y:pad_y+image_shape[2], pad_x:pad_x+image_shape[3]]
self.params = []
def get_pseudo_likelihood_cost(self, updates):
"""Stochastic approximation to the pseudo-likelihood"""
# index of bit i in expression p(x_i | x_{\i})
bit_i_idx = theano.shared(value=0, name = 'bit_i_idx')
# binarize the input image by rounding to nearest integer
xi = T.iround(self.input)
# calculate free energy for the given bit configuration
fe_xi = self.free_energy(xi)
# flip bit x_i of matrix xi and preserve all other bits x_{\i}
# Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx]
# NB: slice(start,stop,step) is the python object used for
# slicing, e.g. to index matrix x as follows: x[start:stop:step]
xi_flip = T.setsubtensor(xi, 1-xi[:, bit_i_idx],
idx_list=(slice(None,None,None),bit_i_idx))
# calculate free energy with bit flipped
fe_xi_flip = self.free_energy(xi_flip)
# equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i})))
cost = self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip - fe_xi))
# increment bit_i_idx % number as part of updates
updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible
return cost
def float_max_pooling(roi, target_y, target_x):
nb_channels, size_y, size_x = roi.shape
scale_y = 1.0 * size_y / target_y
start_indices_y = tt.iround(tt.arange(target_y) *
scale_y) # start indices for each block
start_indices_y = tt.clip(start_indices_y, 0, size_y - 1)
next_indices_y = tt.concatenate([start_indices_y[1:], [size_y]], axis=0)
scale_x = 1.0 * size_x / target_x
start_indices_x = tt.iround(tt.arange(target_x) * scale_x)
start_indices_x = tt.clip(start_indices_x, 0, size_x - 1)
next_indices_x = tt.concatenate([start_indices_x[1:], [size_x]], axis=0)
pool_out, _ = scan(fn=float_for_x,
sequences=[start_indices_y, next_indices_y],
non_sequences=[roi, start_indices_x, next_indices_x])
return pool_out.dimshuffle(2, 0, 1)
def get_pseudo_likelihood_cost(self, updates):
"""Stochastic approximation to the pseudo-likelihood"""
# index of bit i in expression p(x_i | x_{\i})
bit_i_idx = theano.shared(value=0, name = 'bit_i_idx')
# binarize the input image by rounding to nearest integer
xi = T.iround(self.input)
# calculate free energy for the given bit configuration
fe_xi = self.free_energy(xi)
# flip bit x_i of matrix xi and preserve all other bits x_{\i}
# Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx]
# NB: slice(start,stop,step) is the python object used for
# slicing, e.g. to index matrix x as follows: x[start:stop:step]
# In our case, idx_list is a tuple. The first element of the tuple
# describes what slice we want from the first dimension.
# ``slice(None,None,None)`` means that we want all values, equivalent
# to numpy notation ``:``. The second element of the tuple is the
# value bit_i_idx, meaning that we are looking for [:,bit_i_idx].
xi_flip = T.setsubtensor(xi, 1-xi[:, bit_i_idx],
idx_list=(slice(None,None,None),bit_i_idx))
# calculate free energy with bit flipped
fe_xi_flip = self.free_energy(xi_flip)
# equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i})))
cost = T.mean(self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip - fe_xi)))
# increment bit_i_idx % number as part of updates
updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible
return cost
def get_pseudo_likelihood_cost(self, updates):
"""Stochastic approximation to the pseudo-likelihood."""
# index of bit i in expression p(x_i | x_{\i})
bit_i_idx = theano.shared(value=0, name='bit_i_idx')
# binarize the input image by rounding to nearest integer
xi = T.iround(self.input)
# calculate free energy for the given bit configuration
fe_xi = self.free_energy(xi)
# flip bit x_i of matrix xi and preserve all other bits x_{\i}
# Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx], but assigns
# the result to xi_flip, instead of working in place on xi.
xi_flip = T.set_subtensor(xi[:, bit_i_idx], 1 - xi[:, bit_i_idx])
# calculate free energy with bit flipped
fe_xi_flip = self.free_energy(xi_flip)
# equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i})))
cost = T.mean(self.n_visible * T.log(T.nnet.sigmoid(fe_xi_flip - fe_xi)))
# increment bit_i_idx % number as part of updates
updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible
return cost
def _theano_cpu_multi_batch_beam(array, start_idxs, batch_lens, beam_width, wrap_mode, pad_left=0, pad_right=0, idx_dim=0, batch_dim=1):
array = T.as_tensor(array)
start_idxs = T.as_tensor(start_idxs)
if start_idxs.dtype.startswith("float"):
start_idxs = T.iround(start_idxs)
batch_lens = T.as_tensor(batch_lens)
if batch_lens.dtype.startswith("float"):
batch_lens = T.iround(batch_lens)
beam_width = T.as_tensor(beam_width)
if beam_width.dtype.startswith("float"):
beam_width = T.iround(beam_width)
pad_left = T.as_tensor(pad_left)
pad_right = T.as_tensor(pad_right)
assert array.ndim >= 2
assert start_idxs.ndim == 1
assert batch_lens.ndim == 1
assert beam_width.ndim == 0
assert idx_dim < array.ndim
assert batch_dim < array.ndim
assert idx_dim != batch_dim
n_batch = array.shape[batch_dim]
if idx_dim != 0: raise NotImplementedError
if batch_dim != 1: raise NotImplementedError
if wrap_mode != "wrap_around": raise NotImplementedError
idxs_0 = start_idxs.dimshuffle('x', 0) # (beam,batch)
idxs = idxs_0 + T.arange(beam_width).dimshuffle(0, 'x') # (beam,batch)
idxs_wrapped = idxs % batch_lens.dimshuffle('x', 0) # (beam,batch)
batches = T.arange(n_batch) # (batch,)
beam = array[idxs_wrapped[:, batches], batches] # (beam,batch,...)
if wrap_mode == "wrap_around":
pass # Done that.
elif wrap_mode == "pad":
cond_left = T.lt(idxs, 0) # (beam,batch)
cond_right = T.ge(idxs, batch_lens.dimshuffle('x', 0)) # (beam,batch)
cond_left_bc = cond_left.dimshuffle(beam_width, n_batch, *([1] * (array.ndim - 2)))
cond_right_bc = cond_right.dimshuffle(beam_width, n_batch, *([1] * (array.ndim - 2)))
pad_left_bc = pad_left.dimshuffle(*(['x'] * (array.ndim - pad_left.ndim) +
[pad_left.shape[i] for i in range(pad_left.ndim)]))
pad_right_bc = pad_left.dimshuffle(*(['x'] * (array.ndim - pad_right.ndim) +
[pad_right.shape[i] for i in range(pad_right.ndim)]))
beam = T.switch(cond_left_bc, beam, T.cast(pad_left_bc, dtype=array.dtype))
beam = T.switch(cond_right_bc, beam, T.cast(pad_right_bc, dtype=array.dtype))
else:
raise Exception("MultiBatchBeam: unknown wrap mode: %r" % wrap_mode)
return beam
def slice_pooling(roi, target_y, target_x):
nb_channels, size_y, size_x = roi.shape
# WxH -> axH
scale_x = 1.0 * size_x / target_x # scale to shrink/expand
slice_indices_x = tt.iround(tt.arange(target_x) *
scale_x) # indices from which to take slices
slice_indices_x = tt.clip(slice_indices_x, 0,
size_x - 1) # min = 0, max= size_x-1
# slice along x axis
roi = roi[:, :, slice_indices_x]
# axH -> axb
scale_y = 1.0 * size_y / target_y
slice_indices_y = tt.iround(tt.arange(target_y) * scale_y)
slice_indices_y = tt.clip(slice_indices_y, 0, size_y -
1) # maximum can not be greater than size_y-1
roi = roi[:, slice_indices_y, :]
return roi
def slice_pooling1(roi, target_y, target_x):
nb_channels, size_y, size_x = roi.shape
# WxH -> Wxb
scale_y = 1.0 * size_y / target_y
slice_indices_y = tt.iround(tt.arange(target_y) * scale_y)
slice_indices_y = tt.clip(slice_indices_y, 0, size_y - 1)
roi = roi[:, slice_indices_y, :]
# Wxb -> axb
scale_x = 1.0 * size_x / target_x # scale to shrink
slice_indices_x = tt.iround(tt.arange(target_x) *
scale_x) # indices from which to take slices
slice_indices_x = tt.clip(slice_indices_x, 0,
size_x - 1) # min = 0, max= size_x-1
# slice along x axis
roi = roi[:, :, slice_indices_x]
return roi
def input_row_from_variables(ori_ip, dest_ip, ori_lat, ori_long, dest_lat,
dest_long, ori_type, dest_type, dist,
latency):
'''Create an input row for the MLP from the inputs'''
input_row = tensor.zeros([input_size])
offset = 0
ips = [ori_ip, dest_ip]
for ip in ips:
for _ in range(4):
input_row = add_one_shot(input_row, offset,
tensor.mod(ip, 256))
ip = tensor.int_div(ip, 256)
offset += 256
for lat_, long_ in [(ori_lat, ori_long), (dest_lat, dest_long)]:
translated_lat = tensor.iround(
(coordinate_size - 1) * (lat_ / 180 + 0.5))
input_row = add_thermo(input_row, offset, translated_lat)
offset += coordinate_size
translated_long = tensor.iround(
(coordinate_size - 1) * (long_ / 360 + 0.5))
input_row = add_thermo(input_row, offset, translated_long)
offset += coordinate_size
for type_ in [ori_type, dest_type]:
input_row = add_one_shot(input_row, offset, type_ + 1)
offset += type_size
translated_dist = tensor.iround(
(dist_size - 1) * (tensor.minimum(1, dist / max_earth_distance)))
input_row = add_thermo(input_row, offset, translated_dist)
offset += dist_size
translated_dist = tensor.iround(
(small_dist_size - 1) *
(tensor.minimum(1, dist / max_earth_distance)))
input_row = add_thermo(input_row, offset, translated_dist)
#could be useful if we want to add something
offset += small_dist_size
return input_row
def reconstruct(self, testX, rounding=0):
if rounding:
z = T.iround(self.propdown(self.propup(testX)))
else:
z = self.propdown(self.propup(testX))
fn = theano.function([], theano.Out(z, borrow=True), name='ae_recon')
return fn()
def get_output_for(self, input, **kwargs):
# this is a bit hacky but should allow us to feed
# input in here that stems from an array containing
# multiple inputs of different types
#input = T.cast(input * self.scale_task, 'int32')
input = T.iround(input * self.scale_task)
res = self.W[input]
return res
def ShiftConv(w_t_g, s_t, N, num_shifts):
# pad = (num_shifts//2, (num_shifts-1)//2)
# w_t_g_pd_ = T.concatenate([w_t_g[(-pad[0]-1):-1], w_t_g, w_t_g[:(pad[1])]])
# w_t_g_pd = w_t_g_pd_.dimshuffle('x','x','x', 0)
# filter = s_t.dimshuffle('x', 'x', 'x', 0)
# convolution = T.nnet.conv2d(w_t_g_pd, filter,
# input_shape=(1, 1, 1, N + pad[0] + pad[1]),
# filter_shape=(1, 1, 1, num_shifts),
# subsample=(1, 1),
# border_mode='valid')
# w_t_s = convolution[0, 0, 0, :]
shift = 2.*s_t-1.
Z = T.mod(shift+N, N)
simj = 1 - (Z - T.floor(Z))
imj = T.mod(T.arange(N) + T.iround(T.floor(Z)),N)
w_t_g_roll_1 = T.roll(w_t_g, -T.iround(T.floor(Z)))
w_t_g_roll_2 = T.roll(w_t_g, -(T.iround(T.floor(Z))+1))
w_t_s = w_t_g_roll_1*simj + w_t_g_roll_2*(1-simj)
return w_t_s
def project(self, dataX, rounding=0):
"""
project dataX into hidden space. In other words, when rbm was trained,
we can get new representation
"""
if rounding:
h1_mean = T.iround(self.propup(dataX))
else:
h1_mean = self.propup(dataX)
fn = theano.function([], theano.Out(h1_mean, borrow=True), name='project')
return fn()
def input_row_from_variables(ip_, lat_, long_, type_):
'''Create an input row for the MLP from the inputs'''
input_row = tensor.zeros([input_size])
offset = 0
for _ in range(4):
input_row = add_one_shot(input_row, offset, tensor.mod(ip_, 256))
ip_ = tensor.int_div(ip_, 256)
offset += 256
translated_lat = tensor.iround((coordinate_size-1) * (lat_/180 + 0.5))
input_row = add_thermo(input_row, offset, translated_lat)
offset += coordinate_size
translated_long = tensor.iround((coordinate_size-1) * (long_/360 + 0.5))
input_row = add_thermo(input_row, offset, translated_long)
offset += coordinate_size
input_row = add_one_shot(input_row, offset, type_ +1)
offset += type_size
return input_row
def TopAccuracy2C(pred=None, truth=None, symmetric=False):
M1s = T.ones_like(truth, dtype=np.int8)
LRsel = T.triu(M1s, 24)
MLRsel = T.triu(M1s, 12)
SMLRsel = T.triu(M1s, 6)
MRsel = MLRsel - LRsel
SRsel = SMLRsel - MLRsel
dataLen = truth.shape[0]
pred0 = pred[:, :, 0]
if symmetric:
avg_pred = (pred0 + pred0.dimshuffle(1, 0)) / 2.0
else:
avg_pred = pred0
#pred_truth = T.concatenate( (avg_pred, truth.dimshuffle(0, 1, 'x') ), axis=2)
pred_truth = T.stack([avg_pred, T.cast(truth, 'int32')], axis=2)
accuracyList = []
for Rsel in [LRsel, MRsel, MLRsel, SRsel]:
selected_pred_truth = pred_truth[Rsel.nonzero()]
## sort by the predicted value for label 0 from the largest to the smallest
selected_pred_truth_sorted = selected_pred_truth[(
selected_pred_truth[:, 0]).argsort()[::-1]]
#print 'topRatio =', topRatio
numTops = T.minimum(T.iround(dataLen * topRatio),
selected_pred_truth_sorted.shape[0])
selected_sorted_truth = T.cast(
selected_pred_truth_sorted[:, -1], 'int32')
numTruths = T.bincount(selected_sorted_truth, minlength=2)
numCorrects = T.bincount(selected_sorted_truth[0:numTops],
minlength=2)
#numTops = T.minimum(numTops, numTruths[0])
accuracyList.append(
T.stack([
numCorrects[0] * 1. /
(numTops + 0.001), numTops, numTruths[0]
],
axis=0))
return T.stacklists(accuracyList)
def _per_roi_pooling(coord, x):
#x = tt.tensor3() # 512x7x7 float tensor
#coord = tt.fvector() # [ xmin, ymin, xmax, ymax ] in [0,1] x-width,y-height
# step 1: float coord to int
nb_rows = x.shape[1] # height,y
nb_cols = x.shape[2] # width,x
icoords = tt.iround(
coord * [nb_cols, nb_rows, nb_cols, nb_rows
]) # xmin,xmax multiply nb_cols, ymin,ymax multiply nb_rows
# 0 <= xmin < nb_cols
xmin = tt.clip(icoords[0], 0, nb_cols - 1)
# 0 <= ymin < nb_rows
ymin = tt.clip(icoords[1], 0, nb_rows - 1)
xmax = tt.clip(icoords[2], 1 + xmin,
nb_cols) # min(xmax) = 1+xmin, max(xmax) = nb_cols
ymax = tt.clip(icoords[3], 1 + ymin,
nb_rows) # min (ymax) = 1+ymin, max(ymax) = nb_rows
# if xmin == xmax == nb_cols
xmin = ifelse(tt.eq(xmax, xmin), xmax - 1, xmin)
# if ymin == ymax == nb_rows
ymin = ifelse(tt.eq(ymax, ymin), ymax - 1, ymin)
# step 2: extract raw sub-stensor
roi = x[:, ymin:ymax, xmin:xmax]
# step 3: resize raw to target_hx target_w
'''
# method1 (slow): upsampling -> downsampling
subtensor_h = ymax - ymin
subtensor_w = xmax - xmin
# upsample by ( target_h, target_w ) -> ( subtensor_h * target_h, subtensor_w * target_w )
kernel = tt.ones((target_h, target_w)) # create ones filter
roi_up,_ =scan(fn=lambda r2d, kernel: kron(r2d,kernel),sequences = roi,non_sequences = kernel)
# downsample to (target_h, target_w)
#target = roi_up[:,::subtensor_h,::subtensor_w]
target = max_pooling(roi_up, subtensor_h, subtensor_w)
'''
# method 2
if cfg.NET.POOL_METHOD == 'slicepool':
target = slice_pooling(roi, target_h, target_w)
else:
target = float_max_pooling(roi, target_h, target_w)
return K.flatten(target)
def get_output_for(self, input, **kwargs):
# this is a bit hacky but should allow us to feed
# input in here that stems from an array containing
# multiple inputs of different types
#input = T.cast(input * self.scale_task, 'int32')
input = T.iround(input * self.scale_task)
if self.L is not None:
#if self.rank1:
W = T.dot(self.W, self.W.T)
#else:
# W = self.W
if self.multiplicative:
#W += T.eye(self.n_tasks)
res = (1 + W[input]) * self.L[input]
else:
res = W[input] + self.L[input]
else:
res = self.W[input]
return res
def unsup_cost_pseudo_likelihood(self, updates):
index_bit_i = theano.shared(value=0, name='index_bit_i')
#visible inputs rounded to the nearest bit
vi_bit_i = T.iround(self.vis_input)
#unsupervised free energy for rounded visible inputs
vi_free_energy = self.unsup_free_energy(vi_bit_i)
#visible input with bit i flipped (i.e. 0->1, 1->0)
vi_flipped_bit_i = T.set_subtensor(vi_bit_i[:, index_bit_i], 1 - vi_bit_i[:, index_bit_i])
#unsupervised free energy for visible inputs with bit i flipped
vi_flipped_free_energy = self.unsup_free_energy(vi_flipped_bit_i)
cost = T.mean(self.vis_num * T.log(T.nnet.sigmoid(vi_flipped_free_energy - vi_free_energy)))
updates[index_bit_i] = (index_bit_i + 1) % self.vis_num
return cost
def unsup_cost_pseudo_likelihood(self, updates):
index_bit_i = theano.shared(value=0, name='index_bit_i')
#visible inputs rounded to the nearest bit
vi_bit_i = T.iround(self.vis_input)
#unsupervised free energy for rounded visible inputs
vi_free_energy = self.unsup_free_energy(vi_bit_i)
#visible input with bit i flipped (i.e. 0->1, 1->0)
vi_flipped_bit_i = T.set_subtensor(vi_bit_i[:, index_bit_i],
1 - vi_bit_i[:, index_bit_i])
#unsupervised free energy for visible inputs with bit i flipped
vi_flipped_free_energy = self.unsup_free_energy(vi_flipped_bit_i)
cost = T.mean(
self.vis_num *
T.log(T.nnet.sigmoid(vi_flipped_free_energy - vi_free_energy)))
updates[index_bit_i] = (index_bit_i + 1) % self.vis_num
return cost
def get_pseudo_likelihood_cost(self, updates):
"""Stochastic approximation to the pseudo-likelihood"""
# index of bit i in expression p(x_i | x_{\i})
bit_i_idx = theano.shared(value=0, name='bit_i_idx')
# binarize the input image by rounding to nearest integer
xi = T.iround(self.input)
# calculate free energy for the given bit configuration
fe_xi = self.free_energy(xi)
# flip bit x_i of matrix xi and preserve all other bits x_{\i}
# Equivalent to xi[:,bit_i_idx] = 1-xi[:, bit_i_idx]
# NB: slice(start,stop,step) is the python object used for
# slicing, e.g. to index matrix x as follows: x[start:stop:step]
# In our case, idx_list is a tuple. The first element of the tuple
# describes what slice we want from the first dimension.
# ``slice(None,None,None)`` means that we want all values, equivalent
# to numpy notation ``:``. The second element of the tuple is the
# value bit_i_idx, meaning that we are looking for [:,bit_i_idx].
xi_flip = T.setsubtensor(xi,
1 - xi[:, bit_i_idx],
idx_list=(slice(None, None, None), bit_i_idx))
# calculate free energy with bit flipped
fe_xi_flip = self.free_energy(xi_flip)
# equivalent to e^(-FE(x_i)) / (e^(-FE(x_i)) + e^(-FE(x_{\i})))
cost = T.mean(self.n_visible *
T.log(T.nnet.sigmoid(fe_xi_flip - fe_xi)))
# increment bit_i_idx % number as part of updates
updates[bit_i_idx] = (bit_i_idx + 1) % self.n_visible
return cost
def do_preproc(input,label):
if not self.need_proc:
return input,label
if self.deal_label:
assert len(label_shape)==3
else:
outlabel=label
srs=self.rng
w = self.image_shape[-1]
h = self.image_shape[-2]
target = T.as_tensor_variable(np.indices((self.image_shape[-2], self.image_shape[-1])))
if self.deal_label:
tarlab = T.as_tensor_variable(np.indices((self.label_shape[-2], label_shape[-1])))
lw = self.label_shape[-1]
lh = self.label_shape[-2]
# Translate
if self.translation:
transln = self.translation * srs.uniform((2, 1, 1), -1)
target += transln
if self.deal_label:
tarlab += transln
# Apply elastic transform
if self.magnitude:
# Build a gaussian filter
var = self.sigma ** 2
filt = np.array([[np.exp(-.5 * (i * i + j * j) / var)
for i in range(-self.sigma, self.sigma + 1)]
for j in range(-self.sigma, self.sigma + 1)], dtype=float_x)
filt /= 2 * np.pi * var
# Elastic
elast = self.magnitude * srs.normal((2, h, w))
elast = sigconv.conv2d(elast, filt, (2, h, w), filt.shape, 'full')
elast = elast[:, self.sigma:h + self.sigma, self.sigma:w + self.sigma]
target += elast
if deal_label:
raise NotImplementedError()
# Center at 'about' half way
if self.zoom-1 or self.angle:
origin = srs.uniform((2, 1, 1), .25, .75) * \
np.array((h, w)).reshape((2, 1, 1))
if self.deal_label:
lorigin = srs.uniform((2, 1, 1), .25, .75) * \
np.array((lh, lw)).reshape((2, 1, 1))
tarlab -= lorigin
target -= origin
# Zoom
if self.zoom-1:
zoomer = T.exp(np.log(self.zoom) * srs.uniform((2, 1, 1), -1))
target *= zoomer
if self.deal_label:
tarlab *= zoomer
# Rotate
if self.angle:
theta = self.angle * np.pi / 180 * srs.uniform(low=-1)
c, s = T.cos(theta), T.sin(theta)
rotate = T.stack(c, -s, s, c).reshape((2,2))
target = T.tensordot(rotate, target, axes=((0, 0)))
if self.deal_label:
tarlab = T.tensordot(rotate, targlab, axes=((0, 0)))
# Uncenter
target += origin
if self.deal_label:
tarlab -= lorigin
# Clip the mapping to valid range and linearly interpolate
transy = T.clip(target[0], 0, h - 1 - .001)
transx = T.clip(target[1], 0, w - 1 - .001)
if self.deal_label:
ltransy = T.clip(tarlab[0], 0, lh - 1 - .001)
ltransx = T.clip(tarlab[1], 0, lw - 1 - .001)
if self.nearest:
vert = T.iround(transy)
horz = T.iround(transx)
output = input[:, :, vert, horz]
else:
topp = T.cast(transy, 'int32')
left = T.cast(transx, 'int32')
fraction_y = T.cast(transy - topp, float_x)
fraction_x = T.cast(transx - left, float_x)
output = input[:, :, topp, left] * (1 - fraction_y) * (1 - fraction_x) + \
input[:, :, topp, left + 1] * (1 - fraction_y) * fraction_x + \
input[:, :, topp + 1, left] * fraction_y * (1 - fraction_x) + \
input[:, :, topp + 1, left + 1] * fraction_y * fraction_x
if self.deal_lab:
vert = T.iround(ltransy)
horz = T.iround(ltransx)
outlabel = label[:, vert, horz]
# Now add some noise
if self.pflip:
mask = srs.binomial(n=1, p=self.pflip, size=input.shape, dtype=float_x)
output = (1 - output) * mask + output * (1 - mask)
return output,outlabel
def get_output_for(self, grids, **kwargs):
height = width = depth = self.grid_side
# np.indices() returns 3 train_grids exactly as big as the original one.
# The first grid contains the X coordinate of each point at the location of the point.
# The second grid contains the Y coordinate of each point at the location of the point.
# The third grid contains the Z coordinate of each point at the location of the point.
indices_grids = T.as_tensor_variable(np.indices((width, height, depth),
dtype=floatX),
name="grid_indices")
# Translate:
# the translation vector will be broad-casted:
# t_x will be added to all values in the first indices grid
# t_y will be added to all values in the second indices grid
# t_z will be added to all values in the third indices grid
# resulting in a translation in the direction of translation_vector
indices_grids = T.add(indices_grids, self._translation_vector())
# Rotate:
# the origin is just the center point in the grid
origin = T.as_tensor_variable(np.array(
(width // 2, height // 2, depth // 2), dtype=floatX).reshape(
(3, 1, 1, 1)),
name='origin')
# We first center all indices, just as in the translation above
indices_grids = T.sub(indices_grids, origin)
# T.tensordot is a generalized version of a dot product.
# The axes parameter is of length 2, and it gives the axis for each of the two tensors
# passed, over which the summation will occur. Of course, those two axis need to be of the
# same dimension.
# Here we have a (3 x 3) matrix <dot product> (3, width, height, depth) grid, and the
# summation happens over the first axis (index 0). The result is of size
# (3 x width x height x depth) and contains again 3 train_grids of this time
# **rotated** indices for each dimension X, Y, Z respectively.
indices_grids = T.tensordot(self._rotation_matrix(),
indices_grids,
axes=(0, 0))
# Decenter
indices_grids = T.add(indices_grids, origin)
# Since indices_grids was transformed, we now might have indices at certain locations
# that are out of the range of the original grid. We this need to clip them to valid values.
# For the first grid: between 0 and width - 1
# For the second grid: between 0 and height - 1
# For the third grid: between 0 and depth - 1
# Note that now te index train_grids might contain real numbers (not only integers).
x_indices = T.clip(indices_grids[0], 0, width - 1 - .001)
y_indices = T.clip(indices_grids[1], 0, height - 1 - .001)
z_indices = T.clip(indices_grids[2], 0, depth - 1 - .001)
if self.interpolation == "nearest":
# Here we just need to round the indices for each spatial dimension to the closest
# integer, and than index the original input grid with the 3 indices train_grids
# (numpy style indexing with arrays) to obtain the final result. Note that here,
# as usual, the multi-dim array that you index with has the
# same spatial dimensionality as the multi-dim array being index.
# We intentionally flatten everything before indexing, so that Theano can use
# ArraySubtensor1 instead of ArraySubtensor, because only the former can be run
# on the GPU.
# https://groups.google.com/forum/#!topic/theano-users/XkPJP6on50Y
flat_grids, flat_indices = grids.reshape(
(grids.shape[0], grids.shape[1], -1)), \
width * height * T.iround(
x_indices).flatten() + \
height * T.iround(y_indices).flatten() + \
T.iround(z_indices).flatten()
output = flat_grids[:, :, flat_indices]
output = output.reshape(grids.shape)
else:
flat_grids = grids.reshape((grids.shape[0], grids.shape[1], -1))
# For linear interpolation, we use the transformed indices x_indices, y_indices and
# z_indices to linearly calculate the desired values at each of the original indices
# in each dimension.
# Again, everything is flattened so that Theano can put it on the GPU, just as in
# the other part of this if block.
# https://groups.google.com/forum/#!topic/theano-users/XkPJP6on50Y
top = T.cast(y_indices, 'int32').flatten()
left = T.cast(x_indices, 'int32').flatten()
forward = T.cast(z_indices, 'int32').flatten()
x_indices = x_indices.flatten()
y_indices = y_indices.flatten()
z_indices = z_indices.flatten()
# this computes the amount of shift into each direction from the original position
fraction_y = T.cast(y_indices - top,
theano.config.floatX).flatten()
fraction_x = T.cast(x_indices - left,
theano.config.floatX).flatten()
fraction_z = T.cast(z_indices - forward,
theano.config.floatX).flatten()
# then the new value is the linear combination based on the shifts in all
# of the 8 possible directions in 3D
output = flat_grids[:, :, self.grid_side ** 2 * top + self.grid_side * left + forward] \
* (1 - fraction_y) * (1 - fraction_x) * (1 - fraction_z) + \
flat_grids[:, :,
self.grid_side ** 2 * top + self.grid_side * left + (forward + 1)] \
* (1 - fraction_y) * (1 - fraction_x) * fraction_z + \
flat_grids[:, :,
self.grid_side ** 2 * top + self.grid_side * (left + 1) + forward] \
* (1 - fraction_y) * fraction_x * (1 - fraction_z) + \
flat_grids[:, :,
self.grid_side ** 2 * top + self.grid_side * (left + 1) + (forward + 1)] \
* (1 - fraction_y) * fraction_x * fraction_z + \
flat_grids[:, :,
self.grid_side ** 2 * (top + 1) + self.grid_side * left + forward] \
* fraction_y * (1 - fraction_x) * (1 - fraction_z) + \
flat_grids[:, :,
self.grid_side ** 2 * (top + 1) + self.grid_side * left + (forward + 1)] \
* fraction_y * (1 - fraction_x) * fraction_z + \
flat_grids[:, :,
self.grid_side ** 2 * (top + 1) + self.grid_side * (left + 1) + forward] \
* fraction_y * fraction_x * (1 - fraction_z) + \
flat_grids[:, :,
self.grid_side ** 2 * (top + 1) + self.grid_side * (left + 1) + (forward + 1)] \
* fraction_y * fraction_x * fraction_z
output = output.reshape(grids.shape)
return output
network, output = load_network(netfile)
except Exception, e:
print("Could not load network: %s" % e)
print("Loading test dataset...")
# load test data chunk
dl = DataLoader(image_size=IMAGE_SIZE)
test_filenames = dl.test_images
n_predictions = len(test_filenames)
print("Compiling theano functions...")
# set up symbolic variables
X = T.tensor4('X')
X_batch = T.tensor4('X_batch')
batch_index = T.iscalar('batch_index')
pred = T.iround(output.get_output(X_batch, deterministic=True))
predict = theano.function(
[theano.Param(X_batch)],
pred,
givens={
X: X_batch
},
)
print("Predicting...")
predictions = []
i = 0
for test_chunk in dl.test_gen():
n_batches = int(np.ceil(len(test_chunk) * 1. / BATCH_SIZE))
for b in xrange(n_batches):
predictions.append(predict(test_chunk[b * BATCH_SIZE: (b + 1) * BATCH_SIZE]))
def _theano_cpu_multi_batch_beam(array,
start_idxs,
batch_lens,
beam_width,
wrap_mode,
pad_left=0,
pad_right=0,
idx_dim=0,
batch_dim=1):
array = T.as_tensor(array)
start_idxs = T.as_tensor(start_idxs)
if start_idxs.dtype.startswith("float"):
start_idxs = T.iround(start_idxs)
batch_lens = T.as_tensor(batch_lens)
if batch_lens.dtype.startswith("float"):
batch_lens = T.iround(batch_lens)
beam_width = T.as_tensor(beam_width)
if beam_width.dtype.startswith("float"):
beam_width = T.iround(beam_width)
pad_left = T.as_tensor(pad_left)
pad_right = T.as_tensor(pad_right)
assert array.ndim >= 2
assert start_idxs.ndim == 1
assert batch_lens.ndim == 1
assert beam_width.ndim == 0
assert idx_dim < array.ndim
assert batch_dim < array.ndim
assert idx_dim != batch_dim
n_batch = array.shape[batch_dim]
if idx_dim != 0: raise NotImplementedError
if batch_dim != 1: raise NotImplementedError
if wrap_mode != "wrap_around": raise NotImplementedError
idxs_0 = start_idxs.dimshuffle('x', 0) # (beam,batch)
idxs = idxs_0 + T.arange(beam_width).dimshuffle(0, 'x') # (beam,batch)
idxs_wrapped = idxs % batch_lens.dimshuffle('x', 0) # (beam,batch)
batches = T.arange(n_batch) # (batch,)
beam = array[idxs_wrapped[:, batches], batches] # (beam,batch,...)
if wrap_mode == "wrap_around":
pass # Done that.
elif wrap_mode == "pad":
cond_left = T.lt(idxs, 0) # (beam,batch)
cond_right = T.ge(idxs, batch_lens.dimshuffle('x', 0)) # (beam,batch)
cond_left_bc = cond_left.dimshuffle(beam_width, n_batch,
*([1] * (array.ndim - 2)))
cond_right_bc = cond_right.dimshuffle(beam_width, n_batch,
*([1] * (array.ndim - 2)))
pad_left_bc = pad_left.dimshuffle(
*(['x'] * (array.ndim - pad_left.ndim) +
[pad_left.shape[i] for i in range(pad_left.ndim)]))
pad_right_bc = pad_left.dimshuffle(
*(['x'] * (array.ndim - pad_right.ndim) +
[pad_right.shape[i] for i in range(pad_right.ndim)]))
beam = T.switch(cond_left_bc, beam,
T.cast(pad_left_bc, dtype=array.dtype))
beam = T.switch(cond_right_bc, beam,
T.cast(pad_right_bc, dtype=array.dtype))
else:
raise Exception("MultiBatchBeam: unknown wrap mode: %r" % wrap_mode)
return beam
def get_output_for(self, input, **kwargs):
return T.iround(T.clip(input.pop(0), self.min, self.max),
mode="half_away_from_zero")
def __init__(self,
n_input=3,
n_memblock=100,
n_output=2,
lr=0.0001,
m=0.9,
l2rate=0.0001,
dense=True):
self.dense = dense
input_sequence = T.matrix()
gold_sequence = T.matrix() # 1, n_output
#input_sequence.tag.test_value = [[0,0,1],[0,1,0],[1,0,0]]
#gold_sequence.tag.test_value = [[1,0],[0,1],[0,0]]
''' START WEIGHTS - 0=forward; 1=backward'''
wiig = shared_normal(n_input, n_memblock,
0.01, "wiig0"), shared_normal(
n_input, n_memblock, 0.01,
"wiig1") # Weights from inputs to gates
wmig = shared_normal(
n_memblock, n_memblock, 0.01, "wmig0"), shared_normal(
n_memblock, n_memblock, 0.01,
"wmig1") # Weights from cells to gates - peepholes
#big = shared_zeros(n_memblock,"big0"),shared_zeros(n_memblock,"big1")
big = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"big0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "big1")
wifg = shared_normal(n_input, n_memblock, 0.01,
"wifg0"), shared_normal(n_input, n_memblock, 0.01,
"wifg1")
wmfg = shared_normal(n_memblock, n_memblock, 0.01,
"wmfg0"), shared_normal(n_memblock, n_memblock,
0.01, "wmfg1")
#bfg = shared_zeros(n_memblock,"bfg0"),shared_zeros(n_memblock,"bfg1")
bfg = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"bfg0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "bfg1")
wiog = shared_normal(n_input, n_memblock, 0.01,
"wiog0"), shared_normal(n_input, n_memblock, 0.01,
"wiog1")
wmog = shared_normal(n_memblock, n_memblock, 0.01,
"wmog0"), shared_normal(n_memblock, n_memblock,
0.01, "wmog1")
#bog = shared_zeros(n_memblock,"bog0"),shared_zeros(n_memblock,"bog1")
bog = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"bog0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "bog1")
wim = shared_normal(n_input, n_memblock, 0.01, "wim0"), shared_normal(
n_input, n_memblock, 0.01, "wim1") # Weight from input to mem
#bm = shared_zeros(n_memblock,"bm0"),shared_zeros(n_memblock,"bm1") # Bias from input to mem
bm = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),
"bm0"), theano.shared(
numpy.zeros(n_memblock,
dtype=theano.config.floatX), "bm1")
wmo = shared_normal(n_memblock, n_output, 0.01, "wmo0"), shared_normal(
n_memblock, n_output, 0.01, "wmo1") # Weight from input to mem
slo = theano.shared(numpy.random.normal(scale=0.01),
name="slo0"), theano.shared(
numpy.random.normal(scale=0.01), name="slo1")
bo = theano.shared(numpy.zeros(n_output, dtype=theano.config.floatX),
"bo") # Bias from input to mem
''' END OF WEIGHTS '''
self.params = wiig[0], wiig[1], big[0], big[1], wifg[0], wifg[1], bfg[
0], bfg[1], wiog[0], wiog[1], bog[0], bog[1], wmig[0], wmig[
1], wmfg[0], wmfg[1], wmog[0], wmog[1], wim[0], wim[1], bm[
0], bm[1], wmo[0], wmo[1], slo[0], slo[1], bo
''' START DELTAS - 0=forward; 1=backward'''
dwiig = shared_normal(n_input, n_memblock,
0.01, "dwiig0"), shared_normal(
n_input, n_memblock, 0.01,
"dwiig1") # Weights from inputs to gates
dwmig = shared_normal(
n_memblock, n_memblock, 0.01, "dwmig0"), shared_normal(
n_memblock, n_memblock, 0.01,
"dwmig1") # Weights from cells to gates - peepholes
#dbig = shared_zeros(n_memblock,"big0"),shared_zeros(n_memblock,"dbig1")
dbig = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"dbig0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "dbig1")
dwifg = shared_normal(n_input, n_memblock, 0.01,
"dwifg0"), shared_normal(n_input, n_memblock,
0.01, "dwifg1")
dwmfg = shared_normal(n_memblock, n_memblock, 0.01,
"dwmfg0"), shared_normal(n_memblock, n_memblock,
0.01, "dwmfg1")
#dbfg = shared_zeros(n_memblock,"bfg0"),shared_zeros(n_memblock,"dbfg1")
dbfg = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"dbfg0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "dbfg1")
dwiog = shared_normal(n_input, n_memblock, 0.01,
"dwiog0"), shared_normal(n_input, n_memblock,
0.01, "dwiog1")
dwmog = shared_normal(n_memblock, n_memblock, 0.01,
"dwmog0"), shared_normal(n_memblock, n_memblock,
0.01, "dwmog1")
#dbog = shared_zeros(n_memblock,"bog0"),shared_zeros(n_memblock,"dbog1")
dbog = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"dbog0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "dbog1")
dwim = shared_normal(n_input, n_memblock,
0.01, "dwim0"), shared_normal(
n_input, n_memblock, 0.01,
"dwim1") # Weight from input to mem
#dbm = shared_zeros(n_memblock,"bm0"),shared_zeros(n_memblock,"dbm1") # Bias from input to mem
dbm = theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX),
"dbm0"), theano.shared(
numpy.zeros(n_memblock, dtype=theano.config.floatX), "dbm1")
dwmo = shared_normal(n_memblock, n_output,
0.01, "dwmo0"), shared_normal(
n_memblock, n_output, 0.01,
"dwmo1") # Weight from input to mem
dslo = theano.shared(numpy.random.normal(scale=0.01),
name="dslo0"), theano.shared(
numpy.random.normal(scale=0.01), name="dslo1")
dbo = theano.shared(numpy.zeros(n_output, dtype=theano.config.floatX),
"dbo") # Bias from input to mem
''' END OF DELTAS '''
self.deltas = dwiig[0], dwiig[1], dbig[0], dbig[1], dwifg[0], dwifg[
1], dbfg[0], dbfg[1], dwiog[0], dwiog[1], dbog[0], dbog[1], dwmig[
0], dwmig[1], dwmfg[0], dwmfg[1], dwmog[0], dwmog[1], dwim[
0], dwim[1], dbm[0], dbm[1], dwmo[0], dwmo[1], dslo[
0], dslo[1], dbo
init_mem = shared_zeros(n_memblock)
# EXPRESSIONS - Forward
def recurrence(input, pmem, i):
i = i.value
ingate = sig(T.dot(input, wiig[i]) + T.dot(pmem, wmig[i]) + big[i])
forgate = sig(
T.dot(input, wifg[i]) + T.dot(pmem, wmfg[i]) + bfg[i])
#mem = forgate * pmem + ingate * T.tanh(T.dot(input, wim[i]) + bm[i]) # Use sig or tan???
mem = T.tanh(forgate * pmem +
ingate * T.tanh(T.dot(input, wim[i]) + bm[i])
) # instead of identity, use tanh for mem out
outgate = sig(T.dot(input, wiog[i]) + T.dot(mem, wmog[i]) + bog[i])
layerout = T.tanh(T.dot(outgate * mem, wmo[i]))
#print layerout.shape.eval()
return mem, layerout
#Forward Pass
(_,
output_sequencef), updf = theano.scan(fn=recurrence,
sequences=input_sequence,
non_sequences=0,
outputs_info=[init_mem, None])
(_,
output_sequencebp), updb = theano.scan(fn=recurrence,
sequences=input_sequence,
non_sequences=1,
outputs_info=[init_mem, None],
go_backwards=True)
output_sequenceb = output_sequencebp[::-1]
presig_output_sequence, train_updates = theano.scan(
fn=lambda x, y: (x * slo[0] + y * slo[1] + bo),
sequences=[output_sequencef, output_sequenceb],
outputs_info=[None])
# avoid log(0) for log(scan(sigmoid()))
output_sequence = sig(presig_output_sequence)
# output_sequence become a batch of output vectors
train_updates.update(updf)
train_updates.update(updb)
l2 = 0
for p in self.params:
l2 += T.sum(p * p)
# Loss Function
outloss = T.nnet.binary_crossentropy(
output_sequence, gold_sequence).mean(
) + l2 * l2rate # TODO: check if the dimensions match here
# consider using multi-category? because binary allows multiple 1's in the vector
# Backward Pass
gradient = T.grad(outloss,
self.params,
consider_constant=[input_sequence, gold_sequence])
train_updates.update(
((p, p + m * d - lr * g)
for p, g, d in zip(self.params, gradient, self.deltas)))
train_updates.update(
((d, m * d - lr * g)
for p, g, d in zip(self.params, gradient, self.deltas)))
target = T.iround(gold_sequence)
output = T.iround(output_sequence)
tp = T.sum(T.and_(target, output))
p = tp / (T.sum(target))
r = tp / (T.sum(output))
f = (2 * p * r) / (p + r)
ct = T.sum(target)
co = T.sum(output)
#self.train_function = theano.function([input_sequence,gold_sequence], [output_sequence], updates=train_updates)
self.train_function = theano.function([input_sequence, gold_sequence],
[],
updates=train_updates)
#self.validate_function = theano.function([input_sequence,gold_sequence], [outloss,output_sequence])
self.test_function = theano.function([input_sequence, gold_sequence],
[outloss, ct, co, tp])
self.generate_function = theano.function([input_sequence], output)
def __init__(self,
inpt,
img_sz,
num_maps=1,
translation=0,
zoom=1,
magnitude=0,
sigma=1,
pflip=0,
angle=0,
rand_gen=None,
invert_image=False,
nearest=False):
self.inpt = inpt
self.img_sz = img_sz
self.translation = translation
self.zoom = zoom
self.magnitude = magnitude
self.sigma = sigma
self.invert = invert_image
self.nearest = nearest
self.out_sz = img_sz
self.num_maps = num_maps
self.n_out = self.num_maps * self.out_sz**2
self.params = []
self.representation = ('Elastic Maps:{:d} Size:{:2d} Translation:{:} '
'Zoom:{} Mag:{:2d} Sig:{:2d} Noise:{} '
'Angle:{} Invert:{} '
'Interpolation: {}'.format(
self.num_maps, img_sz, translation, zoom,
magnitude, sigma, pflip, angle,
invert_image,
'Nearest' if nearest else 'Linear'))
if invert_image:
inpt = 1 - inpt
assert zoom > 0
if not (magnitude or translation or pflip or angle) and zoom == 1:
self.output = inpt
self.debugout = [self.output, tt.as_tensor_variable((0, 0))]
return
srs = tt.shared_randomstreams.RandomStreams(
rand_gen.randint(1e6) if rand_gen else None)
h = w = img_sz
# Humble as-is beginning
target = tt.as_tensor_variable(np.indices((h, w)))
# Translate
if translation:
transln = translation * srs.uniform((2, 1, 1), -1)
target += transln
# Apply elastic transform
if magnitude:
# Build a gaussian filter
var = sigma**2
filt = np.array([[
np.exp(-.5 * (i * i + j * j) / var)
for i in range(-sigma, sigma + 1)
] for j in range(-sigma, sigma + 1)],
dtype=float_x)
filt /= 2 * np.pi * var
# Elastic
elast = magnitude * srs.normal((2, h, w))
elast = sigconv.conv2d(elast, filt, (2, h, w), filt.shape, 'full')
elast = elast[:, sigma:h + sigma, sigma:w + sigma]
target += elast
# Center at 'about' half way
if zoom - 1 or angle:
origin = srs.uniform((2, 1, 1), .25, .75) * \
np.array((h, w)).reshape((2, 1, 1))
target -= origin
# Zoom
if zoom - 1:
zoomer = tt.exp(np.log(zoom) * srs.uniform((2, 1, 1), -1))
target *= zoomer
# Rotate
if angle:
theta = angle * np.pi / 180 * srs.uniform(low=-1)
c, s = tt.cos(theta), tt.sin(theta)
rotate = tt.stack(c, -s, s, c).reshape((2, 2))
target = tt.tensordot(rotate, target, axes=((0, 0)))
# Uncenter
target += origin
# Clip the mapping to valid range and linearly interpolate
transy = tt.clip(target[0], 0, h - 1 - .001)
transx = tt.clip(target[1], 0, w - 1 - .001)
if nearest:
vert = tt.iround(transy)
horz = tt.iround(transx)
output = inpt[:, :, vert, horz]
else:
topp = tt.cast(transy, 'int32')
left = tt.cast(transx, 'int32')
fraction_y = tt.cast(transy - topp, float_x)
fraction_x = tt.cast(transx - left, float_x)
output = inpt[:, :, topp, left] * (1 - fraction_y) * (1 - fraction_x) + \
inpt[:, :, topp, left + 1] * (1 - fraction_y) * fraction_x + \
inpt[:, :, topp + 1, left] * fraction_y * (1 - fraction_x) + \
inpt[:, :, topp + 1, left + 1] * fraction_y * fraction_x
# Now add some noise
if pflip:
mask = srs.binomial(n=1, p=pflip, size=inpt.shape, dtype=float_x)
output = (1 - output) * mask + output * (1 - mask)
self.output = output
self.debugout = [
self.output,
target - np.indices((h, w)),
]
if translation:
self.debugout.append(transln)
if zoom - 1 or angle:
self.debugout.append(origin)
if angle:
self.debugout.append(theta * 180 / np.pi)
if zoom - 1:
self.debugout.append(zoomer)
def theano_get_proj(rot, datar,datai): ret=[datar[T.iround(rot[2]), T.iround(rot[1]), T.iround(rot[0])].T, datai[T.iround(rot[2]), T.iround(rot[1]), T.iround(rot[0])].T] return ret
def __init__(self, inpt, img_sz,
num_maps = 1,
translation=0,
zoom=1,
magnitude=0,
sigma=1,
pflip=0,
angle=0,
rand_gen=None,
invert_image=False,
nearest=False):
self.inpt = inpt
self.img_sz = img_sz
self.translation = translation
self.zoom = zoom
self.magnitude = magnitude
self.sigma = sigma
self.invert = invert_image
self.nearest = nearest
self.out_sz = img_sz
self.num_maps = num_maps
self.n_out = self.num_maps * self.out_sz ** 2
self.params = []
self.representation = ('Elastic Maps:{:d} Size:{:2d} Translation:{:} '
'Zoom:{} Mag:{:d} Sig:{:d} Noise:{} '
'Angle:{} Invert:{} '
'Interpolation:{}'.format(
self.num_maps, img_sz,
translation, zoom, magnitude, sigma,
pflip, angle, invert_image,
'Nearest' if nearest else 'Linear'))
if invert_image:
inpt = 1 - inpt
assert zoom > 0
if not (magnitude or translation or pflip or angle) and zoom == 1:
self.output = inpt
self.debugout = [self.output, tt.as_tensor_variable((0, 0))]
return
srs = tt.shared_randomstreams.RandomStreams(rand_gen.randint(1e6)
if rand_gen else None)
h = w = img_sz
# Humble as-is beginning
target = tt.as_tensor_variable(np.indices((h, w)))
# Translate
if translation:
transln = translation * srs.uniform((2, 1, 1), -1)
target += transln
# Apply elastic transform
if magnitude:
# Build a gaussian filter
var = sigma ** 2
filt = np.array([[np.exp(-.5 * (i * i + j * j) / var)
for i in range(-sigma, sigma + 1)]
for j in range(-sigma, sigma + 1)], dtype=float_x)
filt /= 2 * np.pi * var
# Elastic
elast = magnitude * srs.normal((2, h, w))
elast = sigconv.conv2d(elast, filt, (2, h, w), filt.shape, 'full')
elast = elast[:, sigma:h + sigma, sigma:w + sigma]
target += elast
# Center at 'about' half way
if zoom-1 or angle:
origin = srs.uniform((2, 1, 1), .25, .75) * \
np.array((h, w)).reshape((2, 1, 1))
target -= origin
# Zoom
if zoom-1:
zoomer = tt.exp(np.log(zoom) * srs.uniform((2, 1, 1), -1))
target *= zoomer
# Rotate
if angle:
theta = angle * np.pi / 180 * srs.uniform(low=-1)
c, s = tt.cos(theta), tt.sin(theta)
rotate = tt.stack(c, -s, s, c).reshape((2,2))
target = tt.tensordot(rotate, target, axes=((0, 0)))
# Uncenter
target += origin
# Clip the mapping to valid range and linearly interpolate
transy = tt.clip(target[0], 0, h - 1 - .001)
transx = tt.clip(target[1], 0, w - 1 - .001)
if nearest:
vert = tt.iround(transy)
horz = tt.iround(transx)
output = inpt[:, :, vert, horz]
else:
topp = tt.cast(transy, 'int32')
left = tt.cast(transx, 'int32')
fraction_y = tt.cast(transy - topp, float_x)
fraction_x = tt.cast(transx - left, float_x)
output = inpt[:, :, topp, left] * (1 - fraction_y) * (1 - fraction_x) + \
inpt[:, :, topp, left + 1] * (1 - fraction_y) * fraction_x + \
inpt[:, :, topp + 1, left] * fraction_y * (1 - fraction_x) + \
inpt[:, :, topp + 1, left + 1] * fraction_y * fraction_x
# Now add some noise
if pflip:
mask = srs.binomial(n=1, p=pflip, size=inpt.shape, dtype=float_x)
output = (1 - output) * mask + output * (1 - mask)
self.output = output
self.debugout = [self.output,
target - np.indices((h, w)),]
if translation:
self.debugout.append(transln)
if zoom-1 or angle:
self.debugout.append(origin)
if angle:
self.debugout.append(theta*180/np.pi)
if zoom-1:
self.debugout.append(zoomer)
def __init__(self, n_input=3, n_memblock=100, n_output=2, lr=0.0001, m=0.9):
input_sequence = T.matrix()
gold_sequence = T.matrix() # 1, n_output
#input_sequence.tag.test_value = [[0,0,1],[0,1,0],[1,0,0]]
#gold_sequence.tag.test_value = [[1,0],[0,1],[0,0]]
''' START WEIGHTS - 0=forward; 1=backward'''
wiig = shared_normal(n_input, n_memblock, 0.01,"wiig0"),shared_normal(n_input, n_memblock, 0.01,"wiig1") # Weights from inputs to gates
wmig = shared_normal(n_memblock, n_memblock, 0.01,"wmig0"),shared_normal(n_memblock, n_memblock, 0.01,"wmig1") # Weights from cells to gates - peepholes
#big = shared_zeros(n_memblock,"big0"),shared_zeros(n_memblock,"big1")
big = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"big0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"big1")
wifg = shared_normal(n_input, n_memblock, 0.01,"wifg0"),shared_normal(n_input, n_memblock, 0.01,"wifg1")
wmfg = shared_normal(n_memblock, n_memblock, 0.01,"wmfg0"),shared_normal(n_memblock, n_memblock, 0.01,"wmfg1")
#bfg = shared_zeros(n_memblock,"bfg0"),shared_zeros(n_memblock,"bfg1")
bfg = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"bfg0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"bfg1")
wiog = shared_normal(n_input, n_memblock, 0.01,"wiog0"),shared_normal(n_input, n_memblock, 0.01,"wiog1")
wmog = shared_normal(n_memblock, n_memblock, 0.01,"wmog0"),shared_normal(n_memblock, n_memblock, 0.01,"wmog1")
#bog = shared_zeros(n_memblock,"bog0"),shared_zeros(n_memblock,"bog1")
bog = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"bog0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"bog1")
wim = shared_normal(n_input, n_memblock, 0.01,"wim0"),shared_normal(n_input, n_memblock, 0.01,"wim1") # Weight from input to mem
#bm = shared_zeros(n_memblock,"bm0"),shared_zeros(n_memblock,"bm1") # Bias from input to mem
bm = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"bm0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"bm1")
wmo = shared_normal(n_memblock, n_output, 0.01,"wmo0"),shared_normal(n_memblock, n_output, 0.01,"wmo1") # Weight from input to mem
bo = theano.shared(numpy.zeros(n_output, dtype=theano.config.floatX),"bo") # Bias from input to mem
''' END OF WEIGHTS '''
self.params = wiig[0], big[0], wifg[0], bfg[0], wiog[0], bog[0], wmig[0], wmfg[0], wmog[0], wim[0], bm[0], wmo[0], wiig[1], big[1], wifg[1], bfg[1], wiog[1], bog[1], wmig[1], wmfg[1], wmog[1], wim[1], bm[1], wmo[1], bo
''' START DELTAS - 0=forward; 1=backward'''
dwiig = shared_normal(n_input, n_memblock, 0.01,"dwiig0"),shared_normal(n_input, n_memblock, 0.01,"dwiig1") # Weights from inputs to gates
dwmig = shared_normal(n_memblock, n_memblock, 0.01,"dwmig0"),shared_normal(n_memblock, n_memblock, 0.01,"dwmig1") # Weights from cells to gates - peepholes
#dbig = shared_zeros(n_memblock,"big0"),shared_zeros(n_memblock,"dbig1")
dbig = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbig0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbig1")
dwifg = shared_normal(n_input, n_memblock, 0.01,"dwifg0"),shared_normal(n_input, n_memblock, 0.01,"dwifg1")
dwmfg = shared_normal(n_memblock, n_memblock, 0.01,"dwmfg0"),shared_normal(n_memblock, n_memblock, 0.01,"dwmfg1")
#dbfg = shared_zeros(n_memblock,"bfg0"),shared_zeros(n_memblock,"dbfg1")
dbfg = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbfg0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbfg1")
dwiog = shared_normal(n_input, n_memblock, 0.01,"dwiog0"),shared_normal(n_input, n_memblock, 0.01,"dwiog1")
dwmog = shared_normal(n_memblock, n_memblock, 0.01,"dwmog0"),shared_normal(n_memblock, n_memblock, 0.01,"dwmog1")
#dbog = shared_zeros(n_memblock,"bog0"),shared_zeros(n_memblock,"dbog1")
dbog = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbog0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbog1")
dwim = shared_normal(n_input, n_memblock, 0.01,"dwim0"),shared_normal(n_input, n_memblock, 0.01,"dwim1") # Weight from input to mem
#dbm = shared_zeros(n_memblock,"bm0"),shared_zeros(n_memblock,"dbm1") # Bias from input to mem
dbm = theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbm0"),theano.shared(numpy.zeros(n_memblock, dtype=theano.config.floatX),"dbm1")
dwmo = shared_normal(n_memblock, n_output, 0.01,"dwmo0"),shared_normal(n_memblock, n_output, 0.01,"dwmo1") # Weight from input to mem
dbo = theano.shared(numpy.zeros(n_output, dtype=theano.config.floatX),"dbo") # Bias from input to mem
''' END OF DELTAS '''
self.deltas = dwiig[0], dbig[0], dwifg[0], dbfg[0], dwiog[0], dbog[0], dwmig[0], dwmfg[0], dwmog[0], dwim[0], dbm[0], dwmo[0], dwiig[1], dbig[1], dwifg[1], dbfg[1], dwiog[1], dbog[1], dwmig[1], dwmfg[1], dwmog[1], dwim[1], dbm[1], dwmo[1], dbo
init_mem = shared_zeros(n_memblock)
# EXPRESSIONS - Forward
def recurrence(input, pmem, i):
i = i.value
ingate = sig(T.dot(input, wiig[i]) + T.dot(pmem, wmig[i]) + big[i])
forgate = sig(T.dot(input, wifg[i]) + T.dot(pmem, wmfg[i]) + bfg[i])
mem = forgate * pmem + ingate * T.tanh(T.dot(input, wim[i]) + bm[i]) # Use sig or tan???
outgate = sig(T.dot(input, wiog[i]) + T.dot(mem, wmog[i]) + bog[i])
layerout = T.dot(outgate * mem, wmo[i])
#output = sig(T.dot(outgate * mem, wmo) + bo)
return mem, layerout
#Forward Pass
(mem_sequencef, output_sequencef), updf = theano.scan(fn=recurrence,
sequences = input_sequence,
non_sequences = 0,
outputs_info = [init_mem, None])
(mem_sequenceb, output_sequenceb), updb = theano.scan(fn=recurrence,
sequences = input_sequence,
non_sequences = 1,
outputs_info = [init_mem, None],
go_backwards=True)
output_sequenceb = output_sequenceb[::-1]
output_sequence, train_updates = theano.scan(fn=lambda x, y: sig(x + y + bo),
sequences = [output_sequencef, output_sequenceb],
outputs_info=[None])
train_updates.update(updf)
train_updates.update(updb)
# output_sequence become a batch of output vectors
# Loss Function
outloss = T.nnet.binary_crossentropy(output_sequence, gold_sequence).mean() # TODO: check if the dimensions match here
# consider using multi-category? because binary allows multiple 1's in the vector
# Backward Pass
gradient = T.grad(outloss, self.params, consider_constant=[input_sequence, gold_sequence])
train_updates.update(((p, p + m * d - lr * g) for p, g, d in zip(self.params, gradient, self.deltas)))
train_updates.update(((d, m * d - lr * g) for p, g, d in zip(self.params, gradient, self.deltas)))
target = T.iround(gold_sequence)
output = T.iround(output_sequence)
tp = T.sum(T.and_(target,output))
p = tp/(T.sum(target))
r = tp/(T.sum(output))
f = ( 2 * p * r )/(p+r)
ct = T.sum(target)
co = T.sum(output)
#self.train_function = theano.function([input_sequence,gold_sequence], [output_sequence], updates=train_updates)
self.train_function = theano.function([input_sequence,gold_sequence], [], updates=train_updates)
#self.validate_function = theano.function([input_sequence,gold_sequence], [outloss,output_sequence])
self.test_function = theano.function([input_sequence,gold_sequence], [outloss, ct, co, tp])
self.generate_function = theano.function([input_sequence], output)
def train(self, samples, labels, valid_samples=[], valid_labels=None):
# Check parameters.
for i in range(labels.size):
assert labels[i] in range(self.nb_classes)
nb_samples = len(samples) if self.nb_samples == None else self.nb_samples
# Set and compile the theano gradient descent update function.
# Tensor for latent vectors.
lat = theano.shared(
np.empty([self.nb_classes, nb_samples, self.nb_features],
dtype=theano.config.floatX),
name='lat'
)
# Tensor for latent constants.
lat_cst = theano.shared(
np.empty([self.nb_classes, nb_samples],
dtype=theano.config.floatX),
name='lat_cst'
)
# Cost function.
regularization = (
0.5 * T.dot(T.flatten(self.beta), T.flatten(self.beta))
)
scores = T.batched_dot(lat, self.beta[1:,:].T).T + lat_cst.T + self.beta[0,:]
losses = T.log(T.nnet.softmax(scores)[T.arange(nb_samples), labels])
cost_sym = (
regularization - self.C * T.sum(losses)
)
# Optimization of the cost with RPROP.
weights_shape = self.beta.get_value().shape
nb_weights = np.prod(weights_shape)
# Keep step for each weight into a shared theano vector.
steps = theano.shared(
np.array(
[self.learning_rate] * nb_weights,
theano.config.floatX
).reshape(weights_shape),
name='steps'
)
# Gradient of the cost function.
grad = T.grad(cost_sym, self.beta)
grad_f = theano.function(
[],
grad
)
# Perform the first iteration "manually", to initialize prev_grad properly.
lat_val, cst_val = self.latent_function(
self.beta.get_value()[1:,:],
samples,
labels,
self.latent_args
)
assert lat_val.shape == (self.nb_classes, nb_samples, self.nb_features)
assert cst_val.shape == (self.nb_classes, nb_samples)
lat.set_value(lat_val)
lat_cst.set_value(cst_val)
init_grad = grad_f()
self.beta.set_value(self.beta.get_value() - steps.get_value()
* np.sign(init_grad))
# Keep the gradient from the previous iteration into another shared theano
# vector.
prev_grad = theano.shared(
init_grad,
name='g(t-1)'
)
# Update rules for RPROP, scaling weight-wise step up when gradient signs
# agree, down when they disagree.
# Vector containing 0 if the sign of the gradients disagree, 1 otherwise.
sign_idx = T.iround((T.sgn(prev_grad * grad) + 1.)/2.).flatten()
# Using the previously defined indices, we index into a matrix to get the
# new step vector.
new_steps = T.stack(
self.dec_rate * steps.flatten(), self.inc_rate * steps.flatten()
)[sign_idx, T.arange(nb_weights)].reshape(weights_shape)
# Specifies the updates at each iteration. We update the steps, the
# gradient from the previous iteration, and the actual descent.
updates = [
(steps, T.clip(new_steps, 10E-5, 0.1)),
(prev_grad, grad),
(self.beta, self.beta - steps * T.sgn(grad))
]
# Full theano rprop function. Outputs the cost and gradient norm at the
# current point, and updates all the variables accordingly.
rprop_descent = theano.function(
[],
[cost_sym, grad.norm(2)],
updates=updates
)
eps = 10E-3
for t_gd in range(self.nb_gd_iter):
# Compute the best negative latent vectors.
lat_val, cst_val = self.latent_function(
self.beta.get_value()[1:,:],
samples,
labels,
self.latent_args
)
lat.set_value(lat_val)
lat_cst.set_value(cst_val)
cost_val, grad_norm = rprop_descent()
if self.verbose:
print "Epoch " + repr(t_gd + 1)
print "Cost: " + repr(cost_val)
print "Gradient norm: " + repr(grad_norm)
print "Mean step size: " + repr(steps.get_value().mean())
if grad_norm <= eps:
break
self.intercept_ = self.beta.get_value()[0,:]
self.coef_ = self.beta.get_value()[1:,:]
# Trick for Theano to free GPU memory, hopefully.
lat.set_value([[[]]])
lat_cst.set_value([[]])
