def show_patches_on_frames(ims, locations_, scales_,
image_shape=(100, 100), patch_shape=(16, 16)):
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
location = T.fmatrix()
scale = T.fmatrix()
x = T.fvector()
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch = cropper.apply(
x.reshape((1, 1,) + image_shape),
np.array([list(image_shape)]),
location,
scale)
get_patch = theano.function([x, location, scale], patch,
allow_input_downcast=True)
final_shape = (image_shape[0], image_shape[0] + patch_shape[0] + 5)
ret = np.ones((ims.shape[0], ) + final_shape + (3,), dtype=np.float32)
for i in range(ims.shape[0]):
im = ims[i]
location_ = locations_[i]
scale_ = scales_[i]
patch_on_frame = show_patch_on_frame(im, location_, scale_)
ret[i, :, :image_shape[1], :] = patch_on_frame
ret[i, -patch_shape[0]:, image_shape[1] + 5:, :] = to_rgb1(
get_patch(im, [location_], [scale_])[0, 0])
return ret
def apply_crop(image, image_shape, patch_shape, location, scale):
from crop import LocallySoftRectangularCropper
from crop import Gaussian
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch = cropper.apply(image, image_shape, location, scale)
return patch
def __init__(self,
input_dim,
dim,
mlp_hidden_dims,
batch_size,
image_shape,
patch_shape,
activation=None,
**kwargs):
super(LSTMAttention, self).__init__(**kwargs)
self.dim = dim
self.image_shape = image_shape
self.patch_shape = patch_shape
self.batch_size = batch_size
non_lins = [Rectifier()] * (len(mlp_hidden_dims) - 1) + [None]
mlp_dims = [input_dim + dim] + mlp_hidden_dims
mlp = MLP(non_lins,
mlp_dims,
weights_init=self.weights_init,
biases_init=self.biases_init,
name=self.name + '_mlp')
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
if not activation:
activation = Tanh()
self.children = [activation, mlp, cropper]
def __init__(self, configs, **kwargs):
super(LSTMAttention, self).__init__(**kwargs)
self.attention_mlp_hidden_dims =\
configs['attention_mlp_hidden_dims'] + [4]
self.conv_layers = configs['conv_layers']
self.fc_layers = configs['fc_layers']
self.num_layers_first_half_of_conv =\
configs['num_layers_first_half_of_conv']
self.lstm_dim = configs['lstm_dim']
self.cropper_input_shape = configs['cropper_input_shape']
self.patch_shape = configs['patch_shape']
self.batch_size = configs['batch_size']
self.num_channels = configs['num_channels']
cropper = LocallySoftRectangularCropper(patch_shape=self.patch_shape,
hyperparameters={
'cutoff': 3000,
'batched_window': True
},
kernel=Gaussian())
self.min_scale = [
(self.patch_shape[0] + 0.0) / self.cropper_input_shape[0],
(self.patch_shape[1] + 0.0) / self.cropper_input_shape[1]
]
self.children = [cropper, Tanh(), Rectifier(), HardTanh()]
patch_shape = (28, 28)
image_shape = (100, 100)
hyperparameters = {}
hyperparameters["cutoff"] = 3000
hyperparameters["batched_window"] = True
# tds, _ = get_cooking_streams(batch_size)
tds, _ = get_bmnist_streams(1)
res = tds.get_epoch_iterator(as_dict=True).next()['features']
# shape: 3 x 125 x 200
img = res[5, 0]
draw(img)
plt.savefig('img.png')
cropper = LocallySoftRectangularCropper(patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch1, matrix, dx2 = cropper.apply(
x.reshape((
batch_size,
num_channel,
) + image_shape), np.array([list(image_shape)]), location, scale, alpha)
grads = T.grad(T.mean(patch1), x)
grad_scale = abs(T.grad(T.mean(patch1), scale))
grad_location = abs(T.grad(T.mean(patch1), location))
grad_alpha = abs(T.grad(T.mean(patch1), alpha))
f = theano.function(
[x, location, scale, alpha],
[patch1, grads, grad_scale + grad_location + grad_alpha, matrix, dx2],
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
location = T.fmatrix()
scale = T.fmatrix()
x = T.fvector()
patch_shape = (28, 28)
image_shape = (100, 100)
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch = cropper.apply(
x.reshape((1, 1,) + image_shape),
np.array([list(image_shape)]),
location,
scale)
f = theano.function([x, location, scale], patch, allow_input_downcast=True)
tds, vds = get_mnist_video_streams(1)
image = tds.get_epoch_iterator().next()[0][10, 0, :]
patch1 = f(image, [[60.0, 65.0]], [[1.0, 1.0]])[0, 0]
patch2 = f(image, [[60.0, 65.0]], [[2.0, 2.0]])[0, 0]
from crop import LocallySoftRectangularCropper
from crop import Gaussian
import numpy as np
from datasets import get_bmnist_streams
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('Agg')
import matplotlib.pyplot as plt
tensor5 = T.TensorType(config.floatX, (False, ) * 4)
# shape: B x C x X x Y
input_ = tensor5('features')
cropper = LocallySoftRectangularCropper(patch_shape=(28, 28),
hyperparameters={
'cutoff': 3000,
'batched_window': True
},
kernel=Gaussian())
down, W, dx2 = cropper.apply(
input_, np.array([list((100, 100))]),
T.constant(99 * [[80, 70]]).astype('float32'),
T.constant(99 * [[0.28, 0.28]]).astype('float32'),
T.constant(99 * [[
0.001,
] * 2]).astype('float32'))
f = theano.function([input_], [down, W, dx2])
data = get_bmnist_streams(99)[0].get_epoch_iterator().next()
res = f(data[0][0])
from crop import LocallySoftRectangularCropper
from crop import Gaussian
import numpy as np
from datasets import get_bmnist_streams
import matplotlib
# Force matplotlib to not use any Xwindows backend.
matplotlib.use('Agg')
import matplotlib.pyplot as plt
tensor5 = T.TensorType(config.floatX, (False,) * 4)
# shape: B x C x X x Y
input_ = tensor5('features')
cropper = LocallySoftRectangularCropper(
patch_shape=(28, 28),
hyperparameters={'cutoff': 3000, 'batched_window': True},
kernel=Gaussian())
down, W, dx2 = cropper.apply(
input_,
np.array([list((100, 100))]),
T.constant(
99 *
[[80, 70]]).astype('float32'),
T.constant(
99 *
[[0.28, 0.28]]).astype('float32'),
T.constant(
99 *
[[0.001, ] * 2]).astype('float32'))
matplotlib.use('Agg')
import matplotlib.pyplot as plt
location = T.fmatrix()
scale = T.fmatrix()
x = T.fvector()
batch_size = 100
num_channel = 3
patch_shape = (14, 14)
image_shape = (224, 224)
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch = cropper.apply(
x.reshape((batch_size, num_channel,) + image_shape),
np.array([list(image_shape)]),
location,
scale)
f = theano.function([x, location, scale], patch, allow_input_downcast=True)
grad_l = T.grad(T.mean(patch), location)
grad_s = T.grad(T.mean(patch), scale)
g = theano.function([x, location, scale], [grad_l, grad_s], allow_input_downcast=True)
image = np.random.randn(
patch_shape = (28, 28)
image_shape = (100, 100)
hyperparameters = {}
hyperparameters["cutoff"] = 3000
hyperparameters["batched_window"] = True
# tds, _ = get_cooking_streams(batch_size)
tds, _ = get_bmnist_streams(1)
res = tds.get_epoch_iterator(as_dict=True).next()['features']
# shape: 3 x 125 x 200
img = res[5, 0]
draw(img)
plt.savefig('img.png')
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch1, matrix, dx2 = cropper.apply(
x.reshape((batch_size, num_channel,) + image_shape),
np.array([list(image_shape)]),
location,
scale,
alpha)
grads = T.grad(T.mean(patch1), x)
grad_scale = abs(T.grad(T.mean(patch1), scale))
grad_location = abs(T.grad(T.mean(patch1), location))
grad_alpha = abs(T.grad(T.mean(patch1), alpha))
f = theano.function(
[x, location, scale, alpha],
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
location = T.fmatrix()
scale = T.fmatrix()
x = T.fvector()
patch_shape = (28, 28)
image_shape = (100, 100)
hyperparameters = {}
hyperparameters["cutoff"] = 3
hyperparameters["batched_window"] = True
cropper = LocallySoftRectangularCropper(
patch_shape=patch_shape,
hyperparameters=hyperparameters,
kernel=Gaussian())
patch = cropper.apply(
x.reshape((1, 1,) + image_shape),
np.array([list(image_shape)]),
location,
scale)
f = theano.function([x, location, scale], patch, allow_input_downcast=True)
tds, vds = get_mnist_video_streams(1)
image = tds.get_epoch_iterator().next()[0][2, 0, :]
import ipdb; ipdb.set_trace()
