def show_patches_on_frames(ims, locations_, scales_, image_shape=(100, 100), patch_shape=(16, 16)): hyperparameters = {} hyperparameters["cutoff"] = 3 hyperparameters["batched_window"] = True location = T.fmatrix() scale = T.fmatrix() x = T.fvector() cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch = cropper.apply( x.reshape((1, 1,) + image_shape), np.array([list(image_shape)]), location, scale) get_patch = theano.function([x, location, scale], patch, allow_input_downcast=True) final_shape = (image_shape[0], image_shape[0] + patch_shape[0] + 5) ret = np.ones((ims.shape[0], ) + final_shape + (3,), dtype=np.float32) for i in range(ims.shape[0]): im = ims[i] location_ = locations_[i] scale_ = scales_[i] patch_on_frame = show_patch_on_frame(im, location_, scale_) ret[i, :, :image_shape[1], :] = patch_on_frame ret[i, -patch_shape[0]:, image_shape[1] + 5:, :] = to_rgb1( get_patch(im, [location_], [scale_])[0, 0]) return ret
def apply_crop(image, image_shape, patch_shape, location, scale): from crop import LocallySoftRectangularCropper from crop import Gaussian hyperparameters = {} hyperparameters["cutoff"] = 3 hyperparameters["batched_window"] = True cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch = cropper.apply(image, image_shape, location, scale) return patch
def __init__(self, input_dim, dim, mlp_hidden_dims, batch_size, image_shape, patch_shape, activation=None, **kwargs): super(LSTMAttention, self).__init__(**kwargs) self.dim = dim self.image_shape = image_shape self.patch_shape = patch_shape self.batch_size = batch_size non_lins = [Rectifier()] * (len(mlp_hidden_dims) - 1) + [None] mlp_dims = [input_dim + dim] + mlp_hidden_dims mlp = MLP(non_lins, mlp_dims, weights_init=self.weights_init, biases_init=self.biases_init, name=self.name + '_mlp') hyperparameters = {} hyperparameters["cutoff"] = 3 hyperparameters["batched_window"] = True cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) if not activation: activation = Tanh() self.children = [activation, mlp, cropper]
def __init__(self, configs, **kwargs): super(LSTMAttention, self).__init__(**kwargs) self.attention_mlp_hidden_dims =\ configs['attention_mlp_hidden_dims'] + [4] self.conv_layers = configs['conv_layers'] self.fc_layers = configs['fc_layers'] self.num_layers_first_half_of_conv =\ configs['num_layers_first_half_of_conv'] self.lstm_dim = configs['lstm_dim'] self.cropper_input_shape = configs['cropper_input_shape'] self.patch_shape = configs['patch_shape'] self.batch_size = configs['batch_size'] self.num_channels = configs['num_channels'] cropper = LocallySoftRectangularCropper(patch_shape=self.patch_shape, hyperparameters={ 'cutoff': 3000, 'batched_window': True }, kernel=Gaussian()) self.min_scale = [ (self.patch_shape[0] + 0.0) / self.cropper_input_shape[0], (self.patch_shape[1] + 0.0) / self.cropper_input_shape[1] ] self.children = [cropper, Tanh(), Rectifier(), HardTanh()]
patch_shape = (28, 28) image_shape = (100, 100) hyperparameters = {} hyperparameters["cutoff"] = 3000 hyperparameters["batched_window"] = True # tds, _ = get_cooking_streams(batch_size) tds, _ = get_bmnist_streams(1) res = tds.get_epoch_iterator(as_dict=True).next()['features'] # shape: 3 x 125 x 200 img = res[5, 0] draw(img) plt.savefig('img.png') cropper = LocallySoftRectangularCropper(patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch1, matrix, dx2 = cropper.apply( x.reshape(( batch_size, num_channel, ) + image_shape), np.array([list(image_shape)]), location, scale, alpha) grads = T.grad(T.mean(patch1), x) grad_scale = abs(T.grad(T.mean(patch1), scale)) grad_location = abs(T.grad(T.mean(patch1), location)) grad_alpha = abs(T.grad(T.mean(patch1), alpha)) f = theano.function( [x, location, scale, alpha], [patch1, grads, grad_scale + grad_location + grad_alpha, matrix, dx2],
import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt location = T.fmatrix() scale = T.fmatrix() x = T.fvector() patch_shape = (28, 28) image_shape = (100, 100) hyperparameters = {} hyperparameters["cutoff"] = 3 hyperparameters["batched_window"] = True cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch = cropper.apply( x.reshape((1, 1,) + image_shape), np.array([list(image_shape)]), location, scale) f = theano.function([x, location, scale], patch, allow_input_downcast=True) tds, vds = get_mnist_video_streams(1) image = tds.get_epoch_iterator().next()[0][10, 0, :] patch1 = f(image, [[60.0, 65.0]], [[1.0, 1.0]])[0, 0] patch2 = f(image, [[60.0, 65.0]], [[2.0, 2.0]])[0, 0]
from crop import LocallySoftRectangularCropper from crop import Gaussian import numpy as np from datasets import get_bmnist_streams import matplotlib # Force matplotlib to not use any Xwindows backend. matplotlib.use('Agg') import matplotlib.pyplot as plt tensor5 = T.TensorType(config.floatX, (False, ) * 4) # shape: B x C x X x Y input_ = tensor5('features') cropper = LocallySoftRectangularCropper(patch_shape=(28, 28), hyperparameters={ 'cutoff': 3000, 'batched_window': True }, kernel=Gaussian()) down, W, dx2 = cropper.apply( input_, np.array([list((100, 100))]), T.constant(99 * [[80, 70]]).astype('float32'), T.constant(99 * [[0.28, 0.28]]).astype('float32'), T.constant(99 * [[ 0.001, ] * 2]).astype('float32')) f = theano.function([input_], [down, W, dx2]) data = get_bmnist_streams(99)[0].get_epoch_iterator().next() res = f(data[0][0])
from crop import LocallySoftRectangularCropper from crop import Gaussian import numpy as np from datasets import get_bmnist_streams import matplotlib # Force matplotlib to not use any Xwindows backend. matplotlib.use('Agg') import matplotlib.pyplot as plt tensor5 = T.TensorType(config.floatX, (False,) * 4) # shape: B x C x X x Y input_ = tensor5('features') cropper = LocallySoftRectangularCropper( patch_shape=(28, 28), hyperparameters={'cutoff': 3000, 'batched_window': True}, kernel=Gaussian()) down, W, dx2 = cropper.apply( input_, np.array([list((100, 100))]), T.constant( 99 * [[80, 70]]).astype('float32'), T.constant( 99 * [[0.28, 0.28]]).astype('float32'), T.constant( 99 * [[0.001, ] * 2]).astype('float32'))
matplotlib.use('Agg') import matplotlib.pyplot as plt location = T.fmatrix() scale = T.fmatrix() x = T.fvector() batch_size = 100 num_channel = 3 patch_shape = (14, 14) image_shape = (224, 224) hyperparameters = {} hyperparameters["cutoff"] = 3 hyperparameters["batched_window"] = True cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch = cropper.apply( x.reshape((batch_size, num_channel,) + image_shape), np.array([list(image_shape)]), location, scale) f = theano.function([x, location, scale], patch, allow_input_downcast=True) grad_l = T.grad(T.mean(patch), location) grad_s = T.grad(T.mean(patch), scale) g = theano.function([x, location, scale], [grad_l, grad_s], allow_input_downcast=True) image = np.random.randn(
patch_shape = (28, 28) image_shape = (100, 100) hyperparameters = {} hyperparameters["cutoff"] = 3000 hyperparameters["batched_window"] = True # tds, _ = get_cooking_streams(batch_size) tds, _ = get_bmnist_streams(1) res = tds.get_epoch_iterator(as_dict=True).next()['features'] # shape: 3 x 125 x 200 img = res[5, 0] draw(img) plt.savefig('img.png') cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch1, matrix, dx2 = cropper.apply( x.reshape((batch_size, num_channel,) + image_shape), np.array([list(image_shape)]), location, scale, alpha) grads = T.grad(T.mean(patch1), x) grad_scale = abs(T.grad(T.mean(patch1), scale)) grad_location = abs(T.grad(T.mean(patch1), location)) grad_alpha = abs(T.grad(T.mean(patch1), alpha)) f = theano.function( [x, location, scale, alpha],
import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt location = T.fmatrix() scale = T.fmatrix() x = T.fvector() patch_shape = (28, 28) image_shape = (100, 100) hyperparameters = {} hyperparameters["cutoff"] = 3 hyperparameters["batched_window"] = True cropper = LocallySoftRectangularCropper( patch_shape=patch_shape, hyperparameters=hyperparameters, kernel=Gaussian()) patch = cropper.apply( x.reshape((1, 1,) + image_shape), np.array([list(image_shape)]), location, scale) f = theano.function([x, location, scale], patch, allow_input_downcast=True) tds, vds = get_mnist_video_streams(1) image = tds.get_epoch_iterator().next()[0][2, 0, :] import ipdb; ipdb.set_trace()