def model(self, X, w1, w2, w3, w4, wo, p_drop_conv, p_drop_hidden): # print X l1a = self.rectify(conv2d(X, w1, border_mode = "full")) l1 = max_pool_2d(l1a, (2, 2)) l1 = self.dropout(l1, p_drop_conv) # print np.mean(l1) l2a = self.rectify(conv2d(l1, w2)) l2 = max_pool_2d(l2a, (2, 2)) l2 = self.dropout(l2, p_drop_conv) # print np.mean(l2) l3a = self.rectify(conv2d(l2, w3)) l3b = max_pool_2d(l3a, (2, 2)) l3 = T.flatten(l3b, outdim = 2) l3 = self.dropout(l3, p_drop_conv) # print np.mean(l3) l4 = self.rectify(T.dot(l3, w4)) l4 = self.dropout(l4, p_drop_hidden) # print np.mean(l4) # l4 = T.dot(l4, wo) sig = T.dot(l4, wo) # pyx = self.softmax(T.dot(l4, wo)) return l1, l2, l3, l4, sig
def LCNinput(data, kernel_shape): X = T.ftensor4() filter_shape = (1, 1, kernel_shape, kernel_shape) filters = sharedX(gaussian_filter(kernel_shape).reshape(filter_shape)) convout = conv2d(X, filters=filters, border_mode='full') # For each pixel, remove mean of 9x9 neighborhood mid = int(np.floor(kernel_shape/ 2.)) centered_X = X - convout[:,:,mid:-mid,mid:-mid] # Scale down norm of 9x9 patch if norm is bigger than 1 sum_sqr_XX = conv2d(T.sqr(X), filters=filters, border_mode='full') denom = T.sqrt(sum_sqr_XX[:,:,mid:-mid,mid:-mid]) per_img_mean = denom.mean(axis = [2,3]) divisor = T.largest(per_img_mean.dimshuffle(0,1, 'x', 'x'), denom) new_X = centered_X / T.maximum(1., divisor) # new_X = new_X[:,:,mid:-mid, mid:-mid] f = theano.function([X], new_X) return f(data)
def model(X, filter_params, bias_params, p_dropout, srng): inp = X half = int(len(filter_params)/2) conv_params = filter_params[:half] deconv_params = filter_params[half:] conv_biases = bias_params[:half] deconv_biases = bias_params[half:] for f, b in zip(conv_params, conv_biases): outa = rectify(conv2d(inp, f, border_mode='valid') + b.dimshuffle('x', 0, 'x', 'x')) outb = dropout(outa, srng, p_dropout) inp = outb c = 0 for f, b in zip(deconv_params, deconv_biases): if c == len(deconv_params): outa = T.nnet.sigmoid(conv2d(inp, f, border_mode='full') + b.dimshuffle('x', 0, 'x', 'x')) else: outa = rectify(conv2d(inp, f, border_mode='full') + b.dimshuffle('x', 0, 'x', 'x')) outb = dropout(outa, srng, p_dropout) inp = outb c += 1 output = inp return output
def testmodel(X, w, w2, w3, w_o, p_drop_conv, p_drop_hidden): l1a = rectify(conv2d(X, w, border_mode='valid')) l1 = max_pool_2d(l1a, (2, 2)) l1 = dropout(l1, p_drop_conv) l2a = rectify(conv2d(l1, w2)) l2b = max_pool_2d(l2a, (2, 2)) l2 = T.flatten(l2b, outdim=2) l2 = dropout(l2, p_drop_conv) l3 = rectify(T.dot(l2, w3)) l3 = dropout(l3, p_drop_hidden) pyx = softmax(T.dot(l3, w_o)) # l3a = rectify(conv2d(l2, w3)) # l3b = max_pool_2d(l3a, (2, 2)) # l3 = T.flatten(l3b, outdim=2) # l3 = dropout(l3, p_drop_conv) # problem happening here # l4 = rectify(T.dot(l3, w4)) # l4 = dropout(l4, p_drop_hidden) # pyx = softmax(T.dot(l4, w_o)) return l1, l2, l3, pyx
def LCN(data, kernel_shape): # X = T.ftensor4() filter_shape = (1, 1, kernel_shape, kernel_shape) filters = sharedX(gaussian_filter(kernel_shape).reshape(filter_shape)) convout = conv2d(data, filters=filters, border_mode='full') # For each pixel, remove mean of 9x9 neighborhood mid = int(np.floor(kernel_shape/ 2.)) centered_X = data - convout[:,:,mid:-mid,mid:-mid] # Scale down norm of 9x9 patch if norm is bigger than 1 sum_sqr_XX = conv2d(T.sqr(data), filters=filters, border_mode='full') denom = T.sqrt(sum_sqr_XX[:,:,mid:-mid,mid:-mid]) per_img_mean = denom.mean(axis = [2,3]) divisor = T.largest(per_img_mean.dimshuffle(0, 1, 'x', 'x'), denom) new_X = centered_X / T.maximum(1., divisor) # new_X = new_X[:,:,mid:-mid, mid:-mid] new_X = T.extra_ops.squeeze(new_X) # remove broadcastable dimension new_X = new_X[:, 0, :, :] # TODO: check whether this forced squeeze is good return new_X
def model3(X, w, w2, w22, w222, w3, w4, p_drop_conv, p_drop_hidden): l1a = rectify(conv2d(X, w, border_mode='full')) l1 = max_pool_2d(l1a, (2, 2)) l1 = dropout(l1, p_drop_conv) l2a = rectify(conv2d(l1, w2)) l2 = max_pool_2d(l2a, (2, 2)) l2 = dropout(l2, p_drop_conv) l22a = rectify(conv2d(l2, w22)) l22 = max_pool_2d(l22a, (2, 2)) l22 = dropout(l22, p_drop_conv) l222a = rectify(conv2d(l22, w222)) l222 = max_pool_2d(l222a, (2, 2)) l222 = dropout(l222, p_drop_conv) l3a = rectify(conv2d(l222, w3)) l3b = max_pool_2d(l3a, (2, 2)) l3 = T.flatten(l3b, outdim=2) l3 = dropout(l3, p_drop_conv) l4 = rectify(T.dot(l3, w4)) l4 = dropout(l4, p_drop_hidden) pyx = softmax(T.dot(l4, w_o)) return l1, l2, l22, l222, l3, l4, pyx
def bench_ConvMed(batchsize): data_x.value = randn(n_examples, 1, 96, 96) w0 = shared(rand(6, 1, 7, 7) * numpy.sqrt(6 / (25.))) b0 = shared(zeros(6)) w1 = shared(rand(16, 6, 7, 7) * numpy.sqrt(6 / (25.))) b1 = shared(zeros(16)) vv = shared(rand(16*8*8, 120) * numpy.sqrt(6.0/16./25)) cc = shared(zeros(120)) v = shared(zeros(120, outputs)) c = shared(zeros(outputs)) params = [w0, b0, w1, b1, v, c, vv, cc] c0 = tanh(conv2d(sx, w0, image_shape=(batchsize, 1, 96, 96), filter_shape=(6,1,7,7)) + b0.dimshuffle(0, 'x', 'x')) s0 = tanh(max_pool_2d(c0, (3,3))) # this is not the correct leNet5 model, but it's closer to c1 = tanh(conv2d(s0, w1, image_shape=(batchsize, 6, 30, 30), filter_shape=(16,6,7,7)) + b1.dimshuffle(0, 'x', 'x')) s1 = tanh(max_pool_2d(c1, (3,3))) p_y_given_x = softmax(dot(tanh(dot(s1.flatten(2), vv)+cc), v)+c) nll = -log(p_y_given_x)[arange(sy.shape[0]), sy] cost = nll.mean() gparams = grad(cost, params) train = function([si, nsi], cost, updates=[(p,p-lr*gp) for p,gp in zip(params, gparams)]) eval_and_report(train, "ConvMed", [batchsize], N=120)
def apply(self, dataset, can_fit=True): x = dataset.get_design_matrix() denseX = T.matrix(dtype=x.dtype) image_shape = (len(x),) + self.img_shape X = denseX.reshape(image_shape) filters = gaussian_filter_9x9().reshape((1,1,9,9)) convout = conv.conv2d(input = X, filters = filters, image_shape = image_shape, filter_shape = (1, 1, 9, 9), border_mode='full') # For each pixel, remove mean of 9x9 neighborhood centered_X = X - convout[:,:,4:-4,4:-4] # Scale down norm of 9x9 patch if norm is bigger than 1 sum_sqr_XX = conv.conv2d(input = centered_X**2, filters = filters, image_shape = image_shape, filter_shape = (1, 1, 9, 9), border_mode='full') denom = T.sqrt(sum_sqr_XX[:,:,4:-4,4:-4]) per_img_mean = T.mean(T.flatten(denom, outdim=3), axis=2) divisor = T.largest(per_img_mean.dimshuffle((0,1,'x','x')), denom) new_X = centered_X / divisor new_X = T.flatten(new_X, outdim=2) f = theano.function([denseX], new_X) dataset.set_design_matrix(f(x))
def convolutional(X, X_test, input_shape, n_filters, filter_size): """ Implementation of a convolutional layer Parameters ---------- X input_shape n_filters filter_size Note ---- The convolutions are implemented using border_mode=same, that is the output shape is the same as the input shape for the 2 last dimensions """ filters_shape = (n_filters, input_shape[1], filter_size[0], filter_size[1]) filters = theano.shared( numpy.random.uniform(low=-0.1, high=0.1, size=filters_shape).astype(numpy.float32), 'conv_filters' ) output_shape = (input_shape[0], n_filters, input_shape[2], input_shape[3]) output = conv2d(input=X, filters=filters, filter_shape=filters_shape, image_shape=input_shape, border_mode='full') output_test = conv2d(input=X_test, filters=filters, filter_shape=filters_shape, image_shape=input_shape, border_mode='full') shift_x = (filter_size[0] - 1) // 2 shift_y = (filter_size[1] - 1) // 2 output = output[:,:,shift_x:input_shape[2]+shift_x,shift_y:input_shape[3]+shift_y] output_test = output_test[:,:,shift_x:input_shape[2]+shift_x,shift_y:input_shape[3]+shift_y] return output, output_test, [filters], output_shape
def apply(self, dataset, can_fit=True): x = dataset.get_design_matrix() denseX = T.matrix(dtype=x.dtype) image_shape = (len(x),) + self.img_shape X = denseX.reshape(image_shape) ones_patch = T.ones((1,1,9,9), dtype=x.dtype) convout = conv.conv2d(input = X, filters = ones_patch / (9.*9.), image_shape = image_shape, filter_shape = (1, 1, 9, 9), border_mode='full') # For each pixel, remove mean of 3x3 neighborhood centered_X = X - convout[:,:,4:-4,4:-4] # Scale down norm of 3x3 patch if norm is bigger than 1 sum_sqr_XX = conv.conv2d(input = centered_X**2, filters = ones_patch, image_shape = image_shape, filter_shape = (1, 1, 9, 9), border_mode='full') denom = T.sqrt(sum_sqr_XX[:,:,4:-4,4:-4]) xdenom = denom.reshape(X.shape) new_X = centered_X / T.largest(1.0, xdenom) new_X = T.flatten(new_X, outdim=2) f = theano.function([denseX], new_X) dataset.set_design_matrix(f(x))
def model(X, w, w2, w3, w4, w5, w_o, b_h1, b_h2, b_o, p_drop_conv, p_drop_hidden): l1_lin = conv2d(X, w, border_mode='full')+b_c1.dimshuffle('x', 0, 'x', 'x') l1a = alpha_c1 * rectify(l1_lin) + (1.- alpha_c1) * T.tanh(l1_lin) l1 = max_pool_2d(l1a, (2, 2)) l1 = dropout(l1, p_drop_conv) l2_lin = conv2d(l1, w2) + b_c2.dimshuffle('x', 0, 'x', 'x') l2a = alpha_c2 * rectify(l2_lin) + (1. - alpha_c2) * T.tanh(l2_lin) l2 = max_pool_2d(l2a, (2, 2)) l2 = dropout(l2, p_drop_conv) l3_lin = conv2d(l2, w3) + b_c3.dimshuffle('x', 0, 'x', 'x') l3a = alpha_c3 * rectify(l3_lin) + ( 1 - alpha_c3) * T.tanh(l3_lin) l3b = max_pool_2d(l3a, (2, 2)) l3 = T.flatten(l3b, outdim=2) l3 = dropout(l3, p_drop_conv) l4_lin = T.dot(l3, w4) + b_h1 l4 = alpha_h1 * rectify(l4_lin) + (1.-alpha_h1) * T.tanh(l4_lin) l4 = dropout(l4, p_drop_hidden) l5_lin = T.dot(l4, w5) + b_h2 l5 = alpha_h1 * rectify(l5_lin) + (1.-alpha_h2) * T.tanh(l5_lin) l5 = dropout(l5, p_drop_hidden) pyx = softmax(T.dot(l5, w_o) + b_o ) return l1, l2, l3, l4, l5, pyx
def output(self, input, mask=None): if mask is None: drop_in = input * self.drop else: drop_in = input * mask conv_out1 = conv.conv2d(input=drop_in, filters=self.W1, filter_shape=self.filter_shape1, image_shape=self.shape_in) linout1 = T.nnet.relu(conv_out1 + self.b1.dimshuffle('x', 0, 'x', 'x')) output1 = ( linout1 if self.activation is None else self.activation(linout1) ) pooled_out1 = downsample.max_pool_2d(input=output1, ds=self.poolsize1, ignore_border=True) conv_out2 = conv.conv2d(input=drop_in, filters=self.W2, filter_shape=self.filter_shape2, image_shape=self.shape_in) linout2 = T.nnet.relu(conv_out2 + self.b2.dimshuffle('x', 0, 'x', 'x')) output2 = ( linout2 if self.activation is None else self.activation(linout2) ) pooled_out2 = downsample.max_pool_2d(input=output2, ds=self.poolsize2, ignore_border=True) conv_out3 = conv.conv2d(input=drop_in, filters=self.W3, filter_shape=self.filter_shape3, image_shape=self.shape_in) linout3 = T.nnet.relu(conv_out3 + self.b3.dimshuffle('x', 0, 'x', 'x')) output3 = ( linout3 if self.activation is None else self.activation(linout3) ) pooled_out3 = downsample.max_pool_2d(input=output3, ds=self.poolsize3, ignore_border=True) output = T.concatenate([pooled_out1, pooled_out2, pooled_out3], axis=1) return output
def _lcn(image, im_shape, fmaps, pool_depth, width, sigma): """ """ import theano import theano.tensor as T from theano.tensor.nnet import conv border = width//2 filters = _lcn_filters(fmaps, pool_depth, width, sigma) filter_shape = filters.shape blurred_mean = conv.conv2d(input=image, filters=filters, image_shape=im_shape, filter_shape=filter_shape, border_mode='full') image -= blurred_mean[:, :, border:-border, border:-border] image_sqr = T.sqr(image) blurred_sqr = conv.conv2d(input=image_sqr, filters=filters, image_shape=im_shape, filter_shape=filter_shape, border_mode='full') div = T.sqrt(blurred_sqr[:, :, border:-border, border:-border]) fm_mean = div.mean(axis=[2, 3]) div = T.largest(fm_mean.dimshuffle(0, 1, 'x', 'x'), div) + 1e-6 image = image/div return T.cast(image, theano.config.floatX)
def __init__(self, input, input_shape, filter_shape, border_mode="valid") : # input : theano symbolic variable of input, 4D tensor # input_shape : shape of input / (minibatch size, input channel num, image height, image width) # filter_shape : shape of filter / (# of new channels to make, input channel num, filter height, filter width) # initialize W (weight) randomly rng = np.random.RandomState(int(time.time())) w_bound = math.sqrt(filter_shape[1] * filter_shape[2] * filter_shape[3]) self.W1 = theano.shared(np.asarray(rng.uniform(low=-1.0/w_bound, high=1.0/w_bound, size=filter_shape), dtype=theano.config.floatX), name='W', borrow=True) self.W2 = theano.shared(np.asarray(rng.uniform(low=-1.0/w_bound, high=1.0/w_bound, size=filter_shape), dtype=theano.config.floatX), name='W', borrow=True) self.W3 = theano.shared(np.asarray(rng.uniform(low=-1.0/w_bound, high=1.0/w_bound, size=filter_shape), dtype=theano.config.floatX), name='W', borrow=True) # initialize b (bias) with zeros self.b1 = theano.shared(np.asarray(np.zeros(filter_shape[0],), dtype=theano.config.floatX), name='b', borrow=True) self.b2 = theano.shared(np.asarray(np.zeros(filter_shape[0],), dtype=theano.config.floatX), name='b', borrow=True) self.b3 = theano.shared(np.asarray(np.zeros(filter_shape[0],), dtype=theano.config.floatX), name='b', borrow=True) # convolution & sigmoid calculation #self.conv_out = conv.conv2d(input, self.W, image_shape=input_shape, filter_shape=filter_shape) #self.output = 1.7159*T.tanh((self.conv_out + self.b.dimshuffle('x', 0, 'x', 'x'))*(2.0/3.0)) # maxout : 3 out1 = conv.conv2d(input, self.W1, image_shape=input_shape, filter_shape=filter_shape, border_mode=border_mode) + self.b1.dimshuffle('x', 0, 'x', 'x') out2 = conv.conv2d(input, self.W2, image_shape=input_shape, filter_shape=filter_shape, border_mode=border_mode) + self.b2.dimshuffle('x', 0, 'x', 'x') out3 = conv.conv2d(input, self.W3, image_shape=input_shape, filter_shape=filter_shape, border_mode=border_mode) + self.b3.dimshuffle('x', 0, 'x', 'x') self.output = T.maximum(out1, T.maximum(out2, out3)) # save parameter of this layer for back-prop convinience self.params = [self.W1, self.W2, self.W3, self.b1, self.b2, self.b3] insize = input_shape[1] * input_shape[2] * input_shape[3] self.paramins = [insize, insize, insize, insize, insize, insize]
def initialise(self): activation = self.activation rng = np.random.RandomState(235) inpt = self.inpt # initialise layer 1 weight vector. #w_shp = (self.no_of_filters, 1.,self.in_channels, self.filter_length) w_shp = (self.no_of_filters, self.in_channels, self.filter_length,1.) w_bound = np.sqrt(self.in_channels* self.filter_length) W = theano.shared(value = np.asarray( rng.normal(0.,0.001,size=w_shp), dtype=inpt.dtype), name =self.param_names[0],borrow = True) b_shp = (self.no_of_filters,) b = theano.shared(value = np.asarray( rng.uniform(low=-.0, high=.0, size=b_shp), dtype=inpt.dtype), name =self.param_names[1],borrow = True) upsampled = self.inpt.repeat(int(self.pool),axis = 2) conv_out = conv.conv2d(upsampled, W.dimshuffle(0,3,2,1),subsample=(1,1),border_mode = "full") conv_out = conv_out[:,:,:,int(self.in_channels-1):-int(self.in_channels-1)] self.params = [W,b] if self.distribution==True: W_sigma = theano.shared(value = np.asarray( rng.normal(0.,0.001,size=w_shp), dtype=inpt.dtype), name ='lik_sigma',borrow = True) b_sigma = theano.shared(value = np.asarray( rng.uniform(low=-.0, high=.0, size=b_shp), dtype=inpt.dtype), name ='b_sigm',borrow = True) #self.output =conv_out + b.dimshuffle('x', 0, 'x', 'x') conv_out_sigma = conv.conv2d(upsampled, W_sigma.dimshuffle(0,3,2,1),subsample=(1,1),border_mode = "full",) conv_out_sigma = conv_out_sigma[:,:,:,int(self.in_channels-1):-int(self.in_channels-1)] self.log_sigma = conv_out_sigma + b_sigma.dimshuffle('x', 0, 'x', 'x') self.params +=[W_sigma,b_sigma] if activation!=None: self.output = self.activation(conv_out + b.dimshuffle('x', 0, 'x', 'x')).astype(theano.config.floatX) else: self.output = conv_out + b.dimshuffle('x', 0, 'x', 'x').astype(theano.config.floatX)
def decode(self, hidden): hidden_ = T.alloc(0.,*self.hidden_shape) deconv_out = T.alloc(0.,*self.output_shape) # Zero padding How can I code easily? hidden_ = T.set_subtensor(hidden_[:,:,:,self.filter_shape[3]-1:],hidden) # Calculate output conv_odd = conv.conv2d( input = hidden_, filters = self.W_odd, filter_shape = self.filter_shape, image_shape = self.hidden_shape,) conv_even = conv.conv2d( input = hidden_, filters = self.W_even, filter_shape = self.filter_shape, image_shape = self.hidden_shape,) deconv_out = T.set_subtensor(deconv_out[:,:,:,::2], conv_odd) deconv_out = T.set_subtensor(deconv_out[:,:,:,1::2], conv_even) linout = deconv_out + self.b.dimshuffle('x',0,'x','x') if self.dec_hid == 'tanh': convout= T.tanh(linout) elif self.dec_hid == 'lin': convout=linout elif self.dec_hid == 'relu': convout=linout * (linout > 0.) + 0. * (linout < 0.) else: raise ValueError('Invalid dec_hid') #### Recurrent connection#### return convout
def model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden): # conv + ReLU + pool # border_mode = full, then zero-padding, default is valid l1a = rectify(conv2d(X, w, border_mode='full')) # pooling at 2*2 kernel and select the largest in the kernel l1 = max_pool_2d(l1a, (2, 2)) l1 = dropout(l1, p_drop_conv) # conv + ReLU + pool l2a = rectify(conv2d(l1, w2)) l2 = max_pool_2d(l2a, (2, 2)) l2 = dropout(l2, p_drop_conv) # conv + ReLU + pool l3a = rectify(conv2d(l2, w3)) l3b = max_pool_2d(l3a, (2, 2)) # convert a ndim array to 2 dim. if l3b dim larger than 2 then the rest dim collapsed. # flatten for enter the FC layer l3 = T.flatten(l3b, outdim=2) l3 = dropout(l3, p_drop_conv) # FC + ReLU l4 = rectify(T.dot(l3, w4)) l4 = dropout(l4, p_drop_hidden) # output layer + softmax pyx = softmax(T.dot(l4, w_o)) return l1, l2, l3, l4, pyx
def bench_ConvLarge(batchsize, variant=True): name = "ConvLarge_b" + str(GlobalBenchReporter.batch_size) name += "_" + config.linker # Image shape 256x256 GlobalBenchReporter.batch_size = batchsize data_x.set_value(randn(n_examples, 1, 256, 256)) w0 = shared(rand(6, 1, 7, 7) * numpy.sqrt(6 / (25.))) b0 = shared(zeros(6)) w1 = shared(rand(16, 6, 7, 7) * numpy.sqrt(6 / (25.))) b1 = shared(zeros(16)) vv = shared(rand(16 * 11 * 11, 120) * numpy.sqrt(6.0 / 16. / 25)) cc = shared(zeros(120)) v = shared(zeros(120, outputs)) c = shared(zeros(outputs)) params = [w0, b0, w1, b1, v, c, vv, cc] c0 = tanh(conv2d(sx, w0, image_shape=(batchsize, 1, 256, 256), filter_shape=(6, 1, 7, 7)) + b0.dimshuffle(0, 'x', 'x')) # this is not the correct leNet5 model, but it's closer to s0 = tanh(max_pool_2d(c0, (5, 5))) c1 = tanh(conv2d(s0, w1, image_shape=(batchsize, 6, 50, 50), filter_shape=(16, 6, 7, 7)) + b1.dimshuffle(0, 'x', 'x')) s1 = tanh(max_pool_2d(c1, (4, 4))) p_y_given_x = softmax(dot(tanh(dot(s1.flatten(2), vv) + cc), v) + c) nll = -log(p_y_given_x)[arange(sy.shape[0]), sy] cost = nll.mean() gparams = grad(cost, params) train = function([si, nsi], cost, updates=[(p, p - lr * gp) for p, gp in zip(params, gparams)], name=name) GlobalBenchReporter.eval_model(train, name) if not variant: return # Versions with no inputs snsi.set_value(GlobalBenchReporter.batch_size) c0 = tanh(conv2d(ssx, w0, image_shape=(batchsize, 1, 256, 256), filter_shape=(6, 1, 7, 7)) + b0.dimshuffle(0, 'x', 'x')) # this is not the correct leNet5 model, but it's closer to s0 = tanh(max_pool_2d(c0, (5, 5))) c1 = tanh(conv2d(s0, w1, image_shape=(batchsize, 6, 50, 50), filter_shape=(16, 6, 7, 7)) + b1.dimshuffle(0, 'x', 'x')) s1 = tanh(max_pool_2d(c1, (4, 4))) p_y_given_x = softmax(dot(tanh(dot(s1.flatten(2), vv) + cc), v) + c) nll = -log(p_y_given_x)[arange(ssy.shape[0]), ssy] cost = nll.mean() gparams = grad(cost, params) train2 = function([], cost, updates=[(p, p - lr * gp) for p, gp in zip(params, gparams)] + [(ssi, ssi + snsi)], name=name) GlobalBenchReporter.bypass_eval_model(train2, name, init_to_zero=ssi)
def get_theano_function(self, input): print self.name print 'in',self.in_size print 'filt',self.theano_filter_shape print 'bias', self.bias.shape conv_out = None if self.pad[0] != 0: tmp_new = T.zeros_like(self.dpad) input = T.set_subtensor(tmp_new[:,:,self.pad[0]:(-self.pad[0]), self.pad[3]:(-self.pad[3])], input) if self.groups == 1: conv_out = conv.conv2d(input, self.w, filter_shape=self.theano_filter_shape, subsample=self.stride, image_shape=self.theano_in_size) else: in_1 = input[:,:input.shape[1]/2,:,:] in_2 = input[:,input.shape[1]/2:,:,:] fs = self.theano_filter_shape fs = (fs[0]/2,fs[1],fs[2],fs[3]) conv_1 = conv.conv2d(in_1, self.w1, filter_shape=fs, subsample=self.stride, image_shape=self.theano_in_size) conv_2 = conv.conv2d(in_2, self.w2, filter_shape=fs, subsample=self.stride, image_shape=self.theano_in_size) conv_out = T.concatenate([conv_1, conv_2], axis=1) return conv_out + self.b.dimshuffle('x',0,'x','x')
def __init__(self, rng, input_A, input_B, filter_shape, image_shape, poolsize=(2, 2)): print image_shape print filter_shape assert image_shape[1] == filter_shape[1] #calc the W_bound and init the W fan_in = numpy.prod(filter_shape[1:]) fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared(numpy.asarray( rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype = theano.config.floatX), borrow = True) b_value = numpy.zeros((filter_shape[0],), dtype = theano.config.floatX) self.b = theano.shared(value = b_value, borrow = True) conv_out_A = conv.conv2d(input = input_A, filters = self.W, filter_shape = filter_shape, image_shape = image_shape) conv_out_B = conv.conv2d(input = input_B, filters = self.W, filter_shape = filter_shape, image_shape = image_shape) pooled_out_A = downsample.max_pool_2d(input = conv_out_A, ds = poolsize, ignore_border = True) pooled_out_B = downsample.max_pool_2d(input = conv_out_B, ds = poolsize, ignore_border = True) self.output_A = T.tanh(pooled_out_A + self.b.dimshuffle('x',0,'x','x')) self.output_B = T.tanh(pooled_out_B + self.b.dimshuffle('x',0,'x','x')) self.params = [self.W, self.b]
def __init__(self,name,rng,inputs, hiden_size,input_size,window_size,feature_num,init_W=None,init_b=None): ''' inputs shape (batch size,input_size,1,max_sentence_length * feature_num) ''' self.hiden_size = hiden_size self.window_size = window_size self.feature_num = feature_num self.conv_window = self.window_size * self.feature_num if init_W == None: self.W = theano.shared(np.asarray(rng.uniform(low=-2.0, high=2.0, size=(hiden_size,1,1,self.conv_window)), dtype=theano.config.floatX) ,name='cov_1d_layer_W_%s' %(name)) else: self.W = theano.shared(init_W,name='cov_1d_layer_W_%s' %(name)) if init_b == None: self.b = theano.shared(np.asarray(rng.uniform(low=-2.0, high=2.0, size=(hiden_size)), dtype=theano.config.floatX) ,name='cov_1d_layer_b_%s' % (name)) else: self.b = theano.shared(init_b,name='cov_1d_layer_b_%s' % (name)) if input_size == 1: self.linear = conv.conv2d(inputs,self.W,subsample=(1,self.feature_num)) + self.b.dimshuffle('x', 0, 'x', 'x') #self.out = self.linear.dimshuffle(0,2,1,3) self.output = self.linear.dimshuffle('x',0,1,2,3) else: self.linear,_updates = theano.scan(lambda x_i: conv.conv2d(inputs[:,x_i:x_i+1,:,:],self.W,\ subsample=(1,self.feature_num)) + self.b.dimshuffle('x', 0, 'x', 'x'), sequences=[T.arange(input_size)]) #self.output = self.linear.dimshuffle(1,0,2,3,4).reshape((inputs.shape[0],inputs.shape[1],hiden_size,-1)) self.output = self.linear
def initialise(self): rng = np.random.RandomState(235) inpt = self.inpt # initialise layer 1 weight vector. w_shp = (self.no_of_filters,self.in_channels,self.filter_length,1.) w_bound = np.sqrt(self.in_channels* self.filter_length) W = theano.shared(value = np.asarray( rng.normal(0.,0.01,size=w_shp), dtype=inpt.dtype), name =self.param_names[0],borrow = True) b_shp = (self.no_of_filters,) b = theano.shared(value = np.asarray( rng.uniform(low=-.0, high=.0, size=b_shp), dtype=inpt.dtype), name =self.param_names[1],borrow = True) if self.unpooling == "repeat": upsampled = self.inpt.repeat(int(self.pool),axis = 2) elif self.unpooling == "zeros": zeros = T.zeros_like(self.inpt.repeat(int(self.pool-1),axis = 3)) # It should do: create array of size (batch, channels in, length_signal, upsample_factor-1) with zeros in it upsampled = T.flatten(T.concatenate((self.inpt,zeros),axis=3),outdim=3)[:,:,:,None] self.upsampled = upsampled conv_out = conv.conv2d(upsampled, W,subsample=(1,1),border_mode = "full") self.params = [W,b] if self.distribution==True: W_sigma = theano.shared(value = np.asarray( rng.normal(0.,0.01,size=w_shp), dtype=inpt.dtype), name ='lik_sigma',borrow = True) b_sigma = theano.shared(value = np.asarray( rng.uniform(low=-.0, high=.0, size=b_shp), dtype=inpt.dtype), name ='b_sigm',borrow = True) #self.output =conv_out + b.dimshuffle('x', 0, 'x', 'x') conv_out_sigma = conv.conv2d(upsampled, W_sigma,subsample=(1,1),border_mode = "full") self.log_sigma = conv_out_sigma + b_sigma.dimshuffle('x', 0, 'x', 'x') self.params +=[W_sigma,b_sigma] self.output = conv_out + b.dimshuffle('x', 0, 'x', 'x').astype(theano.config.floatX)
def get_fprop_fn(variable_shape=False, include_pool=True): """ build a theano function that use SAE weights to get convolved(or pooled if include_pool is True) features from a given input """ conf = utils.get_config() paths = utils.get_paths() ae = serial.load(paths['sae']['model']) cnn_layer = 'cnn_layer_%i' % (conf['cnn_layers']) batch_size = conf[cnn_layer]['batch_size'] nhid = conf['sae']['nhid'] patch_size = conf['patch_size'] region_size = conf['region_size'] input = T.tensor4('input') filter_shape = (nhid, 1, patch_size, patch_size) filters = theano.shared(ae.get_weights().T.reshape(filter_shape)) if variable_shape: out = conv.conv2d(input, filters) else: image_shape = [batch_size, 1, region_size, region_size] out = conv.conv2d(input, filters, filter_shape=filter_shape, image_shape=image_shape) if include_pool: pool_fn = getattr(out, conf['pool_fn']) out = pool_fn(axis=(2, 3)) return theano.function([input], out)
def drop_output(self, input, drop=0, rng=None, p=0.5): ###--- Unpool if self.poolsize[0] == 1 and self.poolsize[1] == 1: unpool_out = input else: unpool_out = Textra.repeat(Textra.repeat(input, self.poolsize[0], axis = 2), self.poolsize[1], axis = 3) * self.mask image_shape = list(self.image_shape) if n_batch is not None: image_shape[0] = n_batch ###--- Unpool + conv # convolve input feature maps with filters if self.border_mode == 'valid': conv_out = conv.conv2d( input=unpool_out, filters=self.W, filter_shape=self.filter_shape, image_shape=image_shape, border_mode='valid' ) elif self.border_mode == 'same': conv_out = conv.conv2d( input=unpool_out, filters=self.W, filter_shape=self.filter_shape, image_shape=image_shape, border_mode='full' ) padding_w = theano.shared((self.filter_shape[2] - 1) / 2) padding_h = theano.shared((self.filter_shape[3] - 1) / 2) conv_out = conv_out[:,:,padding_w:-padding_w,padding_h:-padding_h] elif self.border_mode == 'full': conv_out = conv.conv2d( input=unpool_out, filters=self.W, filter_shape=self.filter_shape, image_shape=image_shape, border_mode='full' ) else: raise Exception('Unknown conv type') # downsample each feature map individually, using maxpooling # add the bias term. Since the bias is a vector (1D array), we first # reshape it to a tensor of shape (1, n_filters, 1, 1). Each bias will # thus be broadcasted across mini-batches and feature map # width & height lin_output = conv_out + self.b.dimshuffle('x', 0, 'x', 'x') output= ( lin_output if self.activation is None else self.activation(lin_output) ) droppedOutput = nonlinearity.dropout(rng, output, p) return T.switch(T.neq(drop, 0), droppedOutput, output)
def __init__(self, input, filter_shape, corruption_level = 0.1, shared_W = None, shared_b = None, image_shape = None, poolsize = (2,2)): theano_rng = RandomStreams() fan_in = numpy.prod(filter_shape[1:]) fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) center = theano.shared(value = 1, name="center") scale = theano.shared(value = 2, name="scale") if shared_W != None and shared_b != None : self.W = shared_W self.b = shared_b else: initial_W = numpy.asarray( numpy.random.uniform( low = -numpy.sqrt(6./(fan_in+fan_out)), high = numpy.sqrt(6./(fan_in+fan_out)), size = filter_shape), dtype = theano.config.floatX) initial_b = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX) self.W = theano.shared(value = initial_W, name = "W") self.b = theano.shared(value = initial_b, name = "b") initial_b_prime= numpy.zeros((filter_shape[1],),dtype=theano.config.floatX) self.b_prime = theano.shared(value = initial_b_prime, name = "b_prime") self.x = input self.tilde_x = theano_rng.binomial( self.x.shape, 1, 1 - corruption_level,dtype=theano.config.floatX) * self.x conv1_out = conv.conv2d(self.tilde_x, self.W, filter_shape=filter_shape, image_shape=image_shape, border_mode='valid') self.y = T.tanh(conv1_out + self.b.dimshuffle('x', 0, 'x', 'x')) da_filter_shape = [ filter_shape[1], filter_shape[0], filter_shape[2], filter_shape[3] ] initial_W_prime = numpy.asarray( numpy.random.uniform( \ low = -numpy.sqrt(6./(fan_in+fan_out)), \ high = numpy.sqrt(6./(fan_in+fan_out)), \ size = da_filter_shape), dtype = theano.config.floatX) self.W_prime = theano.shared(value = initial_W_prime, name = "W_prime") conv2_out = conv.conv2d(self.y, self.W_prime, filter_shape = da_filter_shape, border_mode='full') self.z = (T.tanh(conv2_out + self.b_prime.dimshuffle('x', 0, 'x', 'x'))+center) / scale scaled_x = (self.x + center) / scale self.L = - T.sum( scaled_x*T.log(self.z) + (1-scaled_x)*T.log(1-self.z), axis=1 ) self.cost = T.mean(self.L) self.params = [ self.W, self.b, self.b_prime ]
def model(X, w1, w2, w3, Max_Pooling_Shape, p_drop_conv): # Max_Pooling_Shape has to be large enough to pool only one element out l1 = dropout(max_pool_2d(rectify(conv2d(X, w1, border_mode='valid')), (2,2)), p_drop_conv) l2 = T.flatten(dropout(max_pool_2d(rectify(conv2d(l1, w1, border_mode='valid')), Max_Pooling_Shape), p_drop_conv), outdim=2) l2 = dropout(rectify(T.dot(l1, w2)), p_drop_hidden) pyx = softmax(T.dot(l2, w3)) return pyx
def model(X, w_1, w_2, w_3, w_4, w_5, p_1, p_2): # T.maximum is the rectify activation l1 = dropout(max_pool_2d(T.maximum(conv2d(X, w_1, border_mode='full'), 0.), (2, 2)), p_1) l2 = dropout(max_pool_2d(T.maximum(conv2d(l1, w_2), 0.), (2, 2)), p_1) # flatten to switch back to 1d layers - with "outdim = 2" (2d) output l3 = dropout(T.flatten(max_pool_2d(T.maximum(conv2d(l2, w_3), 0.), (2, 2)), outdim=2), p_1) l4 = dropout(T.maximum(T.dot(l3, w_4), 0.), p_2) return T.dot(l4, w_5) #T.nnet.softmax(T.dot(l4, w_5))
def model(X, w_1, w_2, w_3, w_4, w_5, w_6, w_7, p_1, p_2): l1 = T.maximum(conv2d(X, w_1, border_mode='full'),0.) l2 = dropout(max_pool_2d(T.maximum(conv2d(l1, w_2), 0.), (2, 2)), p_1) l3 = dropout(max_pool_2d(T.maximum(conv2d(l2, w_3), 0.), (2, 2)), p_1) l4 = dropout(max_pool_2d(T.maximum(conv2d(l3, w_4), 0.), (2, 2)), p_1) l5 = dropout(T.flatten(max_pool_2d(T.maximum(conv2d(l4, w_5), 0.), (2, 2)), outdim=2), p_1) l6 = dropout(T.maximum(T.dot(l5, w_6), 0.), p_2) return T.nnet.softmax(T.dot(l6, w_7))
def decode(self, hidden): hidden_shape=( self.input_shape[0], self.input_shape[1], self.input_shape[2]+1, self.input_shape[3],) hidden_ = theano.shared(np.zeros((hidden_shape), dtype=theano.config.floatX), borrow=True) deconv_out = theano.shared(np.zeros((self.output_shape), dtype=theano.config.floatX),borrow=True) #self.W = self.W.dimshuffle(2,1,0,3) #c01b to 10cb #deconv_out = deconv_out.dimshuffle(2,1,0,3) #c01b to 10cb hidden = hidden.dimshuffle(2,1,0,3) hidden_ =hidden_.dimshuffle(2,1,0,3) # Zero padding How can I code easily? hidden_ = T.set_subtensor(hidden_[1:],hidden) hidden_ = hidden.dimshuffle(2,1,0,3) # Calculate output deconv_out = T.set_subtensor(deconv_out[::2], conv.conv2d( input = hidden_, filters = self.W_odd, filter_shape = self.filter_shape,)) deconv_out = T.set_subtensor(deconv_out[1::2], conv.conv2d( input = hidden_, filters = self.W_even, filter_shape = self.filter_shape,)) #deconv_out = T.set_subtensor(deconv_out[::2], # self.W[0]*hidden_[1:] + self.W[2]*hidden_[:-1]) #deconv_out = T.set_subtensor(deconv_out[1::2], # self.W[1]*hidden_[1:] + self.W[3]*hidden_[:-1]) #self.W = self.W.dimshuffle(2,1,0,3) #deconv_out = deconv_out.dimshuffle(2,1,0,3) #c01b to 10cb #deconv_out = np.zeros((self.output_shape)) #deconv_out[::2] = self.W[0]*hidden[1:] + self.W[2]*hidden[:-1] #deconv_out[1::2] = self.W[1]*hidden[1:] + self.W[3]*hidden[:-1] linout = deconv_out + self.b.dimshuffle('x','x','x',0) if self.dec_hid == 'tanh': convout= T.tanh(linout) elif self.dec_hid == 'lin': convout=linout elif self.dec_hid == 'relu': convout=linout * (linout > 0.) + 0. * (linout < 0.) else: raise ValueError('Invalid dec_hid') #### Recurrent connection#### return convout
def convlayer(X, w, first=False): if first: #half perfoms as scipy.signals 'full' for odd filter sizes conv = conv2d(X,w, border_mode='full') else: conv = conv2d(X, w) nonl = rectify(conv) max_pool = pool_2d(nonl, ds=(2,2), ignore_border=False, mode='max') return max_pool
def __init__(self, rng, input, image_shape, filter_shape): # 入力のチャンネル数とフィルタを定義するときに指定する入力のチャンネル数の一致を確認 assert image_shape[1] == filter_shape[1] # 入力の保存 self.input = input # 黒魔術的なfilterの初期化 # フィルターマップ数 * フィルターのheight * フィルターのwidth (prodはnpの配列要素全部の掛け算) fan_in = np.prod(filter_shape[1:]) # 出力特徴Map数 * フィルターのheight * フィルターのwidth fan_out = filter_shape[0] * np.prod(filter_shape[2:]) # filterの定義 W_bound = np.sqrt(6. / (fan_in + fan_out)) # ランダムな値を割り振る self.W = theano.shared(np.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), borrow=True) # biasの定義 b_values = np.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, borrow=True) # conv conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape) # biasとactivate function # poolingの結果にbias項として加える(全項に追加) # biasは1dimのvectorなので(1, n_filters, 1, 1)にreshapeする # biasを加えたら、activate function(ここではtanh)を適用する self.output = func.sym_ReLU(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) # パラメータの保存 self.params = [self.W, self.b] # 入力の保存 self.input = input
def g_deconv(self, z_ver, in_dims, out_dims, weight_name, fspec): """ Inverse operation for each type of f used in convnets """ f_type, f_dims = fspec assert z_ver is not None num_channels = in_dims[0] if in_dims is not None else None num_filters, width, height = out_dims[:3] if f_type in ['globalmeanpool']: u = T.addbroadcast(z_ver, 2, 3) assert in_dims[1] == 1 and in_dims[2] == 1, \ "global pooling needs in_dims (1,1): %s" % str(in_dims) elif f_type in ['maxpool']: sh, str, size = z_ver.shape, f_dims[0], f_dims[1] assert str == size, "depooling requires stride == size" u = T.zeros((sh[0], sh[1], sh[2] * str, sh[3] * str), dtype=z_ver.dtype) for x in xrange(str): for y in xrange(str): u = T.set_subtensor(u[:, :, x::str, y::str], z_ver) u = u[:, :, :width, :height] elif f_type in ['convv', 'convf']: filter_size, str = (f_dims[1], f_dims[1]), f_dims[2] W_shape = (num_filters, num_channels) + filter_size W = self.weight(self.rand_init_conv(W_shape), weight_name) if str > 1: # upsample if strided version sh = z_ver.shape u = T.zeros((sh[0], sh[1], sh[2] * str, sh[3] * str), dtype=z_ver.dtype) u = T.set_subtensor(u[:, :, ::str, ::str], z_ver) else: u = z_ver # no strides, only deconv u = conv2d(u, W, filter_shape=W_shape, border_mode='valid' if 'convf' in f_type else 'full') u = u[:, :, :width, :height] else: raise NotImplementedError('Layer %s has no convolutional decoder' % f_type) return u
def lmul(self, x): """ .. todo:: WRITEME """ """ dot(x, A) This method overrides the original Conv2D lmul to make it work with arbitrary axis orders """ # x must be formatted as batch index, channel, topo dim 0, topo dim 1 # for use with conv2d, so check what the current input space format is assert x.ndim == 4 axes = self.input_space.axes assert len(axes) == 4 op_axes = ('b', 'c', 0, 1) if tuple(axes) != op_axes: x = x.dimshuffle(axes.index('b'), axes.index('c'), axes.index(0), axes.index(1)) rval = conv2d( x, self._filters, image_shape=self._img_shape, filter_shape=self._filters_shape, subsample=self._subsample, border_mode=self._border_mode, ) # Format the output based on the output space axes = self.output_axes assert len(axes) == 4 if tuple(axes) != op_axes: rval = rval.dimshuffle(op_axes.index(axes[0]), op_axes.index(axes[1]), op_axes.index(axes[2]), op_axes.index(axes[3])) return rval
def model(X, img_x, filter_params, fc_params, p_drop_conv, p_drop_hidden): inp = X params = [] for f in filter_params: outa = rectify(conv2d(inp, f, border_mode='valid')) outb = max_pool_2d(outa, (2, 2)) outc = dropout(outb, p_drop_conv) f_shp = f.get_value().shape inp = outc inp = T.flatten(inp, outdim=2) for w in fc_params[:-1]: out = rectify(T.dot(inp, w)) out = dropout(out, p_drop_hidden) inp = out w = fc_params[-1] pyx = softmax(T.dot(out, w)) return pyx
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): assert filter_shape[1] == image_shape[1] self.input = input # (batches, feature, Ih, Iw) fan_in = numpy.prod(filter_shape[1:]) # number of connections for each filter W_value = numpy.asarray( rng.uniform(low=-numpy.sqrt(3. / fan_in), high=numpy.sqrt(3. / fan_in), size=filter_shape), dtype=theano.config.floatX) self.W = theano.shared(W_value, name='W') # (filters, feature, Fh, Fw) b_value = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX) self.b = theano.shared(b_value, name='b') conv_res = conv.conv2d(input, self.W, image_shape=image_shape, filter_shape=filter_shape) # (batches, filters, Ih-Fh+1, Iw-Fw+1) pooled = downsample.max_pool_2d(conv_res, poolsize) #pooled = tf.nn.max_pool(conv_res,[1,2,2,1],[1,2,2,1],padding=padding) self.output = T.tanh(pooled + self.b.dimshuffle('x', 0, 'x', 'x')) self.params = [self.W, self.b]
def __init__(self, image_shape, filter_shape, pool_size=(2, 2), X=None, y=None, W=None, b=None): """ filter_shape = (n_filters, n_input_feats_maps, filter_ht, filter_wd) image_shape = (batch_size, n_input_feats_maps, img_ht, img_wd) pool_size = the downsampling (pooling) factor (n_rows, n_cols) """ assert image_shape[1] == filter_shape[1] rng = np.random.RandomState(0) ## model inputs - integer y for classification self.X = X or T.matrix('X') ## model params ## n_inputs_feats_maps * filter_ht * filter_wd inputs to each hidden unit fan_in = np.prod(filter_shape[1:]) ## each unit receives n_output_feats_maps * filter_ht * filter_wd / pool_size gradient fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) / np.prod(pool_size)) W_bound = np.sqrt(6. / (fan_in + fan_out)) self.W = W or theano.shared(np.asarray(rng.uniform( low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), name='LeNet_W', borrow=True) b_values = np.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = b or theano.shared( value=b_values, borrow=True, name='LeNet_b') self.params = (self.W, self.b) ## model prediction conv_out = conv.conv2d(input=X, filters=self.W, filter_shape=filter_shape, image_shape=image_shape) pooled_out = downsample.max_pool_2d(input=conv_out, ds=pool_size, ignore_border=True) self.prediction = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x'))
def __init__(self, inpts, filter_shape, image_shape, pool_size=(2, 2)): self.image_shape = image_shape self.inpts = inpts.reshape(self.image_shape) self.W = theano.shared(np.random.randn(5, 5)) self.B = theano.shared(np.random.randn(12, )) self.params = [self.W, self.B] self.z = theano.dot(self.inpts, self.W) + self.B self.pool_size = pool_size self.filter_shape = filter_shape conv_out = conv.conv2d(self.inpts, image_shape=self.image_shape, filters=self.W, filter_shape=self.filter_shape) pool_out = downsample.pool_2d(input=conv_out, ds=self.pool_size, ignore_border=True) self.out = T.nnet.sigmoid(pool_out)
def get_reconstructed_input(self): """ Computes the reconstructed input given the values of the hidden layer """ repeated_conv = conv.conv2d(input = self.hidden, filters = self.W_prime, border_mode='full') #repeated_conv=repeated_conv[:,:,1:-1,1:-1] bp=(self.filter_shape[2]-1)/2 repeated_conv=repeated_conv[:,:,bp:-bp,bp:-bp] multiple_conv_out = [repeated_conv.flatten()] * np.prod(self.poolsize) stacked_conv_neibs = T.stack(*multiple_conv_out).T stretch_unpooling_out = T.nnet.neighbours.neibs2images(stacked_conv_neibs, self.poolsize, self.x1.shape) #return self.hidden z=T.tanh(stretch_unpooling_out + self.b_prime.dimshuffle('x', 0, 'x', 'x')) #return T.sum(T.sum((self.x1-z)**2,axis=3),axis=2) #rectified_linear_activation = lambda x: T.maximum(0.0, x) return z
def predict(self, new_data, batch_size): """ predict for new data """ img_shape = (batch_size, 1, self.image_shape[2], self.image_shape[3]) conv_out = conv.conv2d(input=new_data, filters=self.W, filter_shape=self.filter_shape, image_shape=img_shape) # print conv_out if self.non_linear=="tanh": conv_out_tanh = T.tanh(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) output = downsample.max_pool_2d(input=conv_out_tanh, ds=self.poolsize, ignore_border=True) if self.non_linear=="relu": conv_out_tanh = ReLU(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) output = downsample.max_pool_2d(input=conv_out_tanh, ds=self.poolsize, ignore_border=True) else: pooled_out = downsample.max_pool_2d(input=conv_out, ds=self.poolsize, ignore_border=True) output = pooled_out + self.b.dimshuffle('x', 0, 'x', 'x') return output
def get(self, y_p, i, g): W_att_re = self.item("W_att_re", i) b_att_re = self.item("b_att_re", i) B = self.item("B", i) C = self.item("C", i) I = self.item("I", i) beam_size = T.minimum(numpy.int32(abs(self.attrs['beam'])), C.shape[0]) loc = T.cast( T.maximum( T.minimum( T.sum(I, axis=0) * self.n / self.bound - beam_size / 2, T.sum(I, axis=0) - beam_size), 0), 'int32') if self.attrs['beam'] > 0: beam_idx = (self.custom_vars[('P_%d' % i)][loc].dimshuffle( 1, 0).flatten() > 0).nonzero() I = I.reshape((I.shape[0] * I.shape[1], ))[beam_idx].reshape( (beam_size, I.shape[1])) C = C.reshape( (C.shape[0] * C.shape[1], C.shape[2]))[beam_idx].reshape( (beam_size, C.shape[1], C.shape[2])) B = B.reshape( (B.shape[0] * B.shape[1], B.shape[2]))[beam_idx].reshape( (beam_size, B.shape[1], B.shape[2])) if self.attrs['template'] != self.layer.unit.n_out: z_p = T.dot(y_p, W_att_re) + b_att_re else: z_p = y_p if self.attrs['momentum'] == 'conv1d': from theano.tensor.nnet import conv att = self.item('att', i) F = self.item("F", i) v = T.dot( T.sum(conv.conv2d(border_mode='full', input=att.dimshuffle(1, 'x', 0, 'x'), filters=F).dimshuffle(2, 3, 0, 1), axis=1)[F.shape[2] / 2:-F.shape[2] / 2 + 1], self.item("U", i)) v = I * v / v.sum(axis=0, keepdims=True) z_p += T.sum(C * v, axis=0) if g > 0: z_p += self.glimpses[i][-1] h_p = T.tanh(z_p) return B, C, I, h_p, self.item("W_att_in", i), self.item("b_att_in", i)
def init_fbcorr(self, x, x_shp, n_filters, filter_shape, min_out=fbcorr_.DEFAULT_MIN_OUT, max_out=fbcorr_.DEFAULT_MAX_OUT, stride=fbcorr_.DEFAULT_STRIDE, mode=fbcorr_.DEFAULT_MODE, generate=None): # Reference implementation: # ../pythor3/pythor3/operation/fbcorr_/plugins/scipy_naive/scipy_naive.py if stride != fbcorr_.DEFAULT_STRIDE: raise NotImplementedError('stride is not used in reference impl.') fake_x = np.empty((x_shp[2], x_shp[3], x_shp[1]), x.dtype) kerns = self.SLMP._get_filterbank(fake_x, dict(n_filters=n_filters, filter_shape=filter_shape, generate=generate)) kerns = kerns.transpose(0, 3, 1, 2).copy()[:,:,::-1,::-1] x = conv.conv2d( x, kerns, image_shape=x_shp, filter_shape=kerns.shape, border_mode=mode) if mode == 'valid': x_shp = (x_shp[0], n_filters, x_shp[2] - filter_shape[0] + 1, x_shp[3] - filter_shape[1] + 1) elif mode == 'full': x_shp = (x_shp[0], n_filters, x_shp[2] + filter_shape[0] - 1, x_shp[3] + filter_shape[1] - 1) else: raise NotImplementedError('fbcorr mode', mode) if min_out is None and max_out is None: return x, x_shp elif min_out is None: return tensor.minimum(x, max_out), x_shp elif max_out is None: return tensor.maximum(x, min_out), x_shp else: return tensor.clip(x, min_out, max_out), x_shp
def conv_layer_output(self, input, image_shape): conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=self.filter_shape, image_shape=image_shape) if self.non_linear == "tanh": conv_out_tanh = T.tanh(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) output = downsample.max_pool_2d(input=conv_out_tanh, ds=self.poolsize, ignore_border=True) elif self.non_linear == "relu": conv_out_relu = ReLU(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) conv_out = conv_out_relu output = T.max(conv_out_relu, axis=2) argmax = T.argmax(conv_out_relu, axis=2) conv_out = conv_out_relu return output
def __init__(self, name, input, initial_filter_values, filter_shape, image_shape, stride=(1, 1)): self.Name = name self.Filter = theano.shared( value=initial_filter_values, name='FilterFCLayer' + self.Name, ) self.Out = conv.conv2d( input=input, filters=self.Filter, subsample=stride, # Stride filter_shape=filter_shape, image_shape=image_shape)
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2)): assert image_shape[1] == filter_shape[1] self.input = input fan_in = numpy.prod(filter_shape[1:]) fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), borrow=True) # the bias is a 1D tensor -- one bias per output feature map b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, borrow=True) # 卷积 conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape) # 子采样 for 0.8x """ pooled_out = downsample.max_pool_2d( input=conv_out, ds=poolsize, ignore_border=True ) """ #子采样for 0.9x pooled_out = pool.pool_2d(input=conv_out, ws=poolsize, ignore_border=True) self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) # store parameters of this layer self.params = [self.W, self.b]
def lmul_sq_T(self, x): """ .. todo:: WRITEME properly Kind of a stupid hacky method used to support convolutional score matching. Ought to find a way to make _filters symbolic rather than shared. """ assert x.dtype == self._filters.dtype op_axes = ('b', 'c', 0, 1) axes = self.output_axes if tuple(axes) != op_axes: x = x.dimshuffle( axes.index('b'), axes.index('c'), axes.index(0), axes.index(1)) # dot(x, sq(A).T) dummy_v = T.tensor4() sqfilt = T.square(self._filters) z_hs = conv2d(dummy_v, sqfilt, image_shape=self._img_shape, filter_shape=self._filters_shape, subsample=self._subsample, border_mode=self._border_mode, ) rval, xdummy = z_hs.owner.op.grad((dummy_v, sqfilt), (x,)) # Format the output based on the input space axes = self.input_space.axes assert len(axes) == 4 if tuple(axes) != op_axes: rval = rval.dimshuffle( op_axes.index(axes[0]), op_axes.index(axes[1]), op_axes.index(axes[2]), op_axes.index(axes[3])) return rval
def buildConnections(self, rng, input, input_shape, filter_shape): # Table of feature map connectivity # Row represents input feature map and cols represent output feature maps # # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 # 0 x x x x x x x x x x # 1 x x x x x x x x x x # 2 x x x x x x x x x x # 3 x x x x x x x x x x # 4 x x x x x x x x x x # 5 x x x x x x x x x x #implementation of above table connections = [(0, 1, 2), (1, 2, 3), (2, 3, 4), (3, 4, 5), (4, 5, 0), (5, 0, 1), (0, 1, 2, 3), (1, 2, 3, 4), (2, 3, 4, 5), (3, 4, 5, 0), (4, 5, 0, 1), (5, 0, 1, 2), (0, 1, 3, 4), (1, 2, 4, 5), (2, 3, 5, 0), (0, 1, 2, 3, 4, 5)] convs = [] W = [] #fan_in = numpy.prod(filter_shape[1:]) #fan_out = filter_shape[0] * numpy.prod(filter_shape[2:]) #W_bound = numpy.sqrt(6.0 / (fan_in + fan_out)) for i in xrange(len(connections)): fmap_shape = (1, len(connections[i]), filter_shape[2], filter_shape[3]) #note: this should also be multiplied by len(connections[i]) but wont converge fan_in = numpy.prod(filter_shape[2:]) #2.4 is magic from LeCun W_bound = 2.4 / fan_in W.append(self.generateWeights(rng, fmap_shape, W_bound)) convs.append( conv.conv2d(input=input[:, connections[i], :, :], image_shape=(input_shape[0], len(connections[i]), input_shape[2], input_shape[3]), filters=W[i], filter_shape=fmap_shape)) return [convs, W]
def __init__(self, rng, input, filter_shape, image_shape = None, Nonlinear = "tanh", zeroone = False, inc=[0], dropout = False, dropoutrnd = None): if isinstance(input, Layer): self.input = input.output if image_shape==None: image_shape = input.output_shape else: self.input = input assert image_shape[1] == filter_shape[1] assert filter_shape[2]%2 == 1 assert filter_shape[3]%2 == 1 med = (filter_shape[2]-1)/2,(filter_shape[3]-1)/2 fan_in = np.prod(filter_shape[1:]) W_values = np.asarray(rng.uniform( low=-np.sqrt(0.5/fan_in), high=np.sqrt(0.5/fan_in), size=filter_shape), dtype=theano.config.floatX) self.W = theano.shared(value=W_values, name='W_%s'%inc[0]) b_values = np.zeros((filter_shape[0],), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, name='b_%s'%inc[0]) conv_out = conv.conv2d(self.input, self.W, filter_shape=filter_shape, image_shape=image_shape, border_mode="full") #Get middle area conv_out = conv_out[:,:,med[0]:-med[0],med[1]:-med[1]] if Nonlinear: self.output = T.tanh(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) if zeroone: self.output = (self.output+1) * 0.5 else: self.output = conv_out + self.b.dimshuffle('x', 0, 'x', 'x') self.output_shape = (image_shape[0], filter_shape[0], image_shape[2], image_shape[3]) if not (dropout is False): #Embed a layerwise dropout layer self.output = LayerbasedDropOut(self, dropoutrnd, dropout).output self.params = [self.W, self.b] inc[0] = inc[0]+1
def __init__(self, rng, input, filter_shape, image_shape): assert image_shape[1] == filter_shape[1] self.input = input # there are "num input feature maps * filter height * filter width" # inputs to each hidden unit fan_in = numpy.prod(filter_shape[1:]) # each unit in the lower layer receives a gradient from: # "num output feature maps * filter height * filter width" / # pooling size fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:])) # initialize weights with random weights W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), borrow=True) # the bias is a 1D tensor -- one bias per output feature map b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, borrow=True) # convolve input feature maps with filters conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape) # downsample each feature map individually, using maxpooling #pooled_out = downsample.max_pool_2d(input=conv_out, # ds=poolsize, ignore_border=True) # add the bias term. Since the bias is a vector (1D array), we first # reshape it to a tensor of shape (1,n_filters,1,1). Each bias will # thus be broadcasted across mini-batches and feature map # width & height self.output = T.tanh(conv_out + self.b.dimshuffle('x', 0, 'x', 'x')) # store parameters of this layer self.params = [self.W, self.b]
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2), read_file=False, W_input=None, b_input=None): assert image_shape[1] == filter_shape[1] self.input = input fan_in = numpy.prod(filter_shape[1:]) fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared( numpy.asarray( rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX ), borrow=True ) b_values = numpy.zeros((filter_shape[0],), dtype=theano.config.floatX) self.b = theano.shared(value=b_values, borrow=True) if read_file==True: self.W = W_input self.b = b_input conv_out = conv.conv2d( input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape ) pooled_out = downsample.max_pool_2d( input=conv_out, ds=poolsize, ignore_border=True ) self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) self.params = [self.W, self.b] self.input = input
def __init__(self, rng, input, filter_shape, image_shape, poolsize, activation, weights_variance, subsample): assert image_shape[1] == filter_shape[1] self.input = input if activation == 'tanh': activation_function = lambda x: T.tanh(x) fan_in = np.prod(filter_shape[1:]) fan_out = (filter_shape[0] * np.prod(filter_shape[2:]) / np.prod(poolsize)) W_bound = np.sqrt(6. / (fan_in + fan_out)) W_values = np.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype='float32') b_values = np.zeros((filter_shape[0], ), dtype='float32') elif activation == 'relu': activation_function = lambda x: T.maximum(0.0, x) W_values = np.asarray(rng.normal(0.0, weights_variance, size=filter_shape), dtype='float32') b_values = np.ones((filter_shape[0], ), dtype='float32') / 10.0 else: raise ValueError('unknown activation function') self.W = theano.shared(value=W_values, name='W', borrow=True) self.b = theano.shared(value=b_values, name='b', borrow=True) conv_out = conv.conv2d(input, self.W, filter_shape=filter_shape, image_shape=image_shape, subsample=subsample) pooled_out = downsample.max_pool_2d( conv_out, poolsize, ignore_border=True) if poolsize[1] > 1 else conv_out self.output = activation_function(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) self.weights = [self.W, self.b]
def __init__(self, rng, input, filter_shape, image_shape, poolsize=(2, 2), W=None, b=None, activation=T.tanh): assert image_shape[1] == filter_shape[1] self.input = input fan_in = numpy.prod(filter_shape[1:]) fan_out = filter_shape[0] * numpy.prod( filter_shape[2:]) / numpy.prod(poolsize) if W is None: W_bound = numpy.sqrt(6. / (fan_in + fan_out)) W = theano.shared(numpy.asarray(rng.uniform(low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), borrow=True) if b is None: b_values = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) b = theano.shared(value=b_values, borrow=True) self.W = W self.b = b conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape) conv_out = conv_out + self.b.dimshuffle('x', 0, 'x', 'x') if not activation is None: conv_out = activation(conv_out) pooled_out = pool.pool_2d(input=conv_out, ds=poolsize, ignore_border=True) self.output = pooled_out self.params = [self.W, self.b]
def __init__(self, rng, input, filter_shape, image_shape, W=None, b=None, poolsize=(2, 2), activation=T.tanh): if W is None: #calc the W_bound and init the W fan_in = numpy.prod(filter_shape[1:]) fan_out = (filter_shape[0] * numpy.prod(filter_shape[2:]) / numpy.prod(poolsize)) W_bound = numpy.sqrt(6. / (fan_in + fan_out)) self.W = theano.shared(numpy.asarray(rng.uniform( low=-W_bound, high=W_bound, size=filter_shape), dtype=theano.config.floatX), borrow=True) else: self.W = W if b is None: b_value = numpy.zeros((filter_shape[0], ), dtype=theano.config.floatX) self.b = theano.shared(value=b_value, borrow=True) else: self.b = b conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape) pooled_out = downsample.max_pool_2d(input=conv_out, ds=poolsize, ignore_border=True) self.output = activation(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) self.params = [self.W, self.b]
def __init__(self, rng, input, dim, n_feature_maps, window_sizes): """ :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.matrix :param input: symbolic sentence tensor :type dim: int :param dim: dimensions of a word vector :type n_feature_maps: int :param n_feature_maps: number of feature maps :type window_sizes: tuple of int :param window_sizes: convolution kernel window sizes """ self.input = input self.dim = dim self.n_feature_maps = n_feature_maps self.window_sizes = window_sizes self.params = [] reshaped_input = self.input.dimshuffle('x', 'x', 0, 1) self.output = None for window_size in window_sizes: W_init = numpy.asarray(rng.uniform(low=-0.1, high=0.1, size=(self.n_feature_maps, 1, window_size, self.dim)), dtype=theano.config.floatX) W = theano.shared(value=W_init) self.params.append(W) conv_out = conv.conv2d(input=reshaped_input, filters=W) max_out = T.max(conv_out, axis=2).flatten() self.output = max_out if self.output is None else T.concatenate( [self.output, max_out]) b_init = numpy.zeros((self.n_feature_maps * len(self.window_sizes), ), dtype=theano.config.floatX) self.b = theano.shared(value=b_init) self.params.append(self.b) self.output = self.output + self.b
def _setUp(self, input, inputDimensions): self.inputDimensions = inputDimensions nrKernelsPrevious = inputDimensions[0] initialWeights = random.normal(loc=0.0, scale=0.1, size=(self.nrKernels, nrKernelsPrevious, self.kernelSize[0], self.kernelSize[1])) initialBiases = np.zeros(self.nrKernels) W = theano.shared(value=np.asarray(initialWeights, dtype=theanoFloat), name='Wconv') b = theano.shared(value=np.asarray(initialBiases, dtype=theanoFloat), name='bconv') self.output = self.activationFun.deterministic( conv.conv2d(input, W) + b.dimshuffle('x', 0, 'x', 'x')) self.params = [W, b]
def process(self, data, batchSize): ''' >>>process newly input data >>>type data: T.tensor4 >>>para data: newly input data >>>type batchSize: int >>>para batchSize: minibatch size ''' shape = (batchSize, self.shape[1], self.shape[2], self.shape[3]) conv_out = conv.conv2d(input=data, filters=self.w, filter_shape=self.filters, image_shape=shape) pool_out = downsample.max_pool_2d(input=conv_out, ds=self.pool, ignore_border=True) output = T.tanh(pool_out + self.b.dimshuffle('x', 0, 'x', 'x')) return output
def compute_cnn(self, input, filter_shape, image_shape, border_mode, pool_size): assert image_shape[1] == filter_shape[1] self.input = input conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape, border_mode=border_mode) pooled_out = downsample.max_pool_2d( input=conv_out, ds=pool_size, ignore_border=True, ) self.output = theano.tensor.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) self.params = [self.W, self.b]
def __init__(self, input, rng, input_shape, filter_shape, pool_size=(2, 2)): self.input = input self.rng = rng fan_in = np.prod(filter_shape[1:]) w_value = np.array(self.rng.uniform(low=-3.0 / fan_in, high=3.0 / fan_in, size=filter_shape), dtype=theano.config.floatX) self.W = theano.shared(value=w_value, name='c_w', borrow=True) b_value = np.zeros(filter_shape[0], dtype=theano.config.floatX) self.b = theano.shared(value=b_value, name='c_b', borrow=True) self.params = [self.W, self.b] conv_out = conv.conv2d(self.input, self.W, image_shape=input_shape, filter_shape=filter_shape) max_pool_out = pool.pool_2d(conv_out, pool_size) self.outputs = T.tanh(max_pool_out + self.b.dimshuffle('x', 0, 'x', 'x'))
def __init__(self, input, params_W, params_b, filter_shape, image_shape, poolsize=(2, 2)): assert image_shape[1] == filter_shape[1] self.input = input self.W = params_W self.b = params_b # 卷积 conv_out = conv.conv2d( input=input, filters=self.W, filter_shape=filter_shape, image_shape=image_shape ) # 子采样 pooled_out = downsample.max_pool_2d( input=conv_out, ds=poolsize, ignore_border=True ) self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) self.params = [self.W, self.b]
def gradient_sum(vmap): if self.visible_shape is not None: i_shape = [self.visible_shape[k] for k in [1, 0, 2, 3]] else: i_shape = None if self.hidden_shape is not None: f_shape = [self.hidden_shape[k] for k in [1, 0, 2, 3]] else: f_shape = None v_shuffled = vmap[self.vu].dimshuffle(1, 0, 2, 3) h_shuffled = vmap[self.hu].dimshuffle(1, 0, 2, 3) c = conv.conv2d(v_shuffled, h_shuffled, border_mode='valid', image_shape=i_shape, filter_shape=f_shape) return c.dimshuffle(1, 0, 2, 3)
def __init__(self, input, filter_w, filter_h, filter_num, img_w, img_h, input_feature, batch_size, poolsize, params_W, params_b): self.input = input self.W = params_W self.b = params_b conv_out = conv.conv2d(input=input, filters=self.W, filter_shape=(filter_num, input_feature, filter_h, filter_w), image_shape=(batch_size, input_feature, img_h, img_w)) pooled_out = pool.pool_2d(input=conv_out, ds=poolsize, ignore_border=True) self.output = T.tanh(pooled_out + self.b.dimshuffle('x', 0, 'x', 'x')) self.params = [self.W, self.b]