def __init__(self, input, input_shape=None, printinfo=True): from pylearn2.expr.normalize import CrossChannelNormalization self.lrn_func = CrossChannelNormalization() self.get_input_shape(input, input_shape) self.output = self.lrn_func(self.input) self.output_shape = self.input_shape self.name = 'LRN\t' if printinfo: self.print_shape()
def __init__(self, input, filter_shape, image_shape, f_params_w, f_params_b, lrn=False, t_style=None, t_content=None, convstride=1, padsize =0, group=1, poolsize = 3, poolstride = 1): self.input = input #theano.shared(np.asarray(np.input)) if t_style is not None: self.t_style = np.asarray(np.load(t_style),dtype=theano.config.floatX) if t_content is not None: self.t_content = np.asarray(np.load(t_content),dtype=theano.config.floatX) if lrn is True: self.lrn_func = CrossChannelNormalization() #if padsize==(0,0): #padsize='valid' if group == 1: self.W = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w)),(3,0,1,2)),dtype=theano.config.floatX), borrow=True) self.b = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b)),dtype=theano.config.floatX), borrow=True) conv_out = conv2d(input=self.input,filters=self.W,filter_shape=filter_shape,border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) #self.params = [self.W, self.b] elif group == 2: self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[1] = self.filter_shape[1] / 2 #self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[1] = self.image_shape[1] / 2 self.W0 = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w[0])),(3,0,1,2)),dtype=theano.config.floatX), borrow=True) self.W1 = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w[1])),(3,0,1,2)),dtype=theano.config.floatX), borrow=True) self.b0 = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b[0])),dtype=theano.config.floatX), borrow=True) self.b1 = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b[1])),dtype=theano.config.floatX), borrow=True) conv_out0 = conv2d(input=self.input[:,:self.image_shape[1],:,:],filters=self.W0,filter_shape=tuple(self.filter_shape),border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) + self.b0.dimshuffle('x', 0, 'x', 'x') conv_out1 = conv2d(input=self.input[:,self.image_shape[1]:,:,:],filters=self.W1,filter_shape=tuple(self.filter_shape),border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) + self.b1.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1],axis=1) #self.params = [self.W0, self.b0, self.W1, self.b1] else: raise AssertionError() relu_out = T.maximum(conv_out, 0) if poolsize != 1: self.output = pool.pool_2d(input=relu_out,ds=(poolsize,poolsize),ignore_border=True, st=(poolstride,poolstride),mode='average_exc_pad') #self.output = downsample.max_pool_2d(input=relu_out,ds=(poolsize,poolsize),ignore_border=True, st=(poolstride,poolstride)) else: self.output = relu_out if lrn is True: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output)
def basic_test(): channels = 15 rows = 3 cols = 4 batch_size = 2 shape = [channels, rows, cols, batch_size] k = 2 n = 5 # use a big value of alpha so mistakes involving alpha show up strong alpha = 1.5 beta = 0.75 # Perform test for C01B rng = np.random.RandomState([2013, 2]) c01b = rng.randn(*shape).astype(config.floatX) normalizer = CrossChannelNormalization(k=k, n=n, alpha=alpha, beta=beta) warnings.warn("TODO: add test for the CudaConvnet version.") X = T.TensorType(dtype=config.floatX, broadcastable=tuple([False] * 4))() out = normalizer(X) out = function([X], out)(c01b) ground_out = ground_truth_normalizer(c01b, n=n, k=k, alpha=alpha, beta=beta) assert out.shape == ground_out.shape diff = out - ground_out err = np.abs(diff) max_err = err.max() if not np.allclose(out, ground_out): print('C01B test failed') print('error range: ', (err.min(), err.max())) print('output: ') print(out) print('expected output: ') print(ground_out) assert False # Perform test for BC01 bc01 = np.transpose(c01b, [3, 0, 1, 2]) normalizerBC01 = CrossChannelNormalizationBC01(k=k, n=n, alpha=alpha, beta=beta) X = T.TensorType(dtype=config.floatX, broadcastable=tuple([False] * 4))() out = normalizerBC01(X) out = function([X], out)(bc01) ground_out_BC01 = np.transpose(ground_out, [3, 0, 1, 2]) assert out.shape == ground_out_BC01.shape diff = out - ground_out_BC01 err = np.abs(diff) max_err = err.max() if not np.allclose(out, ground_out_BC01): print('BC01 test failed') print('error range: ', (err.min(), err.max())) print('output: ') print(out) print('expected output: ') print(ground_out) assert False
def __init__( self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', ): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous(input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous(input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu self.output = T.maximum(conv_out, 0) conv_out = gpu_contiguous(conv_out) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) elif lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv( img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudaconvnet or cudnn") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv ({}) layer with shape_in: {}".format( lib_conv, str(image_shape))
def __init__(self, input_layer, alpha=0.0001, beta=0.75, n=5): self.input_layer = input_layer self.n_channel = self.input_layer.n_channel # self._shape = input_layer.shape() self.lrn_func = CrossChannelNormalization(alpha=alpha, beta=beta, n=n)
def __init__( self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', poolpadsize=(0, 0), caffe_style=False, Bn=False, ): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv # assert input.shape==image_shape assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization(alpha=0.0005, k=1) # self.lrn_func = CrossChannelNormalization(alpha=0.0005) if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous(input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous(input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) elif lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv( img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) self.conv_out = conv_out if Bn: #Warning this just used for testing phase!!!! self.mean = theano.shared( value=np.zeros((1, filter_shape[3], 1, 1), dtype=theano.config.floatX), broadcastable=[True, False, True, True], name='mean', borrow=True) self.var = theano.shared( value=np.ones((1, filter_shape[3], 1, 1), dtype=theano.config.floatX), broadcastable=[True, False, True, True], name='var', borrow=True) self.gamma = theano.shared(value=np.ones( (filter_shape[3], ), dtype=theano.config.floatX), name='gamma', borrow=True) self.beta = theano.shared(value=np.zeros( (filter_shape[3], ), dtype=theano.config.floatX), name='beta', borrow=True) conv_out = batch_normalization(inputs=conv_out, gamma=self.gamma, beta=self.beta, mean=self.mean, std=T.sqrt(self.var), mode='high_mem') # ReLu self.Bn = conv_out self.output = T.maximum(conv_out, 0) # # Pooling if caffe_style: self.output = self.output[:, :, ::-1, ::-1] if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride), pad=poolpadsize) if caffe_style: self.output = self.output[:, :, ::-1, ::-1] self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudaconvnet or cudnn") if group == 1: if Bn: #self.params = [self.W.val, self.b.val,self.beta,self.gamma,self.mean,self.var] self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] #self.weight_type = ['W', 'b','b','b','b','b'] pass else: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv ({}) layer with shape_in: {}".format( lib_conv, str(image_shape))
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', ): """ lib_conv can be cudnn or cudaconvet """ assert group in [1, 2] assert lib_conv in ['cudnn', 'cudaconvnet'] self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride if lib_conv == 'cudnn': self.channel = image_shape[1] else: self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) if lib_conv == 'cudnn': self.b = Weight(self.filter_shape[0], bias_init, std=0) else: self.b = Weight(self.filter_shape[3], bias_init, std=0) else: if lib_conv == 'cudnn': self.filter_shape[1] /= 2 self.filter_shape[0] /= 2 self.image_shape[1] /= 2 self.image_shape[0] /= 2 else: self.filter_shape[0] /= 2 self.filter_shape[3] /= 2 self.image_shape[0] /= 2 self.image_shape[3] /= 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) if lib_conv=='cudnn': self.b0 = Weight(self.filter_shape[0], bias_init, std=0) self.b1 = Weight(self.filter_shape[0], bias_init, std=0) else: self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous( input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op( contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous( input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op( contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu conv_out = T.maximum(conv_out, 0) self.output = gpu_contiguous(conv_out) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) else: if group == 1: conv_out = dnn.dnn_conv(img=input, kerns=self.W.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: input1, input2 = \ theano.tensor.split(input, [self.channel/2, self.channel/2], 2, axis=1) conv_out0 = \ dnn.dnn_conv(img=input1, kerns=self.W0.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + self.b0.val.dimshuffle('x', 0, 'x', 'x') conv_out1 = \ dnn.dnn_conv(img=input2, kerns=self.W1.val, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + self.b1.val.dimshuffle('x', 0, 'x', 'x') # self.conv_out = conv_out1 conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) # LRN if self.lrn: self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv ({}) layer with shape_in: {}".format(lib_conv, str(image_shape))
def __init__(self, config): ModelBase.__init__(self) self.verbose = config['verbose'] self.config = config if self.verbose: print 'GoogLeNet 7/5' batch_size = config['batch_size'] input_width = config['input_width'] input_height = config['input_height'] n_softmax_out = config['n_softmax_out'] self.name = 'googlenet' self.batch_size = batch_size self.input_width = input_width self.input_height = input_height self.n_softmax_out = n_softmax_out self.lrn_func = CrossChannelNormalization() x = T.ftensor4('x') y = T.lvector('y') self.x = x # c01b self.y = y layers = [] params = [] weight_types = [] conv_7x7 = ConvPool_LRN( input=x, image_shape=(3, 224, 224, batch_size), #c01b (3, 224, 224, batch_size) filter_shape=(3, 7, 7, 64), convstride=2, padsize=3, poolsize=3, poolstride=2, poolpad=1, W=Weight((3, 7, 7, 64), mean=0.0, std=0.1), b=Weight((64, ), mean=0.2, std=0), lrn=True, lib_conv='cudnn', ) layers.append(conv_7x7) params += conv_7x7.params weight_types += conv_7x7.weight_type # output shape = (112x112x64) # output shape = (56x56x64) conv_r3x3 = Conv( input=conv_7x7.output, image_shape=(64, 56, 56, batch_size), filter_shape=(64, 1, 1, 64), convstride=1, padsize=0, W=Weight((64, 1, 1, 64), mean=0.0, std=0.1), b=Weight((64, ), mean=0.2, std=0), lib_conv='cudnn', ) layers.append(conv_r3x3) params += conv_r3x3.params weight_types += conv_r3x3.weight_type # output shape = (56x56x64) conv_3x3 = ConvPool_LRN( input=conv_r3x3.output, image_shape=(64, 56, 56, batch_size), filter_shape=(64, 3, 3, 192), convstride=1, padsize=1, poolsize=3, poolstride=2, poolpad=1, W=Weight((64, 3, 3, 192), mean=0.0, std=0.03), b=Weight((192, ), mean=0.2, std=0), lrn=True, lib_conv='cudnn', ) layers.append(conv_3x3) params += conv_3x3.params weight_types += conv_3x3.weight_type # output shape = (56x56x192) # output shape = (28x28x192) incep3a = Incept(conv_3x3.output, input_shape=(192, 28, 28, batch_size)) layers += incep3a.layers params += incep3a.params weight_types += incep3a.weight_types print 'incep3a output shape: (28x28x256)' # output shape = (28x28x256) incep3b = Incept(incep3a.output, input_shape=(256, 28, 28, batch_size), n1x1=128, nr3x3=128, n3x3=192, nr5x5=32, n5x5=96, npj=64) layers += incep3b.layers params += incep3b.params weight_types += incep3b.weight_types print 'incep3b output shape: (28x28x480)' # output shape = (28x28x480) # lrn3 = self.lrn_func(incep3b.output) # print 'LRN(added)' pool3 = Pool(input=incep3b.output, poolsize=3, poolstride=2, poolpad=1, mode='max') # output shape = (14x14x480) incep4a = Incept(pool3.output, input_shape=(480, 14, 14, batch_size), n1x1=192, nr3x3=96, n3x3=208, nr5x5=16, n5x5=48, npj=64) layers += incep4a.layers params += incep4a.params weight_types += incep4a.weight_types print 'incep4a output shape: (14x14x512)' # output shape = (14x14x512) incep4b = Incept(incep4a.output, input_shape=(512, 14, 14, batch_size), n1x1=160, nr3x3=112, n3x3=224, nr5x5=24, n5x5=64, npj=64) layers += incep4b.layers params += incep4b.params weight_types += incep4b.weight_types print 'incep4b output shape: (14x14x512)' # output shape = (14x14x512) incep4c = Incept(incep4b.output, input_shape=(512, 14, 14, batch_size), n1x1=128, nr3x3=128, n3x3=256, nr5x5=24, n5x5=64, npj=64) layers += incep4c.layers params += incep4c.params weight_types += incep4c.weight_types print 'incep4c output shape: (14x14x512)' # output shape = (14x14x512) incep4d = Incept(incep4c.output, input_shape=(512, 14, 14, batch_size), n1x1=112, nr3x3=144, n3x3=288, nr5x5=32, n5x5=64, npj=64) layers += incep4d.layers params += incep4d.params weight_types += incep4d.weight_types print 'incep4d output shape: (14x14x528)' # output shape = (14x14x528) incep4e = Incept(incep4d.output, input_shape=(528, 14, 14, batch_size), n1x1=256, nr3x3=160, n3x3=320, nr5x5=32, n5x5=128, npj=128) layers += incep4e.layers params += incep4e.params weight_types += incep4e.weight_types print 'incep4e output shape: (14x14x832)' # output shape = (14x14x832) lrn4 = self.lrn_func( incep4e.output) # turn on only this for 16data, 53s/5120images print 'LRN(added)' pool4 = Pool( input=lrn4, #incep4e.output, poolsize=3, poolstride=2, poolpad=1, mode='max') # output shape = (7x7x832) incep5a = Incept(pool4.output, input_shape=(832, 7, 7, batch_size), n1x1=256, nr3x3=160, n3x3=320, nr5x5=32, n5x5=128, npj=128) layers += incep5a.layers params += incep5a.params weight_types += incep5a.weight_types print 'incep5a output shape: (7x7x832)' # output shape = (7x7x832) incep5b = Incept(incep5a.output, input_shape=(832, 7, 7, batch_size), n1x1=384, nr3x3=192, n3x3=384, nr5x5=48, n5x5=128, npj=128) layers += incep5b.layers params += incep5b.params weight_types += incep5b.weight_types print 'incep5b output shape: (7x7x1024)' # output shape = (7x7x1024) # lrn5 = self.lrn_func(incep5b.output) # turn on only this for 16data, 51s/5120images # print 'LRN(added)' poolx = Pool(input=incep5b.output, poolsize=7, poolstride=1, poolpad=0, mode='average') # output shape = (1x1x1024) l_flatten = T.flatten(poolx.output.dimshuffle(3, 0, 1, 2), 2) # output shape = (1024) dropout = Dropout(input=l_flatten, n_in=1024, n_out=1024, prob_drop=0.4) # output shape = (1024) softmax_layer = Softmax(input=dropout.output, n_in=1024, n_out=n_softmax_out) # output shape = (n_softmax_out) layers.append(softmax_layer) params += softmax_layer.params weight_types += softmax_layer.weight_type # auxilary classifier print 'auxilary classifier 1:' aux1 = aux_tower(input=incep4a.output, input_shape=(512, 14, 14, batch_size), config=config) layers += aux1.layers params += aux1.params weight_types += aux1.weight_types print 'auxilary classifier 2:' aux2 = aux_tower(input=incep4d.output, input_shape=(528, 14, 14, batch_size), config=config) layers += aux2.layers params += aux2.params weight_types += aux2.weight_types self.layers = layers self.params = params self.weight_types = weight_types self.output = softmax_layer.p_y_given_x self.cost = softmax_layer.negative_log_likelihood(y)+\ 0.3*aux1.negative_log_likelihood(y)+0.3*aux2.negative_log_likelihood(y) self.error = softmax_layer.errors(y) self.error_top_5 = softmax_layer.errors_top_x(y) # training related self.base_lr = np.float32(config['learning_rate']) self.shared_lr = theano.shared(self.base_lr) self.mu = config['momentum'] # def: 0.9 # momentum self.eta = config['weight_decay'] #0.0002 # weight decay self.shared_x = theano.shared(np.zeros( (3, config['input_width'], config['input_height'], config['file_batch_size']), dtype=theano.config.floatX), borrow=True) self.shared_y = theano.shared(np.zeros((config['file_batch_size'], ), dtype=int), borrow=True) # shared variable for storing momentum before exchanging momentum(delta w) self.vels = [ theano.shared(param_i.get_value() * 0.) for param_i in self.params ] # shared variable for accepting momentum during exchanging momentum(delta w) self.vels2 = [ theano.shared(param_i.get_value() * 0.) for param_i in self.params ] self.train = None self.get_vel = None self.descent_vel = None self.val = None self.inference = None
def __init__( self, input, image_shape, filter_shape, convstride, padsize, poolsize, poolstride, poolpad, W, b, lrn=False, lib_conv='cudnn', ): self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) self.W = W #Weight(self.filter_shape) self.b = b #Weight(self.filter_shape[3])#, bias_init, std=0) input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv( img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') # ReLu self.output = T.maximum(conv_out, 0) # Pool self.poolsize = poolsize self.poolstride = poolstride self.poolpad = poolpad if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride), mode='max', pad=(poolpad, poolpad)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b # LRN if self.lrn: self.lrn_func = CrossChannelNormalization() # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] print "conv ({}) layer with shape_in: {}".format( lib_conv, str(image_shape))
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False): self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous(input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous(input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu self.output = T.maximum(conv_out, 0) conv_out = gpu_contiguous(conv_out) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] print "conv layer with shape_in: " + str(image_shape)
def __init__(self, input, image_shape, filter_shape, convstride, padsize, group, poolsize, poolstride, bias_init, lrn=False, lib_conv='cudnn', verbose=False ): ''' lib_conv can be cudnn (recommended)or cudaconvnet ''' self.filter_size = filter_shape self.convstride = convstride self.padsize = padsize self.poolsize = poolsize self.poolstride = poolstride self.channel = image_shape[0] self.lrn = lrn self.lib_conv = lib_conv self.verbose = verbose assert group in [1, 2] self.filter_shape = np.asarray(filter_shape) self.image_shape = np.asarray(image_shape) if self.lrn: self.lrn_func = CrossChannelNormalization() if group == 1: self.W = Weight(self.filter_shape) self.b = Weight(self.filter_shape[3], bias_init, std=0) else: self.filter_shape[0] = self.filter_shape[0] / 2 self.filter_shape[3] = self.filter_shape[3] / 2 self.image_shape[0] = self.image_shape[0] / 2 self.image_shape[3] = self.image_shape[3] / 2 self.W0 = Weight(self.filter_shape) self.W1 = Weight(self.filter_shape) self.b0 = Weight(self.filter_shape[3], bias_init, std=0) self.b1 = Weight(self.filter_shape[3], bias_init, std=0) if lib_conv == 'cudaconvnet': self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride, partial_sum=1) from theano.sandbox.cuda.basic_ops import gpu_contiguous # Conv if group == 1: contiguous_input = gpu_contiguous(input) contiguous_filters = gpu_contiguous(self.W.val) conv_out = self.conv_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x') else: contiguous_input0 = gpu_contiguous( input[:self.channel / 2, :, :, :]) contiguous_filters0 = gpu_contiguous(self.W0.val) conv_out0 = self.conv_op( contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle(0, 'x', 'x', 'x') contiguous_input1 = gpu_contiguous( input[self.channel / 2:, :, :, :]) contiguous_filters1 = gpu_contiguous(self.W1.val) conv_out1 = self.conv_op( contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle(0, 'x', 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=0) # ReLu conv_out = gpu_contiguous(conv_out) self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.pool_op = MaxPool(ds=poolsize, stride=poolstride) self.output = self.pool_op(self.output) elif lib_conv == 'corrmm': from theano.sandbox.cuda.basic_ops import gpu_contiguous from theano.sandbox.cuda.blas import GpuCorrMM border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize) self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride), border_mode=border_mode) flip_filters=True input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 if group==1: filters = self.W.val.dimshuffle(3, 0, 1, 2) if flip_filters: filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right contiguous_filters = gpu_contiguous(filters) contiguous_input = gpu_contiguous(input_shuffled) conv_out = self.corr_mm_op(contiguous_input, contiguous_filters) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 if flip_filters: W0_shuffled = W0_shuffled[:, :, ::-1, ::-1] contiguous_filters0 = gpu_contiguous(W0_shuffled) contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :]) conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 if flip_filters: W1_shuffled = W1_shuffled[:, :, ::-1, ::-1] contiguous_filters1 = gpu_contiguous(W1_shuffled) contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :]) conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: from theano.tensor.signal import downsample self.output = downsample.max_pool_2d(self.output, ds=(poolsize,poolsize), st=(poolstride,poolstride), ignore_border=False, padding=(0,0), mode='max', ) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b elif lib_conv == 'cudnn': input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01 # in01out to outin01 # print image_shape_shuffled # print filter_shape_shuffled if group == 1: W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out = dnn.dnn_conv(img=input_shuffled, kerns=W_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x') else: W0_shuffled = \ self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out0 = \ dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2, :, :], kerns=W0_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out0 = conv_out0 + \ self.b0.val.dimshuffle('x', 0, 'x', 'x') W1_shuffled = \ self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01 conv_out1 = \ dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:, :, :], kerns=W1_shuffled, subsample=(convstride, convstride), border_mode=padsize, ) conv_out1 = conv_out1 + \ self.b1.val.dimshuffle('x', 0, 'x', 'x') conv_out = T.concatenate([conv_out0, conv_out1], axis=1) # ReLu self.output = T.maximum(conv_out, 0) # Pooling if self.poolsize != 1: self.output = dnn.dnn_pool(self.output, ws=(poolsize, poolsize), stride=(poolstride, poolstride)) self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b else: NotImplementedError("lib_conv can only be cudaconvnet or cudnn") # LRN if self.lrn: # lrn_input = gpu_contiguous(self.output) self.output = self.lrn_func(self.output) if group == 1: self.params = [self.W.val, self.b.val] self.weight_type = ['W', 'b'] else: self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val] self.weight_type = ['W', 'b', 'W', 'b'] if self.verbose: print "conv ({}) layer with shape_in: {}".format(lib_conv, str(image_shape))