def __init__(self, input, input_shape=None, printinfo=True):
from pylearn2.expr.normalize import CrossChannelNormalization
self.lrn_func = CrossChannelNormalization()
self.get_input_shape(input, input_shape)
self.output = self.lrn_func(self.input)
self.output_shape = self.input_shape
self.name = 'LRN\t'
if printinfo: self.print_shape()
def __init__(self, input, filter_shape, image_shape, f_params_w, f_params_b, lrn=False, t_style=None, t_content=None, convstride=1, padsize =0, group=1, poolsize = 3, poolstride = 1):
self.input = input
#theano.shared(np.asarray(np.input))
if t_style is not None:
self.t_style = np.asarray(np.load(t_style),dtype=theano.config.floatX)
if t_content is not None:
self.t_content = np.asarray(np.load(t_content),dtype=theano.config.floatX)
if lrn is True:
self.lrn_func = CrossChannelNormalization()
#if padsize==(0,0):
#padsize='valid'
if group == 1:
self.W = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w)),(3,0,1,2)),dtype=theano.config.floatX), borrow=True)
self.b = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b)),dtype=theano.config.floatX), borrow=True)
conv_out = conv2d(input=self.input,filters=self.W,filter_shape=filter_shape,border_mode = padsize,subsample=(convstride, convstride),filter_flip=True)
#self.params = [self.W, self.b]
elif group == 2:
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[1] = self.filter_shape[1] / 2
#self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[1] = self.image_shape[1] / 2
self.W0 = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w[0])),(3,0,1,2)),dtype=theano.config.floatX), borrow=True)
self.W1 = theano.shared(np.asarray(np.transpose(np.load(os.path.join(params_path,f_params_w[1])),(3,0,1,2)),dtype=theano.config.floatX), borrow=True)
self.b0 = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b[0])),dtype=theano.config.floatX), borrow=True)
self.b1 = theano.shared(np.asarray(np.load(os.path.join(params_path,f_params_b[1])),dtype=theano.config.floatX), borrow=True)
conv_out0 = conv2d(input=self.input[:,:self.image_shape[1],:,:],filters=self.W0,filter_shape=tuple(self.filter_shape),border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) + self.b0.dimshuffle('x', 0, 'x', 'x')
conv_out1 = conv2d(input=self.input[:,self.image_shape[1]:,:,:],filters=self.W1,filter_shape=tuple(self.filter_shape),border_mode = padsize,subsample=(convstride, convstride),filter_flip=True) + self.b1.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1],axis=1)
#self.params = [self.W0, self.b0, self.W1, self.b1]
else:
raise AssertionError()
relu_out = T.maximum(conv_out, 0)
if poolsize != 1:
self.output = pool.pool_2d(input=relu_out,ds=(poolsize,poolsize),ignore_border=True, st=(poolstride,poolstride),mode='average_exc_pad')
#self.output = downsample.max_pool_2d(input=relu_out,ds=(poolsize,poolsize),ignore_border=True, st=(poolstride,poolstride))
else:
self.output = relu_out
if lrn is True:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
def basic_test():
channels = 15
rows = 3
cols = 4
batch_size = 2
shape = [channels, rows, cols, batch_size]
k = 2
n = 5
# use a big value of alpha so mistakes involving alpha show up strong
alpha = 1.5
beta = 0.75
# Perform test for C01B
rng = np.random.RandomState([2013, 2])
c01b = rng.randn(*shape).astype(config.floatX)
normalizer = CrossChannelNormalization(k=k, n=n, alpha=alpha, beta=beta)
warnings.warn("TODO: add test for the CudaConvnet version.")
X = T.TensorType(dtype=config.floatX, broadcastable=tuple([False] * 4))()
out = normalizer(X)
out = function([X], out)(c01b)
ground_out = ground_truth_normalizer(c01b,
n=n,
k=k,
alpha=alpha,
beta=beta)
assert out.shape == ground_out.shape
diff = out - ground_out
err = np.abs(diff)
max_err = err.max()
if not np.allclose(out, ground_out):
print('C01B test failed')
print('error range: ', (err.min(), err.max()))
print('output: ')
print(out)
print('expected output: ')
print(ground_out)
assert False
# Perform test for BC01
bc01 = np.transpose(c01b, [3, 0, 1, 2])
normalizerBC01 = CrossChannelNormalizationBC01(k=k,
n=n,
alpha=alpha,
beta=beta)
X = T.TensorType(dtype=config.floatX, broadcastable=tuple([False] * 4))()
out = normalizerBC01(X)
out = function([X], out)(bc01)
ground_out_BC01 = np.transpose(ground_out, [3, 0, 1, 2])
assert out.shape == ground_out_BC01.shape
diff = out - ground_out_BC01
err = np.abs(diff)
max_err = err.max()
if not np.allclose(out, ground_out_BC01):
print('BC01 test failed')
print('error range: ', (err.min(), err.max()))
print('output: ')
print(out)
print('expected output: ')
print(ground_out)
assert False
def __init__(
self,
input,
image_shape,
filter_shape,
convstride,
padsize,
group,
poolsize,
poolstride,
bias_init,
lrn=False,
lib_conv='cudnn',
):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize,
stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(input[:self.channel /
2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(contiguous_input0,
contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(input[self.channel /
2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(contiguous_input1,
contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
self.output = T.maximum(conv_out, 0)
conv_out = gpu_contiguous(conv_out)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
elif lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(
img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudaconvnet or cudnn")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv ({}) layer with shape_in: {}".format(
lib_conv, str(image_shape))
def __init__(self, input_layer, alpha=0.0001, beta=0.75, n=5):
self.input_layer = input_layer
self.n_channel = self.input_layer.n_channel
# self._shape = input_layer.shape()
self.lrn_func = CrossChannelNormalization(alpha=alpha, beta=beta, n=n)
def __init__(
self,
input,
image_shape,
filter_shape,
convstride,
padsize,
group,
poolsize,
poolstride,
bias_init,
lrn=False,
lib_conv='cudnn',
poolpadsize=(0, 0),
caffe_style=False,
Bn=False,
):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
# assert input.shape==image_shape
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization(alpha=0.0005, k=1)
# self.lrn_func = CrossChannelNormalization(alpha=0.0005)
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize,
stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(input[:self.channel /
2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(contiguous_input0,
contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(input[self.channel /
2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(contiguous_input1,
contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
elif lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(
img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
self.conv_out = conv_out
if Bn:
#Warning this just used for testing phase!!!!
self.mean = theano.shared(
value=np.zeros((1, filter_shape[3], 1, 1),
dtype=theano.config.floatX),
broadcastable=[True, False, True, True],
name='mean',
borrow=True)
self.var = theano.shared(
value=np.ones((1, filter_shape[3], 1, 1),
dtype=theano.config.floatX),
broadcastable=[True, False, True, True],
name='var',
borrow=True)
self.gamma = theano.shared(value=np.ones(
(filter_shape[3], ), dtype=theano.config.floatX),
name='gamma',
borrow=True)
self.beta = theano.shared(value=np.zeros(
(filter_shape[3], ), dtype=theano.config.floatX),
name='beta',
borrow=True)
conv_out = batch_normalization(inputs=conv_out,
gamma=self.gamma,
beta=self.beta,
mean=self.mean,
std=T.sqrt(self.var),
mode='high_mem')
# ReLu
self.Bn = conv_out
self.output = T.maximum(conv_out, 0)
# # Pooling
if caffe_style:
self.output = self.output[:, :, ::-1, ::-1]
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride),
pad=poolpadsize)
if caffe_style:
self.output = self.output[:, :, ::-1, ::-1]
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudaconvnet or cudnn")
if group == 1:
if Bn:
#self.params = [self.W.val, self.b.val,self.beta,self.gamma,self.mean,self.var]
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
#self.weight_type = ['W', 'b','b','b','b','b']
pass
else:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv ({}) layer with shape_in: {}".format(
lib_conv, str(image_shape))
def __init__(self, input, image_shape, filter_shape, convstride, padsize,
group, poolsize, poolstride, bias_init, lrn=False,
lib_conv='cudnn',
):
"""
lib_conv can be cudnn or cudaconvet
"""
assert group in [1, 2]
assert lib_conv in ['cudnn', 'cudaconvnet']
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
if lib_conv == 'cudnn':
self.channel = image_shape[1]
else:
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
if lib_conv == 'cudnn':
self.b = Weight(self.filter_shape[0], bias_init, std=0)
else:
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
if lib_conv == 'cudnn':
self.filter_shape[1] /= 2
self.filter_shape[0] /= 2
self.image_shape[1] /= 2
self.image_shape[0] /= 2
else:
self.filter_shape[0] /= 2
self.filter_shape[3] /= 2
self.image_shape[0] /= 2
self.image_shape[3] /= 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
if lib_conv=='cudnn':
self.b0 = Weight(self.filter_shape[0], bias_init, std=0)
self.b1 = Weight(self.filter_shape[0], bias_init, std=0)
else:
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(
input[:self.channel / 2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(
contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(
input[self.channel / 2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(
contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
conv_out = T.maximum(conv_out, 0)
self.output = gpu_contiguous(conv_out)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
else:
if group == 1:
conv_out = dnn.dnn_conv(img=input,
kerns=self.W.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
input1, input2 = \
theano.tensor.split(input,
[self.channel/2, self.channel/2],
2, axis=1)
conv_out0 = \
dnn.dnn_conv(img=input1,
kerns=self.W0.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + self.b0.val.dimshuffle('x', 0, 'x', 'x')
conv_out1 = \
dnn.dnn_conv(img=input2,
kerns=self.W1.val,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + self.b1.val.dimshuffle('x', 0, 'x', 'x')
# self.conv_out = conv_out1
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
# LRN
if self.lrn:
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv ({}) layer with shape_in: {}".format(lib_conv,
str(image_shape))
def __init__(self, config):
ModelBase.__init__(self)
self.verbose = config['verbose']
self.config = config
if self.verbose: print 'GoogLeNet 7/5'
batch_size = config['batch_size']
input_width = config['input_width']
input_height = config['input_height']
n_softmax_out = config['n_softmax_out']
self.name = 'googlenet'
self.batch_size = batch_size
self.input_width = input_width
self.input_height = input_height
self.n_softmax_out = n_softmax_out
self.lrn_func = CrossChannelNormalization()
x = T.ftensor4('x')
y = T.lvector('y')
self.x = x # c01b
self.y = y
layers = []
params = []
weight_types = []
conv_7x7 = ConvPool_LRN(
input=x,
image_shape=(3, 224, 224,
batch_size), #c01b (3, 224, 224, batch_size)
filter_shape=(3, 7, 7, 64),
convstride=2,
padsize=3,
poolsize=3,
poolstride=2,
poolpad=1,
W=Weight((3, 7, 7, 64), mean=0.0, std=0.1),
b=Weight((64, ), mean=0.2, std=0),
lrn=True,
lib_conv='cudnn',
)
layers.append(conv_7x7)
params += conv_7x7.params
weight_types += conv_7x7.weight_type
# output shape = (112x112x64)
# output shape = (56x56x64)
conv_r3x3 = Conv(
input=conv_7x7.output,
image_shape=(64, 56, 56, batch_size),
filter_shape=(64, 1, 1, 64),
convstride=1,
padsize=0,
W=Weight((64, 1, 1, 64), mean=0.0, std=0.1),
b=Weight((64, ), mean=0.2, std=0),
lib_conv='cudnn',
)
layers.append(conv_r3x3)
params += conv_r3x3.params
weight_types += conv_r3x3.weight_type
# output shape = (56x56x64)
conv_3x3 = ConvPool_LRN(
input=conv_r3x3.output,
image_shape=(64, 56, 56, batch_size),
filter_shape=(64, 3, 3, 192),
convstride=1,
padsize=1,
poolsize=3,
poolstride=2,
poolpad=1,
W=Weight((64, 3, 3, 192), mean=0.0, std=0.03),
b=Weight((192, ), mean=0.2, std=0),
lrn=True,
lib_conv='cudnn',
)
layers.append(conv_3x3)
params += conv_3x3.params
weight_types += conv_3x3.weight_type
# output shape = (56x56x192)
# output shape = (28x28x192)
incep3a = Incept(conv_3x3.output,
input_shape=(192, 28, 28, batch_size))
layers += incep3a.layers
params += incep3a.params
weight_types += incep3a.weight_types
print 'incep3a output shape: (28x28x256)'
# output shape = (28x28x256)
incep3b = Incept(incep3a.output,
input_shape=(256, 28, 28, batch_size),
n1x1=128,
nr3x3=128,
n3x3=192,
nr5x5=32,
n5x5=96,
npj=64)
layers += incep3b.layers
params += incep3b.params
weight_types += incep3b.weight_types
print 'incep3b output shape: (28x28x480)'
# output shape = (28x28x480)
# lrn3 = self.lrn_func(incep3b.output)
# print 'LRN(added)'
pool3 = Pool(input=incep3b.output,
poolsize=3,
poolstride=2,
poolpad=1,
mode='max')
# output shape = (14x14x480)
incep4a = Incept(pool3.output,
input_shape=(480, 14, 14, batch_size),
n1x1=192,
nr3x3=96,
n3x3=208,
nr5x5=16,
n5x5=48,
npj=64)
layers += incep4a.layers
params += incep4a.params
weight_types += incep4a.weight_types
print 'incep4a output shape: (14x14x512)'
# output shape = (14x14x512)
incep4b = Incept(incep4a.output,
input_shape=(512, 14, 14, batch_size),
n1x1=160,
nr3x3=112,
n3x3=224,
nr5x5=24,
n5x5=64,
npj=64)
layers += incep4b.layers
params += incep4b.params
weight_types += incep4b.weight_types
print 'incep4b output shape: (14x14x512)'
# output shape = (14x14x512)
incep4c = Incept(incep4b.output,
input_shape=(512, 14, 14, batch_size),
n1x1=128,
nr3x3=128,
n3x3=256,
nr5x5=24,
n5x5=64,
npj=64)
layers += incep4c.layers
params += incep4c.params
weight_types += incep4c.weight_types
print 'incep4c output shape: (14x14x512)'
# output shape = (14x14x512)
incep4d = Incept(incep4c.output,
input_shape=(512, 14, 14, batch_size),
n1x1=112,
nr3x3=144,
n3x3=288,
nr5x5=32,
n5x5=64,
npj=64)
layers += incep4d.layers
params += incep4d.params
weight_types += incep4d.weight_types
print 'incep4d output shape: (14x14x528)'
# output shape = (14x14x528)
incep4e = Incept(incep4d.output,
input_shape=(528, 14, 14, batch_size),
n1x1=256,
nr3x3=160,
n3x3=320,
nr5x5=32,
n5x5=128,
npj=128)
layers += incep4e.layers
params += incep4e.params
weight_types += incep4e.weight_types
print 'incep4e output shape: (14x14x832)'
# output shape = (14x14x832)
lrn4 = self.lrn_func(
incep4e.output) # turn on only this for 16data, 53s/5120images
print 'LRN(added)'
pool4 = Pool(
input=lrn4, #incep4e.output,
poolsize=3,
poolstride=2,
poolpad=1,
mode='max')
# output shape = (7x7x832)
incep5a = Incept(pool4.output,
input_shape=(832, 7, 7, batch_size),
n1x1=256,
nr3x3=160,
n3x3=320,
nr5x5=32,
n5x5=128,
npj=128)
layers += incep5a.layers
params += incep5a.params
weight_types += incep5a.weight_types
print 'incep5a output shape: (7x7x832)'
# output shape = (7x7x832)
incep5b = Incept(incep5a.output,
input_shape=(832, 7, 7, batch_size),
n1x1=384,
nr3x3=192,
n3x3=384,
nr5x5=48,
n5x5=128,
npj=128)
layers += incep5b.layers
params += incep5b.params
weight_types += incep5b.weight_types
print 'incep5b output shape: (7x7x1024)'
# output shape = (7x7x1024)
# lrn5 = self.lrn_func(incep5b.output) # turn on only this for 16data, 51s/5120images
# print 'LRN(added)'
poolx = Pool(input=incep5b.output,
poolsize=7,
poolstride=1,
poolpad=0,
mode='average')
# output shape = (1x1x1024)
l_flatten = T.flatten(poolx.output.dimshuffle(3, 0, 1, 2), 2)
# output shape = (1024)
dropout = Dropout(input=l_flatten,
n_in=1024,
n_out=1024,
prob_drop=0.4)
# output shape = (1024)
softmax_layer = Softmax(input=dropout.output,
n_in=1024,
n_out=n_softmax_out)
# output shape = (n_softmax_out)
layers.append(softmax_layer)
params += softmax_layer.params
weight_types += softmax_layer.weight_type
# auxilary classifier
print 'auxilary classifier 1:'
aux1 = aux_tower(input=incep4a.output,
input_shape=(512, 14, 14, batch_size),
config=config)
layers += aux1.layers
params += aux1.params
weight_types += aux1.weight_types
print 'auxilary classifier 2:'
aux2 = aux_tower(input=incep4d.output,
input_shape=(528, 14, 14, batch_size),
config=config)
layers += aux2.layers
params += aux2.params
weight_types += aux2.weight_types
self.layers = layers
self.params = params
self.weight_types = weight_types
self.output = softmax_layer.p_y_given_x
self.cost = softmax_layer.negative_log_likelihood(y)+\
0.3*aux1.negative_log_likelihood(y)+0.3*aux2.negative_log_likelihood(y)
self.error = softmax_layer.errors(y)
self.error_top_5 = softmax_layer.errors_top_x(y)
# training related
self.base_lr = np.float32(config['learning_rate'])
self.shared_lr = theano.shared(self.base_lr)
self.mu = config['momentum'] # def: 0.9 # momentum
self.eta = config['weight_decay'] #0.0002 # weight decay
self.shared_x = theano.shared(np.zeros(
(3, config['input_width'], config['input_height'],
config['file_batch_size']),
dtype=theano.config.floatX),
borrow=True)
self.shared_y = theano.shared(np.zeros((config['file_batch_size'], ),
dtype=int),
borrow=True)
# shared variable for storing momentum before exchanging momentum(delta w)
self.vels = [
theano.shared(param_i.get_value() * 0.) for param_i in self.params
]
# shared variable for accepting momentum during exchanging momentum(delta w)
self.vels2 = [
theano.shared(param_i.get_value() * 0.) for param_i in self.params
]
self.train = None
self.get_vel = None
self.descent_vel = None
self.val = None
self.inference = None
def __init__(
self,
input,
image_shape,
filter_shape,
convstride,
padsize,
poolsize,
poolstride,
poolpad,
W,
b,
lrn=False,
lib_conv='cudnn',
):
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
self.W = W #Weight(self.filter_shape)
self.b = b #Weight(self.filter_shape[3])#, bias_init, std=0)
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(
img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
# ReLu
self.output = T.maximum(conv_out, 0)
# Pool
self.poolsize = poolsize
self.poolstride = poolstride
self.poolpad = poolpad
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride),
mode='max',
pad=(poolpad, poolpad))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
# LRN
if self.lrn:
self.lrn_func = CrossChannelNormalization()
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
print "conv ({}) layer with shape_in: {}".format(
lib_conv, str(image_shape))
def __init__(self,
input,
image_shape,
filter_shape,
convstride,
padsize,
group,
poolsize,
poolstride,
bias_init,
lrn=False):
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
self.conv_op = FilterActs(pad=self.padsize,
stride=self.convstride,
partial_sum=1)
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(input[:self.channel /
2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(input[self.channel /
2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
self.output = T.maximum(conv_out, 0)
conv_out = gpu_contiguous(conv_out)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
print "conv layer with shape_in: " + str(image_shape)
def __init__(self, input, image_shape, filter_shape, convstride, padsize,
group, poolsize, poolstride, bias_init, lrn=False,
lib_conv='cudnn',
verbose=False
):
'''
lib_conv can be cudnn (recommended)or cudaconvnet
'''
self.filter_size = filter_shape
self.convstride = convstride
self.padsize = padsize
self.poolsize = poolsize
self.poolstride = poolstride
self.channel = image_shape[0]
self.lrn = lrn
self.lib_conv = lib_conv
self.verbose = verbose
assert group in [1, 2]
self.filter_shape = np.asarray(filter_shape)
self.image_shape = np.asarray(image_shape)
if self.lrn:
self.lrn_func = CrossChannelNormalization()
if group == 1:
self.W = Weight(self.filter_shape)
self.b = Weight(self.filter_shape[3], bias_init, std=0)
else:
self.filter_shape[0] = self.filter_shape[0] / 2
self.filter_shape[3] = self.filter_shape[3] / 2
self.image_shape[0] = self.image_shape[0] / 2
self.image_shape[3] = self.image_shape[3] / 2
self.W0 = Weight(self.filter_shape)
self.W1 = Weight(self.filter_shape)
self.b0 = Weight(self.filter_shape[3], bias_init, std=0)
self.b1 = Weight(self.filter_shape[3], bias_init, std=0)
if lib_conv == 'cudaconvnet':
self.conv_op = FilterActs(pad=self.padsize, stride=self.convstride,
partial_sum=1)
from theano.sandbox.cuda.basic_ops import gpu_contiguous
# Conv
if group == 1:
contiguous_input = gpu_contiguous(input)
contiguous_filters = gpu_contiguous(self.W.val)
conv_out = self.conv_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle(0, 'x', 'x', 'x')
else:
contiguous_input0 = gpu_contiguous(
input[:self.channel / 2, :, :, :])
contiguous_filters0 = gpu_contiguous(self.W0.val)
conv_out0 = self.conv_op(
contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle(0, 'x', 'x', 'x')
contiguous_input1 = gpu_contiguous(
input[self.channel / 2:, :, :, :])
contiguous_filters1 = gpu_contiguous(self.W1.val)
conv_out1 = self.conv_op(
contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle(0, 'x', 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=0)
# ReLu
conv_out = gpu_contiguous(conv_out)
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.pool_op = MaxPool(ds=poolsize, stride=poolstride)
self.output = self.pool_op(self.output)
elif lib_conv == 'corrmm':
from theano.sandbox.cuda.basic_ops import gpu_contiguous
from theano.sandbox.cuda.blas import GpuCorrMM
border_mode = 'half' if padsize == (filter_shape[1]-1)/2 else (padsize, padsize)
self.corr_mm_op = GpuCorrMM(subsample=(convstride,convstride),
border_mode=border_mode)
flip_filters=True
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
if group==1:
filters = self.W.val.dimshuffle(3, 0, 1, 2)
if flip_filters:
filters = filters[:, :, ::-1, ::-1] # flip top-down, left-right
contiguous_filters = gpu_contiguous(filters)
contiguous_input = gpu_contiguous(input_shuffled)
conv_out = self.corr_mm_op(contiguous_input, contiguous_filters)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
if flip_filters:
W0_shuffled = W0_shuffled[:, :, ::-1, ::-1]
contiguous_filters0 = gpu_contiguous(W0_shuffled)
contiguous_input0 = gpu_contiguous(input_shuffled[:, :self.channel / 2,:, :])
conv_out0 = self.corr_mm_op(contiguous_input0, contiguous_filters0)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
if flip_filters:
W1_shuffled = W1_shuffled[:, :, ::-1, ::-1]
contiguous_filters1 = gpu_contiguous(W1_shuffled)
contiguous_input1 = gpu_contiguous(input_shuffled[:, self.channel / 2:,:, :])
conv_out1 = self.corr_mm_op(contiguous_input1, contiguous_filters1)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
from theano.tensor.signal import downsample
self.output = downsample.max_pool_2d(self.output,
ds=(poolsize,poolsize),
st=(poolstride,poolstride),
ignore_border=False,
padding=(0,0),
mode='max',
)
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
elif lib_conv == 'cudnn':
input_shuffled = input.dimshuffle(3, 0, 1, 2) # c01b to bc01
# in01out to outin01
# print image_shape_shuffled
# print filter_shape_shuffled
if group == 1:
W_shuffled = self.W.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out = dnn.dnn_conv(img=input_shuffled,
kerns=W_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out = conv_out + self.b.val.dimshuffle('x', 0, 'x', 'x')
else:
W0_shuffled = \
self.W0.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out0 = \
dnn.dnn_conv(img=input_shuffled[:, :self.channel / 2,
:, :],
kerns=W0_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out0 = conv_out0 + \
self.b0.val.dimshuffle('x', 0, 'x', 'x')
W1_shuffled = \
self.W1.val.dimshuffle(3, 0, 1, 2) # c01b to bc01
conv_out1 = \
dnn.dnn_conv(img=input_shuffled[:, self.channel / 2:,
:, :],
kerns=W1_shuffled,
subsample=(convstride, convstride),
border_mode=padsize,
)
conv_out1 = conv_out1 + \
self.b1.val.dimshuffle('x', 0, 'x', 'x')
conv_out = T.concatenate([conv_out0, conv_out1], axis=1)
# ReLu
self.output = T.maximum(conv_out, 0)
# Pooling
if self.poolsize != 1:
self.output = dnn.dnn_pool(self.output,
ws=(poolsize, poolsize),
stride=(poolstride, poolstride))
self.output = self.output.dimshuffle(1, 2, 3, 0) # bc01 to c01b
else:
NotImplementedError("lib_conv can only be cudaconvnet or cudnn")
# LRN
if self.lrn:
# lrn_input = gpu_contiguous(self.output)
self.output = self.lrn_func(self.output)
if group == 1:
self.params = [self.W.val, self.b.val]
self.weight_type = ['W', 'b']
else:
self.params = [self.W0.val, self.b0.val, self.W1.val, self.b1.val]
self.weight_type = ['W', 'b', 'W', 'b']
if self.verbose:
print "conv ({}) layer with shape_in: {}".format(lib_conv,
str(image_shape))
