def __init__(self, channels):
super(ResidualBlock, self).__init__()
self.conv1 = ConvLayer(channels, channels, kernel_size=3, stride=1)
self.bn1 = torch.nn.BatchNorm2d(channels, affine=True)
self.conv2 = ConvLayer(channels, channels, kernel_size=3, stride=1)
self.bn2 = torch.nn.BatchNorm2d(channels, affine=True)
self.relu = torch.nn.ReLU()
def __init__(self, channels, kernel):
super(ResUnit, self).__init__()
self.bn1 = nn.BatchNorm2d(channels, affine=True)
self.conv1 = ConvLayer(channels, channels, kernel, stride=1)
self.bn2 = nn.BatchNorm2d(channels, affine=True)
self.conv2 = ConvLayer(channels, channels, kernel, stride=1)
self.relu = nn.ReLU(inplace=True)
def __init__(self, dense_rate, inchannels):
super(Hourglass, self).__init__()
# self.modules1 = []
# self.modules2 = []
# self.modules3 = []
# self.modules4 = []
self.channels = inchannels
assert dense_rate % 2 == 0, 'The dense_rate must be even'
half_rate = dense_rate // 2
self.rate = half_rate
# for i in range(half_rate):
self.block1 = nn.Sequential(
nn.MaxPool2d(kernel_size=2, stride=2), ResUnit(self.channels, 3),
Hrgls_ResUnit(self.channels, (self.channels * 2)))
self.channels = self.channels * 2
# self.modules1.append(block)
# for i in range(half_rate):
self.block2 = nn.Sequential(
nn.MaxPool2d(kernel_size=2, stride=2), ResUnit(self.channels, 3),
Hrgls_ResUnit(self.channels, (self.channels * 2)))
self.channels = self.channels * 2
# self.modules2.append(block)
# for i in range(half_rate):
self.block3 = nn.Sequential(
ResUnit(self.channels, 3),
Hrgls_ResUnit(self.channels, (self.channels // 2)),
nn.Upsample(scale_factor=2, mode='nearest'),
ConvLayer((self.channels // 2), (self.channels // 2), 3, 1),
nn.BatchNorm2d((self.channels // 2)))
self.channels = self.channels // 2
# self.modules3.append(block)
# for i in range(half_rate):
self.block4 = nn.Sequential(
ResUnit(self.channels, 3),
Hrgls_ResUnit(self.channels, (self.channels // 2)),
nn.Upsample(scale_factor=2, mode='nearest'),
ConvLayer((self.channels // 2), (self.channels // 2), 3, 1),
nn.BatchNorm2d((self.channels // 2)))
self.channels = self.channels // 2
# self.modules4.append(block)
self.final = nn.Sequential(ResUnit(self.channels, 1),
ResUnit(self.channels, 1))
def __init__(self, inchannels, outchannels):
super(Hrgls_ResUnit, self).__init__()
self.bn1 = nn.BatchNorm2d(inchannels, affine=True)
self.conv1 = ConvLayer(inchannels, inchannels, 1, 1)
self.bn2 = nn.BatchNorm2d(inchannels, affine=True)
self.conv2 = ConvLayer(inchannels, outchannels, 3, 1)
self.bn3 = nn.BatchNorm2d(outchannels, affine=True)
self.conv3 = ConvLayer(outchannels, outchannels, 1, 1)
self.bn4 = nn.BatchNorm2d(inchannels, affine=True)
self.conv4 = ConvLayer(inchannels, outchannels, 1, 1)
self.relu = nn.ReLU(inplace=True)
def __init__(self):
super(AlignNet2, self).__init__()
#self.vgg = Vgg16().eval()
self.kron = KronEmbed()
self.res1 = ResidualBlock(512)
self.res2 = ResidualBlock(512)
self.res3 = ResidualBlock(512)
self.up1 = UpLayer(512, 256, kernel_size=3, stride=1, upsample=2)
self.bn1 = nn.BatchNorm2d(256, affine=True)
self.up2 = UpLayer(256, 128, kernel_size=3, stride=1, upsample=2)
self.bn2 = nn.BatchNorm2d(128, affine=True)
self.up3 = UpLayer(128, 64, kernel_size=3, stride=1, upsample=2)
self.bn3 = nn.BatchNorm2d(64, affine=True)
self.up4 = UpLayer(64, 32, kernel_size=3, stride=1, upsample=2)
self.bn4 = nn.BatchNorm2d(32, affine=True)
self.relu = nn.ReLU(inplace=True)
self.final = ConvLayer(32, 3, kernel_size=9, stride=1)
def create_layers(self):
next_input_shape = self.input_shape
layers = []
for layer in self.layers_description:
if type(layer) == ConvLayer_w:
layers.append(
ConvLayer(input_shape=next_input_shape,
output_images=layer.output_images,
kernel_size=layer.kernel_size,
activation_fn=layer.activation_fn))
next_input_shape = (layer.output_images, next_input_shape[1] -
layer.kernel_size + 1,
next_input_shape[2] - layer.kernel_size +
1)
elif type(layer) == PoolLayer_w:
layers.append(PoolLayer(shape=layer.shape))
next_input_shape = (next_input_shape[0],
int(next_input_shape[1] // layer.shape[0]),
int(next_input_shape[2] // layer.shape[1]))
elif type(layer) == FullyConectedLayer_w:
layers.append(
FullyConectedLayer(in_size=int(np.prod(next_input_shape)),
out_size=layer.size,
activation_fn=layer.activation_fn))
next_input_shape = (layer.size, )
return layers
def __init__(self, inchannels, outchannels):
super(Up_Unit, self).__init__()
self.up = nn.Sequential(nn.Upsample(scale_factor=2, mode='nearest'),
ConvLayer(inchannels, outchannels, 3, 1),
nn.BatchNorm2d(outchannels, affine=True),
nn.ReLU(inplace=True))
def __init__(self, x, y, hidden_dims, input_dim = 23, dropout = 0, reg = 0.0,
weight_scale = 1e-2, dtype = np.float32):
"""
Initialize a new fullyconnectednet
Inputs:
- hidden_dims: A list of integers giving the size of each hidden layer
- input_dim: An integer giving the size of the input
- dropout: scalar between 0 and 1 giving dropout strength, if dropout==0, then
do not use dropout at all
- reg: scalar giving l2 regularization strength
- dtype: A numpy datatype object: all variables will use this type.
"""
self.reg = reg
self.layers = []
self.num_layers = len(hidden_dims)
self.y = y
for i in xrange(self.num_layers):
if i == 0:
layer = ConvLayer(
None,
None,
input = x,
weight_scale = weight_scale,
activation = tf.nn.relu,
n_visible = input_dim * 2,
n_hidden = hidden_dims[i],
dropout = dropout
)
else:
layer = ConvLayer(
None,
None,
input = self.layers[-1].output,
weight_scale = weight_scale,
activation = tf.nn.relu,
n_visible = hidden_dims[i - 1] * 2,
n_hidden = hidden_dims[i],
dropout = dropout
)
self.layers.append(layer)
def __init__(self,inchannel,outchannel,kernel_size=3,stride=1):
super(DownLayer,self).__init__()
#self.conv1=ConvLayer(inchannel,inchannel,kernel_size,stride=1 )
self.conv1 = ResidualBlock(inchannel)
self.bn1 = torch.nn.BatchNorm2d(inchannel, affine=True )
self.conv2 = ConvLayer(inchannel, outchannel, kernel_size, stride =2 )
self.bn2 = torch.nn.BatchNorm2d(outchannel, affine=True )
#self.conv3=ConvLayer(outchannel,outchannel,kernel_size,stride =1)
self.conv3 = ResidualBlock(outchannel)
self.bn3 = torch.nn.BatchNorm2d(outchannel , affine=True )
self.relu = torch.nn.ReLU(inplace=True)
def __init__(self):
super(AlignNet, self).__init__()
self.down = Res_Module()
self.match1 = Matching_Module(inchannels=256)
self.mid1 = nn.Sequential(ResUnit(channels=256, kernel=3),
ResUnit(channels=256, kernel=3))
self.mid2 = nn.Sequential(ResUnit(channels=256, kernel=3),
ResUnit(channels=256, kernel=3))
self.match2 = Matching_Module(inchannels=256)
self.up1 = Up_Unit(inchannels=256, outchannels=128)
self.up2 = Up_Unit(inchannels=128, outchannels=64)
self.up3 = Up_Unit(inchannels=64, outchannels=16)
self.up4 = Up_Unit(inchannels=16, outchannels=3)
self.final = nn.Sequential(
ConvLayer(in_channels=3, out_channels=3, kernel_size=7),
ConvLayer(in_channels=3, out_channels=3, kernel_size=3),
ConvLayer(in_channels=3, out_channels=3, kernel_size=1))
def init_params(options):
params = OrderedDict()
if not use_conv:
params = get_layer('ff')[0](options,
params,
prefix='layer_1',
nin=INPUT_SIZE,
nout=args.dims[0],
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_2',
nin=args.dims[0],
nout=args.dims[0],
ortho=False)
if use_conv and args.dataset == "lsun":
bn = True
params = ConvLayer(3, 64, 5, 2, params=params, prefix='conv_1', bn=bn)
params = ConvLayer(64,
128,
5,
2,
params=params,
prefix='conv_2',
bn=bn)
params = ConvLayer(128,
256,
5,
2,
params=params,
prefix='conv_3',
bn=bn)
params = get_layer('ff')[0](options,
params,
prefix='layer_1',
nin=8 * 8 * 256,
nout=1024,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_2',
nin=1024,
nout=1024,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_3',
nin=1024,
nout=1024,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_4',
nin=1024,
nout=1024,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_5',
nin=1024,
nout=8 * 8 * 256,
ortho=False)
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_mu',
bn=bn)
params = ConvLayer(128,
64,
5,
-2,
params=params,
prefix='conv_5_mu',
bn=bn)
params = ConvLayer(64, 3, 5, -2, params=params, prefix='conv_6_mu')
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_s',
bn=bn)
params = ConvLayer(128,
64,
5,
-2,
params=params,
prefix='conv_5_s',
bn=bn)
params = ConvLayer(64, 3, 5, -2, params=params, prefix='conv_6_s')
elif use_conv and args.dataset == "MNIST":
bn = True
params = ConvLayer(1, 128, 5, 2, params=params, prefix='conv_1', bn=bn)
params = ConvLayer(128,
256,
5,
2,
params=params,
prefix='conv_2',
bn=bn)
params = get_layer('ff')[0](options,
params,
prefix='layer_1',
nin=7 * 7 * 256,
nout=1024,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_2',
nin=1024,
nout=256,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_3',
nin=256,
nout=7 * 7 * 256,
ortho=False)
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_mu',
bn=bn)
params = ConvLayer(128, 1, 5, -2, params=params, prefix='conv_5_mu')
else:
#TODO: Ideally, only in the output layer, flag=True should be set.
if len(args.dims) == 1:
params = get_layer('ff')[0](options,
params,
prefix='mu_0',
nin=args.dims[0],
nout=INPUT_SIZE,
ortho=False,
flag=True)
if args.noise == 'gaussian':
params = get_layer('ff')[0](options,
params,
prefix='sigma_0',
nin=args.dims[0],
nout=INPUT_SIZE,
ortho=False)
for i in range(len(args.dims) - 1):
params = get_layer('ff')[0](options,
params,
prefix='mu_' + str(i),
nin=args.dims[i],
nout=args.dims[i + 1],
ortho=False)
if args.noise == 'gaussian':
params = get_layer('ff')[0](options,
params,
prefix='sigma_' + str(i),
nin=args.dims[i],
nout=args.dims[i + 1],
ortho=False,
flag=True)
if len(args.dims) > 1:
params = get_layer('ff')[0](options,
params,
prefix='mu_' + str(i + 1),
nin=args.dims[i + 1],
nout=INPUT_SIZE,
ortho=False,
flag=True)
if args.noise == 'gaussian':
params = get_layer('ff')[0](options,
params,
prefix='sigma_' + str(i + 1),
nin=args.dims[i + 1],
nout=INPUT_SIZE,
ortho=False)
return params
def init_params(options):
params = OrderedDict()
if use_conv:
bn = True
params = ConvLayer(3, 64, 5, 2, params=params, prefix='conv_1', bn=bn)
params = ConvLayer(64,
128,
5,
2,
params=params,
prefix='conv_2',
bn=bn)
params = ConvLayer(128,
256,
5,
2,
params=params,
prefix='conv_3',
bn=bn)
'''
params = get_layer('ff')[0](options, params, prefix='layer_1',nin=4*4*256, nout=2048,ortho=False)
params = get_layer('ff')[0](options, params, prefix='layer_2',nin=2048, nout=2048,ortho=False)
params = get_layer('ff')[0](options, params, prefix='layer_3',nin=2048, nout=2048,ortho=False)
params = get_layer('ff')[0](options, params, prefix='layer_4',nin=2048, nout=2048,ortho=False)
params = get_layer('ff')[0](options, params, prefix='layer_5',nin=2048, nout=4*4*256,ortho=False)
'''
'''
params = param_init_convlayer(options, params, prefix='conv_1', nin=3, nout=64, kernel_len=5, batch_norm=bn)
params[_p('conv_1', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 64)).astype('float32')
params[_p('conv_1', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 64)).astype('float32')
params = param_init_convlayer(options, params, prefix='conv_2', nin=64, nout=128, kernel_len=5, batch_norm=bn)
params[_p('conv_2', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 128)).astype('float32')
params[_p('conv_2', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 128)).astype('float32')
params = param_init_convlayer(options, params, prefix='conv_3', nin=128, nout=256, kernel_len=5, batch_norm=bn)
params[_p('conv_3', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 256)).astype('float32')
params[_p('conv_3', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 256)).astype('float32')
'''
params = get_layer('ff')[0](options,
params,
prefix='layer_1',
prefix_bnorm='layer_1_step_0',
nin=4 * 4 * 256,
nout=2048,
ortho=False,
batch_norm=True)
params = get_layer('ff')[0](options,
params,
prefix='layer_2',
prefix_bnorm='layer_2_step_0',
nin=2048,
nout=2048,
ortho=False,
batch_norm=True)
params = get_layer('ff')[0](options,
params,
prefix='layer_3',
prefix_bnorm='layer_3_step_0',
nin=2048,
nout=2048,
ortho=False,
batch_norm=True)
params = get_layer('ff')[0](options,
params,
prefix='layer_4',
prefix_bnorm='layer_4_step_0',
nin=2048,
nout=2048,
ortho=False,
batch_norm=True)
params = get_layer('ff')[0](options,
params,
prefix='layer_5',
prefix_bnorm='layer_5_step_0',
nin=2048,
nout=4 * 4 * 256,
ortho=False,
batch_norm=True)
params[_p('layer1_bnorm',
'newmu')] = np.zeros(shape=(args.num_steps * args.meta_steps,
2048)).astype('float32')
params[_p('layer1_bnorm', 'newsigma')] = np.ones(
shape=(args.num_steps * args.meta_steps, 2048)).astype('float32')
params[_p('layer2_bnorm',
'newmu')] = np.zeros(shape=(args.num_steps * args.meta_steps,
2048)).astype('float32')
params[_p('layer2_bnorm', 'newsigma')] = np.ones(
shape=(args.num_steps * args.meta_steps, 2048)).astype('float32')
params[_p('layer3_bnorm',
'newmu')] = np.zeros(shape=(args.num_steps * args.meta_steps,
2048)).astype('float32')
params[_p('layer3_bnorm', 'newsigma')] = np.ones(
shape=(args.num_steps * args.meta_steps, 2048)).astype('float32')
params[_p('layer4_bnorm',
'newmu')] = np.zeros(shape=(args.num_steps * args.meta_steps,
2048)).astype('float32')
params[_p('layer4_bnorm', 'newsigma')] = np.ones(
shape=(args.num_steps * args.meta_steps, 2048)).astype('float32')
params[_p('layer5_bnorm',
'newmu')] = np.zeros(shape=(args.num_steps * args.meta_steps,
4 * 4 * 256)).astype('float32')
params[_p('layer5_bnorm',
'newsigma')] = np.ones(shape=(args.num_steps *
args.meta_steps,
4 * 4 * 256)).astype('float32')
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_mu',
bn=bn)
params = ConvLayer(128,
64,
5,
-2,
params=params,
prefix='conv_5_mu',
bn=bn)
params = ConvLayer(64, 3, 5, -2, params=params, prefix='conv_6_mu')
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_s',
bn=bn)
params = ConvLayer(128,
64,
5,
-2,
params=params,
prefix='conv_5_s',
bn=bn)
params = ConvLayer(64, 3, 5, -2, params=params, prefix='conv_6_s')
'''
params = param_init_convlayer(options, params, prefix='conv_4_mu', nin=256, nout=128, kernel_len=5, batch_norm=bn)
params[_p('conv_4_mu', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 128)).astype('float32')
params[_p('conv_4_mu', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 128)).astype('float32')
params = param_init_convlayer(options, params, prefix='conv_5_mu', nin=128, nout=64, kernel_len=5, batch_norm=bn)
params[_p('conv_5_mu', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 64)).astype('float32')
params[_p('conv_5_mu', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 64)).astype('float32')
params = param_init_convlayer(options, params, prefix='conv_6_mu', nin=64, nout=3, kernel_len=5, batch_norm =False)
params = param_init_convlayer(options, params, prefix='conv_4_s', nin=256, nout=128, kernel_len=5, batch_norm=bn)
params[_p('conv_4_s', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 128)).astype('float32')
params[_p('conv_4_s', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 128)).astype('float32')
params = param_init_convlayer(options, params, prefix='conv_5_s', nin=128, nout=64, kernel_len=5, batch_norm=bn)
params[_p('conv_5_s', 'newmu')] = np.zeros(shape=(args.num_steps *args.meta_steps, 64)).astype('float32')
params[_p('conv_5_s', 'newsigma')] = np.ones(shape=(args.num_steps *args.meta_steps, 64)).astype('float32')
params = param_init_convlayer(options, params, prefix='conv_6_s', nin=64, nout=3, kernel_len=5, batch_norm = False)
'''
return params
def init_params(options):
params = OrderedDict()
if not use_conv:
params = get_layer('ff')[0](options,
params,
prefix='layer_1',
nin=INPUT_SIZE,
nout=args.dims[0],
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_2',
nin=args.dims[0],
nout=args.dims[0],
ortho=False)
dilated_conv = False
if dilated_conv:
bn = True
filter_size = 5
c1 = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(1, 1),
border_mode='same')
c1.build((100, 3, 32, 32))
qw = c1.get_weights()
params[_p('c1', 'w')] = qw[0]
params[_p('c1', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c1_bn')
c2 = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(2, 2),
border_mode='same')
c2.build((100, 3, 32, 128))
qw = c2.get_weights()
params[_p('c2', 'w')] = qw[0]
params[_p('c2', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c2_bn')
c3 = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(4, 4),
border_mode='same')
c3.build((100, 3, 32, 128))
qw = c3.get_weights()
params[_p('c3', 'w')] = qw[0]
params[_p('c3', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c3_bn')
c4_mu = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(4, 4),
border_mode='same')
c4_mu.build((100, 3, 32, 128))
qw = c4_mu.get_weights()
params[_p('c4_mu', 'w')] = qw[0]
params[_p('c4_mu', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c4_mu_bn')
c5_mu = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(2, 2),
border_mode='same')
c5_mu.build((100, 3, 32, 128))
qw = c5_mu.get_weights()
params[_p('c5_mu', 'w')] = qw[0]
params[_p('c5_mu', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c5_mu_bn')
c6_mu = AtrousConvolution2D(32,
filter_size,
filter_size,
atrous_rate=(1, 1),
border_mode='same')
c6_mu.build((100, 3, 32, 128))
qw = c6_mu.get_weights()
params[_p('c6_mu', 'w')] = qw[0]
params[_p('c6_mu', 'b')] = qw[1]
c4_s = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(4, 4),
border_mode='same')
c4_s.build((100, 3, 32, 128))
qw = c4_s.get_weights()
params[_p('c4_s', 'w')] = qw[0]
params[_p('c4_s', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c4_s_bn')
c5_s = AtrousConvolution2D(128,
filter_size,
filter_size,
atrous_rate=(2, 2),
border_mode='same')
c5_s.build((100, 3, 32, 128))
qw = c5_s.get_weights()
params[_p('c5_s', 'w')] = qw[0]
params[_p('c5_s', 'b')] = qw[1]
if bn:
params = bnorm_layer_init((100, 3, 32, 128), params, 'c5_s_bn')
c6_s = AtrousConvolution2D(32,
filter_size,
filter_size,
atrous_rate=(1, 1),
border_mode='same')
c6_s.build((100, 3, 32, 128))
qw = c6_s.get_weights()
params[_p('c6_s', 'w')] = qw[0]
params[_p('c6_s', 'b')] = qw[1]
if use_conv:
bn = True
params = ConvLayer(3, 64, 5, 2, params=params, prefix='conv_1', bn=bn)
params = ConvLayer(64,
128,
5,
2,
params=params,
prefix='conv_2',
bn=bn)
params = ConvLayer(128,
256,
5,
2,
params=params,
prefix='conv_3',
bn=bn)
params = get_layer('ff')[0](options,
params,
prefix='layer_1',
nin=4 * 4 * 256,
nout=2048,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_2',
nin=2048,
nout=2048,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_3',
nin=2048,
nout=2048,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_4',
nin=2048,
nout=2048,
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='layer_5',
nin=2048,
nout=4 * 4 * 256,
ortho=False)
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_mu',
bn=bn)
params = ConvLayer(128,
64,
5,
-2,
params=params,
prefix='conv_5_mu',
bn=bn)
params = ConvLayer(64, 3, 5, -2, params=params, prefix='conv_6_mu')
params = ConvLayer(256,
128,
5,
-2,
params=params,
prefix='conv_4_s',
bn=bn)
params = ConvLayer(128,
64,
5,
-2,
params=params,
prefix='conv_5_s',
bn=bn)
params = ConvLayer(64, 3, 5, -2, params=params, prefix='conv_6_s')
else:
#TODO: Ideally, only in the output layer, flag=True should be set.
if len(args.dims) == 1:
params = get_layer('ff')[0](options,
params,
prefix='mu_0',
nin=args.dims[0],
nout=INPUT_SIZE,
ortho=False,
flag=True)
if args.noise == 'gaussian':
params = get_layer('ff')[0](options,
params,
prefix='sigma_0',
nin=args.dims[0],
nout=INPUT_SIZE,
ortho=False)
for i in range(len(args.dims) - 1):
params = get_layer('ff')[0](options,
params,
prefix='mu_' + str(i),
nin=args.dims[i],
nout=args.dims[i + 1],
ortho=False)
if args.noise == 'gaussian':
params = get_layer('ff')[0](options,
params,
prefix='sigma_' + str(i),
nin=args.dims[i],
nout=args.dims[i + 1],
ortho=False,
flag=True)
if len(args.dims) > 1:
params = get_layer('ff')[0](options,
params,
prefix='mu_' + str(i + 1),
nin=args.dims[i + 1],
nout=INPUT_SIZE,
ortho=False,
flag=True)
if args.noise == 'gaussian':
params = get_layer('ff')[0](options,
params,
prefix='sigma_' + str(i + 1),
nin=args.dims[i + 1],
nout=INPUT_SIZE,
ortho=False)
return params
epoch = 50 # learning epoches batch_size = 32 # Use mini batch n_sample = 500 # Using for Error, Accuracy samples alpha = 1 # how many times do you want to learn in 1 epoch. if 1, whole data learns. input_train, input_test, correct_train, correct_test = cifar10_call(N) n_train = input_train.shape[0] n_test = input_test.shape[0] img_h = 32 img_w = 32 img_ch = 3 # -- 각 층의 초기화 -- cl1 = ConvLayer(img_ch, img_h, img_w, 30, 3, 3, stride=1, pad=1) # 앞3개:인풋 중간3개:필터 cl2 = ConvLayer(cl1.y_ch, cl1.y_h, cl1.y_w, 30, 3, 3, stride=1, pad=1) pl1 = PoolingLayer(cl2.y_ch, cl2.y_h, cl2.y_w, pool=2, pad=0) # pool:풀링크기(2*2), pad:패딩 너비 c_dr1 = fn.dropout(0.25) cl3 = ConvLayer(pl1.y_ch, pl1.y_h, pl1.y_w, 60, 3, 3, stride=1, pad=1) pl2 = PoolingLayer(cl3.y_ch, cl3.y_h, cl3.y_w, pool=2, pad=0) c_dr2 = fn.dropout(0.25) cl4 = ConvLayer(pl2.y_ch, pl2.y_h, pl2.y_w, 120, 3, 3, stride=1, pad=1) pl3 = PoolingLayer(cl4.y_ch, cl4.y_h, cl4.y_w, pool=2, pad=0) n_fc_in = pl3.y_ch * pl3.y_h * pl3.y_w ml1 = MiddleLayer(n_fc_in, 500) dr1 = fn.dropout(0.5) ml2 = MiddleLayer(500, 500)
def test_something():
layer0 = ConvLayer((28, 28, 1), (9,9,2,2))
layer0.W = 1
print layer0.W
res /= y_valid.shape[0]
res *= 100
print("\n Epoch: {0}, validate accuracy: {1:.2f}, total loss: {2:.4f}".
format(epo, res, tloss))
isLeNet5 = False
if 'lenet5' in sys.argv[1:]:
isLeNet5 = True
Num = 55000
ln_layers = []
print("About to train model, lenet-5: {0}".format(isLeNet5))
if isLeNet5:
lenet_act = 'tanh'
# lenet_act = 'sigmoid'
ln_layers.append(ConvLayer(1, 6, KSize=5, activation=lenet_act))
ln_layers.append(MaxPoolLayer())
ln_layers.append(ConvLayer(6, 16, KSize=5, activation=lenet_act))
ln_layers.append(MaxPoolLayer())
ln_layers.append(FullConLayer(256, 120, activation=lenet_act))
ln_layers.append(FullConLayer(120, 84, activation=lenet_act, dropout=True))
ln_layers.append(SoftMaxLayer(84, 10))
else:
Num = 49920
lenet_act = 'relu'
ln_layers.append(ConvLayer(1, 32, KSize=5, activation=lenet_act))
ln_layers.append(MaxPoolLayer())
ln_layers.append(ConvLayer(32, 64, KSize=5, activation=lenet_act))
ln_layers.append(MaxPoolLayer())
ln_layers.append(
FullConLayer(1024, 1024, activation=lenet_act, dropout=True))
def __init__(self, rng, weight, use_last_layer_weight=False):
# weight: list type,0-31,the first 32 W and b
x = T.tensor4('x')
y = T.imatrix('y')
learning_rate = T.scalar('learning_rate')
self.layer1_input = x #224
self.layer1 = ConvLayer(rng,
input=self.layer1_input,
filter_shape=(64, 3, 3, 3),
layer_index=1,
W=weight[0],
b=weight[1])
self.layer2 = ConvLayer(rng,
input=self.layer1.output,
filter_shape=(64, 64, 3, 3),
layer_index=2,
W=weight[2],
b=weight[3])
self.pool1 = PoolLayer(input=self.layer2.output,
poolsize=(2, 2),
layer_index=1) #112
self.layer3 = ConvLayer(rng,
input=self.pool1.output,
filter_shape=(128, 64, 3, 3),
layer_index=3,
W=weight[4],
b=weight[5])
self.layer4 = ConvLayer(rng,
input=self.layer3.output,
filter_shape=(128, 128, 3, 3),
layer_index=4,
W=weight[6],
b=weight[7])
self.pool2 = PoolLayer(input=self.layer4.output,
poolsize=(2, 2),
layer_index=2) #56
self.layer5 = ConvLayer(rng,
input=self.pool2.output,
filter_shape=(256, 128, 3, 3),
layer_index=5,
W=weight[8],
b=weight[9])
self.layer6 = ConvLayer(rng,
input=self.layer5.output,
filter_shape=(256, 256, 3, 3),
layer_index=6,
W=weight[10],
b=weight[11])
self.layer7 = ConvLayer(rng,
input=self.layer6.output,
filter_shape=(256, 256, 3, 3),
layer_index=7,
W=weight[12],
b=weight[13])
self.pool3 = PoolLayer(input=self.layer7.output,
poolsize=(2, 2),
layer_index=3) #28
self.layer8 = ConvLayer(rng,
input=self.pool3.output,
filter_shape=(512, 256, 3, 3),
layer_index=8,
W=weight[14],
b=weight[15])
self.layer9 = ConvLayer(rng,
input=self.layer8.output,
filter_shape=(512, 512, 3, 3),
layer_index=9,
W=weight[16],
b=weight[17])
self.layer10 = ConvLayer(rng,
input=self.layer9.output,
filter_shape=(512, 512, 3, 3),
layer_index=10,
W=weight[18],
b=weight[19])
self.pool4 = PoolLayer(input=self.layer10.output,
poolsize=(2, 2),
layer_index=4) #14
self.layer11 = ConvLayer(rng,
input=self.pool4.output,
filter_shape=(512, 512, 3, 3),
layer_index=11,
W=weight[20],
b=weight[21])
self.layer12 = ConvLayer(rng,
input=self.layer11.output,
filter_shape=(512, 512, 3, 3),
layer_index=12,
W=weight[22],
b=weight[23])
self.layer13 = ConvLayer(rng,
input=self.layer12.output,
filter_shape=(512, 512, 3, 3),
layer_index=13,
W=weight[24],
b=weight[25])
self.pool5 = PoolLayer(input=self.layer13.output,
poolsize=(2, 2),
layer_index=5) #7
if use_last_layer_weight: #use weight of layer14/15/16
self.layer14 = HiddenLayer(rng,
input=self.pool5.output.flatten(ndim=2),
n_in=25088,
n_out=1024,
layer_index=14,
W=weight[26],
b=weight[27])
self.layer15 = HiddenLayer(rng,
input=self.layer14.output,
n_in=1024,
n_out=1024,
layer_index=15,
W=weight[28],
b=weight[29])
self.layer16 = SoftmaxLayer(rng,
input=self.layer15.output,
n_in=1024,
n_out=7,
layer_index=16,
W=weight[30],
b=weight[31])
else:
self.layer14 = HiddenLayer(rng,
input=self.pool5.output.flatten(ndim=2),
n_in=25088,
n_out=1024,
layer_index=14)
self.layer15 = HiddenLayer(rng,
input=self.layer14.output,
n_in=1024,
n_out=1024,
layer_index=15)
self.layer16 = SoftmaxLayer(rng,
input=self.layer15.output,
n_in=1024,
n_out=7,
layer_index=16)
# the objective loss
self.loss = self.layer16.squared_error(y)
self.params = self.layer16.params+self.layer15.params+self.layer14.params+self.layer13.params+self.layer12.params\
+self.layer11.params+self.layer10.params+self.layer9.params +self.layer8.params +self.layer7.params\
+self.layer6.params +self.layer5.params +self.layer4.params +self.layer3.params +self.layer2.params\
+self.layer1.params
# optimization methods
updates = lasagne.updates.nesterov_momentum(
self.loss, self.params, learning_rate=learning_rate, momentum=0.9)
self.train = theano.function(
inputs=[x, y, learning_rate],
outputs=[self.loss, self.layer16.errors(y)],
updates=updates,
allow_input_downcast=True)
self.test = theano.function(
inputs=[x, y],
outputs=[self.loss, self.layer16.errors(y)],
allow_input_downcast=True)
def __init__(self, rng, weight, use_last_layer_weight=False):
# weight: list type,0-31,the first 32 W and b
x = T.tensor4('x')
y = T.imatrix('y')
learning_rate = T.scalar('learning_rate')
self.layer1_input = x #224
self.layer1 = ConvLayer(rng,
input=self.layer1_input,
filter_shape=(96, 3, 7, 7),
layer_index=1,
stride=(2, 2),
W=weight[0],
b=weight[1]) #output:
self.LRN_1 = LRN(input=self.layer1.nopad_output, layer_index=1)
self.pool1 = PoolLayer(input=self.LRN_1.output,
poolsize=(3, 3),
layer_index=1) # 138
self.layer2 = ConvLayer(rng,
input=self.pool1.output,
filter_shape=(256, 96, 5, 5),
layer_index=2,
W=weight[2],
b=weight[3]) #output:
self.pool2 = PoolLayer(input=self.layer2.nopad_output,
poolsize=(2, 2),
layer_index=2) #67
self.layer3 = ConvLayer(rng,
input=self.pool2.output,
filter_shape=(512, 256, 3, 3),
layer_index=3,
W=weight[4],
b=weight[5]) #
self.layer4 = ConvLayer(rng,
input=self.layer3.output,
filter_shape=(512, 512, 3, 3),
layer_index=4,
W=weight[6],
b=weight[7]) #
self.layer5 = ConvLayer(rng,
input=self.layer4.output,
filter_shape=(512, 512, 3, 3),
layer_index=5,
W=weight[8],
b=weight[9]) #
self.pool3 = PoolLayer(input=self.layer5.output,
poolsize=(3, 3),
layer_index=3) #23
if use_last_layer_weight: #use weight of layer14/15/16
self.layer6 = HiddenLayer(rng,
input=self.pool3.output.flatten(ndim=2),
n_in=18432,
n_out=1024,
layer_index=6,
W=weight[10],
b=weight[11])
self.layer7 = HiddenLayer(rng,
input=self.layer6.output,
n_in=1024,
n_out=1024,
layer_index=7,
W=weight[12],
b=weight[13])
self.layer8 = SoftmaxLayer(rng,
input=self.layer7.output,
n_in=1024,
n_out=7,
layer_index=8,
W=weight[14],
b=weight[15])
else:
self.layer6 = HiddenLayer(rng,
input=self.pool3.output.flatten(ndim=2),
n_in=18432,
n_out=1024,
layer_index=6)
self.layer7 = HiddenLayer(rng,
input=self.layer6.output,
n_in=1024,
n_out=1024,
layer_index=7)
self.layer8 = SoftmaxLayer(rng,
input=self.layer7.output,
n_in=1024,
n_out=7,
layer_index=8)
# the objective loss
self.loss = self.layer8.squared_error(y)
self.params = self.layer8.params +self.layer7.params+self.layer6.params +self.layer5.params\
+self.layer4.params +self.layer3.params +self.layer2.params +self.layer1.params
# optimization methods
updates = lasagne.updates.nesterov_momentum(
self.loss, self.params, learning_rate=learning_rate, momentum=0.9)
#updates=lasagne.updates.rmsprop(self.loss, self.params, learning_rate, rho=0.9, epsilon=1e-06)
self.train = theano.function(
inputs=[x, y, learning_rate],
outputs=[self.loss, self.layer8.errors(y)],
updates=updates,
allow_input_downcast=True)
self.test = theano.function(inputs=[x, y],
outputs=[self.loss,
self.layer8.errors(y)],
allow_input_downcast=True)
fully_connected_settings = [500, 10]
# resnet
# layer_desc = [(64, 7), (256, 1), (64, 1), (64, 3), (256, 1), (64, 1), (64, 3), (256, 1),
# (64, 1), (64, 3), (256, 1), (512, 1), (128, 1), (128, 3), (512, 1), (128, 1), (128, 3),
# (512, 1), (128, 1), (128, 3), (512, 1), (128, 1), (128, 3)
# ]
fully_connected_settings = [500, 10]
PEs = PE_gen(layer_desc[0][0], PE_PER_LEAF, LEAF_COUNT, PES)
mems = mem_gen(first_layer_mems, LEAF_COUNT, PE_PER_LEAF, MEMs)
assign_dests(PEs, mems, LEAF_COUNT)
# generate layers
layers = []
layers.append(
ConvLayer(1, layer_desc[0][0], PEs, mems, PE_PER_LEAF, LEAF_COUNT,
layer_desc[0][1], 20))
packets = layers[0].generate_packets(INJECTION_RATE, 0, start_at_z)
for i in range(len(layer_desc) - 1):
mem_count = layers[i].output_count
filter_count = layer_desc[i + 1][0]
PEs = PE_gen(filter_count, PE_PER_LEAF, LEAF_COUNT, PES)
mems = mem_gen(mem_count, PE_PER_LEAF, LEAF_COUNT, MEMs)
assign_dests(PEs, mems, LEAF_COUNT)
layers.append(
ConvLayer(mem_count, filter_count, PEs, mems, PE_PER_LEAF, LEAF_COUNT,
layer_desc[i + 1][1], layers[i].out_img_size))
packets = layers[-1].generate_packets(INJECTION_RATE,
def __init__(self,
rng,
weight,
agg_func='soft',
use_last_layer_weight=False):
# weight: list type,0-15,the first 16 W and b
x = T.tensor4('x')
y = T.imatrix('y')
learning_rate = T.scalar('learning_rate')
self.layer1_input = x #832
self.layer1 = ConvLayer(rng,
input=self.layer1_input,
filter_shape=(96, 3, 7, 7),
layer_index=1,
stride=(2, 2),
W=weight[0],
b=weight[1]) #output:413
self.LRN_1 = LRN(input=self.layer1.nopad_output, layer_index=1)
self.pool1 = PoolLayer(input=self.LRN_1.output,
poolsize=(3, 3),
layer_index=1) # 138
self.layer2 = ConvLayer(rng,
input=self.pool1.output,
filter_shape=(256, 96, 5, 5),
layer_index=2,
W=weight[2],
b=weight[3]) #output: 134
self.pool2 = PoolLayer(input=self.layer2.nopad_output,
poolsize=(2, 2),
layer_index=2) #67
self.layer3 = ConvLayer(rng,
input=self.pool2.output,
filter_shape=(512, 256, 3, 3),
layer_index=3,
W=weight[4],
b=weight[5]) #67
self.layer4 = ConvLayer(rng,
input=self.layer3.output,
filter_shape=(512, 512, 3, 3),
layer_index=4,
W=weight[6],
b=weight[7]) #67
self.layer5 = ConvLayer(rng,
input=self.layer4.output,
filter_shape=(512, 512, 3, 3),
layer_index=5,
W=weight[8],
b=weight[9]) #67
self.pool3 = PoolLayer(input=self.layer5.output,
poolsize=(3, 3),
layer_index=3) #23
## change FC1/FC2/FC3 layers to Convlayers14/15/16
if use_last_layer_weight: #use weight of layer14/15/16
self.layer6 = ConvLayer(rng,
input=self.pool3.output,
filter_shape=(1024, 512, 6, 6),
layer_index=6,
W=weight[10],
b=weight[11]) #18
self.layer7 = ConvLayer(rng,
input=self.layer6.nopad_output,
filter_shape=(1024, 1024, 1, 1),
layer_index=7,
W=weight[12],
b=weight[13]) #18
self.layer8 = AggregationLayer(rng,
input=self.layer7.nopad_output,
filter_shape=(7, 1024, 1, 1),
layer_index=8,
W=weight[14],
b=weight[15],
agg_func=agg_func) #18
else:
self.layer6 = ConvLayer(rng,
input=self.pool3.output,
filter_shape=(1024, 512, 6, 6),
layer_index=6) #18
self.layer7 = ConvLayer(rng,
input=self.layer6.nopad_output,
filter_shape=(1024, 1024, 1, 1),
layer_index=7) #18
self.layer8 = AggregationLayer(rng,
input=self.layer7.nopad_output,
filter_shape=(7, 1024, 1, 1),
layer_index=8,
agg_func=agg_func) #18
# the objective loss
self.loss = self.layer8.squared_error(y)
self.params = self.layer8.params+self.layer7.params+self.layer6.params+self.layer5.params\
+self.layer4.params+self.layer3.params+self.layer2.params +self.layer1.params
# optimization methods
updates = lasagne.updates.nesterov_momentum(
self.loss, self.params, learning_rate=learning_rate, momentum=0.9)
self.train = theano.function(
inputs=[x, y, learning_rate],
outputs=[self.loss, self.layer8.errors(y)],
updates=updates,
allow_input_downcast=True)
self.test = theano.function(inputs=[x, y],
outputs=[self.loss,
self.layer8.errors(y)],
allow_input_downcast=True)
self.detection = theano.function(inputs=[x, y],
outputs=[
self.layer8.pred,
self.layer8.errors(y),
self.layer8.pro_instance,
self.layer8.pro_bag
],
allow_input_downcast=True)
def testGradient():
"""Test the backprop implementation by checking the gradients on a small network"""
# load the training data
images, labels = load_mnist()
images /= 255.0
grad_images = images[:,:,0:10] #use 10 image subset for gradient checking
grad_labels = labels[0,0:10] #respective labels for the images--going to have to encode these labels
# create a small network, 1 conv layer + 1 pooling layer + 1 fully connected softmax
# convolutional layer, taking in a 28x28 image, using 2 9x9 filters
# output should be 2 28-9+1x28-9+1 = 2 20x20 feature maps in a (20, 20, 2) form
layer0 = ConvLayer(grad_images[:,:,0].reshape((28,28,1)), (28, 28, 1), (9, 9, 2, 1))
print "initalized convolutional layer"
layer0.forwardprop(grad_images[:,:,0].reshape((28,28,1)))
print "finished forward pass of convolutional layer"
# pooling layer, taking in 2 20x20 feature maps
# output should be 2 10x10 feature maps (though may want to downsample 5x for gradient check)
layer1 = PoolingLayer(layer0.output, (20, 20, 2))
print "initialized pooling layer"
layer1.downsample(layer0.output, (20, 20, 2))
print "finished forward pass of pooling layer"
# fully-connected softmax layer, taking in 2 10x10 feature maps (if downsampled by 2)
# or taking in 2 4x4 feature maps (if downsampled by 5)
# either way, flattened into a long input vector
full_conn_input = layer1.output.flatten()
layer2 = FullyConnectedLayer(full_conn_input.reshape((full_conn_input.size, 1)), full_conn_input.size, 10)
print "initialized fully-conn layer"
layer2.softmax_output(full_conn_input.reshape((full_conn_input.size, 1)))
print "finished forward pass of fully-conn layer"
# perform backpropagation
target = np.zeros((10,1))
for i in range(0, 10):
if grad_labels[i] == 1:
target[i] = 1
layer2.backprop(0, 0, target)
print "finished layer 2 backprop"
layer1.upsample(layer2, 0)
print "finished layer 1 backprop"
layer0.backprop(layer1)
print "finished layer 0 backprop"
# # after initialization, finish training
# for i in range(1, grad_labels.size):
# # forward propagation
# layer0.forwardprop(grad_images[:,:,i].reshape((28,28,1)))
# layer1.downsample(layer0.output, (20,20,2))
# full_conn_input = layer1.output.flatten()
# layer2.softmax_output(full_conn_input.reshape((full_conn_input.size, 1)))
#
# # backpropagation
# target = np.zeros((10,1))
# for j in range(0,10):
# if grad_labels[i] == 1:
# target[i] = 1
# layer2.backprop(0, 0, target)
# layer1.upsample(layer2, 0)
# layer0.backprop(layer1)
# check the gradient
epsilon = 1.0e-4
layer0_check = layer0
layer1_check = layer1
layer2_check = layer2
layer0_w_vec = layer0.W.flatten()
layer0_bias_vec = layer0.bias.flatten()
layer0_gradw = layer0.gradient_w.flatten()
layer0_gradb = layer0.gradient_b.flatten()
layer2_w_vec = layer2.W.flatten()
layer2_bias_vec = layer2.bias.flatten()
layer2_gradw = layer2.gradient_w.flatten()
layer2_gradb = layer2.gradient_b.flatten()
w_vec = np.concatenate((layer0_w_vec, layer0_bias_vec, layer2_w_vec, layer2_bias_vec))
backprop_vec = np.concatenate((layer0_gradw, layer0_gradb, layer2_gradw, layer2_gradb))
print layer0_gradw
gradient_check = np.zeros(w_vec.size)
for i in range(0, w_vec.size):
pos = w_vec
pos[i] += epsilon
neg = w_vec
neg[i] -= epsilon
# feed-forward to get J(w+e), J(w-e), subtract and calculate gradient
# J(w+e)
layer0_check.W = pos[0:layer0_w_vec.size].reshape(layer0.filter_shape)
layer0_check.bias = pos[layer0_w_vec.size : layer0_w_vec.size+layer0_bias_vec.size].reshape(layer0.bias_shape)
layer2_check.W = pos[layer0_w_vec.size+layer0_bias_vec.size : layer0.W.size+layer0.bias.size+layer2_w_vec.size].reshape(layer2.W.shape)
layer2_check.bias = pos[layer0.W.size+layer0.bias.size+layer2_w_vec.size:].reshape(layer2.bias.shape)
layer0_check.forwardprop(grad_images[:,:,0].reshape((28,28,1)))
layer1_check.downsample(layer0_check.output, (20,20,2))
full_conn_input = layer1.output.flatten()
layer2_check.softmax_output(full_conn_input.reshape((full_conn_input.size, 1)))
pos_out = J(layer2_check.output, grad_labels[0])
# J(w-e)
layer0_check.W = neg[0:layer0_w_vec.size].reshape(layer0.filter_shape)
layer0_check.bias = neg[layer0_w_vec.size : layer0_w_vec.size+layer0_bias_vec.size].reshape(layer0.bias_shape)
layer2_check.W = neg[layer0_w_vec.size+layer0_bias_vec.size : layer0.W.size+layer0.bias.size+layer2_w_vec.size].reshape(layer2.W.shape)
layer2_check.bias = neg[layer0.W.size+layer0.bias.size+layer2_w_vec.size:].reshape(layer2.bias.shape)
layer0_check.forwardprop(grad_images[:,:,0].reshape((28,28,1)))
layer1_check.downsample(layer0_check.output, (20,20,2))
full_conn_input = layer1.output.flatten()
layer2_check.softmax_output(full_conn_input.reshape((full_conn_input.size, 1)))
neg_out = J(layer2_check.output, grad_labels[0])
# compute gradient for i
gradient_check[i] = (pos_out - neg_out)/(2*epsilon)
# print gradient_check
print gradient_check[0:layer0_w_vec.size]
def SGD_train(minibatch_size, data, labels, alpha, momentum, epochs):
"""Train the network with stochastic gradient descent
:type minibatch_size: an integer
:param minibatch_size: the size of the minibatches (usually something like 256)
:type data: 3D matrix height x width x num training data pts.
:param data: A 3D matrix that contains all of the training data points of the set
:type labels: num training data pts x 1 vector
:param labels: the labels for each image
:type alpha: float
:param alpha: the learning rate
:type momentum: float
:param momentum: the momentum
:type epochs: an integer
:param epochs: the number of epochs (ie. iterations) through the training
"""
it = 0
# convolutional layer, taking in a 28x28 image, using 2 9x9 filters
# output should be 2 28-9+1x28-9+1 = 2 20x20 feature maps in a (20, 20, 2) form
layer0 = ConvLayer((28, 28, 1), (9,9,2))
print "initialized convolutional layer"
# pooling layer, taking in 2 20x20 feature maps
# output should be 2 10x10 feature maps
layer1 = PoolingLayer((20, 20, 2))
print "initialized pooling layer"
# fully-connected softmax layer, taking in 2 10x10 feature maps (if downsampled by 2)
# flattened into a long input vector
layer2 = FullyConnectedLayer(200, 10)
print "initialized fully-connected layer"
params = np.concatenate((layer0.W.flatten(), layer0.bias.flatten(), layer2.W.flatten(), layer2.bias.flatten()))
velocity = np.zeros(params.shape)
for i in range(0, epochs):
correct_class = 0
cost = 0.0
# shuffle the dataset--shuffle_vec will be used as indices
shuffle_vec = rand.permutation(data.shape[2])
for j in range(0, data.shape[2] - minibatch_size + 1, minibatch_size):
# perform gradient descent w/each batch
it += 1
if it == 20:
# increase momentum after 20 iterations
momentum = 0.9
# gradient should be an unrolled vector of the avg. sum of the 256 gradients gotten
# from the forward pass and backprop
for k in range(0, minibatch_size):
layer0.forwardprop(data[:,:,shuffle_vec[j+k]].reshape((28,28,1)))
layer1.downsample(layer0.output, (20,20,2))
layer2_input = layer1.output.flatten()
layer2.softmax_output(layer2_input.reshape((layer2_input.size, 1)))
cost += J(layer2.output, labels[shuffle_vec[j+k]])
# print "%d %d" % (np.argmax(layer2.output), labels[shuffle_vec[j+k]])
if np.argmax(layer2.output) == labels[shuffle_vec[j+k]]:
correct_class += 1
# backprop
layer2.backprop(0, 0, encode_label(labels[shuffle_vec[j+k]]))
layer1.upsample(layer2, 0)
layer0.backprop(layer1)
# flatten the gradient vector
if k == 0:
grad = np.concatenate((layer0.gradient_w.flatten(), layer0.gradient_b.flatten(), layer2.gradient_w.flatten(), layer2.gradient_b.flatten()))
else:
grad += np.concatenate((layer0.gradient_w.flatten(), layer0.gradient_b.flatten(), layer2.gradient_w.flatten(), layer2.gradient_b.flatten()))
grad /= minibatch_size
# update velocity vector
velocity = momentum*velocity + alpha*grad
params = params - velocity
# update the parameters
layer0.W = params[0:layer0.W.flatten().size].reshape(layer0.W.shape)
next_begin = layer0.W.flatten().size
layer0.bias = params[next_begin:next_begin+layer0.bias.flatten().size].reshape(layer0.bias.shape)
next_begin += layer0.bias.flatten().size
layer2.W = params[next_begin:next_begin+layer2.W.flatten().size].reshape(layer2.W.shape)
next_begin += layer2.W.flatten().size
layer2.bias = params[next_begin:].reshape(layer2.bias.shape)
# reduce learning rate by half after each epoch
alpha /= 2.0
print "%d correct classifications" % correct_class
print "cost function is ", cost/(minibatch_size*(data.shape[2] - minibatch_size + 1))
def __init__(self,rng,weight,agg_func='soft',use_last_layer_weight=False):
# weight: list type,0-31,the first 32 W and b
x = T.tensor4('x')
y = T.imatrix('y')
learning_rate = T.scalar('learning_rate')
self.layer1_input = x #832
self.layer1 = ConvLayer(rng,input=self.layer1_input, filter_shape=(64,3,3,3), layer_index=1, W=weight[0], b=weight[1])
self.layer2 = ConvLayer(rng,input=self.layer1.output,filter_shape=(64,64,3,3),layer_index=2, W=weight[2], b=weight[3])
self.pool1 = PoolLayer(input = self.layer2.output, poolsize=(2,2), layer_index=1) #416
self.layer3 = ConvLayer(rng,input=self.pool1.output, filter_shape=(128,64,3,3),layer_index=3, W=weight[4], b=weight[5])
self.layer4 = ConvLayer(rng,input=self.layer3.output,filter_shape=(128,128,3,3),layer_index=4,W=weight[6], b=weight[7])
self.pool2 = PoolLayer(input = self.layer4.output, poolsize=(2,2), layer_index=2) #208
self.layer5 = ConvLayer(rng,input=self.pool2.output, filter_shape=(256,128,3,3),layer_index=5,W=weight[8], b=weight[9])
self.layer6 = ConvLayer(rng,input=self.layer5.output,filter_shape=(256,256,3,3),layer_index=6,W=weight[10], b=weight[11])
self.layer7 = ConvLayer(rng,input=self.layer6.output,filter_shape=(256,256,3,3),layer_index=7,W=weight[12], b=weight[13])
self.pool3 = PoolLayer(input = self.layer7.output, poolsize=(2,2), layer_index=3) #104
self.layer8 = ConvLayer(rng,input=self.pool3.output, filter_shape=(512,256,3,3),layer_index=8,W=weight[14], b=weight[15])
self.layer9 = ConvLayer(rng,input=self.layer8.output,filter_shape=(512,512,3,3),layer_index=9,W=weight[16], b=weight[17])
self.layer10= ConvLayer(rng,input=self.layer9.output,filter_shape=(512,512,3,3),layer_index=10,W=weight[18],b=weight[19])
self.pool4 = PoolLayer(input = self.layer10.output, poolsize=(2,2), layer_index=4) #52
self.layer11 = ConvLayer(rng,input=self.pool4.output,filter_shape=(512,512,3,3),layer_index=11,W=weight[20],b=weight[21])
self.layer12 = ConvLayer(rng,input=self.layer11.output,filter_shape=(512,512,3,3),layer_index=12,W=weight[22],b=weight[23])
self.layer13 = ConvLayer(rng,input=self.layer12.output,filter_shape=(512,512,3,3),layer_index=13,W=weight[24],b=weight[25])
self.pool5 = PoolLayer(input = self.layer13.output, poolsize=(2,2), layer_index=5) #26
## change FC1/FC2/FC3 layers to Convlayers14/15/16
if use_last_layer_weight: #use weight of layer14/15/16
self.layer14 = ConvLayer(rng,input=self.pool5.output,filter_shape=(1024,512,7,7),layer_index=14,
W=weight[26],b=weight[27]) #20
self.layer15 = ConvLayer(rng,input=self.layer14.nopad_output,filter_shape=(1024,1024,1,1),layer_index=15,
W=weight[28],b=weight[29]) #20
self.layer16 = AggregationLayer(rng,input=self.layer15.nopad_output,filter_shape=(7,1024,1,1),layer_index=16,
W=weight[30],b=weight[31],agg_func=agg_func) #20
else:
self.layer14 = ConvLayer(rng,input=self.pool5.output,filter_shape=(1024,512,7,7),layer_index=14)
self.layer15 = ConvLayer(rng,input=self.layer14.nopad_output,filter_shape=(1024,1024,1,1),layer_index=15)
self.layer16 = AggregationLayer(rng,input=self.layer15.nopad_output,filter_shape=(7,1024,1,1),layer_index=16,
agg_func=agg_func)
# the objective loss
self.loss = self.layer16.squared_error(y)
self.params = self.layer16.params+self.layer15.params+self.layer14.params+self.layer13.params+self.layer12.params\
+self.layer11.params+self.layer10.params+self.layer9.params +self.layer8.params +self.layer7.params\
+self.layer6.params +self.layer5.params +self.layer4.params +self.layer3.params +self.layer2.params\
+self.layer1.params
# optimization methods
updates = lasagne.updates.nesterov_momentum(self.loss,self.params,
learning_rate=learning_rate,
momentum=0.9)
self.train = theano.function(inputs = [x,y,learning_rate],
outputs= [self.loss,self.layer16.errors(y)],
updates= updates,
allow_input_downcast=True)
self.test = theano.function(inputs = [x,y],
outputs= [self.loss,self.layer16.errors(y)],
allow_input_downcast=True)
self.detection = theano.function(
inputs = [x,y],
outputs= [self.layer16.pred,self.layer16.errors(y),
self.layer16.pro_instance,self.layer16.pro_bag],
allow_input_downcast=True )
self.feamap = theano.function(
inputs=[x],
outputs=[self.pool1.output, self.pool2.output, self.pool3.output, self.pool4.output,self.pool5.output],
allow_input_downcast=True)
