def discriminator(x, maxh=256, test=False, output_hidden=False):
"""
Building discriminator network which maps a (B, 1, 28, 28) input to
a (B, 1).
"""
# Define shortcut functions
def bn(xx):
# Batch normalization
return PF.batch_normalization(xx, batch_stat=not test)
def downsample2(xx, c):
return PF.convolution(xx, c, (3, 3), pad=(1, 1), stride=(2, 2), with_bias=False)
assert maxh / 8 > 0
with nn.parameter_scope("dis"):
# (1, 28, 28) --> (32, 16, 16)
with nn.parameter_scope("conv1"):
c1 = F.elu(bn(PF.convolution(x, maxh / 8,
(3, 3), pad=(3, 3), stride=(2, 2), with_bias=False)))
# (32, 16, 16) --> (64, 8, 8)
with nn.parameter_scope("conv2"):
c2 = F.elu(bn(downsample2(c1, maxh / 4)))
# (64, 8, 8) --> (128, 4, 4)
with nn.parameter_scope("conv3"):
c3 = F.elu(bn(downsample2(c2, maxh / 2)))
# (128, 4, 4) --> (256, 4, 4)
with nn.parameter_scope("conv4"):
c4 = bn(PF.convolution(c3, maxh, (3, 3),
pad=(1, 1), with_bias=False))
# (256, 4, 4) --> (1,)
with nn.parameter_scope("fc1"):
f = PF.affine(c4, 1)
if output_hidden:
return f, [c1, c2, c3, c4]
return f
def discriminator(x, maxh=256, test=False, output_hidden=False):
"""
Building discriminator network which maps a (B, 1, 28, 28) input to
a (B, 1).
"""
# Define shortcut functions
def bn(xx):
# Batch normalization
return PF.batch_normalization(xx, batch_stat=not test)
def downsample2(xx, c):
return PF.convolution(xx, c, (3, 3), pad=(1, 1), stride=(2, 2), with_bias=False)
assert maxh / 8 > 0
with nn.parameter_scope("dis"):
# (1, 28, 28) --> (32, 16, 16)
with nn.parameter_scope("conv1"):
c1 = F.elu(bn(PF.convolution(x, maxh / 8,
(3, 3), pad=(3, 3), stride=(2, 2), with_bias=False)))
# (32, 16, 16) --> (64, 8, 8)
with nn.parameter_scope("conv2"):
c2 = F.elu(bn(downsample2(c1, maxh / 4)))
# (64, 8, 8) --> (128, 4, 4)
with nn.parameter_scope("conv3"):
c3 = F.elu(bn(downsample2(c2, maxh / 2)))
# (128, 4, 4) --> (256, 4, 4)
with nn.parameter_scope("conv4"):
c4 = bn(PF.convolution(c3, maxh, (3, 3),
pad=(1, 1), with_bias=False))
# (256, 4, 4) --> (1,)
with nn.parameter_scope("fc1"):
f = PF.affine(c4, 1)
if output_hidden:
return f, [c1, c2, c3, c4]
return f
def vectorizer(x, maxh=256, test=False, output_hidden=False):
"""
Building discriminator network which maps a (B, 1, 28, 28) input to
a (B, 100).
"""
# Define shortcut functions
def bn(xx):
# Batch normalization
return PF.batch_normalization(xx, batch_stat=not test)
def downsample2(xx, c):
return PF.convolution(xx,
c, (3, 3),
pad=(1, 1),
stride=(2, 2),
with_bias=False)
assert maxh / 8 > 0
with nn.parameter_scope("dis"):
# (1, 28, 28) --> (32, 16, 16)
if not test:
x_ = F.image_augmentation(x, min_scale=0.9, max_scale=1.08)
x2 = F.random_shift(x_, (2, 2))
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(
PF.convolution(x2,
maxh / 8, (3, 3),
pad=(3, 3),
stride=(2, 2),
with_bias=False)))
else:
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(
PF.convolution(x,
maxh / 8, (3, 3),
pad=(3, 3),
stride=(2, 2),
with_bias=False)))
# (32, 16, 16) --> (64, 8, 8)
with nn.parameter_scope("conv2"):
c2 = F.elu(bn(downsample2(c1, maxh / 4)))
# (64, 8, 8) --> (128, 4, 4)
with nn.parameter_scope("conv3"):
c3 = F.elu(bn(downsample2(c2, maxh / 2)))
# (128, 4, 4) --> (256, 4, 4)
with nn.parameter_scope("conv4"):
c4 = bn(
PF.convolution(c3, maxh, (3, 3), pad=(1, 1), with_bias=False))
# (256, 4, 4) --> (1,)
with nn.parameter_scope("fc1"):
#print "c4fdafafa",c4.shape
#f = PF.affine(c4, 100)
f = PF.convolution(c4, 1000, (4, 4), pad=(0, 0), with_bias=False)
if output_hidden:
return f, [c1, c2, c3, c4]
return f
def generator(z, maxh=256, test=False, output_hidden=False):
"""
Building generator network which takes (B, Z, 1, 1) inputs and generates
(B, 1, 28, 28) outputs.
"""
# Define shortcut functions
def bn(x):
# Batch normalization
return PF.batch_normalization(x, batch_stat=not test)
def upsample2(x, c):
# Twise upsampling with deconvolution.
return PF.deconvolution(x,
c,
kernel=(4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False)
assert maxh / 4 > 0
with nn.parameter_scope("gen"):
# (Z, 1, 1) --> (256, 4, 4)
with nn.parameter_scope("deconv1"):
d1 = F.elu(bn(PF.deconvolution(z, maxh, (4, 4), with_bias=False)))
# (256, 4, 4) --> (128, 8, 8)
with nn.parameter_scope("deconv2"):
d2 = F.elu(bn(upsample2(d1, maxh / 2)))
# (128, 8, 8) --> (64, 16, 16)
with nn.parameter_scope("deconv3"):
d3 = F.elu(bn(upsample2(d2, maxh / 4)))
# (64, 16, 16) --> (32, 28, 28)
with nn.parameter_scope("deconv4"):
# Convolution with kernel=4, pad=3 and stride=2 transforms a 28 x 28 map
# to a 16 x 16 map. Deconvolution with those parameters behaves like an
# inverse operation, i.e. maps 16 x 16 to 28 x 28.
d4 = F.elu(
bn(
PF.deconvolution(d3,
maxh / 8, (4, 4),
pad=(3, 3),
stride=(2, 2),
with_bias=False)))
# (32, 28, 28) --> (32, 56, 56)
with nn.parameter_scope("deconv5"):
# Convolution with kernel=4, pad=3 and stride=2 transforms a 28 x 28 map
# to a 16 x 16 map. Deconvolution with those parameters behaves like an
# inverse operation, i.e. maps 16 x 16 to 28 x 28.
d5 = F.elu(bn(upsample2(d4, maxh / 8)))
# (32, 56, 56) --> (1, 56, 56)
with nn.parameter_scope("conv6"):
x = F.tanh(PF.convolution(d5, 1, (3, 3), pad=(1, 1)))
if output_hidden:
return x, [d1, d2, d3, d4]
return x
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
def mnist_lenet_feature(image, test=False):
"""
Construct LeNet for MNIST.
"""
c1 = F.elu(PF.convolution(image, 20, (5, 5), name='conv1'))
c1 = F.average_pooling(c1, (2, 2))
c2 = F.elu(PF.convolution(c1, 50, (5, 5), name='conv2'))
c2 = F.average_pooling(c2, (2, 2))
c3 = F.elu(PF.affine(c2, 500, name='fc3'))
c4 = PF.affine(c3, 10, name='fc4')
c5 = PF.affine(c4, 2, name='fc_embed')
return c5
def mnist_lenet_feature(image, test=False):
"""
Construct LeNet for MNIST.
"""
c1 = F.elu(PF.convolution(image, 20, (5, 5), name='conv1'))
c1 = F.average_pooling(c1, (2, 2))
c2 = F.elu(PF.convolution(c1, 50, (5, 5), name='conv2'))
c2 = F.average_pooling(c2, (2, 2))
c3 = F.elu(PF.affine(c2, 500, name='fc3'))
c4 = PF.affine(c3, 10, name='fc4')
c5 = PF.affine(c4, 2, name='fc_embed')
return c5
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.binary_connect_convolution(
x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.binary_connect_convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.binary_connect_convolution(
h, C, (1, 1), with_bias=False))
return F.elu(x + h)
def __call__(self, x, test=False):
with nn.parameter_scope(self.scope):
with nn.parameter_scope('conv1'):
h1 = F.elu(bn(downsample(x, self.hidden_channel), test))
with nn.parameter_scope('conv2'):
h2 = F.elu(bn(downsample(h1, self.hidden_channel // 2), test))
with nn.parameter_scope('conv3'):
h3 = F.elu(bn(downsample(h2, self.hidden_channel // 4), test))
with nn.parameter_scope('conv4'):
h4 = F.elu(bn(downsample(h3, self.hidden_channel // 8), test))
with nn.parameter_scope('fc1'):
f = PF.affine(h4, 1)
return f
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.binary_connect_convolution(
x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.binary_connect_convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.binary_connect_convolution(
h, C, (1, 1), with_bias=False))
return F.elu(x + h)
def __call__(self, x, test=False):
with nn.parameter_scope(self.scope):
with nn.parameter_scope('conv1'):
h1 = F.elu(bn(downsample(x, self.hidden_channel), test))
with nn.parameter_scope('conv2'):
h2 = F.elu(bn(downsample(h1, self.hidden_channel // 8), test))
with nn.parameter_scope('conv3'):
h3 = F.elu(bn(downsample(h2, self.hidden_channel // 4), test))
with nn.parameter_scope('conv4'):
h4 = F.elu(bn(downsample(h3, self.hidden_channel // 2), test))
with nn.parameter_scope('deconv1'):
h5 = F.elu(bn(upsample(h4, self.hidden_channel), test))
with nn.parameter_scope('deconv2'):
h6 = F.elu(bn(upsample(h5, self.hidden_channel // 2), test))
with nn.parameter_scope('deconv3'):
h7 = F.elu(bn(upsample(h6, self.hidden_channel // 4), test))
with nn.parameter_scope('deconv4'):
h8 = F.elu(bn(upsample(h7, self.hidden_channel // 8), test))
with nn.parameter_scope('conv5'):
y = F.tanh(
PF.convolution(h8,
self.out_channel,
kernel=(3, 3),
pad=(1, 1)))
return y
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Elu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x,
C / 2,
kernel=(1, 1),
pad=(0, 0),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.elu(h)
# Conv -> BN -> Elu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h,
C / 2,
kernel=(3, 3),
pad=(1, 1),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.elu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Elu
h = F.elu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def mnist_binary_weight_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (Binary Weight Network version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_weight_convolution(image, 16, (5, 5))
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_weight_convolution(c1, 16, (5, 5))
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = F.elu(PF.binary_weight_affine(c2, 50))
with nn.parameter_scope("fc4"):
c4 = PF.binary_weight_affine(c3, 10)
return c4
def mnist_binary_weight_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (Binary Weight Network version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_weight_convolution(image, 16, (5, 5))
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_weight_convolution(c1, 16, (5, 5))
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = F.elu(PF.binary_weight_affine(c2, 50))
with nn.parameter_scope("fc4"):
c4 = PF.binary_weight_affine(c3, 10)
return c4
def mnist_inq_resnet_prediction(image, num_bits=3,
inq_iterations=(5000, 6000, 7000, 8000, 9000),
selection_algorithm='largest_abs', test=False):
"""
Construct ResNet for MNIST (INQ Version).
"""
image /= 255.0
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.inq_convolution(x, C / 2, (1, 1),
num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.inq_convolution(h, C / 2, (3, 3), pad=(1, 1),
num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.inq_convolution(h, C, (1, 1), num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(PF.inq_convolution(image, 64, (3, 3), pad=(3, 3),
num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = PF.inq_affine(pl, 10, num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm)
return y
def mnist_binary_connect_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (BinaryNet version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_connect_convolution(image, 16, (5, 5))
c1 = PF.batch_normalization(c1, batch_stat=not test)
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_connect_convolution(c1, 16, (5, 5))
c2 = PF.batch_normalization(c2, batch_stat=not test)
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = PF.binary_connect_affine(c2, 50)
c3 = PF.batch_normalization(c3, batch_stat=not test)
c3 = F.elu(c3)
with nn.parameter_scope("fc4"):
c4 = PF.binary_connect_affine(c3, 10)
c4 = PF.batch_normalization(c4, batch_stat=not test)
return c4
def encoder(inp, test=False):
with nn.parameter_scope('encoder'):
c1 = PF.convolution(inp, 64, (3, 3), pad=(1, 1), name='c1')
c1 = F.elu(c1)
c2 = conv_bn_relu(c1,
128, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c2')
c3 = conv_bn_relu(c2,
256, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c3')
c4 = conv_bn_relu(c3,
512, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c4')
c5 = conv_bn_relu(c4,
512, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c5')
c6 = conv_bn_relu(c5,
512, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c6')
c7 = conv_bn_relu(c6,
512, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c7')
c8 = conv_bn_relu(c7,
512, (4, 4),
stride=(2, 2),
pad=(1, 1),
relu=F.elu,
test=test,
name='c8')
return [c8, c7, c6, c5, c4, c3, c2, c1]
def mnist_binary_connect_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (BinaryNet version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_connect_convolution(image, 16, (5, 5))
c1 = PF.batch_normalization(c1, batch_stat=not test)
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_connect_convolution(c1, 16, (5, 5))
c2 = PF.batch_normalization(c2, batch_stat=not test)
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = PF.binary_connect_affine(c2, 50)
c3 = PF.batch_normalization(c3, batch_stat=not test)
c3 = F.elu(c3)
with nn.parameter_scope("fc4"):
c4 = PF.binary_connect_affine(c3, 10)
c4 = PF.batch_normalization(c4, batch_stat=not test)
return c4
def mnist_binary_connect_resnet_prediction(image, test=False):
"""
Construct ResNet for MNIST (BinaryNet version).
"""
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(
bn(
PF.binary_connect_convolution(x,
C / 2, (1, 1),
with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(
PF.binary_connect_convolution(h,
C / 2, (3, 3),
pad=(1, 1),
with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(
PF.binary_connect_convolution(h,
C, (1, 1),
with_bias=False))
return F.elu(x + h)
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(
PF.binary_connect_convolution(image,
64, (3, 3),
pad=(3, 3),
with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = bn(PF.binary_connect_affine(pl, 10))
return y
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.inq_convolution(x, C / 2, (1, 1),
num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.inq_convolution(h, C / 2, (3, 3), pad=(1, 1),
num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.inq_convolution(h, C, (1, 1), num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm,
with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
def generator(z, maxh=256, test=False, output_hidden=False):
"""
Building generator network which takes (B, Z, 1, 1) inputs and generates
(B, 1, 28, 28) outputs.
"""
# Define shortcut functions
def bn(x):
# Batch normalization
return PF.batch_normalization(x, batch_stat=not test)
def upsample2(x, c):
# Twise upsampling with deconvolution.
return PF.deconvolution(x, c, kernel=(4, 4), pad=(1, 1), stride=(2, 2), with_bias=False)
assert maxh / 4 > 0
with nn.parameter_scope("gen"):
# (Z, 1, 1) --> (256, 4, 4)
with nn.parameter_scope("deconv1"):
d1 = F.elu(bn(PF.deconvolution(z, maxh, (4, 4), with_bias=False)))
# (256, 4, 4) --> (128, 8, 8)
with nn.parameter_scope("deconv2"):
d2 = F.elu(bn(upsample2(d1, maxh / 2)))
# (128, 8, 8) --> (64, 16, 16)
with nn.parameter_scope("deconv3"):
d3 = F.elu(bn(upsample2(d2, maxh / 4)))
# (64, 16, 16) --> (32, 28, 28)
with nn.parameter_scope("deconv4"):
# Convolution with kernel=4, pad=3 and stride=2 transforms a 28 x 28 map
# to a 16 x 16 map. Deconvolution with those parameters behaves like an
# inverse operation, i.e. maps 16 x 16 to 28 x 28.
d4 = F.elu(bn(PF.deconvolution(
d3, maxh / 8, (4, 4), pad=(3, 3), stride=(2, 2), with_bias=False)))
# (32, 28, 28) --> (1, 28, 28)
with nn.parameter_scope("conv5"):
x = F.tanh(PF.convolution(d4, 1, (3, 3), pad=(1, 1)))
if output_hidden:
return x, [d1, d2, d3, d4]
return x
def mnist_resnet_prediction(image, net="teacher", maps=64, test=False):
"""
Construct ResNet for MNIST.
"""
image /= 255.0
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1),
with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(
PF.convolution(h,
C / 2, (3, 3),
pad=(1, 1),
with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
with nn.parameter_scope(net):
# Conv1 --> maps x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(
PF.convolution(image,
maps, (3, 3),
pad=(3, 3),
with_bias=False)))
# Conv2 --> maps x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> maps x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> maps x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> maps x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> maps x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = PF.affine(pl, 10)
return y
def network(x, y_index, test=False):
# Input -> 3,64,64
# Convolution -> 16,31,31
with nn.parameter_scope('Convolution'):
h = PF.convolution(x, 16, (3, 3), (0, 0), (2, 2))
# Tanh
h = F.tanh(h)
# MaxPooling -> 16,16,11
h = F.max_pooling(h, (2, 3), (2, 3))
# Dropout
if not test:
h = F.dropout(h)
# Convolution_2 -> 32,6,5
with nn.parameter_scope('Convolution_2'):
h = PF.convolution(h, 32, (5, 3), (0, 0), (2, 2))
# ReLU_4
h = F.relu(h, True)
# MaxPooling_2 -> 32,3,3
h = F.max_pooling(h, (2, 2), (2, 2))
# Dropout_2
if not test:
h = F.dropout(h)
# Convolution_3 -> 64,1,1
with nn.parameter_scope('Convolution_3'):
h = PF.convolution(h, 64, (3, 3), (0, 0), (2, 2))
# Tanh_2
h = F.tanh(h)
# Dropout_3
if not test:
h = F.dropout(h)
# Affine -> 50
with nn.parameter_scope('Affine'):
h = PF.affine(h, (50, ))
# ReLU_2
h = F.relu(h, True)
# Dropout_4
if not test:
h = F.dropout(h)
# Affine_2 -> 5
with nn.parameter_scope('Affine_2'):
h = PF.affine(h, (5, ))
# ELU
h = F.elu(h)
# Affine_3 -> 1
with nn.parameter_scope('Affine_3'):
h = PF.affine(h, (1, ))
# SquaredError
#h = F.squared_error(h, y_index)
return h
def ref_activation(x, nonlinearity, nonlinearity_args):
if nonlinearity == 'identity' or not nonlinearity:
return x
elif nonlinearity == 'relu':
return F.relu(x)
elif nonlinearity == 'sigmoid':
return F.sigmoid(x)
elif nonlinearity == 'tanh':
return F.tanh(x)
elif nonlinearity == 'leaky_relu':
return F.leaky_relu(x, nonlinearity_args[0])
elif nonlinearity == 'elu':
return F.elu(x, nonlinearity_args[0])
elif nonlinearity == 'relu6':
return F.relu6(x)
raise ValueError("unknown nonlinearity type {}".format(nonlinearity))
def mnist_resnet_prediction(image, test=False):
"""
Construct ResNet for MNIST.
"""
image /= 255.0
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(PF.convolution(image, 64, (3, 3), pad=(3, 3), with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = PF.affine(pl, 10)
return y
def network(x, r, test=False):
# Input:x -> 1,128,128
# Input_2:r -> 1
# Convolution -> 16,42,42
h = PF.convolution(x, 16, (7, 7), (1, 1), (3, 3), name='Convolution')
# BatchNormalization
h = PF.batch_normalization(h, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization')
# ELU
h = F.elu(h, 1.0)
# MaxPooling -> 16,14,14
h = F.max_pooling(h, (3, 3), (3, 3))
# Convolution_2 -> 32,6,6
h = PF.convolution(h, 32, (5, 5), (1, 1), (2, 2), name='Convolution_2')
# BatchNormalization_2
h = PF.batch_normalization(h, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization_2')
# ELU_2
h = F.elu(h, 1.0)
# Convolution_3 -> 64,2,2
h = PF.convolution(h, 64, (5, 5), (0, 0), name='Convolution_3')
# BatchNormalization_7
h = PF.batch_normalization(h, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization_7')
# ELU_7
h = F.elu(h, 1.0)
# MaxPooling_2 -> 64,1,1
h = F.max_pooling(h, (2, 2), (2, 2))
# Reshape -> 64
h = F.reshape(h, (
h.shape[0],
64,
))
# Concatenate -> 65
h1 = F.concatenate(r, h, axis=1)
# Affine_3 -> 512
h2 = PF.affine(h1, (512, ), name='Affine_3')
# Affine_2 -> 512
h3 = PF.affine(h1, (512, ), name='Affine_2')
# BatchNormalization_3
h2 = PF.batch_normalization(h2, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization_3')
# BatchNormalization_4
h3 = PF.batch_normalization(h3, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization_4')
# ELU_3
h2 = F.elu(h2, 1.0)
# ELU_4
h3 = F.elu(h3, 1.0)
# Affine_7
h2 = PF.affine(h2, (512, ), name='Affine_7')
# Affine_6
h3 = PF.affine(h3, (512, ), name='Affine_6')
# BatchNormalization_5
h2 = PF.batch_normalization(h2, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization_5')
# BatchNormalization_6
h3 = PF.batch_normalization(h3, (1, ),
0.8999999761581421,
9.999999747378752e-05,
not test,
name='BatchNormalization_6')
# ELU_5
h2 = F.elu(h2, 1.0)
# ELU_6
h3 = F.elu(h3, 1.0)
# Affine_5 -> 50,2
h2 = PF.affine(h2, (50, 2), name='Affine_5')
# Affine_4 -> 2,2
h3 = PF.affine(h3, (2, 2), name='Affine_4')
return h2, h3
def vae(x, shape_z, test=False):
"""
Function for calculate Elbo(evidence lowerbound) loss.
This sample is a Bernoulli generator version.
Args:
x(`~nnabla.Variable`): N-D array
shape_z(tuple of int): size of z
test : True=train, False=test
Returns:
~nnabla.Variable: Elbo loss
"""
#############################################
# Encoder of 2 fully connected layers #
#############################################
# Normalize input
xa = x / 256.
batch_size = x.shape[0]
# 2 fully connected layers, and Elu replaced from original Softplus.
h = F.elu(PF.affine(xa, (500, ), name='fc1'))
h = F.elu(PF.affine(h, (500, ), name='fc2'))
# The outputs are the parameters of Gauss probability density.
mu = PF.affine(h, shape_z, name='fc_mu')
logvar = PF.affine(h, shape_z, name='fc_logvar')
sigma = F.exp(0.5 * logvar)
# The prior variable and the reparameterization trick
if not test:
# training with reparameterization trick
epsilon = F.randn(mu=0, sigma=1, shape=(batch_size, ) + shape_z)
z = mu + sigma * epsilon
else:
# test without randomness
z = mu
#############################################
# Decoder of 2 fully connected layers #
#############################################
# 2 fully connected layers, and Elu replaced from original Softplus.
h = F.elu(PF.affine(z, (500, ), name='fc3'))
h = F.elu(PF.affine(h, (500, ), name='fc4'))
# The outputs are the parameters of Bernoulli probabilities for each pixel.
prob = PF.affine(h, (1, 28, 28), name='fc5')
#############################################
# Elbo components and loss objective #
#############################################
# Binarized input
xb = F.greater_equal_scalar(xa, 0.5)
# E_q(z|x)[log(q(z|x))]
# without some constant terms that will canceled after summation of loss
logqz = 0.5 * F.sum(1.0 + logvar, axis=1)
# E_q(z|x)[log(p(z))]
# without some constant terms that will canceled after summation of loss
logpz = 0.5 * F.sum(mu * mu + sigma * sigma, axis=1)
# E_q(z|x)[log(p(x|z))]
logpx = F.sum(F.sigmoid_cross_entropy(prob, xb), axis=(1, 2, 3))
# Vae loss, the negative evidence lowerbound
loss = F.mean(logpx + logpz - logqz)
return loss
#!/usr/bin/env python
import nnabla as nn
import nnabla.functions as F
import nnabla.parametric_functions as PF
import numpy as np
import matplotlib.pyplot as plt
batch_size = 1
x = nn.Variable((batch_size,1))
h1 = F.elu(PF.affine(x, 16,name="affine1"))
h2 = F.elu(PF.affine(h1, 32,name="affine2"))
y = F.elu(PF.affine(h2, 1,name="affine3"))
nn.load_parameters("exp_net.h5")
xi=np.linspace(0,1,100)
plt.plot(xi,np.exp(xi))
ys=[]
for i in range(100):
x.d = xi[i]
y.forward()
ys.append(y.d.copy())
_=plt.plot(xi, np.array(ys).reshape((100,)),"r")
plt.show()
def cifar100_resnet23_prediction(image,
ctx, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Elu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.elu(h)
# Conv -> BN -> Elu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.elu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Elu
h = F.elu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
rng = np.random.RandomState(0)
nmaps = 384
ncls = 100
# Conv -> BN -> Elu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
# Preprocess
if not test:
image = F.image_augmentation(image, contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
w_init = UniformInitializer(
calc_uniform_lim_glorot(3, nmaps, kernel=(3, 3)),
rng=rng)
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.elu(h)
h = res_unit(h, "conv2", rng, False) # -> 32x32
h = res_unit(h, "conv3", rng, True) # -> 16x16
h = res_unit(h, "conv4", rng, False) # -> 16x16
h = res_unit(h, "conv5", rng, True) # -> 8x8
h = res_unit(h, "conv6", rng, False) # -> 8x8
h = res_unit(h, "conv7", rng, True) # -> 4x4
h = res_unit(h, "conv8", rng, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
w_init = UniformInitializer(
calc_uniform_lim_glorot(int(np.prod(h.shape[1:])), ncls, kernel=(1, 1)), rng=rng)
pred = PF.affine(h, ncls, w_init=w_init)
return pred