def mnist_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST.
"""
image /= 255.0
c1 = PF.convolution(image, 16, (5, 5), name='conv1')
c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
c2 = PF.convolution(c1, 16, (5, 5), name='conv2')
c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
c3 = F.relu(PF.affine(c2, 50, name='fc3'), inplace=True)
c4 = PF.affine(c3, 10, name='fc4')
return c4
def cnn_model_003(ctx, x, act=F.relu, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
# Convblock 3
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h_branch = h
# Convblock 3
h = conv_unit(h_branch, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u0 = F.average_pooling(u0, (6, 6))
with nn.parameter_scope("u0bn"):
u0 = PF.batch_normalization(u0, batch_stat=not test)
log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:])))
# Uncertainty for uncertainty
u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test)
u1 = F.average_pooling(u1, (6, 6))
with nn.parameter_scope("u1bn"):
u1 = PF.batch_normalization(u1, batch_stat=not test)
log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:])))
return pred, log_var, log_s
def cnn_model_003(ctx, h, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
if not test:
b, c, s, s = h.shape
h = F.image_augmentation(h, (c, s, s),
min_scale=1.0, max_scale=1.5,
angle=0.5, aspect_ratio=1.3, distortion=0.2,
flip_lr=True)
# Convblock0
h = conv_unit(h, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (6, 6))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 28 -> 14
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 14 -> 7
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 7 -> 5
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (5, 5))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (5, 5))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def __call__(self, x):
'''
Defines a ResNet-like network according to the configuration specified.
Args:
x:
A Variable object which has a shape with a format
`NCHW` if `channel_last=False` else `NHWC`.
Returns:
* An output `Variable` of classification layer
* Intermediate `Variable` outputs from input and output of each
cell
'''
logger.debug(x.shape)
# First convolution
axes = [get_channel_axis(self.channel_last)]
with nn.parameter_scope("conv1"):
r = pf_convolution(x,
64, (7, 7),
stride=(2, 2),
channel_last=self.channel_last)
r = PF.fused_batch_normalization(r,
axes=axes,
batch_stat=not self.test)
mp_opts = dict(
ignore_border=False) if self.max_pooling_ceil_border else dict(
pad=(1, 1))
r = F.max_pooling(r, (3, 3), (2, 2),
channel_last=self.channel_last,
**mp_opts)
hidden = {}
hidden['r0'] = r
logger.debug(r.shape)
# Create cells each of which consists of blocks repeatedly applied
cell_configs = self.get_cell_configurations(self.num_layers)
for i, (counts, ochannels, strides) in enumerate(zip(*cell_configs)):
with nn.parameter_scope("res{}".format(i + 1)):
r = self.cell(r, ochannels, counts, (strides, ) * 2)
hidden['r{}'.format(i + 1)] = r
logger.debug(r.shape)
# Global average pooling
pool_shape = get_spatial_shape(r.shape, self.channel_last)
r = F.average_pooling(r, pool_shape, channel_last=self.channel_last)
# Final classification layer
with nn.parameter_scope("fc"):
r = pf_affine(r, self.num_classes, channel_last=self.channel_last)
return r, hidden
def net(h):
import nnabla.functions as F
import nnabla.parametric_functions as PF
h = PF.convolution(h, 3, (3, 3), name="conv1")
h = PF.batch_normalization(h, name="bn1")
h = F.relu(h)
h = F.max_pooling(h, (2, 2))
h = PF.convolution(h, 3, (3, 3), name="conv2")
h = PF.batch_normalization(h, name="bn2")
pred = F.relu(h)
return pred
def zero(x, output_filter, scope,
input_node_id, is_reduced, test, is_search):
"""
Zero operation, i.e. all elements become 0.
"""
if is_reduced and input_node_id < 2:
h = F.max_pooling(x, kernel=(1, 1), stride=(2, 2)) # downsampling
h = F.mul_scalar(h, 0)
else:
h = F.mul_scalar(x, 0)
return h
def lenet(image, test=False):
h = PF.convolution(image,
16, (5, 5), (1, 1),
with_bias=False,
name='conv1')
h = PF.batch_normalization(h, batch_stat=not test, name='conv1-bn')
h = F.max_pooling(h, (2, 2))
h = F.relu(h)
h = PF.convolution(h, 16, (5, 5), (1, 1), with_bias=True, name='conv2')
h = PF.batch_normalization(h, batch_stat=not test, name='conv2-bn')
h = F.max_pooling(h, (2, 2))
h = F.relu(h)
h = PF.affine(h, 10, with_bias=False, name='fc1')
h = PF.batch_normalization(h, batch_stat=not test, name='fc1-bn')
h = F.relu(h)
pred = PF.affine(h, 10, with_bias=True, name='fc2')
return pred
def mnist_lenet_prediction_slim(image, scope="slim", rrate=0.75, test=False):
"""
Construct LeNet for MNIST.
"""
with nn.parameter_scope(scope):
image /= 255.0
c1 = PF.convolution(image, 16, (5, 5), name='conv1')
c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
c2 = PF.convolution(c1, 16, (5, 5), name='conv2')
c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
# SVD applied
inmaps = np.prod(c2.shape[1:]) # c * h * w
outmaps0 = 50 # original outmaps
outmaps1 = reduce_maps(inmaps, outmaps0, rrate)
d0 = F.relu(PF.affine(c2, outmaps1, name='fc-d0'), inplace=True)
d1 = F.relu(PF.affine(d0, outmaps0, name='fc-d1'), inplace=True)
c4 = PF.affine(d1, 10, name='fc4')
return c4
def resnet_imagenet(x,
num_classes,
num_layers,
shortcut_type,
test,
tiny=False):
"""
Args:
x : Variable
num_classes : Number of classes of outputs
num_layers : Number of layers of ResNet chosen from (18, 34, 50, 101, 152)
shortcut_type : 'c', 'b', ''
'c' : Use Convolution anytime
'b' : Use Convolution if numbers of channels of input
and output mismatch.
'' : Use Identity mapping if channels match, otherwise zero padding.
test : Construct net for testing.
tiny (bool): Tiny imagenet mode. Input image must be (3, 56, 56).
"""
layers = {
18: ((2, 2, 2, 2), basicblock, 1),
34: ((3, 4, 6, 3), basicblock, 1),
50: ((3, 4, 6, 3), bottleneck, 4),
101: ((3, 4, 23, 3), bottleneck, 4),
152: ((3, 8, 36, 3), bottleneck, 4)
}
counts, block, ocoef = layers[num_layers]
logger.debug(x.shape)
with nn.parameter_scope("conv1"):
stride = (1, 1) if tiny else (2, 2)
r = PF.convolution(x,
64, (7, 7),
pad=(3, 3),
stride=stride,
with_bias=False)
r = F.relu(PF.batch_normalization(r, batch_stat=not test))
r = F.max_pooling(r, (3, 3), stride, pad=(1, 1))
hidden = {}
hidden['r0'] = r
ochannels = [64, 128, 256, 512]
strides = [1, 2, 2, 2]
logger.debug(r.shape)
for i in range(4):
with nn.parameter_scope("res{}".format(i + 1)):
r = layer(r, block, ochannels[i] * ocoef, counts[i],
(strides[i], strides[i]), shortcut_type, test)
hidden['r{}'.format(i + 1)] = r
logger.debug(r.shape)
r = F.average_pooling(r, r.shape[-2:])
with nn.parameter_scope("fc"):
r = PF.affine(r, num_classes)
logger.debug(r.shape)
return r, hidden
def down_block(input, output_channels=64, stride=1, scope='down_block'):
with nn.parameter_scope(scope):
net = conv2d(input,
output_channels, (3, 3), (stride, stride),
name='conv_1')
net = F.leaky_relu(net, 0.2)
net = conv2d(net,
output_channels, (3, 3), (stride, stride),
name='conv_2')
net = F.leaky_relu(net, 0.2)
net = F.max_pooling(net, (2, 2), channel_last=True)
return net
def cnn_dni(image, y=None, maps=128, ncls=10, test=False):
with nn.parameter_scope("ref"):
image /= 255.
h = act_bn_conv(image, maps, test, name="conv0")
h = F.max_pooling(h, (2, 2)) # 28x28 -> 14x14
h = act_bn_conv(h, maps, act=None, test=test, name="conv1")
# decoupled here
h_d, h_copy, x_input, g_label = decouple(h)
g_pred = cnn_gradient_synthesizer(x_input, y, test)
h_d.grad = g_pred.data
h = F.relu(h) # decouple after non-linearity
h = F.max_pooling(h, (2, 2)) # 14x14 -> 7x7
with nn.parameter_scope("ref"):
h = act_bn_conv(h, maps, test, name="conv2")
h = F.average_pooling(h, (7, 7)) # 7x7 -> 1x1
pred = PF.affine(h, ncls, name="fc")
pred.persistent = True
return h_d, h_copy, pred, g_pred, g_label
def small_bn_rm_resnet(image, test=False, w_bias=False, name='bn-rm-graph-ref'):
h = image
h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1),
with_bias=w_bias, name='first-conv')
h = F.relu(h)
h = F.max_pooling(h, (2, 2))
h = bn_rm_resblock(h, maps=16, test=test, w_bias=w_bias, name='cb1')
h = bn_rm_resblock(h, maps=16, test=test, w_bias=w_bias, name='cb2')
h = bn_rm_resblock(h, maps=16, test=test, w_bias=w_bias, name='cb3')
h = bn_rm_resblock(h, maps=16, test=test, w_bias=w_bias, name='cb4')
h = F.average_pooling(h, (2, 2))
pred = PF.affine(h, 10, name='bn-rm-fc')
return pred
def mnist_resnet_prediction(image, test=False, aug=None):
"""
Construct ResNet for MNIST.
"""
image /= 255.0
image = augmentation(image, test, aug)
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(PF.convolution(image, 64, (3, 3), pad=(3, 3), with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = PF.affine(pl, 10)
return y
def mnist_binary_connect_resnet_prediction(image, test=False):
"""
Construct ResNet for MNIST (BinaryNet version).
"""
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.binary_connect_convolution(
x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.binary_connect_convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.binary_connect_convolution(
h, C, (1, 1), with_bias=False))
return F.elu(x + h)
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(PF.binary_connect_convolution(image, 64, (3, 3), pad=(3, 3), with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = bn(PF.binary_connect_affine(pl, 10))
return y
def fpq_lenet(image,
test=False,
n=8,
delta=2e-4,
name="fixed-point-graph-ref"):
with nn.parameter_scope(name):
h = PF.fixed_point_quantized_convolution(image,
16, (5, 5), (1, 1),
with_bias=False,
delta_w=delta,
name='conv1')
h = PF.batch_normalization(h, batch_stat=not test, name='conv1-bn')
h = F.max_pooling(h, (2, 2))
h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False)
h = PF.fixed_point_quantized_convolution(h,
16, (5, 5), (1, 1),
with_bias=True,
delta_w=delta,
name='conv2')
h = PF.batch_normalization(h, batch_stat=not test, name='conv2-bn')
h = F.max_pooling(h, (2, 2))
h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False)
h = PF.fixed_point_quantized_affine(h,
10,
with_bias=False,
delta_w=delta,
name='fc1')
h = PF.batch_normalization(h, batch_stat=not test, name='fc1-bn')
h = F.fixed_point_quantize(h, n=n, delta=delta, sign=False)
pred = PF.fixed_point_quantized_affine(h,
10,
with_bias=True,
delta_w=delta,
name='fc2')
return pred
def get_alex_feat(input_var):
"""
Exactly the same architecture used for LPIPS.
This is a little bit modified version (can't use nnabla models!).
"""
assert input_var.shape[1] == 3
act1 = F.relu(
PF.convolution(input_var,
outmaps=64,
kernel=(11, 11),
pad=(2, 2),
stride=(4, 4),
name="conv0"), True)
act2 = F.relu(
PF.convolution(F.max_pooling(act1, kernel=(3, 3), stride=(2, 2)),
outmaps=192,
kernel=(5, 5),
pad=(2, 2),
name="conv3"), True)
act3 = F.relu(
PF.convolution(F.max_pooling(act2, kernel=(3, 3), stride=(2, 2)),
outmaps=384,
kernel=(3, 3),
pad=(1, 1),
name="conv6"), True)
act4 = F.relu(
PF.convolution(act3,
outmaps=256,
kernel=(3, 3),
pad=(1, 1),
name="conv8"), True)
act5 = F.relu(
PF.convolution(act4,
outmaps=256,
kernel=(3, 3),
pad=(1, 1),
name="conv10"), True)
return [act1, act2, act3, act4, act5]
def mnist_resnet_prediction(image, test=False):
"""
Construct ResNet for MNIST.
"""
image /= 255.0
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(PF.convolution(image, 64, (3, 3), pad=(3, 3), with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = PF.affine(pl, 10)
return y
def small_clbn_folding_resnet(image, test=False):
h = image
h /= 255.0
h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1), channel_last=True,
with_bias=True, name='first-clbn-conv')
h = F.relu(h)
h = F.max_pooling(h, (2, 2), channel_last=True)
h = clbn_folding_resblock(h, maps=16, test=test, name='clbn-cb1')
h = clbn_folding_resblock(h, maps=16, test=test, name='clbn-cb2')
h = clbn_folding_resblock(h, maps=16, test=test, name='clbn-cb3')
h = clbn_folding_resblock(h, maps=16, test=test, name='clbn-cb4')
h = F.average_pooling(h, (2, 2), channel_last=True)
pred = PF.affine(h, 10, name='clbn-fc')
return pred
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x,
C / 2,
kernel=(1, 1),
pad=(0, 0),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h,
C / 2,
kernel=(3, 3),
pad=(1, 1),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def mnist_inq_lenet_prediction(image, num_bits=3,
inq_iterations=(5000, 6000, 7000, 8000, 9000),
selection_algorithm='largest_abs', test=False):
"""
Construct LeNet for MNIST (INQ Version).
"""
image /= 255.0
c1 = PF.inq_convolution(image, 16, (5, 5), name='conv1', num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm)
c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
c2 = PF.inq_convolution(c1, 16, (5, 5), name='conv2', num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm)
c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
c3 = F.relu(PF.inq_affine(c2, 50, name='fc3', num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm),
inplace=True)
c4 = PF.inq_affine(c3, 10, name='fc4', num_bits=num_bits,
inq_iterations=inq_iterations,
selection_algorithm=selection_algorithm)
return c4
def bn_opp_lenet(image, test=False, channel_last=False, w_bias=False):
axes = get_channel_axes(image, channel_last)
h = PF.batch_normalization(image,
axes=axes,
batch_stat=not test,
name='conv1-bn')
h = PF.convolution(h,
16, (5, 5), (1, 1),
with_bias=w_bias,
channel_last=channel_last,
name='conv1')
h = F.max_pooling(h, (2, 2))
h = F.relu(h)
axes = get_channel_axes(h, channel_last)
h = PF.batch_normalization(h,
axes=axes,
batch_stat=not test,
name='conv2-bn')
h = PF.convolution(h,
16, (5, 5), (1, 1),
with_bias=w_bias,
channel_last=channel_last,
name='conv2')
h = F.max_pooling(h, (2, 2), channel_last=channel_last)
h = F.relu(h)
axes = get_channel_axes(h, channel_last)
h = PF.batch_normalization(h,
axes=axes,
batch_stat=not test,
name='fc1-bn')
h = PF.affine(h, 10, with_bias=True, name='fc1')
h = F.relu(h)
pred = PF.affine(h, 10, with_bias=True, name='fc2')
return pred
def small_bn_resnet(image,
test=False,
w_bias=False,
channel_last=False,
name='bn-graph-ref'):
axes = get_channel_axes(channel_last)
h = image
h /= 255.0
h = PF.convolution(h,
16,
kernel=(3, 3),
pad=(1, 1),
channel_last=channel_last,
with_bias=w_bias,
name='first-conv')
h = PF.batch_normalization(h,
axes=axes,
batch_stat=not test,
name='first-bn')
h = F.relu(h)
h = F.max_pooling(h, (2, 2), channel_last=channel_last)
h = bn_resblock(h,
maps=16,
test=test,
w_bias=w_bias,
channel_last=channel_last,
name='cb1')
h = bn_resblock(h,
maps=16,
test=test,
w_bias=w_bias,
channel_last=channel_last,
name='cb2')
h = bn_resblock(h,
maps=16,
test=test,
w_bias=w_bias,
channel_last=channel_last,
name='cb3')
h = bn_resblock(h,
maps=16,
test=test,
w_bias=w_bias,
channel_last=channel_last,
name='cb4')
h = F.average_pooling(h, (2, 2), channel_last=channel_last)
pred = PF.affine(h, 10, name='fc')
return pred
def bn_linear_lenet(image, test=False, name="bn-linear-graph-ref"):
with nn.parameter_scope(name):
h = PF.convolution(image,
16, (5, 5), (1, 1),
with_bias=False,
name='conv1')
a, b = create_scale_bias(1, h.shape[1])
h = a * h + b
h = F.max_pooling(h, (2, 2))
h = F.relu(h)
h = PF.convolution(h, 16, (5, 5), (1, 1), with_bias=True, name='conv2')
a, b = create_scale_bias(2, h.shape[1])
h = a * h + b
h = F.max_pooling(h, (2, 2))
h = F.relu(h)
h = PF.affine(h, 10, with_bias=False, name='fc1')
a, b = create_scale_bias(4, h.shape[1], 2)
h = a * h + b
h = F.relu(h)
pred = PF.affine(h, 10, with_bias=True, name='fc2')
return pred
def small_bsf_resnet(image, w_bias=False, name='bn-graph-ref'):
h = image
h /= 255.0
h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1),
with_bias=w_bias, name='first-conv')
h = PF.batch_normalization(h, batch_stat=False, name='first-bn-bsf')
h = F.relu(h)
h = F.max_pooling(h, (2, 2))
h = bsf_resblock(h, maps=16, test=False, w_bias=w_bias, name='cb1')
h = bsf_resblock(h, maps=16, test=False, w_bias=w_bias, name='cb2')
h = bsf_resblock(h, maps=16, test=False, w_bias=w_bias, name='cb3')
h = bsf_resblock(h, maps=16, test=False, w_bias=w_bias, name='cb4')
h = F.average_pooling(h, (2, 2))
pred = PF.affine(h, 10, name='fc')
return pred
def cnn_model_003(ctx, scope, x, act=F.relu, test=False):
with nn.context_scope(ctx):
with nn.parameter_scope(scope):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act,
test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
# Convblock 3
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def stochastic_res_unit(x, scope_name, act=F.relu, dn=False, test=False):
if not test:
flag = np.random.randint(2)
if flag:
h = res_block(x, scope_name, act=act, dn=dn, test=test)
h = F.add2(h, x)
else:
h = x
else:
h = res_block(x, scope_name, act=act, dn=dn, test=test)
h = F.add2(h, x)
h = act(h)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def stochastic_res_unit(x, scope_name, act=F.relu, dn=False, test=False):
if not test:
flag = np.random.randint(2)
if flag:
h = res_block(x, scope_name, act=act, dn=dn, test=test)
h = F.add2(h, x)
else:
h = x
else:
h = res_block(x, scope_name, act=act, dn=dn, test=test)
h = F.add2(h, x)
h = act(h)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def small_clbn_self_folding_resnet(image, name='clbn-self-folding-graph-ref'):
h = image
h /= 255.0
h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1), channel_last=True,
with_bias=False, name='first-conv')
a, b = create_scale_bias(1, h.shape[3], axes=[3])
h = a * h + b
h = F.relu(h)
h = F.max_pooling(h, (2, 2), channel_last=True)
h = clbn_self_folding_resblock(h, 2, maps=16, name='cb1')
h = clbn_self_folding_resblock(h, 3, maps=16, name='cb2')
h = clbn_self_folding_resblock(h, 4, maps=16, name='cb3')
h = clbn_self_folding_resblock(h, 5, maps=16, name='cb4')
h = F.average_pooling(h, (2, 2), channel_last=True)
pred = PF.affine(h, 10, name='fc')
return pred
def max_pool_3x3(x, output_filter, scope,
input_node_id, is_reduced, test, is_search):
"""
max pooling (with no spatial downsampling).
"""
if is_reduced and input_node_id < 2:
stride = (2, 2)
else:
stride = (1, 1)
h = F.max_pooling(x, kernel=(3, 3), stride=stride, pad=(1, 1))
with nn.parameter_scope(scope + "bn"):
h = PF.batch_normalization(h, batch_stat=not test,
fix_parameters=is_search)
return h
def small_cl_resnet(image, test=False):
h = image
h /= 255.0
h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1), channel_last=True,
with_bias=False, name='first-cl-conv')
h = PF.batch_normalization(
h, axes=[3], batch_stat=not test, name='first-cl-bn')
h = F.relu(h)
h = F.max_pooling(h, (2, 2), channel_last=True)
h = cl_resblock(h, maps=16, test=test, name='cl-cb1')
h = cl_resblock(h, maps=16, test=test, name='cl-cb2')
h = cl_resblock(h, maps=16, test=test, name='cl-cb3')
h = cl_resblock(h, maps=16, test=test, name='cl-cb4')
h = F.average_pooling(h, (2, 2), channel_last=True)
pred = PF.affine(h, 10, name='cl-fc')
return pred
def convblock(x, nmaps, layer_idx, with_bias, with_bn=False):
h = x
scopenames = ["conv{}".format(_) for _ in layer_idx]
for scopename in scopenames:
with nn.parameter_scope(scopename):
if scopename not in ["conv1", "conv13"] and scopename == scopenames[-1]:
nmaps *= 2
h = PF.convolution(h, nmaps, kernel=(3, 3), pad=(
1, 1), with_bias=with_bias, fix_parameters=finetune)
if with_bn:
h = PF.batch_normalization(
h, batch_stat=not test, fix_parameters=finetune)
h = F.relu(h)
if len(scopenames) != 1 and scopename == scopenames[-2]:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def _make_tree_level1(x,
children,
block,
ochannels,
level,
test,
level_root=False,
stride=1,
channel_last=False):
axes = 3 if channel_last else 1
ichannels = x.shape[axes]
bottom = F.max_pooling(x,
kernel=(stride, stride),
stride=(stride, stride),
channel_last=channel_last) if stride > 1 else x
if ichannels != ochannels:
residual = pf_convolution(bottom,
ochannels, (1, 1),
stride=(1, 1),
pad=None,
with_bias=False,
channel_last=channel_last)
residual = PF.batch_normalization(residual,
axes=[axes],
batch_stat=not test)
else:
residual = bottom
with nn.parameter_scope('block1'):
b1 = block(x,
residual,
ochannels,
stride,
test,
channel_last=channel_last)
with nn.parameter_scope('block2'):
b2 = block(b1, b1, ochannels, 1, test, channel_last=channel_last)
_children = [bottom, b2] if level_root else [b2]
if children:
_children += children
x = root(b1,
_children,
ochannels,
test,
kernel_size=1,
channel_last=channel_last)
return x, bottom
def res_unit(x, scope_name, act=F.relu, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x,
C / 2,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h,
C / 2,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
# Residual -> Relu
if not test:
h = F.dropout(h)
with nn.parameter_scope(scope_name):
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = F.add2(h, x)
h = act(h)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def conv4(x, test=False):
'''
Embedding function
This network is a typical embedding network for the one-shot learning benchmark task.
Args:
x (~nnabla.Variable) : input images.
test (boolean) : whether test or training
Returns:
h (~nnabla.Variable): embedding vector.
'''
h = x
for i in range(4):
h = PF.convolution(h, 64, [3, 3], pad=[1, 1], name='conv' + str(i))
h = PF.batch_normalization(h, batch_stat=not test, name='bn' + str(i))
h = F.relu(h)
h = F.max_pooling(h, [2, 2])
h = F.reshape(h, [h.shape[0], np.prod(h.shape[1:])])
return h
def hg_module(n, x):
with nn.parameter_scope(f"{n - 1}.0.0"):
up1 = ops[n - 1][0](x)
low1 = F.max_pooling(x, kernel=(2, 2), stride=(2, 2))
with nn.parameter_scope(f"{n - 1}.1.0"):
low1 = ops[n - 1][1](low1)
if n > 1:
low2 = hg_module(n - 1, low1)
else:
with nn.parameter_scope(f"{n - 1}.3.0"):
low2 = ops[n - 1][3](low1)
with nn.parameter_scope(f"{n - 1}.2.0"):
low3 = ops[n - 1][2](low2)
up2 = F.interpolate(low3, scale=(2, 2), mode="nearest")
out = up1 + up2
return out
def cnn_model_003_with_cross_attention(ctx, x_list, act=F.relu, test=False):
"""With attention before pooling
"""
with nn.context_scope(ctx):
# Convblock0
h0_list = []
for x in x_list:
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h0_list.append(h)
# Corss attention
ca0 = attention(h0_list[0], h0_list[1], h0_list[1],
div_dim=True, softmax=True)
ca1 = attention(h0_list[1], h0_list[0], h0_list[0],
div_dim=True, softmax=True)
# Maxpooing, Batchnorm, Dropout
h0_list = []
for h in [ca0, ca1]:
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h0_list.append(h)
# Convblock 1
h1_list = []
for h in h0_list:
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h1_list.append(h)
# Corss attention
ca0 = attention(h1_list[0], h1_list[1], h1_list[1],
div_dim=True, softmax=True)
ca1 = attention(h1_list[1], h1_list[0], h1_list[0],
div_dim=True, softmax=True)
# Maxpooing, Batchnorm, Dropout
h1_list = []
for h in [ca0, ca1]:
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h1_list.append(h)
# Convblock 2
h2_list = []
for h in h1_list:
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h2_list.append(h)
# Corss attention
ca0 = attention(h2_list[0], h2_list[1], h2_list[1],
div_dim=True, softmax=True)
ca1 = attention(h2_list[1], h2_list[0], h2_list[0],
div_dim=True, softmax=True)
# Convblock 3
h3_list = []
for h in [ca0, ca1]:
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
h3_list.append(h)
return h3_list