def small_multiple_inputs_outputs_resnet(images, test=False, w_bias=False):
# Branches
outputs = []
for i, image in enumerate(images):
h = image
h /= 255.0
h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1),
with_bias=w_bias, name='first-mo-conv-{}'.format(i))
h = PF.batch_normalization(
h, axes=[1], batch_stat=not test, name='first-mo-bn-{}'.format(i))
h = F.relu(h)
h = F.max_pooling(h, (2, 2))
outputs.append(h)
# Merge branches
z = sum(outputs)
h = multiple_inputs_outputs_resblock(
z, maps=16, w_bias=w_bias, test=test, name='mo-cb1')
h = F.average_pooling(h, (2, 2))
pred1 = PF.affine(h, 10, name='mo-fc1')
h = multiple_inputs_outputs_resblock(
z, maps=16, w_bias=w_bias, test=test, name='mo-cb2')
h = F.average_pooling(h, (2, 2))
pred2 = PF.affine(h, 10, name='mo-fc2')
return [pred1, pred2]
def resblock_d(h, y, scopename,
n_classes, maps, kernel=(3, 3), pad=(1, 1), stride=(1, 1),
downsample=True, test=False, sn=True):
"""Residual block for discriminator"""
s = h
_, c, _, _ = h.shape
assert maps // 2 == c or maps == c
maps1 = c if maps // 2 == c else maps
maps2 = maps
with nn.parameter_scope(scopename):
# LeakyRelu -> Conv
with nn.parameter_scope("conv1"):
#h = F.leaky_relu(h, 0.2, False)
h = F.relu(h, False)
h = convolution(h, maps1, kernel=kernel, pad=pad, stride=stride,
with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2))
# LeakyRelu -> Conv -> Downsample
with nn.parameter_scope("conv2"):
#h = F.leaky_relu(h, 0.2, True)
h = F.relu(h, True)
h = convolution(h, maps2, kernel=kernel, pad=pad, stride=stride,
with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2))
if downsample:
h = F.average_pooling(h, kernel=(2, 2))
# Shortcut: Conv -> Downsample
if c != maps2 or downsample:
with nn.parameter_scope("shortcut"):
s = convolution(s, maps2, kernel=(1, 1), pad=(0, 0), stride=(1, 1),
with_bias=True, sn=sn, test=test)
if downsample:
s = F.average_pooling(s, kernel=(2, 2))
return F.add2(h, s, True)
def factorized_reduction(x, output_filter, scope, test):
"""
Applying spatial reduction to input variable.
Input variable is passed to:
Skip path 1, applied average pooling with stride 2.
Skip path 2, first padded with 0 on the right and bottom,
then shifted by 1 (so that those 0-padded sides will be added,
whereas its shape is the same as the original),
Then these 2 variables are concatenated along the depth dimension.
"""
with nn.parameter_scope(scope):
path1 = F.average_pooling(x, (1, 1), (2, 2))
with nn.parameter_scope("path1_conv"):
path1 = PF.convolution(
path1, output_filter // 2, (1, 1), with_bias=False)
path2 = F.pad(x, (0, 1, 0, 1), mode='constant')
path2 = F.slice(path2, (0, 0, 1, 1))
path2 = F.average_pooling(path2, (1, 1), (2, 2))
with nn.parameter_scope("path2_conv"):
path2 = PF.convolution(
path2, output_filter // 2, (1, 1), with_bias=False)
final_path = F.concatenate(path1, path2, axis=1)
with nn.parameter_scope("reduction_bn"):
final_path = PF.batch_normalization(
final_path, batch_stat=not test)
return final_path
def optblock_d(h, y, scopename,
n_classes, maps, kernel=(3, 3), pad=(1, 1), stride=(1, 1),
downsample=True, test=False, sn=True):
"""Optimized block for discriminator"""
s = h
_, c, _, _ = h.shape
with nn.parameter_scope(scopename):
# Conv
with nn.parameter_scope("conv1"):
h = convolution(h, maps, kernel=kernel, pad=pad, stride=stride,
with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2))
# ReLU -> Conv
with nn.parameter_scope("conv2"):
#h = F.leaky_relu(h, 0.2, True)
h = F.relu(h, True)
h = convolution(h, maps, kernel=kernel, pad=pad, stride=stride,
with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2))
if downsample:
h = F.average_pooling(h, kernel=(2, 2))
# Shortcut: Conv -> Downsample
with nn.parameter_scope("shortcut"):
if downsample:
s = F.average_pooling(s, kernel=(2, 2))
s = convolution(s, maps, kernel=(1, 1), pad=(0, 0), stride=(1, 1),
with_bias=True, sn=sn, test=test)
return F.add2(h, s, True)
def res_block(x, dim_out, kernel, pad, scope, training):
r"""Residual block.
Args:
x (nn.Variable): Input variable.
dim_out (int): Number of output channels.
kernel (tuple of int): Kernel size.
pad (tuple of int): Padding.
scope (str): The scope.
Returns:
nn.Variable: Output variable.
"""
dim_in = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('shortcut'):
sc = x
if dim_in != dim_out:
sc = wn_conv(sc, dim_out, (1, ), with_bias=False, name='conv')
sc = F.average_pooling(sc, kernel=(1, 2))
with nn.parameter_scope('residual'):
x = F.leaky_relu(x, 0.2)
x = wn_conv(x, dim_in, kernel, pad, name='conv1')
x = F.average_pooling(x, kernel=(1, 2))
x = F.leaky_relu(x, 0.2, inplace=True)
x = wn_conv(x, dim_out, (1, ), name='conv2')
return sc + x
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act,
test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h_branch = h
# Convblock 3
h = conv_unit(h_branch,
"conv23",
10,
k=1,
s=1,
p=0,
act=act,
test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u0 = F.average_pooling(u0, (6, 6))
with nn.parameter_scope("u0bn"):
u0 = PF.batch_normalization(u0, batch_stat=not test)
log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:])))
# Uncertainty for uncertainty
u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test)
u1 = F.average_pooling(u1, (6, 6))
with nn.parameter_scope("u1bn"):
u1 = PF.batch_normalization(u1, batch_stat=not test)
log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:])))
return pred, log_var, log_s
def mnist_lenet_feature(image, test=False):
"""
Construct LeNet for MNIST.
"""
c1 = F.elu(PF.convolution(image, 20, (5, 5), name='conv1'))
c1 = F.average_pooling(c1, (2, 2))
c2 = F.elu(PF.convolution(c1, 50, (5, 5), name='conv2'))
c2 = F.average_pooling(c2, (2, 2))
c3 = F.elu(PF.affine(c2, 500, name='fc3'))
c4 = PF.affine(c3, 10, name='fc4')
c5 = PF.affine(c4, 2, name='fc_embed')
return c5
def mnist_lenet_feature(image, test=False):
"""
Construct LeNet for MNIST.
"""
c1 = F.elu(PF.convolution(image, 20, (5, 5), name='conv1'))
c1 = F.average_pooling(c1, (2, 2))
c2 = F.elu(PF.convolution(c1, 50, (5, 5), name='conv2'))
c2 = F.average_pooling(c2, (2, 2))
c3 = F.elu(PF.affine(c2, 500, name='fc3'))
c4 = PF.affine(c3, 10, name='fc4')
c5 = PF.affine(c4, 2, name='fc_embed')
return c5
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h_branch = h
# Convblock 3
h = conv_unit(h_branch, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u0 = F.average_pooling(u0, (6, 6))
with nn.parameter_scope("u0bn"):
u0 = PF.batch_normalization(u0, batch_stat=not test)
log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:])))
# Uncertainty for uncertainty
u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test)
u1 = F.average_pooling(u1, (6, 6))
with nn.parameter_scope("u1bn"):
u1 = PF.batch_normalization(u1, batch_stat=not test)
log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:])))
return pred, log_var, log_s
def cnn_model_003(ctx, h, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
if not test:
b, c, s, s = h.shape
h = F.image_augmentation(h, (c, s, s),
min_scale=1.0, max_scale=1.5,
angle=0.5, aspect_ratio=1.3, distortion=0.2,
flip_lr=True)
# Convblock0
h = conv_unit(h, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (6, 6))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def mnist_binary_weight_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (Binary Weight Network version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_weight_convolution(image, 16, (5, 5))
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_weight_convolution(c1, 16, (5, 5))
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = F.elu(PF.binary_weight_affine(c2, 50))
with nn.parameter_scope("fc4"):
c4 = PF.binary_weight_affine(c3, 10)
return c4
def mnist_binary_weight_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (Binary Weight Network version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_weight_convolution(image, 16, (5, 5))
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_weight_convolution(c1, 16, (5, 5))
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = F.elu(PF.binary_weight_affine(c2, 50))
with nn.parameter_scope("fc4"):
c4 = PF.binary_weight_affine(c3, 10)
return c4
def downblock(x, out_features, norm=False, kernel_size=4, pool=False, sn=False, test=False):
out = x
if sn:
def apply_w(w): return PF.spectral_norm(w, dim=0, test=test)
else:
apply_w = None
inmaps, outmaps = out.shape[1], out_features
k_w = I.calc_normal_std_he_forward(
inmaps, outmaps, kernel=(kernel_size, kernel_size)) / np.sqrt(2.)
k_b = I.calc_normal_std_he_forward(inmaps, outmaps) / np.sqrt(2.)
w_init = I.UniformInitializer((-k_w, k_w))
b_init = I.UniformInitializer((-k_b, k_b))
out = PF.convolution(out, out_features,
kernel=(kernel_size, kernel_size), pad=(0, 0),
stride=(1, 1), w_init=w_init, b_init=b_init,
apply_w=apply_w)
if norm:
out = PF.instance_normalization(out)
out = F.leaky_relu(out, 0.2, inplace=True)
if pool:
out = F.average_pooling(out, kernel=(2, 2))
return out
def transition_cnn(self, h, pre_resolution, nxt_resolution, pre_channel,
nxt_channel, alpha):
lhs = self.from_RGB(F.average_pooling(h, kernel=(2, 2)),
pre_resolution, pre_channel)
rhs = alpha * self.cnn(self.from_RGB(h, nxt_resolution, nxt_channel),
nxt_resolution, nxt_channel, pre_channel)
return (1 - alpha) * lhs + alpha * rhs
def DownsampleComp(h, nmap_out, scope_name):
with nn.parameter_scope(scope_name):
def sn_w(w):
return PF.spectral_norm(w, dim=0)
# Main
h0 = PF.convolution(h,
nmap_out, (4, 4),
stride=(2, 2),
pad=(1, 1),
apply_w=sn_w,
with_bias=False,
name="main_conv1")
h0 = PF.batch_normalization(h0, name="bn_main1")
h0 = F.leaky_relu(h0, 0.2)
h0 = PF.convolution(h0,
nmap_out, (3, 3),
pad=(1, 1),
apply_w=sn_w,
with_bias=False,
name="main_conv2")
h0 = PF.batch_normalization(h0, name="bn_main2")
h0 = F.leaky_relu(h0, 0.2)
# Direct
h1 = F.average_pooling(h, (2, 2), stride=(2, 2))
h1 = PF.convolution(h1,
nmap_out, (1, 1),
apply_w=sn_w,
with_bias=False,
name="direct_conv1")
h1 = PF.batch_normalization(h1, name="direct_bn1")
h1 = F.leaky_relu(h1, 0.2)
return (h0 + h1) / 2.0
def __call__(self, x):
# First conv
h = self.conv_bn_act(x, int(self.maps0 * self.depth_mul),
stride=(2, 2), act="hswish", name="first-conv")
# Inverted residual blocks
for i, elm in enumerate(self.settings):
maps, kernel, stride, ef, act, se = elm
maps = round(maps * self.depth_mul)
name = "mbconv-{:03d}".format(i)
h = self.inverted_residual(
h, maps, kernel, stride, ef, act, se, name=name)
# Conv -> Avepool -> Conv
h = self.conv_bn_act(h, int(self.maps1 * self.depth_mul), (1, 1), act="hswish",
name="last-conv-1")
pool_shape = get_spatial_shape(x.shape, self.channel_last)
h = F.average_pooling(h, pool_shape, channel_last=self.channel_last)
h = self.conv_act(h, int(self.maps2 * self.depth_mul), (1, 1), act="hswish",
name="last-conv-2")
# Classifier
if not self.test:
h = F.dropout(h, 0.2)
h = PF.affine(h, self.num_classes,
w_init=I.NormalInitializer(0.01), name="linear")
return h, {}
def shortcut(x, ochannels, stride, shortcut_type, test):
ichannels = x.shape[1]
use_conv = shortcut_type.lower() == 'c'
if ichannels != ochannels:
assert (ichannels * 2 == ochannels) or (ichannels * 4 == ochannels)
if shortcut_type.lower() == 'b':
use_conv = True
if use_conv:
# Convolution does everything.
# Matching channels, striding.
with nn.parameter_scope("shortcut_conv"):
x = PF.convolution(x,
ochannels, (1, 1),
stride=stride,
with_bias=False)
x = PF.batch_normalization(x, batch_stat=not test)
else:
if stride != (1, 1):
# Stride
x = F.average_pooling(x, (1, 1), stride)
if ichannels != ochannels:
# Zero-padding to channel axis
ishape = x.shape
zeros = F.constant(0, (ishape[0], ochannels - ichannels) +
ishape[-2:])
x = F.concatenate(x, zeros, axis=1)
return x
def cifar10_resnet23_prediction(ctx, image, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = bn_dropout(h, "bn_dropout1", test)
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = bn_dropout(h, "bn_dropout2", test)
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = bn_dropout(h, "bn_dropout3", test)
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def feature_view(ctx, h):
"""feature connection only"""
with nn.parameter_scope("fv"):
b, c, s, s = h.shape
h = F.average_pooling(h, (s, s))
h = PF.affine(h, 10)
return h
def cnn_model_003(ctx, x, act=F.relu, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 28 -> 14
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 14 -> 7
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act,
test=test) # 7 -> 5
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
# Convblock 3
h = F.average_pooling(h, (5, 5))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def backward_impl(self, inputs, outputs, prop_down, accum):
# inputs: [inputs_fwd_graph] + [inputs_bwd_graph] or
# [inputs_fwd_graph] + [outputs_fwd_graph] + [inputs_bwd_graph]
# Args
kernel = self.forward_func.info.args["kernel"]
stride = self.forward_func.info.args["stride"]
ignore_border = self.forward_func.info.args["ignore_border"]
pad = self.forward_func.info.args["pad"]
channel_last = self.forward_func.info.args["channel_last"]
including_pad = self.forward_func.info.args["including_pad"]
# TODO: BHWC
assert channel_last == False, "`channel_last = False` is only supported now."
# Inputs
x0 = inputs[0].data
dy = inputs[1].data
# Outputs
dx0 = outputs[0].data
# Grads of inputs
g_x0 = inputs[0].grad
g_dy = inputs[1].grad
# Grads of outputs
g_dx0 = outputs[0].grad
# Computation
if prop_down[1]:
g_dy_ = F.average_pooling(g_dx0, kernel, stride, ignore_border, pad,
channel_last, including_pad)
if accum[1]:
g_dy += g_dy_
else:
g_dy.copy_from(g_dy_)
def __call__(self, x):
# First conv
h = self.conv_bn_relu6(x, int(self.init_maps * self.depth_mul),
stride=(2, 2), name="first-conv")
# Inverted residual blocks
for i, elm in enumerate(self.settings):
t, c, n, s = elm
# TODO: where to multiply depth_mul
c = round(c * self.depth_mul)
mbconv_s = partial(self.inverted_residual,
maps=c, stride=(s, s), ef=t)
mbconv_1 = partial(self.inverted_residual,
maps=c, stride=(1, 1), ef=t)
for j in range(n):
name = "mbconv-{:02d}-{:02d}".format(i, j)
h = mbconv_s(h, name=name) if j == 0 else mbconv_1(
h, name=name)
# Last conv
h = self.conv_bn_relu6(h, int(1280 * self.depth_mul),
kernel=(1, 1), name="last-conv")
# Classifier
if not self.test:
h = F.dropout(h, 0.2)
pool_shape = get_spatial_shape(x.shape, self.channel_last)
h = F.average_pooling(h, pool_shape, channel_last=self.channel_last)
h = PF.affine(h, self.num_classes,
w_init=I.NormalInitializer(0.01), name="linear")
return h, {}
def shuffle_unit(x, scope_name, dn=False):
"""
Figure. 2 (b) and (c) in https://arxiv.org/pdf/1707.01083.pdf
"""
C = x.shape[1]
h = x
with nn.parameter_scope(scope_name):
with nn.parameter_scope("gconv1"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
group=groups,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h, True)
with nn.parameter_scope("shuffle"): # no meaning but semantics
h = shuffle(h)
with nn.parameter_scope("dconv"):
stride = (2, 2) if dn else (1, 1)
h = PF.depthwise_convolution(h, kernel=(3, 3), pad=(1, 1),
stride=stride,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
with nn.parameter_scope("gconv2"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
group=groups,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
s = F.average_pooling(x, (2, 2)) if dn else x
h = F.concatenate(*[h, s], axis=1) if dn else h + s
h = F.relu(h)
return h
def call(self, input):
out = F.average_pooling(input,
kernel=self._kernel,
stride=self._stride,
pad=self._pad,
channel_last=self._channel_last)
return out
def resnet_model(ctx, x, inmaps=64, act=F.relu, test=False):
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(x,
inmaps,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
h = res_unit(h, "conv2", act, False) # -> 32x32
h = res_unit(h, "conv3", act, True) # -> 16x16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv4", act, False) # -> 16x16
h = res_unit(h, "conv5", act, True) # -> 8x8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv6", act, False) # -> 8x8
h = res_unit(h, "conv7", act, True) # -> 4x4
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv8", act, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, 10)
return pred
def cnn_model_003(ctx, x, act=F.relu, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
# Convblock 3
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def cnn(x):
"""Unnecessarily Deep CNN.
Args:
x : Variable, shape (B, 1, 8, 8)
Returns:
y : Variable, shape (B, 10)
"""
with nn.parameter_scope("cnn"): # Parameter scope can be nested
with nn.parameter_scope("conv1"):
h = F.tanh(
PF.batch_normalization(PF.convolution(x, 64, (3, 3), pad=(1, 1)))
)
for i in range(10): # unnecessarily deep
with nn.parameter_scope("conv{}".format(i + 2)):
h = F.tanh(
PF.batch_normalization(PF.convolution(h, 128, (3, 3), pad=(1, 1)))
)
with nn.parameter_scope("conv_last"):
h = F.tanh(
PF.batch_normalization(PF.convolution(h, 512, (3, 3), pad=(1, 1)))
)
h = F.average_pooling(h, (2, 2))
with nn.parameter_scope("fc"):
h = F.tanh(PF.affine(h, 1024))
with nn.parameter_scope("classifier"):
y = PF.affine(h, 10)
return y
def __call__(self, x):
h = self.conv_bn_relu(x, 32, stride=(2, 2), name="first-conv")
h = self.depthwise_separable_conv(
h, 64, stride=(1, 1), name="conv-ds-1")
h = self.depthwise_separable_conv(
h, 128, stride=(2, 2), name="conv-ds-2")
h = self.depthwise_separable_conv(
h, 128, stride=(1, 1), name="conv-ds-3")
h = self.depthwise_separable_conv(
h, 256, stride=(2, 2), name="conv-ds-4")
h = self.depthwise_separable_conv(
h, 256, stride=(1, 1), name="conv-ds-5")
h = self.depthwise_separable_conv(
h, 512, stride=(2, 2), name="conv-ds-6")
h = self.depthwise_separable_conv(
h, 512, stride=(1, 1), name="conv-ds-7")
h = self.depthwise_separable_conv(
h, 512, stride=(1, 1), name="conv-ds-8")
h = self.depthwise_separable_conv(
h, 512, stride=(1, 1), name="conv-ds-9")
h = self.depthwise_separable_conv(
h, 512, stride=(1, 1), name="conv-ds-10")
h = self.depthwise_separable_conv(
h, 512, stride=(1, 1), name="conv-ds-11")
h = self.depthwise_separable_conv(
h, 1024, stride=(2, 2), name="conv-ds-12")
h = self.depthwise_separable_conv(
h, 1024, stride=(1, 1), name="conv-ds-13")
pool_shape = get_spatial_shape(x.shape, self.channel_last)
h = F.average_pooling(h, pool_shape, channel_last=self.channel_last)
h = PF.affine(h, self.num_classes,
w_init=I.NormalInitializer(0.01), name="linear")
return h, {}
def discriminator(x, scopename="discriminator", maps=128):
with nn.parameter_scope(scopename):
h = PF.convolution(x, maps * 1, (3, 3), name="conv-1")
h = F.average_pooling(h, (2, 2)) # 32x32 -> 16x16
h = F.leaky_relu(h, 0.01)
h = PF.convolution(h, maps * 2, (3, 3), name="conv-2")
h = F.average_pooling(h, (2, 2)) # 16x16 -> 8x8
h = F.leaky_relu(h, 0.01)
h = PF.convolution(h, maps * 4, (3, 3), name="conv-3")
h = F.average_pooling(h, (2, 2)) # 8x8 -> 4x4
h = F.leaky_relu(h, 0.01)
h = PF.affine(h, 1)
return h
def shortcut(x, ochannels, stride, shortcut_type, test, channel_last=False):
axes = [3 if channel_last else 1]
ichannels = x.shape[axes[0]]
use_conv = shortcut_type.lower() == 'c'
if ichannels != ochannels:
assert (ichannels * 2 == ochannels) or (ichannels * 4 == ochannels)
if shortcut_type.lower() == 'b':
use_conv = True
if use_conv:
# Convolution does everything.
# Matching channels, striding.
with nn.parameter_scope("shortcut_conv"):
x = pf_convolution(x,
ochannels, (1, 1),
stride=stride,
channel_last=channel_last)
x = PF.batch_normalization(x, axes=axes, batch_stat=not test)
else:
if stride != (1, 1):
# Stride
x = F.average_pooling(x, (1, 1), stride, channel_last=channel_last)
if ichannels != ochannels:
# Zero-padding to channel axis
ishape = x.shape
if channel_last:
zero_shape = (ishape[0],) + ishape[1:3] + \
(ochannels - ichannels,)
else:
zero_shape = (ishape[0], ochannels - ichannels) + ishape[-2:]
zeros = F.constant(zero_shape, 0)
x = F.concatenate(x, zeros, axis=1)
return x
def resnet_model(ctx, x, inmaps=64, act=F.relu, test=False):
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(x, inmaps, kernel=(3, 3), pad=(1, 1), with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
h = res_unit(h, "conv2", act, False) # -> 32x32
h = res_unit(h, "conv3", act, True) # -> 16x16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv4", act, False) # -> 16x16
h = res_unit(h, "conv5", act, True) # -> 8x8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv6", act, False) # -> 8x8
h = res_unit(h, "conv7", act, True) # -> 4x4
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv8", act, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, 10)
return pred
def fan(x, num_modules=1, test=True):
x = PF.convolution(x,
64,
kernel=(7, 7),
stride=(2, 2),
pad=(3, 3),
name='conv1')
x = PF.batch_normalization(x, batch_stat=not test, name='bn1')
x = F.relu(x, True)
with nn.parameter_scope('conv2'):
x = conv_block(x, 64, 128)
x = F.average_pooling(x, (2, 2), stride=(2, 2))
with nn.parameter_scope('conv3'):
x = conv_block(x, 128, 128)
with nn.parameter_scope('conv4'):
x = conv_block(x, 128, 256)
previous = x
outputs = []
for i in range(num_modules):
with nn.parameter_scope('m' + str(i)):
hg = hour_glass(previous, 4, 256)
ll = hg
with nn.parameter_scope('top_m_' + str(i)):
ll = conv_block(ll, 256, 256)
ll = PF.convolution(ll,
256,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='conv_last' + str(i))
ll = PF.batch_normalization(ll,
batch_stat=not test,
name='bn_end' + str(i))
ll = F.relu(ll, True)
# Predict heatmaps
tmp_out = PF.convolution(ll,
68,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='l' + str(i))
outputs.append(tmp_out)
if i < num_modules - 1:
ll = PF.convolution(ll,
256,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='bl' + str(i))
tmp_out_ = PF.convolution(tmp_out,
256,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='al' + str(i))
previous = previous + ll + tmp_out_
return outputs
def cnn(self, h, resolution, ch_in, ch_out):
"""CNN block
The following operations are performed two times.
1. Conv
2. Layer normalization
3. Leaky relu
"""
with nn.parameter_scope("phase_{}".format(resolution)):
with nn.parameter_scope("conv1"):
h = conv(h,
ch_in,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=not self.use_ln,
use_wscale=self.use_wscale,
use_he_backward=self.use_he_backward)
h = LN(h, use_ln=self.use_ln)
h = self.activation(h)
with nn.parameter_scope("conv2"):
h = conv(h,
ch_out,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=not self.use_ln,
use_wscale=self.use_wscale,
use_he_backward=self.use_he_backward)
h = LN(h, use_ln=self.use_ln)
h = self.activation(h)
h = F.average_pooling(h, kernel=(2, 2))
return h
def csc(x, scope_name, dn=False):
C = x.shape[1]
h = x
with nn.parameter_scope(scope_name):
with nn.parameter_scope("conv1"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h, True)
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
with nn.parameter_scope("shift"): # no meaning but semantics
h = shift(h)
with nn.parameter_scope("conv2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h, True)
stride = (2, 2) if dn else (1, 1)
if p > 0:
h = F.dropout(h, p=0.5) if not test else h
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
stride=stride,
with_bias=False)
s = F.average_pooling(x, (2, 2)) if dn else x
return h + s
def network(x, y, test=False):
# Input:x -> 3,64,64
# AveragePooling -> 3,12,21
h = F.average_pooling(x, (5, 3), (5, 3))
# LeakyReLU_2
h = F.leaky_relu(h, 0.1, True)
# Convolution_2 -> 20,13,21
h = PF.convolution(h, 20, (2, 3), (1, 1), name='Convolution_2')
# BatchNormalization
h = PF.batch_normalization(h, (1, ),
0.9,
0.0001,
not test,
name='BatchNormalization')
# ReLU
h = F.relu(h, True)
# DepthwiseConvolution
h = PF.depthwise_convolution(h, (5, 5), (2, 2),
name='DepthwiseConvolution')
# MaxPooling_2 -> 20,6,7
h = F.max_pooling(h, (2, 3), (2, 3))
# LeakyReLU
h = F.leaky_relu(h, 0.1, True)
# Affine -> 2
h = PF.affine(h, (2, ), name='Affine')
# Softmax
h = F.softmax(h)
return h
def average_pool(x, output_filter, scope, test):
"""
average pooling (with no spatial downsampling).
"""
with nn.parameter_scope(scope):
h = conv1x1(x, output_filter, scope, test)
h = F.average_pooling(h, (3, 3), (1, 1), pad=(1, 1))
return h
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 28 -> 14
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 14 -> 7
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 7 -> 5
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (5, 5))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (5, 5))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def mnist_binary_connect_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (BinaryNet version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_connect_convolution(image, 16, (5, 5))
c1 = PF.batch_normalization(c1, batch_stat=not test)
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_connect_convolution(c1, 16, (5, 5))
c2 = PF.batch_normalization(c2, batch_stat=not test)
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = PF.binary_connect_affine(c2, 50)
c3 = PF.batch_normalization(c3, batch_stat=not test)
c3 = F.elu(c3)
with nn.parameter_scope("fc4"):
c4 = PF.binary_connect_affine(c3, 10)
c4 = PF.batch_normalization(c4, batch_stat=not test)
return c4
def mnist_resnet_prediction(image, test=False):
"""
Construct ResNet for MNIST.
"""
image /= 255.0
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def res_unit(x, scope):
C = x.shape[1]
with nn.parameter_scope(scope):
with nn.parameter_scope('conv1'):
h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False)))
with nn.parameter_scope('conv2'):
h = F.elu(
bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False)))
with nn.parameter_scope('conv3'):
h = bn(PF.convolution(h, C, (1, 1), with_bias=False))
return F.elu(F.add2(h, x, inplace=True))
# Conv1 --> 64 x 32 x 32
with nn.parameter_scope("conv1"):
c1 = F.elu(
bn(PF.convolution(image, 64, (3, 3), pad=(3, 3), with_bias=False)))
# Conv2 --> 64 x 16 x 16
c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2))
# Conv3 --> 64 x 8 x 8
c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2))
# Conv4 --> 64 x 8 x 8
c4 = res_unit(c3, "conv4")
# Conv5 --> 64 x 4 x 4
c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2))
# Conv5 --> 64 x 4 x 4
c6 = res_unit(c5, "conv6")
pl = F.average_pooling(c6, (4, 4))
with nn.parameter_scope("classifier"):
y = PF.affine(pl, 10)
return y
def cifar100_resnet23_prediction(image,
ctx, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
rng = np.random.RandomState(0)
nmaps = 384
ncls = 100
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
# Preprocess
if not test:
image = F.image_augmentation(image, contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
w_init = UniformInitializer(
calc_uniform_lim_glorot(3, nmaps, kernel=(3, 3)),
rng=rng)
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", rng, False) # -> 32x32
h = res_unit(h, "conv3", rng, True) # -> 16x16
h = res_unit(h, "conv4", rng, False) # -> 16x16
h = res_unit(h, "conv5", rng, True) # -> 8x8
h = res_unit(h, "conv6", rng, False) # -> 8x8
h = res_unit(h, "conv7", rng, True) # -> 4x4
h = res_unit(h, "conv8", rng, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
w_init = UniformInitializer(
calc_uniform_lim_glorot(int(np.prod(h.shape[1:])), ncls, kernel=(1, 1)), rng=rng)
pred = PF.affine(h, ncls, w_init=w_init)
return pred
def cnn_model_003_with_cross_attention(ctx, x_list, act=F.relu, test=False):
"""With attention before pooling
"""
with nn.context_scope(ctx):
# Convblock0
h0_list = []
for x in x_list:
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h0_list.append(h)
# Corss attention
ca0 = attention(h0_list[0], h0_list[1], h0_list[1],
div_dim=True, softmax=True)
ca1 = attention(h0_list[1], h0_list[0], h0_list[0],
div_dim=True, softmax=True)
# Maxpooing, Batchnorm, Dropout
h0_list = []
for h in [ca0, ca1]:
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h0_list.append(h)
# Convblock 1
h1_list = []
for h in h0_list:
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h1_list.append(h)
# Corss attention
ca0 = attention(h1_list[0], h1_list[1], h1_list[1],
div_dim=True, softmax=True)
ca1 = attention(h1_list[1], h1_list[0], h1_list[0],
div_dim=True, softmax=True)
# Maxpooing, Batchnorm, Dropout
h1_list = []
for h in [ca0, ca1]:
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h1_list.append(h)
# Convblock 2
h2_list = []
for h in h1_list:
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h2_list.append(h)
# Corss attention
ca0 = attention(h2_list[0], h2_list[1], h2_list[1],
div_dim=True, softmax=True)
ca1 = attention(h2_list[1], h2_list[0], h2_list[0],
div_dim=True, softmax=True)
# Convblock 3
h3_list = []
for h in [ca0, ca1]:
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
h3_list.append(h)
return h3_list