def small_multiple_inputs_outputs_resnet(images, test=False, w_bias=False): # Branches outputs = [] for i, image in enumerate(images): h = image h /= 255.0 h = PF.convolution(h, 16, kernel=(3, 3), pad=(1, 1), with_bias=w_bias, name='first-mo-conv-{}'.format(i)) h = PF.batch_normalization( h, axes=[1], batch_stat=not test, name='first-mo-bn-{}'.format(i)) h = F.relu(h) h = F.max_pooling(h, (2, 2)) outputs.append(h) # Merge branches z = sum(outputs) h = multiple_inputs_outputs_resblock( z, maps=16, w_bias=w_bias, test=test, name='mo-cb1') h = F.average_pooling(h, (2, 2)) pred1 = PF.affine(h, 10, name='mo-fc1') h = multiple_inputs_outputs_resblock( z, maps=16, w_bias=w_bias, test=test, name='mo-cb2') h = F.average_pooling(h, (2, 2)) pred2 = PF.affine(h, 10, name='mo-fc2') return [pred1, pred2]
def resblock_d(h, y, scopename, n_classes, maps, kernel=(3, 3), pad=(1, 1), stride=(1, 1), downsample=True, test=False, sn=True): """Residual block for discriminator""" s = h _, c, _, _ = h.shape assert maps // 2 == c or maps == c maps1 = c if maps // 2 == c else maps maps2 = maps with nn.parameter_scope(scopename): # LeakyRelu -> Conv with nn.parameter_scope("conv1"): #h = F.leaky_relu(h, 0.2, False) h = F.relu(h, False) h = convolution(h, maps1, kernel=kernel, pad=pad, stride=stride, with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2)) # LeakyRelu -> Conv -> Downsample with nn.parameter_scope("conv2"): #h = F.leaky_relu(h, 0.2, True) h = F.relu(h, True) h = convolution(h, maps2, kernel=kernel, pad=pad, stride=stride, with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2)) if downsample: h = F.average_pooling(h, kernel=(2, 2)) # Shortcut: Conv -> Downsample if c != maps2 or downsample: with nn.parameter_scope("shortcut"): s = convolution(s, maps2, kernel=(1, 1), pad=(0, 0), stride=(1, 1), with_bias=True, sn=sn, test=test) if downsample: s = F.average_pooling(s, kernel=(2, 2)) return F.add2(h, s, True)
def factorized_reduction(x, output_filter, scope, test): """ Applying spatial reduction to input variable. Input variable is passed to: Skip path 1, applied average pooling with stride 2. Skip path 2, first padded with 0 on the right and bottom, then shifted by 1 (so that those 0-padded sides will be added, whereas its shape is the same as the original), Then these 2 variables are concatenated along the depth dimension. """ with nn.parameter_scope(scope): path1 = F.average_pooling(x, (1, 1), (2, 2)) with nn.parameter_scope("path1_conv"): path1 = PF.convolution( path1, output_filter // 2, (1, 1), with_bias=False) path2 = F.pad(x, (0, 1, 0, 1), mode='constant') path2 = F.slice(path2, (0, 0, 1, 1)) path2 = F.average_pooling(path2, (1, 1), (2, 2)) with nn.parameter_scope("path2_conv"): path2 = PF.convolution( path2, output_filter // 2, (1, 1), with_bias=False) final_path = F.concatenate(path1, path2, axis=1) with nn.parameter_scope("reduction_bn"): final_path = PF.batch_normalization( final_path, batch_stat=not test) return final_path
def optblock_d(h, y, scopename, n_classes, maps, kernel=(3, 3), pad=(1, 1), stride=(1, 1), downsample=True, test=False, sn=True): """Optimized block for discriminator""" s = h _, c, _, _ = h.shape with nn.parameter_scope(scopename): # Conv with nn.parameter_scope("conv1"): h = convolution(h, maps, kernel=kernel, pad=pad, stride=stride, with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2)) # ReLU -> Conv with nn.parameter_scope("conv2"): #h = F.leaky_relu(h, 0.2, True) h = F.relu(h, True) h = convolution(h, maps, kernel=kernel, pad=pad, stride=stride, with_bias=True, sn=sn, test=test, init_scale=np.sqrt(2)) if downsample: h = F.average_pooling(h, kernel=(2, 2)) # Shortcut: Conv -> Downsample with nn.parameter_scope("shortcut"): if downsample: s = F.average_pooling(s, kernel=(2, 2)) s = convolution(s, maps, kernel=(1, 1), pad=(0, 0), stride=(1, 1), with_bias=True, sn=sn, test=test) return F.add2(h, s, True)
def res_block(x, dim_out, kernel, pad, scope, training): r"""Residual block. Args: x (nn.Variable): Input variable. dim_out (int): Number of output channels. kernel (tuple of int): Kernel size. pad (tuple of int): Padding. scope (str): The scope. Returns: nn.Variable: Output variable. """ dim_in = x.shape[1] with nn.parameter_scope(scope): with nn.parameter_scope('shortcut'): sc = x if dim_in != dim_out: sc = wn_conv(sc, dim_out, (1, ), with_bias=False, name='conv') sc = F.average_pooling(sc, kernel=(1, 2)) with nn.parameter_scope('residual'): x = F.leaky_relu(x, 0.2) x = wn_conv(x, dim_in, kernel, pad, name='conv1') x = F.average_pooling(x, kernel=(1, 2)) x = F.leaky_relu(x, 0.2, inplace=True) x = wn_conv(x, dim_out, (1, ), name='conv2') return sc + x
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False): with nn.context_scope(ctx): # Convblock0 h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 32 -> 16 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test and do: h = F.dropout(h) # Convblock 1 h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 16 -> 8 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test and do: h = F.dropout(h) # Convblock 2 h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6 h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test) h_branch = h # Convblock 3 h = conv_unit(h_branch, "conv23", 10, k=1, s=1, p=0, act=act, test=test) h = F.average_pooling(h, (6, 6)) with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:]))) # Uncertainty u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test) u0 = F.average_pooling(u0, (6, 6)) with nn.parameter_scope("u0bn"): u0 = PF.batch_normalization(u0, batch_stat=not test) log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:]))) # Uncertainty for uncertainty u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test) u1 = F.average_pooling(u1, (6, 6)) with nn.parameter_scope("u1bn"): u1 = PF.batch_normalization(u1, batch_stat=not test) log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:]))) return pred, log_var, log_s
def mnist_lenet_feature(image, test=False): """ Construct LeNet for MNIST. """ c1 = F.elu(PF.convolution(image, 20, (5, 5), name='conv1')) c1 = F.average_pooling(c1, (2, 2)) c2 = F.elu(PF.convolution(c1, 50, (5, 5), name='conv2')) c2 = F.average_pooling(c2, (2, 2)) c3 = F.elu(PF.affine(c2, 500, name='fc3')) c4 = PF.affine(c3, 10, name='fc4') c5 = PF.affine(c4, 2, name='fc_embed') return c5
def cnn_model_003(ctx, h, act=F.elu, do=True, test=False): with nn.context_scope(ctx): if not test: b, c, s, s = h.shape h = F.image_augmentation(h, (c, s, s), min_scale=1.0, max_scale=1.5, angle=0.5, aspect_ratio=1.3, distortion=0.2, flip_lr=True) # Convblock0 h = conv_unit(h, "conv00", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 32 -> 16 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test and do: h = F.dropout(h) # Convblock 1 h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 16 -> 8 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test and do: h = F.dropout(h) # Convblock 2 h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6 h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test) u = h # Convblock 3 h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test) h = F.average_pooling(h, (6, 6)) with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:]))) # Uncertainty u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test) u = F.average_pooling(u, (6, 6)) with nn.parameter_scope("u0bn"): u = PF.batch_normalization(u, batch_stat=not test) log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:]))) return pred, log_var
def mnist_binary_weight_lenet_prediction(image, test=False): """ Construct LeNet for MNIST (Binary Weight Network version). """ with nn.parameter_scope("conv1"): c1 = PF.binary_weight_convolution(image, 16, (5, 5)) c1 = F.elu(F.average_pooling(c1, (2, 2))) with nn.parameter_scope("conv2"): c2 = PF.binary_weight_convolution(c1, 16, (5, 5)) c2 = F.elu(F.average_pooling(c2, (2, 2))) with nn.parameter_scope("fc3"): c3 = F.elu(PF.binary_weight_affine(c2, 50)) with nn.parameter_scope("fc4"): c4 = PF.binary_weight_affine(c3, 10) return c4
def downblock(x, out_features, norm=False, kernel_size=4, pool=False, sn=False, test=False): out = x if sn: def apply_w(w): return PF.spectral_norm(w, dim=0, test=test) else: apply_w = None inmaps, outmaps = out.shape[1], out_features k_w = I.calc_normal_std_he_forward( inmaps, outmaps, kernel=(kernel_size, kernel_size)) / np.sqrt(2.) k_b = I.calc_normal_std_he_forward(inmaps, outmaps) / np.sqrt(2.) w_init = I.UniformInitializer((-k_w, k_w)) b_init = I.UniformInitializer((-k_b, k_b)) out = PF.convolution(out, out_features, kernel=(kernel_size, kernel_size), pad=(0, 0), stride=(1, 1), w_init=w_init, b_init=b_init, apply_w=apply_w) if norm: out = PF.instance_normalization(out) out = F.leaky_relu(out, 0.2, inplace=True) if pool: out = F.average_pooling(out, kernel=(2, 2)) return out
def transition_cnn(self, h, pre_resolution, nxt_resolution, pre_channel, nxt_channel, alpha): lhs = self.from_RGB(F.average_pooling(h, kernel=(2, 2)), pre_resolution, pre_channel) rhs = alpha * self.cnn(self.from_RGB(h, nxt_resolution, nxt_channel), nxt_resolution, nxt_channel, pre_channel) return (1 - alpha) * lhs + alpha * rhs
def DownsampleComp(h, nmap_out, scope_name): with nn.parameter_scope(scope_name): def sn_w(w): return PF.spectral_norm(w, dim=0) # Main h0 = PF.convolution(h, nmap_out, (4, 4), stride=(2, 2), pad=(1, 1), apply_w=sn_w, with_bias=False, name="main_conv1") h0 = PF.batch_normalization(h0, name="bn_main1") h0 = F.leaky_relu(h0, 0.2) h0 = PF.convolution(h0, nmap_out, (3, 3), pad=(1, 1), apply_w=sn_w, with_bias=False, name="main_conv2") h0 = PF.batch_normalization(h0, name="bn_main2") h0 = F.leaky_relu(h0, 0.2) # Direct h1 = F.average_pooling(h, (2, 2), stride=(2, 2)) h1 = PF.convolution(h1, nmap_out, (1, 1), apply_w=sn_w, with_bias=False, name="direct_conv1") h1 = PF.batch_normalization(h1, name="direct_bn1") h1 = F.leaky_relu(h1, 0.2) return (h0 + h1) / 2.0
def __call__(self, x): # First conv h = self.conv_bn_act(x, int(self.maps0 * self.depth_mul), stride=(2, 2), act="hswish", name="first-conv") # Inverted residual blocks for i, elm in enumerate(self.settings): maps, kernel, stride, ef, act, se = elm maps = round(maps * self.depth_mul) name = "mbconv-{:03d}".format(i) h = self.inverted_residual( h, maps, kernel, stride, ef, act, se, name=name) # Conv -> Avepool -> Conv h = self.conv_bn_act(h, int(self.maps1 * self.depth_mul), (1, 1), act="hswish", name="last-conv-1") pool_shape = get_spatial_shape(x.shape, self.channel_last) h = F.average_pooling(h, pool_shape, channel_last=self.channel_last) h = self.conv_act(h, int(self.maps2 * self.depth_mul), (1, 1), act="hswish", name="last-conv-2") # Classifier if not self.test: h = F.dropout(h, 0.2) h = PF.affine(h, self.num_classes, w_init=I.NormalInitializer(0.01), name="linear") return h, {}
def shortcut(x, ochannels, stride, shortcut_type, test): ichannels = x.shape[1] use_conv = shortcut_type.lower() == 'c' if ichannels != ochannels: assert (ichannels * 2 == ochannels) or (ichannels * 4 == ochannels) if shortcut_type.lower() == 'b': use_conv = True if use_conv: # Convolution does everything. # Matching channels, striding. with nn.parameter_scope("shortcut_conv"): x = PF.convolution(x, ochannels, (1, 1), stride=stride, with_bias=False) x = PF.batch_normalization(x, batch_stat=not test) else: if stride != (1, 1): # Stride x = F.average_pooling(x, (1, 1), stride) if ichannels != ochannels: # Zero-padding to channel axis ishape = x.shape zeros = F.constant(0, (ishape[0], ochannels - ichannels) + ishape[-2:]) x = F.concatenate(x, zeros, axis=1) return x
def cifar10_resnet23_prediction(ctx, image, test=False): """ Construct ResNet 23 """ # Residual Unit def res_unit(x, scope_name, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Relu with nn.parameter_scope("conv1"): h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN -> Relu with nn.parameter_scope("conv2"): h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Relu h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h # Random generator for using the same init parameters in all devices nmaps = 64 ncls = 10 # Conv -> BN -> Relu with nn.context_scope(ctx): with nn.parameter_scope("conv1"): h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) h = res_unit(h, "conv2", False) # -> 32x32 h = res_unit(h, "conv3", True) # -> 16x16 h = bn_dropout(h, "bn_dropout1", test) h = res_unit(h, "conv4", False) # -> 16x16 h = res_unit(h, "conv5", True) # -> 8x8 h = bn_dropout(h, "bn_dropout2", test) h = res_unit(h, "conv6", False) # -> 8x8 h = res_unit(h, "conv7", True) # -> 4x4 h = bn_dropout(h, "bn_dropout3", test) h = res_unit(h, "conv8", False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 pred = PF.affine(h, ncls) return pred
def feature_view(ctx, h): """feature connection only""" with nn.parameter_scope("fv"): b, c, s, s = h.shape h = F.average_pooling(h, (s, s)) h = PF.affine(h, 10) return h
def cnn_model_003(ctx, x, act=F.relu, test=False): with nn.context_scope(ctx): # Convblock0 h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 28 -> 14 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) # Convblock 1 h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 14 -> 7 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) # Convblock 2 h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 7 -> 5 h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test) # Convblock 3 h = F.average_pooling(h, (5, 5)) with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:]))) return h
def backward_impl(self, inputs, outputs, prop_down, accum): # inputs: [inputs_fwd_graph] + [inputs_bwd_graph] or # [inputs_fwd_graph] + [outputs_fwd_graph] + [inputs_bwd_graph] # Args kernel = self.forward_func.info.args["kernel"] stride = self.forward_func.info.args["stride"] ignore_border = self.forward_func.info.args["ignore_border"] pad = self.forward_func.info.args["pad"] channel_last = self.forward_func.info.args["channel_last"] including_pad = self.forward_func.info.args["including_pad"] # TODO: BHWC assert channel_last == False, "`channel_last = False` is only supported now." # Inputs x0 = inputs[0].data dy = inputs[1].data # Outputs dx0 = outputs[0].data # Grads of inputs g_x0 = inputs[0].grad g_dy = inputs[1].grad # Grads of outputs g_dx0 = outputs[0].grad # Computation if prop_down[1]: g_dy_ = F.average_pooling(g_dx0, kernel, stride, ignore_border, pad, channel_last, including_pad) if accum[1]: g_dy += g_dy_ else: g_dy.copy_from(g_dy_)
def __call__(self, x): # First conv h = self.conv_bn_relu6(x, int(self.init_maps * self.depth_mul), stride=(2, 2), name="first-conv") # Inverted residual blocks for i, elm in enumerate(self.settings): t, c, n, s = elm # TODO: where to multiply depth_mul c = round(c * self.depth_mul) mbconv_s = partial(self.inverted_residual, maps=c, stride=(s, s), ef=t) mbconv_1 = partial(self.inverted_residual, maps=c, stride=(1, 1), ef=t) for j in range(n): name = "mbconv-{:02d}-{:02d}".format(i, j) h = mbconv_s(h, name=name) if j == 0 else mbconv_1( h, name=name) # Last conv h = self.conv_bn_relu6(h, int(1280 * self.depth_mul), kernel=(1, 1), name="last-conv") # Classifier if not self.test: h = F.dropout(h, 0.2) pool_shape = get_spatial_shape(x.shape, self.channel_last) h = F.average_pooling(h, pool_shape, channel_last=self.channel_last) h = PF.affine(h, self.num_classes, w_init=I.NormalInitializer(0.01), name="linear") return h, {}
def shuffle_unit(x, scope_name, dn=False): """ Figure. 2 (b) and (c) in https://arxiv.org/pdf/1707.01083.pdf """ C = x.shape[1] h = x with nn.parameter_scope(scope_name): with nn.parameter_scope("gconv1"): h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), group=groups, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h, True) with nn.parameter_scope("shuffle"): # no meaning but semantics h = shuffle(h) with nn.parameter_scope("dconv"): stride = (2, 2) if dn else (1, 1) h = PF.depthwise_convolution(h, kernel=(3, 3), pad=(1, 1), stride=stride, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) with nn.parameter_scope("gconv2"): h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), group=groups, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) s = F.average_pooling(x, (2, 2)) if dn else x h = F.concatenate(*[h, s], axis=1) if dn else h + s h = F.relu(h) return h
def call(self, input): out = F.average_pooling(input, kernel=self._kernel, stride=self._stride, pad=self._pad, channel_last=self._channel_last) return out
def resnet_model(ctx, x, inmaps=64, act=F.relu, test=False): # Conv -> BN -> Relu with nn.context_scope(ctx): with nn.parameter_scope("conv1"): h = PF.convolution(x, inmaps, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test) h = act(h) h = res_unit(h, "conv2", act, False) # -> 32x32 h = res_unit(h, "conv3", act, True) # -> 16x16 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) h = res_unit(h, "conv4", act, False) # -> 16x16 h = res_unit(h, "conv5", act, True) # -> 8x8 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) h = res_unit(h, "conv6", act, False) # -> 8x8 h = res_unit(h, "conv7", act, True) # -> 4x4 with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) h = res_unit(h, "conv8", act, False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 pred = PF.affine(h, 10) return pred
def cnn_model_003(ctx, x, act=F.relu, test=False): with nn.context_scope(ctx): # Convblock0 h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 32 -> 16 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) # Convblock 1 h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 16 -> 8 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) # Convblock 2 h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6 h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test) # Convblock 3 h = F.average_pooling(h, (6, 6)) with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:]))) return h
def cnn(x): """Unnecessarily Deep CNN. Args: x : Variable, shape (B, 1, 8, 8) Returns: y : Variable, shape (B, 10) """ with nn.parameter_scope("cnn"): # Parameter scope can be nested with nn.parameter_scope("conv1"): h = F.tanh( PF.batch_normalization(PF.convolution(x, 64, (3, 3), pad=(1, 1))) ) for i in range(10): # unnecessarily deep with nn.parameter_scope("conv{}".format(i + 2)): h = F.tanh( PF.batch_normalization(PF.convolution(h, 128, (3, 3), pad=(1, 1))) ) with nn.parameter_scope("conv_last"): h = F.tanh( PF.batch_normalization(PF.convolution(h, 512, (3, 3), pad=(1, 1))) ) h = F.average_pooling(h, (2, 2)) with nn.parameter_scope("fc"): h = F.tanh(PF.affine(h, 1024)) with nn.parameter_scope("classifier"): y = PF.affine(h, 10) return y
def __call__(self, x): h = self.conv_bn_relu(x, 32, stride=(2, 2), name="first-conv") h = self.depthwise_separable_conv( h, 64, stride=(1, 1), name="conv-ds-1") h = self.depthwise_separable_conv( h, 128, stride=(2, 2), name="conv-ds-2") h = self.depthwise_separable_conv( h, 128, stride=(1, 1), name="conv-ds-3") h = self.depthwise_separable_conv( h, 256, stride=(2, 2), name="conv-ds-4") h = self.depthwise_separable_conv( h, 256, stride=(1, 1), name="conv-ds-5") h = self.depthwise_separable_conv( h, 512, stride=(2, 2), name="conv-ds-6") h = self.depthwise_separable_conv( h, 512, stride=(1, 1), name="conv-ds-7") h = self.depthwise_separable_conv( h, 512, stride=(1, 1), name="conv-ds-8") h = self.depthwise_separable_conv( h, 512, stride=(1, 1), name="conv-ds-9") h = self.depthwise_separable_conv( h, 512, stride=(1, 1), name="conv-ds-10") h = self.depthwise_separable_conv( h, 512, stride=(1, 1), name="conv-ds-11") h = self.depthwise_separable_conv( h, 1024, stride=(2, 2), name="conv-ds-12") h = self.depthwise_separable_conv( h, 1024, stride=(1, 1), name="conv-ds-13") pool_shape = get_spatial_shape(x.shape, self.channel_last) h = F.average_pooling(h, pool_shape, channel_last=self.channel_last) h = PF.affine(h, self.num_classes, w_init=I.NormalInitializer(0.01), name="linear") return h, {}
def discriminator(x, scopename="discriminator", maps=128): with nn.parameter_scope(scopename): h = PF.convolution(x, maps * 1, (3, 3), name="conv-1") h = F.average_pooling(h, (2, 2)) # 32x32 -> 16x16 h = F.leaky_relu(h, 0.01) h = PF.convolution(h, maps * 2, (3, 3), name="conv-2") h = F.average_pooling(h, (2, 2)) # 16x16 -> 8x8 h = F.leaky_relu(h, 0.01) h = PF.convolution(h, maps * 4, (3, 3), name="conv-3") h = F.average_pooling(h, (2, 2)) # 8x8 -> 4x4 h = F.leaky_relu(h, 0.01) h = PF.affine(h, 1) return h
def shortcut(x, ochannels, stride, shortcut_type, test, channel_last=False): axes = [3 if channel_last else 1] ichannels = x.shape[axes[0]] use_conv = shortcut_type.lower() == 'c' if ichannels != ochannels: assert (ichannels * 2 == ochannels) or (ichannels * 4 == ochannels) if shortcut_type.lower() == 'b': use_conv = True if use_conv: # Convolution does everything. # Matching channels, striding. with nn.parameter_scope("shortcut_conv"): x = pf_convolution(x, ochannels, (1, 1), stride=stride, channel_last=channel_last) x = PF.batch_normalization(x, axes=axes, batch_stat=not test) else: if stride != (1, 1): # Stride x = F.average_pooling(x, (1, 1), stride, channel_last=channel_last) if ichannels != ochannels: # Zero-padding to channel axis ishape = x.shape if channel_last: zero_shape = (ishape[0],) + ishape[1:3] + \ (ochannels - ichannels,) else: zero_shape = (ishape[0], ochannels - ichannels) + ishape[-2:] zeros = F.constant(zero_shape, 0) x = F.concatenate(x, zeros, axis=1) return x
def fan(x, num_modules=1, test=True): x = PF.convolution(x, 64, kernel=(7, 7), stride=(2, 2), pad=(3, 3), name='conv1') x = PF.batch_normalization(x, batch_stat=not test, name='bn1') x = F.relu(x, True) with nn.parameter_scope('conv2'): x = conv_block(x, 64, 128) x = F.average_pooling(x, (2, 2), stride=(2, 2)) with nn.parameter_scope('conv3'): x = conv_block(x, 128, 128) with nn.parameter_scope('conv4'): x = conv_block(x, 128, 256) previous = x outputs = [] for i in range(num_modules): with nn.parameter_scope('m' + str(i)): hg = hour_glass(previous, 4, 256) ll = hg with nn.parameter_scope('top_m_' + str(i)): ll = conv_block(ll, 256, 256) ll = PF.convolution(ll, 256, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='conv_last' + str(i)) ll = PF.batch_normalization(ll, batch_stat=not test, name='bn_end' + str(i)) ll = F.relu(ll, True) # Predict heatmaps tmp_out = PF.convolution(ll, 68, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='l' + str(i)) outputs.append(tmp_out) if i < num_modules - 1: ll = PF.convolution(ll, 256, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='bl' + str(i)) tmp_out_ = PF.convolution(tmp_out, 256, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='al' + str(i)) previous = previous + ll + tmp_out_ return outputs
def cnn(self, h, resolution, ch_in, ch_out): """CNN block The following operations are performed two times. 1. Conv 2. Layer normalization 3. Leaky relu """ with nn.parameter_scope("phase_{}".format(resolution)): with nn.parameter_scope("conv1"): h = conv(h, ch_in, kernel=(3, 3), pad=(1, 1), stride=(1, 1), with_bias=not self.use_ln, use_wscale=self.use_wscale, use_he_backward=self.use_he_backward) h = LN(h, use_ln=self.use_ln) h = self.activation(h) with nn.parameter_scope("conv2"): h = conv(h, ch_out, kernel=(3, 3), pad=(1, 1), stride=(1, 1), with_bias=not self.use_ln, use_wscale=self.use_wscale, use_he_backward=self.use_he_backward) h = LN(h, use_ln=self.use_ln) h = self.activation(h) h = F.average_pooling(h, kernel=(2, 2)) return h
def csc(x, scope_name, dn=False): C = x.shape[1] h = x with nn.parameter_scope(scope_name): with nn.parameter_scope("conv1"): h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h, True) h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) with nn.parameter_scope("shift"): # no meaning but semantics h = shift(h) with nn.parameter_scope("conv2"): h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h, True) stride = (2, 2) if dn else (1, 1) if p > 0: h = F.dropout(h, p=0.5) if not test else h h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), stride=stride, with_bias=False) s = F.average_pooling(x, (2, 2)) if dn else x return h + s
def network(x, y, test=False): # Input:x -> 3,64,64 # AveragePooling -> 3,12,21 h = F.average_pooling(x, (5, 3), (5, 3)) # LeakyReLU_2 h = F.leaky_relu(h, 0.1, True) # Convolution_2 -> 20,13,21 h = PF.convolution(h, 20, (2, 3), (1, 1), name='Convolution_2') # BatchNormalization h = PF.batch_normalization(h, (1, ), 0.9, 0.0001, not test, name='BatchNormalization') # ReLU h = F.relu(h, True) # DepthwiseConvolution h = PF.depthwise_convolution(h, (5, 5), (2, 2), name='DepthwiseConvolution') # MaxPooling_2 -> 20,6,7 h = F.max_pooling(h, (2, 3), (2, 3)) # LeakyReLU h = F.leaky_relu(h, 0.1, True) # Affine -> 2 h = PF.affine(h, (2, ), name='Affine') # Softmax h = F.softmax(h) return h
def average_pool(x, output_filter, scope, test): """ average pooling (with no spatial downsampling). """ with nn.parameter_scope(scope): h = conv1x1(x, output_filter, scope, test) h = F.average_pooling(h, (3, 3), (1, 1), pad=(1, 1)) return h
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False): with nn.context_scope(ctx): # Convblock0 h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 28 -> 14 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test and do: h = F.dropout(h) # Convblock 1 h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test) h = F.max_pooling(h, (2, 2)) # 14 -> 7 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test and do: h = F.dropout(h) # Convblock 2 h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 7 -> 5 h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test) u = h # Convblock 3 h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test) h = F.average_pooling(h, (5, 5)) with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:]))) # Uncertainty u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test) u = F.average_pooling(u, (5, 5)) with nn.parameter_scope("u0bn"): u = PF.batch_normalization(u, batch_stat=not test) log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:]))) return pred, log_var
def mnist_binary_connect_lenet_prediction(image, test=False): """ Construct LeNet for MNIST (BinaryNet version). """ with nn.parameter_scope("conv1"): c1 = PF.binary_connect_convolution(image, 16, (5, 5)) c1 = PF.batch_normalization(c1, batch_stat=not test) c1 = F.elu(F.average_pooling(c1, (2, 2))) with nn.parameter_scope("conv2"): c2 = PF.binary_connect_convolution(c1, 16, (5, 5)) c2 = PF.batch_normalization(c2, batch_stat=not test) c2 = F.elu(F.average_pooling(c2, (2, 2))) with nn.parameter_scope("fc3"): c3 = PF.binary_connect_affine(c2, 50) c3 = PF.batch_normalization(c3, batch_stat=not test) c3 = F.elu(c3) with nn.parameter_scope("fc4"): c4 = PF.binary_connect_affine(c3, 10) c4 = PF.batch_normalization(c4, batch_stat=not test) return c4
def mnist_resnet_prediction(image, test=False): """ Construct ResNet for MNIST. """ image /= 255.0 def bn(x): return PF.batch_normalization(x, batch_stat=not test) def res_unit(x, scope): C = x.shape[1] with nn.parameter_scope(scope): with nn.parameter_scope('conv1'): h = F.elu(bn(PF.convolution(x, C / 2, (1, 1), with_bias=False))) with nn.parameter_scope('conv2'): h = F.elu( bn(PF.convolution(h, C / 2, (3, 3), pad=(1, 1), with_bias=False))) with nn.parameter_scope('conv3'): h = bn(PF.convolution(h, C, (1, 1), with_bias=False)) return F.elu(F.add2(h, x, inplace=True)) # Conv1 --> 64 x 32 x 32 with nn.parameter_scope("conv1"): c1 = F.elu( bn(PF.convolution(image, 64, (3, 3), pad=(3, 3), with_bias=False))) # Conv2 --> 64 x 16 x 16 c2 = F.max_pooling(res_unit(c1, "conv2"), (2, 2)) # Conv3 --> 64 x 8 x 8 c3 = F.max_pooling(res_unit(c2, "conv3"), (2, 2)) # Conv4 --> 64 x 8 x 8 c4 = res_unit(c3, "conv4") # Conv5 --> 64 x 4 x 4 c5 = F.max_pooling(res_unit(c4, "conv5"), (2, 2)) # Conv5 --> 64 x 4 x 4 c6 = res_unit(c5, "conv6") pl = F.average_pooling(c6, (4, 4)) with nn.parameter_scope("classifier"): y = PF.affine(pl, 10) return y
def cifar100_resnet23_prediction(image, ctx, test=False): """ Construct ResNet 23 """ # Residual Unit def res_unit(x, scope_name, rng, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Relu with nn.parameter_scope("conv1"): w_init = UniformInitializer( calc_uniform_lim_glorot(C, C / 2, kernel=(1, 1)), rng=rng) h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN -> Relu with nn.parameter_scope("conv2"): w_init = UniformInitializer( calc_uniform_lim_glorot(C / 2, C / 2, kernel=(3, 3)), rng=rng) h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): w_init = UniformInitializer( calc_uniform_lim_glorot(C / 2, C, kernel=(1, 1)), rng=rng) h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Relu h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h # Random generator for using the same init parameters in all devices rng = np.random.RandomState(0) nmaps = 384 ncls = 100 # Conv -> BN -> Relu with nn.context_scope(ctx): with nn.parameter_scope("conv1"): # Preprocess if not test: image = F.image_augmentation(image, contrast=1.0, angle=0.25, flip_lr=True) image.need_grad = False w_init = UniformInitializer( calc_uniform_lim_glorot(3, nmaps, kernel=(3, 3)), rng=rng) h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) h = res_unit(h, "conv2", rng, False) # -> 32x32 h = res_unit(h, "conv3", rng, True) # -> 16x16 h = res_unit(h, "conv4", rng, False) # -> 16x16 h = res_unit(h, "conv5", rng, True) # -> 8x8 h = res_unit(h, "conv6", rng, False) # -> 8x8 h = res_unit(h, "conv7", rng, True) # -> 4x4 h = res_unit(h, "conv8", rng, False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 w_init = UniformInitializer( calc_uniform_lim_glorot(int(np.prod(h.shape[1:])), ncls, kernel=(1, 1)), rng=rng) pred = PF.affine(h, ncls, w_init=w_init) return pred
def cnn_model_003_with_cross_attention(ctx, x_list, act=F.relu, test=False): """With attention before pooling """ with nn.context_scope(ctx): # Convblock0 h0_list = [] for x in x_list: h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test) h0_list.append(h) # Corss attention ca0 = attention(h0_list[0], h0_list[1], h0_list[1], div_dim=True, softmax=True) ca1 = attention(h0_list[1], h0_list[0], h0_list[0], div_dim=True, softmax=True) # Maxpooing, Batchnorm, Dropout h0_list = [] for h in [ca0, ca1]: h = F.max_pooling(h, (2, 2)) # 32 -> 16 with nn.parameter_scope("bn0"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) h0_list.append(h) # Convblock 1 h1_list = [] for h in h0_list: h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test) h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test) h1_list.append(h) # Corss attention ca0 = attention(h1_list[0], h1_list[1], h1_list[1], div_dim=True, softmax=True) ca1 = attention(h1_list[1], h1_list[0], h1_list[0], div_dim=True, softmax=True) # Maxpooing, Batchnorm, Dropout h1_list = [] for h in [ca0, ca1]: h = F.max_pooling(h, (2, 2)) # 16 -> 8 with nn.parameter_scope("bn1"): h = PF.batch_normalization(h, batch_stat=not test) if not test: h = F.dropout(h) h1_list.append(h) # Convblock 2 h2_list = [] for h in h1_list: h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6 h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test) h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test) h2_list.append(h) # Corss attention ca0 = attention(h2_list[0], h2_list[1], h2_list[1], div_dim=True, softmax=True) ca1 = attention(h2_list[1], h2_list[0], h2_list[0], div_dim=True, softmax=True) # Convblock 3 h3_list = [] for h in [ca0, ca1]: h = F.average_pooling(h, (6, 6)) with nn.parameter_scope("bn2"): h = PF.batch_normalization(h, batch_stat=not test) h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:]))) h3_list.append(h) return h3_list