def mnist_lenet_prediction(image, test=False): """ Construct LeNet for MNIST. """ image /= 255.0 c1 = PF.convolution(image, 16, (5, 5), name='conv1') c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True) c2 = PF.convolution(c1, 16, (5, 5), name='conv2') c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True) c3 = F.relu(PF.affine(c2, 50, name='fc3'), inplace=True) c4 = PF.affine(c3, 10, name='fc4') return c4
def sep_conv_3x3(x, output_filter, scope, input_node_id, is_reduced, test, is_search): """ depthwise separable convolution with kernel 3x3. """ if is_reduced and input_node_id < 2: stride = (2, 2) else: stride = (1, 1) input_filter = x.shape[1] with nn.parameter_scope(scope + "_0"): h = F.relu(x) with nn.parameter_scope("depthwise"): h = PF.convolution(h, input_filter, kernel=(3, 3), pad=(1, 1), stride=stride, group=input_filter, with_bias=False) with nn.parameter_scope("pointwise"): h = PF.convolution(h, input_filter, (1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test, fix_parameters=is_search) with nn.parameter_scope(scope + "_1"): h = F.relu(h) with nn.parameter_scope("depthwise"): h = PF.convolution(h, input_filter, kernel=(3, 3), pad=(1, 1), stride=(1, 1), group=input_filter, with_bias=False) with nn.parameter_scope("pointwise"): h = PF.convolution(h, output_filter, (1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test, fix_parameters=is_search) return h
def resblock_hg(x, in_channels, bottleneck, out_channels, batch_stat=True): # (bn --> relu --> conv) * 3 with nn.parameter_scope('bn1'): h = PF.batch_normalization(x, batch_stat=batch_stat) h = F.relu(h, True) with nn.parameter_scope('conv1'): h = PF.convolution(h, bottleneck, kernel=(1, 1)) with nn.parameter_scope('bn2'): h = PF.batch_normalization(h, batch_stat=batch_stat) h = F.relu(h, True) with nn.parameter_scope('conv2'): h = PF.convolution(h, bottleneck, kernel=(3, 3), pad=(1, 1)) with nn.parameter_scope('bn3'): h = PF.batch_normalization(h, batch_stat=batch_stat) h = F.relu(h, True) with nn.parameter_scope('conv3'): h = PF.convolution(h, out_channels, kernel=(1, 1)) if in_channels != out_channels: with nn.parameter_scope('downsample'): x = PF.convolution(x, out_channels, kernel=(1, 1)) return x + h
def res_unit(x, scope_name, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Relu with nn.parameter_scope("conv1"): h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN -> Relu with nn.parameter_scope("conv2"): h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Relu h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h
def entry_flow(x, num_blocks, depth_list, test=False, fix_params=False): shortcut_channels = [128, 256, 728] global endpoints with nn.parameter_scope("1"): h = PF.convolution(x, 32, (3, 3), pad=(1, 1), stride=(2, 2), with_bias=False, fix_parameters=fix_params) h = F.relu( PF.batch_normalization(h, batch_stat=not test, eps=1e-03, fix_parameters=fix_params)) with nn.parameter_scope("2"): h = PF.convolution(h, 64, (3, 3), pad=(1, 1), with_bias=False, fix_parameters=fix_params) h = F.relu( PF.batch_normalization(h, batch_stat=not test, eps=1e-03, fix_parameters=fix_params)) x = h sh = x for i in range(0, num_blocks): with nn.parameter_scope("block" + str(i + 1)): if i == 1: x = block(x, sh, 1, True, True, depth_list[i], True, shortcut_channels[i], test=test) else: x = block(x, sh, 1, True, False, depth_list[i], True, shortcut_channels[i], test=test) sh = x if i == 2: endpoints['conv1'] = x return x
def bn_lenet(image, test=False, channel_last=False, w_bias=False): h = PF.convolution(image, 16, (5, 5), (1, 1), with_bias=w_bias, channel_last=channel_last, name='conv1') axes = get_channel_axes(h, channel_last) h = PF.batch_normalization(h, axes=axes, batch_stat=not test, name='conv1-bn') h = F.max_pooling(h, (2, 2), channel_last=channel_last) h = F.relu(h) h = PF.convolution(h, 16, (5, 5), (1, 1), with_bias=w_bias, channel_last=channel_last, name='conv2') axes = get_channel_axes(h, channel_last) h = PF.batch_normalization(h, axes=axes, batch_stat=not test, name='conv2') h = F.max_pooling(h, (2, 2), channel_last=channel_last) h = F.relu(h) h = PF.affine(h, 10, with_bias=True, name='fc1') axes = get_channel_axes(h, channel_last) h = PF.batch_normalization(h, axes=axes, batch_stat=not test, name='fc1') h = F.relu(h) pred = PF.affine(h, 10, with_bias=True, name='fc2') return pred
def convolution(x): x = x.reshape([BATCH_SIZE, IMAGE_DEPTH, IMAGE_HEIGHT, IMAGE_WIDTH]) with nn.parameter_scope("conv1"): output = PF.convolution(x, 16, (5, 5), stride=(2, 2), pad=(1, 1)) output = F.relu(output) with nn.parameter_scope("conv2"): output = PF.convolution(output, 32, (3, 3), stride=(1, 1), pad=(1, 1)) output = F.relu(output) with nn.parameter_scope("conv3"): output = PF.convolution(output, 64, (3, 3), stride=(1, 1), pad=(1, 1)) output = F.relu(output) output = output.reshape([BATCH_SIZE, int(output.size / BATCH_SIZE)]) with nn.parameter_scope("fc1"): output = PF.affine(output, 1024) output = F.relu(output) with nn.parameter_scope("fc2"): output = PF.affine(output, 256) output = F.relu(output) with nn.parameter_scope("softmax"): output = PF.affine(output, 10) output = F.softmax(output) return output
def fan(x, num_modules=1, test=True): x = PF.convolution(x, 64, kernel=(7, 7), stride=(2, 2), pad=(3, 3), name='conv1') x = PF.batch_normalization(x, batch_stat=not test, name='bn1') x = F.relu(x, True) with nn.parameter_scope('conv2'): x = conv_block(x, 64, 128) x = F.average_pooling(x, (2, 2), stride=(2, 2)) with nn.parameter_scope('conv3'): x = conv_block(x, 128, 128) with nn.parameter_scope('conv4'): x = conv_block(x, 128, 256) previous = x outputs = [] for i in range(num_modules): with nn.parameter_scope('m' + str(i)): hg = hour_glass(previous, 4, 256) ll = hg with nn.parameter_scope('top_m_' + str(i)): ll = conv_block(ll, 256, 256) ll = PF.convolution(ll, 256, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='conv_last' + str(i)) ll = PF.batch_normalization(ll, batch_stat=not test, name='bn_end' + str(i)) ll = F.relu(ll, True) # Predict heatmaps tmp_out = PF.convolution(ll, 68, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='l' + str(i)) outputs.append(tmp_out) if i < num_modules - 1: ll = PF.convolution(ll, 256, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='bl' + str(i)) tmp_out_ = PF.convolution(tmp_out, 256, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name='al' + str(i)) previous = previous + ll + tmp_out_ return outputs
def bottleneck(x, ochannels, shortcut_type, stride, test, channel_last=False): def bn(h): axes = [3 if channel_last else 1] return PF.batch_normalization(h, axes=axes, batch_stat=not test) assert ochannels % 4 == 0 hchannels = ochannels / 4 with nn.parameter_scope("bottleneck1"): h = F.relu( bn( PF.convolution(x, hchannels, (1, 1), with_bias=False, channel_last=channel_last))) with nn.parameter_scope("bottleneck2"): h = F.relu( bn( PF.convolution(h, hchannels, (3, 3), pad=(1, 1), stride=stride, with_bias=False, channel_last=channel_last))) with nn.parameter_scope("bottleneck3"): h = bn( PF.convolution(h, ochannels, (1, 1), with_bias=False, channel_last=channel_last)) with nn.parameter_scope("bottleneck_s"): s = shortcut(x, ochannels, stride, shortcut_type, test, channel_last) return F.relu(F.add2(h, s))
def bn_folding_lenet(image, test=False, channel_last=False, name="bn-folding-graph-ref"): h = PF.convolution(image, 16, (5, 5), (1, 1), with_bias=True, channel_last=channel_last, name='conv1') h = F.max_pooling(h, (2, 2), channel_last=channel_last) h = F.relu(h) h = PF.convolution(h, 16, (5, 5), (1, 1), with_bias=True, channel_last=channel_last, name='conv2') h = F.max_pooling(h, (2, 2), channel_last=channel_last) h = F.relu(h) h = PF.affine(h, 10, with_bias=True, name='fc1') h = F.relu(h) pred = PF.affine(h, 10, with_bias=True, name='fc2') return pred
def capsule_loss(v_norm, t_onehot, recon=None, x=None, m_pos=0.9, m_neg=0.1, wn=0.5, wr=0.0005): ''' Compute a margin loss given a length vector of output capsules and one-hot labels, and.optionally computes a reconstruction loss. Margin loss is given in eq 4. Reconstruction loss is given in Sec 4.1. Args: v_norm (nnabla.Variable): A length vector of capsules. A shape of [B, capsules]. t_onehot (nnabla.Variable): A shape of [B, capsules]. recon (nnabla.Variable): Reconstruction output with a shape of [B, 1, 28, 28]. The values are in [0, 0.1]. x (nnabla.Variable): Reconstruction target (i.e. input) with a shape of [B, 1, 28, 28]. The values are in [0, 0.1]. m_pos (float): Margin of capsules corresponding targets. m_neg (float): Margin of capsules corresponding non-targets. wn (float): Weight of the non-target margin loss. wr (float): Weight of the reconstruction loss. Returns: nnabla.Variable: 0-dim ''' # Classification loss lp = F.sum(t_onehot * F.relu(m_pos - v_norm) ** 2) ln = F.sum((1 - t_onehot) * F.relu(v_norm - m_neg) ** 2) lmargin = lp + wn * ln if recon is None or x is None: return lmargin / v_norm.shape[0] # Reconstruction loss lr = F.sum(F.squared_error(recon, x)) # return (lmargin + wr * lr) / v_norm.shape[0] lmargin = lmargin / v_norm.shape[0] lmargin.persistent = True lreconst = (wr * lr) / v_norm.shape[0] lreconst.persistent = True return lmargin, lreconst, lmargin + lreconst
def res_unit(x, scope_name, dn=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Relu with nn.parameter_scope("conv1"): h = PF.convolution(x, C // 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN -> Relu with nn.parameter_scope("conv2"): h = PF.convolution(h, C // 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Relu h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h
def call(self, x, speaker_emb=None): # causal convolution with nn.parameter_scope("causal"): pad = causal_padding(x, kernel_size=2, dilation=1) current = PF.convolution(pad, self.skip_dims, kernel=(2, ), dilation=(1, )) # residual skips = [] for index, dilation in enumerate(self.dilations): with nn.parameter_scope("residual_{}".format(index)): current, skip = self.residual_block(current, dilation, speaker_emb) skips.append(skip) # output out = F.relu(sum(skips)) with nn.parameter_scope("out1"): y1 = F.relu(PF.convolution(out, self.skip_dims, kernel=(1, ))) with nn.parameter_scope("out2"): y2 = PF.convolution(y1, self.output_channels, kernel=(1, )) return y2
def mlp_net(x, n_h, n_y, test=False): """ Function for building multi-layer-perceptron with batch_normalization Args: x(`~nnabla.Variable`): N-D array n_h(int): number of units in an intermediate layer n_y(int): number of classes test: operation type train=True, test=False Returns: ~nnabla.Variable: h """ h = x with nn.parameter_scope("fc1"): h = F.relu(PF.batch_normalization(PF.affine(h, n_h), batch_stat=not test), inplace=True) with nn.parameter_scope("fc2"): h = F.relu(PF.batch_normalization(PF.affine(h, n_h), batch_stat=not test), inplace=True) with nn.parameter_scope("fc3"): h = PF.affine(h, n_y) return h
def network(x, test=False): # Input:x -> 1,128,128 # ImageAugmentation h = F.image_augmentation(x, (1,128,128), (0,0), 1, 1, 0, 1, 0, False, False, 0, False, 1, 0.5, False, 0) # Convolution -> 16,124,124 h = PF.convolution(h, 16, (5,5), (0,0), name='Convolution') # ReLU h = F.relu(h, True) # MaxPooling -> 16,62,62 h = F.max_pooling(h, (2,2), (2,2)) # Convolution_2 -> 30,60,60 h = PF.convolution(h, 30, (3,3), (0,0), name='Convolution_2') # MaxPooling_2 -> 30,30,30 h = F.max_pooling(h, (2,2), (2,2)) # Tanh_2 h = F.tanh(h) # Affine -> 150 h = PF.affine(h, (150,), name='Affine') # ReLU_2 h = F.relu(h, True) # Affine_2 -> 2 h = PF.affine(h, (2,), name='Affine_2') # Softmax h = F.softmax(h) return h
def bottleneck(x, planes, stride=1, downsample=None, test=True): residual = x out = PF.convolution(x, planes, kernel=(1, 1), name='conv1', with_bias=False) out = PF.batch_normalization(out, batch_stat=not test, name='bn1') out = F.relu(out, True) out = PF.convolution(out, planes, kernel=(3, 3), stride=(stride, stride), pad=(1, 1), name='conv2', with_bias=False) out = PF.batch_normalization(out, batch_stat=not test, name='bn2') out = F.relu(out, True) out = PF.convolution(out, planes * 4, kernel=(1, 1), name='conv3', with_bias=False) out = PF.batch_normalization(out, batch_stat=not test, name='bn3') if downsample is not None: residual = downsample out += residual out = F.relu(out, True) return out
def gated_conv(self, x, kernel_shape, h=None, mask_type='', gated=True, payload=None, return_payload=False, scope_name='gated_conv'): pad_dim_0 = (kernel_shape[0] - 1) / 2 pad_dim_1 = (kernel_shape[1] - 1) / 2 if mask_type == '': mask_type = self.mask_type_B with nn.parameter_scope(scope_name): if gated: out_f = PF.convolution(x, self.num_features, kernel_shape, pad=(pad_dim_0, pad_dim_1), apply_w=mask_type, name='conv_f') out_g = PF.convolution(x, self.num_features, kernel_shape, pad=(pad_dim_0, pad_dim_1), apply_w=mask_type, name='conv_g') if isinstance(payload, nn.Variable): out_f += payload[:, :self.num_features, :, :] out_g += payload[:, self.num_features:, :, :] if self.conditional: h_out_f = PF.affine(h, self.num_features, name='h_out_f') h_out_f = h_out_f.reshape( (h_out_f.shape[0], h_out_f.shape[1], 1, 1)) h_out_g = PF.affine(h, self.num_features, name='h_out_g') h_out_g = h_out_g.reshape( (h_out_g.shape[0], h_out_g.shape[1], 1, 1)) out = F.tanh(out_f + h_out_f) * F.sigmoid(out_g + h_out_g) else: out = F.tanh(out_f) * F.sigmoid(out_g) if return_payload: payload = PF.convolution(F.concatenate(out_f, out_g, axis=1), 2 * self.num_features, (1, 1), name='conv_1x1') payload = F.relu(payload) return out, payload else: out = PF.convolution(x, self.num_features, kernel_shape, stride=(1, 1), pad=(pad_dim_0, pad_dim_1), apply_w=mask_type) out = F.relu(out) return out
def separable_conv_with_bn(x, f, stride=False, aspp=False, atrous_rate=1, act_fn=True, last_block=False, end_point=False, eps=1e-03, out=False, test=False, fix_params=False): with nn.parameter_scope("depthwise"): if (stride == True): h = PF.depthwise_convolution(x, (3, 3), stride=(2, 2), pad=(1, 1), with_bias=False, fix_parameters=fix_params) elif (aspp == True): h = PF.depthwise_convolution(x, (3, 3), pad=(atrous_rate, atrous_rate), stride=(1, 1), dilation=(atrous_rate, atrous_rate), with_bias=False, fix_parameters=fix_params) else: h = PF.depthwise_convolution(x, (3, 3), pad=(1, 1), with_bias=False, fix_parameters=fix_params) h = PF.batch_normalization(h, batch_stat=not test, eps=eps, fix_parameters=fix_params) if last_block == True: h = F.relu(h) with nn.parameter_scope("pointwise"): h = PF.convolution(h, f, (1, 1), stride=(1, 1), with_bias=False, fix_parameters=fix_params) h = PF.batch_normalization(h, batch_stat=not test, eps=eps, fix_parameters=fix_params) if end_point == True: global endpoints endpoints['Decoder End Point 1'] = h if act_fn == True: h = F.relu(h) return h
def q_network(obs, action, name): with nn.parameter_scope(name): out = F.concatenate(obs, action, axis=1) out = PF.affine(out, 256, name='fc1') out = F.relu(out) out = PF.affine(out, 256, name='fc2') out = F.relu(out) return PF.affine(out, 1, name='fc3')
def policy_network(obs, action_size, name): with nn.parameter_scope(name): out = PF.affine(obs, 256, name='fc1') out = F.relu(out) out = PF.affine(out, 256, name='fc2') out = F.relu(out) out = PF.affine(out, action_size, name='fc3') return F.tanh(out)
def __init__(self, channels, stride=1, skip_by_conv=True): self.conv1 = ConvBn(channels // 4, 1, 1, act=lambda x: F.relu(x, inplace=True)) self.conv2 = ConvBn(channels // 4, 3, stride, act=lambda x: F.relu(x, inplace=True)) self.conv3 = ConvBn(channels, 1) self.skip_by_conv = skip_by_conv self.skip = ConvBn(channels, 1, stride)
def nips_head(obs): out = PF.convolution(obs, 16, (8, 8), stride=(4, 4), name='conv1') out = F.relu(out) out = PF.convolution(out, 32, (4, 4), stride=(2, 2), name='conv2') out = F.relu(out) out = PF.affine(out, 256, name='fc1') out = F.relu(out) return out
def mlp(image, test=False): image /= 255.0 c1 = F.relu(PF.convolution(image, 32, (3, 3), name='conv1'), inplace=True) c2 = F.relu(PF.convolution(c1, 128, (3, 3), name='conv2'), inplace=True) c3 = F.relu(PF.convolution(c2, 256, (3, 3), name='conv3'), inplace=True) c4 = F.relu(PF.affine(c3, 512, name='fc3'), inplace=True) c5 = PF.affine(c3, 10, name='fc4') return F.softmax(c5)
def res_unit(x, scope_name, dn=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Pow2Quantize -> Relu with nn.parameter_scope("conv1"): h = PF.pow2_quantized_convolution(x, C // 2, kernel=(1, 1), pad=(0, 0), n_w=n, m_w=m, n_b=n, m_b=m, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h, n=n, m=m, ste_fine_grained=ste_fine_grained) h = F.relu(h) # Conv -> BN -> Pow2Quantize -> Relu with nn.parameter_scope("conv2"): h = PF.pow2_quantized_convolution(h, C // 2, kernel=(3, 3), pad=(1, 1), n_w=n, m_w=m, n_b=n, m_b=m, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.pow2_quantize(h, n=n, m=m, ste_fine_grained=ste_fine_grained) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.pow2_quantized_convolution(h, C, kernel=(1, 1), pad=(0, 0), n_w=n, m_w=m, n_b=n, m_b=m, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Pow2Quantize -> Relu h = F.pow2_quantize(h, n=n, m=m, ste_fine_grained=ste_fine_grained) h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h
def hinge_gan_loss(r_out, f_out): # D d_gan_real = F.mean(F.relu(1. - r_out)) d_gan_fake = F.mean(F.relu(1. + f_out)) # G g_gan = -1 * F.mean(f_out) return d_gan_real, d_gan_fake, g_gan
def res_block(x, out_ch, name): """ Create residual block """ with nn.parameter_scope(name): h = conv_2d(F.relu(x), out_ch, kernel=(3, 3), name='conv/0') h = conv_2d(F.relu(h), out_ch, kernel=(3, 3), name='conv/1') h = x + h return h
def res_block_concat(x, out_ch, name): """ Basic residual block -> [conv-relu | conv-relu] + input """ with nn.parameter_scope(name): h = conv_2d(F.relu(x), out_ch, kernel=(3, 3), name='conv/0') h = conv_2d(F.relu(h), out_ch, kernel=(3, 3), name='conv/1') h = x[:, :, :, :out_ch] + h return h
def test_graph_model(model, seed): np.random.seed(313) rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4, 4], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definintion nn.clear_parameters() if model == "mlp": with nn.parameter_scope('fc1'): z = PF.affine(x, 3) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) elif model == "recurrent": with nn.parameter_scope('fc1'): z = PF.affine(x, 3) z2 = F.relu(z, inplace=True) h = z2 for _ in range(2): with nn.parameter_scope('fc2'): h = PF.affine(h, 3) h = F.relu(h, inplace=True) with nn.parameter_scope('fc3'): z3 = PF.affine(h, 5) elif model == "convolution": with nn.parameter_scope('conv1'): z = PF.convolution(x, 3, (2, 2)) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) else: raise ValueError() l = F.softmax_cross_entropy(z3, t, 1) L = F.mean(l) # Forwardprop L.forward(clear_no_need_grad=True) # Backprop # Diff should be initialized since they are always accumulated x.grad.zero() L.backward(clear_buffer=True) x.g = rng.randn(*x.shape) parameters = nn.get_parameters() for param in parameters.values(): param.grad.zero() inputs = [x] + list(parameters.values()) from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L, inputs, 1e-3) assert np.allclose(ngrad, agrad, atol=1.05e-2)
def cnn(x, n_class): c1 = PF.convolution(x, 16, (5, 5), name='conv1') c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True) c2 = PF.convolution(c1, 16, (5, 5), name='conv2') c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True) c3 = F.relu(PF.affine(c2, 50, name='fc3'), inplace=True) c4 = PF.affine(c3, n_class, name='fc4') return c4
def test_graph_model(model, seed): np.random.seed(313) rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4, 4], need_grad=True) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) t.d = rng.randint(0, 5, size=t.shape) nn.set_default_context(nn.Context()) # Forwardprop by definition nn.clear_parameters() if model == "mlp": with nn.parameter_scope('fc1'): z = PF.affine(x, 3) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) elif model == "recurrent": with nn.parameter_scope('fc1'): z = PF.affine(x, 8) z2 = F.relu(z, inplace=True) h = z2 for _ in range(2): with nn.parameter_scope('fc2'): h = PF.affine(h, 8) h = F.relu(h, inplace=True) with nn.parameter_scope('fc3'): z3 = PF.affine(h, 5) elif model == "convolution": with nn.parameter_scope('conv1'): z = PF.convolution(x, 3, (2, 2)) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) else: raise ValueError() l = F.softmax_cross_entropy(z3, t, 1) L = F.mean(l) # Forwardprop L.forward(clear_no_need_grad=True) # Backprop # Diff should be initialized since they are always accumulated x.grad.zero() L.backward(clear_buffer=True) x.g = rng.randn(*x.shape) parameters = nn.get_parameters() for param in parameters.values(): param.grad.zero() inputs = [x] + list(parameters.values()) from nbla_test_utils import \ compute_analytical_and_numerical_grad_graph as grads agrad, ngrad = grads(L, inputs, 1e-3) assert_allclose(ngrad, agrad, atol=1.05e-2)
def model(img, sf): """ Define JSInet model """ with nn.parameter_scope('Network'): with nn.parameter_scope('local_contrast_enhancement'): ## ================= Local Contrast Enhancement Subnet ============================ ## ch = 64 b = guided_filter(img, 5, 0.01) n1 = conv_2d(b, ch, kernel=(3, 3), name='conv/0') for i in range(4): n1 = res_block(n1, ch, 'res_block/%d' % i) n1 = F.relu(n1, inplace=True) local_filter_2d = conv_2d( n1, (9**2) * (sf**2), kernel=(3, 3), name='conv_k') # [B, H, W, (9x9)*(sfxsf)] # dynamic 2D upsampling with 2D local filters pred_C = dyn_2d_up_operation(b, local_filter_2d, (9, 9), sf) # local contrast mask pred_C = 2 * F.sigmoid(pred_C) ## ================= Detail Restoration Subnet ============================ ## ch = 64 d = F.div2(img, b + 1e-15) with nn.parameter_scope('detail_restoration'): n3 = conv_2d(d, ch, kernel=(3, 3), name='conv/0') for i in range(4): n3 = res_block(n3, ch, 'res_block/%d' % i) if i == 0: d_feature = n3 n3 = F.relu(n3, inplace=True) # separable 1D filters dr_k_h = conv_2d(n3, 41 * sf**2, kernel=(3, 3), name='conv_k_h') dr_k_v = conv_2d(n3, 41 * sf**2, kernel=(3, 3), name='conv_k_v') # dynamic separable upsampling with with separable 1D local filters pred_D = dyn_sep_up_operation(d, dr_k_v, dr_k_h, 41, sf) ## ================= Image Reconstruction Subnet ============================ ## with nn.parameter_scope('image_reconstruction'): n4 = conv_2d(img, ch, kernel=(3, 3), name='conv/0') for i in range(4): if i == 1: n4 = F.concatenate(n4, d_feature, axis=3) n4 = res_block_concat(n4, ch, 'res_block/%d' % i) else: n4 = res_block(n4, ch, 'res_block/%d' % i) n4 = F.relu(n4, inplace=True) n4 = F.relu(conv_2d(n4, ch * sf * sf, kernel=(3, 3), name='conv/1'), inplace=True) # (1,100,170,1024) -> (1,100,170,4,4,64) -> (1,100,4,170,4,64) # pixel shuffle n4 = depth_to_space(n4, sf) pred_I = conv_2d(n4, 3, kernel=(3, 3), name='conv/2') pred = F.add2(pred_I, pred_D, inplace=True) * pred_C jsinet = namedtuple('jsinet', ['pred']) return jsinet(pred)
def mlp(x, maps, num_res=4, num_layers=2, name="mlp"): h = x with nn.parameter_scope(name): h = PF.affine(h, maps, name="affine-first") h = F.relu(h, True) h = PF.affine(h, maps, name="affine-mid") h = F.relu(h, True) h = PF.affine(h, 2 * maps * num_res * num_layers, name="affine-last") return h
def _check_context(ctx): try: x = nn.Variable() F.relu(x) except: logger.warn('Fallback to CPU context.') import nnabla_ext.cpu ctx = nnabla_ext.cpu.context() return ctx
def cifar10_resnet23_prediction(ctx, image, test=False): """ Construct ResNet 23 """ # Residual Unit def res_unit(x, scope_name, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Relu with nn.parameter_scope("conv1"): h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN -> Relu with nn.parameter_scope("conv2"): h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Relu h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h # Random generator for using the same init parameters in all devices nmaps = 64 ncls = 10 # Conv -> BN -> Relu with nn.context_scope(ctx): with nn.parameter_scope("conv1"): h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1), with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) h = res_unit(h, "conv2", False) # -> 32x32 h = res_unit(h, "conv3", True) # -> 16x16 h = bn_dropout(h, "bn_dropout1", test) h = res_unit(h, "conv4", False) # -> 16x16 h = res_unit(h, "conv5", True) # -> 8x8 h = bn_dropout(h, "bn_dropout2", test) h = res_unit(h, "conv6", False) # -> 8x8 h = res_unit(h, "conv7", True) # -> 4x4 h = bn_dropout(h, "bn_dropout3", test) h = res_unit(h, "conv8", False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 pred = PF.affine(h, ncls) return pred
def test_cuda_large_blocks(m): CUDA_THREAD_PER_BLOCK = 512 CUDA_MAX_BLOCKS = 65536 size = CUDA_MAX_BLOCKS * CUDA_THREAD_PER_BLOCK * m + 3 print "Variable size:", size x = np.zeros((size,), np.float32) v = nn.Variable(x.shape) v.d = x ctx = nn.Context(backend='cuda') y = F.relu(v)
def mlp_net(x, n_h, n_y, test=False): """ Function for building multi-layer-perceptron with batch_normalization Args: x(`~nnabla.Variable`): N-D array n_h(int): number of units in an intermediate layer n_y(int): number of classes test: operation type train=True, test=False Returns: ~nnabla.Variable: log(p(y|x)) """ h = x with nn.parameter_scope("fc1"): h = F.relu(PF.batch_normalization( PF.affine(h, n_h), batch_stat=not test), inplace=True) with nn.parameter_scope("fc2"): h = F.relu(PF.batch_normalization( PF.affine(h, n_h), batch_stat=not test), inplace=True) with nn.parameter_scope("fc3"): h = PF.affine(h, n_y) return h
def test_graph_clear_buffer(seed): np.random.seed(313) rng = np.random.RandomState(seed) x = nn.Variable([2, 3, 4, 4]) t = nn.Variable([2, 1]) x.d = rng.randn(*x.shape) t.d = rng.randint(0, 5, size=t.shape) # Network definition nn.set_default_context(nn.Context()) nn.clear_parameters() x1 = x + 1 x2 = x1 - 1 with nn.parameter_scope('conv1'): z = PF.convolution(x2, 3, (2, 2)) z2 = F.relu(z, inplace=True) with nn.parameter_scope('fc2'): z3 = PF.affine(z2, 5) l = F.softmax_cross_entropy(z3, t, 1) L = F.mean(l) # Forwardprop import tempfile import os tmpd = tempfile.mkdtemp() nn.save_parameters(os.path.join(tmpd, 'parameter.h5')) first = False for cnng in [False, True]: for cb in [False, True]: _ = nn.load_parameters(os.path.join(tmpd, 'parameter.h5')) for v in nn.get_parameters().values(): v.grad.zero() L.forward(clear_no_need_grad=cnng) L.backward(clear_buffer=cb) if not first: first = True g = list(nn.get_parameters().values())[0].g.copy() else: g2 = list(nn.get_parameters().values())[0].g.copy() assert np.all(g == g2)
def cifar100_resnet23_prediction(image, ctx, test=False): """ Construct ResNet 23 """ # Residual Unit def res_unit(x, scope_name, rng, dn=False, test=False): C = x.shape[1] with nn.parameter_scope(scope_name): # Conv -> BN -> Relu with nn.parameter_scope("conv1"): w_init = UniformInitializer( calc_uniform_lim_glorot(C, C / 2, kernel=(1, 1)), rng=rng) h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN -> Relu with nn.parameter_scope("conv2"): w_init = UniformInitializer( calc_uniform_lim_glorot(C / 2, C / 2, kernel=(3, 3)), rng=rng) h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) # Conv -> BN with nn.parameter_scope("conv3"): w_init = UniformInitializer( calc_uniform_lim_glorot(C / 2, C, kernel=(1, 1)), rng=rng) h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) # Residual -> Relu h = F.relu(h + x) # Maxpooling if dn: h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2)) return h # Random generator for using the same init parameters in all devices rng = np.random.RandomState(0) nmaps = 384 ncls = 100 # Conv -> BN -> Relu with nn.context_scope(ctx): with nn.parameter_scope("conv1"): # Preprocess if not test: image = F.image_augmentation(image, contrast=1.0, angle=0.25, flip_lr=True) image.need_grad = False w_init = UniformInitializer( calc_uniform_lim_glorot(3, nmaps, kernel=(3, 3)), rng=rng) h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1), w_init=w_init, with_bias=False) h = PF.batch_normalization(h, batch_stat=not test) h = F.relu(h) h = res_unit(h, "conv2", rng, False) # -> 32x32 h = res_unit(h, "conv3", rng, True) # -> 16x16 h = res_unit(h, "conv4", rng, False) # -> 16x16 h = res_unit(h, "conv5", rng, True) # -> 8x8 h = res_unit(h, "conv6", rng, False) # -> 8x8 h = res_unit(h, "conv7", rng, True) # -> 4x4 h = res_unit(h, "conv8", rng, False) # -> 4x4 h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1 w_init = UniformInitializer( calc_uniform_lim_glorot(int(np.prod(h.shape[1:])), ncls, kernel=(1, 1)), rng=rng) pred = PF.affine(h, ncls, w_init=w_init) return pred