def resblock(x, dim_out, w_init=None, epsilon=1e-05): assert dim_out == x.shape[1], "The number of input / output channels must match." h = PF.convolution(x, dim_out, kernel=(3, 3), pad=( 1, 1), with_bias=False, w_init=w_init, name="1st") h = PF.instance_normalization(h, eps=epsilon, name="1st") h = F.relu(h, inplace=True) h = PF.convolution(h, dim_out, kernel=(3, 3), pad=( 1, 1), with_bias=False, w_init=w_init, name="2nd") h = PF.instance_normalization(h, eps=epsilon, name="2nd") return x + h
def downblock(x, out_features, norm=False, kernel_size=4, pool=False, sn=False, test=False): out = x if sn: def apply_w(w): return PF.spectral_norm(w, dim=0, test=test) else: apply_w = None inmaps, outmaps = out.shape[1], out_features k_w = I.calc_normal_std_he_forward( inmaps, outmaps, kernel=(kernel_size, kernel_size)) / np.sqrt(2.) k_b = I.calc_normal_std_he_forward(inmaps, outmaps) / np.sqrt(2.) w_init = I.UniformInitializer((-k_w, k_w)) b_init = I.UniformInitializer((-k_b, k_b)) out = PF.convolution(out, out_features, kernel=(kernel_size, kernel_size), pad=(0, 0), stride=(1, 1), w_init=w_init, b_init=b_init, apply_w=apply_w) if norm: out = PF.instance_normalization(out) out = F.leaky_relu(out, 0.2, inplace=True) if pool: out = F.average_pooling(out, kernel=(2, 2)) return out
def resblock(x, n=256, test=False, norm_type="batch_norm"): r = x r = F.pad(r, (1, 1, 1, 1), 'reflect') with nn.parameter_scope('block1'): r = PF.convolution(r, n, (3, 3), with_bias=False) if norm_type == "instance_norm": r = PF.instance_normalization(r, eps=1e-05) else: r = PF.batch_normalization(r, batch_stat=not test) r = F.relu(r) r = F.pad(r, (1, 1, 1, 1), 'reflect') with nn.parameter_scope('block2'): r = PF.convolution(r, n, (3, 3), with_bias=False) if norm_type == "instance_norm": r = PF.instance_normalization(r, eps=1e-05) else: r = PF.batch_normalization(r, batch_stat=not test) return x + r
def generator(x, c, conv_dim=64, c_dim=5, num_downsample=2, num_upsample=2, repeat_num=6, w_init=None, epsilon=1e-05): assert len(c.shape) == 4 c = F.tile(c, (1, 1) + x.shape[2:]) concat_input = F.concatenate(x, c, axis=1) h = PF.convolution(concat_input, conv_dim, kernel=(7, 7), pad=( 3, 3), stride=(1, 1), with_bias=False, w_init=w_init, name="init_conv") h = PF.instance_normalization(h, eps=epsilon, name="init_inst_norm") h = F.relu(h, inplace=True) # Down-sampling layers. curr_dim = conv_dim for i in range(num_downsample): h = PF.convolution(h, curr_dim*2, kernel=(4, 4), pad=(1, 1), stride=(2, 2), with_bias=False, w_init=w_init, name="downsample_{}".format(i)) h = PF.instance_normalization( h, eps=epsilon, name="downsample_{}".format(i)) h = F.relu(h, inplace=True) curr_dim = curr_dim * 2 # Bottleneck layers. for i in range(repeat_num): with nn.parameter_scope("bottleneck_{}".format(i)): h = resblock(h, dim_out=curr_dim) # Up-sampling layers. for i in range(num_upsample): h = PF.deconvolution(h, curr_dim//2, kernel=(4, 4), pad=(1, 1), stride=( 2, 2), w_init=w_init, with_bias=False, name="upsample_{}".format(i)) h = PF.instance_normalization( h, eps=epsilon, name="upsample_{}".format(i)) h = F.relu(h, inplace=True) curr_dim = curr_dim // 2 h = PF.convolution(h, 3, kernel=(7, 7), pad=(3, 3), stride=( 1, 1), with_bias=False, w_init=w_init, name="last_conv") h = F.tanh(h) return h
def residual_block(self, x, o_channels): pad_width = get_symmetric_padwidth(1, channel_last=self.channel_last) with nn.parameter_scope("residual_1"): h = F.pad(x, pad_width=pad_width, mode=self.padding_type) h = PF.convolution(h, o_channels, (3, 3), **self.conv_opts) h = self.instance_norm_relu(h) with nn.parameter_scope("residual_2"): h = F.pad(h, pad_width=pad_width, mode=self.padding_type) h = PF.convolution(h, o_channels, (3, 3), **self.conv_opts) h = PF.instance_normalization(h, **self.norm_opts) return x + h
def convblock(x, n=64, k=(3, 3), s=(2, 2), p=(1, 1), test=False, norm_type="batch_norm"): x = PF.convolution(x, n, k, pad=p, stride=s, with_bias=False) if norm_type == "instance_norm": x = PF.instance_normalization(x, eps=1e-05) else: x = PF.batch_normalization(x, batch_stat=not test) x = F.relu(x) return x
def test_pf_instance_normalization(g_rng, inshape, batch_axis, channel_axis, output_stat, fix_parameters, param_init): from nnabla.normalization_functions import _force_list, _get_axes_excluding def ref_instance_normalization(x, beta, gamma, channel_axis, batch_axis, eps, output_stat): ignore_axes = _force_list(batch_axis) + [ channel_axis, ] axes = tuple(_get_axes_excluding(len(x.shape), ignore_axes)) x_mean = x.mean(axis=axes, keepdims=True) x_std = x.std(axis=axes, keepdims=True) if output_stat: return (x - x_mean) / (x_std + eps) * gamma + beta, x_mean, x_std return (x - x_mean) / (x_std + eps) * gamma + beta eps = 1e-5 p_shape = tuple( [inshape[i] if i == channel_axis else 1 for i in range(len(inshape))]) x_npy = g_rng.randn(*inshape) if param_init: beta_init = np.ones(p_shape) gamma_init = np.ones(p_shape) * 2 param_init = dict(beta=beta_init, gamma=gamma_init) else: beta_init = np.zeros(p_shape) gamma_init = np.ones(p_shape) x = nn.Variable.from_numpy_array(x_npy) kw = {} insert_if_not_default(kw, 'channel_axis', channel_axis, 1) insert_if_not_default(kw, 'batch_axis', batch_axis, 0) insert_if_not_default(kw, 'eps', eps, 1e-5) insert_if_not_default(kw, 'output_stat', output_stat, False) insert_if_not_default(kw, 'fix_parameters', fix_parameters, False) insert_if_not_none(kw, 'param_init', param_init) # Check creation y = PF.instance_normalization(x, **kw) y = _force_list(y) # just to simplify after execution # Check parameter values before execution h = y[0] b = h.parent.inputs[1] g = h.parent.inputs[0].parent.inputs[1] assert np.allclose(b.d, beta_init) assert np.allclose(g.d, gamma_init) # Check execution forward_backward_all(*y) # Check values ref = ref_instance_normalization(x_npy, beta_init, gamma_init, channel_axis, batch_axis, eps, output_stat) if not output_stat: ref = [ref] for i in range(len(ref)): assert np.allclose(y[i].d, ref[i], atol=1e-2, rtol=1e-5) # Check created parameters assert len(nn.get_parameters()) == 2 assert len(nn.get_parameters(grad_only=False)) == 2 beta, gamma = [ nn.get_parameters()['instance_normalization/' + name] for name in ['beta', 'gamma'] ] assert beta.shape == p_shape assert gamma.shape == p_shape assert beta.need_grad assert gamma.need_grad b = h.parent.inputs[1] g = h.parent.inputs[0].parent.inputs[1] assert b.need_grad == (not fix_parameters) assert g.need_grad == (not fix_parameters)
def instance_norm_lrelu(self, x, alpha=0.2): norm = PF.instance_normalization(x, no_scale=True, no_bias=True) return F.leaky_relu(norm, alpha=alpha, inplace=True)
def instance_norm_relu(self, x): # return F.relu(PF.layer_normalization(x, **self.norm_opts)) return F.relu(PF.instance_normalization(x, **self.norm_opts))