def mnist_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST.
"""
image /= 255.0
c1 = PF.convolution(image, 16, (5, 5), name='conv1')
c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
c2 = PF.convolution(c1, 16, (5, 5), name='conv2')
c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
c3 = F.relu(PF.affine(c2, 50, name='fc3'), inplace=True)
c4 = PF.affine(c3, 10, name='fc4')
return c4
def sep_conv_3x3(x, output_filter, scope,
input_node_id, is_reduced, test, is_search):
"""
depthwise separable convolution with kernel 3x3.
"""
if is_reduced and input_node_id < 2:
stride = (2, 2)
else:
stride = (1, 1)
input_filter = x.shape[1]
with nn.parameter_scope(scope + "_0"):
h = F.relu(x)
with nn.parameter_scope("depthwise"):
h = PF.convolution(h, input_filter, kernel=(3, 3),
pad=(1, 1), stride=stride,
group=input_filter, with_bias=False)
with nn.parameter_scope("pointwise"):
h = PF.convolution(h, input_filter, (1, 1), with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test,
fix_parameters=is_search)
with nn.parameter_scope(scope + "_1"):
h = F.relu(h)
with nn.parameter_scope("depthwise"):
h = PF.convolution(h, input_filter, kernel=(3, 3),
pad=(1, 1), stride=(1, 1),
group=input_filter, with_bias=False)
with nn.parameter_scope("pointwise"):
h = PF.convolution(h, output_filter, (1, 1), with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test,
fix_parameters=is_search)
return h
def resblock_hg(x, in_channels, bottleneck, out_channels, batch_stat=True):
# (bn --> relu --> conv) * 3
with nn.parameter_scope('bn1'):
h = PF.batch_normalization(x, batch_stat=batch_stat)
h = F.relu(h, True)
with nn.parameter_scope('conv1'):
h = PF.convolution(h, bottleneck, kernel=(1, 1))
with nn.parameter_scope('bn2'):
h = PF.batch_normalization(h, batch_stat=batch_stat)
h = F.relu(h, True)
with nn.parameter_scope('conv2'):
h = PF.convolution(h, bottleneck, kernel=(3, 3), pad=(1, 1))
with nn.parameter_scope('bn3'):
h = PF.batch_normalization(h, batch_stat=batch_stat)
h = F.relu(h, True)
with nn.parameter_scope('conv3'):
h = PF.convolution(h, out_channels, kernel=(1, 1))
if in_channels != out_channels:
with nn.parameter_scope('downsample'):
x = PF.convolution(x, out_channels, kernel=(1, 1))
return x + h
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def entry_flow(x, num_blocks, depth_list, test=False, fix_params=False):
shortcut_channels = [128, 256, 728]
global endpoints
with nn.parameter_scope("1"):
h = PF.convolution(x,
32, (3, 3),
pad=(1, 1),
stride=(2, 2),
with_bias=False,
fix_parameters=fix_params)
h = F.relu(
PF.batch_normalization(h,
batch_stat=not test,
eps=1e-03,
fix_parameters=fix_params))
with nn.parameter_scope("2"):
h = PF.convolution(h,
64, (3, 3),
pad=(1, 1),
with_bias=False,
fix_parameters=fix_params)
h = F.relu(
PF.batch_normalization(h,
batch_stat=not test,
eps=1e-03,
fix_parameters=fix_params))
x = h
sh = x
for i in range(0, num_blocks):
with nn.parameter_scope("block" + str(i + 1)):
if i == 1:
x = block(x,
sh,
1,
True,
True,
depth_list[i],
True,
shortcut_channels[i],
test=test)
else:
x = block(x,
sh,
1,
True,
False,
depth_list[i],
True,
shortcut_channels[i],
test=test)
sh = x
if i == 2:
endpoints['conv1'] = x
return x
def bn_lenet(image, test=False, channel_last=False, w_bias=False):
h = PF.convolution(image,
16, (5, 5), (1, 1),
with_bias=w_bias,
channel_last=channel_last,
name='conv1')
axes = get_channel_axes(h, channel_last)
h = PF.batch_normalization(h,
axes=axes,
batch_stat=not test,
name='conv1-bn')
h = F.max_pooling(h, (2, 2), channel_last=channel_last)
h = F.relu(h)
h = PF.convolution(h,
16, (5, 5), (1, 1),
with_bias=w_bias,
channel_last=channel_last,
name='conv2')
axes = get_channel_axes(h, channel_last)
h = PF.batch_normalization(h, axes=axes, batch_stat=not test, name='conv2')
h = F.max_pooling(h, (2, 2), channel_last=channel_last)
h = F.relu(h)
h = PF.affine(h, 10, with_bias=True, name='fc1')
axes = get_channel_axes(h, channel_last)
h = PF.batch_normalization(h, axes=axes, batch_stat=not test, name='fc1')
h = F.relu(h)
pred = PF.affine(h, 10, with_bias=True, name='fc2')
return pred
def convolution(x):
x = x.reshape([BATCH_SIZE, IMAGE_DEPTH, IMAGE_HEIGHT, IMAGE_WIDTH])
with nn.parameter_scope("conv1"):
output = PF.convolution(x, 16, (5, 5), stride=(2, 2), pad=(1, 1))
output = F.relu(output)
with nn.parameter_scope("conv2"):
output = PF.convolution(output, 32, (3, 3), stride=(1, 1), pad=(1, 1))
output = F.relu(output)
with nn.parameter_scope("conv3"):
output = PF.convolution(output, 64, (3, 3), stride=(1, 1), pad=(1, 1))
output = F.relu(output)
output = output.reshape([BATCH_SIZE, int(output.size / BATCH_SIZE)])
with nn.parameter_scope("fc1"):
output = PF.affine(output, 1024)
output = F.relu(output)
with nn.parameter_scope("fc2"):
output = PF.affine(output, 256)
output = F.relu(output)
with nn.parameter_scope("softmax"):
output = PF.affine(output, 10)
output = F.softmax(output)
return output
def fan(x, num_modules=1, test=True):
x = PF.convolution(x,
64,
kernel=(7, 7),
stride=(2, 2),
pad=(3, 3),
name='conv1')
x = PF.batch_normalization(x, batch_stat=not test, name='bn1')
x = F.relu(x, True)
with nn.parameter_scope('conv2'):
x = conv_block(x, 64, 128)
x = F.average_pooling(x, (2, 2), stride=(2, 2))
with nn.parameter_scope('conv3'):
x = conv_block(x, 128, 128)
with nn.parameter_scope('conv4'):
x = conv_block(x, 128, 256)
previous = x
outputs = []
for i in range(num_modules):
with nn.parameter_scope('m' + str(i)):
hg = hour_glass(previous, 4, 256)
ll = hg
with nn.parameter_scope('top_m_' + str(i)):
ll = conv_block(ll, 256, 256)
ll = PF.convolution(ll,
256,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='conv_last' + str(i))
ll = PF.batch_normalization(ll,
batch_stat=not test,
name='bn_end' + str(i))
ll = F.relu(ll, True)
# Predict heatmaps
tmp_out = PF.convolution(ll,
68,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='l' + str(i))
outputs.append(tmp_out)
if i < num_modules - 1:
ll = PF.convolution(ll,
256,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='bl' + str(i))
tmp_out_ = PF.convolution(tmp_out,
256,
kernel=(1, 1),
stride=(1, 1),
pad=(0, 0),
name='al' + str(i))
previous = previous + ll + tmp_out_
return outputs
def bottleneck(x, ochannels, shortcut_type, stride, test, channel_last=False):
def bn(h):
axes = [3 if channel_last else 1]
return PF.batch_normalization(h, axes=axes, batch_stat=not test)
assert ochannels % 4 == 0
hchannels = ochannels / 4
with nn.parameter_scope("bottleneck1"):
h = F.relu(
bn(
PF.convolution(x,
hchannels, (1, 1),
with_bias=False,
channel_last=channel_last)))
with nn.parameter_scope("bottleneck2"):
h = F.relu(
bn(
PF.convolution(h,
hchannels, (3, 3),
pad=(1, 1),
stride=stride,
with_bias=False,
channel_last=channel_last)))
with nn.parameter_scope("bottleneck3"):
h = bn(
PF.convolution(h,
ochannels, (1, 1),
with_bias=False,
channel_last=channel_last))
with nn.parameter_scope("bottleneck_s"):
s = shortcut(x, ochannels, stride, shortcut_type, test, channel_last)
return F.relu(F.add2(h, s))
def bn_folding_lenet(image,
test=False,
channel_last=False,
name="bn-folding-graph-ref"):
h = PF.convolution(image,
16, (5, 5), (1, 1),
with_bias=True,
channel_last=channel_last,
name='conv1')
h = F.max_pooling(h, (2, 2), channel_last=channel_last)
h = F.relu(h)
h = PF.convolution(h,
16, (5, 5), (1, 1),
with_bias=True,
channel_last=channel_last,
name='conv2')
h = F.max_pooling(h, (2, 2), channel_last=channel_last)
h = F.relu(h)
h = PF.affine(h, 10, with_bias=True, name='fc1')
h = F.relu(h)
pred = PF.affine(h, 10, with_bias=True, name='fc2')
return pred
def capsule_loss(v_norm, t_onehot, recon=None, x=None, m_pos=0.9, m_neg=0.1, wn=0.5, wr=0.0005):
'''
Compute a margin loss given a length vector of output capsules and one-hot labels, and.optionally computes a reconstruction loss.
Margin loss is given in eq 4. Reconstruction loss is given in Sec 4.1.
Args:
v_norm (nnabla.Variable): A length vector of capsules. A shape of [B, capsules].
t_onehot (nnabla.Variable): A shape of [B, capsules].
recon (nnabla.Variable): Reconstruction output with a shape of [B, 1, 28, 28]. The values are in [0, 0.1].
x (nnabla.Variable): Reconstruction target (i.e. input) with a shape of [B, 1, 28, 28]. The values are in [0, 0.1].
m_pos (float): Margin of capsules corresponding targets.
m_neg (float): Margin of capsules corresponding non-targets.
wn (float): Weight of the non-target margin loss.
wr (float): Weight of the reconstruction loss.
Returns:
nnabla.Variable: 0-dim
'''
# Classification loss
lp = F.sum(t_onehot * F.relu(m_pos - v_norm) ** 2)
ln = F.sum((1 - t_onehot) * F.relu(v_norm - m_neg) ** 2)
lmargin = lp + wn * ln
if recon is None or x is None:
return lmargin / v_norm.shape[0]
# Reconstruction loss
lr = F.sum(F.squared_error(recon, x))
# return (lmargin + wr * lr) / v_norm.shape[0]
lmargin = lmargin / v_norm.shape[0]
lmargin.persistent = True
lreconst = (wr * lr) / v_norm.shape[0]
lreconst.persistent = True
return lmargin, lreconst, lmargin + lreconst
def res_unit(x, scope_name, dn=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C // 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C // 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def call(self, x, speaker_emb=None):
# causal convolution
with nn.parameter_scope("causal"):
pad = causal_padding(x, kernel_size=2, dilation=1)
current = PF.convolution(pad,
self.skip_dims,
kernel=(2, ),
dilation=(1, ))
# residual
skips = []
for index, dilation in enumerate(self.dilations):
with nn.parameter_scope("residual_{}".format(index)):
current, skip = self.residual_block(current, dilation,
speaker_emb)
skips.append(skip)
# output
out = F.relu(sum(skips))
with nn.parameter_scope("out1"):
y1 = F.relu(PF.convolution(out, self.skip_dims, kernel=(1, )))
with nn.parameter_scope("out2"):
y2 = PF.convolution(y1, self.output_channels, kernel=(1, ))
return y2
def mlp_net(x, n_h, n_y, test=False):
"""
Function for building multi-layer-perceptron with batch_normalization
Args:
x(`~nnabla.Variable`): N-D array
n_h(int): number of units in an intermediate layer
n_y(int): number of classes
test: operation type train=True, test=False
Returns:
~nnabla.Variable: h
"""
h = x
with nn.parameter_scope("fc1"):
h = F.relu(PF.batch_normalization(PF.affine(h, n_h),
batch_stat=not test),
inplace=True)
with nn.parameter_scope("fc2"):
h = F.relu(PF.batch_normalization(PF.affine(h, n_h),
batch_stat=not test),
inplace=True)
with nn.parameter_scope("fc3"):
h = PF.affine(h, n_y)
return h
def network(x, test=False):
# Input:x -> 1,128,128
# ImageAugmentation
h = F.image_augmentation(x, (1,128,128), (0,0), 1, 1, 0, 1, 0, False, False, 0, False, 1, 0.5, False, 0)
# Convolution -> 16,124,124
h = PF.convolution(h, 16, (5,5), (0,0), name='Convolution')
# ReLU
h = F.relu(h, True)
# MaxPooling -> 16,62,62
h = F.max_pooling(h, (2,2), (2,2))
# Convolution_2 -> 30,60,60
h = PF.convolution(h, 30, (3,3), (0,0), name='Convolution_2')
# MaxPooling_2 -> 30,30,30
h = F.max_pooling(h, (2,2), (2,2))
# Tanh_2
h = F.tanh(h)
# Affine -> 150
h = PF.affine(h, (150,), name='Affine')
# ReLU_2
h = F.relu(h, True)
# Affine_2 -> 2
h = PF.affine(h, (2,), name='Affine_2')
# Softmax
h = F.softmax(h)
return h
def bottleneck(x, planes, stride=1, downsample=None, test=True):
residual = x
out = PF.convolution(x,
planes,
kernel=(1, 1),
name='conv1',
with_bias=False)
out = PF.batch_normalization(out, batch_stat=not test, name='bn1')
out = F.relu(out, True)
out = PF.convolution(out,
planes,
kernel=(3, 3),
stride=(stride, stride),
pad=(1, 1),
name='conv2',
with_bias=False)
out = PF.batch_normalization(out, batch_stat=not test, name='bn2')
out = F.relu(out, True)
out = PF.convolution(out,
planes * 4,
kernel=(1, 1),
name='conv3',
with_bias=False)
out = PF.batch_normalization(out, batch_stat=not test, name='bn3')
if downsample is not None:
residual = downsample
out += residual
out = F.relu(out, True)
return out
def gated_conv(self,
x,
kernel_shape,
h=None,
mask_type='',
gated=True,
payload=None,
return_payload=False,
scope_name='gated_conv'):
pad_dim_0 = (kernel_shape[0] - 1) / 2
pad_dim_1 = (kernel_shape[1] - 1) / 2
if mask_type == '':
mask_type = self.mask_type_B
with nn.parameter_scope(scope_name):
if gated:
out_f = PF.convolution(x,
self.num_features,
kernel_shape,
pad=(pad_dim_0, pad_dim_1),
apply_w=mask_type,
name='conv_f')
out_g = PF.convolution(x,
self.num_features,
kernel_shape,
pad=(pad_dim_0, pad_dim_1),
apply_w=mask_type,
name='conv_g')
if isinstance(payload, nn.Variable):
out_f += payload[:, :self.num_features, :, :]
out_g += payload[:, self.num_features:, :, :]
if self.conditional:
h_out_f = PF.affine(h, self.num_features, name='h_out_f')
h_out_f = h_out_f.reshape(
(h_out_f.shape[0], h_out_f.shape[1], 1, 1))
h_out_g = PF.affine(h, self.num_features, name='h_out_g')
h_out_g = h_out_g.reshape(
(h_out_g.shape[0], h_out_g.shape[1], 1, 1))
out = F.tanh(out_f + h_out_f) * F.sigmoid(out_g + h_out_g)
else:
out = F.tanh(out_f) * F.sigmoid(out_g)
if return_payload:
payload = PF.convolution(F.concatenate(out_f,
out_g,
axis=1),
2 * self.num_features, (1, 1),
name='conv_1x1')
payload = F.relu(payload)
return out, payload
else:
out = PF.convolution(x,
self.num_features,
kernel_shape,
stride=(1, 1),
pad=(pad_dim_0, pad_dim_1),
apply_w=mask_type)
out = F.relu(out)
return out
def separable_conv_with_bn(x,
f,
stride=False,
aspp=False,
atrous_rate=1,
act_fn=True,
last_block=False,
end_point=False,
eps=1e-03,
out=False,
test=False,
fix_params=False):
with nn.parameter_scope("depthwise"):
if (stride == True):
h = PF.depthwise_convolution(x, (3, 3),
stride=(2, 2),
pad=(1, 1),
with_bias=False,
fix_parameters=fix_params)
elif (aspp == True):
h = PF.depthwise_convolution(x, (3, 3),
pad=(atrous_rate, atrous_rate),
stride=(1, 1),
dilation=(atrous_rate, atrous_rate),
with_bias=False,
fix_parameters=fix_params)
else:
h = PF.depthwise_convolution(x, (3, 3),
pad=(1, 1),
with_bias=False,
fix_parameters=fix_params)
h = PF.batch_normalization(h,
batch_stat=not test,
eps=eps,
fix_parameters=fix_params)
if last_block == True:
h = F.relu(h)
with nn.parameter_scope("pointwise"):
h = PF.convolution(h,
f, (1, 1),
stride=(1, 1),
with_bias=False,
fix_parameters=fix_params)
h = PF.batch_normalization(h,
batch_stat=not test,
eps=eps,
fix_parameters=fix_params)
if end_point == True:
global endpoints
endpoints['Decoder End Point 1'] = h
if act_fn == True:
h = F.relu(h)
return h
def q_network(obs, action, name):
with nn.parameter_scope(name):
out = F.concatenate(obs, action, axis=1)
out = PF.affine(out, 256, name='fc1')
out = F.relu(out)
out = PF.affine(out, 256, name='fc2')
out = F.relu(out)
return PF.affine(out, 1, name='fc3')
def policy_network(obs, action_size, name):
with nn.parameter_scope(name):
out = PF.affine(obs, 256, name='fc1')
out = F.relu(out)
out = PF.affine(out, 256, name='fc2')
out = F.relu(out)
out = PF.affine(out, action_size, name='fc3')
return F.tanh(out)
def __init__(self, channels, stride=1, skip_by_conv=True):
self.conv1 = ConvBn(channels // 4, 1, 1,
act=lambda x: F.relu(x, inplace=True))
self.conv2 = ConvBn(channels // 4, 3, stride,
act=lambda x: F.relu(x, inplace=True))
self.conv3 = ConvBn(channels, 1)
self.skip_by_conv = skip_by_conv
self.skip = ConvBn(channels, 1, stride)
def nips_head(obs):
out = PF.convolution(obs, 16, (8, 8), stride=(4, 4), name='conv1')
out = F.relu(out)
out = PF.convolution(out, 32, (4, 4), stride=(2, 2), name='conv2')
out = F.relu(out)
out = PF.affine(out, 256, name='fc1')
out = F.relu(out)
return out
def mlp(image, test=False):
image /= 255.0
c1 = F.relu(PF.convolution(image, 32, (3, 3), name='conv1'), inplace=True)
c2 = F.relu(PF.convolution(c1, 128, (3, 3), name='conv2'), inplace=True)
c3 = F.relu(PF.convolution(c2, 256, (3, 3), name='conv3'), inplace=True)
c4 = F.relu(PF.affine(c3, 512, name='fc3'), inplace=True)
c5 = PF.affine(c3, 10, name='fc4')
return F.softmax(c5)
def res_unit(x, scope_name, dn=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Pow2Quantize -> Relu
with nn.parameter_scope("conv1"):
h = PF.pow2_quantized_convolution(x,
C // 2,
kernel=(1, 1),
pad=(0, 0),
n_w=n,
m_w=m,
n_b=n,
m_b=m,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.pow2_quantize(h,
n=n,
m=m,
ste_fine_grained=ste_fine_grained)
h = F.relu(h)
# Conv -> BN -> Pow2Quantize -> Relu
with nn.parameter_scope("conv2"):
h = PF.pow2_quantized_convolution(h,
C // 2,
kernel=(3, 3),
pad=(1, 1),
n_w=n,
m_w=m,
n_b=n,
m_b=m,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.pow2_quantize(h,
n=n,
m=m,
ste_fine_grained=ste_fine_grained)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.pow2_quantized_convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
n_w=n,
m_w=m,
n_b=n,
m_b=m,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Pow2Quantize -> Relu
h = F.pow2_quantize(h, n=n, m=m, ste_fine_grained=ste_fine_grained)
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def hinge_gan_loss(r_out, f_out):
# D
d_gan_real = F.mean(F.relu(1. - r_out))
d_gan_fake = F.mean(F.relu(1. + f_out))
# G
g_gan = -1 * F.mean(f_out)
return d_gan_real, d_gan_fake, g_gan
def res_block(x, out_ch, name):
"""
Create residual block
"""
with nn.parameter_scope(name):
h = conv_2d(F.relu(x), out_ch, kernel=(3, 3), name='conv/0')
h = conv_2d(F.relu(h), out_ch, kernel=(3, 3), name='conv/1')
h = x + h
return h
def res_block_concat(x, out_ch, name):
"""
Basic residual block -> [conv-relu | conv-relu] + input
"""
with nn.parameter_scope(name):
h = conv_2d(F.relu(x), out_ch, kernel=(3, 3), name='conv/0')
h = conv_2d(F.relu(h), out_ch, kernel=(3, 3), name='conv/1')
h = x[:, :, :, :out_ch] + h
return h
def test_graph_model(model, seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4], need_grad=True)
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
nn.set_default_context(nn.Context())
# Forwardprop by definintion
nn.clear_parameters()
if model == "mlp":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
elif model == "recurrent":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
h = z2
for _ in range(2):
with nn.parameter_scope('fc2'):
h = PF.affine(h, 3)
h = F.relu(h, inplace=True)
with nn.parameter_scope('fc3'):
z3 = PF.affine(h, 5)
elif model == "convolution":
with nn.parameter_scope('conv1'):
z = PF.convolution(x, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
else:
raise ValueError()
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
L.forward(clear_no_need_grad=True)
# Backprop
# Diff should be initialized since they are always accumulated
x.grad.zero()
L.backward(clear_buffer=True)
x.g = rng.randn(*x.shape)
parameters = nn.get_parameters()
for param in parameters.values():
param.grad.zero()
inputs = [x] + list(parameters.values())
from nbla_test_utils import \
compute_analytical_and_numerical_grad_graph as grads
agrad, ngrad = grads(L, inputs, 1e-3)
assert np.allclose(ngrad, agrad, atol=1.05e-2)
def cnn(x, n_class):
c1 = PF.convolution(x, 16, (5, 5), name='conv1')
c1 = F.relu(F.max_pooling(c1, (2, 2)), inplace=True)
c2 = PF.convolution(c1, 16, (5, 5), name='conv2')
c2 = F.relu(F.max_pooling(c2, (2, 2)), inplace=True)
c3 = F.relu(PF.affine(c2, 50, name='fc3'), inplace=True)
c4 = PF.affine(c3, n_class, name='fc4')
return c4
def test_graph_model(model, seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4], need_grad=True)
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
nn.set_default_context(nn.Context())
# Forwardprop by definition
nn.clear_parameters()
if model == "mlp":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
elif model == "recurrent":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 8)
z2 = F.relu(z, inplace=True)
h = z2
for _ in range(2):
with nn.parameter_scope('fc2'):
h = PF.affine(h, 8)
h = F.relu(h, inplace=True)
with nn.parameter_scope('fc3'):
z3 = PF.affine(h, 5)
elif model == "convolution":
with nn.parameter_scope('conv1'):
z = PF.convolution(x, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
else:
raise ValueError()
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
L.forward(clear_no_need_grad=True)
# Backprop
# Diff should be initialized since they are always accumulated
x.grad.zero()
L.backward(clear_buffer=True)
x.g = rng.randn(*x.shape)
parameters = nn.get_parameters()
for param in parameters.values():
param.grad.zero()
inputs = [x] + list(parameters.values())
from nbla_test_utils import \
compute_analytical_and_numerical_grad_graph as grads
agrad, ngrad = grads(L, inputs, 1e-3)
assert_allclose(ngrad, agrad, atol=1.05e-2)
def model(img, sf):
"""
Define JSInet model
"""
with nn.parameter_scope('Network'):
with nn.parameter_scope('local_contrast_enhancement'):
## ================= Local Contrast Enhancement Subnet ============================ ##
ch = 64
b = guided_filter(img, 5, 0.01)
n1 = conv_2d(b, ch, kernel=(3, 3), name='conv/0')
for i in range(4):
n1 = res_block(n1, ch, 'res_block/%d' % i)
n1 = F.relu(n1, inplace=True)
local_filter_2d = conv_2d(
n1, (9**2) * (sf**2), kernel=(3, 3),
name='conv_k') # [B, H, W, (9x9)*(sfxsf)]
# dynamic 2D upsampling with 2D local filters
pred_C = dyn_2d_up_operation(b, local_filter_2d, (9, 9), sf)
# local contrast mask
pred_C = 2 * F.sigmoid(pred_C)
## ================= Detail Restoration Subnet ============================ ##
ch = 64
d = F.div2(img, b + 1e-15)
with nn.parameter_scope('detail_restoration'):
n3 = conv_2d(d, ch, kernel=(3, 3), name='conv/0')
for i in range(4):
n3 = res_block(n3, ch, 'res_block/%d' % i)
if i == 0:
d_feature = n3
n3 = F.relu(n3, inplace=True)
# separable 1D filters
dr_k_h = conv_2d(n3, 41 * sf**2, kernel=(3, 3), name='conv_k_h')
dr_k_v = conv_2d(n3, 41 * sf**2, kernel=(3, 3), name='conv_k_v')
# dynamic separable upsampling with with separable 1D local filters
pred_D = dyn_sep_up_operation(d, dr_k_v, dr_k_h, 41, sf)
## ================= Image Reconstruction Subnet ============================ ##
with nn.parameter_scope('image_reconstruction'):
n4 = conv_2d(img, ch, kernel=(3, 3), name='conv/0')
for i in range(4):
if i == 1:
n4 = F.concatenate(n4, d_feature, axis=3)
n4 = res_block_concat(n4, ch, 'res_block/%d' % i)
else:
n4 = res_block(n4, ch, 'res_block/%d' % i)
n4 = F.relu(n4, inplace=True)
n4 = F.relu(conv_2d(n4, ch * sf * sf, kernel=(3, 3),
name='conv/1'),
inplace=True)
# (1,100,170,1024) -> (1,100,170,4,4,64) -> (1,100,4,170,4,64)
# pixel shuffle
n4 = depth_to_space(n4, sf)
pred_I = conv_2d(n4, 3, kernel=(3, 3), name='conv/2')
pred = F.add2(pred_I, pred_D, inplace=True) * pred_C
jsinet = namedtuple('jsinet', ['pred'])
return jsinet(pred)
def mlp(x, maps, num_res=4, num_layers=2, name="mlp"):
h = x
with nn.parameter_scope(name):
h = PF.affine(h, maps, name="affine-first")
h = F.relu(h, True)
h = PF.affine(h, maps, name="affine-mid")
h = F.relu(h, True)
h = PF.affine(h, 2 * maps * num_res * num_layers, name="affine-last")
return h
def _check_context(ctx):
try:
x = nn.Variable()
F.relu(x)
except:
logger.warn('Fallback to CPU context.')
import nnabla_ext.cpu
ctx = nnabla_ext.cpu.context()
return ctx
def cifar10_resnet23_prediction(ctx, image, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = bn_dropout(h, "bn_dropout1", test)
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = bn_dropout(h, "bn_dropout2", test)
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = bn_dropout(h, "bn_dropout3", test)
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def test_cuda_large_blocks(m):
CUDA_THREAD_PER_BLOCK = 512
CUDA_MAX_BLOCKS = 65536
size = CUDA_MAX_BLOCKS * CUDA_THREAD_PER_BLOCK * m + 3
print "Variable size:", size
x = np.zeros((size,), np.float32)
v = nn.Variable(x.shape)
v.d = x
ctx = nn.Context(backend='cuda')
y = F.relu(v)
def mlp_net(x, n_h, n_y, test=False):
"""
Function for building multi-layer-perceptron with batch_normalization
Args:
x(`~nnabla.Variable`): N-D array
n_h(int): number of units in an intermediate layer
n_y(int): number of classes
test: operation type train=True, test=False
Returns:
~nnabla.Variable: log(p(y|x))
"""
h = x
with nn.parameter_scope("fc1"):
h = F.relu(PF.batch_normalization(
PF.affine(h, n_h), batch_stat=not test), inplace=True)
with nn.parameter_scope("fc2"):
h = F.relu(PF.batch_normalization(
PF.affine(h, n_h), batch_stat=not test), inplace=True)
with nn.parameter_scope("fc3"):
h = PF.affine(h, n_y)
return h
def test_graph_clear_buffer(seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4])
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
# Network definition
nn.set_default_context(nn.Context())
nn.clear_parameters()
x1 = x + 1
x2 = x1 - 1
with nn.parameter_scope('conv1'):
z = PF.convolution(x2, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
import tempfile
import os
tmpd = tempfile.mkdtemp()
nn.save_parameters(os.path.join(tmpd, 'parameter.h5'))
first = False
for cnng in [False, True]:
for cb in [False, True]:
_ = nn.load_parameters(os.path.join(tmpd, 'parameter.h5'))
for v in nn.get_parameters().values():
v.grad.zero()
L.forward(clear_no_need_grad=cnng)
L.backward(clear_buffer=cb)
if not first:
first = True
g = list(nn.get_parameters().values())[0].g.copy()
else:
g2 = list(nn.get_parameters().values())[0].g.copy()
assert np.all(g == g2)
def cifar100_resnet23_prediction(image,
ctx, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
rng = np.random.RandomState(0)
nmaps = 384
ncls = 100
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
# Preprocess
if not test:
image = F.image_augmentation(image, contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
w_init = UniformInitializer(
calc_uniform_lim_glorot(3, nmaps, kernel=(3, 3)),
rng=rng)
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", rng, False) # -> 32x32
h = res_unit(h, "conv3", rng, True) # -> 16x16
h = res_unit(h, "conv4", rng, False) # -> 16x16
h = res_unit(h, "conv5", rng, True) # -> 8x8
h = res_unit(h, "conv6", rng, False) # -> 8x8
h = res_unit(h, "conv7", rng, True) # -> 4x4
h = res_unit(h, "conv8", rng, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
w_init = UniformInitializer(
calc_uniform_lim_glorot(int(np.prod(h.shape[1:])), ncls, kernel=(1, 1)), rng=rng)
pred = PF.affine(h, ncls, w_init=w_init)
return pred
