def test_save_load_parameters():
v = nn.Variable([64, 1, 28, 28], need_grad=False)
with nn.parameter_scope("param1"):
with nn.parameter_scope("conv1"):
h = PF.convolution(v, 32, (3, 3))
b = PF.batch_normalization(h, batch_stat=True)
with nn.parameter_scope("conv2"):
h1 = PF.convolution(v, 32, (3, 3))
b2 = PF.batch_normalization(h1, batch_stat=True)
for k, v in iteritems(nn.get_parameters(grad_only=False)):
v.data.cast(np.float32)[...] = np.random.randn(*v.shape)
with nn.parameter_scope("param1"):
param1 = nn.get_parameters(grad_only=False)
nn.save_parameters("tmp.h5")
nn.save_parameters("tmp.protobuf")
with nn.parameter_scope("param2"):
nn.load_parameters('tmp.h5')
param2 = nn.get_parameters(grad_only=False)
with nn.parameter_scope("param3"):
nn.load_parameters('tmp.protobuf')
param3 = nn.get_parameters(grad_only=False)
for par2 in [param2, param3]:
assert param1.keys() == par2.keys() # Check order
for (n1, p1), (n2, p2) in zip(sorted(param1.items()), sorted(par2.items())):
assert n1 == n2
assert np.all(p1.d == p2.d)
assert p1.data.dtype == p2.data.dtype
assert p1.need_grad == p2.need_grad
def cnn_model_003(ctx, x, act=F.relu, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
# Convblock 3
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def resnet_model(ctx, x, inmaps=64, act=F.relu, test=False):
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(x, inmaps, kernel=(3, 3), pad=(1, 1), with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
h = res_unit(h, "conv2", act, False) # -> 32x32
h = res_unit(h, "conv3", act, True) # -> 16x16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv4", act, False) # -> 16x16
h = res_unit(h, "conv5", act, True) # -> 8x8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv6", act, False) # -> 8x8
h = res_unit(h, "conv7", act, True) # -> 4x4
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv8", act, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, 10)
return pred
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h_branch = h
# Convblock 3
h = conv_unit(h_branch, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u0 = F.average_pooling(u0, (6, 6))
with nn.parameter_scope("u0bn"):
u0 = PF.batch_normalization(u0, batch_stat=not test)
log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:])))
# Uncertainty for uncertainty
u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test)
u1 = F.average_pooling(u1, (6, 6))
with nn.parameter_scope("u1bn"):
u1 = PF.batch_normalization(u1, batch_stat=not test)
log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:])))
return pred, log_var, log_s
def cnn_model_003(ctx, h, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
if not test:
b, c, s, s = h.shape
h = F.image_augmentation(h, (c, s, s),
min_scale=1.0, max_scale=1.5,
angle=0.5, aspect_ratio=1.3, distortion=0.2,
flip_lr=True)
# Convblock0
h = conv_unit(h, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (6, 6))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def cifar10_resnet23_prediction(ctx, image, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = bn_dropout(h, "bn_dropout1", test)
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = bn_dropout(h, "bn_dropout2", test)
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = bn_dropout(h, "bn_dropout3", test)
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def conv_unit(x, scope, maps, k=4, s=2, p=1, act=F.relu, test=False):
with nn.parameter_scope(scope):
h = PF.convolution(x, maps, kernel=(k, k), stride=(s, s), pad=(p, p))
if act is None:
return h
h = PF.batch_normalization(h, batch_stat=not test)
h = act(h)
return h
def res_block(x, scope_name, act=F.relu, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C/2, kernel=(1, 1), pad=(0, 0), with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C/2, kernel=(3, 3), pad=(1, 1), with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0), with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
return h
def conv_unit(x, scope, maps, k=4, s=2, p=1, act=F.prelu, test=False):
with nn.parameter_scope(scope):
h = PF.convolution(x, maps, kernel=(k, k), stride=(s, s), pad=(p, p))
h = PF.batch_normalization(h, batch_stat=not test)
shape = h.shape
w = nn.Variable()
w.d = 0.3
h = act(h, w)
return h
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 28 -> 14
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 14 -> 7
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 7 -> 5
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (5, 5))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (5, 5))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def mnist_binary_connect_lenet_prediction(image, test=False):
"""
Construct LeNet for MNIST (BinaryNet version).
"""
with nn.parameter_scope("conv1"):
c1 = PF.binary_connect_convolution(image, 16, (5, 5))
c1 = PF.batch_normalization(c1, batch_stat=not test)
c1 = F.elu(F.average_pooling(c1, (2, 2)))
with nn.parameter_scope("conv2"):
c2 = PF.binary_connect_convolution(c1, 16, (5, 5))
c2 = PF.batch_normalization(c2, batch_stat=not test)
c2 = F.elu(F.average_pooling(c2, (2, 2)))
with nn.parameter_scope("fc3"):
c3 = PF.binary_connect_affine(c2, 50)
c3 = PF.batch_normalization(c3, batch_stat=not test)
c3 = F.elu(c3)
with nn.parameter_scope("fc4"):
c4 = PF.binary_connect_affine(c3, 10)
c4 = PF.batch_normalization(c4, batch_stat=not test)
return c4
def mlp_net(x, n_h, n_y, test=False):
"""
Function for building multi-layer-perceptron with batch_normalization
Args:
x(`~nnabla.Variable`): N-D array
n_h(int): number of units in an intermediate layer
n_y(int): number of classes
test: operation type train=True, test=False
Returns:
~nnabla.Variable: log(p(y|x))
"""
h = x
with nn.parameter_scope("fc1"):
h = F.relu(PF.batch_normalization(
PF.affine(h, n_h), batch_stat=not test), inplace=True)
with nn.parameter_scope("fc2"):
h = F.relu(PF.batch_normalization(
PF.affine(h, n_h), batch_stat=not test), inplace=True)
with nn.parameter_scope("fc3"):
h = PF.affine(h, n_y)
return h
def conv_unit(x, scope, maps, k=4, s=2, p=1, act=F.relu, test=False):
with nn.parameter_scope(scope):
h = PF.convolution(x, maps, kernel=(k, k), stride=(s, s), pad=(p, p))
h = PF.batch_normalization(h, batch_stat=not test)
h = act(h)
return h
def batch_normalization(h, cnt=0, test=False):
with nn.parameter_scope("{}".format(cnt)):
h = PF.batch_normalization(h, batch_stat=not test)
return h
def bn(xx):
# Batch normalization
return PF.batch_normalization(xx, batch_stat=not test)
def construct_architecture(image, num_class, operations, output_filter, test, connect_patterns):
"""
Architecture Construction.
"""
ops = {0: conv3x3, 1: conv5x5, 2: depthwise_separable_conv3x3,
3: depthwise_separable_conv5x5, 4: max_pool, 5: average_pool}
used_weights = set()
pool_distance = len(operations) // 3
pool_layers = [pool_distance - 1, 2*pool_distance - 1]
# exclude negative indices
pool_layers = [idx for idx in pool_layers if idx > 0]
ref_groups = len(operations) * [0]
tmp_list = pool_layers + [len(operations) - 1]
index = 0
for n in range(len(operations)):
if n <= tmp_list[index]:
ref_groups[n] = index
else:
index += 1
ref_groups[n] = index
# elements in ref_groups tell you how many times you need to do pooling.
# e.g. [0, 0, 0, 1, 1, 1, ..., 2] : the 1st layer needs no pooling,
# but the last needs 2 poolings, to get spatially reduced variables.
#required_indices = get_requirement_soft(ref_groups)
required_indices = get_requirement_strict(
ref_groups, connect_patterns, pool_layers)
num_of_pooling = len(pool_layers)
normal_layers = [list()]
pooled_layers = [list() for j in range(num_of_pooling)]
prev_layers = normal_layers + pooled_layers
# prev_layer consists of: [[initial_size_layers], [1x pooled_layers], [2x pooled_layers], ...]
if not test:
image = F.image_augmentation(image, angle=0.25, flip_lr=True)
image.need_grad = False
x = image
# next comes the basic operation. for the first layer,
# just apply a convolution (to make the size of the input the same as that of successors)
with nn.parameter_scope("stem_conv"):
x = PF.convolution(x, output_filter, (3, 3), (1, 1), with_bias=False)
x = PF.batch_normalization(x, batch_stat=not test)
used_weights.update(
{"stem_conv/conv/W", "stem_conv/bn/gamma", "stem_conv/bn/beta"})
prev_layers[0].append(x)
# "unpooled" variable is stored in normal_layers (prev_layers[0]).
# then apply factorized reduction (kind of pooling),
# but ONLY IF the spatially-reduced variable is required.
# for example, when this layer has skip connection with latter layers.
for j in range(1, len(prev_layers)):
if required_indices[0][j]:
nested_scope = "stem_pool_{}".format(j)
reduced_var = factorized_reduction(
prev_layers[j - 1][-1], output_filter, nested_scope, test)
used_weights.update(get_factorized_weights_name(nested_scope))
else:
# dummy variable. Should never be used.
reduced_var = nn.Variable([1, 1, 1, 1])
prev_layers[j].append(reduced_var)
# reduced (or "pooled") variable is stored in pooled_layers (prev_layers[1:]).
# basically, repeat the same process, for whole layers.
for i, elem in enumerate(operations):
scope = 'w{}_{}'.format(i, elem)
# basic operation (and connects it with previous layers if it has skip connections)
using_layer_index = ref_groups[i]
connect_pattern = connect_patterns[i]
x, local_used_weights = apply_ops_and_connect(prev_layers[using_layer_index][-1],
prev_layers[using_layer_index], connect_pattern, ops, elem, output_filter, scope, test)
used_weights.update(local_used_weights)
prev_layers[using_layer_index].append(x)
# factorized reduction
for j in range(using_layer_index + 1, len(prev_layers)):
if required_indices[i + 1][j]:
nested_scope = "{0}_pool{1}".format(scope, j)
reduced_var = factorized_reduction(
prev_layers[j - 1][-1], output_filter, nested_scope, test)
used_weights.update(get_factorized_weights_name(nested_scope))
else:
reduced_var = nn.Variable([1, 1, 1, 1]) # dummy variable.
prev_layers[j].append(reduced_var)
x = F.global_average_pooling(x)
if not test:
dropout_rate = 0.5
x = F.dropout(x, dropout_rate)
with nn.parameter_scope("fc"):
pred = PF.affine(x, num_class, with_bias=False)
used_weights.add("fc/affine/W")
return pred, used_weights
def atrous_spatial_pyramid_pooling(x,
output_stride,
test=False,
fix_params=False):
if output_stride not in [8, 16]:
raise ValueError('output_stride neither 8 nor 16.')
depth = 256
atrous_rates = [6, 12, 18]
if output_stride == 8:
atrous_rates = [2 * rate for rate in atrous_rates]
with nn.parameter_scope("aspp0"):
atrous_conv0 = PF.convolution(x,
depth, (1, 1),
with_bias=False,
fix_parameters=fix_params)
atrous_conv0 = F.relu(
PF.batch_normalization(atrous_conv0,
batch_stat=not test,
fix_parameters=fix_params))
atrous_conv = []
for i in range(3):
with nn.parameter_scope("aspp" + str(i + 1)):
atrous_conv.append(
xception.separable_conv_with_bn(x,
depth,
stride=False,
aspp=True,
atrous_rate=atrous_rates[i],
last_block=True,
eps=1e-05,
test=test,
fix_params=fix_params))
# Image-level features
with nn.parameter_scope("image_pooling"):
x_shape = x.shape[2]
h = F.average_pooling(x, (x.shape[2], x.shape[3]),
stride=(x.shape[2], x.shape[2]))
h = PF.convolution(h,
depth, (1, 1),
with_bias=False,
fix_parameters=fix_params)
h = F.relu(
PF.batch_normalization(h,
batch_stat=not test,
fix_parameters=fix_params))
h = F.interpolate(h,
output_size=(x.shape[2], x.shape[3]),
mode='linear')
with nn.parameter_scope("concat_projection"):
h5 = F.concatenate(h,
atrous_conv0,
atrous_conv[0],
atrous_conv[1],
atrous_conv[2],
axis=1)
return h5
def cifar100_resnet23_prediction(image,
ctx, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(
calc_uniform_lim_glorot(C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
rng = np.random.RandomState(0)
nmaps = 384
ncls = 100
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
# Preprocess
if not test:
image = F.image_augmentation(image, contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
w_init = UniformInitializer(
calc_uniform_lim_glorot(3, nmaps, kernel=(3, 3)),
rng=rng)
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
w_init=w_init, with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", rng, False) # -> 32x32
h = res_unit(h, "conv3", rng, True) # -> 16x16
h = res_unit(h, "conv4", rng, False) # -> 16x16
h = res_unit(h, "conv5", rng, True) # -> 8x8
h = res_unit(h, "conv6", rng, False) # -> 8x8
h = res_unit(h, "conv7", rng, True) # -> 4x4
h = res_unit(h, "conv8", rng, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
w_init = UniformInitializer(
calc_uniform_lim_glorot(int(np.prod(h.shape[1:])), ncls, kernel=(1, 1)), rng=rng)
pred = PF.affine(h, ncls, w_init=w_init)
return pred
def bn_dropout(h, scope_name, test=False):
with nn.parameter_scope(scope_name):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
return h
def bn(h):
axes = [3 if channel_last else 1]
return PF.batch_normalization(h, axes=axes, batch_stat=not test)
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def __call__(self, x):
if not isinstance(x, nn._variable.Variable):
input_variable = nn.Variable(x.shape)
if isinstance(x, np.ndarray):
input_variable.d = x
else:
input_variable.data = x
else:
input_variable = x
axes = 3 if self.channel_last else 1
r, hidden = self.backbone_model(input_variable,
num_classes=1000,
num_layers=self.num_layers,
shortcut_type='b',
test=not self.training,
channel_last=self.channel_last)
with nn.parameter_scope("upsample1"):
kernel_size = self.kernels_size[0]
features = pf_deconvolution(hidden['r4'],
self.ochannels[0],
(kernel_size, kernel_size),
pad=(1, 1),
stride=(2, 2),
dilation=(1, 1),
w_init=self.n_init,
with_bias=False,
channel_last=self.channel_last)
features = PF.batch_normalization(
features,
axes=[axes],
batch_stat=self.training,
param_init={
'gamma': ConstantInitializer(1),
'beta': ConstantInitializer(0)
},
)
features = F.relu(features)
with nn.parameter_scope("upsample2"):
kernel_size = self.kernels_size[1]
features = pf_deconvolution(features,
self.ochannels[1],
(kernel_size, kernel_size),
pad=(1, 1),
stride=(2, 2),
dilation=(1, 1),
w_init=self.n_init,
with_bias=False,
channel_last=self.channel_last)
features = F.relu(
PF.batch_normalization(features,
axes=[axes],
batch_stat=self.training,
param_init={
'gamma': ConstantInitializer(1),
'beta': ConstantInitializer(0)
}))
with nn.parameter_scope("upsample3"):
kernel_size = self.kernels_size[2]
features = pf_deconvolution(features,
self.ochannels[2],
(kernel_size, kernel_size),
pad=(1, 1),
stride=(2, 2),
dilation=(1, 1),
w_init=self.n_init,
with_bias=False,
channel_last=self.channel_last)
features = F.relu(
PF.batch_normalization(features,
axes=[axes],
batch_stat=self.training,
param_init={
'gamma': ConstantInitializer(1),
'beta': ConstantInitializer(0)
}))
output = []
for head in sorted(self.heads):
num_output = self.heads[head]
rng = np.random.RandomState(313)
b_init_param = -2.19 if head == 'hm' else 0.0
if self.head_conv > 0:
with nn.parameter_scope(head + "_conv1"):
w_init_param = torch_initializer(
features.shape[axes],
(3, 3)) if head == 'hm' else self.n_init
out = pf_convolution(
features,
self.head_conv,
(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=True,
w_init=w_init_param,
b_init=ConstantInitializer(b_init_param),
channel_last=self.channel_last,
)
out = F.relu(out)
with nn.parameter_scope(head + "_final"):
w_init_param = torch_initializer(
out.shape[axes],
(1, 1)) if head == 'hm' else self.n_init
out = pf_convolution(
out,
num_output, (1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=True,
w_init=w_init_param,
b_init=ConstantInitializer(b_init_param),
channel_last=self.channel_last)
else:
with nn.parameter_scope(head + "_final"):
out = pf_convolution(
features,
num_output, (1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=True,
w_init=w_init_param,
b_init=ConstantInitializer(b_init_param),
channel_last=self.channel_last)
output.append(out)
return output
def cifar10_resnet23_prediction(ctx, scope, image, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x,
C / 2,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h,
C / 2,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope(scope):
with nn.parameter_scope("conv1"):
h = PF.convolution(image,
nmaps,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def module_C(input_variable, use_max_pool=False, eps=1e-3):
with nn.parameter_scope(f"Conv"):
with nn.parameter_scope("Convolution"):
h0 = PF.convolution(input_variable,
outmaps=320,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h0 = PF.batch_normalization(h0, batch_stat=False, eps=eps)
h0 = F.relu(h0)
#################################################################
with nn.parameter_scope(f"Conv_2"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(input_variable,
outmaps=384,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
with nn.parameter_scope(f"Conv_3"):
with nn.parameter_scope("Convolution"):
h11 = PF.convolution(h1,
outmaps=384,
kernel=(1, 3),
pad=(0, 1),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h11 = PF.batch_normalization(h11, batch_stat=False, eps=eps)
h11 = F.relu(h11)
with nn.parameter_scope(f"Conv_8"):
with nn.parameter_scope("Convolution"):
h12 = PF.convolution(h1,
outmaps=384,
kernel=(3, 1),
pad=(1, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h12 = PF.batch_normalization(h12, batch_stat=False, eps=eps)
h12 = F.relu(h12)
#################################################################
with nn.parameter_scope(f"Conv_4"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(input_variable,
outmaps=448,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_5"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=384,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_6"):
with nn.parameter_scope("Convolution"):
h21 = PF.convolution(h2,
outmaps=384,
kernel=(1, 3),
pad=(0, 1),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h21 = PF.batch_normalization(h21, batch_stat=False, eps=eps)
h21 = F.relu(h21)
with nn.parameter_scope(f"Conv_9"):
with nn.parameter_scope("Convolution"):
h22 = PF.convolution(h2,
outmaps=384,
kernel=(3, 1),
pad=(1, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h22 = PF.batch_normalization(h22, batch_stat=False, eps=eps)
h22 = F.relu(h22)
#################################################################
with nn.parameter_scope(f"Conv_7"):
if use_max_pool:
h3 = F.max_pooling(input_variable,
kernel=(3, 3),
stride=(1, 1),
pad=(1, 1))
else:
h3 = F.average_pooling(input_variable,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
including_pad=False)
with nn.parameter_scope("Convolution"):
h3 = PF.convolution(h3,
outmaps=192,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h3 = PF.batch_normalization(h3, batch_stat=False, eps=eps)
h3 = F.relu(h3)
h = F.concatenate(*[h0, h11, h12, h21, h22, h3], axis=1)
return h
def module_A(input_variable, is_first=False, eps=1e-3):
with nn.parameter_scope(f"Conv"):
with nn.parameter_scope("Convolution"):
h0 = PF.convolution(input_variable,
outmaps=64,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h0 = PF.batch_normalization(h0, batch_stat=False, eps=eps)
h0 = F.relu(h0)
#################################################################
with nn.parameter_scope(f"Conv_2"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(input_variable,
outmaps=48,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
with nn.parameter_scope(f"Conv_3"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(h1,
outmaps=64,
kernel=(5, 5),
pad=(2, 2),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
#################################################################
with nn.parameter_scope(f"Conv_4"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(input_variable,
outmaps=64,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_5"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=96,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_6"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=96,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
#################################################################
with nn.parameter_scope(f"Conv_7"):
h3 = F.average_pooling(input_variable,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
including_pad=False)
with nn.parameter_scope("Convolution"):
if is_first:
outmaps = 32
else:
outmaps = 64
h3 = PF.convolution(h3,
outmaps=outmaps,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h3 = PF.batch_normalization(h3, batch_stat=False, eps=eps)
h3 = F.relu(h3)
h = F.concatenate(*[h0, h1, h2, h3], axis=1)
return h
def grid_size_reduction_B(input_variable, eps=1e-3):
with nn.parameter_scope(f"Conv_2"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(input_variable,
outmaps=192,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
with nn.parameter_scope(f"Conv"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(h1,
outmaps=320,
kernel=(3, 3),
pad=(0, 0),
stride=(2, 2),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
###########################################################
with nn.parameter_scope(f"Conv_4"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(input_variable,
outmaps=192,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_5"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=192,
kernel=(1, 7),
pad=(0, 3),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_3"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=192,
kernel=(7, 1),
pad=(3, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_6"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=192,
kernel=(3, 3),
pad=(0, 0),
stride=(2, 2),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
#################################################################
h3 = F.max_pooling(input_variable,
kernel=(3, 3),
pad=(0, 0),
stride=(2, 2))
h = F.concatenate(*[h1, h2, h3], axis=1)
return h
def module_B(input_variable, internal_outmaps=128, eps=1e-3):
with nn.parameter_scope(f"Conv"):
with nn.parameter_scope("Convolution"):
h0 = PF.convolution(input_variable,
outmaps=192,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h0 = PF.batch_normalization(h0, batch_stat=False, eps=eps)
h0 = F.relu(h0)
#################################################################
with nn.parameter_scope(f"Conv_2"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(input_variable,
outmaps=internal_outmaps,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
with nn.parameter_scope(f"Conv_8"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(h1,
outmaps=internal_outmaps,
kernel=(1, 7),
pad=(0, 3),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
with nn.parameter_scope(f"Conv_3"):
with nn.parameter_scope("Convolution"):
h1 = PF.convolution(h1,
outmaps=192,
kernel=(7, 1),
pad=(3, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h1 = PF.batch_normalization(h1, batch_stat=False, eps=eps)
h1 = F.relu(h1)
#################################################################
with nn.parameter_scope(f"Conv_4"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(input_variable,
outmaps=internal_outmaps,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_9"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=internal_outmaps,
kernel=(7, 1),
pad=(3, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_10"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=internal_outmaps,
kernel=(1, 7),
pad=(0, 3),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_5"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=internal_outmaps,
kernel=(7, 1),
pad=(3, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
with nn.parameter_scope(f"Conv_6"):
with nn.parameter_scope("Convolution"):
h2 = PF.convolution(h2,
outmaps=192,
kernel=(1, 7),
pad=(0, 3),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h2 = PF.batch_normalization(h2, batch_stat=False, eps=eps)
h2 = F.relu(h2)
#################################################################
with nn.parameter_scope(f"Conv_7"):
h3 = F.average_pooling(input_variable,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
including_pad=False)
with nn.parameter_scope("Convolution"):
h3 = PF.convolution(h3,
outmaps=192,
kernel=(1, 1),
pad=(0, 0),
stride=(1, 1),
with_bias=False)
with nn.parameter_scope("BatchNormalization"):
h3 = PF.batch_normalization(h3, batch_stat=False, eps=eps)
h3 = F.relu(h3)
h = F.concatenate(*[h0, h1, h2, h3], axis=1)
return h
def BN(h, decay_rate=0.999, test=False):
"""Batch Normalization"""
return PF.batch_normalization(h,
decay_rate=decay_rate,
batch_stat=not test)
def resnet_prediction(image, test=False, ncls=2, nmaps=128, act=F.relu):
# Residual Unit
def res_unit(x, scope_name, dn=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Nonlinear
with nn.parameter_scope("conv1"):
h = PF.convolution(x,
C,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = act(h)
# Conv -> BN -> Nonlinear
with nn.parameter_scope("conv2"):
h = PF.convolution(h,
C,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = act(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Nonlinear
h = act(F.add2(h, x, inplace=False))
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Conv -> BN -> Nonlinear
with nn.parameter_scope("conv1"):
# Preprocess
if not test:
image = F.image_augmentation(image,
contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
h = PF.convolution(image,
nmaps,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = act(h)
image_size = image.shape[-1]
for i in range(int(np.log2(image_size)) - 1):
h = res_unit(h, f'conv{i*2+2}', False)
if i != np.log2(image_size) - 2:
h = res_unit(h, f'conv{i*2+3}', True)
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def cifar10_resnet23_prediction(image, maps=64, test=False):
"""
Construct Resnet23.
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x,
C / 2,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h,
C / 2,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
ncls = 10
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
# Preprocess
image /= 255.0
if not test:
image = F.image_augmentation(image,
contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
h = PF.convolution(image,
maps,
kernel=(3, 3),
pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def bn(xx):
# Batch normalization
return PF.batch_normalization(xx, batch_stat=not test)
def netG_decoder(x, test=False):
# x: (1, 15, 64, 64) -> c0: (1, 15, 128, 128)
with nn.parameter_scope('ReluDeconvBN1'):
c0 = PF.batch_normalization(PF.deconvolution(F.relu(x),
15, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# c0: (1, 15, 128, 128) -> c1: (1, 15, 256, 256)
with nn.parameter_scope('ReluDeconvBN2'):
c1 = F.tanh(
PF.deconvolution(F.relu(c0), 15, (4, 4), pad=(1, 1),
stride=(2, 2)))
# c1: (1, 15, 256, 256) -> down_0: (1, 64, 128, 128)
with nn.parameter_scope('down0'):
down_0 = PF.convolution(c1,
64, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False)
# down_0: (1, 64, 128, 128) -> down_1: (1, 128, 64, 64)
with nn.parameter_scope('down1'):
down_1 = PF.batch_normalization(PF.convolution(F.leaky_relu(down_0,
alpha=0.2),
128, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_1: (1, 128, 64, 64) -> down_2: (1, 256, 32, 32)
with nn.parameter_scope('down2'):
down_2 = PF.batch_normalization(PF.convolution(F.leaky_relu(down_1,
alpha=0.2),
256, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_2: (1, 256, 32, 32) -> down_3: (1, 512, 16, 16)
with nn.parameter_scope('down3'):
down_3 = PF.batch_normalization(PF.convolution(F.leaky_relu(down_2,
alpha=0.2),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_3: (1, 512, 16, 16) -> down_4: (1, 512, 8, 8)
with nn.parameter_scope('down4'):
down_4 = PF.batch_normalization(PF.convolution(F.leaky_relu(down_3,
alpha=0.2),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_4: (1, 512, 8, 8) -> down_5: (1, 512, 4, 4)
with nn.parameter_scope('down5'):
down_5 = PF.batch_normalization(PF.convolution(F.leaky_relu(down_4,
alpha=0.2),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_5: (1, 512, 4, 4) -> down_6: (1, 512, 2, 2)
with nn.parameter_scope('down6'):
down_6 = PF.batch_normalization(PF.convolution(F.leaky_relu(down_5,
alpha=0.2),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_6: (1, 512, 2, 2) -> down_7: (1, 512, 1, 1)
with nn.parameter_scope('down7'):
down_7 = PF.convolution(F.leaky_relu(down_6, alpha=0.2),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False)
# down_7: (1, 512, 1, 1) -> up_0: (1, 512, 2, 2)
with nn.parameter_scope('up0'):
up_0 = PF.batch_normalization(PF.deconvolution(F.relu(down_7),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_6: (1, 512, 2, 2) + up_0: (1, 512, 2, 2) -> up_1: (1, 512, 4, 4)
with nn.parameter_scope('up1'):
up_1 = PF.batch_normalization(PF.deconvolution(F.relu(
F.concatenate(down_6, up_0, axis=1)),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
if not test:
up_1 = F.dropout(up_1, 0.5)
# down_5: (1, 512, 4, 4) + up_1: (1, 512, 4, 4)-> up_2: (1, 512, 8, 8)
with nn.parameter_scope('up2'):
up_2 = PF.batch_normalization(PF.deconvolution(F.relu(
F.concatenate(down_5, up_1, axis=1)),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
if not test:
up_2 = F.dropout(up_2, 0.5)
# down_4: (1, 512, 8, 8) + up_2: (1, 512, 8, 8) -> up_3: (1, 512, 16, 16)
with nn.parameter_scope('up3'):
up_3 = PF.batch_normalization(PF.deconvolution(F.relu(
F.concatenate(down_4, up_2, axis=1)),
512, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
if not test:
up_3 = F.dropout(up_3, 0.5)
# down_3: (1, 512, 16, 16) + up_3: (1, 512, 16, 16) -> up_4: (1, 256, 32, 32)
with nn.parameter_scope('up4'):
up_4 = PF.batch_normalization(PF.deconvolution(F.relu(
F.concatenate(down_3, up_3, axis=1)),
256, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_2: (1, 256, 32, 32) + up_4: (1, 256, 32, 32) -> up_5: (1, 128, 64, 64)
with nn.parameter_scope('up5'):
up_5 = PF.batch_normalization(PF.deconvolution(F.relu(
F.concatenate(down_2, up_4, axis=1)),
128, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_1: (1, 128, 64, 64) + up_5: (1, 128, 64, 64) -> up_6: (1, 64, 128, 128)
with nn.parameter_scope('up6'):
up_6 = PF.batch_normalization(PF.deconvolution(F.relu(
F.concatenate(down_1, up_5, axis=1)),
64, (4, 4),
pad=(1, 1),
stride=(2, 2),
with_bias=False),
batch_stat=not test)
# down_0: (1, 64, 128, 128) + up_6: (1, 64, 128, 128) -> output: (1, 3, 256, 256)
with nn.parameter_scope('up7'):
output = F.tanh(
PF.deconvolution(F.relu(F.concatenate(down_0, up_6, axis=1)),
3, (4, 4),
pad=(1, 1),
stride=(2, 2)))
return output
def align_resnet(x, channel_basic=16, test=False, fix_parameters=False):
def resblock_align(x,
channel,
stride=(1, 1),
test=False,
downsample=False,
fix_parameters=False):
residual = x
with nn.parameter_scope('conv1'):
h = PF.convolution(x,
channel,
kernel=(3, 3),
stride=stride,
pad=(1, 1),
with_bias=False,
fix_parameters=fix_parameters)
with nn.parameter_scope('bn1'):
h = PF.batch_normalization(h,
batch_stat=not test,
fix_parameters=fix_parameters)
h = F.relu(h)
with nn.parameter_scope('conv2'):
h = PF.convolution(h,
channel,
kernel=(3, 3),
stride=(1, 1),
pad=(1, 1),
with_bias=False,
fix_parameters=fix_parameters)
with nn.parameter_scope('bn2'):
h = PF.batch_normalization(h,
batch_stat=not test,
fix_parameters=fix_parameters)
if downsample:
with nn.parameter_scope('downsample'):
residual = PF.convolution(x,
channel,
kernel=(1, 1),
stride=stride,
with_bias=False,
fix_parameters=fix_parameters)
residual = PF.batch_normalization(
residual,
batch_stat=not test,
fix_parameters=fix_parameters)
out = h + residual
out = F.relu(out)
return out
with nn.parameter_scope('layer0'):
h = PF.convolution(x,
3,
kernel=(3, 3),
stride=(1, 1),
pad=(1, 1),
with_bias=True,
fix_parameters=fix_parameters)
with nn.parameter_scope('layer1'):
h = PF.convolution(h,
16,
kernel=(7, 7),
stride=(2, 2),
pad=(3, 3),
with_bias=False,
fix_parameters=fix_parameters)
with nn.parameter_scope('layer2'):
h = PF.batch_normalization(h,
batch_stat=not test,
fix_parameters=fix_parameters)
h = F.relu(h)
h = F.max_pooling(h, kernel=(3, 3), stride=(2, 2), pad=(1, 1))
use_downsample = False
stride = (1, 1)
for i in range(5, 9):
with nn.parameter_scope(f'layer{i}_0'):
h = resblock_align(h,
channel_basic * (2**(i - 5)),
stride=stride,
test=False,
downsample=use_downsample,
fix_parameters=fix_parameters)
with nn.parameter_scope(f'layer{i}_1'):
h = resblock_align(h,
channel_basic * (2**(i - 5)),
stride=(1, 1),
test=False,
fix_parameters=fix_parameters)
use_downsample = True
stride = (2, 2)
with nn.parameter_scope('mlp1'):
h = F.relu(
PF.affine(h, 128, with_bias=True, fix_parameters=fix_parameters))
with nn.parameter_scope('mlp3'):
h = F.relu(
PF.affine(h, 128, with_bias=True, fix_parameters=fix_parameters))
with nn.parameter_scope('mlp5'):
h = PF.affine(h, 212, with_bias=True, fix_parameters=fix_parameters)
return h
def feature_transform_net(
feature: nn.Variable,
train: bool,
K: int = 64) -> Tuple[nn.Variable, Dict[str, nn.Variable]]:
"""T net, create transformation matrix
Args:
feature (nn.Variable): feature, shape(batch, number of points, 1, K)
train (bool): training flag
K (int): transformation matrix size, default is 64.
Returns:
Tuple[nn.Variable, Dict[str, nn.Variable]]: transformation matrix and internal variables
"""
batch_size, num_points, *_ = feature.shape
# B*H(=num_points)*W(=dim)*C(=K) to B*C(=K)*H(=num_points)*W(=dim)
feature = F.transpose(feature, (0, 3, 1, 2))
with nn.parameter_scope("conv1"):
conv_h1 = PF.convolution(feature,
64, (1, 1),
stride=(1, 1),
with_bias=False)
conv_h1 = PF.batch_normalization(conv_h1, batch_stat=train)
conv_h1 = F.relu(conv_h1)
with nn.parameter_scope("conv2"):
conv_h2 = PF.convolution(conv_h1,
128, (1, 1),
stride=(1, 1),
with_bias=False)
conv_h2 = PF.batch_normalization(conv_h2, batch_stat=train)
conv_h2 = F.relu(conv_h2)
with nn.parameter_scope("conv3"):
conv_h3 = PF.convolution(conv_h2,
1024, (1, 1),
stride=(1, 1),
with_bias=False)
conv_h3 = PF.batch_normalization(conv_h3, batch_stat=train)
conv_h3 = F.relu(conv_h3)
pool_h = F.max_pooling(conv_h3, (num_points, 1))
pool_h = F.reshape(pool_h, (batch_size, -1))
with nn.parameter_scope("affine1"):
affine_h1 = PF.affine(pool_h, 512, with_bias=False)
affine_h1 = PF.batch_normalization(affine_h1, batch_stat=train)
affine_h1 = F.relu(affine_h1)
with nn.parameter_scope("affine2"):
affine_h2 = PF.affine(affine_h1, 256, with_bias=False)
affine_h2 = PF.batch_normalization(affine_h2, batch_stat=train)
affine_h2 = F.relu(affine_h2)
with nn.parameter_scope("affine3"):
transform_h = PF.affine(affine_h2, K * K)
eye_mat = nn.Variable.from_numpy_array(
np.eye(K, dtype=np.float32).flatten())
eye_mat = F.reshape(eye_mat, (1, K * K))
transform_h = transform_h + eye_mat
transform_h = F.reshape(transform_h, (batch_size, K, K))
return transform_h, {
"conv_h1": conv_h1,
"conv_h2": conv_h2,
"conv_h3": conv_h3,
"pool_h": pool_h,
"affine_h1": affine_h1,
"affine_h2": affine_h2,
"transform_h": transform_h,
}
def bn(x):
return PF.batch_normalization(x, batch_stat=not test)
def batch_normalization(h, cnt=0, test=False):
with nn.parameter_scope("{}".format(cnt)):
h = PF.batch_normalization(h, batch_stat=not test)
return h
def cifar10_resnet23_prediction(image, ctx, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, rng, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C, C / 2, kernel=(1, 1)),
rng=rng)
h = PF.convolution(x,
C / 2,
kernel=(1, 1),
pad=(0, 0),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C / 2, C / 2, kernel=(3, 3)),
rng=rng)
h = PF.convolution(h,
C / 2,
kernel=(3, 3),
pad=(1, 1),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
w_init = UniformInitializer(calc_uniform_lim_glorot(
C / 2, C, kernel=(1, 1)),
rng=rng)
h = PF.convolution(h,
C,
kernel=(1, 1),
pad=(0, 0),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
rng = np.random.RandomState(0)
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
# Preprocess
if not test:
image = F.image_augmentation(image,
contrast=1.0,
angle=0.25,
flip_lr=True)
image.need_grad = False
w_init = UniformInitializer(calc_uniform_lim_glorot(3,
nmaps,
kernel=(3, 3)),
rng=rng)
h = PF.convolution(image,
nmaps,
kernel=(3, 3),
pad=(1, 1),
w_init=w_init,
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", rng, False) # -> 32x32
h = res_unit(h, "conv3", rng, True) # -> 16x16
h = res_unit(h, "conv4", rng, False) # -> 16x16
h = res_unit(h, "conv5", rng, True) # -> 8x8
h = res_unit(h, "conv6", rng, False) # -> 8x8
h = res_unit(h, "conv7", rng, True) # -> 4x4
h = res_unit(h, "conv8", rng, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
w_init = UniformInitializer(calc_uniform_lim_glorot(int(
np.prod(h.shape[1:])),
ncls,
kernel=(1, 1)),
rng=rng)
pred = PF.affine(h, ncls, w_init=w_init)
return pred
def dla_imagenet(x,
num_classes,
num_layers,
test,
residual_root=False,
tiny=False,
channel_last=False):
"""
Args:
x : Variable
num_classes : Number of classes of outputs
num_layers : Number of layers of DLA chosen from (34).
test : Construct net for testing.
tiny (bool): Tiny imagenet mode. Input image must be (3, 56, 56).
"""
layers = {
# 18: ((2, 2, 2, 2), basicblock, 1),
34: ((1, 1, 1, 2, 2, 1), (False, False, False, True, True, True),
basicblock)
# 50: ((3, 4, 6, 3), bottleneck, 4),
# 101: ((3, 4, 23, 3), bottleneck, 4),
# 152: ((3, 8, 36, 3), bottleneck, 4)
}
ochannels = [16, 32, 64, 128, 256, 512]
levels, levels_root, block = layers[num_layers]
strides = [1, 2, 2, 2, 2, 2]
logger.debug(x.shape)
axes = 3 if channel_last else 1
with nn.parameter_scope("conv1"):
stride = (1, 1)
r = pf_convolution(x,
16, (7, 7),
pad=(3, 3),
stride=stride,
with_bias=False,
channel_last=channel_last)
r = F.relu(PF.batch_normalization(r, axes=[axes], batch_stat=not test))
hidden = {}
hidden['conv0'] = r
logger.debug(r.shape)
with nn.parameter_scope("level0"):
r = _make_conv_level(r,
ochannels[0],
levels[0],
test=test,
stride=strides[0],
channel_last=channel_last)
hidden['level0'] = r
logger.debug(r.shape)
with nn.parameter_scope("level1"):
r = _make_conv_level(r,
ochannels[1],
levels[1],
test=test,
stride=strides[1],
channel_last=channel_last)
hidden['level1'] = r
logger.debug(r.shape)
with nn.parameter_scope("level2"):
r, _ = _make_tree_level1(r,
None,
block,
ochannels[2],
levels[2],
test,
levels_root[2],
stride=strides[2],
channel_last=channel_last)
hidden['level2'] = r
logger.debug(r.shape)
with nn.parameter_scope("level3"):
r = _make_tree_level2(r,
None,
block,
ochannels[3],
levels[3],
test,
levels_root[3],
stride=strides[3],
channel_last=channel_last)
hidden['level3'] = r
logger.debug(r.shape)
with nn.parameter_scope("level4"):
r = _make_tree_level2(r,
None,
block,
ochannels[4],
levels[4],
test,
levels_root[4],
stride=strides[4],
channel_last=channel_last)
hidden['level4'] = r
logger.debug(r.shape)
with nn.parameter_scope("level5"):
r, _ = _make_tree_level1(r,
None,
block,
ochannels[5],
levels[5],
test,
levels_root[5],
stride=strides[5],
channel_last=channel_last)
hidden['level5'] = r
logger.debug(r.shape)
pool_shape = r.shape[-2:]
if channel_last:
pool_shape = r.shape[1:3]
r = F.average_pooling(r, pool_shape, channel_last=channel_last)
with nn.parameter_scope("fc"):
r = pf_affine(r, num_classes, channel_last=channel_last)
logger.debug(r.shape)
return r, hidden
def stacked_hourglass_net(x,
batch_stat=True,
planes=64,
output_nc=15,
num_stacks=2,
activation='none'):
with nn.parameter_scope('conv1'):
x = PF.convolution(x, planes, kernel=(7, 7), pad=(3, 3), stride=(2, 2))
with nn.parameter_scope('bn1'):
x = PF.batch_normalization(x, batch_stat=batch_stat)
x = F.relu(x)
with nn.parameter_scope('layer1'):
x = resblock_hg(x, planes, planes, planes * 2, batch_stat=batch_stat)
x = F.max_pooling(x, kernel=(2, 2), stride=(2, 2))
with nn.parameter_scope('layer2'):
x = resblock_hg(x,
planes * 2,
planes * 2,
planes * 4,
batch_stat=batch_stat)
with nn.parameter_scope('layer3'):
x = resblock_hg(x,
planes * 4,
planes * 2,
planes * 4,
batch_stat=batch_stat)
planes = planes * 4
scores = []
for i in range(1, num_stacks):
# applied only once
with nn.parameter_scope(f'hourglass{i-1}'):
y = hourglass(x, planes, batch_stat=batch_stat)
with nn.parameter_scope('res0'):
y = resblock_hg(y,
planes,
planes // 2,
planes,
batch_stat=batch_stat)
with nn.parameter_scope('fc0'):
y = fc(y, planes, batch_stat=batch_stat) # True
score = PF.convolution(y, output_nc, kernel=(1, 1), name='score0')
score.persistent = True
scores.append(score)
fc_ = PF.convolution(y, planes, kernel=(1, 1), name='fc_')
score_ = PF.convolution(score, planes, kernel=(1, 1), name='score_')
x = x + fc_ + score_
with nn.parameter_scope('hourglass1'):
y = hourglass(x, planes, batch_stat=batch_stat)
with nn.parameter_scope('res1'):
y = resblock_hg(y, planes, planes // 2, planes, batch_stat=batch_stat)
with nn.parameter_scope('fc1'):
y = fc(y, planes, batch_stat=batch_stat) # mistakenly set as True
score = PF.convolution(y, output_nc, kernel=(1, 1), name='score1')
score.persistent = True
scores.append(score)
return scores
def discriminator_fm(x, sf, scope="Discriminator_FM"):
"""
Feature matching discriminator
"""
with nn.parameter_scope(scope):
fm_list = []
ch = 32
n = F.leaky_relu(conv(x, ch, 3, 1, 1, scope='d_conv/1'), alpha=0.2)
for i in range(4):
n, out_fm = dis_block(n, ch, i, train=True)
ch = ch * 2
fm_list.append(out_fm)
n = F.leaky_relu(PF.batch_normalization(conv(n,
channels=ch,
kernel=4,
stride=2,
pad=1,
use_bias=False,
scope='d_conv/10'),
axes=[3],
batch_stat=True,
name='d_bn/9'),
alpha=0.2,
inplace=True)
if sf == 1:
n = F.leaky_relu(PF.batch_normalization(conv(n,
channels=ch,
kernel=5,
stride=1,
pad=1,
use_bias=False,
scope='d_conv/11'),
axes=[3],
batch_stat=True,
name='d_bn/10'),
alpha=0.2,
inplace=True)
else:
n = F.leaky_relu(PF.batch_normalization(conv(n,
channels=ch,
kernel=5,
stride=1,
use_bias=False,
scope='d_conv/11'),
axes=[3],
batch_stat=True,
name='d_bn/10'),
alpha=0.2,
inplace=True)
n = PF.batch_normalization(conv(n,
channels=1,
kernel=1,
stride=1,
use_bias=False,
scope='d_conv/12'),
axes=[3],
batch_stat=True,
name='d_bn/11')
out_logit = n
out = F.sigmoid(out_logit) # [B,1]
return out, out_logit, fm_list
def point_cloud_transform_net(
point_cloud: nn.Variable,
train: bool) -> Tuple[nn.Variable, Dict[str, nn.Variable]]:
"""T net, create transformation matrix for point cloud
Args:
point_cloud (nn.Variable): point cloud, shape(batch, number of points, 3)
train (bool): training flag
Returns:
Tuple[nn.Variable, Dict[str, nn.Variable]]: transformation matrix and internal variables
"""
batch_size, num_points, _ = point_cloud.shape
# expand dim to B*C(=K)*H(=num_points)*W(=dim)
point_cloud = F.reshape(point_cloud, shape=(batch_size, 1, num_points, 3))
with nn.parameter_scope("conv1"):
conv_h1 = PF.convolution(point_cloud,
64, (1, 3),
stride=(1, 1),
with_bias=False)
conv_h1 = PF.batch_normalization(conv_h1, batch_stat=train)
conv_h1 = F.relu(conv_h1)
with nn.parameter_scope("conv2"):
conv_h2 = PF.convolution(conv_h1,
128, (1, 1),
stride=(1, 1),
with_bias=False)
conv_h2 = PF.batch_normalization(conv_h2, batch_stat=train)
conv_h2 = F.relu(conv_h2)
with nn.parameter_scope("conv3"):
conv_h3 = PF.convolution(conv_h2,
1024, (1, 1),
stride=(1, 1),
with_bias=False)
conv_h3 = PF.batch_normalization(conv_h3, batch_stat=train)
conv_h3 = F.relu(conv_h3)
pool_h = F.max_pooling(conv_h3, (num_points, 1))
pool_h = F.reshape(pool_h, (batch_size, -1))
with nn.parameter_scope("affine1"):
affine_h1 = PF.affine(pool_h, 512, with_bias=False)
affine_h1 = PF.batch_normalization(affine_h1, batch_stat=train)
affine_h1 = F.relu(affine_h1)
with nn.parameter_scope("affine2"):
affine_h2 = PF.affine(affine_h1, 256, with_bias=False)
affine_h2 = PF.batch_normalization(affine_h2, batch_stat=train)
affine_h2 = F.relu(affine_h2)
with nn.parameter_scope("affine3"):
# transform points (3 dim) so the matrix size is (3*3)
transform_h = PF.affine(affine_h2, 3 * 3)
eye_mat = nn.Variable.from_numpy_array(
np.array([1, 0, 0, 0, 1, 0, 0, 0, 1], dtype=np.float32))
eye_mat = F.reshape(eye_mat, (1, 9))
transform_h = transform_h + eye_mat
transform_h = F.reshape(transform_h, (batch_size, 3, 3))
return transform_h, {
"conv_h1": conv_h1,
"conv_h2": conv_h2,
"conv_h3": conv_h3,
"pool_h": pool_h,
"affine_h1": affine_h1,
"affine_h2": affine_h2,
"transform_h": transform_h,
}
def construct_architecture(image, num_class, num_cells, num_nodes, both_archs,
output_filter, test):
"""
Construct an architecture based on the given lists.
Note that first 2 layers are stem conv and have nothing to do with node operations.
"""
conv_arch, reduc_arch = both_archs
aux_logits = None
used_weights = set()
pool_distance = num_cells // 3
pool_layers = [pool_distance - 1, 2 * pool_distance - 1]
pool_layers = [_ for _ in pool_layers if _ > 0]
if len(pool_layers) > 0:
aux_head_indices = [pool_layers[-1] + 1]
else:
# this must not be happened. since num_cells needs to be more than 3.
aux_head_indices = [1]
ref_groups, required_indices = get_reference_layers(num_cells, pool_layers)
prev_layers = [list() for _ in range(ref_groups[-1] + 1)]
# Note that this implementation is slightly different from the one written by tensorflow.
if not test:
image = F.image_augmentation(image, angle=0.25,
flip_lr=True) # random_crop, min_scale
image.need_grad = False
x = image
# --------------------------------------- 1st cell ---------------------------------------
with nn.parameter_scope("stem_conv1"):
x = PF.convolution(x, output_filter, (3, 3), (1, 1), with_bias=False)
x = PF.batch_normalization(x, batch_stat=not test)
used_weights.update(
{"stem_conv1/conv/W", "stem_conv1/bn/gamma", "stem_conv1/bn/beta"})
prev_layers[0].append(x) # store to the "unpooled" layer
# spatial reduction (this might be skipped)
for i in range(1, len(required_indices[0])):
curr_scope = "stem1_reduc{}".format(i)
x = factorized_reduction(x, 2 * x.shape[1], curr_scope, test)
local_used_weights = get_factorized_weights_name(curr_scope)
used_weights.update(local_used_weights)
prev_layers[i].append(x)
# --------------------------------------- 2nd cell ---------------------------------------
with nn.parameter_scope("stem_conv2"):
x = PF.convolution(prev_layers[0][-1],
output_filter, (3, 3), (1, 1),
with_bias=False)
x = PF.batch_normalization(x, batch_stat=not test)
used_weights.update(
{"stem_conv2/conv/W", "stem_conv2/bn/gamma", "stem_conv2/bn/beta"})
prev_layers[0].append(x) # store to the "unpooled" layer
# spatial reduction (this might be skipped)
for i in range(1, len(required_indices[1])):
curr_scope = "stem2_reduc{}".format(i)
x = factorized_reduction(x, 2 * x.shape[1], curr_scope, test)
local_used_weights = get_factorized_weights_name(curr_scope)
used_weights.update(local_used_weights)
prev_layers[i].append(x)
# ------------------------------- Normal / Reduction cells -------------------------------
for layer_id in range(2, num_cells):
using_layer_index = ref_groups[layer_id]
required_index = list(required_indices[layer_id])
required_index.sort()
scope = 'w{}'.format(layer_id)
if layer_id in pool_layers:
architecture = reduc_arch
else:
architecture = conv_arch
previous_outputs = prev_layers[using_layer_index]
x, local_used_weights = construct_cell(previous_outputs, architecture,
num_nodes,
previous_outputs[-1].shape[1],
scope, test)
used_weights.update(local_used_weights)
prev_layers[using_layer_index].append(x)
required_index.remove(using_layer_index) # discard an index used above
# if this output (x) is reused as an input in other cells and
# its shape needs to be changed, apply downsampling in advance
for i in required_index:
curr_scope = "scope{0}_reduc{1}".format(layer_id, i)
x = factorized_reduction(x, 2 * x.shape[1], curr_scope, test)
local_used_weights = get_factorized_weights_name(curr_scope)
used_weights.update(local_used_weights)
prev_layers[i].append(x)
# auxiliary head, to use the intermediate output for training
if layer_id in aux_head_indices and not test:
print("Using aux_head at layer {}".format(layer_id))
aux_logits = F.relu(x)
aux_logits = F.average_pooling(aux_logits, (5, 5), (3, 3))
with nn.parameter_scope("proj"):
aux_logits = PF.convolution(aux_logits,
128, (3, 3), (1, 1),
with_bias=False)
aux_logits = PF.batch_normalization(aux_logits,
batch_stat=not test)
aux_logits = F.relu(aux_logits)
used_weights.update(
{"proj/conv/W", "proj/bn/gamma", "proj/bn/beta"})
with nn.parameter_scope("avg_pool"):
aux_logits = PF.convolution(aux_logits,
768, (3, 3), (1, 1),
with_bias=False)
aux_logits = PF.batch_normalization(aux_logits,
batch_stat=not test)
aux_logits = F.relu(aux_logits)
used_weights.update(
{"avg_pool/conv/W", "avg_pool/bn/gamma", "avg_pool/bn/beta"})
with nn.parameter_scope("additional_fc"):
aux_logits = F.global_average_pooling(aux_logits)
aux_logits = PF.affine(aux_logits, num_class, with_bias=False)
used_weights.update({"additional_fc/affine/W"})
x = F.global_average_pooling(prev_layers[-1][-1])
if not test:
dropout_rate = 0.5
x = F.dropout(x, dropout_rate)
with nn.parameter_scope("fc"):
pred = PF.affine(x, num_class, with_bias=False)
used_weights.add("fc/affine/W")
return pred, aux_logits, used_weights
def cnn_model_003_with_cross_attention(ctx, x_list, act=F.relu, test=False):
"""With attention before pooling
"""
with nn.context_scope(ctx):
# Convblock0
h0_list = []
for x in x_list:
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h0_list.append(h)
# Corss attention
ca0 = attention(h0_list[0], h0_list[1], h0_list[1],
div_dim=True, softmax=True)
ca1 = attention(h0_list[1], h0_list[0], h0_list[0],
div_dim=True, softmax=True)
# Maxpooing, Batchnorm, Dropout
h0_list = []
for h in [ca0, ca1]:
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h0_list.append(h)
# Convblock 1
h1_list = []
for h in h0_list:
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h1_list.append(h)
# Corss attention
ca0 = attention(h1_list[0], h1_list[1], h1_list[1],
div_dim=True, softmax=True)
ca1 = attention(h1_list[1], h1_list[0], h1_list[0],
div_dim=True, softmax=True)
# Maxpooing, Batchnorm, Dropout
h1_list = []
for h in [ca0, ca1]:
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h1_list.append(h)
# Convblock 2
h2_list = []
for h in h1_list:
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h2_list.append(h)
# Corss attention
ca0 = attention(h2_list[0], h2_list[1], h2_list[1],
div_dim=True, softmax=True)
ca1 = attention(h2_list[1], h2_list[0], h2_list[0],
div_dim=True, softmax=True)
# Convblock 3
h3_list = []
for h in [ca0, ca1]:
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
h3_list.append(h)
return h3_list
def test_pf_batch_normalization_execution(g_rng, inshape, decay_rate, eps,
batch_stat, output_stat, param_init,
fix_parameters, rng):
axis = 1 # Assume axes=[1]
p_shape = [1] * len(inshape)
p_shape[axis] = inshape[axis]
p_shape = tuple(p_shape)
if param_init:
beta_init = np.ones(p_shape) * 1
gamma_init = np.ones(p_shape) * 2
mean_init = np.ones(p_shape) * 0.5
var_init = np.ones(p_shape) * 1.5
param_init = dict(beta=beta_init,
gamma=gamma_init,
mean=mean_init,
var=var_init)
rng = process_rng(rng)
x = nn.Variable.from_numpy_array(g_rng.randn(*inshape))
kw = {}
insert_if_not_default(kw, 'decay_rate', decay_rate, 0.9)
insert_if_not_default(kw, 'eps', eps, 1e-5)
insert_if_not_default(kw, 'batch_stat', batch_stat, True)
insert_if_not_default(kw, 'output_stat', output_stat, False)
insert_if_not_default(kw, 'fix_parameters', fix_parameters, False)
insert_if_not_none(kw, 'param_init', param_init)
# Check creation
y = PF.batch_normalization(x, **kw)
# Check parameter values before execution
h = y[0] if output_stat else y
_, b, g, m, v = h.parent.inputs
if param_init:
assert np.allclose(b.d, beta_init)
assert np.allclose(g.d, gamma_init)
assert np.allclose(m.d, mean_init)
assert np.allclose(v.d, var_init)
else:
assert np.allclose(b.d, 0)
assert np.allclose(g.d, 1)
assert np.allclose(m.d, 0)
assert np.allclose(v.d, 1)
# Check execution
if output_stat:
forward_backward_all(*y)
else:
y.forward()
# TODO: Enable when implemented
if batch_stat:
y.backward()
# Check values
# TODO
# Check args
assert h.parent.info.type_name == 'BatchNormalization'
args = h.parent.info.args
assert np.isclose(args['decay_rate'], decay_rate)
assert np.isclose(args['eps'], eps)
assert args['batch_stat'] == batch_stat
# Check created parameters
assert h.parent.inputs[0] == x
assert len(h.parent.inputs) == 5
assert len(nn.get_parameters()) == 2
assert len(nn.get_parameters(grad_only=False)) == 4
beta, gamma, mean, var = [
nn.get_parameters(grad_only=False)['bn/' + name]
for name in ['beta', 'gamma', 'mean', 'var']
]
assert beta.shape == p_shape
assert gamma.shape == p_shape
assert mean.shape == p_shape
assert var.shape == p_shape
assert beta.need_grad
assert gamma.need_grad
assert not mean.need_grad
assert not var.need_grad
_, b, g, m, v = h.parent.inputs
assert b.need_grad == (not fix_parameters)
assert g.need_grad == (not fix_parameters)
assert not m.need_grad
assert not v.need_grad
def __call__(self, x, test=False):
# x = PF.mean_subtraction(x, base_axis=0)
if not self.input_is_spectrogram:
x = Spectrogram(*STFT(x, n_fft=self.n_fft, n_hop=self.n_hop),
power=self.power,
mono=(self.nb_channels == 1))
nb_frames, nb_samples, nb_channels, nb_bins = x.shape
mix = x
x = x[..., :self.nb_bins]
x += F.reshape(self.input_mean,
shape=(1, 1, 1, self.nb_bins),
inplace=False)
x *= F.reshape(self.input_scale,
shape=(1, 1, 1, self.nb_bins),
inplace=False)
with nn.parameter_scope("fc1"):
x = PF.affine(x, self.hidden_size, base_axis=2)
x = PF.batch_normalization(x, batch_stat=not test)
x = F.tanh(x)
with nn.parameter_scope("lstm"):
if self.unidirectional:
lstm_hidden_size = self.hidden_size
else:
lstm_hidden_size = self.hidden_size // 2
h = nn.Variable((self.nb_layers, self.nb_of_directions, nb_samples,
lstm_hidden_size),
need_grad=False)
h.d = np.zeros(h.shape)
c = nn.Variable((self.nb_layers, self.nb_of_directions, nb_samples,
lstm_hidden_size),
need_grad=False)
c.d = np.zeros(c.shape)
lstm_out, _, _ = PF.lstm(x,
h,
c,
num_layers=self.nb_layers,
bidirectional=not self.unidirectional,
training=not test)
x = F.concatenate(x, lstm_out) # concatenate along last axis
with nn.parameter_scope("fc2"):
x = PF.affine(
x,
(self.hidden_size),
base_axis=2,
)
x = PF.batch_normalization(x, batch_stat=not test)
x = F.relu(x)
with nn.parameter_scope("fc3"):
x = PF.affine(
x,
(nb_channels, nb_bins),
base_axis=2,
)
x = PF.batch_normalization(x, batch_stat=not test)
x = x.reshape(
(nb_frames, nb_samples, nb_channels, self.nb_output_bins))
# apply output scaling
x *= F.reshape(self.output_scale,
shape=(1, 1, 1, self.nb_output_bins),
inplace=False)
x += F.reshape(self.output_mean,
shape=(1, 1, 1, self.nb_output_bins),
inplace=False)
x = F.relu(x) * mix
return x
def conv_bn_relu(inp,
maps,
size,
stride=(1, 1),
pad=(0, 0),
deconv=False,
bn=True,
dropout=False,
relu=F.relu,
test=False,
name=''):
if not deconv:
if isinstance(inp, tuple):
h = F.add2(PF.convolution(inp[0],
maps,
size,
stride=stride,
pad=pad,
with_bias=not bn,
name=name + '_conv_0'),
PF.convolution(inp[1],
maps,
size,
stride=stride,
pad=pad,
with_bias=False,
name=name + '_conv_1'),
inplace=True)
else:
h = PF.convolution(inp,
maps,
size,
stride=stride,
pad=pad,
with_bias=not bn,
name=name + '_conv')
else:
if isinstance(inp, tuple):
h = F.add2(PF.deconvolution(inp[0],
maps,
kernel=size,
stride=stride,
pad=pad,
with_bias=not bn,
name=name + '_deconv_0'),
PF.deconvolution(inp[1],
maps,
kernel=size,
stride=stride,
pad=pad,
with_bias=False,
name=name + '_deconv_1'),
inplace=True)
else:
h = PF.deconvolution(inp,
maps,
kernel=size,
stride=stride,
pad=pad,
with_bias=not bn,
name=name + '_deconv')
if bn:
h = PF.batch_normalization(h, batch_stat=not test, name=name + '_bn')
if dropout and not test:
h = F.dropout(h, 0.5)
if relu is not None:
if relu is F.relu:
h = relu(h, inplace=True)
else:
h = relu(h)
return h
def fc(x, planes, batch_stat=True):
h = PF.convolution(x, planes, kernel=(1, 1))
h = PF.batch_normalization(h, batch_stat=batch_stat)
h = F.relu(h)
return h
def test_imperative_pf():
import nnabla.parametric_functions as PF
x = nn.NdArray([2, 3, 4, 5])
y = PF.batch_normalization(x)
