def forward(self, input):
x = self.DownBlock(input)
print('x: '+str(x.shape))
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(layers.reshape(gap, [x.shape[0], -1]))
gap_weight = list(self.gap_fc.parameters())[0]
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(layers.reshape(gmp, [x.shape[0], -1]))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = layers.reduce_sum(x, dim=1, keepdim=True)
if self.light:
x_ = layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(layers.reshape(x_, [x_.shape[0], -1]))
else:
x_ = self.FC(layers.reshape(x, [x.shape[0], -1]))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i+1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def forward(self, input):
x = self.model(input)
gap = layers.adaptive_pool2d(x, 1, pool_type="avg")
gap_logit = self.gap_fc(fluid.layers.reshape(gap, (x.shape[0], -1)))
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = fluid.layers.reshape(gap_weight, (-1, gap_weight.shape[0]))
gap = x * fluid.layers.unsqueeze(fluid.layers.unsqueeze(gap_weight, 2), 3)
gmp = layers.adaptive_pool2d(x, 1, pool_type="max")
gmp_logit = self.gmp_fc(fluid.layers.reshape(gmp, (x.shape[0], -1)))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = fluid.layers.reshape(gmp_weight, (-1, gmp_weight.shape[0]))
gmp = x * fluid.layers.unsqueeze(fluid.layers.unsqueeze(gmp_weight, 2), 3)
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.leaky_relu(self.conv1x1(x))
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit, heatmap
def forward(self, input):
x = self.DownBlock(input)
# gap = torch.nn.functional.adaptive_avg_pool2d(x, 1)
# gap_logit = self.gap_fc(gap.view(x.shape[0], -1))
# gap_weight = list(self.gap_fc.parameters())[0]
# gap = x * gap_weight.unsqueeze(2).unsqueeze(3)
# adaptive_avg_pool2d_1 = dygraph.Pool2D(pool_size=x.shape[-2:], pool_type='avg') # pool into 1x1 feature map
# gap = adaptive_avg_pool2d_1(x)
# print('x', x.shape)
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
# print('gap', gap.shape)
gap_logit = self.gap_fc(layers.reshape(gap, shape=(x.shape[0], -1)))
# print('gap_logit', gap_logit.shape)
gap_weight = self.gap_fc.parameters()[0]
gap_weight = layers.reshape(gap_weight, shape=(1, -1))
# print('gap_weight', gap_weight.shape)
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
# print('gap', gap.shape)
# gmp = torch.nn.functional.adaptive_max_pool2d(x, 1)
# gmp_logit = self.gmp_fc(gmp.view(x.shape[0], -1))
# gmp_weight = list(self.gmp_fc.parameters())[0]
# gmp = x * gmp_weight.unsqueeze(2).unsqueeze(3)
# adaptive_max_pool2d_1 = dygraph.Pool2D(pool_size=x.shape[-2:], pool_type='max') # pool into 1x1 feature map
# gmp = adaptive_max_pool2d_1(x)
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(layers.reshape(gmp, shape=(x.shape[0], -1)))
gmp_weight = self.gmp_fc.parameters()[0]
gmp_weight = layers.reshape(gmp_weight, shape=(1, -1))
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
# cam_logit = torch.cat([gap_logit, gmp_logit], 1)
# x = torch.cat([gap, gmp], 1)
# x = self.relu(self.conv1x1(x))
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
# heatmap = torch.sum(x, dim=1, keepdim=True)
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
if self.light:
# x_ = torch.nn.functional.adaptive_avg_pool2d(x, 1)
# x_ = self.FC(x_.view(x_.shape[0], -1))
# adaptive_avg_pool2d_1 = dygraph.Pool2D(pool_size=x.shape[-2:], pool_type='avg')
# x_ = adaptive_avg_pool2d_1(x)
x_ = layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(layers.reshape(x_, shape=(x_.shape[0], -1)))
else:
# x_ = self.FC(x.view(x.shape[0], -1))
x_ = self.FC(layers.reshape(x, shape=(x.shape[0], -1)))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def forward(self, input):
x = self.DownBlock(input)
gap = L.adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(L.reshape(gap, (x.shape[0], -1)))
gap_weight = self.gap_fc.weight
gap = x * L.unsqueeze(gap_weight, (2, 3))
gmp = L.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(L.reshape(gmp, (x.shape[0], -1)))
gmp_weight = self.gmp_fc.weight
gmp = x * L.unsqueeze(gmp_weight, (2, 3))
cam_logit = L.concat([gap_logit, gmp_logit], 1)
x = L.concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = L.reduce_sum(x, dim=1, keep_dim=True)
if self.light:
x_ = L.adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(L.reshape(x_, (x_.shape[0], -1)))
else:
x_ = self.FC(L.reshape(x, (x.shape[0], -1)))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def forward(self, input):
x = self.model(input)
gap = adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(reshape(gap, shape=[x.shape[0], -1]))
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = transpose(gap_weight, perm=[1, 0])
gap = x * unsqueeze(unsqueeze(gap_weight, 2), 3)
gmp = adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(reshape(gmp, shape=[x.shape[0], -1]))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = transpose(gmp_weight, perm=[1, 0])
gmp = x * unsqueeze(unsqueeze(gmp_weight, 2), 3)
cam_logit = concat([gap_logit, gmp_logit], 1)
x = concat([gap, gmp], 1)
x = self.leaky_relu(self.conv1x1(x))
heatmap = reduce_sum(x, dim=1, keep_dim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit, heatmap
def forward(self, input):
x = self.DownBlock(input)
gap = adaptive_pool2d(x, pool_size=[1, 1], pool_type='avg')
gap_ = reshape(x=gap, shape=(x.shape[0], -1))
gap_logit = self.gap_fc(gap_)
gap_weight = self.gap_fc.parameters()[0]
gap_weight = transpose(gap_weight, perm=[1, 0])
gap_weight = unsqueeze(gap_weight, axes=2)
gap_weight = unsqueeze(gap_weight, axes=3)
gap = x * gap_weight
gmp = adaptive_pool2d(x, pool_size=[1, 1], pool_type='max')
gmp_ = reshape(x=gmp, shape=(x.shape[0], -1))
gmp_logit = self.gmp_fc(gmp_)
gmp_weight = self.gmp_fc.parameters()[0]
gmp_weight = transpose(gmp_weight, perm=[1, 0])
gmp_weight = unsqueeze(gmp_weight, axes=2)
gmp_weight = unsqueeze(gmp_weight, axes=3)
gmp = x * gmp_weight
cam_logit = concat(input=[gap_logit, gmp_logit], axis=1)
x = concat(input=[gap, gmp], axis=1)
x = self.relu(self.conv1x1(x))
heatmap = reduce_sum(x, dim=1, keep_dim=True)
if self.light:
x_ = adaptive_pool2d(x, pool_size=[1, 1], pool_type='avg')
x_ = reshape(x=x_, shape=(x_.shape[0], -1))
x_ = self.FC(x_)
else:
x_ = reshape(x, shape=(x.shape[0], -1))
x_ = self.FC(x_)
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def test_adaptive_pool2d(self):
program = Program()
with program_guard(program):
x = layers.data(name='x', shape=[3, 224, 224], dtype='float32')
self.assertIsNotNone(
layers.adaptive_pool2d(x, [3, 3], pool_type='avg'))
pool, mask = layers.adaptive_pool2d(x, [3, 3], require_index=True)
self.assertIsNotNone(pool)
self.assertIsNotNone(mask)
self.assertIsNotNone(layers.adaptive_pool2d(x, 3, pool_type='avg'))
pool, mask = layers.adaptive_pool2d(x, 3, require_index=True)
self.assertIsNotNone(pool)
self.assertIsNotNone(mask)
def get_conv_weight(self, x, i):
"""
Adaptively generate weights for layer i in main branch convolutions.
Args:
x (NxCxHxW): input features
i (int): layer index
Returns:
conv_weights (list of tensors): Weights for the conv layers in the main branch.
"""
if not self.mul_ref_label:
x = L.adaptive_pool2d(x,
pool_size=(self.sh_fix, self.sw_fix),
pool_type='avg')
in_ch = self.num_filters_each_layer[i]
out_ch = self.num_filters_each_layer[i + 1]
cks = self.conv_kernel_size
b = x.shape[0]
weight_reshaper = WeightReshaper()
x = weight_reshaper.reshape_embed_input(x)
fc_0 = L.reshape(getattr(self, 'fc_conv_0_' + str(i))(x), (b, -1))
fc_1 = L.reshape(getattr(self, 'fc_conv_1_' + str(i))(x), (b, -1))
fc_s = L.reshape(getattr(self, 'fc_conv_s_' + str(i))(x), (b, -1))
weight_0 = weight_reshaper.reshape_weight(fc_0,
[in_ch, out_ch, cks, cks])
weight_1 = weight_reshaper.reshape_weight(fc_1,
[in_ch, in_ch, cks, cks])
weight_s = weight_reshaper.reshape_weight(fc_s, [in_ch, out_ch, 1, 1])
return [weight_0, weight_1, weight_s]
def forward(self, input):
x = self.DownBlock1_1(input)
x = self.DownBlock1_2(x)
x = instance_norm(x)
x = self.DownBlock1_4(x)
x = self.DownBlock2_1(x)
x = self.DownBlock2_2(x)
x = instance_norm(x)
x = self.DownBlock2_4(x)
x = self.DownBlock3_1(x)
x = self.DownBlock3_2(x)
x = instance_norm(x)
x = self.DownBlock3_4(x)
gap = adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(reshape(gap, [x.shape[0], -1]))
gap_weight = self.gap_fc.parameters()[0]
gap_weight = reshape(gap_weight, shape=[1, -1])
gap = x * unsqueeze(unsqueeze(gap_weight, 2), 3)
gmp = adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(reshape(gmp, [x.shape[0], -1]))
gmp_weight = self.gmp_fc.parameters()[0]
gmp_weight = reshape(gmp_weight, shape=[1, -1])
gmp = x * unsqueeze(unsqueeze(gmp_weight, 2), 3)
cam_logit = concat([gap_logit, gmp_logit], 1)
x = concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = reduce_sum(x, dim=1, keep_dim=True)
if self.light:
x_ = adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(reshape(x_, [x_.shape[0], -1]))
else:
x_ = self.FC(reshape(x, [x.shape[0], -1]))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
def forward(self, input):
x = self.model(input)
gap = adaptive_pool2d(x, pool_size=[1, 1], pool_type='avg')
gap_ = reshape(gap, shape=[x.shape[0], -1])
gap_logit = self.gap_fc(gap_)
gap_weight = self.gap_fc.parameters()[0]
gap_weight = transpose(gap_weight, perm=[1, 0])
gap_weight = unsqueeze(gap_weight, axes=2)
gap_weight = unsqueeze(gap_weight, axes=3)
gap = x * gap_weight
gmp = adaptive_pool2d(x, pool_size=[1, 1], pool_type='max')
gmp_ = reshape(gmp, shape=[x.shape[0], -1])
gmp_logit = self.gmp_fc(gmp_)
gmp_weight = self.gmp_fc.parameters()[0]
gmp_weight = transpose(gmp_weight, perm=[1, 0])
gmp_weight = unsqueeze(gmp_weight, axes=2)
gmp_weight = unsqueeze(gmp_weight, axes=3)
gmp = x * gmp_weight
cam_logit = concat(input=[gap_logit, gmp_logit], axis=1)
x = concat(input=[gap, gmp], axis=1)
x = self.leaky_relu(self.conv1x1(x))
heatmap = reduce_sum(x, dim=1, keep_dim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit, heatmap
def forward(self, input):
shape_print('Discriminator')
shape_print('input shape:' + str(input.shape))
x = self.model(input)
shape_print('x shape:' + str(x.shape))
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
shape_print('gap shape:' + str(gap.shape))
gap_logit = self.gap_fc(layers.reshape(gap, [x.shape[0], -1]))
shape_print('gap_logit shape:' + str(gap_logit.shape))
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = layers.reshape(gap_weight, [x.shape[0], -1])
shape_print('gap_weight shape:' + str(gap_weight.shape))
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
shape_print('gap shape:' + str(gap.shape))
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
shape_print('gmp shape:' + str(gmp.shape))
gmp_logit = self.gmp_fc(layers.reshape(gmp, [x.shape[0], -1]))
shape_print('gap_logit shape:' + str(gap_logit.shape))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = layers.reshape(gmp_weight, [x.shape[0], -1])
shape_print('gmp_weight shape:' + str(gmp_weight.shape))
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
shape_print('gmp shape:' + str(gmp.shape))
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
shape_print('cam_logit shape:' + str(cam_logit.shape))
x = layers.concat([gap, gmp], 1)
shape_print('x shape:' + str(x.shape))
x = self.leaky_relu(self.conv1x1(x))
shape_print('x shape:' + str(x.shape))
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
shape_print('heatmap shape:' + str(heatmap.shape))
x = self.pad(x)
shape_print('x shape:' + str(x.shape))
out = self.conv(x)
shape_print('out shape:' + str(out.shape))
return out, cam_logit, heatmap
def forward(self, input):
x = self.model(input)
# gap = torch.nn.functional.adaptive_avg_pool2d(x, 1)
# gap_logit = self.gap_fc(gap.view(x.shape[0], -1))
# gap_weight = list(self.gap_fc.parameters())[0]
# gap = x * gap_weight.unsqueeze(2).unsqueeze(3)
# adaptive_avg_pool2d_1 = dygraph.Pool2D(pool_size=x.shape[-2:], pool_type='avg')
# gap = adaptive_avg_pool2d_1(x)
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(layers.reshape(gap, shape=(x.shape[0], -1)))
gap_weight = self.gap_fc.parameters()[0]
gap_weight = layers.reshape(gap_weight, shape=(1, -1))
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
# gmp = torch.nn.functional.adaptive_max_pool2d(x, 1)
# gmp_logit = self.gmp_fc(gmp.view(x.shape[0], -1))
# gmp_weight = list(self.gmp_fc.parameters())[0]
# gmp = x * gmp_weight.unsqueeze(2).unsqueeze(3)
# adaptive_max_pool2d_1 = dygraph.Pool2D(pool_size=x.shape[-2:], pool_type='max')
# gmp = adaptive_max_pool2d_1(x)
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(layers.reshape(gmp, shape=(x.shape[0], -1)))
gmp_weight = self.gmp_fc.parameters()[0]
gmp_weight = layers.reshape(gmp_weight, shape=(1, -1))
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
# cam_logit = torch.cat([gap_logit, gmp_logit], 1)
# x = torch.cat([gap, gmp], 1)
# x = self.leaky_relu(self.conv1x1(x))
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.leaky_relu(self.conv1x1(x))
# heatmap = torch.sum(x, dim=1, keepdim=True)
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit, heatmap
def get_norm_weight(self, x, i):
"""
Adaptively generate weights for SPADE in layer i of generator.
Args:
x (NxCxHxW): input features.
i (int): Layer index
Returns:
embedding_weights (list of tensors): weights for the label embedding network
norm_weights (list of tensors): weights for the SPADE layers
"""
if not self.mul_ref_label:
x = L.adaptive_pool2d(x,
pool_size=(self.sh_fix, self.sw_fix),
pool_type='avg')
in_ch = self.num_filters_each_layer[i]
out_ch = self.num_filters_each_layer[i + 1]
spade_ch = self.spade_in_channels[i]
eks, sks = self.embed_kernel_size, self.kernel_size
b = x.shape[0]
weight_reshaper = WeightReshaper()
x = weight_reshaper.reshape_embed_input(x)
# Weights for the label embedding network.
embedding_weights = None
if self.use_hyper_embed:
fc_e = L.reshape(getattr(self, 'fc_spade_e_' + str(i))(x), (b, -1))
if 'decoder' in self.embed_arch:
weight_shape = [in_ch, out_ch, eks, eks]
fc_e = fc_e[:, :-in_ch]
else:
weight_shape = [out_ch, in_ch, eks, eks]
embedding_weights = weight_reshaper.reshape_weight(
fc_e, weight_shape)
# weights for the 3 layers in SPADE module: conv_0, conv_1, and shortcut.
fc_0 = L.reshape(getattr(self, 'fc_spade_0_' + str(i))(x), (b, -1))
fc_1 = L.reshape(getattr(self, 'fc_spade_1_' + str(i))(x), (b, -1))
fc_s = L.reshape(getattr(self, 'fc_spade_s_' + str(i))(x), (b, -1))
if self.conv_before_norm:
out_ch = in_ch
weight_0 = weight_reshaper.reshape_weight(
fc_0, [out_ch * 2, spade_ch, sks, sks])
weight_1 = weight_reshaper.reshape_weight(
fc_1, [in_ch * 2, spade_ch, sks, sks])
weight_s = weight_reshaper.reshape_weight(
fc_s, [out_ch * 2, spade_ch, sks, sks])
norm_weights = [weight_0, weight_1, weight_s]
return embedding_weights, norm_weights
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,
groups=1, width_per_group=64, replace_stride_with_dilation=None,
norm_layer=None):
super(ResNet, self).__init__()
if norm_layer is None:
norm_layer = dg.BatchNorm
self._norm_layer = norm_layer
self.inplanes = 64
self.dilation = 1
if replace_stride_with_dilation is None:
# each element in the tuple indicates if we should replace
# the 2x2 stride with a dilated convolution instead
replace_stride_with_dilation = [False, False, False]
if len(replace_stride_with_dilation) != 3:
raise ValueError("replace_stride_with_dilation should be None "
"or a 3-element tuple, got {}".format(replace_stride_with_dilation))
self.groups = groups
self.base_width = width_per_group
self.conv1 = dg.Conv2D(3, self.inplanes, filter_size=7, stride=2,
padding=3, bias_attr=False)
self.bn1 = norm_layer(self.inplanes)
self.relu = ReLU()
self.maxpool = dg.Pool2D(pool_size=3, pool_type='max', pool_stride=2, pool_padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2,
dilate=replace_stride_with_dilation[0])
self.layer3 = self._make_layer(block, 256, layers[2], stride=2,
dilate=replace_stride_with_dilation[1])
self.layer4 = self._make_layer(block, 512, layers[3], stride=2,
dilate=replace_stride_with_dilation[2])
self.avgpool = lambda x: L.adaptive_pool2d(x, (1, 1), pool_type='avg')
self.fc = dg.Linear(512 * block.expansion, num_classes)
for m in self.sublayers():
if isinstance(m, dg.Conv2D):
m.param_attr = F.ParamAttr(initializer=F.initializer.MSRAInitializer())
elif isinstance(m, (dg.BatchNorm, dg.GroupNorm)):
m.param_attr = F.ParamAttr(initializer=F.initializer.ConstantInitializer(value=0.0))
m.bias_attr = F.ParamAttr(initializer=F.initializer.ConstantInitializer(value=0.0))
# Zero-initialize the last BN in each residual branch,
# so that the residual branch starts with zeros, and each residual block behaves like an identity.
# This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
if zero_init_residual:
for m in self.sublayers():
if isinstance(m, Bottleneck):
m.bn3.param_attr = F.ParamAttr(initializer=F.initializer.ConstantInitializer(value=0.0))
elif isinstance(m, BasicBlock):
m.bn2.param_attr = F.ParamAttr(initializer=F.initializer.ConstantInitializer(value=0.0))
def get_feature(self, im: np.ndarray):
"""Get the feature. Generally, call this function.
args:
im: image patch
"""
# Return empty tensor if it should not be used
is_color = im.shape[1] == 3
if is_color and not self.use_for_color or not is_color and not self.use_for_gray:
return np.array([])
feat_list = self.extract(im)
output_sz = [None] * len(
feat_list) if self.output_size is None else self.output_size
# Pool/downsample
with fluid.dygraph.guard():
feat_list = [n2p(f) for f in feat_list]
for i, (sz, s) in enumerate(zip(output_sz, self.pool_stride)):
if sz is not None:
feat_list[i] = layers.adaptive_pool2d(feat_list[i],
sz,
pool_type='avg')
elif s != 1:
feat_list[i] = layers.pool2d(feat_list[i],
s,
pool_stride=s,
pool_type='avg')
# Normalize
if self.normalize_power is not None:
new_feat_list = []
for feat in feat_list:
norm = (layers.reduce_sum(layers.reshape(
layers.abs(feat), [feat.shape[0], 1, 1, -1])**
self.normalize_power,
dim=3,
keep_dim=True) /
(feat.shape[1] * feat.shape[2] * feat.shape[3]) +
1e-10)**(1 / self.normalize_power)
feat = broadcast_op(feat, norm, 'div')
new_feat_list.append(feat)
feat_list = new_feat_list
# To numpy
feat_list = TensorList([f.numpy() for f in feat_list])
return feat_list
def forward(self, input):
x = self.model(input)
gap = L.adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(L.reshape(gap, (x.shape[0], -1)))
gap_weight = self.gap_fc.weight_orig
gap = x * L.unsqueeze(gap_weight, (2, 3))
gmp = L.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(L.reshape(gmp, (x.shape[0], -1)))
gmp_weight = self.gmp_fc.weight_orig
gmp = x * L.unsqueeze(gmp_weight, (2, 3))
cam_logit = L.concat([gap_logit, gmp_logit], 1)
x = L.concat([gap, gmp], 1)
x = self.leaky_relu(self.conv1x1(x))
heatmap = L.reduce_sum(x, dim=1, keep_dim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit, heatmap
def get_feature(self, im: np.ndarray):
"""Get the feature. Generally, call this function.
args:
im: image patch
"""
# Return empty tensor if it should not be used
is_color = im.shape[1] == 3
if is_color and not self.use_for_color or not is_color and not self.use_for_gray:
return np.array([])
# Extract feature
feat = self.extract(im)
# Pool/downsample
with fluid.dygraph.guard():
feat = n2p(feat)
if self.output_size is not None:
feat = layers.adaptive_pool2d(feat, self.output_size, 'avg')
elif self.pool_stride != 1:
feat = layers.pool2d(
feat,
self.pool_stride,
pool_stride=self.pool_stride,
pool_type='avg')
# Normalize
if self.normalize_power is not None:
feat /= (
layers.reduce_sum(
layers.reshape(
layers.abs(feat), [feat.shape[0], 1, 1, -1])**
self.normalize_power,
dim=3,
keep_dim=True) /
(feat.shape[1] * feat.shape[2] * feat.shape[3]) + 1e-10)**(
1 / self.normalize_power)
feat = feat.numpy()
return feat
def forward(self, input):
shape_print('Generator')
shape_print('input shape:' + str(input.shape))
x = self.DownBlock(input)
shape_print('downblock shape:' + str(x.shape))
debug_print('DownBlock Pass')
debug_save_img(x, 'DownBlock')
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
shape_print('gap shape:' + str(gap.shape))
gap_logit = self.gap_fc(layers.reshape(gap, [x.shape[0], -1]))
shape_print('gap_logit shape:' + str(gap_logit.shape))
debug_print('GAP logit Pass')
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = layers.reshape(gap_weight, [x.shape[0], -1])
shape_print('gap_weight shape:' + str(gap_weight.shape))
debug_print('GAP weight Pass')
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
shape_print('gap shape:' + str(gap.shape))
debug_print('GAP Pass')
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(layers.reshape(gmp, [x.shape[0], -1]))
shape_print('gmp_logit shape:' + str(gmp_logit.shape))
debug_print('GMP logit Pass')
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = layers.reshape(gmp_weight, [x.shape[0], -1])
shape_print('gmp_weight shape:' + str(gmp_weight.shape))
debug_print('GMP weight Pass')
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
shape_print('gmp shape:' + str(gmp.shape))
debug_print('GMP Pass')
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
shape_print('cam logit shape:' + str(cam_logit.shape))
debug_print('CAM logit Pass')
x = layers.concat([gap, gmp], 1)
shape_print('x shape:' + str(x.shape))
x = self.relu(self.conv1x1(x))
shape_print('x shape:' + str(x.shape))
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
shape_print('heatmap shape:' + str(heatmap.shape))
if self.light:
x_ = layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(layers.reshape(x_, [x_.shape[0], -1]))
shape_print('FC shape:' + str(x_.shape))
debug_print('FC Pass')
else:
x_ = self.FC(layers.reshape(x, [x.shape[0], -1]))
shape_print('FC shape:' + str(x_.shape))
gamma, beta = self.gamma(x_), self.beta(x_)
shape_print('gamma shape:' + str(gamma.shape))
shape_print('beta shape:' + str(beta.shape))
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
debug_save_img(x, 'UpBlock1_' + str(i + 1))
shape_print('UpBlock1_' + str(i + 1) + 'shape:' + str(x.shape))
debug_print('UpBlock1_' + str(i + 1) + 'Pass')
out = self.UpBlock2(x)
debug_save_img(out, 'out')
debug_print('UpBlock2 Pass')
return out, cam_logit, heatmap
def __call__(self, image):
"""
Estimating parameters of geometric transformation
Args:
image: input
Return:
batch_C_prime: the matrix of the geometric transformation
"""
F = self.F
loc_lr = self.loc_lr
if self.model_name == "large":
num_filters_list = [64, 128, 256, 512]
fc_dim = 256
else:
num_filters_list = [16, 32, 64, 128]
fc_dim = 64
for fno in range(len(num_filters_list)):
num_filters = num_filters_list[fno]
name = "loc_conv%d" % fno
if fno == 0:
conv = self.conv_bn_layer(image,
num_filters,
3,
act='relu',
name=name)
else:
conv = self.conv_bn_layer(pool,
num_filters,
3,
act='relu',
name=name)
if fno == len(num_filters_list) - 1:
pool = layers.adaptive_pool2d(input=conv,
pool_size=[1, 1],
pool_type='avg')
else:
pool = layers.pool2d(input=conv,
pool_size=2,
pool_stride=2,
pool_padding=0,
pool_type='max')
name = "loc_fc1"
stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0)
fc1 = layers.fc(input=pool,
size=fc_dim,
param_attr=fluid.param_attr.ParamAttr(
learning_rate=loc_lr,
initializer=fluid.initializer.Uniform(-stdv, stdv),
name=name + "_w"),
act='relu',
name=name)
initial_bias = self.get_initial_fiducials()
initial_bias = initial_bias.reshape(-1)
name = "loc_fc2"
param_attr = fluid.param_attr.ParamAttr(
learning_rate=loc_lr,
initializer=fluid.initializer.NumpyArrayInitializer(
np.zeros([fc_dim, F * 2])),
name=name + "_w")
bias_attr = fluid.param_attr.ParamAttr(
learning_rate=loc_lr,
initializer=fluid.initializer.NumpyArrayInitializer(initial_bias),
name=name + "_b")
fc2 = layers.fc(input=fc1,
size=F * 2,
param_attr=param_attr,
bias_attr=bias_attr,
name=name)
batch_C_prime = layers.reshape(x=fc2, shape=[-1, F, 2], inplace=False)
return batch_C_prime
