class Discriminator(dygraph.Layer):
def __init__(self, input_nc, ndf=64, n_layers=5):
super(Discriminator, self).__init__()
model = [ReflectionPad2d([1,1,1,1]),
# TODO 谱归一化
Conv2D(input_nc, ndf, filter_size=4, stride=2, padding=0, bias_attr=True),
LeakyReLU(0.2, True)]
for i in range(1, n_layers - 2):
mult = 2 ** (i - 1)
model += [ReflectionPad2d([1,1,1,1]),
Conv2D(ndf * mult, ndf * mult * 2, filter_size=4, stride=2, padding=0, bias_attr=True),
LeakyReLU(0.2, True)]
mult = 2 ** (n_layers - 2 - 1)
model += [ReflectionPad2d([1,1,1,1]),
Conv2D(ndf * mult, ndf * mult * 2, filter_size=4, stride=1, padding=0, bias_attr=True),
LeakyReLU(0.2, True)]
# Class Activation Map
mult = 2 ** (n_layers - 2)
self.gap_fc = Linear(ndf * mult, 1, bias_attr=False)
self.gmp_fc = Linear(ndf * mult, 1, bias_attr=False)
self.conv1x1 = Conv2D(ndf * mult * 2, ndf * mult, filter_size=1, stride=1, bias_attr=True)
self.leaky_relu = LeakyReLU(0.2, True)
self.pad = ReflectionPad2d([1,1,1,1])
self.conv = Conv2D(ndf * mult, 1, filter_size=4, stride=1, padding=0, bias_attr=False)
self.model = Sequential(*model)
def forward(self, input):
x = self.model(input)
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(layers.reshape(gap, [x.shape[0], -1]))
gap_weight = list(self.gap_fc.parameters())[0]
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gap_logit = self.gmp_fc(layers.reshape(gmp, [x.shape[0], -1]))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.leaky_relu(self.conv1x1(x))
heatmap = layers.reduce_sum(x, dim=1, keepdim=True)
x = self.pad(x)
out = self.conv(x)
return out, cam_logit, heatmap
def func_test_parameter_list(self):
with fluid.dygraph.guard():
linear_1 = Linear(10, 10)
linear_2 = Linear(10, 10)
sgd = SGDOptimizer(
1.0,
parameter_list=itertools.chain(linear_1.parameters(),
linear_2.parameters()))
in_np = np.random.uniform(-0.1, 0.1, [10, 10]).astype("float32")
in_data = fluid.dygraph.to_variable(in_np)
y = linear_1(in_data)
y = linear_2(y)
loss = fluid.layers.reduce_mean(y)
loss.backward()
sgd.minimize(loss)
self.assertTrue(
len(sgd._parameter_list) ==
len(linear_1.parameters() + linear_2.parameters()))
class ResnetGenerator(fluid.dygraph.Layer):
def __init__(self,
input_nc,
output_nc,
ngf=64,
n_blocks=6,
img_size=256,
light=False):
assert (n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = []
DownBlock += [
#fluid.layers.pad2d(3),
ReflectionPad2d(3),
Conv2D(num_channels=input_nc,
num_filters=ngf,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
InstanceNorm(ngf),
ReLU(False)
#BatchNorm(ngf,act='relu')
#fluid.layers.instance_norm(ngf)
]
# self.conv1=Conv2D(input_nc, ngf, 7)
# self.instance_norm=InstanceNorm(ngf)
#self.n_downsampling=n_downsampling
# Down-Sampling
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
DownBlock += [
#fluid.layers.pad2d(1),
ReflectionPad2d(1),
Conv2D(ngf * mult,
ngf * mult * 2,
filter_size=3,
stride=2,
padding=0,
bias_attr=False),
InstanceNorm(ngf * mult * 2),
ReLU(False)
#BatchNorm(ngf * mult * 2,act='relu')
#fluid.layers.instance_norm(ngf * mult * 2)
]
# Down-Sampling Bottleneck
mult = 2**n_downsampling
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
#self.renetblock=ResnetBlock(ngf * mult, use_bias=False)
# Class Activation Map
self.gap_fc = Linear(ngf * mult, 1, bias_attr=False)
self.gmp_fc = Linear(ngf * mult, 1, bias_attr=False)
self.conv1x1 = Conv2D(ngf * mult * 2,
ngf * mult,
filter_size=1,
stride=1,
bias_attr=True)
self.relu = ReLU(False)
# Gamma, Beta block
if self.light:
FC = [
Linear(ngf * mult, ngf * mult, bias_attr=False, act='relu'),
Linear(ngf * mult, ngf * mult, bias_attr=False, act='relu')
]
else:
FC = [
Linear(img_size // mult * img_size // mult * ngf * mult,
ngf * mult,
bias_attr=False,
act='relu'),
Linear(ngf * mult, ngf * mult, bias_attr=False, act='relu')
]
self.gamma = Linear(ngf * mult, ngf * mult, bias_attr=False)
self.beta = Linear(ngf * mult, ngf * mult, bias_attr=False)
# Up-Sampling Bottleneck
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i + 1),
ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [ #nn.Upsample(scale_factor=2, mode='nearest'),
#fluid.layers.pad2d(1),
Upsample(),
ReflectionPad2d(1),
Conv2D(ngf * mult,
int(ngf * mult / 2),
filter_size=3,
stride=1,
padding=0,
bias_attr=False),
ILN(int(ngf * mult / 2)),
ReLU(False)
]
UpBlock2 += [
#fluid.layers.pad2d(3),
ReflectionPad2d(3),
Conv2D(ngf,
output_nc,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
Tanh()
]
self.DownBlock = Sequential(*DownBlock)
self.FC = Sequential(*FC)
self.UpBlock2 = Sequential(*UpBlock2)
def forward(self, input):
x = self.DownBlock(input)
#torch.Size([1, 256, 64, 64])
#gap torch.Size([1, 256, 1, 1])
gap = fluid.layers.adaptive_pool2d(x, 1, pool_type='avg')
# print(gap.shape)
# gap =Pool2D(pool_size=x.shape[-1],pool_stride=x.shape[-1],pool_type='avg')(x)
gap = fluid.layers.reshape(gap, shape=[x.shape[0],
-1]) #torch.Size([1, 1])
# print(gap.shape)
gap_logit = self.gap_fc(gap)
gap_weight = list(self.gap_fc.parameters())[0] #torch.Size([1, 256])
gap_weight = fluid.layers.reshape(gap_weight, shape=[x.shape[0], -1])
gap_weight = fluid.layers.unsqueeze(input=gap_weight, axes=[2])
gap_weight = fluid.layers.unsqueeze(input=gap_weight, axes=[3])
# print(gap_weight.shape)
gap = x * gap_weight #torch.Size([1, 256, 64, 64])
#gap = x * gap_weight.unsqueeze(2).unsqueeze(3)
# print(1111111)
gmp = fluid.layers.adaptive_pool2d(x, 1, pool_type='max')
# print(gmp.shape)
#gmp =Pool2D(pool_size=x.shape[-1],pool_stride=x.shape[-1],pool_type='max')(x)
gmp = fluid.layers.reshape(gmp, shape=[x.shape[0], -1])
# print(gmp.shape)
gmp_logit = self.gmp_fc(gmp)
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = fluid.layers.reshape(gmp_weight, shape=[x.shape[0], -1])
gmp_weight = fluid.layers.unsqueeze(input=gmp_weight, axes=[2])
gmp_weight = fluid.layers.unsqueeze(input=gmp_weight, axes=[3])
# print(gap_weight.shape)
gmp = x * gmp_weight #torch.Size([1, 256, 64, 64])
#gmp = x * gmp_weight.unsqueeze(2).unsqueeze(3)
#cam_logit = torch.cat([gap_logit, gmp_logit], 1)
#x = torch.cat([gap, gmp], 1)
cam_logit = fluid.layers.concat([gap_logit, gmp_logit], 1)
x = fluid.layers.concat([gap, gmp], 1) #torch.Size([1, 512, 64, 64])
# x = self.conv1x1(x) #torch.Size([1, 256, 64, 64])
x = self.relu(self.conv1x1(x))
#torch.Size([1, 256, 64, 64])
#heatmap = torch.sum(x, dim=1, keepdim=True)
heatmap = fluid.layers.reduce_sum(x, dim=1, keep_dim=True)
#heatmap torch.Size([1, 1, 64, 64])
if self.light:
x_ = fluid.layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = fluid.layers.reshape(x_, shape=[x_.shape[0], -1]) ########
x_ = self.FC(x_)
else:
x_ = fluid.layers.reshape(x, shape=[x.shape[0], -1])
x_ = self.FC(x_)
gamma, beta = self.gamma(x_), self.beta(x_)
# gamma torch.Size([1, 256]) beta torch.Size([1, 256])
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
#print(heatmap.shape)
#out torch.Size([1, 3, 256, 256]) cam_logit torch.Size([1, 2]) heatmap torch.Size([1, 1, 64, 64])
return out, cam_logit, heatmap
class ResnetGenerator(dygraph.Layer):
def __init__(self, input_nc, output_nc, ngf=64, n_blocks=6, img_size=256, light=False):
super(ResnetGenerator, self).__init__()
'''
Args:
input_cn: 输入通道数
output_nc: 输出通道数，此处二者都为3
ngf: base channel number per layer
n_blocks: The number of resblock
'''
assert(n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = []
# 3 ReflectionPad2d 抵消了紧接着的7*7Conv层
#TODO 此处由于Paddle的pad2d在fluid.layer中，不能作为对象定义，比较麻烦，因此暂时使用普通padding
DownBlock += [ReflectionPad2d([1,1,1,1]),
Conv2D(input_nc, ngf, filter_size=7, stride=1, padding=0, bias_attr=False),
InstanceNorm(ngf),
#TODO paddle没有单独的ReLU对象，暂时用PReLU代替，后期将const改成0
ReLU(True)]
# Down-Sampling
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
# 通道以2倍增，stride为2，大小以2减少
DownBlock += [ReflectionPad2d([1,1,1,1]),
Conv2D(ngf * mult, ngf * mult * 2, filter_size=3, stride=2, padding=0, bias_attr=False),
InstanceNorm(ngf * mult * 2),
ReLU(True)]
# Down-Sampling Bottleneck
mult = 2**n_downsampling
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
# Class Activation Map
self.gap_fc = Linear(ngf * mult, 1, bias_attr=False)
self.gmp_fc = Linear(ngf * mult, 1, bias_attr=False)
self.conv1x1 = Conv2D(ngf * mult * 2, ngf * mult, filter_size=1, stride=1, padding=0, bias_attr=True)
self.relu = ReLU(True)
# Gamma, Beta block
if self.light:
FC = [Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(True),
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(True)]
else:
# TODO 这里没太理解
FC = [Linear(img_size // mult * img_size // mult * ngf * mult, ngf * mult, bias_attr=False),
ReLU(True),
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(True)]
self.gamma = Linear(ngf * mult, ngf * mult, bias_attr=False)
self.beta = Linear(ngf * mult, ngf * mult, bias_attr=False)
# Up-Sampling Bottleneck
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i+1), ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [Upsample(scale_factor=2),
ReflectionPad2d([1,1,1,1]),
Conv2D(ngf * mult, int(ngf * mult / 2), filter_size=3, stride=1, padding=0, bias_attr=False),
ILN(int(ngf * mult / 2)),
ReLU(True)]
UpBlock2 += [ReflectionPad2d([3,3,3,3]),
Conv2D(ngf, output_nc, filter_size=7, stride=1, padding=0, bias_attr=False),
Tanh(False)]
self.DownBlock = Sequential(*DownBlock)
self.FC = Sequential(*FC)
self.UpBlock2 = Sequential(*UpBlock2)
def forward(self, input):
x = self.DownBlock(input)
print('x: '+str(x.shape))
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(layers.reshape(gap, [x.shape[0], -1]))
gap_weight = list(self.gap_fc.parameters())[0]
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(layers.reshape(gmp, [x.shape[0], -1]))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = layers.reduce_sum(x, dim=1, keepdim=True)
if self.light:
x_ = layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(layers.reshape(x_, [x_.shape[0], -1]))
else:
x_ = self.FC(layers.reshape(x, [x.shape[0], -1]))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i+1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
class ResnetGenerator(fluid.dygraph.Layer):
def __init__(self, input_nc, output_nc, ngf=64, n_blocks=6, img_size=256, light=False):
assert(n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = []
DownBlock += [
ReflectionPad2D(3),
Conv2D(num_channels=input_nc, num_filters=ngf, filter_size=7, stride=1, padding=0, bias_attr=False),
InstanceNorm(self.ngf),
Relu(),
]
# Down-Sampling
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
DownBlock += [
ReflectionPad2D(1),
Conv2D(num_channels=ngf * mult, num_filters=ngf * mult * 2, filter_size=3, stride=2, padding=0, bias_attr=False),
InstanceNorm(ngf * mult * 2),
Relu(),
]
# Down-Sampling Bottleneck
mult = 2**n_downsampling
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
# Class Activation Map
self.gap_fc = Linear(ngf * mult, 1, bias_attr=False, act='sigmoid')
self.gmp_fc = Linear(ngf * mult, 1, bias_attr=False, act='sigmoid')
self.conv1x1 = Conv2D(ngf * mult * 2, ngf * mult, filter_size=1, stride=1, bias_attr=True)
self.relu = Relu()
# Gamma, Beta block
if self.light:
FC = [
Linear(ngf * mult, ngf * mult, bias_attr=False),
Relu(),
Linear(ngf * mult, ngf * mult, bias_attr=False),
Relu(),
]
else:
FC = [
Linear(img_size // mult * img_size // mult * ngf * mult, ngf * mult, bias_attr=False),
Relu(),
Linear(ngf * mult, ngf * mult, bias_attr=False),
Relu(),
]
self.gamma = Linear(ngf * mult, ngf * mult, bias_attr=False)
self.beta = Linear(ngf * mult, ngf * mult, bias_attr=False)
# Up-Sampling Bottleneck
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i+1), ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [
UpSample(2),
ReflectionPad2D(1),
Conv2D(num_channels=ngf * mult, num_filters=int(ngf * mult / 2),
filter_size=3, stride=1, padding=0, bias_attr=False),
ILN(int(ngf * mult / 2)),
Relu(),
]
UpBlock2 += [
ReflectionPad2D(3),
Conv2D(num_channels=ngf, num_filters=output_nc,
filter_size=7, stride=1, padding=0, bias_attr=False),
Tanh(),
]
self.DownBlock = Sequential(*DownBlock)
self.DownBlock_list = DownBlock
self.FC = Sequential(*FC)
self.UpBlock2 = Sequential(*UpBlock2)
def forward(self, input):
x = self.DownBlock(input)
gap = layers.adaptive_pool2d(x, 1, pool_type="avg")
gap_logit = self.gap_fc(fluid.layers.reshape(gap, (x.shape[0], -1)))
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = fluid.layers.reshape(gap_weight, (-1, gap_weight.shape[0]))
gap = x * fluid.layers.unsqueeze(fluid.layers.unsqueeze(gap_weight, 2), 3)
gmp = layers.adaptive_pool2d(x, 1, pool_type="max")
gmp_logit = self.gmp_fc(fluid.layers.reshape(gmp, (x.shape[0], -1)))
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = fluid.layers.reshape(gmp_weight, (-1, gmp_weight.shape[0]))
gmp = x * fluid.layers.unsqueeze(fluid.layers.unsqueeze(gmp_weight, 2), 3)
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
x = layers.concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
if self.light:
x_ = layers.adaptive_pool2d(x, 1, pool_type="avg")
x_ = self.FC(fluid.layers.reshape(x_, (x_.shape[0], -1)))
else:
x_ = self.FC(fluid.layers.reshape(x, (x.shape[0], -1)))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i+1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
class ResnetGenerator(fluid.dygraph.Layer):
def __init__(self,
input_nc,
output_nc,
ngf=64,
n_blocks=6,
img_size=256,
light=False):
assert (n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = []
# 先通过一个卷积核尺寸为7的卷积层，图片大小不变，通道数变为64
DownBlock += [
ReflectionPad2d(3),
Conv2D(input_nc,
ngf,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
InstanceNorm(ngf),
PRelu(mode="all")
]
# Down-Sampling --> 下采样模块
n_downsampling = 2
# 两层下采样，img_size缩小4倍（64），通道数扩大4倍（256）
for i in range(n_downsampling):
mult = 2**i
DownBlock += [
ReflectionPad2d(1),
Conv2D(ngf * mult,
ngf * mult * 2,
filter_size=3,
stride=2,
padding=0,
bias_attr=False),
InstanceNorm(ngf * mult * 2),
PRelu(mode="all")
]
# Down-Sampling Bottleneck --> 编码器中的残差模块
mult = 2**n_downsampling
# 6个残差块，尺寸和通道数都不变
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
# Class Activation Map --> 产生类别激活图
# 接着global average pooling后的全连接层
self.gap_fc = Linear(ngf * mult, 1, bias_attr=False)
# 接着global max pooling后的全连接层
self.gmp_fc = Linear(ngf * mult, 1, bias_attr=False)
#下面1x1卷积和激活函数，是为了得到两个pooling合并后的特征图
self.conv1x1 = Conv2D(ngf * mult * 2,
ngf * mult,
filter_size=1,
stride=1,
bias_attr=True,
act='relu')
# self.relu = nn.ReLU(True)
# Gamma, Beta block --> 生成自适应 L-B Normalization(AdaILN)中的Gamma, Beta
# 确定轻量级，FC使用的是两个256 --> 256的全连接层
if self.light:
FC = [
Linear(ngf * mult, ngf * mult, bias_attr=False, act='relu'),
# nn.ReLU(True),
Linear(ngf * mult, ngf * mult, bias_attr=False, act='relu'),
# nn.ReLU(True)
]
else:
# 不是轻量级，则下面的1024x1024 --> 256的全连接层和一个256 --> 256的全连接层
FC = [
Linear(img_size // mult * img_size // mult * ngf * mult,
ngf * mult,
bias_attr=False,
act='relu'),
# nn.ReLU(True),
Linear(ngf * mult, ngf * mult, bias_attr=False, act='relu'),
# nn.ReLU(True)
]
# AdaILN中的Gamma, Beta
self.gamma = Linear(ngf * mult, ngf * mult, bias_attr=False)
self.beta = Linear(ngf * mult, ngf * mult, bias_attr=False)
# Up-Sampling Bottleneck --> 解码器中的自适应残差模块
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i + 1),
ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling --> 解码器中的上采样模块
UpBlock2 = []
# 上采样与编码器的下采样对应
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [
Upsample(),
ReflectionPad2d(1),
Conv2D(ngf * mult,
int(ngf * mult / 2),
filter_size=3,
stride=1,
padding=0,
bias_attr=False,
act='relu'),
ILN(int(ngf * mult / 2)), # 注:只有自适应残差块使用AdaILN
# nn.ReLU(True)
]
# 最后一层卷积层，与最开始的卷积层对应
UpBlock2 += [
ReflectionPad2d(3),
Conv2D(ngf,
output_nc,
filter_size=7,
stride=1,
padding=0,
bias_attr=False,
act='tanh'),
# nn.Tanh()
]
self.DownBlock = Sequential(*DownBlock) # 编码器整个模块
self.FC = Sequential(*FC) # 生成gamma,beta的全连接层模块
self.UpBlock2 = Sequential(*UpBlock2) # 只包含上采样后的模块，不包含残差块
def forward(self, input):
x = self.DownBlock(input)
# 得到编码器的输出,对应途中encoder feature map
# torch.Size([1, 256, 64, 64])
# gap torch.Size([1, 256, 1, 1])
gap = Pool2D(pool_size=x.shape[-1],
pool_stride=x.shape[-1],
pool_type='avg')(x) #全局平均池化
gap = fluid.layers.reshape(gap, shape=[x.shape[0],
-1]) #torch.Size([1, 1])
gap_logit = self.gap_fc(gap) #gap的预测
gap_weight = list(self.gap_fc.parameters())[
0] #self.gap_fc的权重参数 torch.Size([1, 256])
gap_weight = fluid.layers.unsqueeze(input=gap_weight, axes=[0])
gap_weight = fluid.layers.unsqueeze(input=gap_weight, axes=[3])
gap = x * gap_weight #得到全局平均池化加持权重的特征图 torch.Size([1, 256, 64, 64])
gmp = Pool2D(pool_size=x.shape[-1],
pool_stride=x.shape[-1],
pool_type='max')(x)
gmp = fluid.layers.reshape(gmp, shape=[x.shape[0], -1])
gmp_logit = self.gmp_fc(gmp)
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = fluid.layers.unsqueeze(input=gmp_weight, axes=[0])
gmp_weight = fluid.layers.unsqueeze(input=gmp_weight, axes=[3])
gmp = x * gmp_weight #torch.Size([1, 256, 64, 64])
cam_logit = fluid.layers.concat([gap_logit, gmp_logit],
1) #结合gap和gmp的cam_logit预测
x = fluid.layers.concat([gap, gmp], 1) #torch.Size([1, 512, 64, 64])
x = self.conv1x1(x) #接入一个卷积层，通道数512转换为256 torch.Size([1, 256, 64, 64])
#x = self.relu(self.conv1x1(x))
#torch.Size([1, 256, 64, 64])
# heatmap = torch.sum(x, dim=1, keepdim=True)
heatmap = fluid.layers.reduce_sum(x, dim=1, keep_dim=True) #得到注意力热力图
#heatmap torch.Size([1, 1, 64, 64])
if self.light:
#轻量级则先经过一个gap
x_ = fluid.layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = fluid.layers.reshape(x_, shape=[x.shape[0], -1])
x_ = self.FC(x_)
else:
x_ = fluid.layers.reshape(x, shape=[x.shape[0], -1])
x_ = self.FC(x_)
gamma, beta = self.gamma(x_), self.beta(x_) #得到自适应gamma和beta
# gamma torch.Size([1, 256]) beta torch.Size([1, 256])
for i in range(self.n_blocks):
# 将自适应gamma和beta送入到AdaILN
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x) #通过上采样后的模块，得到生成结果
#out torch.Size([1, 3, 256, 256]) cam_logit torch.Size([1, 2]) heatmap torch.Size([1, 1, 64, 64])
return out, cam_logit, heatmap #模型输出为生成结果，cam预测以及热力图
class ResnetGenerator(dygraph.Layer):
def __init__(self,
input_nc,
output_nc,
ngf=64,
n_blocks=6,
img_size=256,
light=False):
super(ResnetGenerator, self).__init__()
'''
Args:
input_cn: 输入通道数
output_nc: 输出通道数，此处二者都为3
ngf: base channel number per layer
n_blocks: The number of resblock
'''
assert (n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = []
DownBlock += [
ReflectionPad2d([3, 3, 3, 3]),
Conv2D(input_nc,
ngf,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
InstanceNorm(ngf),
ReLU(True)
]
# Down-Sampling
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
DownBlock += [
ReflectionPad2d([1, 1, 1, 1]),
Conv2D(ngf * mult,
ngf * mult * 2,
filter_size=3,
stride=2,
padding=0,
bias_attr=False),
InstanceNorm(ngf * mult * 2),
ReLU(True)
]
# Down-Sampling Bottleneck
mult = 2**n_downsampling
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
# Class Activation Map
self.gap_fc = Linear(ngf * mult, 1, bias_attr=False)
self.gmp_fc = Linear(ngf * mult, 1, bias_attr=False)
self.conv1x1 = Conv2D(ngf * mult * 2,
ngf * mult,
filter_size=1,
stride=1,
padding=0,
bias_attr=True)
self.relu = ReLU(True)
# Gamma, Beta block
if self.light:
FC = [
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(True),
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(True)
]
else:
FC = [
Linear(img_size // mult * img_size // mult * ngf * mult,
ngf * mult,
bias_attr=False),
ReLU(True),
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(True)
]
self.gamma = Linear(ngf * mult, ngf * mult, bias_attr=False)
self.beta = Linear(ngf * mult, ngf * mult, bias_attr=False)
# Up-Sampling Bottleneck
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i + 1),
ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [
Upsample(scale_factor=2),
Debug('Upsample Pass'),
ReflectionPad2d([1, 1, 1, 1]),
Debug('ReflectionPad2d Pass'),
Conv2D(ngf * mult,
int(ngf * mult / 2),
filter_size=3,
stride=1,
padding=0,
bias_attr=False),
Debug('Conv2D Pass'),
ILN(int(ngf * mult / 2)),
Debug('ILN Pass'),
ReLU(True)
]
UpBlock2 += [
ReflectionPad2d([3, 3, 3, 3]),
Conv2D(ngf,
output_nc,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
Tanh()
]
self.DownBlock = Sequential(*DownBlock)
self.FC = Sequential(*FC)
self.UpBlock2 = Sequential(*UpBlock2)
def forward(self, input):
shape_print('Generator')
shape_print('input shape:' + str(input.shape))
x = self.DownBlock(input)
shape_print('downblock shape:' + str(x.shape))
debug_print('DownBlock Pass')
debug_save_img(x, 'DownBlock')
gap = layers.adaptive_pool2d(x, 1, pool_type='avg')
shape_print('gap shape:' + str(gap.shape))
gap_logit = self.gap_fc(layers.reshape(gap, [x.shape[0], -1]))
shape_print('gap_logit shape:' + str(gap_logit.shape))
debug_print('GAP logit Pass')
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = layers.reshape(gap_weight, [x.shape[0], -1])
shape_print('gap_weight shape:' + str(gap_weight.shape))
debug_print('GAP weight Pass')
gap = x * layers.unsqueeze(layers.unsqueeze(gap_weight, 2), 3)
shape_print('gap shape:' + str(gap.shape))
debug_print('GAP Pass')
gmp = layers.adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(layers.reshape(gmp, [x.shape[0], -1]))
shape_print('gmp_logit shape:' + str(gmp_logit.shape))
debug_print('GMP logit Pass')
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = layers.reshape(gmp_weight, [x.shape[0], -1])
shape_print('gmp_weight shape:' + str(gmp_weight.shape))
debug_print('GMP weight Pass')
gmp = x * layers.unsqueeze(layers.unsqueeze(gmp_weight, 2), 3)
shape_print('gmp shape:' + str(gmp.shape))
debug_print('GMP Pass')
cam_logit = layers.concat([gap_logit, gmp_logit], 1)
shape_print('cam logit shape:' + str(cam_logit.shape))
debug_print('CAM logit Pass')
x = layers.concat([gap, gmp], 1)
shape_print('x shape:' + str(x.shape))
x = self.relu(self.conv1x1(x))
shape_print('x shape:' + str(x.shape))
heatmap = layers.reduce_sum(x, dim=1, keep_dim=True)
shape_print('heatmap shape:' + str(heatmap.shape))
if self.light:
x_ = layers.adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(layers.reshape(x_, [x_.shape[0], -1]))
shape_print('FC shape:' + str(x_.shape))
debug_print('FC Pass')
else:
x_ = self.FC(layers.reshape(x, [x.shape[0], -1]))
shape_print('FC shape:' + str(x_.shape))
gamma, beta = self.gamma(x_), self.beta(x_)
shape_print('gamma shape:' + str(gamma.shape))
shape_print('beta shape:' + str(beta.shape))
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
debug_save_img(x, 'UpBlock1_' + str(i + 1))
shape_print('UpBlock1_' + str(i + 1) + 'shape:' + str(x.shape))
debug_print('UpBlock1_' + str(i + 1) + 'Pass')
out = self.UpBlock2(x)
debug_save_img(out, 'out')
debug_print('UpBlock2 Pass')
return out, cam_logit, heatmap
class ResnetGenerator(fluid.dygraph.Layer):
def __init__(self,
input_nc,
output_nc,
ngf=64,
n_blocks=6,
img_size=256,
light=False):
assert (n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
DownBlock = []
DownBlock += [
ReflectionPad2d(pad=3),
Conv2D(num_channels=input_nc,
num_filters=ngf,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
InstanceNorm(num_channels=ngf),
ReLU(inplace=True)
]
# Down-Sampling
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
DownBlock += [
ReflectionPad2d(pad=1),
Conv2D(num_channels=ngf * mult,
num_filters=ngf * mult * 2,
filter_size=3,
stride=2,
padding=0,
bias_attr=False),
InstanceNorm(num_channels=ngf * mult * 2),
ReLU(inplace=True)
]
# Down-Sampling Bottleneck
mult = 2**n_downsampling
for i in range(n_blocks):
DownBlock += [ResnetBlock(ngf * mult, use_bias=False)]
# Class Activation Map
self.gap_fc = Linear(input_dim=ngf * mult,
output_dim=1,
bias_attr=False)
self.gmp_fc = Linear(input_dim=ngf * mult,
output_dim=1,
bias_attr=False)
self.conv1x1 = Conv2D(num_channels=ngf * mult * 2,
num_filters=ngf * mult,
filter_size=1,
stride=1,
bias_attr=True)
self.relu = ReLU(inplace=True)
# Gamma, Beta block
if self.light:
FC = [
Linear(input_dim=ngf * mult,
output_dim=ngf * mult,
bias_attr=False),
ReLU(inplace=True),
Linear(input_dim=ngf * mult,
output_dim=ngf * mult,
bias_attr=False),
ReLU(True)
]
else:
FC = [
Linear(input_dim=img_size // mult * img_size // mult * ngf *
mult,
output_dim=ngf * mult,
bias_attr=False),
ReLU(inplace=True),
Linear(input_dim=ngf * mult,
output_dim=ngf * mult,
bias_attr=False),
ReLU(True)
]
self.gamma = Linear(input_dim=ngf * mult,
output_dim=ngf * mult,
bias_attr=False)
self.beta = Linear(input_dim=ngf * mult,
output_dim=ngf * mult,
bias_attr=False)
# Up-Sampling Bottleneck
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i + 1),
ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [
Upsample(scale=2),
ReflectionPad2d(pad=1),
Conv2D(num_channels=ngf * mult,
num_filters=int(ngf * mult / 2),
filter_size=3,
stride=1,
padding=0,
bias_attr=False),
ILN(int(ngf * mult / 2)),
ReLU(True)
]
UpBlock2 += [
ReflectionPad2d(pad=3),
Conv2D(num_channels=ngf,
num_filters=output_nc,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
Tanh()
]
self.DownBlock = Sequential(*DownBlock)
self.FC = Sequential(*FC)
self.UpBlock2 = Sequential(*UpBlock2)
def forward(self, input):
x = self.DownBlock(input)
gap = fluid.layers.adaptive_pool2d(input=x,
pool_size=1,
pool_type='avg')
gap = fluid.layers.reshape(x=gap, shape=[x.shape[0], -1])
gap_logit = self.gap_fc(gap)
gap_weight = list(self.gap_fc.parameters())[0]
gap_weight = fluid.layers.reshape(
x=gap_weight, shape=[gap_weight.shape[1], gap_weight.shape[0]])
gap_weight = fluid.layers.unsqueeze(input=gap_weight, axes=[2, 3])
gap = x * gap_weight
gmp = fluid.layers.adaptive_pool2d(input=x,
pool_size=1,
pool_type='max')
gmp = fluid.layers.reshape(x=gmp, shape=[x.shape[0], -1])
gmp_logit = self.gmp_fc(gmp)
gmp_weight = list(self.gmp_fc.parameters())[0]
gmp_weight = fluid.layers.reshape(
x=gmp_weight, shape=[gmp_weight.shape[1], gmp_weight.shape[0]])
gmp_weight = fluid.layers.unsqueeze(input=gmp_weight, axes=[2, 3])
gmp = x * gmp_weight
cam_logit = fluid.layers.concat(input=[gap_logit, gmp_logit], axis=1)
x = fluid.layers.concat(input=[gap, gmp], axis=1)
x = self.relu(self.conv1x1(x))
heatmap = fluid.layers.reduce_sum(input=x, dim=1, keep_dim=True)
if self.light:
x_ = fluid.layers.adaptive_pool2d(input=x,
pool_size=1,
pool_type='avg')
x_ = fluid.layers.reshape(x=x_, shape=[x_.shape[0], -1])
x_ = self.FC(x_)
else:
x = fluid.layers.reshape(x=x, shape=[x.shape[0], -1])
x_ = self.FC(x)
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap
class ResnetGenerator(Layer):
def __init__(self,
input_nc,
output_nc,
ngf=64,
n_blocks=6,
img_size=256,
light=False):
assert (n_blocks >= 0)
super(ResnetGenerator, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.n_blocks = n_blocks
self.img_size = img_size
self.light = light
self.DownBlock1_1 = ReflectionPad2D(3)
self.DownBlock1_2 = Conv2D(3,
64,
filter_size=7,
stride=1,
padding=0,
bias_attr=False)
self.DownBlock1_4 = ReLU(False)
self.DownBlock2_1 = ReflectionPad2D(1)
self.DownBlock2_2 = Conv2D(64,
128,
filter_size=3,
stride=2,
padding=0,
bias_attr=False)
self.DownBlock2_4 = ReLU(False)
self.DownBlock3_1 = ReflectionPad2D(1)
self.DownBlock3_2 = Conv2D(128,
256,
filter_size=3,
stride=2,
padding=0,
bias_attr=False)
self.DownBlock3_4 = ReLU(False)
n_downsampling = 2
# Down-Sampling
self.DownBlock1 = ResnetBlock(256, use_bias=False)
self.DownBlock2 = ResnetBlock(256, use_bias=False)
self.DownBlock3 = ResnetBlock(256, use_bias=False)
self.DownBlock4 = ResnetBlock(256, use_bias=False)
# Down-Sampling Bottleneck
mult = 4
# Class Activation Map
self.gap_fc = Linear(ngf * mult, 1, bias_attr=False)
self.gmp_fc = Linear(ngf * mult, 1, bias_attr=False)
self.conv1x1 = Conv2D(
ngf * mult * 2,
ngf * mult,
filter_size=1,
stride=1,
bias_attr=fluid.ParamAttr(initializer=fluid.initializer.Uniform(
low=-1 / math.sqrt(ngf * mult * 2),
high=1 / math.sqrt(ngf * mult * 2))))
self.relu = ReLU(False)
# Gamma, Beta block
if self.light:
FC = [
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(False),
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(False)
]
else:
FC = [
Linear(img_size // mult * img_size // mult * ngf * mult,
ngf * mult,
bias_attr=False),
ReLU(False),
Linear(ngf * mult, ngf * mult, bias_attr=False),
ReLU(False)
]
self.gamma = Linear(ngf * mult, ngf * mult, bias_attr=False)
self.beta = Linear(ngf * mult, ngf * mult, bias_attr=False)
# Up-Sampling Bottleneck
for i in range(n_blocks):
setattr(self, 'UpBlock1_' + str(i + 1),
ResnetAdaILNBlock(ngf * mult, use_bias=False))
# Up-Sampling
UpBlock2 = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
UpBlock2 += [
Upsample(scales=2, resamples='NEAREST'),
ReflectionPad2D(1),
Conv2D(ngf * mult,
int(ngf * mult / 2),
filter_size=3,
stride=1,
padding=0,
bias_attr=False),
ILN(int(ngf * mult / 2)),
ReLU(False)
]
UpBlock2 += [
ReflectionPad2D(3),
Conv2D(ngf,
output_nc,
filter_size=7,
stride=1,
padding=0,
bias_attr=False),
Tanh()
]
self.FC = Sequential(*FC)
self.UpBlock2 = Sequential(*UpBlock2)
def forward(self, input):
x = self.DownBlock1_1(input)
x = self.DownBlock1_2(x)
x = instance_norm(x)
x = self.DownBlock1_4(x)
x = self.DownBlock2_1(x)
x = self.DownBlock2_2(x)
x = instance_norm(x)
x = self.DownBlock2_4(x)
x = self.DownBlock3_1(x)
x = self.DownBlock3_2(x)
x = instance_norm(x)
x = self.DownBlock3_4(x)
gap = adaptive_pool2d(x, 1, pool_type='avg')
gap_logit = self.gap_fc(reshape(gap, [x.shape[0], -1]))
gap_weight = self.gap_fc.parameters()[0]
gap_weight = reshape(gap_weight, shape=[1, -1])
gap = x * unsqueeze(unsqueeze(gap_weight, 2), 3)
gmp = adaptive_pool2d(x, 1, pool_type='max')
gmp_logit = self.gmp_fc(reshape(gmp, [x.shape[0], -1]))
gmp_weight = self.gmp_fc.parameters()[0]
gmp_weight = reshape(gmp_weight, shape=[1, -1])
gmp = x * unsqueeze(unsqueeze(gmp_weight, 2), 3)
cam_logit = concat([gap_logit, gmp_logit], 1)
x = concat([gap, gmp], 1)
x = self.relu(self.conv1x1(x))
heatmap = reduce_sum(x, dim=1, keep_dim=True)
if self.light:
x_ = adaptive_pool2d(x, 1, pool_type='avg')
x_ = self.FC(reshape(x_, [x_.shape[0], -1]))
else:
x_ = self.FC(reshape(x, [x.shape[0], -1]))
gamma, beta = self.gamma(x_), self.beta(x_)
for i in range(self.n_blocks):
x = getattr(self, 'UpBlock1_' + str(i + 1))(x, gamma, beta)
out = self.UpBlock2(x)
return out, cam_logit, heatmap