def compute_ll(model, image_rgb, image_q):
b, c, h, w = image_rgb[0].size()
# Loss long
l_long = 0
for i in range(1,3):
for j in range(i):
ref_g = image_rgb[j]
if j == 0:
ref_c = image_q[j]
else:
ref_c = outputs
tar_g = image_rgb[j + 1]
outputs = model(ref_g, ref_c, tar_g)
outputs = F.interpolate(outputs, (h, w), mode='bilinear')
for j in range(i,0,-1):
ref_g = image_rgb[j]
ref_c = outputs
tar_g = image_rgb[j-1]
outputs = model(ref_g, ref_c, tar_g)
outputs = F.interpolate(outputs, (h, w), mode='bilinear')
tar_c = image_q[0]
tar_c = torch.squeeze(tar_c, 1)
loss = cross_entropy(outputs, tar_c, size_average=True)
l_long += loss
return l_long
def compute_ls(model, image_rgb, image_q, bi, epoch, n_b):
eps_s, eps_e = 0.9, 0.6
b, c, h, w = image_rgb[0].size()
# Loss similarity
l_sim = 0
for i in range(2): # 3-1
ref_g = image_rgb[i]
tar_g = image_rgb[i + 1]
tar_c = image_q[i + 1]
tar_c = torch.squeeze(tar_c, 1)
total_batch = args.epochs * n_b
current_batch = epoch * n_b + bi
thres = eps_s - (eps_s - eps_e) * current_batch / total_batch
truth = np.random.random() < thres
ref_c = image_q[i] if truth or (i == 0) else outputs
outputs = model(ref_g, ref_c, tar_g)
outputs = F.interpolate(outputs, (h, w), mode='bilinear')
loss = cross_entropy(outputs, tar_c, size_average=True)
l_sim += loss
return l_sim
def forward(self, higher_res_feature, lower_res_feature):
lower_res_feature = F.interpolate(
lower_res_feature, scale_factor=4, mode="bilinear", align_corners=True
)
lower_res_feature = self.dwconv(lower_res_feature)
lower_res_feature = self.conv_lower_res(lower_res_feature)
higher_res_feature = self.conv_higher_res(higher_res_feature)
out = higher_res_feature + lower_res_feature
return self.relu(out)
def forward(self, src, tgt):
x = torch.cat([src, tgt], dim=1)
size = x.size()[2:]
higher_res_features = self.learning_to_downsample(x)
x = self.global_feature_extractor(higher_res_features)
x = self.feature_fusion_module(higher_res_features, x)
deform = self.deformation(x)
deform = F.interpolate(deform, list(map(int, size)), mode="bilinear", align_corners=True) * 10.
y = self.spatial_transform(src, deform)
return y, deform
def forward(self, features, skips):
features = self.ASPP(features)
current_os = self.backbone_OS
# do one upconv and one residual
for matchconv, mixconv in zip(self.matchconvs, self.mixconvs):
current_os //= 2
# upconv (convolution + upsampling)
features = matchconv(features)
# upsample
_, _, H, W = features.shape
features = F.interpolate(features,
size=[int(2 * H), int(2 * W)],
mode="nearest")
# add from skip (no grad)
if current_os in self.skip_os:
# this elementwise product with the sigmoid should scale the features
# so that the ones corresponding to high activations in the encoder
# are scaled up and the other ones scaled down
attention = torch.sigmoid(skips[current_os].detach())
features = features * attention
# do mixing of lower and higher levels
features = mixconv(features)
return features
def upsample(self, x, size):
return F.interpolate(
x, list(map(int, size)), mode="bilinear", align_corners=True
)
out = F.conv2d(x, w, b, stride=1, padding=1)
#P55 pooling下采样-->max/avg pooling
#upsample上采样
x = out
layer = nn.MaxPool2d(2, stride=2) #window的大小和补偿的大小
out = layer(x)
out = F.avg_pool2d(x, 2, stride=2)
#F.interpolate
x = out
out = F.interpolate(x, scale_factor=2,
mode='nearest') #scale_factor是放大倍数，mode是紧邻差值模式
out.shape
out = F.interpolaye(x, scale_factor=3, mode='nearest')
out.shape
#卷积函数常用单元Unit:conv2d-->batch normalization-->pool-->relu(顺序可以颠倒)
x.shape
layer = nn.ReLU(inplace=True) #设置inplace为true的结果是，x'会占据x原本的空间（正常情况下会节省一半的空间）
out = layer(x)
out.shape
out = F.relu(x)
out.shape
writer = SummaryWriter(
os.path.join('tensorboard', args['save_name'] + "_scale" + str(s)))
iteration = 0
for epoch in range(args['start_epoch'], args['epochs']):
print("Starting epoch %i" % (epoch))
for i, (real_heightmaps) in enumerate(dataloader):
real_heightmaps = real_heightmaps.to(args['device'])
real_heightmaps = laplace_pyramid_downscale2D(
real_heightmaps, n - s - 1, 0.75, args['device'])
if (s > 0):
fake_heightmaps = generate(gs[0:s],
real_heightmaps.shape[0],
args['device'])
fake_heightmaps = F.interpolate(fake_heightmaps,
size=g.resolution,
mode="bilinear",
align_corners=False)
else:
fake_heightmaps = torch.zeros(real_heightmaps.shape,
device=args['device'])
fake_heightmaps = g(fake_heightmaps)
# Update discriminator: maximize D(x) + D(G(z))
for j in range(3):
d.zero_grad()
# Train with real
output = d(real_heightmaps)
discrim_error_real = -output.mean()
discrim_error_real.backward(retain_graph=True)
def _forward_impl(self, x):
xin = x.clone()
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.conv2(x)
x = self.bn2(x)
x = self.relu(x)
x = self.conv3(x)
x = self.bn3(x)
# x = F.max_pool2d(x,2,2)
x = self.relu(x)
# x = self.maxpool(x)
# pdb.set_trace()
x1 = self.layer1(x)
# print(x1.shape)
x2 = self.layer2(x1)
# print(x2.shape)
# x3 = self.layer3(x2)
# # print(x3.shape)
# x4 = self.layer4(x3)
# # print(x4.shape)
# x = F.relu(F.interpolate(self.decoder1(x4), scale_factor=(2,2), mode ='bilinear'))
# x = torch.add(x, x4)
# x = F.relu(F.interpolate(self.decoder2(x4) , scale_factor=(2,2), mode ='bilinear'))
# x = torch.add(x, x3)
# x = F.relu(F.interpolate(self.decoder3(x3) , scale_factor=(2,2), mode ='bilinear'))
# x = torch.add(x, x2)
x = F.relu(
F.interpolate(self.decoder4(x2),
scale_factor=(2, 2),
mode='bilinear'))
x = torch.add(x, x1)
x = F.relu(
F.interpolate(self.decoder5(x),
scale_factor=(2, 2),
mode='bilinear'))
# print(x.shape)
# end of full image training
# y_out = torch.ones((1,2,128,128))
x_loc = x.clone()
# x = F.relu(F.interpolate(self.decoder5(x) , scale_factor=(2,2), mode ='bilinear'))
#start
for i in range(0, 4):
for j in range(0, 4):
x_p = xin[:, :, 32 * i:32 * (i + 1), 32 * j:32 * (j + 1)]
# begin patch wise
x_p = self.conv1_p(x_p)
x_p = self.bn1_p(x_p)
# x = F.max_pool2d(x,2,2)
x_p = self.relu(x_p)
x_p = self.conv2_p(x_p)
x_p = self.bn2_p(x_p)
# x = F.max_pool2d(x,2,2)
x_p = self.relu(x_p)
x_p = self.conv3_p(x_p)
x_p = self.bn3_p(x_p)
# x = F.max_pool2d(x,2,2)
x_p = self.relu(x_p)
# x = self.maxpool(x)
# pdb.set_trace()
x1_p = self.layer1_p(x_p)
# print(x1.shape)
x2_p = self.layer2_p(x1_p)
# print(x2.shape)
x3_p = self.layer3_p(x2_p)
# # print(x3.shape)
x4_p = self.layer4_p(x3_p)
x_p = F.relu(
F.interpolate(self.decoder1_p(x4_p),
scale_factor=(2, 2),
mode='bilinear'))
x_p = torch.add(x_p, x4_p)
x_p = F.relu(
F.interpolate(self.decoder2_p(x_p),
scale_factor=(2, 2),
mode='bilinear'))
x_p = torch.add(x_p, x3_p)
x_p = F.relu(
F.interpolate(self.decoder3_p(x_p),
scale_factor=(2, 2),
mode='bilinear'))
x_p = torch.add(x_p, x2_p)
x_p = F.relu(
F.interpolate(self.decoder4_p(x_p),
scale_factor=(2, 2),
mode='bilinear'))
x_p = torch.add(x_p, x1_p)
x_p = F.relu(
F.interpolate(self.decoder5_p(x_p),
scale_factor=(2, 2),
mode='bilinear'))
x_loc[:, :, 32 * i:32 * (i + 1), 32 * j:32 * (j + 1)] = x_p
x = torch.add(x, x_loc)
x = F.relu(self.decoderf(x))
x = self.adjust(F.relu(x))
# pdb.set_trace()
return x