def visualize(self, layer, filt, lr=1e-2, opt_steps=36): sz = self.size img = Image.fromarray( np.uint8(np.random.uniform(150, 180, (sz, sz, 3)))) activations = SaveFeatures(list(self.target.children())[layer]) gaussian_filter = get_gaussian_kernel() self.model.zero_grad() for outer in tqdm(range(self.upscaling_steps), leave=False): img_var = torch.unsqueeze(ToTensor()(img), 0).cuda(ORDINAL).requires_grad_(True) img_var.requires_grad_(True).cuda(ORDINAL) optimizer = torch.optim.Adam([img_var], lr=lr, weight_decay=1e-6) pbar = tqdm(range(opt_steps), leave=False) for n in pbar: optimizer.zero_grad() self.model(img_var) loss = -activations.features[ 0, filt].mean() + 0.00 * torch.norm(img_var) loss.backward() pbar.set_description(f'Loss: {loss.item()}') optimizer.step() sz = int(sz * self.upscaling_factor) img = ToPILImage()(img_var.squeeze(0)) if outer != self.upscaling_steps: img = img.resize((sz, sz)) img = img.filter(ImageFilter.BoxBlur(2)) self.output = img.copy() self.save(layer, filt) activations.close()
def __getitem__(self, index): input = load_img(self.image_filenames[index]) #input是预先合成的4通道RGDB图片 #数据增强 if self.crop: input = RandomCrop(64)(input) #取patch input = RandomHorizontalFlip()(input) #水平翻转 input = RandomVerticalFlip()(input) #竖直翻转 input = RandomRotation(180)(input) #随机旋转 input_tensor = ToTensor()(input) rgb_tensor = torch.zeros(3, input_tensor.shape[1], input_tensor.shape[2]) depth_tensor = torch.zeros(1, input_tensor.shape[1], input_tensor.shape[2]) rgb_tensor[0, :, :] = input_tensor[0, :, :] rgb_tensor[1, :, :] = input_tensor[1, :, :] rgb_tensor[2, :, :] = input_tensor[2, :, :] depth_tensor[0, :, :] = input_tensor[3, :, :] depth = ToPILImage()(depth_tensor) size = min(depth.size[0], depth.size[1]) guide = ToPILImage()(rgb_tensor) target = depth.copy() guide = guide.convert('L') #生成LR depth = downsampling(depth, self.upscale_factor) depth = Resize(size=size, interpolation=Image.BICUBIC)(depth) depth = ToTensor()(depth) guide = ToTensor()(guide) depth = torch.cat((depth, guide), 0) #concatenate 生成输入张量 target = ToTensor()(target) return depth, target
def create_image(net, loader, name, upscaling=2, multiscale=False): patch_size = 30 patches = np.array( [[85, 90, 85 + patch_size, 90 + patch_size], [160, 140, 160 + patch_size, 140 + patch_size], [350, 80, 350 + patch_size, 80 + patch_size]]) fig = plt.figure(figsize=(15, 18), dpi=100) gs = fig.add_gridspec(4, 3, wspace=0.01, hspace=0.3, left=0.05, top=0.95, bottom=0.02, right=0.95) highres, lowres = loader[0] lowres = lowres.to(torch.device("cuda:0")) lowres = lowres.view([1] + list(lowres.size())) net.eval() with torch.no_grad(): if (multiscale): superres = net(lowres, upscaling) else: superres = net(lowres) lowres = ToImage()(lowres.cpu().view(lowres.size()[1:])) superres = ToImage()(superres.cpu().view(superres.size()[1:])) highres = ToImage()(highres.cpu().view(highres.size()[1:])) lowres_draw = lowres.copy() draw = ImageDraw.Draw(lowres_draw) draw.rectangle(list(patches[0]), outline='white') draw.rectangle(list(patches[1]), outline='white') draw.rectangle(list(patches[2]), outline='white') lowres = lowres.resize((lowres.size[0] * upscaling, lowres.size[1] * upscaling), Image.BICUBIC) psnr_low, psnr_super, ssim_low, ssim_super = compute_metrics( net, DataLoader(loader), upscaling, multiscale) lowres_title = "Low-resolution image" +\ "\nAverage PSNR over the dataset: {:.2f}\n".format(psnr_low) +\ "Average SSIM over the dataset: {:.4f}".format(ssim_low) superres_title = "Reconstructed image\n" +\ "Average PSNR over the dataset: {:.2f}\n".format(psnr_super) +\ "Average SSIM over the dataset: {:.4f}".format(ssim_super) fig.add_subplot(gs[0, 0], xticks=[], yticks=[], ylabel=f"Image", title=lowres_title) plt.imshow(np.array(lowres_draw)) fig.add_subplot(gs[0, 1], xticks=[], yticks=[], title=superres_title) plt.imshow(np.array(superres)) fig.add_subplot(gs[0, 2], xticks=[], yticks=[], title="High-resolution image") plt.imshow(np.array(highres)) ylabels = ["Patch 1", "Patch 2", "Patch 3"] for i in range(3): print(lowres.size, highres.size, superres.size) lowres_patch = lowres.crop(patches[i] * upscaling) highres_patch = highres.crop(patches[i] * upscaling) superres_patch = superres.crop(patches[i] * upscaling) fig.add_subplot(gs[1 + i, 0], xticks=[], yticks=[], ylabel=ylabels[i]) plt.imshow(np.array(lowres_patch)) fig.add_subplot(gs[1 + i, 1], xticks=[], yticks=[]) plt.imshow(np.array(superres_patch)) fig.add_subplot(gs[1 + i, 2], xticks=[], yticks=[]) plt.imshow(np.array(highres_patch)) plt.savefig(name)