# mode(str)- 模式选择，有 min 和 max 两种模式， min 表示当指标不再降低(如监测loss)， max 表示当指标不再升高(如监测 accuracy)。
# factor(float)- 学习率调整倍数(等同于其它方法的 gamma)，即学习率更新为 lr = lr * factor
# patience(int)- 忍受该指标多少个 step 不变化，当忍无可忍时，调整学习率。
# verbose(bool)- 是否打印学习率信息， print(‘Epoch {:5d}: reducing learning rate of group {} to {:.4e}.’.format(epoch, i, new_lr))
# threshold_mode(str)- 选择判断指标是否达最优的模式，有两种模式， rel 和 abs。
# 当 threshold_mode == rel，并且 mode == max 时， dynamic_threshold = best * ( 1 +threshold )；
# 当 threshold_mode == rel，并且 mode == min 时， dynamic_threshold = best * ( 1 -threshold )；
# 当 threshold_mode == abs，并且 mode== max 时， dynamic_threshold = best + threshold ；
# 当 threshold_mode == abs，并且 mode == max 时， dynamic_threshold = best - threshold；
# threshold(float)- 配合 threshold_mode 使用。
# cooldown(int)- “冷却时间“，当调整学习率之后，让学习率调整策略冷静一下，让模型再训练一段时间，再重启监测模式。
# min_lr(float or list)- 学习率下限，可为 float，或者 list，当有多个参数组时，可用 list 进行设置。
# eps(float)- 学习率衰减的最小值，当学习率变化小于 eps 时，则不调整学习率。
MSE_fun = nn.MSELoss()
SSIM_fun = SSIM()
CrossEntropyLoss = nn.CrossEntropyLoss()
with torch.no_grad():
loss_network = LossNetwork()
loss_network.to(device)
loss_network.eval()
# Training
mse_train = []
ssim_train = []
loss_train = []
mse_val = []
ssim_val = []
loss_val = []
gradient_train = []
mkdir(train_path)
mean_fid = cfd(mf, sf, mr, sr)
print(FOLDER)
print('FID: ', mean_fid)
print('Mean PSNR: ', mean_psnr / len(file_chunks))
print('Mean SSIM: ', mean_ssim / len(file_chunks))
print('Mean MS_SSIM: ', mean_msssim / len(file_chunks))
with open(FOLDER + '_metrics.txt', 'w') as f:
f.write('FID: %.6f\n' % mean_fid)
f.write('Mean PSNR: %.6f\n' % (mean_psnr / len(file_chunks)))
f.write('Mean SSIM: %.6f\n' % (mean_ssim / len(file_chunks)))
f.write('Mean MS_SSIM: %.6f\n' % (mean_msssim / len(file_chunks)))
if __name__ == '__main__':
ssim = SSIM(window_size=11, window_sigma=1.5, data_range=1., channel=3, use_padding=False).cuda()
msssim = MS_SSIM(window_size=11, window_sigma=1.5, data_range=1., channel=3, use_padding=False).cuda()
block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[2048]
fid_model = InceptionV3([block_idx]).cuda()
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/face_enhan_dixian.wp_16x'
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/face_enhan_lingbo.ylb_16x_subadd_encoder'
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/face_enhan_lingbo.ylb_16x_spade'
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/face_enhan_lingbo.ylb_3mix_LIP_encoder'
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/face_enhan_lingbo.ylb_motion_blur_LIP_encoder'
#FOLDER = '/home/lingbo.ylb/projects/face_sr_sota/PyTorch-ARCNN/results'
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/face_enhan_lingbo.ylb_jpeg_LIP_encoder'
#FOLDER = '/home/lingbo.ylb/projects/pg_spade_face_v4/results/mixed2jpeg'
#FOLDER = '/home/lingbo.ylb/projects/face_sr_sota/ESRGAN/results/ESRGAN-V1-FFHQ-16x'
#FOLDER = '/home/lingbo.ylb/projects/face_sr_sota/VDNet-master/results/noise'
counter = 0
for i, data in enumerate(dataset):
if i >= opt.how_many:
break
counter = i
model.set_input(data)
model.test()
visuals = model.get_current_visuals()
if opt.dataset_mode != 'single':
real_B = util.tensor2im(data['B1'])
avgPSNR += PSNR(visuals['Restored_Train'], real_B)
pilFake = Image.fromarray(visuals['Restored_Train'])
pilReal = Image.fromarray(real_B)
avgSSIM += SSIM(pilFake).cw_ssim_value(pilReal)
img_path = model.get_image_paths()
if opt.dataset_mode != 'single':
real_B = util.tensor2im(data['B1'])
psnr = PSNR(visuals['Restored_Train'], real_B)
avgPSNR += psnr
pilFake = Image.fromarray(visuals['Restored_Train'])
pilReal = Image.fromarray(real_B)
pilBlur = Image.fromarray(visuals['Blurred_Train'])
ssim = SSIM(pilFake).cw_ssim_value(pilReal)
avgSSIM += ssim
img_path = model.get_image_paths()
print('process image... {} {:.4f} {:.4f}'.format(img_path, psnr, ssim))
# visualizer.save_images(webpage, visuals, img_path)
def compare_ssim(im1, im2):
similar = SSIM(im1).cw_ssim_value(im2) * 100
if similar > 90:
return 'Same Image'
else:
return 'Different Image'
def __init__(self):
super(SSIMLoss, self).__init__()
self.ssim = SSIM(data_range=1.)
img = Image.fromarray(img, 'YCbCr')
img = img.convert('RGB')
return np.array(img)
def Y_UV2RGB(y, img):
# print img[:, :, 0] == np.array(y)
img[:, :, 0] = np.array(y)
yuv = img
img = Image.fromarray(yuv, 'YCbCr')
img = img.convert('RGB')
return img
transform = transforms.Compose([transforms.ToTensor()])
ssim_loss = SSIM(window_size=11).cuda()
mse_loss = nn.MSELoss().cuda()
def ssim_covers(stegos, imgs):
imgs = imgs.numpy()
stegos = to_np(stegos)
for i in range(len(stegos)):
img = stegos[i]
img = (((img - img.min()) * 255) / (img.max() - img.min())).astype(
np.uint8)
stegos[i] = img
origin = deepcopy(imgs)
stegos = [Y_UV2RGB(stego, img) for stego, img in zip(stegos, origin)]
covers = [Image.fromarray(YUV2RGB(img), 'RGB') for img in imgs]
else:
print(restored + fileN)
img_restored = cv2.imread(restored + fileN)
img_gray_restored = cv2.cvtColor(img_restored, cv2.COLOR_RGB2GRAY)
image_restored = Image.open(restored + fileN)
# before
img_corrupted = cv2.imread(corrupted + fileN)
image_corrupted = Image.open(corrupted + fileN)
img_gray_corrupted = cv2.cvtColor(img_corrupted, cv2.COLOR_RGB2GRAY)
# restored vs original
if grayscale:
vifp_value_B = QCM.vifp_mscale(img_gray_original, img_gray_corrupted)
psnr_value_B = QCM.psnr(img_gray_original, img_gray_corrupted)
reco_value_B = QCM.reco(img_gray_original, img_gray_corrupted)
ssim_value_B = SSIM(img_gray_original,
gaussian_kernel_1d).ssim_value(img_gray_corrupted)
# restored vs original
vifp_value_A = QCM.vifp_mscale(img_gray_original, img_gray_restored)
psnr_value_A = QCM.psnr(img_gray_original, img_gray_restored)
reco_value_A = QCM.reco(img_gray_original, img_gray_restored)
ssim_value_A = SSIM(img_gray_original,
gaussian_kernel_1d).ssim_value(img_gray_restored)
else:
vifp_value_B = QCM.vifp_mscale(img_clean, img_corrupted)
psnr_value_B = QCM.psnr(img_clean, img_corrupted)
# reco_value_B = QCM.reco(img_clean, img_corrupted)
ssim_value_B = SSIM(image_clean,
gaussian_kernel_1d).ssim_value(image_corrupted)
# restored vs original
vifp_value_A = QCM.vifp_mscale(img_clean, img_restored)
psnr_value_A = QCM.psnr(img_clean, img_restored)
def main():
global opt, name, logger, netG, netD, vgg, curriculum_ssim_mask, curriculum_ssim_clean, loss_mse, rgb2yuv, instance_ssim, loss_bce
opt = parser.parse_args()
name = "ShadowRemoval"
print(opt)
# Tag_ResidualBlocks_BatchSize
logger = SummaryWriter("./runs_sr/" +
time.strftime("/%Y-%m-%d-%H/", time.localtime()))
cuda = opt.cuda
if 'WORLD_SIZE' in os.environ:
opt.distributed = int(os.environ['WORLD_SIZE']) > 1
if cuda and not torch.cuda.is_available():
raise Exception("No GPU found, please run without --cuda")
if opt.distributed:
opt.gpu = opt.local_rank
torch.cuda.set_device(opt.gpu)
torch.distributed.init_process_group(backend='nccl',
init_method='env://')
opt.world_size = torch.distributed.get_world_size()
seed = 1334
torch.manual_seed(seed)
if cuda:
torch.cuda.manual_seed(seed)
cudnn.benchmark = True
print("==========> Loading datasets")
train_dataset = DatasetFromFolder(
opt.train,
transform=Compose([ToTensor()]),
training=True,
experiments="ShadowRemoval",
)
test_dataset = DatasetFromFolder(opt.test,
transform=Compose([ToTensor()]),
training=False,
experiments="ShadowRemoval")
train_data_loader = DataLoader(dataset=train_dataset,
num_workers=4,
batch_size=opt.batchSize,
pin_memory=True,
shuffle=True)
test_data_loader = DataLoader(dataset=test_dataset,
num_workers=4,
batch_size=opt.batchSize,
pin_memory=True,
shuffle=False)
print("==========> Building model")
netG = ShadowRemoval(channels=64)
netD = Discrimator(in_channels=6, channels=64, depth=3)
print("=========> Building criterion")
loss_smooth_l1 = nn.SmoothL1Loss()
loss_l1 = nn.L1Loss()
loss_mse = torch.nn.MSELoss()
loss_bce = torch.nn.BCELoss()
loss_perceptual = perceptual()
instance_ssim = SSIM(reduction='mean', window_size=7)
rgb2yuv = rgb2yuv()
curriculum_ssim_mask = CLBase(lossfunc=nn.BCELoss(reduce=False))
curriculum_ssim_clean = CLBase()
# optionally copy weights from a checkpoint
if opt.pretrained and opt.continue_training:
if os.path.isfile(opt.pretrained):
print("=> loading model '{}'".format(opt.pretrained))
weights = torch.load(opt.pretrained)
netG.load_state_dict(weights['state_dict'])
else:
print("=> no model found at '{}'".format(opt.pretrained))
print("==========> Setting Optimizer")
optimizerG = optim.Adam(filter(lambda p: p.requires_grad,
netG.parameters()),
lr=opt.lr_g,
betas=(0.9, 0.999))
#optimizerD = optim.Adam(filter(lambda p: p.requires_grad, netD.module.parameters() if opt.parallel else netD.parameters()), lr=opt.lr_d, betas = (0.5, 0.999))
optimizerD = optim.SGD(filter(lambda p: p.requires_grad,
netD.parameters()),
lr=opt.lr_d)
print("==========> Setting GPU")
if cuda:
netG = netG.cuda()
netD = netD.cuda()
instance_ssim = instance_ssim.cuda()
loss_smooth_l1 = loss_smooth_l1.cuda()
loss_mse = loss_mse.cuda()
loss_l1 = loss_l1.cuda()
loss_bce = loss_bce.cuda()
curriculum_ssim_mask = curriculum_ssim_mask.cuda()
curriculum_ssim_clean = curriculum_ssim_clean.cuda()
loss_perceptual = loss_perceptual.cuda()
rgb2yuv = rgb2yuv.cuda()
if opt.acceleration:
print("FP 16 Trianing")
amp.register_float_function(torch, 'sigmoid')
[netD,
netG], [optimizerD,
optimizerG] = amp.initialize([netD, netG],
[optimizerD, optimizerG],
opt_level=opt.opt_level)
if opt.parallel:
print("Parallel Training")
netG = nn.DataParallel(netG)
netD = nn.DataParallel(netD)
elif opt.distributed:
netG = DDP(netG, delay_allreduce=True)
netD = DDP(netD, delay_allreduce=True)
else:
netG = netG.cpu()
netD = netD.cpu()
instance_ssim = instance_ssim.cpu()
loss_smooth_l1 = loss_smooth_l1.cpu()
loss_mse = loss_mse.cpu()
loss_l1 = loss_l1.cpu()
loss_bce = loss_bce.cpu()
curriculum_ssim = curriculum_ssim.cpu()
loss_perceptual = loss_perceptual.cpu()
rgb2yuv = rgb2yuv.cpu()
lr_schedulerG = optim.lr_scheduler.CosineAnnealingLR(optimizerG,
opt.epoch,
eta_min=1e-7)
lr_schedulerD = optim.lr_scheduler.CosineAnnealingLR(optimizerD,
opt.epoch,
eta_min=1e-7)
print("==========> Training")
for epoch in range(opt.epoch + 1):
train(train_data_loader,
netG,
netD,
optimizerG,
optimizerD,
epoch,
logger=logger)
#test(test_data_loader, netG)
if epoch % opt.save_model_freq == 0:
save_checkpoint(netG, epoch, name, opt)
lr_schedulerG.step()
lr_schedulerD.step()
logger.close()
def main():
global opt, name, logger, netG, netD, vgg, curriculum_ssim, loss_mse, rgb2yuv, instance_ssim
opt = parser.parse_args()
name = "ShadowSyns"
print(opt)
# Tag_ResidualBlocks_BatchSize
cuda = opt.cuda
if cuda and not torch.cuda.is_available():
raise Exception("No GPU found, please run without --cuda")
seed = 1334
torch.manual_seed(seed)
if cuda:
torch.cuda.manual_seed(seed)
cudnn.benchmark = True
print("==========> Loading datasets")
train_dataset = DatasetFromFolder(
opt.train,
transform=Compose([ToTensor()]),
training = True,
)
test_dataset = DatasetFromFolder(
opt.test,
transform=Compose([ToTensor()]),
training = False,
)
train_data_loader = DataLoader(dataset=train_dataset, num_workers=4, batch_size=opt.batchSize,
pin_memory=True, shuffle=True)
test_data_loader = DataLoader(dataset=test_dataset, num_workers=4, batch_size=opt.batchSize,
pin_memory=True, shuffle=True)
print("==========> Building model")
netG = ShadowMattingNet(channels =64, depth = 9)
netD = Discrimator(in_channels = 7, channels = 64, depth = 5)
print("=========> Building criterion")
loss_smooth_l1 = nn.SmoothL1Loss()
loss_l1 = nn.L1Loss()
loss_mse = torch.nn.MSELoss()
instance_ssim = SSIM(reduction = 'mean', window_size = 7)
curriculum_ssim = CLBase()
loss_perceptual = perceptual()
rgb2yuv = rgb2yuv()
# optionally copy weights from a checkpoint
if opt.pretrained:
if os.path.isfile(opt.pretrained):
print("=> loading model '{}'".format(opt.pretrained))
weights = torch.load(opt.pretrained)
netG.load_state_dict(weights['state_dict_g'])
else:
print("=> no model found at '{}'".format(opt.pretrained))
print("==========> Setting GPU")
if cuda:
if opt.parallel:
netG = nn.DataParallel(netG, [0, 1, 2, 3]).cuda()
netD = nn.DataParallel(netD, [0, 1, 2, 3]).cuda()
instance_ssim = nn.DataParallel(instance_ssim, [0, 1, 2, 3]).cuda()
loss_smooth_l1 = nn.DataParallel(loss_smooth_l1, [0, 1, 2, 3]).cuda()
loss_mse = nn.DataParallel(loss_mse, [0, 1, 2, 3]).cuda()
loss_l1 = nn.DataParallel(loss_l1, [0, 1, 2, 3]).cuda()
curriculum_ssim = nn.DataParallel(curriculum_ssim, [0, 1, 2, 3]).cuda()
rgb2yuv = nn.DataParallel(rgb2yuv, [0, 1, 2, 3]).cuda()
else:
netG = netG.cuda()
netD = netD.cuda()
instance_ssim = instance_ssim.cuda()
loss_smooth_l1 = loss_smooth_l1.cuda()
loss_mse = loss_mse.cuda()
loss_l1 = loss_l1.cuda()
curriculum_ssim = curriculum_ssim.cuda()
loss_perceptual = loss_perceptual.cuda()
rgb2yuv = rgb2yuv.cuda()
else:
netG = netG.cpu()
netD = netD.cpu()
instance_ssim = instance_ssim.cpu()
loss_smooth_l1 = loss_smooth_l1.cpu()
loss_mse = loss_mse.cpu()
loss_l1 = loss_l1.cpu()
curriculum_ssim = curriculum_ssim.cpu()
loss_perceptual = loss_perceptual.cpu()
rgb2yuv = rgb2yuv.cpu()
print("==========> Setting Optimizer")
optimizerG = optim.Adam(filter(lambda p: p.requires_grad, netG.module.parameters() if opt.parallel else netG.parameters()), lr=opt.lr_g, betas = (0.5, 0.99))
#optimizerD = optim.Adam(filter(lambda p: p.requires_grad, netD.module.parameters() if opt.parallel else netD.parameters()), lr = opt.lr_d, betas = (0.5, 0.999))
optimizerD = optim.SGD(filter(lambda p: p.requires_grad, netD.module.parameters() if opt.parallel else netD.parameters()), lr = opt.lr_d)
lr_schedulerG = optim.lr_scheduler.CosineAnnealingLR(optimizerG, opt.epoch, eta_min = 1e-7)
lr_schedulerD = optim.lr_scheduler.CosineAnnealingLR(optimizerD, opt.epoch, eta_min = 1e-7)
print("==========> Training")
for epoch in range(opt.epoch + 1):
train(train_data_loader, netG, netD, optimizerG, optimizerD, epoch)
test(test_data_loader, netG)
if epoch % opt.save_model_freq == 0:
save_checkpoint(netG, epoch, name)
lr_schedulerG.step()
lr_schedulerD.step()
# plt.savefig('figs/' + speaker + '_' + basefile + '_mean.png')
# plt.close()
ult_all[speaker + '-' + basefile] = ult_data_3d_mean
# now all ult files are loaded
SSIM_all = np.zeros((len(ult_all), len(ult_all)))
# calculate SSIM utterance by utterance
ult_keys = list(sorted(ult_all.keys()))
for i in range(len(ult_all)):
for j in range(len(ult_all)):
print('calc SSIM', speaker, i, j, ' ', end='\r')
SSIM_all[i, j] = SSIM( \
Image.fromarray(ult_all[ult_keys[i]].reshape(NumVectors, PixPerVector), 'L'), \
gaussian_kernel_1d).ssim_value( \
Image.fromarray(ult_all[ult_keys[j]].reshape(NumVectors, PixPerVector), 'L'))
print('calc SSIM', speaker, 'done')
# serialize results to file
pickle.dump(SSIM_all,
open('measures/' + speaker + '_SSIM_avg.pkl', 'wb'))
else:
SSIM_all = pickle.load(
open('measures/' + speaker + '_SSIM_avg.pkl', 'rb'))
print(speaker, ', len:', len(SSIM_all), ', SSIM min/max: ',
np.min(SSIM_all), np.max(SSIM_all))
# visualize results
# vmax set according to manual checking