def evaluate_model(self, step):
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netE.eval()
self.netD.eval()
fakes, names = [], []
batch_id = 0
avg_psnr = 0
avg_ssim = 0
eval_dataloader = self.eval_dataloader
for i, data in enumerate(eval_dataloader):
# input
self.set_input(data)
self.test()
avg_psnr += self.psnr
avg_ssim += self.ssim
batch_id += 1
fakes.append(self.decolor_gray.cpu())
for j in range(len(self.image_paths)):
short_path = os.path.basename(self.image_paths[j])
name, _ = os.path.splitext(short_path)
names.append(name)
real_rgb = util.tensor2im(self.real_rgb[j])
decolor_gray = util.tensor2im(self.decolor_gray[j])
util.save_image(real_rgb, os.path.join(save_dir, 'real_rgb', '%s.png' % name), create_dir=True)
util.save_image(decolor_gray, os.path.join(save_dir, 'decolor_gray', '%s.png' % name), create_dir=True)
# metric
self.metric_psnr_eval = avg_psnr / batch_id
self.metric_ssim_eval = avg_ssim / batch_id
self.netE.train()
self.netD.train()
def save_images(webpage, visuals, image_path, opt):
def convert_visuals_to_numpy(visuals):
for key, t in visuals.items():
tile = opt.batch_size > 8
if key == 'labels':
t = util.tensor2label(t, opt.input_nc + 2, tile=tile)
else:
t = util.tensor2im(t, tile=tile)
visuals[key] = t
return visuals
visuals = convert_visuals_to_numpy(visuals)
image_dir = webpage.get_image_dir()
short_path = ntpath.basename(image_path[0])
name = os.path.splitext(short_path)[0]
webpage.add_header(name)
ims = []
txts = []
links = []
for label, image_numpy in visuals.items():
image_name = os.path.join(label, '%s.png' % (name))
save_path = os.path.join(image_dir, image_name)
util.save_image(image_numpy, save_path, create_dir=True)
ims.append(image_name)
txts.append(label)
links.append(image_name)
webpage.add_images(ims, txts, links, width=opt.display_winsize)
def save_images(webpage, visuals, image_path, aspect_ratio=1.0, width=256):
"""Save images to the disk.
Parameters:
webpage (the HTML class) -- the HTML webpage class that stores these imaegs (see myhtml.py for more details)
visuals (OrderedDict) -- an ordered dictionary that stores (name, images (either tensor or numpy) ) pairs
image_path (str) -- the string is used to create image paths
aspect_ratio (float) -- the aspect ratio of saved images
width (int) -- the images will be resized to width x width
This function will save images stored in 'visuals' to the HTML file specified by 'webpage'.
"""
image_dir = webpage.get_image_dir()
short_path = ntpath.basename(image_path[0])
name = os.path.splitext(short_path)[0]
webpage.add_header(name)
ims, txts, links = [], [], []
for label, im_data in visuals.items():
im = util.tensor2im(im_data)
image_name = '%s_%s.png' % (name, label)
save_path = os.path.join(image_dir, image_name)
h, w, _ = im.shape
if aspect_ratio > 1.0:
im = imresize(im, (h, int(w * aspect_ratio)), interp='bicubic')
if aspect_ratio < 1.0:
im = imresize(im, (int(h / aspect_ratio), w), interp='bicubic')
util.save_image(im, save_path)
ims.append(image_name)
txts.append(label)
links.append(image_name)
webpage.add_images(ims, txts, links, width=width)
def main(cfgs):
fluid.enable_imperative()
if 'resnet' in cfgs.netG:
from configs.resnet_configs import get_configs
else:
raise NotImplementedError
configs = get_configs(config_name=cfgs.config_set)
configs = list(configs.all_configs())
data_loader, id2name = create_eval_data(cfgs, direction=cfgs.direction)
model = TestModel(cfgs)
model.setup() ### load_network
### this input used in compute model flops and params
for data in data_loader:
model.set_input(data)
break
npz = np.load(cfgs.real_stat_path)
results = []
for config in configs:
fakes, names = [], []
flops, _ = model.profile(config=config)
s_time = time.time()
for i, data in enumerate(data_loader()):
model.set_input(data)
model.test(config)
generated = model.fake_B
fakes.append(generated.detach().numpy())
name = id2name[i]
save_path = os.path.join(cfgs.save_dir, 'test' + str(config))
if not os.path.exists(save_path):
os.makedirs(save_path)
save_path = os.path.join(save_path, name)
names.append(name)
if i < cfgs.num_test:
image = util.tensor2img(generated)
util.save_image(image, save_path)
result = {'config_str': encode_config(config), 'flops': flops} ### compute FLOPs
fluid.disable_imperative()
if not cfgs.no_fid:
block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[2048]
inception_model = InceptionV3([block_idx])
fid = get_fid(fakes, inception_model, npz, cfgs.inception_model_path, batch_size=cfgs.batch_size, use_gpu=cfgs.use_gpu)
result['fid'] = fid
fluid.enable_imperative()
e_time = (time.time() - s_time) / 60
result['time'] = e_time
print(result)
results.append(result)
if not os.path.exists(cfgs.save_dir):
os.makedirs(os.path.dirname(cfgs.save_dir))
save_file = os.path.join(cfgs.save_dir, 'search_result.pkl')
with open(save_file, 'wb') as f:
pickle.dump(results, f)
print('Successfully finish searching!!!')
def save_test(synthesized, file_path, frame_i):
im = util.tensor2im(synthesized.data)
elements = file_path.split('/')
ident = "-".join((elements[-3], elements[-2], elements[-1][:-4]))
img_path = os.path.join(opt.test_output_dir,
ident + "-" + str(frame_i) + ".png")
util.save_image(im, img_path, opt.final_output_size)
def evaluate_model(self, step, save_image=False):
ret = {}
self.is_best_A = False
self.is_best_B = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG_A.eval()
self.netG_B.eval()
for direction in ['AtoB', 'BtoA']:
eval_dataloader = getattr(self, 'eval_dataloader_' + direction)
fakes, names = [], []
cnt = 0
for i, data_i in enumerate(tqdm(eval_dataloader)):
self.set_single_input(data_i)
self.test_single_side(direction)
fakes.append(self.fake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10 or save_image:
input_im = util.tensor2im(self.real_A[j])
fake_im = util.tensor2im(self.fake_B[j])
util.save_image(input_im,
os.path.join(save_dir, direction,
'input', '%s.png' % name),
create_dir=True)
util.save_image(fake_im,
os.path.join(save_dir, direction,
'fake', '%s.png' % name),
create_dir=True)
cnt += 1
suffix = direction[-1]
fid = get_fid(fakes,
self.inception_model,
getattr(self, 'npz_%s' % direction[-1]),
device=self.device,
batch_size=self.opt.eval_batch_size)
if fid < getattr(self, 'best_fid_%s' % suffix):
setattr(self, 'is_best_%s' % direction[0], True)
setattr(self, 'best_fid_%s' % suffix, fid)
fids = getattr(self, 'fids_%s' % suffix)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % suffix] = fid
ret['metric/fid_%s-mean' %
suffix] = sum(getattr(self, 'fids_%s' % suffix)) / len(
getattr(self, 'fids_%s' % suffix))
ret['metric/fid_%s-best' % suffix] = getattr(
self, 'best_fid_%s' % suffix)
self.netG_A.train()
self.netG_B.train()
return ret
def save_images_reconst(self, webpage, visuals, step):
image_dir = webpage.get_image_dir()
save_paths = []
for label, image_numpy in visuals.items():
image_name = '%s_%s.png' % (label, step)
path1 = os.path.join(image_dir, 'save_images')
if not os.path.exists(path1):
os.makedirs(path1)
save_path = os.path.join(path1, image_name)
util.save_image(image_numpy, save_path)
save_paths.append(save_path)
return save_paths
def save_current_results(self, visuals, epoch, step):
for label, image_numpy in visuals.items():
if isinstance(image_numpy, list):
for i in range(len(image_numpy)):
img_path = self.get_img_path(epoch, step, label, i)
util.save_image(image_numpy[i], img_path)
else:
img_path = self.get_img_path(epoch, step, label)
util.save_image(image_numpy, img_path)
if self.opt.use_html:
self.save_html_log(visuals, epoch, step)
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.cfgs.save_dir, 'mobile', 'eval',
str(step))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
self.netG_A.eval()
self.netG_B.eval()
for direction in ['AtoB', 'BtoA']:
eval_dataloader = getattr(self, 'eval_dataloader_' + direction)
id2name = getattr(self, 'name_' + direction)
fakes = []
cnt = 0
for i, data_i in enumerate(eval_dataloader):
self.set_single_input(data_i)
self.test_single_side(direction)
fakes.append(self.fake_B.detach().numpy())
for j in range(len(self.fake_B)):
if cnt < 10:
name = 'fake_' + direction + str(i + j) + '.png'
save_path = os.path.join(save_dir, name)
fake_im = util.tensor2img(self.fake_B[j])
util.save_image(fake_im, save_path)
cnt += 1
suffix = direction[-1]
fluid.disable_imperative()
fid = get_fid(fakes, self.inception_model,
getattr(self, 'npz_%s' % direction[-1]),
self.cfgs.inception_model)
fluid.enable_imperative(place=self.cfgs.place)
if fid < getattr(self, 'best_fid_%s' % suffix):
self.is_best = True
setattr(self, 'best_fid_%s' % suffix, fid)
print("direction: %s, fid score is: %f, best fid score is %f" %
(direction, fid, getattr(self, 'best_fid_%s' % suffix)))
fids = getattr(self, 'fids_%s' % suffix)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % suffix] = fid
ret['metric/fid_%s-mean' %
suffix] = sum(getattr(self, 'fids_%s' % suffix)) / len(
getattr(self, 'fids_%s' % suffix))
ret['metric/fid_%s-best' % suffix] = getattr(
self, 'best_fid_%s' % suffix)
self.netG_A.train()
self.netG_B.train()
return ret
def evaluate_model(self, step):
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG_student.eval()
fakes, names = [], []
cnt = 0
for i, data_i in enumerate(tqdm(self.eval_dataloader)):
if self.opt.dataset_mode == 'aligned':
self.set_input(data_i)
else:
self.set_single_input(data_i)
self.test()
fakes.append(self.Sfake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2im(self.real_A[j])
Sfake_im = util.tensor2im(self.Sfake_B[j])
Tfake_im = util.tensor2im(self.Tfake_B[j])
util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png') % name, create_dir=True)
util.save_image(Sfake_im, os.path.join(save_dir, 'Sfake', '%s.png' % name), create_dir=True)
util.save_image(Tfake_im, os.path.join(save_dir, 'Tfake', '%s.png' % name), create_dir=True)
if self.opt.dataset_mode == 'aligned':
real_im = util.tensor2im(self.real_B[j])
util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True)
cnt += 1
fid = get_fid(fakes, self.inception_model, self.npz,
device=self.device, batch_size=self.opt.eval_batch_size)
if fid < self.best_fid:
self.is_best = True
self.best_fid = fid
self.fids.append(fid)
if len(self.fids) > 3:
self.fids.pop(0)
ret = {'metric/fid': fid, 'metric/fid-mean': sum(self.fids) / len(self.fids), 'metric/fid-best': self.best_fid}
if 'cityscapes' in self.opt.dataroot and self.opt.direction == 'BtoA':
mAP = get_mAP(fakes, names, self.drn_model, self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads)
if mAP > self.best_mAP:
self.is_best = True
self.best_mAP = mAP
self.mAPs.append(mAP)
if len(self.mAPs) > 3:
self.mAPs = self.mAPs[1:]
ret['metric/mAP'] = mAP
ret['metric/mAP-mean'] = sum(self.mAPs) / len(self.mAPs)
ret['metric/mAP-best'] = self.best_mAP
self.netG_student.train()
return ret
def main(cfgs):
fluid.enable_imperative()
if cfgs.config_str is not None:
assert 'super' in cfgs.netG or 'sub' in cfgs.netG
config = decode_config(cfgs.config_str)
else:
assert 'super' not in cfgs.model
config = None
data_loader, id2name = create_eval_data(cfgs, direction=cfgs.direction)
model = TestModel(cfgs)
model.setup() ### load_network
fakes, names = [], []
for i, data in enumerate(data_loader()):
model.set_input(data)
if i == 0 and cfgs.need_profile:
flops, params = model.profile(config)
print('FLOPs: %.3fG, params: %.3fM' % (flops / 1e9, params / 1e6))
sys.exit(0)
model.test(config)
generated = model.fake_B
fakes.append(generated.detach().numpy())
name = id2name[i]
print(name)
save_path = os.path.join(cfgs.save_dir, 'test')
if not os.path.exists(save_path):
os.makedirs(save_path)
save_path = os.path.join(save_path, name)
names.append(name)
if i < cfgs.num_test:
image = util.tensor2img(generated)
util.save_image(image, save_path)
fluid.disable_imperative()
if not cfgs.no_fid:
print('Calculating FID...')
block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[2048]
inception_model = InceptionV3([block_idx])
npz = np.load(cfgs.real_stat_path)
fid = get_fid(fakes, inception_model, npz, cfgs.inception_model_path)
print('fid score: %#.2f' % fid)
def test():
global T
tensor = get_tensor_from_file(
'/Users/eric/work/codes/py/toutiao/app/mydemos/gan-compression/database/horse2zebra/testA/n02381460_140.jpg',
transform)
if not tensor:
return
# 在这里对 tensor 进行测试并保存
url = tensor.get('A_url')
print(url)
model.set_input(tensor) # unpack data from data loader
if not T:
T = True
model.profile(config)
model.test(config) # run inference
visuals = model.get_current_visuals() # get image results
generated = visuals['fake_B'].cpu()
im = util.tensor2im(generated)
util.save_image(im, '/tmp/res.jpg')
def save_results(self, save_data, save_path, score=None, data_name='none'):
"""Save the training or testing results to disk"""
img_paths = self.get_image_paths()
for i in range(save_data.size(0)):
print('process image ...... %s' % img_paths[i])
short_path = ntpath.basename(img_paths[i]) # get image path
name = os.path.splitext(short_path)[0]
if type(score) == type(None):
img_name = '%s_%s.png' % (name, data_name)
else:
# d_score = score[i].mean()
# img_name = '%s_%s_%s.png' % (name, data_name, str(round(d_score.item(), 3)))
img_name = '%s_%s_%s.png' % (name, data_name, str(score))
# save predicted image with discriminator score
util.mkdir(save_path)
img_path = os.path.join(save_path, img_name)
img_numpy = util.tensor2im(save_data[i].data)
util.save_image(img_numpy, img_path)
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.cfgs.save_dir, 'distiller', 'eval',
str(step))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
self.netG_student.eval()
fakes = []
cnt = 0
for i, data_i in enumerate(self.eval_dataloader):
id2name = self.name
self.set_single_input(data_i)
self.test()
fakes.append(self.Sfake_B.detach().numpy())
for j in range(len(self.Sfake_B)):
if cnt < 10:
Sname = 'Sfake_' + str(i + j) + '.png'
Tname = 'Tfake_' + str(i + j) + '.png'
Sfake_im = util.tensor2img(self.Sfake_B[j])
Tfake_im = util.tensor2img(self.Tfake_B[j])
util.save_image(Sfake_im, os.path.join(save_dir, Sname))
util.save_image(Tfake_im, os.path.join(save_dir, Tname))
cnt += 1
suffix = self.cfgs.direction
fluid.disable_imperative()
fid = get_fid(fakes, self.inception_model, self.npz,
self.cfgs.inception_model)
fluid.enable_imperative(place=self.cfgs.place)
if fid < self.best_fid:
self.is_best = True
self.best_fid = fid
print("fid score is: %f, best fid score is %f" % (fid, self.best_fid))
self.fids.append(fid)
if len(self.fids) > 3:
self.fids.pop(0)
ret['metric/fid'] = fid
ret['metric/fid-mean'] = sum(self.fids) / len(self.fids)
ret['metric/fid-best'] = self.best_fid
self.netG_student.train()
return ret
def save_images(self, webpage, visuals, image_path):
image_dir = webpage.get_image_dir()
short_path = ntpath.basename(image_path[0])
name = os.path.splitext(short_path)[0]
webpage.add_header(name)
ims = []
txts = []
links = []
for label, image_numpy in visuals.items():
image_name = '%s_%s.jpg' % (name, label)
save_path = os.path.join(image_dir, image_name)
util.save_image(image_numpy, save_path)
ims.append(image_name)
txts.append(label)
links.append(image_name)
webpage.add_images(ims, txts, links, width=self.win_size)
def stylize(args):
device = torch.device("cuda" if args.cuda else "cpu")
content_image = load_image(args.content_image, size=args.content_size)
content_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Lambda(lambda x: x.mul(255))])
content_image = content_transform(content_image)
content_image = content_image.unsqueeze(0).to(device)
with torch.no_grad():
style_model = TransformerNet()
model = torch.load(args.model)
state_dict = model['tfm_state_dict']
for k in list(state_dict.keys()):
if re.search(r'in\d+\.running_(mean|var)$', k):
del state_dict[k]
style_model.load_state_dict(checkpoint['tfm_state_dict'])
style_model.to(device)
output = style_model.forward(content_image, weights=None).cpu()
save_image(args.output_image, output[0])
def save_images_test(self, webpage, visuals, image_path, sequence,
test_num):
image_dir = webpage.get_image_dir()
short_path = ntpath.basename(image_path[0])
name = os.path.splitext(short_path)[0]
webpage.add_header(name)
ims = []
txts = []
links = []
for label, image_numpy in visuals.items():
image_name = '%s_%s_%s.png' % (name, label, sequence)
path1 = os.path.join(image_dir, str(test_num))
if not os.path.exists(path1):
os.makedirs(path1)
save_path = os.path.join(path1, image_name)
util.save_image(image_numpy, save_path)
ims.append(image_name)
txts.append(label)
links.append(image_name)
webpage.add_images(ims, txts, links, width=self.win_size)
def stylize_image(config, device, args):
config = config['STYLIZE']
# Load image
input_image = util.load_image(args.img, scale=config['content_scale'])
# Transform input image
input_image = util.transform(input_image)
input_image = input_image.unsqueeze(0).to(device)
image_name = args.img.split("/")[-1][:-4]
net = TransformerNet().to(device)
if torch.cuda.device_count() > 1:
net = torch.nn.DataParallel(net)
for i in os.listdir(config['models_path']):
path = f"{config['models_path']}/{i}"
model_name = path.split("/")[-1][:-4]
net.load_state_dict(torch.load(path))
with torch.no_grad():
output = net(input_image).cpu()
output_image = f"{config['output_path']}/{image_name}_{model_name}.jpg"
# output_image = f"{config['output_path']}/{image_name}_{model_name}_{int(time.time())}.jpg"
util.save_image(output_image, output[0])
def save_images(webpage, visuals, image_path, aspect_ratio=1.0, width=256):
image_dir = webpage.get_image_dir()
short_path = ntpath.basename(image_path[0])
name = os.path.splitext(short_path)[0]
webpage.add_header(name)
ims, txts, links = [], [], []
for label, im_data in visuals.items():
im = im_data
image_name = '%s_%s.png' % (name, label)
save_path = os.path.join(image_dir, image_name)
h, w, _ = im.shape
if aspect_ratio > 1.0:
im = imresize(im, (h, int(w * aspect_ratio)), interp='bicubic')
if aspect_ratio < 1.0:
im = imresize(im, (int(h / aspect_ratio), w), interp='bicubic')
util.save_image(im, save_path)
ims.append(image_name)
txts.append(label)
links.append(image_name)
webpage.add_images(ims, txts, links, width=width)
def evaluate_model(self, step):
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.modules_on_one_gpu.netG.eval()
torch.cuda.empty_cache()
fakes, names = [], []
ret = {}
cnt = 0
for i, data_i in enumerate(tqdm(self.eval_dataloader, desc='Eval ', position=2, leave=False)):
self.set_input(data_i)
self.test()
fakes.append(self.fake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2label(self.input_semantics[j], self.opt.input_nc + 2)
real_im = util.tensor2im(self.real_B[j])
fake_im = util.tensor2im(self.fake_B[j])
util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png' % name), create_dir=True)
util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True)
util.save_image(fake_im, os.path.join(save_dir, 'fake', '%s.png' % name), create_dir=True)
cnt += 1
if not self.opt.no_fid:
fid = get_fid(fakes, self.inception_model, self.npz, device=self.device,
batch_size=self.opt.eval_batch_size, tqdm_position=2)
if fid < self.best_fid:
self.is_best = True
self.best_fid = fid
self.fids.append(fid)
if len(self.fids) > 3:
self.fids.pop(0)
ret['metric/fid'] = fid
ret['metric/fid-mean'] = sum(self.fids) / len(self.fids)
ret['metric/fid-best'] = self.best_fid
if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU:
mIoU = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads, tqdm_position=2)
if mIoU > self.best_mIoU:
self.is_best = True
self.best_mIoU = mIoU
self.mIoUs.append(mIoU)
if len(self.mIoUs) > 3:
self.mIoUs = self.mIoUs[1:]
ret['metric/mIoU'] = mIoU
ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs)
ret['metric/mIoU-best'] = self.best_mIoU
self.modules_on_one_gpu.netG.train()
torch.cuda.empty_cache()
return ret
def save_result(self):
"""Save the results to the disk"""
util.mkdir(self.opt.results_dir)
img_name = self.fname.split('/')[-1]
data_name = self.opt.img_file.split('/')[-1].split('.')[0]
# save the original image
original_name = '%s_%s_%s' % ('original', data_name, img_name)
original_path = os.path.join(self.opt.results_dir, original_name)
img_original = util.tensor2im(self.img_truth)
util.save_image(img_original, original_path)
# save the mask
mask_name = '%s_%s_%d_%s' % ('mask', data_name, self.PaintPanel.iteration, img_name)
mask_path = os.path.join(self.opt.results_dir, mask_name)
img_mask = util.tensor2im(self.img_m)
util.save_image(img_mask, mask_path)
# save the results
result_name = '%s_%s_%d_%s' % ('result', data_name, self.PaintPanel.iteration, img_name)
result_path = os.path.join(self.opt.results_dir, result_name)
img_result = util.tensor2im(self.img_out)
util.save_image(img_result, result_path)
def evaluate_model(self, step):
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG.eval()
fakes, names = [], []
cnt = 0
for i, data_i in enumerate(tqdm(self.eval_dataloader)):
self.set_input(data_i)
self.test()
fakes.append(self.fake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2im(self.real_A[j])
real_im = util.tensor2im(self.real_B[j])
fake_im = util.tensor2im(self.fake_B[j])
util.save_image(input_im,
os.path.join(save_dir, 'input',
'%s.png' % name),
create_dir=True)
util.save_image(real_im,
os.path.join(save_dir, 'real',
'%s.png' % name),
create_dir=True)
util.save_image(fake_im,
os.path.join(save_dir, 'fake',
'%s.png' % name),
create_dir=True)
cnt += 1
fid = get_fid(fakes,
self.inception_model,
self.npz,
device=self.device,
batch_size=self.opt.eval_batch_size)
if fid < self.best_fid:
self.is_best = True
self.best_fid = fid
self.fids.append(fid)
if len(self.fids) > 3:
self.fids.pop(0)
ret = {
'metric/fid': fid,
'metric/fid-mean': sum(self.fids) / len(self.fids),
'metric/fid-best': self.best_fid
}
if 'cityscapes' in self.opt.dataroot:
mIoU = get_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads)
if mIoU > self.best_mIoU:
self.is_best = True
self.best_mIoU = mIoU
self.mIoUs.append(mIoU)
if len(self.mIoUs) > 3:
self.mIoUs = self.mIoUs[1:]
ret['metric/mIoU'] = mIoU
ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs)
ret['metric/mIoU-best'] = self.best_mIoU
self.netG.train()
return ret
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
if self.opt.eval_mode == 'both':
settings = ('largest', 'smallest')
else:
settings = (self.opt.eval_mode, )
for config_name in settings:
config = self.configs(config_name)
fakes, names = [], []
self.modules_on_one_gpu.netG_student.train()
self.calibrate(config, 2)
tqdm_position = 2 + int(not self.opt.no_calibration)
self.modules_on_one_gpu.netG_student.eval()
torch.cuda.empty_cache()
cnt = 0
for i, data_i in enumerate(
tqdm(self.eval_dataloader,
desc='Eval ',
position=tqdm_position,
leave=False)):
self.set_input(data_i)
self.test(config=config)
fakes.append(self.Sfake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2label(self.input_semantics[j],
self.opt.input_nc + 2)
real_im = util.tensor2im(self.real_B[j])
Tfake_im = util.tensor2im(self.Tfake_B[j])
Sfake_im = util.tensor2im(self.Sfake_B[j])
util.save_image(input_im,
os.path.join(save_dir, 'input',
'%s.png' % name),
create_dir=True)
util.save_image(real_im,
os.path.join(save_dir, 'real',
'%s.png' % name),
create_dir=True)
util.save_image(Tfake_im,
os.path.join(save_dir, 'Tfake',
'%s.png' % name),
create_dir=True)
util.save_image(Sfake_im,
os.path.join(save_dir, 'Sfake',
'%s.png' % name),
create_dir=True)
cnt += 1
if not self.opt.no_fid:
fid = get_fid(fakes,
self.inception_model,
self.npz,
device=self.device,
batch_size=self.opt.eval_batch_size,
tqdm_position=2)
if fid < getattr(self, 'best_fid_%s' % config_name):
self.is_best = True
setattr(self, 'best_fid_%s' % config_name, fid)
fids = getattr(self, 'fids_%s' % config_name)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % config_name] = fid
ret['metric/fid_%s-mean' % config_name] = sum(
getattr(self, 'fids_%s' % config_name)) / len(
getattr(self, 'fids_%s' % config_name))
ret['metric/fid_%s-best' % config_name] = getattr(
self, 'best_fid_%s' % config_name)
if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU:
mIoU = get_cityscapes_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads,
tqdm_position=2)
if mIoU > getattr(self, 'best_mIoU_%s' % config_name):
self.is_best = True
setattr(self, 'best_mIoU_%s' % config_name, mIoU)
mIoUs = getattr(self, 'mIoUs_%s' % config_name)
mIoUs.append(mIoU)
if len(mIoUs) > 3:
mIoUs.pop(0)
ret['metric/mIoU_%s' % config_name] = mIoU
ret['metric/mIoU_%s-mean' % config_name] = sum(
getattr(self, 'mIoUs_%s' % config_name)) / len(
getattr(self, 'mIoUs_%s' % config_name))
ret['metric/mIoU_%s-best' % config_name] = getattr(
self, 'best_mIoU_%s' % config_name)
self.modules_on_one_gpu.netG_student.train()
torch.cuda.empty_cache()
return ret
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG_A.eval()
self.netG_B.eval()
for direction in ['AtoB', 'BtoA']:
eval_dataloader = getattr(self, 'eval_dataloader_' + direction)
fakes, names = [], []
cnt = 0
# print(len(eval_dataset))
for i, data_i in enumerate(tqdm(eval_dataloader)):
self.set_single_input(data_i)
self.test_single_side(direction)
# print(self.image_paths)
fakes.append(self.fake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2im(self.real_A[j])
fake_im = util.tensor2im(self.fake_B[j])
util.save_image(input_im,
os.path.join(save_dir, direction,
'input', '%s.png' % name),
create_dir=True)
util.save_image(fake_im,
os.path.join(save_dir, direction,
'fake', '%s.png' % name),
create_dir=True)
cnt += 1
suffix = direction[-1]
fid = get_fid(fakes,
self.inception_model,
getattr(self, 'npz_%s' % direction[-1]),
device=self.device,
batch_size=self.opt.eval_batch_size)
if fid < getattr(self, 'best_fid_%s' % suffix):
self.is_best = True
setattr(self, 'best_fid_%s' % suffix, fid)
fids = getattr(self, 'fids_%s' % suffix)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % suffix] = fid
ret['metric/fid_%s-mean' %
suffix] = sum(getattr(self, 'fids_%s' % suffix)) / len(
getattr(self, 'fids_%s' % suffix))
ret['metric/fid_%s-best' % suffix] = getattr(
self, 'best_fid_%s' % suffix)
if 'cityscapes' in self.opt.dataroot and direction == 'BtoA':
mIoU = get_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads)
if mIoU > self.best_mIoU:
self.is_best = True
self.best_mIoU = mIoU
self.mIoUs.append(mIoU)
if len(self.mIoUs) > 3:
self.mIoUs = self.mIoUs[1:]
ret['metric/mIoU'] = mIoU
ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs)
ret['metric/mIoU-best'] = self.best_mIoU
self.netG_A.train()
self.netG_B.train()
return ret
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG_student.eval()
if self.opt.eval_mode == 'both':
setting = ('largest', 'smallest')
else:
setting = (self.opt.eval_mode, )
for config_name in setting:
config = self.configs(config_name)
fakes, names = [], []
cnt = 0
for i, data_i in enumerate(tqdm(self.eval_dataloader)):
if self.opt.dataset_mode == 'aligned':
self.set_input(data_i)
else:
self.set_single_input(data_i)
self.test(config)
fakes.append(self.Sfake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if i < 10:
Sfake_im = util.tensor2im(self.Sfake_B[j])
real_im = util.tensor2im(self.real_B[j])
Tfake_im = util.tensor2im(self.Tfake_B[j])
util.save_image(real_im,
os.path.join(save_dir, 'real',
'%s.png' % name),
create_dir=True)
util.save_image(Sfake_im,
os.path.join(save_dir,
'Sfake_%s' % config_name,
'%s.png' % name),
create_dir=True)
util.save_image(Tfake_im,
os.path.join(save_dir, 'Tfake',
'%s.png' % name),
create_dir=True)
if self.opt.dataset_mode == 'aligned':
input_im = util.tensor2im(self.real_A[j])
util.save_image(
input_im,
os.path.join(save_dir, 'input', '%s.png') %
name,
create_dir=True)
cnt += 1
fid = get_fid(fakes,
self.inception_model,
self.npz,
device=self.device,
batch_size=self.opt.eval_batch_size)
if fid < getattr(self, 'best_fid_%s' % config_name):
self.is_best = True
setattr(self, 'best_fid_%s' % config_name, fid)
fids = getattr(self, 'fids_%s' % config_name)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % config_name] = fid
ret['metric/fid_%s-mean' % config_name] = sum(
getattr(self, 'fids_%s' % config_name)) / len(
getattr(self, 'fids_%s' % config_name))
ret['metric/fid_%s-best' % config_name] = getattr(
self, 'best_fid_%s' % config_name)
if 'cityscapes' in self.opt.dataroot:
mIoU = get_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads)
if mIoU > getattr(self, 'best_mIoU_%s' % config_name):
self.is_best = True
setattr(self, 'best_mIoU_%s' % config_name, mIoU)
mIoUs = getattr(self, 'mIoUs_%s' % config_name)
mIoUs.append(mIoU)
if len(mIoUs) > 3:
mIoUs.pop(0)
ret['metric/mIoU_%s' % config_name] = mIoU
ret['metric/mIoU_%s-mean' % config_name] = sum(
getattr(self, 'mIoUs_%s' % config_name)) / len(
getattr(self, 'mIoUs_%s' % config_name))
ret['metric/mIoU_%s-best' % config_name] = getattr(
self, 'best_mIoU_%s' % config_name)
self.netG_student.train()
return ret
def display_current_results(self, visuals, epoch, step):
if self.tf_log: # show images in tensorboard output
img_summaries = []
for label, image_numpy in visuals.items():
# Write the image to a string
try:
s = StringIO()
except:
s = BytesIO()
scipy.misc.toimage(image_numpy).save(s, format="jpeg")
# Create an Image object
img_sum = self.tf.Summary.Image(
encoded_image_string=s.getvalue(),
height=image_numpy.shape[0],
width=image_numpy.shape[1])
# Create a Summary value
img_summaries.append(
self.tf.Summary.Value(tag=label, image=img_sum))
# Create and write Summary
summary = self.tf.Summary(value=img_summaries)
self.writer.add_summary(summary, step)
if self.use_html: # save images to a html file
for label, image_numpy in visuals.items():
img_path = os.path.join(self.img_dir,
'epoch%.3d_%s.jpg' % (epoch, label))
# print(label)
if label == 'left_disp' or label == 'right_disp' or label == 'left_disp_ref' or label == 'right_disp_ref' or label == 'disp_out':
util.save_image_(image_numpy, img_path)
elif label == 'edge_map':
util.save_image__(image_numpy, img_path)
else:
util.save_image(image_numpy, img_path)
# update website
if self.opt.isTrain:
webpage = html.HTML(self.web_dir,
'Experiment name = %s' % self.name,
refresh=5)
for n in range(epoch, 0, -1):
webpage.add_header(
'Epoch [%d] steps [%d] updated at [%s]' %
(n, step, time.ctime()))
ims = []
txts = []
links = []
for label, image_numpy in visuals.items():
img_path = 'epoch%.3d_%s.jpg' % (n, label)
ims.append(img_path)
txts.append(label)
links.append(img_path)
if len(ims) < 3:
webpage.add_images(ims,
txts,
links,
width=self.win_size)
else:
num = len(ims) / 2
j = 0
for i in range(num):
start = j
end = j + 2
webpage.add_images(ims[start:end],
txts[start:end],
links[start:end],
width=self.win_size)
j += 2
webpage.save()
def generate_pos_neg_diff(self):
"""
Decide positive and negative responsibility and generate ground truth for box regression
Anchors which has an responsibility for ground truth will be optimised to fit ground truth
"""
gt_box_in_detection = self.ret['target_in_resized_detection_xywh']
pos, neg = self.anchor.pos_neg_anchor(gt_box_in_detection)
diff = self.anchor.diff_anchor_gt(gt_box_in_detection)
pos, neg, diff = pos.reshape((-1, 1)), neg.reshape(
(-1, 1)), diff.reshape((-1, 4))
class_target = np.zeros(
(self.anchor.gen_anchors().shape[0], 2)).astype(np.float32)
# positive anchor
pos_index = np.where(pos == 1)[0]
pos_num = len(pos_index)
print(pos_num)
if pos_num > 0:
class_target[pos_index] = [1., 0.]
# negative anchor
neg_index = np.where(neg == 1)[0]
neg_num = len(neg_index)
class_target[neg_index] = [0., 1.]
# draw pos and neg anchor box
if self.debug:
s = os.path.join(self.debug_dir, '4_check_pos_neg_anchors')
os.makedirs(s, exist_ok=True)
debug_img = self.ret['detection_cropped_resized'].copy()
for i in range(pos_num):
index = pos_index[i]
cx, cy, w, h = self.anchor.gen_anchors()[index]
x1, y1, x2, y2 = int(cx -
w / 2), int(cy -
h / 2), int(cx +
w / 2), int(cy +
h / 2)
cv2.rectangle(debug_img, (x1, y1), (x2, y2), (255, 0, 0), 1)
for i in range(neg_num):
index = neg_index[i]
cx, cy, w, h = self.anchor.gen_anchors()[index]
x1, y1, x2, y2 = int(cx -
w / 2), int(cy -
h / 2), int(cx +
w / 2), int(cy +
h / 2)
cv2.rectangle(debug_img, (x1, y1), (x2, y2), (0, 255, 0), 1)
save_path = os.path.join(s, '{}.jpg'.format('pos_neg_anchor'))
save_image(debug_img, save_path)
if self.debug:
s = os.path.join(self.debug_dir, '5_check_all_anchors')
os.makedirs(s, exist_ok=True)
x1, y1, x2, y2 = self.ret['target_in_resized_detection_x1y1x2y2']
debug_img = self.ret['detection_cropped_resized'].copy()
cv2.rectangle(debug_img, (x1, y1), (x2, y2), (255, 0, 0), 1)
for i in range(self.anchor.gen_anchors().shape[0]):
cx, cy, w, h = self.anchor.gen_anchors()[i]
x1, y1, x2, y2 = int(cx -
w / 2), int(cy -
h / 2), int(cx +
w / 2), int(cy +
h / 2)
cv2.rectangle(debug_img, (x1, y1), (x2, y2), (0, 255, 0), 1)
save_path = os.path.join(s,
'{}.jpg'.format('all_anchors_and_target'))
save_image(debug_img, save_path)
pos_neg_diff = np.hstack((class_target, diff))
self.ret['pos_neg_diff'] = pos_neg_diff
return pos_neg_diff
def get_img_pairs(self, idx):
"""
Get template and detection image pair
max_inter is max interval between template and detection images
"""
if idx >= len(self.sub_class_dir):
raise ValueError('idx should be less than length of sub_class_dir')
frames_basename = self.sub_class_dir[idx]
frames_dir_path = os.path.join(self.img_dir_path, frames_basename)
frames_filename = [
img_name for img_name in os.listdir(frames_dir_path)
if not img_name.find('.jpg') == -1
]
frames_filename = sorted(frames_filename)
frames_num = len(frames_filename)
gt_name = 'groundtruth.txt'
status = True
while status:
if self.max_inter >= frames_num - 1:
self.max_inter = frames_num // 2
template_index = np.clip(
random.choice(range(0, max(1, frames_num - self.max_inter))),
0, frames_num - 1)
detection_index = np.clip(
random.choice(range(1, max(2, self.max_inter))) +
template_index, 0, frames_num - 1)
template_name, detection_name = frames_filename[
template_index], frames_filename[detection_index]
template_img_path, detection_img_path = os.path.join(
frames_dir_path,
template_name), os.path.join(frames_dir_path, detection_name)
gt_path = os.path.join(frames_dir_path, gt_name)
with open(gt_path, 'r') as f:
lines = f.readlines()
cords_of_template_abs = [
abs(int(float(i)))
for i in lines[template_index].strip('\n').split(',')[:4]
]
cords_of_detection_abs = [
abs(int(float(i)))
for i in lines[detection_index].strip('\n').split(',')[:4]
]
if cords_of_template_abs[2] * cords_of_template_abs[
3] * cords_of_detection_abs[2] * cords_of_detection_abs[
3] != 0:
status = False
else:
print('Warning : Encounter object missing, reinitializing ...')
# Save informations of template and detection
self.ret['template_img_path'] = template_img_path
self.ret['detection_img_path'] = detection_img_path
self.ret['template_target_x1y1wh'] = [
int(float(i)) for i in lines[template_index].strip('\n').split(',')
]
self.ret['detection_target_x1y1wh'] = [
int(float(i))
for i in lines[detection_index].strip('\n').split(',')
]
t1, t2 = self.ret['template_target_x1y1wh'], self.ret[
'detection_target_x1y1wh']
# Coordinate transformation (left, top, width, height) => (center_x, center_y, width, height )
self.ret['template_target_xywh'] = np.array(
[t1[0] + t1[2] // 2, t1[1] + t1[3] // 2, t1[2], t1[3]])
self.ret['detection_target_xywh'] = np.array(
[t2[0] + t2[2] // 2, t2[1] + t2[3] // 2, t2[2], t2[3]])
if self.debug:
s = os.path.join(self.debug_dir, '0_check_bbox_groundtruth')
if not os.path.exists(s):
os.makedirs(s)
debug_image = load_image(self.ret['template_img_path'])
x, y, w, h = self.ret['template_target_xywh']
x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
cv2.rectangle(debug_image, (x1, y1), (x2, y2), (255, 0, 0), 1)
save_path = os.path.join(
s, '{}.jpg'.format('template_image_with_target'))
save_image(debug_image, save_path)
debug_image = load_image(self.ret['detection_img_path'])
x, y, w, h = self.ret['detection_target_xywh']
x1, y1, x2, y2 = x - w // 2, y - h // 2, x + w // 2, y + h // 2
cv2.rectangle(debug_image, (x1, y1), (x2, y2), (255, 0, 0), 1)
save_path = os.path.join(
s, '{}.jpg'.format('detection_image_with_target'))
save_image(debug_image, save_path)
def pad_crop_resize(self):
"""
Insert padding to make square image and crop to feed in network with fixed size
Template images are cropped into 127 x 127
Detection images are cropped into 255 x 255
Images are padded with channel mean
"""
template_img = load_image(self.ret['template_img_path'])
detection_img = load_image(self.ret['detection_img_path'])
template_origin_height, template_origin_width, _ = template_img.shape
detection_origin_height, detection_origin_width, _ = detection_img.shape
template_target_cx, template_target_cy, template_target_w, template_target_h = self.ret[
'template_target_xywh']
detecion_target_cx, detecion_target_cy, detecion_target_w, detecion_target_h = self.ret[
'detection_target_xywh']
p = (template_target_w + template_target_h) // 2
template_square_size = int(
np.sqrt((template_target_w + p) * (template_target_h + p)))
detection_square_size = int(template_square_size * 2)
# lt means left top point, rb means right bottom point
template_target_left = template_target_cx - template_square_size // 2
template_target_top = template_target_cy - template_square_size // 2
template_target_right = template_target_cx + template_square_size // 2
template_target_bottom = template_target_cy + template_square_size // 2
detection_target_left = detecion_target_cx - detection_square_size // 2
detection_target_top = detecion_target_cy - detection_square_size // 2
detection_target_right = detecion_target_cx + detection_square_size // 2
detection_target_bottom = detecion_target_cy + detection_square_size // 2
# calculate template padding
template_left_padding = -template_target_left if template_target_left < 0 else 0
template_top_padding = -template_target_top if template_target_top < 0 else 0
template_right_padding = template_target_right - template_origin_width \
if template_target_right > template_origin_width else 0
template_bottom_padding = template_target_bottom - template_origin_height \
if template_target_bottom > template_origin_height else 0
new_template_width = template_left_padding + template_origin_width + template_right_padding
new_template_height = template_top_padding + template_origin_height + template_bottom_padding
# calculate detection padding
detection_left_padding = -detection_target_left if detection_target_left < 0 else 0
detection_top_padding = -detection_target_top if detection_target_top < 0 else 0
detection_right_padding = detection_target_right - detection_origin_width \
if detection_target_right > detection_origin_width else 0
detection_bottom_padding = detection_target_bottom - detection_origin_height \
if detection_target_bottom > detection_origin_height else 0
new_detection_width = detection_left_padding + detection_origin_width + detection_right_padding
new_detection_height = detection_top_padding + detection_origin_height + detection_bottom_padding
if any([
detection_left_padding, detection_top_padding,
detection_right_padding, detection_bottom_padding
]):
img_mean = tuple(map(int, detection_img.mean(axis=(0, 1))))
detection_with_padding = np.zeros(
(new_detection_height, new_detection_width, 3))
detection_with_padding[
detection_top_padding:detection_top_padding +
detection_origin_height,
detection_left_padding:detection_left_padding +
detection_origin_width, :] = detection_img
if detection_top_padding:
detection_with_padding[
0:detection_top_padding,
detection_left_padding:detection_left_padding +
detection_origin_width, :] = img_mean
if detection_bottom_padding:
detection_with_padding[
detection_origin_height + detection_top_padding:,
detection_left_padding:detection_left_padding +
detection_origin_width, :] = img_mean
if detection_left_padding:
detection_with_padding[:,
0:detection_left_padding, :] = img_mean
if detection_right_padding:
detection_with_padding[:, detection_origin_width +
detection_left_padding:, :] = img_mean
self.ret['new_detection_img_padding'] = detection_with_padding
else:
self.ret['new_detection_img_padding'] = detection_img
if any([
template_left_padding, template_top_padding,
template_right_padding, template_bottom_padding
]):
img_mean = tuple(map(int, template_img.mean(axis=(0, 1))))
template_with_padding = np.zeros(
(new_template_height, new_template_width, 3), np.uint8)
template_with_padding[template_top_padding:template_top_padding +
template_origin_height,
template_left_padding:template_left_padding +
template_origin_width, :] = template_img
if template_top_padding:
template_with_padding[
0:template_top_padding,
template_left_padding:template_left_padding +
template_origin_width, :] = img_mean
if template_bottom_padding:
template_with_padding[
template_origin_height + template_top_padding:,
template_left_padding:template_left_padding +
template_origin_width, :] = img_mean
if template_left_padding:
template_with_padding[:, 0:template_left_padding, :] = img_mean
if template_right_padding:
template_with_padding[:, template_origin_width +
template_left_padding:, :] = img_mean
self.ret['new_template_img_padding'] = template_with_padding
else:
self.ret['new_template_img_padding'] = template_img
# crop
tl = int(template_target_cx + template_left_padding -
template_square_size // 2)
tt = int(template_target_cy + template_top_padding -
template_square_size // 2)
self.ret['template_cropped'] = self.ret['new_template_img_padding'][
tt:tt + template_square_size, tl:tl + template_square_size, :]
dl = int(detecion_target_cx + detection_left_padding -
detection_square_size // 2)
dt = int(detecion_target_cy + detection_top_padding -
detection_square_size // 2)
self.ret['detection_cropped'] = self.ret['new_detection_img_padding'][
dt:dt + detection_square_size, dl:dl + detection_square_size, :]
self.ret['detection_tlcords_of_padding_image'] = (
detecion_target_cx - detection_square_size // 2 +
detection_left_padding, detecion_target_cy -
detection_square_size // 2 + detection_top_padding)
self.ret['detection_rbcords_of_padding_image'] = (
detecion_target_cx + detection_square_size // 2 +
detection_left_padding, detecion_target_cy +
detection_square_size // 2 + detection_top_padding)
# resize
self.ret['template_cropped_resized'] = cv2.resize(
self.ret['template_cropped'], (127, 127))
self.ret['detection_cropped_resized'] = cv2.resize(
self.ret['detection_cropped'], (255, 255))
self.ret['detection_cropped_resized_ratio'] = round(
255. / detection_square_size, 2)
self.ret['target_tlcords_of_padding_image'] = np.array([
detecion_target_cx + detection_left_padding -
detecion_target_w // 2,
detecion_target_cy + detection_top_padding - detecion_target_h // 2
])
self.ret['target_rbcords_of_padding_image'] = np.array([
detecion_target_cx + detection_left_padding +
detecion_target_w // 2,
detecion_target_cy + detection_top_padding + detecion_target_h // 2
])
if self.debug:
s = os.path.join(self.debug_dir,
'1_check_detection_target_in_padding')
os.makedirs(s, exist_ok=True)
debug_image = self.ret['new_detection_img_padding']
x1, y1 = self.ret['target_tlcords_of_padding_image']
x2, y2 = self.ret['target_rbcords_of_padding_image']
cv2.rectangle(debug_image, (x1, y1), (x2, y2), (255, 0, 0), 1)
x1, y1 = self.ret['detection_tlcords_of_padding_image']
x2, y2 = self.ret['detection_rbcords_of_padding_image']
cv2.rectangle(debug_image, (x1, y1), (x2, y2), (0, 255, 0), 1)
save_path = os.path.join(
s,
'{}.jpg'.format('target_detection_area_in_padding_detection'))
save_image(debug_image, save_path)
x11, y11 = self.ret['detection_tlcords_of_padding_image']
x12, y12 = self.ret['detection_rbcords_of_padding_image']
x21, y21 = self.ret['target_tlcords_of_padding_image']
x22, y22 = self.ret['target_rbcords_of_padding_image']
# Caculate target's relative coordinate with respect to padded detection image
x1_of_d, y1_of_d, x3_of_d, y3_of_d = x21 - x11, y21 - y11, x22 - x11, y22 - y11
x1 = np.clip(x1_of_d, 0, x12 - x11)
y1 = np.clip(y1_of_d, 0, y12 - y11)
x2 = np.clip(x3_of_d, 0, x12 - x11)
y2 = np.clip(y3_of_d, 0, y12 - y11)
if self.debug:
s = os.path.join(self.debug_dir,
'2_check_target_in_cropped_detection')
os.makedirs(s, exist_ok=True)
debug_image = self.ret['detection_cropped'].copy()
cv2.rectangle(debug_image, (x1, y1), (x2, y2), (255, 0, 0), 1)
save_path = os.path.join(
s, '{}.jpg'.format('target_in_cropped_detection'))
save_image(debug_image, save_path)
cords_in_cropped_detection = np.array([x1, y1, x2, y2])
cords_in_cropped_resized_detection = (
cords_in_cropped_detection *
self.ret['detection_cropped_resized_ratio']).astype(np.int32)
x1, y1, x2, y2 = cords_in_cropped_resized_detection
cx, cy, w, h = (x1 + x2) // 2, (y1 + y2) // 2, x2 - x1, y2 - y1
self.ret[
'target_in_resized_detection_x1y1x2y2'] = cords_in_cropped_resized_detection
self.ret['target_in_resized_detection_xywh'] = np.array([cx, cy, w, h])
self.ret['area_target_in_resized_detection'] = w * h
if self.debug:
s = os.path.join(self.debug_dir,
'3_check_target_in_cropped_resized_detection')
os.makedirs(s, exist_ok=True)
debug_image = self.ret['detection_cropped_resized'].copy()
cv2.rectangle(debug_image, (x1, y1), (x2, y2), (255, 0, 0), 1)
save_path = os.path.join(
s, '{}.jpg'.format('target_in_croppedd_resized_detection'))
save_image(debug_image, save_path)
def main(config, saved_folder=None):
logger = config.get_logger('predict')
# setup data_loader instances
if 'KittiLoader' in config['data_loader']['type']:
init_kwags = {
"kitti_depth_dir":
config['data_loader']['args']['kitti_depth_dir'],
"kitti_raw_dir": config['data_loader']['args']['kitti_raw_dir'],
# "root_dir": config['data_loader']['args']['root_dir'],
"batch_size": 1,
"shuffle": False,
"img_size": config['val_img_size'],
"num_workers": config['data_loader']['args']['num_workers'],
"mode": "val",
"scale_factor": config['data_loader']['args']['scale_factor'],
"seq_size": config['data_loader']['args']['seq_size'],
"cam_ids": config['data_loader']['args']['cam_ids']
}
data_loader = getattr(module_data,
config['data_loader']['type'])(**init_kwags)
else:
init_kwags = {
"root_dir": config['data_loader']['args']['root_dir'],
"batch_size": 1,
"shuffle": False,
"img_size": config['val_img_size'],
"num_workers": config['data_loader']['args']['num_workers'],
"mode": "test",
"scale_factor": config['data_loader']['args']['scale_factor'],
"seq_size": config['data_loader']['args']['seq_size'],
"img_resize": config['data_loader']['args']['img_resize']
}
data_loader = getattr(module_data,
config['data_loader']['type'])(**init_kwags)
# build models architecture
model = config.init_obj('arch', module_arch)
logger.info(model)
# get function handles of loss and metrics
loss_fn = getattr(module_loss, config['loss'])
metric_fns = [getattr(module_metric, met) for met in config['metrics']]
name_metrics = list()
for m in metric_fns:
if m.__name__ != 'deltas':
name_metrics.append(m.__name__)
else:
for i in range(1, 4):
name_metrics.append("delta_%d" % i)
total_metrics = MetricTracker('loss', *name_metrics)
logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(str(config.resume))
state_dict = checkpoint['state_dict']
new_state_dict = {}
for key, val in state_dict.items():
new_state_dict[key.replace('module.', '')] = val
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(new_state_dict)
# prepare models for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
itg_state = None
if saved_folder is not None:
path_idepth = os.path.join(saved_folder, 'final_depth_maps')
if not os.path.exists(path_idepth):
os.makedirs(path_idepth)
path_icfd = os.path.join(saved_folder, 'final_cfd_maps')
if not os.path.exists(path_icfd):
os.makedirs(path_icfd)
with torch.no_grad():
for batch_idx, data in enumerate(data_loader):
data = to_device(tuple(data), device)
print(len(data))
imgs, sdmaps, E, K, scale, is_begin_video = data
is_begin_video = is_begin_video.type(torch.uint8)
if itg_state is None:
init_depth = torch.zeros(sdmaps.size(), dtype=torch.float32)
init_cfd = torch.zeros(sdmaps.size(), dtype=torch.float32)
itg_state = init_depth, init_cfd
itg_state = to_device(itg_state, device)
prev_E = E
else:
if config['trainer']['seq']:
itg_state[0][is_begin_video] = 0.
itg_state[1][is_begin_video] = 0.
prev_E[is_begin_video] = E[is_begin_video]
else:
itg_state[0].zero_()
itg_state[1].zero_()
warped_depth, warped_cfd = h**o.warping(itg_state[0], itg_state[1],
K, prev_E, K, E)
warped_depth *= scale.view(-1, 1, 1, 1)
prev_E = E
final_depth, final_cfd, _, _ = model(
(imgs, sdmaps), prev_state=(warped_depth, warped_cfd))
d = final_depth.detach()
c = final_cfd.detach()
iscale = 1. / scale.view(-1, 1, 1, 1)
itg_state = (d * iscale, c)
if config['data_loader']['type'] == 'KittiLoaderv2':
name, id_data = data_loader.kitti_dataset.generate_img_index[
batch_idx]
video_data = data_loader.kitti_dataset.all_paths[name]
elif config['data_loader']['type'] == 'VISIMLoader':
name, id_data = data_loader.visim_dataset.generate_img_index[
batch_idx]
video_data = data_loader.visim_dataset.all_paths[name]
elif config['data_loader']['type'] == 'SevenSceneLoader':
name, id_data = data_loader.scene7_dataset.generate_img_index[
batch_idx]
video_data = data_loader.scene7_dataset.all_paths[name]
img_path = video_data['img_paths'][id_data]
if config['data_loader']['type'] == 'KittiLoaderv2':
id_img = img_path.split('/')[-1].split('.')[0]
elif config['data_loader']['type'] == 'VISIMLoader':
id_img = img_path.split('/')[-1].split('.')[0][5:]
elif config['data_loader']['type'] == 'SevenSceneLoader':
id_img = img_path.split('/')[-1].split('.')[0].split('-')[-1]
if saved_folder is not None:
if (batch_idx + 1) % 1 == 0:
final_depth = itg_state[0].squeeze(0).squeeze(
0).cpu().numpy() * 100
if config['data_loader']['type'] == 'KittiLoaderv2':
subfolder_idepth = os.path.join(
path_idepth, '_'.join(name.split('_')))
elif config['data_loader']['type'] == 'VISIMLoader':
subfolder_idepth = os.path.join(
path_idepth,
name.split('_')[0])
elif config['data_loader']['type'] == 'SevenSceneLoader':
subfolder_idepth = os.path.join(
path_idepth, '/'.join(name.split('_')))
if not os.path.exists(subfolder_idepth):
os.makedirs(subfolder_idepth)
util.save_image(subfolder_idepth,
'%s.png' % id_img,
final_depth.astype(np.uint16),
saver='opencv')
c_new = c.squeeze(0).squeeze(0).cpu().numpy() * 255
if config['data_loader']['type'] == 'KittiLoaderv2':
subfolder_cfd = os.path.join(path_icfd,
'_'.join(name.split('_')))
elif config['data_loader']['type'] == 'VISIMLoader':
subfolder_cfd = os.path.join(path_icfd,
name.split('_')[0])
elif config['data_loader']['type'] == 'SevenSceneLoader':
subfolder_cfd = os.path.join(path_icfd,
'/'.join(name.split('_')))
if not os.path.exists(subfolder_cfd):
os.makedirs(subfolder_cfd)
util.save_image(subfolder_cfd,
'%s.png' % id_img,
c_new.astype(np.uint8),
saver='opencv')
