def evaluate(self, child_pool):
results = []
for child in tqdm(child_pool, position=1, desc='Evaluate ', leave=False):
result = self.result_cache.get(encode_config(child))
if result is None:
result = {}
fakes, names = [], []
if isinstance(self.model, SPADEModel):
self.model.calibrate(child)
for i, data_i in enumerate(self.dataloader):
self.model.set_input(data_i)
self.model.test(child)
fakes.append(self.model.fake_B.cpu())
for path in self.model.get_image_paths():
short_path = ntpath.basename(path)
name = os.path.splitext(short_path)[0]
names.append(name)
if self.inception_model is not None:
result['fid'] = get_fid(fakes, self.inception_model, self.npz,
self.device, opt.batch_size, tqdm_position=2)
if self.drn_model is not None:
result['mIoU'] = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device,
data_dir=opt.cityscapes_path, batch_size=opt.batch_size,
num_workers=opt.num_threads, tqdm_position=2)
if self.deeplabv2_model is not None:
torch.cuda.empty_cache()
result['accu'], result['mIoU'] = get_coco_scores(fakes, names, self.deeplabv2_model, self.device,
opt.dataroot, 1, num_workers=0, tqdm_position=2)
if len(self.result_cache) < self.opt.max_cache_size:
self.result_cache[encode_config(child)] = result
results.append(result)
return results
def evaluate_model(self, step):
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.modules_on_one_gpu.netG.eval()
torch.cuda.empty_cache()
fakes, names = [], []
ret = {}
cnt = 0
for i, data_i in enumerate(tqdm(self.eval_dataloader, desc='Eval ', position=2, leave=False)):
self.set_input(data_i)
self.test()
fakes.append(self.fake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2label(self.input_semantics[j], self.opt.input_nc + 2)
real_im = util.tensor2im(self.real_B[j])
fake_im = util.tensor2im(self.fake_B[j])
util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png' % name), create_dir=True)
util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True)
util.save_image(fake_im, os.path.join(save_dir, 'fake', '%s.png' % name), create_dir=True)
cnt += 1
if not self.opt.no_fid:
fid = get_fid(fakes, self.inception_model, self.npz, device=self.device,
batch_size=self.opt.eval_batch_size, tqdm_position=2)
if fid < self.best_fid:
self.is_best = True
self.best_fid = fid
self.fids.append(fid)
if len(self.fids) > 3:
self.fids.pop(0)
ret['metric/fid'] = fid
ret['metric/fid-mean'] = sum(self.fids) / len(self.fids)
ret['metric/fid-best'] = self.best_fid
if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU:
mIoU = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads, tqdm_position=2)
if mIoU > self.best_mIoU:
self.is_best = True
self.best_mIoU = mIoU
self.mIoUs.append(mIoU)
if len(self.mIoUs) > 3:
self.mIoUs = self.mIoUs[1:]
ret['metric/mIoU'] = mIoU
ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs)
ret['metric/mIoU-best'] = self.best_mIoU
self.modules_on_one_gpu.netG.train()
torch.cuda.empty_cache()
return ret
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
if self.opt.eval_mode == 'both':
settings = ('largest', 'smallest')
else:
settings = (self.opt.eval_mode, )
for config_name in settings:
config = self.configs(config_name)
fakes, names = [], []
self.modules_on_one_gpu.netG_student.train()
self.calibrate(config, 2)
tqdm_position = 2 + int(not self.opt.no_calibration)
self.modules_on_one_gpu.netG_student.eval()
torch.cuda.empty_cache()
cnt = 0
for i, data_i in enumerate(
tqdm(self.eval_dataloader,
desc='Eval ',
position=tqdm_position,
leave=False)):
self.set_input(data_i)
self.test(config=config)
fakes.append(self.Sfake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2label(self.input_semantics[j],
self.opt.input_nc + 2)
real_im = util.tensor2im(self.real_B[j])
Tfake_im = util.tensor2im(self.Tfake_B[j])
Sfake_im = util.tensor2im(self.Sfake_B[j])
util.save_image(input_im,
os.path.join(save_dir, 'input',
'%s.png' % name),
create_dir=True)
util.save_image(real_im,
os.path.join(save_dir, 'real',
'%s.png' % name),
create_dir=True)
util.save_image(Tfake_im,
os.path.join(save_dir, 'Tfake',
'%s.png' % name),
create_dir=True)
util.save_image(Sfake_im,
os.path.join(save_dir, 'Sfake',
'%s.png' % name),
create_dir=True)
cnt += 1
if not self.opt.no_fid:
fid = get_fid(fakes,
self.inception_model,
self.npz,
device=self.device,
batch_size=self.opt.eval_batch_size,
tqdm_position=2)
if fid < getattr(self, 'best_fid_%s' % config_name):
self.is_best = True
setattr(self, 'best_fid_%s' % config_name, fid)
fids = getattr(self, 'fids_%s' % config_name)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % config_name] = fid
ret['metric/fid_%s-mean' % config_name] = sum(
getattr(self, 'fids_%s' % config_name)) / len(
getattr(self, 'fids_%s' % config_name))
ret['metric/fid_%s-best' % config_name] = getattr(
self, 'best_fid_%s' % config_name)
if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU:
mIoU = get_cityscapes_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads,
tqdm_position=2)
if mIoU > getattr(self, 'best_mIoU_%s' % config_name):
self.is_best = True
setattr(self, 'best_mIoU_%s' % config_name, mIoU)
mIoUs = getattr(self, 'mIoUs_%s' % config_name)
mIoUs.append(mIoU)
if len(mIoUs) > 3:
mIoUs.pop(0)
ret['metric/mIoU_%s' % config_name] = mIoU
ret['metric/mIoU_%s-mean' % config_name] = sum(
getattr(self, 'mIoUs_%s' % config_name)) / len(
getattr(self, 'mIoUs_%s' % config_name))
ret['metric/mIoU_%s-best' % config_name] = getattr(
self, 'best_mIoU_%s' % config_name)
self.modules_on_one_gpu.netG_student.train()
torch.cuda.empty_cache()
return ret
device = copy.deepcopy(model.device)
del model
torch.cuda.empty_cache()
inception_model, drn_model, deeplabv2_model = create_metric_models(
opt, device)
if inception_model is not None:
npz = np.load(opt.real_stat_path)
fid = get_fid(fakes, inception_model, npz, device, opt.batch_size)
print('fid score: %.2f' % fid, flush=True)
if drn_model is not None:
mIoU = get_cityscapes_mIoU(fakes,
names,
drn_model,
device,
table_path=opt.table_path,
data_dir=opt.cityscapes_path,
batch_size=opt.batch_size,
num_workers=opt.num_threads)
print('mIoU: %.2f' % mIoU)
if deeplabv2_model is not None:
accu, mIoU = get_coco_scores(fakes,
names,
deeplabv2_model,
device,
opt.dataroot,
opt.batch_size,
num_workers=0)
print('accu: %.2f\tmIoU: %.2f' % (accu, mIoU))
def evaluate_model(self, step):
ret = {}
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG_A.eval()
self.netG_B.eval()
for direction in ['AtoB', 'BtoA']:
eval_dataloader = getattr(self, 'eval_dataloader_' + direction)
fakes, names = [], []
cnt = 0
for i, data_i in enumerate(
tqdm(eval_dataloader,
desc='Eval %s ' % direction,
position=2,
leave=False)):
self.set_single_input(data_i)
self.test_single_side(direction)
fakes.append(self.fake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2im(self.real_A[j])
fake_im = util.tensor2im(self.fake_B[j])
util.save_image(input_im,
os.path.join(save_dir, direction,
'input', '%s.png' % name),
create_dir=True)
util.save_image(fake_im,
os.path.join(save_dir, direction,
'fake', '%s.png' % name),
create_dir=True)
cnt += 1
suffix = direction[-1]
fid = get_fid(fakes,
self.inception_model,
getattr(self, 'npz_%s' % direction[-1]),
device=self.device,
batch_size=self.opt.eval_batch_size,
tqdm_position=2)
if fid < getattr(self, 'best_fid_%s' % suffix):
self.is_best = True
setattr(self, 'best_fid_%s' % suffix, fid)
fids = getattr(self, 'fids_%s' % suffix)
fids.append(fid)
if len(fids) > 3:
fids.pop(0)
ret['metric/fid_%s' % suffix] = fid
ret['metric/fid_%s-mean' %
suffix] = sum(getattr(self, 'fids_%s' % suffix)) / len(
getattr(self, 'fids_%s' % suffix))
ret['metric/fid_%s-best' % suffix] = getattr(
self, 'best_fid_%s' % suffix)
if 'cityscapes' in self.opt.dataroot and direction == 'BtoA':
mIoU = get_cityscapes_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads,
tqdm_position=2)
if mIoU > self.best_mIoU:
self.is_best = True
self.best_mIoU = mIoU
self.mIoUs.append(mIoU)
if len(self.mIoUs) > 3:
self.mIoUs = self.mIoUs[1:]
ret['metric/mIoU'] = mIoU
ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs)
ret['metric/mIoU-best'] = self.best_mIoU
self.netG_A.train()
self.netG_B.train()
return ret
def evaluate_model(self, step):
self.is_best = False
save_dir = os.path.join(self.opt.log_dir, 'eval', str(step))
os.makedirs(save_dir, exist_ok=True)
self.netG_student.eval()
fakes, names = [], []
cnt = 0
for i, data_i in enumerate(
tqdm(self.eval_dataloader,
desc='Eval ',
position=2,
leave=False)):
if self.opt.dataset_mode == 'aligned':
self.set_input(data_i)
else:
self.set_single_input(data_i)
self.test()
fakes.append(self.Sfake_B.cpu())
for j in range(len(self.image_paths)):
short_path = ntpath.basename(self.image_paths[j])
name = os.path.splitext(short_path)[0]
names.append(name)
if cnt < 10:
input_im = util.tensor2im(self.real_A[j])
Sfake_im = util.tensor2im(self.Sfake_B[j])
Tfake_im = util.tensor2im(self.Tfake_B[j])
util.save_image(input_im,
os.path.join(save_dir, 'input', '%s.png') %
name,
create_dir=True)
util.save_image(Sfake_im,
os.path.join(save_dir, 'Sfake',
'%s.png' % name),
create_dir=True)
util.save_image(Tfake_im,
os.path.join(save_dir, 'Tfake',
'%s.png' % name),
create_dir=True)
if self.opt.dataset_mode == 'aligned':
real_im = util.tensor2im(self.real_B[j])
util.save_image(real_im,
os.path.join(save_dir, 'real',
'%s.png' % name),
create_dir=True)
cnt += 1
fid = get_fid(fakes,
self.inception_model,
self.npz,
device=self.device,
batch_size=self.opt.eval_batch_size,
tqdm_position=2)
if fid < self.best_fid:
self.is_best = True
self.best_fid = fid
self.fids.append(fid)
if len(self.fids) > 3:
self.fids.pop(0)
ret = {
'metric/fid': fid,
'metric/fid-mean': sum(self.fids) / len(self.fids),
'metric/fid-best': self.best_fid
}
if 'cityscapes' in self.opt.dataroot and self.opt.direction == 'BtoA':
mIoU = get_cityscapes_mIoU(fakes,
names,
self.drn_model,
self.device,
table_path=self.opt.table_path,
data_dir=self.opt.cityscapes_path,
batch_size=self.opt.eval_batch_size,
num_workers=self.opt.num_threads,
tqdm_position=2)
if mIoU > self.best_mIoU:
self.is_best = True
self.best_mIoU = mIoU
self.mIoUs.append(mIoU)
if len(self.mIoUs) > 3:
self.mIoUs = self.mIoUs[1:]
ret['metric/mIoU'] = mIoU
ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs)
ret['metric/mIoU-best'] = self.best_mIoU
self.netG_student.train()
return ret
for path in model.get_image_paths():
short_path = ntpath.basename(path)
name = os.path.splitext(short_path)[0]
names.append(name)
tqdm_position = 1
if inception_model is not None:
if qualified:
result['fid'] = get_fid(fakes, inception_model, npz, device, opt.batch_size,
tqdm_position=tqdm_position)
tqdm_position += 1
else:
result['fid'] = 1e9
if drn_model is not None:
if qualified:
result['mIoU'] = get_cityscapes_mIoU(fakes, names, drn_model, device, data_dir=opt.cityscapes_path,
batch_size=opt.batch_size, num_workers=opt.num_threads,
tqdm_position=tqdm_position)
tqdm_position += 1
else:
result['mIoU'] = 0
if deeplabv2_model is not None:
if qualified:
torch.cuda.empty_cache()
result['accu'], result['mIoU'] = get_coco_scores(fakes, names, deeplabv2_model, device, opt.dataroot, 1,
num_workers=0, tqdm_position=tqdm_position)
tqdm_position += 1
else:
result['accu'], result['mIoU'] = 0, 0
results.append(result)
eval_configs.add(config_str)
configs_tqdm.write(str(result))
