def evaluate(self, child_pool):
results = []
for child in tqdm(child_pool, position=1, desc='Evaluate ', leave=False):
result = self.result_cache.get(encode_config(child))
if result is None:
result = {}
fakes, names = [], []
if isinstance(self.model, SPADEModel):
self.model.calibrate(child)
for i, data_i in enumerate(self.dataloader):
self.model.set_input(data_i)
self.model.test(child)
fakes.append(self.model.fake_B.cpu())
for path in self.model.get_image_paths():
short_path = ntpath.basename(path)
name = os.path.splitext(short_path)[0]
names.append(name)
if self.inception_model is not None:
result['fid'] = get_fid(fakes, self.inception_model, self.npz,
self.device, opt.batch_size, tqdm_position=2)
if self.drn_model is not None:
result['mIoU'] = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device,
data_dir=opt.cityscapes_path, batch_size=opt.batch_size,
num_workers=opt.num_threads, tqdm_position=2)
if self.deeplabv2_model is not None:
torch.cuda.empty_cache()
result['accu'], result['mIoU'] = get_coco_scores(fakes, names, self.deeplabv2_model, self.device,
opt.dataroot, 1, num_workers=0, tqdm_position=2)
if len(self.result_cache) < self.opt.max_cache_size:
self.result_cache[encode_config(child)] = result
results.append(result)
return results
del model
torch.cuda.empty_cache()
inception_model, drn_model, deeplabv2_model = create_metric_models(
opt, device)
if inception_model is not None:
npz = np.load(opt.real_stat_path)
fid = get_fid(fakes, inception_model, npz, device, opt.batch_size)
print('fid score: %.2f' % fid, flush=True)
if drn_model is not None:
mIoU = get_cityscapes_mIoU(fakes,
names,
drn_model,
device,
table_path=opt.table_path,
data_dir=opt.cityscapes_path,
batch_size=opt.batch_size,
num_workers=opt.num_threads)
print('mIoU: %.2f' % mIoU)
if deeplabv2_model is not None:
accu, mIoU = get_coco_scores(fakes,
names,
deeplabv2_model,
device,
opt.dataroot,
opt.batch_size,
num_workers=0)
print('accu: %.2f\tmIoU: %.2f' % (accu, mIoU))
tqdm_position=tqdm_position)
tqdm_position += 1
else:
result['fid'] = 1e9
if drn_model is not None:
if qualified:
result['mIoU'] = get_cityscapes_mIoU(fakes, names, drn_model, device, data_dir=opt.cityscapes_path,
batch_size=opt.batch_size, num_workers=opt.num_threads,
tqdm_position=tqdm_position)
tqdm_position += 1
else:
result['mIoU'] = 0
if deeplabv2_model is not None:
if qualified:
torch.cuda.empty_cache()
result['accu'], result['mIoU'] = get_coco_scores(fakes, names, deeplabv2_model, device, opt.dataroot, 1,
num_workers=0, tqdm_position=tqdm_position)
tqdm_position += 1
else:
result['accu'], result['mIoU'] = 0, 0
results.append(result)
eval_configs.add(config_str)
configs_tqdm.write(str(result))
current_time = time.time()
if current_time - last_save_time > opt.save_freq * 60:
last_save_time = current_time
save(opt, results)
configs_tqdm.write('Save the latest results at [%s].' % (opt.output_path))
save(opt, results)
print('Successfully finish searching!!!', flush=True)