def evaluate(self, child_pool): results = [] for child in tqdm(child_pool, position=1, desc='Evaluate ', leave=False): result = self.result_cache.get(encode_config(child)) if result is None: result = {} fakes, names = [], [] if isinstance(self.model, SPADEModel): self.model.calibrate(child) for i, data_i in enumerate(self.dataloader): self.model.set_input(data_i) self.model.test(child) fakes.append(self.model.fake_B.cpu()) for path in self.model.get_image_paths(): short_path = ntpath.basename(path) name = os.path.splitext(short_path)[0] names.append(name) if self.inception_model is not None: result['fid'] = get_fid(fakes, self.inception_model, self.npz, self.device, opt.batch_size, tqdm_position=2) if self.drn_model is not None: result['mIoU'] = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device, data_dir=opt.cityscapes_path, batch_size=opt.batch_size, num_workers=opt.num_threads, tqdm_position=2) if self.deeplabv2_model is not None: torch.cuda.empty_cache() result['accu'], result['mIoU'] = get_coco_scores(fakes, names, self.deeplabv2_model, self.device, opt.dataroot, 1, num_workers=0, tqdm_position=2) if len(self.result_cache) < self.opt.max_cache_size: self.result_cache[encode_config(child)] = result results.append(result) return results
def evaluate_model(self, step): self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.modules_on_one_gpu.netG.eval() torch.cuda.empty_cache() fakes, names = [], [] ret = {} cnt = 0 for i, data_i in enumerate(tqdm(self.eval_dataloader, desc='Eval ', position=2, leave=False)): self.set_input(data_i) self.test() fakes.append(self.fake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10: input_im = util.tensor2label(self.input_semantics[j], self.opt.input_nc + 2) real_im = util.tensor2im(self.real_B[j]) fake_im = util.tensor2im(self.fake_B[j]) util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png' % name), create_dir=True) util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True) util.save_image(fake_im, os.path.join(save_dir, 'fake', '%s.png' % name), create_dir=True) cnt += 1 if not self.opt.no_fid: fid = get_fid(fakes, self.inception_model, self.npz, device=self.device, batch_size=self.opt.eval_batch_size, tqdm_position=2) if fid < self.best_fid: self.is_best = True self.best_fid = fid self.fids.append(fid) if len(self.fids) > 3: self.fids.pop(0) ret['metric/fid'] = fid ret['metric/fid-mean'] = sum(self.fids) / len(self.fids) ret['metric/fid-best'] = self.best_fid if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU: mIoU = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads, tqdm_position=2) if mIoU > self.best_mIoU: self.is_best = True self.best_mIoU = mIoU self.mIoUs.append(mIoU) if len(self.mIoUs) > 3: self.mIoUs = self.mIoUs[1:] ret['metric/mIoU'] = mIoU ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs) ret['metric/mIoU-best'] = self.best_mIoU self.modules_on_one_gpu.netG.train() torch.cuda.empty_cache() return ret
def evaluate_model(self, step): ret = {} self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) if self.opt.eval_mode == 'both': settings = ('largest', 'smallest') else: settings = (self.opt.eval_mode, ) for config_name in settings: config = self.configs(config_name) fakes, names = [], [] self.modules_on_one_gpu.netG_student.train() self.calibrate(config, 2) tqdm_position = 2 + int(not self.opt.no_calibration) self.modules_on_one_gpu.netG_student.eval() torch.cuda.empty_cache() cnt = 0 for i, data_i in enumerate( tqdm(self.eval_dataloader, desc='Eval ', position=tqdm_position, leave=False)): self.set_input(data_i) self.test(config=config) fakes.append(self.Sfake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10: input_im = util.tensor2label(self.input_semantics[j], self.opt.input_nc + 2) real_im = util.tensor2im(self.real_B[j]) Tfake_im = util.tensor2im(self.Tfake_B[j]) Sfake_im = util.tensor2im(self.Sfake_B[j]) util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png' % name), create_dir=True) util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True) util.save_image(Tfake_im, os.path.join(save_dir, 'Tfake', '%s.png' % name), create_dir=True) util.save_image(Sfake_im, os.path.join(save_dir, 'Sfake', '%s.png' % name), create_dir=True) cnt += 1 if not self.opt.no_fid: fid = get_fid(fakes, self.inception_model, self.npz, device=self.device, batch_size=self.opt.eval_batch_size, tqdm_position=2) if fid < getattr(self, 'best_fid_%s' % config_name): self.is_best = True setattr(self, 'best_fid_%s' % config_name, fid) fids = getattr(self, 'fids_%s' % config_name) fids.append(fid) if len(fids) > 3: fids.pop(0) ret['metric/fid_%s' % config_name] = fid ret['metric/fid_%s-mean' % config_name] = sum( getattr(self, 'fids_%s' % config_name)) / len( getattr(self, 'fids_%s' % config_name)) ret['metric/fid_%s-best' % config_name] = getattr( self, 'best_fid_%s' % config_name) if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU: mIoU = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads, tqdm_position=2) if mIoU > getattr(self, 'best_mIoU_%s' % config_name): self.is_best = True setattr(self, 'best_mIoU_%s' % config_name, mIoU) mIoUs = getattr(self, 'mIoUs_%s' % config_name) mIoUs.append(mIoU) if len(mIoUs) > 3: mIoUs.pop(0) ret['metric/mIoU_%s' % config_name] = mIoU ret['metric/mIoU_%s-mean' % config_name] = sum( getattr(self, 'mIoUs_%s' % config_name)) / len( getattr(self, 'mIoUs_%s' % config_name)) ret['metric/mIoU_%s-best' % config_name] = getattr( self, 'best_mIoU_%s' % config_name) self.modules_on_one_gpu.netG_student.train() torch.cuda.empty_cache() return ret
device = copy.deepcopy(model.device) del model torch.cuda.empty_cache() inception_model, drn_model, deeplabv2_model = create_metric_models( opt, device) if inception_model is not None: npz = np.load(opt.real_stat_path) fid = get_fid(fakes, inception_model, npz, device, opt.batch_size) print('fid score: %.2f' % fid, flush=True) if drn_model is not None: mIoU = get_cityscapes_mIoU(fakes, names, drn_model, device, table_path=opt.table_path, data_dir=opt.cityscapes_path, batch_size=opt.batch_size, num_workers=opt.num_threads) print('mIoU: %.2f' % mIoU) if deeplabv2_model is not None: accu, mIoU = get_coco_scores(fakes, names, deeplabv2_model, device, opt.dataroot, opt.batch_size, num_workers=0) print('accu: %.2f\tmIoU: %.2f' % (accu, mIoU))
def evaluate_model(self, step): ret = {} self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.netG_A.eval() self.netG_B.eval() for direction in ['AtoB', 'BtoA']: eval_dataloader = getattr(self, 'eval_dataloader_' + direction) fakes, names = [], [] cnt = 0 for i, data_i in enumerate( tqdm(eval_dataloader, desc='Eval %s ' % direction, position=2, leave=False)): self.set_single_input(data_i) self.test_single_side(direction) fakes.append(self.fake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10: input_im = util.tensor2im(self.real_A[j]) fake_im = util.tensor2im(self.fake_B[j]) util.save_image(input_im, os.path.join(save_dir, direction, 'input', '%s.png' % name), create_dir=True) util.save_image(fake_im, os.path.join(save_dir, direction, 'fake', '%s.png' % name), create_dir=True) cnt += 1 suffix = direction[-1] fid = get_fid(fakes, self.inception_model, getattr(self, 'npz_%s' % direction[-1]), device=self.device, batch_size=self.opt.eval_batch_size, tqdm_position=2) if fid < getattr(self, 'best_fid_%s' % suffix): self.is_best = True setattr(self, 'best_fid_%s' % suffix, fid) fids = getattr(self, 'fids_%s' % suffix) fids.append(fid) if len(fids) > 3: fids.pop(0) ret['metric/fid_%s' % suffix] = fid ret['metric/fid_%s-mean' % suffix] = sum(getattr(self, 'fids_%s' % suffix)) / len( getattr(self, 'fids_%s' % suffix)) ret['metric/fid_%s-best' % suffix] = getattr( self, 'best_fid_%s' % suffix) if 'cityscapes' in self.opt.dataroot and direction == 'BtoA': mIoU = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads, tqdm_position=2) if mIoU > self.best_mIoU: self.is_best = True self.best_mIoU = mIoU self.mIoUs.append(mIoU) if len(self.mIoUs) > 3: self.mIoUs = self.mIoUs[1:] ret['metric/mIoU'] = mIoU ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs) ret['metric/mIoU-best'] = self.best_mIoU self.netG_A.train() self.netG_B.train() return ret
def evaluate_model(self, step): self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.netG_student.eval() fakes, names = [], [] cnt = 0 for i, data_i in enumerate( tqdm(self.eval_dataloader, desc='Eval ', position=2, leave=False)): if self.opt.dataset_mode == 'aligned': self.set_input(data_i) else: self.set_single_input(data_i) self.test() fakes.append(self.Sfake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10: input_im = util.tensor2im(self.real_A[j]) Sfake_im = util.tensor2im(self.Sfake_B[j]) Tfake_im = util.tensor2im(self.Tfake_B[j]) util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png') % name, create_dir=True) util.save_image(Sfake_im, os.path.join(save_dir, 'Sfake', '%s.png' % name), create_dir=True) util.save_image(Tfake_im, os.path.join(save_dir, 'Tfake', '%s.png' % name), create_dir=True) if self.opt.dataset_mode == 'aligned': real_im = util.tensor2im(self.real_B[j]) util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True) cnt += 1 fid = get_fid(fakes, self.inception_model, self.npz, device=self.device, batch_size=self.opt.eval_batch_size, tqdm_position=2) if fid < self.best_fid: self.is_best = True self.best_fid = fid self.fids.append(fid) if len(self.fids) > 3: self.fids.pop(0) ret = { 'metric/fid': fid, 'metric/fid-mean': sum(self.fids) / len(self.fids), 'metric/fid-best': self.best_fid } if 'cityscapes' in self.opt.dataroot and self.opt.direction == 'BtoA': mIoU = get_cityscapes_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads, tqdm_position=2) if mIoU > self.best_mIoU: self.is_best = True self.best_mIoU = mIoU self.mIoUs.append(mIoU) if len(self.mIoUs) > 3: self.mIoUs = self.mIoUs[1:] ret['metric/mIoU'] = mIoU ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs) ret['metric/mIoU-best'] = self.best_mIoU self.netG_student.train() return ret
for path in model.get_image_paths(): short_path = ntpath.basename(path) name = os.path.splitext(short_path)[0] names.append(name) tqdm_position = 1 if inception_model is not None: if qualified: result['fid'] = get_fid(fakes, inception_model, npz, device, opt.batch_size, tqdm_position=tqdm_position) tqdm_position += 1 else: result['fid'] = 1e9 if drn_model is not None: if qualified: result['mIoU'] = get_cityscapes_mIoU(fakes, names, drn_model, device, data_dir=opt.cityscapes_path, batch_size=opt.batch_size, num_workers=opt.num_threads, tqdm_position=tqdm_position) tqdm_position += 1 else: result['mIoU'] = 0 if deeplabv2_model is not None: if qualified: torch.cuda.empty_cache() result['accu'], result['mIoU'] = get_coco_scores(fakes, names, deeplabv2_model, device, opt.dataroot, 1, num_workers=0, tqdm_position=tqdm_position) tqdm_position += 1 else: result['accu'], result['mIoU'] = 0, 0 results.append(result) eval_configs.add(config_str) configs_tqdm.write(str(result))