def test_pix2pix_mIoU(model, opt): opt.phase = 'val' opt.num_threads = 0 opt.batch_size = 1 opt.serial_batches = True opt.no_flip = True opt.load_size = 256 opt.display_id = -1 dataset = create_dataset(opt) model.model_eval() result_dir = os.path.join(opt.checkpoints_dir, opt.name, 'test_results') util.mkdirs(result_dir) fake_B = {} names = [] for i, data in enumerate(dataset): model.set_input(data) with torch.no_grad(): model.forward() visuals = model.get_current_visuals() fake_B[data['A_paths'][0]] = visuals['fake_B'] for path in range(len(model.image_paths)): short_path = ntpath.basename(model.image_paths[0][0]) name = os.path.splitext(short_path)[0] if name not in names: names.append(name) util.save_images(visuals, model.image_paths, result_dir, direction=opt.direction, aspect_ratio=opt.aspect_ratio) drn_model = DRNSeg('drn_d_105', 19, pretrained=False).to(model.device) util.load_network(drn_model, opt.drn_path, verbose=False) drn_model.eval() mIoU = get_mIoU(list(fake_B.values()), names, drn_model, model.device, table_path=os.path.join(opt.dataroot, 'table.txt'), data_dir=opt.dataroot, batch_size=opt.batch_size, num_workers=opt.num_threads) return mIoU
def evaluate_model(self, step): self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.netG_student.eval() fakes, names = [], [] cnt = 0 for i, data_i in enumerate(tqdm(self.eval_dataloader)): if self.opt.dataset_mode == 'aligned': self.set_input(data_i) else: self.set_single_input(data_i) self.test() fakes.append(self.Sfake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10: input_im = util.tensor2im(self.real_A[j]) Sfake_im = util.tensor2im(self.Sfake_B[j]) Tfake_im = util.tensor2im(self.Tfake_B[j]) util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png') % name, create_dir=True) util.save_image(Sfake_im, os.path.join(save_dir, 'Sfake', '%s.png' % name), create_dir=True) util.save_image(Tfake_im, os.path.join(save_dir, 'Tfake', '%s.png' % name), create_dir=True) if self.opt.dataset_mode == 'aligned': real_im = util.tensor2im(self.real_B[j]) util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True) cnt += 1 fid = get_fid(fakes, self.inception_model, self.npz, device=self.device, batch_size=self.opt.eval_batch_size) if fid < self.best_fid: self.is_best = True self.best_fid = fid self.fids.append(fid) if len(self.fids) > 3: self.fids.pop(0) ret = { 'metric/fid': fid, 'metric/fid-mean': sum(self.fids) / len(self.fids), 'metric/fid-best': self.best_fid } if 'cityscapes' in self.opt.dataroot and self.opt.direction == 'BtoA': mIoU = get_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads) if mIoU > self.best_mIoU: self.is_best = True self.best_mIoU = mIoU self.mIoUs.append(mIoU) if len(self.mIoUs) > 3: self.mIoUs = self.mIoUs[1:] ret['metric/mIoU'] = mIoU ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs) ret['metric/mIoU-best'] = self.best_mIoU self.netG_student.train() return ret
def evaluate_model(self, step): ret = {} self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.netG_A.eval() self.netG_B.eval() for direction in ['AtoB', 'BtoA']: eval_dataloader = getattr(self, 'eval_dataloader_' + direction) fakes, names = [], [] cnt = 0 # print(len(eval_dataset)) for i, data_i in enumerate(tqdm(eval_dataloader)): self.set_single_input(data_i) self.test_single_side(direction) # print(self.image_paths) fakes.append(self.fake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10: input_im = util.tensor2im(self.real_A[j]) fake_im = util.tensor2im(self.fake_B[j]) util.save_image(input_im, os.path.join(save_dir, direction, 'input', '%s.png' % name), create_dir=True) util.save_image(fake_im, os.path.join(save_dir, direction, 'fake', '%s.png' % name), create_dir=True) cnt += 1 suffix = direction[-1] fid = get_fid(fakes, self.inception_model, getattr(self, 'npz_%s' % direction[-1]), device=self.device, batch_size=self.opt.eval_batch_size) if fid < getattr(self, 'best_fid_%s' % suffix): self.is_best = True setattr(self, 'best_fid_%s' % suffix, fid) fids = getattr(self, 'fids_%s' % suffix) fids.append(fid) if len(fids) > 3: fids.pop(0) ret['metric/fid_%s' % suffix] = fid ret['metric/fid_%s-mean' % suffix] = sum(getattr(self, 'fids_%s' % suffix)) / len( getattr(self, 'fids_%s' % suffix)) ret['metric/fid_%s-best' % suffix] = getattr( self, 'best_fid_%s' % suffix) if 'cityscapes' in self.opt.dataroot and direction == 'BtoA': mIoU = get_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads) if mIoU > self.best_mIoU: self.is_best = True self.best_mIoU = mIoU self.mIoUs.append(mIoU) if len(self.mIoUs) > 3: self.mIoUs = self.mIoUs[1:] ret['metric/mIoU'] = mIoU ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs) ret['metric/mIoU-best'] = self.best_mIoU self.netG_A.train() self.netG_B.train() return ret
def evaluate_model(self, step): ret = {} self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.netG_student.eval() if self.opt.eval_mode == 'both': setting = ('largest', 'smallest') else: setting = (self.opt.eval_mode, ) for config_name in setting: config = self.configs(config_name) fakes, names = [], [] cnt = 0 for i, data_i in enumerate(tqdm(self.eval_dataloader)): if self.opt.dataset_mode == 'aligned': self.set_input(data_i) else: self.set_single_input(data_i) self.test(config) fakes.append(self.Sfake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if i < 10: Sfake_im = util.tensor2im(self.Sfake_B[j]) real_im = util.tensor2im(self.real_B[j]) Tfake_im = util.tensor2im(self.Tfake_B[j]) util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True) util.save_image(Sfake_im, os.path.join(save_dir, 'Sfake_%s' % config_name, '%s.png' % name), create_dir=True) util.save_image(Tfake_im, os.path.join(save_dir, 'Tfake', '%s.png' % name), create_dir=True) if self.opt.dataset_mode == 'aligned': input_im = util.tensor2im(self.real_A[j]) util.save_image( input_im, os.path.join(save_dir, 'input', '%s.png') % name, create_dir=True) cnt += 1 fid = get_fid(fakes, self.inception_model, self.npz, device=self.device, batch_size=self.opt.eval_batch_size) if fid < getattr(self, 'best_fid_%s' % config_name): self.is_best = True setattr(self, 'best_fid_%s' % config_name, fid) fids = getattr(self, 'fids_%s' % config_name) fids.append(fid) if len(fids) > 3: fids.pop(0) ret['metric/fid_%s' % config_name] = fid ret['metric/fid_%s-mean' % config_name] = sum( getattr(self, 'fids_%s' % config_name)) / len( getattr(self, 'fids_%s' % config_name)) ret['metric/fid_%s-best' % config_name] = getattr( self, 'best_fid_%s' % config_name) if 'cityscapes' in self.opt.dataroot: mIoU = get_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads) if mIoU > getattr(self, 'best_mIoU_%s' % config_name): self.is_best = True setattr(self, 'best_mIoU_%s' % config_name, mIoU) mIoUs = getattr(self, 'mIoUs_%s' % config_name) mIoUs.append(mIoU) if len(mIoUs) > 3: mIoUs.pop(0) ret['metric/mIoU_%s' % config_name] = mIoU ret['metric/mIoU_%s-mean' % config_name] = sum( getattr(self, 'mIoUs_%s' % config_name)) / len( getattr(self, 'mIoUs_%s' % config_name)) ret['metric/mIoU_%s-best' % config_name] = getattr( self, 'best_mIoU_%s' % config_name) self.netG_student.train() return ret
if qualified: fid = get_fid(fakes, inception_model, npz, device, opt.batch_size, use_tqdm=False) result['fid'] = fid else: result['fid'] = 1e9 if 'cityscapes' in opt.dataroot and opt.direction == 'BtoA': if qualified: mIoU = get_mIoU(fakes, names, drn_model, device, data_dir=opt.cityscapes_path, batch_size=opt.batch_size, num_workers=opt.num_threads, use_tqdm=False) result['mIoU'] = mIoU else: result['mIoU'] = 0 print(result, flush=True) results.append(result) os.makedirs(os.path.dirname(opt.output_path), exist_ok=True) with open(opt.output_path, 'wb') as f: pickle.dump(results, f) print('Successfully finish searching!!!', flush=True)
def evaluate_model(self, step, save_image=False): self.is_best = False save_dir = os.path.join(self.opt.log_dir, 'eval', str(step)) os.makedirs(save_dir, exist_ok=True) self.modules_on_one_gpu.netG.eval() torch.cuda.empty_cache() fakes, names = [], [] ret = {} cnt = 0 for i, data_i in enumerate(tqdm(self.eval_dataloader)): self.set_input(data_i) self.test() fakes.append(self.fake_B.cpu()) for j in range(len(self.image_paths)): short_path = ntpath.basename(self.image_paths[j]) name = os.path.splitext(short_path)[0] names.append(name) if cnt < 10 or save_image: input_im = util.tensor2label(self.input_semantics[j], self.opt.input_nc + 2) real_im = util.tensor2im(self.real_B[j]) fake_im = util.tensor2im(self.fake_B[j]) util.save_image(input_im, os.path.join(save_dir, 'input', '%s.png' % name), create_dir=True) util.save_image(real_im, os.path.join(save_dir, 'real', '%s.png' % name), create_dir=True) util.save_image(fake_im, os.path.join(save_dir, 'fake', '%s.png' % name), create_dir=True) cnt += 1 if not self.opt.no_fid: fid = get_fid(fakes, self.inception_model, self.npz, device=self.device, batch_size=self.opt.eval_batch_size) if fid < self.best_fid: self.is_best = True self.best_fid = fid self.fids.append(fid) if len(self.fids) > 3: self.fids.pop(0) ret['metric/fid'] = fid ret['metric/fid-mean'] = sum(self.fids) / len(self.fids) ret['metric/fid-best'] = self.best_fid if 'cityscapes' in self.opt.dataroot and not self.opt.no_mIoU: mIoU = get_mIoU(fakes, names, self.drn_model, self.device, table_path=self.opt.table_path, data_dir=self.opt.cityscapes_path, batch_size=self.opt.eval_batch_size, num_workers=self.opt.num_threads) if mIoU > self.best_mIoU: self.is_best = True self.best_mIoU = mIoU self.mIoUs.append(mIoU) if len(self.mIoUs) > 3: self.mIoUs = self.mIoUs[1:] ret['metric/mIoU'] = mIoU ret['metric/mIoU-mean'] = sum(self.mIoUs) / len(self.mIoUs) ret['metric/mIoU-best'] = self.best_mIoU self.modules_on_one_gpu.netG.train() torch.cuda.empty_cache() return ret
def main(configs, opt, gpu_id, queue, verbose): opt.gpu_ids = [gpu_id] dataloader = create_dataloader(opt, verbose) model = create_model(opt, verbose) model.setup(opt, verbose) device = model.device if not opt.no_fid: block_idx = InceptionV3.BLOCK_INDEX_BY_DIM[2048] inception_model = InceptionV3([block_idx]) inception_model.to(device) inception_model.eval() if 'cityscapes' in opt.dataroot and opt.direction == 'BtoA': drn_model = DRNSeg('drn_d_105', 19, pretrained=False) util.load_network(drn_model, opt.drn_path, verbose=False) if len(opt.gpu_ids) > 0: drn_model = nn.DataParallel(drn_model, opt.gpu_ids) drn_model.eval() npz = np.load(opt.real_stat_path) results = [] for data_i in dataloader: model.set_input(data_i) break for config in tqdm.tqdm(configs): qualified = True macs, _ = model.profile(config) if macs > opt.budget: qualified = False else: qualified = True fakes, names = [], [] if qualified: for i, data_i in enumerate(dataloader): model.set_input(data_i) model.test(config) fakes.append(model.fake_B.cpu()) for path in model.get_image_paths(): short_path = ntpath.basename(path) name = os.path.splitext(short_path)[0] names.append(name) result = {'config_str': encode_config(config), 'macs': macs} if not opt.no_fid: if qualified: fid = get_fid(fakes, inception_model, npz, device, opt.batch_size, use_tqdm=False) result['fid'] = fid else: result['fid'] = 1e9 if 'cityscapes' in opt.dataroot and opt.direction == 'BtoA': if qualified: mIoU = get_mIoU(fakes, names, drn_model, device, data_dir=opt.cityscapes_path, batch_size=opt.batch_size, num_workers=opt.num_threads, use_tqdm=False) result['mIoU'] = mIoU else: result['mIoU'] = mIoU print(result, flush=True) results.append(result) queue.put(results)