def eval_fid2(self, testloader, video_transforms, image_transforms):
from fid.fid_score import fid_score
output_score_filename = os.path.join(self.save_dir, 'fid_score2.csv')
with open(output_score_filename, 'a') as f:
f.write('epoch,fid,vfid\n')
models = os.listdir(self.model_dir)
for epoch in range(121, 0, -1):
if 'netG_epoch_{}.pth'.format(epoch) in models:
if os.path.exists(os.path.join(self.save_dir, 'original')):
shutil.rmtree(os.path.join(self.save_dir, 'original'))
if os.path.exists(os.path.join(self.save_dir, 'generate')):
shutil.rmtree(os.path.join(self.save_dir, 'generate'))
print('Evaluating epoch {}'.format(epoch))
netG = self.load_network_stageI(self.output_dir, load_ckpt=epoch)
with torch.no_grad():
self.generate_story(netG, testloader)
ref_dataloader = FolderStoryDataset(os.path.join(self.save_dir, 'original'), video_transforms)
gen_dataloader = FolderStoryDataset(os.path.join(self.save_dir, 'generate'), video_transforms)
vfid = vfid_score(ref_dataloader,
gen_dataloader, cuda=True, normalize=True, r_cache=None)
ref_dataloader = FolderImageDataset(os.path.join(self.save_dir, 'original'), image_transforms)
gen_dataloader = FolderImageDataset(os.path.join(self.save_dir, 'generate'), image_transforms)
fid = fid_score(ref_dataloader,
gen_dataloader, cuda=True, normalize=True, r_cache=None)
with open(output_score_filename, 'a') as f:
f.write('{},{},{}\n'.format(epoch, fid, vfid))
def calculate_vfid(self, netG, epoch, testloader):
netG.eval()
with torch.no_grad():
eval_modeldataset = StoryGANDataset(netG, len(testloader), testloader.dataset)
vfid_value = vfid_score(IgnoreLabelDataset(testloader.dataset),
eval_modeldataset, cuda=True, normalize=True, r_cache=None
)
fid_value = fid_score(IgnoreLabelDataset(testloader.dataset),
eval_modeldataset, cuda=True, normalize=True, r_cache=None
)
#netG.train()
if self._logger:
self._logger.add_scalar('Off_Evaluation/vfid', vfid_value, epoch)
self._logger.add_scalar('Off_Evaluation/fid', fid_value, epoch)
return fid_value, vfid_value
def calculate_vfid(self, netG, epoch, testloader):
netG.eval()
with torch.no_grad():
eval_modeldataset = StoryGANDataset(netG, len(testloader),
testloader.dataset)
vfid_value = vfid_score(
IgnoreLabelDataset(testloader.dataset),
eval_modeldataset,
cuda=True,
normalize=True,
r_cache='.cache/seg_story_vfid_reference_score.npz')
fid_value = fid_score(
IgnoreLabelDataset(testloader.dataset),
eval_modeldataset,
cuda=True,
normalize=True,
r_cache='.cache/seg_story_fid_reference_score.npz')
netG.train()
if self._logger:
self._logger.add_scalar('Evaluation/vfid', vfid_value, epoch)
self._logger.add_scalar('Evaluation/fid', fid_value, epoch)