def eval_loss_metric(model, resultdict=None):
if resultdict is None:
resultdict = defaultdict(lambda: defaultdict(lambda: {}))
for dataset in datasets:
data_loader = dl.CreateDataLoader(dataset,
dataset_mode=dataset_mode,
batch_size=model.batch_size)
# evaluate model on data
if (dataset_mode == '2afc'):
(score,
results_verbose) = dm.score_2afc_dataset(data_loader,
model.forward)
resultdict[model.model_name][dataset]['score'] = results_verbose[
'scores'].mean()
resultdict[model.model_name][dataset]['std'] = results_verbose[
'scores'].std()
human_judgements = results_verbose['gts']
human_scores = human_judgements**2 + (1 - human_judgements)**2
resultdict['Human'][dataset]['score'] = human_scores.mean()
resultdict['Human'][dataset]['std'] = human_scores.std()
elif (dataset_mode == 'jnd'):
raise Exception('not implemented / validated')
(score,
results_verbose) = dm.score_jnd_dataset(data_loader,
model.forward)
# print results
print(' Model [%s] Dataset [%s]: %.2f' %
(model.model_name, dataset, 100. * score))
return resultdict
use_gpu=opt.use_gpu,
pnet_rand=opt.from_scratch,
pnet_tune=opt.train_trunk,
version=opt.version,
gpu_ids=opt.gpu_ids)
if (opt.model in ['net-lin', 'net']):
print('Testing model [%s]-[%s]' % (opt.model, opt.net))
elif (opt.model in ['l2', 'ssim']):
print('Testing model [%s]-[%s]' % (opt.model, opt.colorspace))
# initialize data loader
for dataset in opt.datasets:
data_loader = dl.CreateDataLoader(dataset,
dataset_mode=opt.dataset_mode,
batch_size=opt.batch_size,
nThreads=opt.nThreads)
# evaluate model on data
if (opt.dataset_mode == '2afc'):
(score, results_verbose) = dm.score_2afc_dataset(data_loader,
model.forward,
name=dataset)
elif (opt.dataset_mode == 'jnd'):
(score, results_verbose) = dm.score_jnd_dataset(data_loader,
model.forward,
name=dataset)
# print results
print(' Dataset [%s]: %.2f' % (dataset, 100. * score))
parser.add_argument('--batch_size', type=int, default=50, help='batch size to test image patches in')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
parser.add_argument('--model_path', type=str, default=None, help='location of model, will default to ./weights/[net_name].pth')
opt = parser.parse_args()
if(opt.model in ['l2','ssim']):
opt.batch_size = 1
# initialize model
model = dm.DistModel()
model.initialize(model=opt.model,net=opt.net,colorspace=opt.colorspace,model_path=opt.model_path,use_gpu=opt.use_gpu)
if(opt.model in ['net-lin','net']):
print('Testing model [%s]-[%s]'%(opt.model,opt.net))
elif(opt.model in ['l2','ssim']):
print('Testing model [%s]-[%s]'%(opt.model,opt.colorspace))
# embed()
# initialize data loader
for dataset in opt.datasets:
data_loader = dl.CreateDataLoader(dataset,dataset_mode=opt.dataset_mode, batch_size=opt.batch_size)
# evaluate model on data
if(opt.dataset_mode=='2afc'):
(score, results_verbose) = dm.score_2afc_dataset(data_loader,model.forward)
elif(opt.dataset_mode=='jnd'):
(score, results_verbose) = dm.score_jnd_dataset(data_loader,model.forward)
# print results
print(' Dataset [%s]: %.2f'%(dataset,100.*score))
