def eval_loss_metric(model, resultdict=None): if resultdict is None: resultdict = defaultdict(lambda: defaultdict(lambda: {})) for dataset in datasets: data_loader = dl.CreateDataLoader(dataset, dataset_mode=dataset_mode, batch_size=model.batch_size) # evaluate model on data if (dataset_mode == '2afc'): (score, results_verbose) = dm.score_2afc_dataset(data_loader, model.forward) resultdict[model.model_name][dataset]['score'] = results_verbose[ 'scores'].mean() resultdict[model.model_name][dataset]['std'] = results_verbose[ 'scores'].std() human_judgements = results_verbose['gts'] human_scores = human_judgements**2 + (1 - human_judgements)**2 resultdict['Human'][dataset]['score'] = human_scores.mean() resultdict['Human'][dataset]['std'] = human_scores.std() elif (dataset_mode == 'jnd'): raise Exception('not implemented / validated') (score, results_verbose) = dm.score_jnd_dataset(data_loader, model.forward) # print results print(' Model [%s] Dataset [%s]: %.2f' % (model.model_name, dataset, 100. * score)) return resultdict
use_gpu=opt.use_gpu, pnet_rand=opt.from_scratch, pnet_tune=opt.train_trunk, version=opt.version, gpu_ids=opt.gpu_ids) if (opt.model in ['net-lin', 'net']): print('Testing model [%s]-[%s]' % (opt.model, opt.net)) elif (opt.model in ['l2', 'ssim']): print('Testing model [%s]-[%s]' % (opt.model, opt.colorspace)) # initialize data loader for dataset in opt.datasets: data_loader = dl.CreateDataLoader(dataset, dataset_mode=opt.dataset_mode, batch_size=opt.batch_size, nThreads=opt.nThreads) # evaluate model on data if (opt.dataset_mode == '2afc'): (score, results_verbose) = dm.score_2afc_dataset(data_loader, model.forward, name=dataset) elif (opt.dataset_mode == 'jnd'): (score, results_verbose) = dm.score_jnd_dataset(data_loader, model.forward, name=dataset) # print results print(' Dataset [%s]: %.2f' % (dataset, 100. * score))
parser.add_argument('--batch_size', type=int, default=50, help='batch size to test image patches in') parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU') parser.add_argument('--model_path', type=str, default=None, help='location of model, will default to ./weights/[net_name].pth') opt = parser.parse_args() if(opt.model in ['l2','ssim']): opt.batch_size = 1 # initialize model model = dm.DistModel() model.initialize(model=opt.model,net=opt.net,colorspace=opt.colorspace,model_path=opt.model_path,use_gpu=opt.use_gpu) if(opt.model in ['net-lin','net']): print('Testing model [%s]-[%s]'%(opt.model,opt.net)) elif(opt.model in ['l2','ssim']): print('Testing model [%s]-[%s]'%(opt.model,opt.colorspace)) # embed() # initialize data loader for dataset in opt.datasets: data_loader = dl.CreateDataLoader(dataset,dataset_mode=opt.dataset_mode, batch_size=opt.batch_size) # evaluate model on data if(opt.dataset_mode=='2afc'): (score, results_verbose) = dm.score_2afc_dataset(data_loader,model.forward) elif(opt.dataset_mode=='jnd'): (score, results_verbose) = dm.score_jnd_dataset(data_loader,model.forward) # print results print(' Dataset [%s]: %.2f'%(dataset,100.*score))
# embed() # initialize data loader if __name__ == '__main__': # Windows adaptations # pool = ThreadPool(processes=len(opt.datasets)) # async_result = [None]*len(opt.datasets) # i=0 # j=0 for dataset in opt.datasets: # print ("Running dataset {0}".format(dataset)) # async_result[i] = pool.apply_async(threaded_function, (dataset,)) data_loader = dl.CreateDataLoader(dataset,dataset_mode=opt.dataset_mode, batch_size=opt.batch_size) # evaluate model on data if(opt.dataset_mode=='2afc'): (score, results_verbose) = dm.score_2afc_dataset(opt.alt,data_loader,model.forward) elif(opt.dataset_mode=='jnd'): (score, results_verbose) = dm.score_jnd_dataset(data_loader,model.forward) with open('ResultsFile.txt', 'a') as f: f.write(' Dataset [%s]: %.2f\n'%(dataset,100.*score)) print(' Dataset [%s]: %.2f'%(dataset,100.*score)) # i += 1 # for dataset in opt.datasets: # (score, results_verbose) = async_result[j].get() # with open("ResltsFile.txt", "w") as f: # f.write(' Dataset [%s]: %.2f'%(dataset,100.*score)) # print(' Dataset [%s]: %.2f'%(dataset,100.*score)) # j += 1