def eval_loss_metric(model, resultdict=None):
if resultdict is None:
resultdict = defaultdict(lambda: defaultdict(lambda: {}))
for dataset in datasets:
data_loader = dl.CreateDataLoader(dataset,
dataset_mode=dataset_mode,
batch_size=model.batch_size)
# evaluate model on data
if (dataset_mode == '2afc'):
(score,
results_verbose) = dm.score_2afc_dataset(data_loader,
model.forward)
resultdict[model.model_name][dataset]['score'] = results_verbose[
'scores'].mean()
resultdict[model.model_name][dataset]['std'] = results_verbose[
'scores'].std()
human_judgements = results_verbose['gts']
human_scores = human_judgements**2 + (1 - human_judgements)**2
resultdict['Human'][dataset]['score'] = human_scores.mean()
resultdict['Human'][dataset]['std'] = human_scores.std()
elif (dataset_mode == 'jnd'):
raise Exception('not implemented / validated')
(score,
results_verbose) = dm.score_jnd_dataset(data_loader,
model.forward)
# print results
print(' Model [%s] Dataset [%s]: %.2f' %
(model.model_name, dataset, 100. * score))
return resultdict
model_path=opt.model_path,
use_gpu=opt.use_gpu,
pnet_rand=opt.from_scratch,
pnet_tune=opt.train_trunk,
version=opt.version,
gpu_ids=opt.gpu_ids)
if (opt.model in ['net-lin', 'net']):
print('Testing model [%s]-[%s]' % (opt.model, opt.net))
elif (opt.model in ['l2', 'ssim']):
print('Testing model [%s]-[%s]' % (opt.model, opt.colorspace))
# initialize data loader
for dataset in opt.datasets:
data_loader = dl.CreateDataLoader(dataset,
dataset_mode=opt.dataset_mode,
batch_size=opt.batch_size,
nThreads=opt.nThreads)
# evaluate model on data
if (opt.dataset_mode == '2afc'):
(score, results_verbose) = dm.score_2afc_dataset(data_loader,
model.forward,
name=dataset)
elif (opt.dataset_mode == 'jnd'):
(score, results_verbose) = dm.score_jnd_dataset(data_loader,
model.forward,
name=dataset)
# print results
print(' Dataset [%s]: %.2f' % (dataset, 100. * score))
os.mkdir(opt.save_dir)
# initialize model
model = dm.DistModel()
model.initialize(model=opt.model,
net=opt.net,
use_gpu=opt.use_gpu,
is_train=True,
pnet_rand=opt.from_scratch,
pnet_tune=opt.train_trunk,
gpu_ids=opt.gpu_ids)
# load data from all training sets
data_loader = dl.CreateDataLoader(opt.datasets,
dataset_mode='2afc',
batch_size=opt.batch_size,
serial_batches=False,
nThreads=opt.nThreads)
dataset = data_loader.load_data()
dataset_size = len(data_loader)
D = len(dataset)
print('Loading %i instances from' % dataset_size, opt.datasets)
visualizer = Visualizer(opt)
total_steps = 0
fid = open(os.path.join(opt.checkpoints_dir, opt.name, 'train_log.txt'), 'w+')
for epoch in range(1, opt.nepoch + opt.nepoch_decay + 1):
epoch_start_time = time.time()
for i, data in enumerate(dataset):
iter_start_time = time.time()
total_steps += opt.batch_size
is_cuda = torch.cuda.is_available()
device = "cuda" if is_cuda else "cpu"
# load dataset
datasets = [
"val/traditional",
"val/cnn",
"val/superres",
"val/deblur",
"val/color",
"val/frameinterp",
]
dataset_mode = "2afc"
data_loader = dl.CreateDataLoader(
datasets,
dataroot="../perceptual_sim_training/dataset",
dataset_mode=dataset_mode,
batch_size=1,
).load_data()
# load functions to evaluate
lp = LossProvider()
metrics = {}
metrics["Watson-fft"] = lp.get_loss_function("Watson-fft",
reduction="sum").to(device)
metrics["L2"] = lp.get_loss_function("L2", reduction="sum").to(device)
metrics["SSIM"] = lp.get_loss_function("SSIM", reduction="sum").to(device)
metrics["Deeploss-vgg"] = lp.get_loss_function("Deeploss-vgg",
reduction="sum").to(device)
# set-up output file
out_data = defaultdict(lambda: [])
opt.save_dir = os.path.join(opt.checkpoints_dir,opt.name)
if(not os.path.exists(opt.save_dir)):
os.mkdir(opt.save_dir)
print(opt)
psnr_df = pd.read_csv(os.path.join(opt.csv_path, 'psnr.csv'), index_col=0)
psnrs = psnr_df.to_numpy()
# initialize model
trainer = lpips.Trainer()
trainer.initialize(model=opt.model, net=opt.net, use_gpu=opt.use_gpu, is_train=True,
pnet_rand=opt.from_scratch, pnet_tune=opt.train_trunk, gpu_ids=opt.gpu_ids, lr=opt.lr)
# load data from all training sets
data_loader = dl.CreateDataLoader(opt, opt.datasets,dataset_mode=opt.dataset_mode, load_size=opt.load_size, batch_size=opt.batch_size, serial_batches=False, nThreads=opt.nThreads)
dataset = data_loader.load_data()
dataset_size = len(data_loader)
D = len(dataset)
print('Loading %i instances from'%dataset_size,opt.datasets)
visualizer = Visualizer(opt)
if opt.dataset_mode == 'tnn':
# data_loader_val = dl.CreateDataLoader(opt,'val', dataset_mode=opt.dataset_mode, load_size=opt.load_size, batch_size=opt.batch_size, serial_batches=False, nThreads=opt.nThreads)
data_loader_test = dl.CreateDataLoader(opt, 'test', dataset_mode=opt.dataset_mode, load_size=opt.load_size, batch_size=opt.batch_size, nThreads=opt.nThreads)
total_steps = 0
fid = open(os.path.join(opt.checkpoints_dir,opt.name,'train_log.txt'),'w+')
if opt.dataset_mode == 'tnn':
trainer.set_eval()
# initialize model
model = dm.DistModel()
# model.initialize(model=opt.model,net=opt.net,use_gpu=opt.use_gpu, is_train=True)
model.initialize(
model=opt.model,
net=opt.net,
use_gpu=opt.use_gpu,
is_train=True,
pnet_rand=opt.from_scratch,
pnet_tune=opt.train_trunk,
)
# load data from all training sets
data_loader = dl.CreateDataLoader(
opt.datasets, dataset_mode="2afc", batch_size=opt.batch_size, serial_batches=False
)
dataset = data_loader.load_data()
dataset_size = len(data_loader)
D = len(dataset)
print("Loading %i instances from" % dataset_size, opt.datasets)
visualizer = Visualizer(opt)
total_steps = 0
fid = open(os.path.join(opt.checkpoints_dir, opt.name, "train_log.txt"), "w+")
for epoch in range(1, opt.nepoch + opt.nepoch_decay + 1):
epoch_start_time = time.time()
for i, data in enumerate(dataset):
iter_start_time = time.time()
total_steps += opt.batch_size
epoch_iter = total_steps - dataset_size * (epoch - 1)
