def main():
timer = Timer()
timer.start('Load word2vec models...')
vocab = load_vocab(config.VOCAB_DATA)
embeddings = get_trimmed_w2v_vectors(config.W2V_DATA)
timer.stop()
timer.start('Load data...')
train = process_data(opt.train, vocab)
if opt.val is not None:
if opt.val != '1vs9':
validation = process_data(opt.val, vocab)
else:
validation, train = train.one_vs_nine()
else:
validation = None
if opt.test is not None:
test = process_data(opt.test, vocab)
else:
test = None
timer.stop()
timer.start('Build model...')
model = CnnModel(embeddings=embeddings)
model.build()
timer.stop()
timer.start('Train model...')
epochs = opt.e
batch_size = opt.b
early_stopping = True if opt.p != 0 else False
patience = opt.p
pre_train = opt.pre if opt.pre != '' else None
model_name = opt.name
model.train(
model_name,
train=train,
validation=validation,
epochs=epochs,
batch_size=batch_size,
early_stopping=early_stopping,
patience=patience,
cont=pre_train,
)
timer.stop()
if test is not None:
timer.start('Test model...')
preds = model.predict(test, model_name)
labels = test.labels
p, r, f1, _ = precision_recall_fscore_support(labels,
preds,
average='binary')
print('Testing result:P=\t{}\tR={}\tF1={}'.format(p, r, f1))
timer.stop()
def get_data(num=3):
metadata = dataset.process_data(mode="test")
test_metadata = random.sample(metadata, num)
data_list = list()
origin_mel_list = list()
wav_list = list()
durations = list()
for i in range(num):
character = np.load(
"/apdcephfs/share_1213607/zhxliu/DURIAN/durian.daji" +
test_metadata[i][4][1:])
data_list.append(character)
mel = np.load("/apdcephfs/share_1213607/zhxliu/DURIAN/durian.daji" +
test_metadata[i][2][1:])
origin_mel_list.append(mel)
wav_list.append(test_metadata[i][0])
frame_ind = np.load(
"/apdcephfs/share_1213607/zhxliu/DURIAN/durian.daji/" +
test_metadata[i][5][1:])
D = [0 for i in range(character.shape[0])]
for ind in frame_ind:
D[ind] += 1
D = np.array(D)
durations.append(D)
return data_list, origin_mel_list, wav_list, durations
def get_input_examples(data_path,
data_name,
test=False,
input_name='text_a',
target_name='label'):
dataset = process_data(data_path,
data_name,
test,
bert=True,
input_name=input_name,
target_name=target_name)
schemas = get_schemas(data_path)
if test:
dataset.apply(lambda x: process_class(schemas, []),
new_field_name=target_name)
print(dataset[0])
#exit()
return dataset
# dir of pre-computed beliefs
DCB_dir_HR = join(dataset_root, 'DCBs/HR/')
DCB_dir_LR = join(dataset_root, 'DCBs/LR/')
data_name = '{}x{}'.format(hparams.Data.im_w, hparams.Data.im_h)
# bounding box of the target object (for search efficiency evaluation)
bbox_annos = np.load(join(dataset_root,
'coco_search_annos_{}.npy'.format(data_name)),
allow_pickle=True).item()
# load ground-truth human scanpaths
fixation_path = join(dataset_root,
'processed_human_scanpaths_TP_trainval.npy')
human_scanpaths = np.load(fixation_path,
allow_pickle=True,
encoding='latin1')
# exclude incorrect scanpaths
if hparams.Train.exclude_wrong_trials:
human_scanpaths = list(
filter(lambda x: x['correct'] == 1, human_scanpaths))
# process fixation data
dataset = process_data(human_scanpaths, DCB_dir_HR, DCB_dir_LR, bbox_annos,
hparams)
built = build(hparams, True, device, dataset['catIds'])
trainer = Trainer(**built, dataset=dataset, device=device, hparams=hparams)
trainer.train()
# images
print("\nadding images ...\n")
I_rgb = utils.make_grid(real_rgb.data[0:16, :, :, :].clamp(0., 1.),
nrow=4,
normalize=True,
scale_each=True)
I_real = utils.make_grid(real_hyper.data[0:16,
0:3, :, :].clamp(0., 1.),
nrow=4,
normalize=True,
scale_each=True)
I_fake = utils.make_grid(fake_hyper.data[0:16,
0:3, :, :].clamp(0., 1.),
nrow=4,
normalize=True,
scale_each=True)
writer.add_image('rgb', I_rgb, epoch)
writer.add_image('real', I_real, epoch)
writer.add_image('fake', I_fake, epoch)
# save model
if epoch >= opt.epochs - 10:
torch.save(model.state_dict(),
os.path.join(opt.outf, 'net_%03d.pth' % epoch))
if __name__ == "__main__":
if opt.preprocess:
process_data(patch_size=64, stride=40, path='data', mode='train')
process_data(patch_size=None, stride=None, path='data', mode='test')
main()
average_RMSE_G += RMSE_G.item()
average_rRMSE += rRMSE.item()
average_rRMSE_G += rRMSE_G.item()
average_SAM += SAM.item()
print("[%d/%d] RMSE: %.4f RMSE_G: %.4f rRMSE: %.4f rRMSE_G: %.4f SAM: %.4f"
% (i+1, num, RMSE.item(), RMSE_G.item(), rRMSE.item(), rRMSE_G.item(), SAM.item()))
# images
print("adding images ...\n")
I_rgb = real_rgb.data.cpu().numpy().squeeze()
I_hyper = real_hyper.data.cpu().numpy().squeeze()
I_hyper_fake = fake_hyper.data.cpu().numpy().squeeze()
im_rgb['rgb_%d'%i] = I_rgb
im_hyper['hyper_%d'%i] = I_hyper
im_hyper_fake['hyper_fake_%d'%i] = I_hyper_fake
print("\naverage RMSE: %.4f" % (average_RMSE/num))
print("average RMSE_G: %.4f" % (average_RMSE_G/num))
print("\naverage rRMSE: %.4f" % (average_rRMSE/num))
print("average rRMSE_G: %.4f" % (average_rRMSE_G/num))
print("\naverage SAM: %.4f" % (average_SAM/num))
print("\nsaving matlab files ...\n")
scio.savemat(os.path.join(opt.logs, 'im_rgb.mat'), im_rgb)
scio.savemat(os.path.join(opt.logs, 'im_hyper.mat'), im_hyper)
scio.savemat(os.path.join(opt.logs, 'im_hyper_fake.mat'), im_hyper_fake)
if __name__ == "__main__":
if opt.preprocess:
process_data(patch_size=None, stride=None, path='NTIRE2018', mode='test')
main()
'coco_search_annos_{}.npy'.format(data_name)),
allow_pickle=True).item()
# load ground-truth human scanpaths
fixation_path = join(dataset_root, 'processed_human_scanpaths_TP_test.npy')
human_scanpaths = np.load(fixation_path,
allow_pickle=True,
encoding='latin1')
fix_clusters = np.load(join('./data', 'clusters.npy'),
allow_pickle=True).item()
# process fixation data
dataset = process_data(human_scanpaths,
DCB_dir_HR,
DCB_dir_LR,
bbox_annos,
hparams,
is_testing=True)
img_loader = DataLoader(dataset['img_test'],
batch_size=64,
shuffle=False,
num_workers=16)
print('num of test images =', len(dataset['img_test']))
# load trained model
input_size = 134 # number of belief maps
task_eye = torch.eye(len(dataset['catIds'])).to(device)
generator = LHF_Policy_Cond_Small(hparams.Data.patch_count,
len(dataset['catIds']), task_eye,
input_size).to(device)
# dir of pre-computed beliefs
DCB_dir_HR = join(dataset_root, 'DCBs/HR/')
DCB_dir_LR = join(dataset_root, 'DCBs/LR/')
# bounding box of the target object (for search efficiency evaluation)
bbox_annos = np.load(join(dataset_root, 'bbox_annos.npy'),
allow_pickle=True).item()
# load ground-truth human scanpaths
with open(join(dataset_root,
'coco_search18_fixations_TP_train.json')) as json_file:
human_scanpaths_train = json.load(json_file)
with open(join(dataset_root,
'coco_search18_fixations_TP_validation.json')) as json_file:
human_scanpaths_valid = json.load(json_file)
# exclude incorrect scanpaths
if hparams.Train.exclude_wrong_trials:
human_scanpaths_train = list(
filter(lambda x: x['correct'] == 1, human_scanpaths_train))
human_scanpaths_valid = list(
filter(lambda x: x['correct'] == 1, human_scanpaths_valid))
# process fixation data
dataset = process_data(human_scanpaths_train, human_scanpaths_valid,
DCB_dir_HR, DCB_dir_LR, bbox_annos, hparams)
built = build(hparams, True, device, dataset['catIds'])
trainer = Trainer(**built, dataset=dataset, device=device, hparams=hparams)
trainer.train()
print("[epoch %d][%d/%d] Loss: %.4f" %
(epoch, i, len(trainLoader), loss_train))
step += 1
# validate
num = len(testDataset)
avg_loss = 0
for k in range(num):
# data
real_hyper, real_rgb = testDataset[k]
real_hyper = torch.unsqueeze(real_hyper, 0).permute((0, 2, 3, 1))
real_rgb = torch.unsqueeze(real_rgb, 0).permute((0, 2, 3, 1))
real_hyper, real_rgb = Variable(real_hyper.cuda()), Variable(
real_rgb.cuda())
# forward
with torch.no_grad():
fake_rgb = model.forward(real_hyper)
avg_loss += criterion(fake_rgb, real_rgb).item()
writer.add_scalar('Loss_val', avg_loss / num, avg_loss / num)
print("[epoch %d] Validation Loss: %.4f" % (epoch, avg_loss / num))
for param in model.parameters():
writer.writerows(param.data.cpu().numpy().T)
torch.save(model.state_dict(), os.path.join(opt.outf, 'net.pth'))
if __name__ == "__main__":
if opt.preprocess:
process_data(patch_size=64, stride=40, path='NTIRE2018', mode='train')
process_data(patch_size=64, stride=40, path='NTIRE2018', mode='test')
main()
