def train(epoch, dir_fn, label_dict, in_dir):
mlp.train()
with open(dir_fn, "r") as fo:
imnames = map(lambda x: x.strip(), fo.readlines())
perm = np.random.permutation(len(imnames)).tolist()
lt, pred_multi, grt_multi = [], [], []
for i in range(0, len(perm), args.batch_size):
optimizer.zero_grad()
fns = imnames[i:i + args.batch_size]
l, pred, grt = run_net(label_dict, in_dir, fns)
pred_multi.append(pred)
grt_multi.append(grt)
l.backward()
torch.nn.utils.clip_grad_norm(mlp.parameters(), 5.0)
optimizer.step()
lt.append(l.cpu().item())
if len(lt) % args.print_interval == 0:
# save model
pred_multi, grt_multi = np.concatenate(
pred_multi, axis=0), np.concatenate(grt_multi, axis=0)
precision, recall, fscore = get_fscore(
pred_multi >= args.threshold, grt_multi)
pcont = "epoch: %d, train loss: %.3f, precision: %.3f, recall: %.3f, fscore: %.3f" % (
epoch, sum(lt) / len(lt), precision, recall, fscore)
print(pcont)
with open(vs_log_fn, "a+") as fo:
fo.write(pcont + "\n")
lt, pred_multi, grt_multi = [], [], []
return 0
def test(epoch, dir_fn, label_dict, in_dir):
mlp.eval()
with open(dir_fn, "r") as fo:
imnames = map(lambda x: x.strip(), fo.readlines())
lt, pred_multi, grt_multi = [], [], []
for i in range(0, len(imnames), args.test_batch_size):
fns = imnames[i:i + args.test_batch_size]
l, pred, grt = run_net(label_dict, in_dir, fns)
lt.append(l.cpu().item())
pred_multi.append(pred)
grt_multi.append(grt)
pred_multi, grt_multi = np.concatenate(pred_multi,
axis=0), np.concatenate(grt_multi,
axis=0)
precision, recall, fscore = get_fscore(pred_multi >= args.threshold,
grt_multi)
pcont = "epoch: %d, dev loss: %.3f, precision: %.3f, recall: %.3f, fscore: %.3f" % (
epoch, sum(lt) / len(lt), precision, recall, fscore)
print(pcont)
with open(vs_log_fn, "a+") as fo:
fo.write(pcont + "\n")
return fscore
def test(epoch, subset):
network.eval()
ids_fn_sem = ddir.sp_testSem_ids_fn
ids_fn = ddir.sp_dev_ids_fn if subset == "dev" else ddir.sp_test_ids_fn
with open(ids_fn, "r") as fo:
ids = map(lambda x: x.strip(), fo.readlines())
with open(ids_fn_sem, "r") as fo:
ids_sem = map(lambda x: x.strip(), fo.readlines())
lt, pred_multi, grt_multi, pred_multiBoW, ltKW, ltBoW = [], [], [], [], [], []
for i in range(0, len(ids), args.test_batch_size):
idx = ids[i:i + args.test_batch_size]
Xs, _ = data_io.load_mfcc(ddir.mfcc_dir, idx, args.n_pad)
Xs = np.transpose(Xs, (0, 2, 1))
Ys = np.stack(map(lambda x: vision_bow_vec[x[4:-2]], idx), axis=0)
caption_Ys = np.stack(map(lambda x: caption_bow_vec[x], idx), axis=0)
if (args.attn): l, pred, _ = run_net(Xs, Ys)
elif (args.mt): l, lBoW, pred, predBoW = run_net(Xs, Ys, caption_Ys)
else: l, pred = run_net(Xs, Ys)
pred_multi.append(pred)
grt_multi.append(caption_Ys)
if (args.mt):
pred_multiBoW.append(predBoW)
lossTot = args.alpha * l + (1 - args.alpha) * lBoW
ltKW.append(l.cpu().item())
ltBoW.append(lBoW.cpu().item())
else:
lossTot = torch.tensor(l, requires_grad=True)
lt.append(lossTot.cpu().item())
pred_multi, grt_multi = np.concatenate(pred_multi,
axis=0), np.concatenate(grt_multi,
axis=0)
pred_multi = np.concatenate(
(pred_multi,
np.zeros(
(pred_multi.shape[0],
grt_multi.shape[1] - pred_multi.shape[1])).astype(np.float32)),
axis=1)
if (args.mt):
pred_multiBoW = np.concatenate(pred_multiBoW, axis=0)
pred_multiBoW = np.concatenate(
(pred_multiBoW,
np.zeros((pred_multiBoW.shape[0], grt_multi.shape[1] -
pred_multiBoW.shape[1])).astype(np.float32)),
axis=1)
precision, recall, fscore = utils.get_fscore(pred_multi >= args.threshold,
grt_multi)
if (args.mt):
precisionBoW, recallBoW, fscoreBoW = utils.get_fscore(
pred_multiBoW >= args.threshold, grt_multi)
pcont1 = "epoch: %d, dev loss: %.3f" % (epoch, sum(lt) / len(lt))
pcont2 = "dev loss KW: %.3f, precision KW: %.3f, recall KW: %.3f, fscore KW: %.3f" % (
sum(ltKW) / len(ltKW), precision, recall, fscore)
pcont3 = "dev loss BoW: %.3f, precision BoW: %.3f, recall BoW: %.3f, fscore BoW: %.3f" % (
sum(ltBoW) / len(ltBoW), precisionBoW, recallBoW, fscoreBoW)
print(pcont1)
print(pcont2)
print(pcont3)
else:
pcont = "epoch: %d, dev loss: %.3f, precision: %.3f, recall: %.3f, fscore: %.3f" % (
epoch, sum(lt) / len(lt), precision, recall, fscore)
print(pcont)
with open(saveLog, "a+") as fo:
if (args.mt):
fo.write("\n" + pcont1 + "\n" + pcont2 + "\n" + pcont3 + "\n")
else:
fo.write(pcont + "\n")
return fscore
def train(epoch):
network.train()
with open(ddir.sp_train_ids_fn, "r") as fo:
train_ids = map(lambda x: x.strip(), fo.readlines())
perm = np.random.permutation(len(train_ids)).tolist()
perm = range(len(train_ids))
lt, pred_multi, grt_multi, pred_multiBoW, ltKW, ltBoW = [], [], [], [], [], []
for i in range(0, len(perm), args.batch_size):
optimizer.zero_grad()
idx = map(lambda x: train_ids[x], perm[i:i + args.batch_size])
train_Xs, _ = data_io.load_mfcc(ddir.mfcc_dir, idx, args.n_pad)
train_Xs = np.transpose(train_Xs, (0, 2, 1))
train_Ys = np.stack(map(lambda x: vision_bow_vec[x[4:-2]], idx),
axis=0)
caption_Ys = np.stack(map(lambda x: caption_bow_vec[x], idx), axis=0)
if (args.attn): l, pred, _ = run_net(train_Xs, train_Ys)
elif (args.mt):
l, lBoW, pred, predBoW = run_net(train_Xs,
train_Ys,
YsBoW=caption_Ys)
else:
l, pred = run_net(train_Xs, train_Ys)
pred_multi.append(pred)
grt_multi.append(caption_Ys)
if (args.mt):
pred_multiBoW.append(predBoW)
lossTot = args.alpha * l + (1 - args.alpha) * lBoW
ltKW.append(l.cpu().item())
ltBoW.append(lBoW.cpu().item())
else:
lossTot = torch.tensor(l, requires_grad=True)
lossTot.backward()
torch.nn.utils.clip_grad_norm(network.parameters(), 5.0)
optimizer.step()
lt.append(lossTot.cpu().item())
if len(lt) % args.print_interval == 0:
# save model
pred_multi, grt_multi = np.concatenate(
pred_multi, axis=0), np.concatenate(grt_multi, axis=0)
pred_multi = np.concatenate(
(pred_multi,
np.zeros((pred_multi.shape[0], grt_multi.shape[1] -
pred_multi.shape[1])).astype(np.float32)),
axis=1)
if (args.mt):
pred_multiBoW = np.concatenate(pred_multiBoW, axis=0)
pred_multiBoW = np.concatenate(
(pred_multiBoW,
np.zeros((pred_multiBoW.shape[0], grt_multi.shape[1] -
pred_multiBoW.shape[1])).astype(np.float32)),
axis=1)
# pred_multi, grt_multi = np.concatenate(pred_multi, axis=0), np.concatenate(grt_multi, axis=0)
# pred_multi = np.concatenate((pred_multi, np.zeros((pred_multi.shape[0], grt_multi.shape[1]-pred_multi.shape[1])).astype(np.float32)), axis=1)
precision, recall, fscore = get_fscore(
pred_multi >= args.threshold, grt_multi)
if (args.mt):
precisionBoW, recallBoW, fscoreBoW = utils.get_fscore(
pred_multiBoW >= args.threshold, grt_multi)
pcont1 = "epoch: %d, train loss: %.3f" % (epoch,
sum(lt) / len(lt))
pcont2 = "train loss KW: %.3f, precision KW: %.3f, recall KW: %.3f, fscore KW: %.3f" % (
sum(ltKW) / len(ltKW), precision, recall, fscore)
pcont3 = "train loss BoW: %.3f, precision BoW: %.3f, recall BoW: %.3f, fscore BoW: %.3f" % (
sum(ltBoW) / len(ltBoW), precisionBoW, recallBoW,
fscoreBoW)
print(pcont1)
print(pcont2)
print(pcont3)
else:
pcont = "epoch: %d, train loss: %.3f, precision: %.3f, recall: %.3f, fscore: %.3f" % (
epoch, sum(lt) / len(lt), precision, recall, fscore)
print(pcont)
with open(saveLog, "a+") as fo:
if (args.mt):
fo.write("\n" + pcont1 + "\n" + pcont2 + "\n" + pcont3 +
"\n")
else:
fo.write(pcont + "\n")
lt, pred_multi, grt_multi, pred_multiBoW, ltKW, ltBoW = [], [], [], [], [], []
return 0
def test(epoch, subset):
network.eval()
pcont4 = " "
ids_fn_sem = ddir.sp_testSem_ids_fn
ids_fn = ddir.sp_dev_ids_fn if subset == "dev" else ddir.sp_test_ids_fn
with open(ids_fn, "r") as fo:
ids = map(lambda x: x.strip(), fo.readlines())
with open(ids_fn_sem, "r") as fo:
ids_sem = map(lambda x: x.strip(), fo.readlines())
lt, pred_multi, grt_multi, pred_multiBoW, ltKW, ltBoW, pred_multi_kw, grt_multi_kw = [], [], [], [], [], [], [], []
for i in range(0, len(ids), args.test_batch_size):
idx = ids[i:i + args.test_batch_size]
Xs, _ = data_io.load_mfcc(ddir.mfcc_dir, idx, args.n_pad)
Xs = np.transpose(Xs, (0, 2, 1))
Ys = np.stack(map(lambda x: vision_bow_vec[x[4:-2]], idx), axis=0)
caption_Ys = np.stack(map(lambda x: caption_bow_vec[x], idx), axis=0)
if (args.mt): l, lBoW, pred, predBoW, _ = run_net(Xs, Ys, caption_Ys)
else: l, pred, _ = run_net(Xs, Ys)
pred_multi.append(pred)
grt_multi.append(caption_Ys)
if (args.mt):
pred_multiBoW.append(predBoW)
lossTot = args.alpha * l + (1 - args.alpha) * lBoW
ltKW.append(l.cpu().item())
ltBoW.append(lBoW.cpu().item())
else:
lossTot = torch.tensor(l, requires_grad=True)
lt.append(lossTot.cpu().item())
# if('taslp' in args.dataset):
predMapped = (pred.T[mapping]).T
pred_multi_kw.append(predMapped)
# pdb.set_trace()
grtMapped = (caption_Ys.T[mapping1]).T
grt_multi_kw.append(grtMapped)
pred_multi, grt_multi = np.concatenate(pred_multi,
axis=0), np.concatenate(grt_multi,
axis=0)
pred_multi = np.concatenate(
(pred_multi,
np.zeros(
(pred_multi.shape[0],
grt_multi.shape[1] - pred_multi.shape[1])).astype(np.float32)),
axis=1)
# if('taslp' in args.dataset):
pred_multi_kw, grt_multi_kw = np.concatenate(
pred_multi_kw, axis=0), np.concatenate(grt_multi_kw, axis=0)
pred_multi_kw = np.concatenate(
(pred_multi_kw,
np.zeros((pred_multi_kw.shape[0], grt_multi_kw.shape[1] -
pred_multi_kw.shape[1])).astype(np.float32)),
axis=1)
precision1, recall1, fscore1 = utils.get_fscore(
pred_multi_kw >= args.threshold, grt_multi_kw)
if (epoch == args.epoch - 1 or subset == "test"):
ap1 = utils.get_metrics(pred_multi, grt_multi, flag='onlyAP')
eer, ap, prec10, precN = utils.get_metrics(pred_multi_kw, grt_multi_kw)
pcont4 = "Overall ratings: EER: %f, Average precision: %f, Average precision on all keywords: %f, Precision@10: %f, Precision@N: %f" % (
eer, ap, ap1, prec10, precN)
with open("tempcsvap.csv", "a+") as fo:
fo.write(args.mtType + ',' + str(args.alpha) + ',' +
str(ap1 * 100) + ',' + str(ap * 100) + '\n')
if (args.mt):
pred_multiBoW = np.concatenate(pred_multiBoW, axis=0)
pred_multiBoW = np.concatenate(
(pred_multiBoW,
np.zeros((pred_multiBoW.shape[0], grt_multi.shape[1] -
pred_multiBoW.shape[1])).astype(np.float32)),
axis=1)
precision, recall, fscore = utils.get_fscore(pred_multi >= args.threshold,
grt_multi)
if (args.mt):
precisionBoW, recallBoW, fscoreBoW = utils.get_fscore(
pred_multiBoW >= args.threshold, grt_multi)
pcont1 = "epoch: %d, dev loss: %.3f" % (epoch, sum(lt) / len(lt))
pcont2 = "dev loss KW: %.3f, precision KW: %.3f, recall KW: %.3f, fscore KW: %.3f" % (
sum(ltKW) / len(ltKW), precision, recall, fscore)
pcont3 = "dev loss BoW: %.3f, precision BoW: %.3f, recall BoW: %.3f, fscore BoW: %.3f" % (
sum(ltBoW) / len(ltBoW), precisionBoW, recallBoW, fscoreBoW)
print(pcont1)
print(pcont2)
print(pcont3)
else:
pcont = "epoch: %d, dev loss: %.3f, precision: %.3f, recall: %.3f, fscore: %.3f" % (
epoch, sum(lt) / len(lt), precision, recall, fscore)
print(pcont)
print(pcont4)
with open(saveLog, "a+") as fo:
if (args.mt):
fo.write("\n" + pcont1 + "\n" + pcont2 + "\n" + pcont3 + "\n" +
pcont4 + "\n")
else:
fo.write(pcont + "\n" + pcont4 + "\n")
# with open(saveLog, "a+") as fo:
# if(args.mt): fo.write("\n"+pcont1+"\n"+pcont2+"\n"+pcont3+"\n")
# else: fo.write(pcont+"\n")
# return fscore
return fscore1