def test(model, crit, dataset, vocab, opt):
loader = DataLoader(dataset, batch_size=opt.batch_size, shuffle=True)
scorer = COCOScorer()
gt_dataframe = json_normalize(json.load(open(opt.input_json))['sentences'])
gts = convert_data_to_coco_scorer_format(gt_dataframe)
results = []
samples = {}
for data in loader:
# forward the model to get loss
fc_feats = Variable(data['fc_feats']).cuda()
labels = Variable(data['labels']).long().cuda()
with torch.no_grad():
# forward the model to also get generated samples for each image
seq_probs, seq_preds = model(fc_feats, labels, teacher_forcing_ratio=0)
print(seq_preds)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
video_id = 'video' + str(data['ix'][k])
samples[video_id] = [{'image_id': video_id, 'caption': sent}]
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
results.append(valid_score)
print(valid_score)
if not os.path.exists(opt.results_path):
os.makedirs(opt.results_path)
with open(os.path.join(opt.results_path, "scores.txt"), 'a') as scores_table:
scores_table.write(json.dumps(results[0]) + "\n")
with open(os.path.join(opt.results_path, opt.model.split("/")[-1].split('.')[0] + ".json"), 'w') as prediction_results:
json.dump({"predictions": samples, "scores": valid_score}, prediction_results)
def test(model, crit, dataset, vocab, opt):
model.eval()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=False)
scorer = COCOScorer()
gt_dataframe = json_normalize(
json.load(open(opt["input_json"]))['sentences'])
gts = convert_data_to_coco_scorer_format(gt_dataframe)
#results = []
samples = {}
for index, data in enumerate(loader):
print 'batch: ' + str((index + 1) * opt["batch_size"])
# forward the model to get loss
fc_feats = Variable(data['fc_feats'], volatile=True).cuda()
labels = Variable(data['labels'], volatile=True).long().cuda()
masks = Variable(data['masks'], volatile=True).cuda()
video_ids = data['video_ids']
# forward the model to also get generated samples for each image
seq_probs, seq_preds = model(fc_feats, mode='inference', opt=opt)
# print(seq_preds)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
video_id = video_ids[k]
samples[video_id] = [{'image_id': video_id, 'caption': sent}]
# break
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
#results.append(valid_score)
#print(valid_score)
if not os.path.exists(opt["results_path"]):
os.makedirs(opt["results_path"])
result = OrderedDict()
result['checkpoint'] = opt["saved_model"][opt["saved_model"].rfind('/') +
1:]
score_sum = 0
for key, value in valid_score.items():
score_sum += float(value)
result['sum'] = str(score_sum)
#result = OrderedDict(result, **valid_score)
result = OrderedDict(result.items() + valid_score.items())
print result
if not os.path.exists(opt["results_path"]):
os.makedirs(opt["results_path"])
with open(os.path.join(opt["results_path"], "scores.txt"),
'a') as scores_table:
scores_table.write(json.dumps(result) + "\n")
with open(
os.path.join(opt["results_path"],
opt["model"].split("/")[-1].split('.')[0] + ".json"),
'w') as prediction_results:
json.dump({
"predictions": samples,
"scores": valid_score
}, prediction_results)
def test(model, crit, dataset, vocab, device, opt):
model.eval()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True)
scorer = COCOScorer()
gt_dataframe = json_normalize(
json.load(open(opt["input_json"]))['sentences'])
gts = convert_data_to_coco_scorer_format(gt_dataframe)
results = []
samples = {}
for data in loader:
# forward the model to get loss
fc_feats = data['fc_feats'].to(device)
labels = data['labels'].to(device)
masks = data['masks'].to(device)
video_ids = data['video_ids']
if opt["model"] == "S2VTACTModel":
action = data['action'].to(device)
# forward the model to also get generated samples for each image
with torch.no_grad():
if opt["model"] == "S2VTModel":
seq_probs, seq_preds = model(fc_feats,
mode='inference',
opt=opt)
else:
seq_probs, seq_preds = model(fc_feats,
action=action,
device=device,
mode='inference',
opt=opt)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
video_id = video_ids[k]
samples[video_id] = [{'image_id': video_id, 'caption': sent}]
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
results.append(valid_score)
print(valid_score)
if not os.path.exists(opt["results_path"]):
os.makedirs(opt["results_path"])
with open(os.path.join(opt["results_path"], "scores.txt"),
'a') as scores_table:
scores_table.write(json.dumps(results[0]) + "\n")
with open(
os.path.join(opt["results_path"],
opt["model"].split("/")[-1].split('.')[0] + ".json"),
'w') as prediction_results:
json.dump({
"predictions": samples,
"scores": valid_score
}, prediction_results)
def test(model, crit, dataset, vocab, opt):
model.eval()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True)
scorer = COCOScorer()
gt_dataframe = json_normalize(
json.load(open(opt["input_json"]))['sentences'])
gts = convert_data_to_coco_scorer_format(gt_dataframe)
results = []
samples = {}
seq_probs_list = []
seq_preds_list = []
masks_list = []
labels_list = []
for data in loader:
# forward the model to get loss
fc_feats = data['fc_feats'].cuda()
if(opt["with_mean"] == 0):
feats_3d = data['feats_3d'].cuda()
labels = data['labels'].cuda()
masks = data['masks'].cuda()
video_ids = data['video_ids']
# forward the model to also get generated samples for each image
with torch.no_grad():
if(opt["with_mean"] == 1):
seq_probs, seq_preds = model(
fc_feats, mode='inference', opt=opt)
else:
seq_probs, seq_preds = model(
fc_feats, feats_3d, mode='inference', opt=opt)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
video_id = video_ids[k]
samples[video_id] = [{'image_id': video_id, 'caption': sent}]
seq_preds_list.append(seq_preds)
seq_probs_list.append(seq_probs)
masks_list.append(masks)
labels_list.append(labels)
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
results.append(valid_score)
print(valid_score)
seq_probs_list = torch.cat(seq_probs_list, 0)
seq_preds_list = torch.cat(seq_preds_list, 0)
labels_list = torch.cat(labels_list, 0)
masks_list = torch.cat(masks_list, 0)
return valid_score, samples, seq_probs_list, seq_preds_list, labels_list, masks_list
def test(model, crit, dataset, vocab, opt):
model.eval()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True)
scorer = COCOScorer()
gt_dataframe = json_normalize(
json.load(open('data_subset/vatex_subsample_v1.0.json')))
gts = convert_data_to_coco_scorer_format(gt_dataframe, 'chinese')
results = []
samples = {}
for data in loader:
# forward the model to get loss
i3d_feats = data['i3d_feats'].squeeze(1) #.cuda()
labels = data['labels'] #.cuda()
masks = data['masks'] #.cuda()
video_ids = data['video_ids']
# forward the model to also get generated samples for each image
with torch.no_grad():
seq_probs, seq_preds = model(i3d_feats, mode='inference', opt=opt)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
video_id = video_ids[k]
samples[video_id] = [{'image_id': video_id, 'caption': sent}]
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
results.append(valid_score)
print(valid_score)
if not os.path.exists(opt["results_path"]):
os.makedirs(opt["results_path"])
with open(
os.path.join(opt["results_path"],
"chinese_LSTM_OPT_epoch601_scores.txt"),
'a') as scores_table:
scores_table.write(json.dumps(results[0]) + "\n")
with open(
os.path.join(
opt["results_path"],
opt["model"].split("/")[-1].split('.')[0] +
"_chinese_LSTM_OPT_epoch601.json"), 'w') as prediction_results:
json.dump({
"predictions": samples,
"scores": valid_score
},
prediction_results,
indent=2)
def test(model, crit, dataset, vocab, opt):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.eval()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=False)
scorer = COCOScorer()
dataset_meta = json.load(open(opt["dataset_json"]))
vid_to_meta = dataset_meta["vid_to_meta"]
vid_ids = dataset_meta["split_to_ids"]["test"]
# gt_dataframe = json_normalize(json.load(open(opt["dataset_json"]))['sentences'])
gts = convert_data_to_coco_scorer_format(vid_ids, vid_to_meta)
#results = []
samples = {}
with torch.no_grad():
for index, data in enumerate(loader):
print('batch: '+str((index+1)*opt["batch_size"]))
# forward the model to get loss
fc_feats = data['fc_feats'].to(device)
video_id = data['video_ids'].cpu()
# forward the model to also get generated samples for each image
seq_probs, seq_preds = model(fc_feats, mode='inference', opt=opt)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
# Iter through each video in batch and convert id back to original msvd key
vid_key = vid_ids[video_id[k]]
samples[vid_key] = [{'image_id': vid_key, 'caption': sent}]
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
print(valid_score)
def test(model, crit, dataset, vocab, opt, writer):
model.eval()
loss_avg = averager()
writer = SummaryWriter()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=True)
scorer = COCOScorer()
gt_dataframe = json_normalize(
json.load(open(opt["input_json"]))['sentences'])
gts = convert_data_to_coco_scorer_format(gt_dataframe)
results = []
samples = {}
for data in loader:
# forward the model to get loss
fc_feats = data['fc_feats'].cuda()
labels = data['labels'].cuda()
masks = data['masks'].cuda()
video_ids = data['video_ids']
# clip_nums = data['clip_num']
# sorted_clip_nums, indices = torch.sort(clip_nums, descending=True)
# _, desorted_indices = torch.sort(indices, descending=False)
# fc_feats = fc_feats[indices]
# pack = rnn.pack_padded_sequence(fc_feats, sorted_clip_nums, batch_first=True)
# forward the model to also get generated samples for each image
with torch.no_grad():
seq_probs, seq_preds = model(fc_feats, mode='inference', opt=opt)
sents = utils.decode_sequence(vocab, seq_preds)
for k, sent in enumerate(sents):
video_id = video_ids[k]
samples[video_id] = [{'image_id': video_id, 'caption': sent}]
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
results.append(valid_score)
print(valid_score)
def test(model, rem, crit, dataset, vocab, opt):
videos = json.load(open('caption1.json', 'r'))
model.eval()
loader = DataLoader(dataset, batch_size=opt["batch_size"], shuffle=False)
scorer = COCOScorer()
gt_dataframe = json_normalize(
json.load(open(opt["input_json"]))['sentence'])
gts = convert_data_to_coco_scorer_format(gt_dataframe)
results = []
results_f = []
results_avg = []
samples = {}
samples_f = {}
samples_avg = {}
sample_all = {}
for i, data in enumerate(loader):
# forward the model to get loss
fc_feats = data['fc_feats'].cuda()
labels = data['labels'].cuda()
masks = data['masks'].cuda()
video_ids = data['video_ids']
# forward the model to also get generated samples for each image
with torch.no_grad():
seq_probs, seq_preds, en_hn, de_hn = model(fc_feats,
mode='inference',
opt=opt)
fake_en_hn = rem(de_hn, seq_probs)
f_seq_probs, f_seq_preds, __, __ = model(fc_feats,
mode='inference',
h=fake_en_hn,
opt=opt)
avg_en_hn = (en_hn + fake_en_hn) / 2
avg_f_seq_probs, avg_f_seq_preds, __, __ = model(fc_feats,
mode='inference',
h=avg_en_hn,
opt=opt)
sents = utils.decode_sequence(vocab, seq_preds)
f_sents = utils.decode_sequence(vocab, f_seq_preds)
avg_sents = utils.decode_sequence(vocab, avg_f_seq_preds)
for k, sent in enumerate(sents):
video_id = video_ids[k]
samples[video_id] = [{'image_id': video_id, 'captions': sent}]
samples_f[video_id] = [{
'image_id': video_id,
'captions': f_sents[k]
}]
samples_avg[video_id] = [{
'image_id': video_id,
'captions': avg_sents[k]
}]
sample_all[video_id] = [{
'ground truth':
videos[video_id]['captions'],
'caption_origin':
sent,
'caption_fake':
f_sents[k],
'caption_average':
avg_sents[k]
}]
if i > 1:
print(seq_preds.size())
break
with suppress_stdout_stderr():
valid_score = scorer.score(gts, samples, samples.keys())
valid_score_f = scorer.score(gts, samples_f, samples_f.keys())
valid_score_avg = scorer.score(gts, samples_avg, samples_avg.keys())
results.append(valid_score)
results_f.append(valid_score_f)
results_avg.append(valid_score_avg)
print(valid_score)
if not os.path.exists(opt["results_path"]):
os.makedirs(opt["results_path"])
with open(os.path.join(opt["results_path"], "scores2.txt"),
'w') as scores_table:
scores_table.write(json.dumps(results[0]) + "\n")
with open(os.path.join(opt["results_path"], "scores_f2.txt"),
'w') as scores_table:
scores_table.write(json.dumps(results_f[0]) + "\n")
with open(os.path.join(opt["results_path"], "scores_avg2.txt"),
'w') as scores_table:
scores_table.write(json.dumps(results_avg[0]) + "\n")
with open(
os.path.join(opt["results_path"],
opt["model"].split("/")[-1].split('.')[0] + "2.json"),
'w') as prediction_results:
json.dump({
"predictions": samples,
"scores": valid_score
}, prediction_results)
with open(
os.path.join(
opt["results_path"],
opt["model"].split("/")[-1].split('.')[0] + "_f2.json"),
'w') as prediction_results:
json.dump({
"predictions": samples_f,
"scores": valid_score_f
}, prediction_results)
with open(
os.path.join(
opt["results_path"],
opt["model"].split("/")[-1].split('.')[0] + "_avg2.json"),
'w') as prediction_results:
json.dump({
"predictions": samples_avg,
"scores": valid_score_avg
}, prediction_results)
with open('./results/total_caption2.json', 'w') as f:
json.dump({"total": sample_all}, f)
keylist = sent.keys()
res = {}
res2 = {}
for k in keylist:
res[k] = []
res2[k] = []
with suppress_stdout_stderr():
for i in range(B):
samples = {}
for key in keylist:
samples[key] = []
samples[key].append({
'image_id': key,
'caption': sent[key][i]['caption']
})
valid_score, detail_scores = scorer.score(gts, samples, samples.keys())
for k in keylist:
res[k].append(detail_scores[k]['CIDEr'] +
detail_scores[k]['METEOR'])
res2[k].append(detail_scores[k]['METEOR'])
with suppress_stdout_stderr():
samples = {}
for key in keylist:
samples[key] = []
index = np.array(res[key]).argmax()
samples[key].append({
'image_id': key,
'caption': sent[key][index]['caption']
})
valid_score, detail_scores = scorer.score(gts, samples, samples.keys())