def main():
args = parser.parse_args()
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
# select device
torch.cuda.set_device(args.gpu_id)
print('[Info] use gpu: {}'.format(torch.cuda.current_device()))
# get parameters
sys.path.insert(0, args.model_dir)
from params import params
assert len(params) > 1
last_cfg = params[0][-1]
last_cfg()
get_data.main()
dataset = VQADataset('test', params[0][1])
itoa = dataset.codebook['itoa']
vote_buff = [{} for i in range(len(dataset))]
conf_buff = np.zeros((len(dataset), len(itoa)))
sm_conf_buff = np.zeros((len(dataset), len(itoa)))
l2_conf_buff = np.zeros((len(dataset), len(itoa)))
que_ids = dataset.que_id
for fpath, mgrp, mname, acc, cfg_func, in params:
# data
if cfg_func != last_cfg:
cfg_func()
get_data.main()
last_cfg = cfg_func
dataset = VQADataset('test', mgrp)
itoa = dataset.codebook['itoa']
dataset.reload_obj(mgrp)
dataloader = torch.utils.data.DataLoader(
dataset, batch_size=args.bs, shuffle=False,
num_workers=2, pin_memory=True)
# model
model_group = import_module('models.' + mgrp)
model = getattr(model_group, mname)
num_words=dataset.num_words,
num_ans=dataset.num_ans,
emb_size=get_emb_size())
cp_file = os.path.join(args.model_dir, fpath)
checkpoint = torch.load(cp_file, map_location=lambda s, l: s.cuda(0))
model.load_state_dict(checkpoint['state_dict'])
model.cuda()
model.eval()
# predict
bar = progressbar.ProgressBar()
start = 0
# sample: (que_id, img, que, [obj])
for sample in bar(dataloader):
sample_var = [Variable(d).cuda() for d in list(sample)[1:]]
score = model(*sample_var)
sm_score = torch.nn.functional.softmax(score)
l2_score = torch.nn.functional.normalize(score)
bs = score.size(0)
conf_buff[start:start+bs] += score.data.cpu().numpy()
sm_conf_buff[start:start+bs] += sm_score.data.cpu().numpy()
l2_conf_buff[start:start+bs] += l2_score.data.cpu().numpy()
_, ans_ids = torch.max(score.data, dim=1)
for i, ans_id in enumerate(ans_ids):
ans = itoa[ans_id]
ans_score = acc + vote_buff[start + i].get(ans, 0)
vote_buff[start + i][ans] = ans_score
start += bs
def __init__(self, model_info, split, save_dir):
assert len(model_info) > 0
assert len(cfg.TEST.SPLITS) == 1 and cfg.TEST.SPLITS[0] == split
model_info = sorted(model_info, key=itemgetter(0))
self._split = split
self.model_info = model_info
self.save_dir = save_dir
# load model
self._pred_ans = []
self._scores = []
self._att_weights = []
dataset = VQADataset('test', model_info[0][0])
emb_size = get_emb_size()
for model_group_name, model_name, cp_file in model_info:
cache_file = cp_file + '.cache'
if os.path.isfile(cache_file):
print("load from cache: '{}".format(cache_file))
cache = pickle.load(open(cache_file, 'rb'))
self._pred_ans.append(cache['pred_ans'])
self._scores.append(cache['scores'])
self._att_weights.append(cache['att_weights'])
continue
# dataset
dataset.reload_obj(model_group_name)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=args.bs,
shuffle=False,
num_workers=2,
pin_memory=True)
# model
model_group = import_module('models.' + model_group_name)
model = getattr(model_group,
model_name)(num_words=dataset.num_words,
num_ans=dataset.num_ans,
emb_size=emb_size)
checkpoint = torch.load(cp_file,
map_location=lambda s, l: s.cuda(0))
model.load_state_dict(checkpoint['state_dict'])
model.cuda()
model.eval()
# predicting
itoa = dataloader.dataset.codebook['itoa']
batch_att_weight = []
pred_ans = []
bar = progressbar.ProgressBar()
print('predicting answers...')
# sample: (que_id, img, que, [obj])
for sample in bar(dataloader):
# setting hook
att_weight_buff = torch.FloatTensor(len(sample[0]), 36)
def get_weight(self, input, output):
att_weight_buff.copy_(output.data.view_as(att_weight_buff))
hook = model.att_net.register_forward_hook(get_weight)
# forward
sample_var = [Variable(d).cuda() for d in list(sample)[1:]]
score = model(*sample_var)
att_weight = F.softmax(Variable(att_weight_buff)).data.numpy()
batch_att_weight.append(att_weight)
pred_ans.extend(format_result(sample[0], score, itoa))
hook.remove()
att_weights = np.vstack(batch_att_weight)
# evaluation
print('evaluting results...')
if split in ('train2014', 'val2014'):
vqa_eval = get_eval(pred_ans, split)
scores = []
for i in range(len(dataset)):
qid = int(dataset[i][0])
score = vqa_eval.evalQA.get(qid)
scores.append(score)
else:
scores = None
self._pred_ans.append(pred_ans)
self._scores.append(scores)
self._att_weights.append(att_weights)
# save cache
cache = {}
cache['pred_ans'] = pred_ans
cache['scores'] = scores
cache['att_weights'] = att_weights
pickle.dump(cache, open(cache_file, 'wb'))
print('done.')
# load data
print('load raw data...')
split_fname = '{}/raw-{}.json'.format(cfg.DATA_DIR, split)
self._data = json.load(open(split_fname))
print('load boxes...')
self._boxes = self._load_box()
# query key
self._question = None
self._answer = None
self._condition = None
# query result
self._r_question = None
self._r_answer = None
self._r_condition = None
# dirty flag
self._d_question = True
self._d_answer = True
self._d_condition = True
self.last_results = None