topK_correct_num = np.zeros(K, dtype=np.float32)
total_num = 0
sample_im = num_im
for n_im in range(sample_im):
print('testing image %d / %d' % (n_im, num_im))
imname = imlist[n_im]
# gt
imcrop_names = imcrop_dict[imname]
candidate_boxes = candidate_boxes_dict[imname]
im = skimage.io.imread(image_dir + imname + '.jpg')
imsize = np.array([im.shape[1], im.shape[0]]) # [width, height]
# Compute local descriptors (local image feature + spatial feature)
descriptors = retriever.compute_descriptors_edgebox(
captioner, im, candidate_boxes, 'fc7') # (100,4096)
spatial_feats = retriever.compute_spatial_feat(candidate_boxes,
imsize) # (100,8)
np.savez('./data/ReferIt/referit_proposal_feature/' + imname,
spatial_feat=spatial_feats,
local_feature=descriptors)
# print intermediate results during testing
if (n_im + 1) % 1000 == 0:
print('Recall on first %d test images' % (n_im + 1))
for k in [0, 10 - 1]:
print('\trecall @ %d = %f' %
(k + 1, topK_correct_num[k] / total_num))
print('Final recall on the whole test set')
for k in [0, 10 - 1]:
query = raw_input("type the input query: ")
#query = 'bike on the red house'
print("query =", query)
print("Find best candidate..!")
for i in range(8):
im_file = './splited_image/test' + str(i) + '.jpg'
edgebox_file = './proposal_box/selective_box' + str(
i) + '.txt' # pre-extracted EdgeBox proposals
im = skimage.io.imread(im_file)
imsize = np.array([im.shape[1], im.shape[0]]) # [width, height]
candidate_boxes = np.loadtxt(edgebox_file).astype(int)
candidate_boxes = np.reshape(candidate_boxes, (-1, 4))
# Compute features
region_feature = retriever.compute_descriptors_edgebox(
captioner, im, candidate_boxes)
spatial_feature = retriever.compute_spatial_feat(
candidate_boxes, imsize)
descriptors = np.concatenate((region_feature, spatial_feature), axis=1)
context_feature = captioner.compute_descriptors([im],
output_name='fc7')
# Compute scores of each candidate region
scores = retriever.score_descriptors_context(descriptors, query,
context_feature,
captioner, vocab_dict)
#candidate_boxes = (i, candidate_boxes)
candidate_boxes = np.insert(candidate_boxes, 0, i, axis=1)
if (i == 0):
sum_candidate_box = candidate_boxes
else:
################################################################################
# Test recall
K = 100 # evaluate recall at 1, 2, ..., K
topK_correct_num = np.zeros(K, dtype=np.float32)
total_num = 0
for n_im in range(num_im):
print('testing image %d / %d' % (n_im, num_im))
imname = imlist[n_im]
imcrop_names = imcrop_dict[imname]
candidate_boxes = candidate_boxes_dict[imname]
im = skimage.io.imread(image_dir + imname + '.jpg')
imsize = np.array([im.shape[1], im.shape[0]]) # [width, height]
# Compute local descriptors (local image feature + spatial feature)
descriptors = retriever.compute_descriptors_edgebox(captioner, im,
candidate_boxes)
spatial_feats = retriever.compute_spatial_feat(candidate_boxes, imsize)
descriptors = np.concatenate((descriptors, spatial_feats), axis=1)
num_imcrop = len(imcrop_names)
num_proposal = candidate_boxes.shape[0]
for n_imcrop in range(num_imcrop):
imcrop_name = imcrop_names[n_imcrop]
if imcrop_name not in query_dict:
continue
gt_bbox = np.array(imcrop_bbox_dict[imcrop_name])
IoUs = retriever.compute_iou(candidate_boxes, gt_bbox)
for n_sentence in range(len(query_dict[imcrop_name])):
sentence = query_dict[imcrop_name][n_sentence]
# Scores for each candidate region
if use_context: