def find_NN(self, images, query, measure, k, npts=None):
"""
Text->Images (Image Search)
Images: (5N, K) matrix of images
query: (1, K) matrix of captions
"""
if npts is None:
npts = images.shape[0] / 5
ims = numpy.array([images[i] for i in range(0, len(images), 5)])
# Compute scores
tic = time.clock()
if measure == 'order':
d2 = order_sim(
torch.Tensor(ims).cuda(),
torch.Tensor(query).cuda())
d2 = d2.cpu().numpy()
d = d2.T
else:
d = numpy.dot(
query, ims.T
) # TODO Try to optimize this computation, see if sorting is bottleneck
imgs_by_similarity = numpy.argsort(numpy.squeeze(d))[::-1]
toc = time.clock()
print('NN search took {} ms over {} images'.format(
(toc - tic) * 1000.0, ims.shape[0]))
return imgs_by_similarity[0:k] * 5, query, ims
def t2i(images,
captions,
images2,
captions2,
npts=None,
measure='cosine',
return_ranks=False):
"""
Text->Images (Image Search)
Images: (5N, K) matrix of images
Captions: (5N, K) matrix of captions
"""
if npts is None:
npts = images.shape[0] / 5
ims = numpy.array([images[i] for i in range(0, len(images), 5)])
ims2 = numpy.array([images2[i] for i in range(0, len(images2), 5)])
ranks = numpy.zeros(5 * npts)
top1 = numpy.zeros(5 * npts)
for index in range(npts):
# Get query captions
queries = captions[5 * index:5 * index + 5]
queries2 = captions2[5 * index:5 * index + 5]
# Compute scores
if measure == 'order':
bs = 100
if 5 * index % bs == 0:
mx = min(captions.shape[0], 5 * index + bs)
q2 = captions[5 * index:mx]
d2 = order_sim(
torch.Tensor(ims).cuda(),
torch.Tensor(q2).cuda())
d2 = d2.cpu().numpy()
d = d2[:, (5 * index) % bs:(5 * index) % bs + 5].T
else:
d = numpy.dot(queries, ims.T)
d2 = numpy.dot(queries2, ims2.T)
d = (d + d2) / 2
inds = numpy.zeros(d.shape)
for i in range(len(inds)):
inds[i] = numpy.argsort(d[i])[::-1]
ranks[5 * index + i] = numpy.where(inds[i] == index)[0][0]
top1[5 * index + i] = inds[i][0]
# Compute metrics
r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
medr = numpy.floor(numpy.median(ranks)) + 1
meanr = ranks.mean() + 1
if return_ranks:
return (r1, r5, r10, medr, meanr), (ranks, top1)
else:
return (r1, r5, r10, medr, meanr)
def i2t(images,
captions,
npts=None,
n=5,
measure='cosine',
return_ranks=False):
"""
Images->Text (Image Annotation)
Images: (5N, K) matrix of images
Captions: (5N, K) matrix of captions
"""
if npts is None:
npts = images.shape[0] / n
index_list = []
ranks = numpy.zeros(npts)
top1 = numpy.zeros(npts)
for index in range(npts):
# Get query image
im = images[n * index].reshape(1, images.shape[1])
# Compute scores
if measure == 'order':
bs = 100
if index % bs == 0:
mx = min(images.shape[0], n * (index + bs))
im2 = images[n * index:mx:n]
d2 = order_sim(
torch.Tensor(im2).cuda(),
torch.Tensor(captions).cuda())
d2 = d2.cpu().numpy()
d = d2[index % bs]
else:
d = numpy.dot(im, captions.T).flatten()
inds = numpy.argsort(d)[::-1]
index_list.append(inds[0])
# Score
rank = 1e20
for i in range(n * index, n * index + n, 1):
tmp = numpy.where(inds == i)[0][0]
if tmp < rank:
rank = tmp
ranks[index] = rank
top1[index] = inds[0]
# Compute metrics
r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
medr = numpy.floor(numpy.median(ranks)) + 1
meanr = ranks.mean() + 1
if return_ranks:
return (r1, r5, r10, medr, meanr), (ranks, top1)
else:
return (r1, r5, r10, medr, meanr)
def i2t_split(images,
captions_orig,
captions_ex,
npts=None,
measure='cosine',
return_ranks=False):
if npts is None:
npts = images.shape[0] // 5
index_list = []
ranks = numpy.zeros(npts)
top1 = numpy.zeros(npts)
for index in range(npts):
# Get query image
im = images[5 * index].reshape(1, images.shape[1])
captions_ex_ind = [captions_orig
] + [captions_ex[5 * index + j] for j in range(5)]
captions = np.concatenate(captions_ex_ind, axis=0)
# Compute scores
if measure == 'order':
bs = 100
if index % bs == 0:
mx = min(images.shape[0], 5 * (index + bs))
im2 = images[5 * index:mx:5]
d2 = order_sim(
torch.Tensor(im2).cuda(),
torch.Tensor(captions).cuda())
d2 = d2.cpu().numpy()
d = d2[index % bs]
else:
d = numpy.dot(im, captions.T).flatten()
# print(captions.shape)
inds = numpy.argsort(d)[::-1]
index_list.append(inds[0])
# Score
rank = 1e20
for i in range(5 * index, 5 * index + 5, 1):
tmp = numpy.where(inds == i)[0][0]
if tmp < rank:
rank = tmp
ranks[index] = rank
top1[index] = inds[0]
# Compute metrics
r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
medr = numpy.floor(numpy.median(ranks)) + 1
meanr = ranks.mean() + 1
if return_ranks:
return (r1, r5, r10, medr, meanr), (ranks, top1)
else:
return r1, r5, r10, medr, meanr
def i2t(videos,
captions,
videos2,
captions2,
npts=None,
measure='cosine',
return_ranks=False):
"""
videos->Text (video Annotation)
videos: (5N, K) matrix of videos
Captions: (5N, K) matrix of captions
"""
if npts is None:
npts = videos.shape[0] / 5
index_list = []
ranks = numpy.zeros(npts)
top1 = numpy.zeros(npts)
for index in range(npts):
# Get query video
vid = videos[5 * index].reshape(1, videos.shape[1])
vid_2 = videos2[5 * index].reshape(1, videos2.shape[1])
# Compute scores
if measure == 'order':
bs = 100
if index % bs == 0:
mx = min(videos.shape[0], 5 * (index + bs))
vid2 = videos[5 * index:mx:5]
d2 = order_sim(
torch.Tensor(vid2).cuda(),
torch.Tensor(captions).cuda())
d2 = d2.cpu().numpy()
d = d2[index % bs]
else:
d = numpy.dot(vid, captions.T).flatten()
d2 = numpy.dot(vid_2, captions2.T).flatten()
d = (d + d2) / 2
inds = numpy.argsort(d)[::-1]
index_list.append(inds[0])
# Score
rank = 1e20
for i in range(5 * index, 5 * index + 5, 1):
tmp = numpy.where(inds == i)[0][0]
if tmp < rank:
rank = tmp
ranks[index] = rank
top1[index] = inds[0]
# Compute metrics
r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks)
r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks)
r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks)
medr = numpy.floor(numpy.median(ranks)) + 1
meanr = ranks.mean() + 1
if return_ranks:
return (r1, r5, r10, medr, meanr), (ranks, top1)
else:
return (r1, r5, r10, medr, meanr)
