def find_NN(self, images, query, measure, k, npts=None): """ Text->Images (Image Search) Images: (5N, K) matrix of images query: (1, K) matrix of captions """ if npts is None: npts = images.shape[0] / 5 ims = numpy.array([images[i] for i in range(0, len(images), 5)]) # Compute scores tic = time.clock() if measure == 'order': d2 = order_sim( torch.Tensor(ims).cuda(), torch.Tensor(query).cuda()) d2 = d2.cpu().numpy() d = d2.T else: d = numpy.dot( query, ims.T ) # TODO Try to optimize this computation, see if sorting is bottleneck imgs_by_similarity = numpy.argsort(numpy.squeeze(d))[::-1] toc = time.clock() print('NN search took {} ms over {} images'.format( (toc - tic) * 1000.0, ims.shape[0])) return imgs_by_similarity[0:k] * 5, query, ims
def t2i(images, captions, images2, captions2, npts=None, measure='cosine', return_ranks=False): """ Text->Images (Image Search) Images: (5N, K) matrix of images Captions: (5N, K) matrix of captions """ if npts is None: npts = images.shape[0] / 5 ims = numpy.array([images[i] for i in range(0, len(images), 5)]) ims2 = numpy.array([images2[i] for i in range(0, len(images2), 5)]) ranks = numpy.zeros(5 * npts) top1 = numpy.zeros(5 * npts) for index in range(npts): # Get query captions queries = captions[5 * index:5 * index + 5] queries2 = captions2[5 * index:5 * index + 5] # Compute scores if measure == 'order': bs = 100 if 5 * index % bs == 0: mx = min(captions.shape[0], 5 * index + bs) q2 = captions[5 * index:mx] d2 = order_sim( torch.Tensor(ims).cuda(), torch.Tensor(q2).cuda()) d2 = d2.cpu().numpy() d = d2[:, (5 * index) % bs:(5 * index) % bs + 5].T else: d = numpy.dot(queries, ims.T) d2 = numpy.dot(queries2, ims2.T) d = (d + d2) / 2 inds = numpy.zeros(d.shape) for i in range(len(inds)): inds[i] = numpy.argsort(d[i])[::-1] ranks[5 * index + i] = numpy.where(inds[i] == index)[0][0] top1[5 * index + i] = inds[i][0] # Compute metrics r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks) r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks) r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks) medr = numpy.floor(numpy.median(ranks)) + 1 meanr = ranks.mean() + 1 if return_ranks: return (r1, r5, r10, medr, meanr), (ranks, top1) else: return (r1, r5, r10, medr, meanr)
def i2t(images, captions, npts=None, n=5, measure='cosine', return_ranks=False): """ Images->Text (Image Annotation) Images: (5N, K) matrix of images Captions: (5N, K) matrix of captions """ if npts is None: npts = images.shape[0] / n index_list = [] ranks = numpy.zeros(npts) top1 = numpy.zeros(npts) for index in range(npts): # Get query image im = images[n * index].reshape(1, images.shape[1]) # Compute scores if measure == 'order': bs = 100 if index % bs == 0: mx = min(images.shape[0], n * (index + bs)) im2 = images[n * index:mx:n] d2 = order_sim( torch.Tensor(im2).cuda(), torch.Tensor(captions).cuda()) d2 = d2.cpu().numpy() d = d2[index % bs] else: d = numpy.dot(im, captions.T).flatten() inds = numpy.argsort(d)[::-1] index_list.append(inds[0]) # Score rank = 1e20 for i in range(n * index, n * index + n, 1): tmp = numpy.where(inds == i)[0][0] if tmp < rank: rank = tmp ranks[index] = rank top1[index] = inds[0] # Compute metrics r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks) r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks) r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks) medr = numpy.floor(numpy.median(ranks)) + 1 meanr = ranks.mean() + 1 if return_ranks: return (r1, r5, r10, medr, meanr), (ranks, top1) else: return (r1, r5, r10, medr, meanr)
def i2t_split(images, captions_orig, captions_ex, npts=None, measure='cosine', return_ranks=False): if npts is None: npts = images.shape[0] // 5 index_list = [] ranks = numpy.zeros(npts) top1 = numpy.zeros(npts) for index in range(npts): # Get query image im = images[5 * index].reshape(1, images.shape[1]) captions_ex_ind = [captions_orig ] + [captions_ex[5 * index + j] for j in range(5)] captions = np.concatenate(captions_ex_ind, axis=0) # Compute scores if measure == 'order': bs = 100 if index % bs == 0: mx = min(images.shape[0], 5 * (index + bs)) im2 = images[5 * index:mx:5] d2 = order_sim( torch.Tensor(im2).cuda(), torch.Tensor(captions).cuda()) d2 = d2.cpu().numpy() d = d2[index % bs] else: d = numpy.dot(im, captions.T).flatten() # print(captions.shape) inds = numpy.argsort(d)[::-1] index_list.append(inds[0]) # Score rank = 1e20 for i in range(5 * index, 5 * index + 5, 1): tmp = numpy.where(inds == i)[0][0] if tmp < rank: rank = tmp ranks[index] = rank top1[index] = inds[0] # Compute metrics r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks) r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks) r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks) medr = numpy.floor(numpy.median(ranks)) + 1 meanr = ranks.mean() + 1 if return_ranks: return (r1, r5, r10, medr, meanr), (ranks, top1) else: return r1, r5, r10, medr, meanr
def i2t(videos, captions, videos2, captions2, npts=None, measure='cosine', return_ranks=False): """ videos->Text (video Annotation) videos: (5N, K) matrix of videos Captions: (5N, K) matrix of captions """ if npts is None: npts = videos.shape[0] / 5 index_list = [] ranks = numpy.zeros(npts) top1 = numpy.zeros(npts) for index in range(npts): # Get query video vid = videos[5 * index].reshape(1, videos.shape[1]) vid_2 = videos2[5 * index].reshape(1, videos2.shape[1]) # Compute scores if measure == 'order': bs = 100 if index % bs == 0: mx = min(videos.shape[0], 5 * (index + bs)) vid2 = videos[5 * index:mx:5] d2 = order_sim( torch.Tensor(vid2).cuda(), torch.Tensor(captions).cuda()) d2 = d2.cpu().numpy() d = d2[index % bs] else: d = numpy.dot(vid, captions.T).flatten() d2 = numpy.dot(vid_2, captions2.T).flatten() d = (d + d2) / 2 inds = numpy.argsort(d)[::-1] index_list.append(inds[0]) # Score rank = 1e20 for i in range(5 * index, 5 * index + 5, 1): tmp = numpy.where(inds == i)[0][0] if tmp < rank: rank = tmp ranks[index] = rank top1[index] = inds[0] # Compute metrics r1 = 100.0 * len(numpy.where(ranks < 1)[0]) / len(ranks) r5 = 100.0 * len(numpy.where(ranks < 5)[0]) / len(ranks) r10 = 100.0 * len(numpy.where(ranks < 10)[0]) / len(ranks) medr = numpy.floor(numpy.median(ranks)) + 1 meanr = ranks.mean() + 1 if return_ranks: return (r1, r5, r10, medr, meanr), (ranks, top1) else: return (r1, r5, r10, medr, meanr)