def getbetatrainLoaded():
imgdata = datasets.Features172K().Get_all_images()
trigdata = datasets.Features172K().Get_all_queries()
trigdata = np.array(trigdata)
imgdata = np.array(imgdata)
Ntrig2 = []
for i in range(trigdata.shape[0]):
trigdata[i, :] /= np.linalg.norm(trigdata[i, :])
for i in range(imgdata.shape[0]):
imgdata[i, :] /= np.linalg.norm(imgdata[i, :])
for i in range(trigdata.shape[0]):
Ntrig2.append(np.insert(trigdata[i], 0, 1))
#print("Ntrig2 shape %d first elemnt %d",Ntrig2[0] )
Ntrig2 = np.array(Ntrig2)
Ntrigdata1 = Ntrig2.transpose()
X1 = np.matmul(Ntrigdata1, Ntrig2)
X2 = np.linalg.inv(X1)
X3 = np.matmul(X2, Ntrigdata1)
Nbeta = np.matmul(X3, imgdata)
with open(Path1 + r"/" + 'BetatrainLoaded.txt', 'wb') as fp:
pickle.dump(Nbeta, fp)
def savesourcevalues():
train = datasets.Fashion200k(
path=Path1,
split='train',
transform=torchvision.transforms.Compose([
torchvision.transforms.Resize(224),
torchvision.transforms.CenterCrop(224),
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]))
test = datasets.Fashion200k(
path=Path1,
split='test',
transform=torchvision.transforms.Compose([
torchvision.transforms.Resize(224),
torchvision.transforms.CenterCrop(224),
torchvision.transforms.ToTensor(),
torchvision.transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
]))
trig = img_text_composition_models.TIRG(
[t.encode().decode('utf-8') for t in train.get_all_texts()], 512)
trig.load_state_dict(
torch.load(Path1 + r'\fashion200k.tirg.iter160k.pth',
map_location=torch.device('cpu'))['model_state_dict'])
opt = argparse.ArgumentParser()
opt.add_argument('--batch_size', type=int, default=2)
opt.add_argument('--dataset', type=str, default='fashion200k')
opt.batch_size = 1
opt.dataset = 'fashion200k'
datasets.Features172K().SavetoFilesImageSource(Path1 + r'/dataset172',
trig, train, opt)
print('172 Finished')
def testLoadedBeta(opt, model, testset,beta):
"""Tests a model over the given testset."""
model.eval()
test_queries = testset.get_test_queries()
all_imgs = []
all_captions = []
all_queries = []
all_queries1=[]
all_target_captions = []
if test_queries:
# compute test query features
all_imgs = datasets.Features33K().Get_all_images()
all_captions = datasets.Features33K().Get_all_captions()
all_queries1 = datasets.Features33K().Get_all_queries()
all_target_captions = datasets.Features33K().Get_target_captions()
for j in range(len(all_queries1)):
all_queries1[j, :] /= np.linalg.norm(all_queries1[j, :])
X1 = np.insert(all_queries1[j],0, 1)
X2=np.matmul(X1,beta)
all_queries.append(X2)
else:
# use training queries to approximate training retrieval performance
all_imgs = datasets.Features172K().Get_all_images()[:10000]
all_captions = datasets.Features172K().Get_all_captions()[:10000]
all_queries1 = datasets.Features172K().Get_all_queries()[:10000]
all_target_captions = datasets.Features172K().Get_all_captions()[:10000]
for j in range(len(all_queries1)):
all_queries1[j, :] /= np.linalg.norm(all_queries1[j, :])
X1 = np.insert(all_queries1[j],0, 1)
X2=np.matmul(X1,beta)
all_queries.append(X2)
all_queries= np.array(all_queries)
# feature normalization
# for i in range(all_queries.shape[0]):
# all_queries[i, :] /= np.linalg.norm(all_queries[i, :])
for i in range(all_imgs.shape[0]):
all_imgs[i, :] /= np.linalg.norm(all_imgs[i, :])
# match test queries to target images, get nearest neighbors
nn_result = []
for i in tqdm(range(all_queries.shape[0])):
sims = all_queries[i:(i+1), :].dot(all_imgs.T)
if test_queries:
sims[0, test_queries[i]['source_img_id']] = -10e10 # remove query image
nn_result.append(np.argsort(-sims[0, :])[:110])
# compute recalls
out = []
nn_result = [[all_captions[nn] for nn in nns] for nns in nn_result]
for k in [1, 5, 10, 50, 100]:
r = 0.0
for i, nns in enumerate(nn_result):
if all_target_captions[i] in nns[:k]:
r += 1
r /= len(nn_result)
#out += [('recall_top' + str(k) + '_correct_composition', r)]
out.append(str(k) + ' ---> '+ str(r*100))
if opt.dataset == 'mitstates':
r = 0.0
for i, nns in enumerate(nn_result):
if all_target_captions[i].split()[0] in [c.split()[0] for c in nns[:k]]:
r += 1
r /= len(nn_result)
out += [('recall_top' + str(k) + '_correct_adj', r)]
r = 0.0
for i, nns in enumerate(nn_result):
if all_target_captions[i].split()[1] in [c.split()[1] for c in nns[:k]]:
r += 1
r /= len(nn_result)
out += [('recall_top' + str(k) + '_correct_noun', r)]
return out
def testLoaded_NLP(opt, model, testset):
"""Tests a model over the given testset."""
model.eval()
test_queries = testset.get_test_queries()
all_imgs = []
all_captions = []
all_queries = []
all_target_captions = []
if test_queries:
# compute test query features
all_imgs = datasets.Features33K().Get_all_images()
all_captions = datasets.Features33K().Get_all_captions()
all_queries = datasets.Features33K().Get_all_queries()
all_target_captions = datasets.Features33K().Get_target_captions()
else:
# use training queries to approximate training retrieval performance
all_imgs = datasets.Features172K().Get_all_images()#[:10000]
all_captions = datasets.Features172K().Get_all_captions()#[:10000]
all_queries = datasets.Features172K().Get_all_queries()#[:10000]
all_target_captions = datasets.Features172K().Get_all_captions()#[:10000]
modelNLR=main2.NLR2(all_queries.shape[1],all_imgs.shape[1],700)
modelNLR.load_state_dict(torch.load(Path1+r'\NLPMohamed3.pth'))
modelNLR.eval()
all_queries=torch.from_numpy(all_queries)
# for t in range(int(len(all_queries))):
# print('get testdata=',t,end='\r')
# f=all_queries[t]
# all_queries[t] = modelNLR.myforward(f)
all_queries = modelNLR.myforward(all_queries)
#all_queries.detach().numpy()
all_queries = torch.tensor(all_queries,requires_grad=False)
all_queries=np.array(all_queries)
# feature normalization
for i in range(all_queries.shape[0]):
all_queries[i, :] /= np.linalg.norm(all_queries[i, :])
for i in range(all_imgs.shape[0]):
all_imgs[i, :] /= np.linalg.norm(all_imgs[i, :])
# match test queries to target images, get nearest neighbors
nn_result = []
for i in tqdm(range(all_queries.shape[0])):
sims = all_queries[i:(i+1), :].dot(all_imgs.T)
if test_queries:
sims[0, test_queries[i]['source_img_id']] = -10e10 # remove query image
nn_result.append(np.argsort(-sims[0, :])[:110])
# compute recalls
out = []
nn_result = [[all_captions[nn] for nn in nns] for nns in nn_result]
for k in [1, 5, 10, 50, 100]:
r = 0.0
for i, nns in enumerate(nn_result):
if all_target_captions[i] in nns[:k]:
r += 1
r /= len(nn_result)
#out += [('recall_top' + str(k) + '_correct_composition', r)]
out.append(str(k) + ' ---> '+ str(r*100))
if opt.dataset == 'mitstates':
r = 0.0
for i, nns in enumerate(nn_result):
if all_target_captions[i].split()[0] in [c.split()[0] for c in nns[:k]]:
r += 1
r /= len(nn_result)
out += [('recall_top' + str(k) + '_correct_adj', r)]
r = 0.0
for i, nns in enumerate(nn_result):
if all_target_captions[i].split()[1] in [c.split()[1] for c in nns[:k]]:
r += 1
r /= len(nn_result)
out += [('recall_top' + str(k) + '_correct_noun', r)]
return out
def ab_MtestLoaded(opt, model, testset,option):
"""Tests a model over the given testset."""
model.eval()
test_queries = testset.get_test_queries()
all_imgs = []
all_captions = []
all_queries = []
all_target_captions = []
if test_queries:
# compute test query features
all_imgs = datasets.Features33K().Get_all_images()
all_captions = datasets.Features33K().Get_all_captions()
all_queries = datasets.Features33K().Get_all_queries()
all_target_captions = datasets.Features33K().Get_target_captions()
new_all_queries=mymodels(all_queries,all_imgs,option)
else:
# use training queries to approximate training retrieval performance
all_imgs = datasets.Features172K().Get_all_images()[:10000]
all_captions = datasets.Features172K().Get_all_captions()[:10000]
all_queries = datasets.Features172K().Get_all_queries()[:10000]
all_target_captions = datasets.Features172K().Get_all_captions()[:10000]
new_all_queries=mymodels(all_queries,all_imgs,option)
# feature normalization
diff=new_all_queries-all_queries
diff=diff**2
diff=np.sum(diff,axis=1)
print(mean(diff))
for i in range(all_queries.shape[0]):
all_queries[i, :] /= np.linalg.norm(all_queries[i, :])
new_all_queries[i, :] /= np.linalg.norm(new_all_queries[i, :])
for i in range(all_imgs.shape[0]):
all_imgs[i, :] /= np.linalg.norm(all_imgs[i, :])
# match test queries to target images, get nearest neighbors
nn_result = []
new_nn_result = []
euc_new_nn_result=[]
for i in tqdm(range(all_queries.shape[0])):
#sims = all_queries[i:(i+1), :].dot(all_imgs.T)
new_sims = new_all_queries[i:(i+1), :].dot(all_imgs.T)
#euc_new_sims=np.sum(abs(all_imgs-all_queries[i, :]),axis=1)
if test_queries:
# sims[0, test_queries[i]['source_img_id']] = -10e10 # remove query image
new_sims[0, test_queries[i]['source_img_id']] = -10e10 # remove query image
# euc_new_sims[test_queries[i]['source_img_id']]=10e10
#nn_result.append(np.argsort(-sims[0, :])[:110])
new_nn_result.append(np.argsort(-new_sims[0, :])[:110])
#euc_new_nn_result.append(np.argsort(euc_new_sims)[:110])
# compute recalls
out = []
out2=[]
out3=[]
#nn_result = [[all_captions[nn] for nn in nns] for nns in nn_result]
new_nn_result = [[all_captions[nn] for nn in nns] for nns in new_nn_result]
#euc_new_nn_result = [[all_captions[nn] for nn in nns] for nns in euc_new_nn_result]
for k in [1, 5, 10, 50, 100]:
# r = 0.0
# for i, nns in enumerate(euc_new_nn_result):
# if all_target_captions[i] in nns[:k]:
# r += 1
# r /= len(euc_new_nn_result)
# #out += [('recall_top' + str(k) + '_correct_composition', r)]
# out3.append(str(k) + ' ---> '+ str(r*100))
r = 0.0
for i, nns in enumerate(new_nn_result):
if all_target_captions[i] in nns[:k]:
r += 1
r /= len(new_nn_result)
#out += [('recall_top' + str(k) + '_correct_composition', r)]
out2.append(str(k) + ' ---> '+ str(r*100))
# r = 0.0
# for i, nns in enumerate(nn_result):
# if all_target_captions[i] in nns[:k]:
# r += 1
# r /= len(nn_result)
# #out += [('recall_top' + str(k) + '_correct_composition', r)]
# out.append(str(k) + ' ---> '+ str(r*100))
return out, out2, out3