def one_nn(self, vector_function=utils.average_embedding, cuda=False):
"""
Nearest Neighbor Baseline: Img2Vec library (https://github.com/christiansafka/img2vec/) is used to obtain
image embeddings, extracted from ResNet-18. For each test image the cosine similarity with all the training images
is computed in order to retrieve similar training images.
The caption of the most similar retrieved image is returned as the generated caption of the test image.
:param vector_function: function that accepts a numpy array nxm and returns a numpy array 1xm
:param cuda: Boolean value of whether to use cuda for image embeddings extraction. Default: False
If a GPU is available pass True
:return: Dictionary with the results
"""
img2vec = Img2Vec(cuda=cuda)
# Load train data
train_data = utils.load_data(self.train_dir)
train_images = dict(zip(train_data.image_ids, train_data.caption))
print("Calculating visual embeddings from train images")
train_images_vec = {}
print("Extracting embeddings for all train images...")
for train_image_lists in tqdm(train_data.img_ids_list):
image_vectors = []
for train_image in train_image_lists:
image_vectors.append(self.image_to_vector(
img2vec, train_image))
train_images_vec[','.join(train_image_lists)] = vector_function(
np.array(image_vectors))
print("Got embeddings for train images.")
# Load test data
test_data = utils.load_data(self.test_dir)
# Save IDs and raw image vectors separately but aligned
ids = list(train_images_vec.keys())
raw = np.array([train_images_vec[i] for i in train_images_vec])
# Normalize image vectors to avoid normalized cosine and use dot
raw = raw / np.array([np.sum(raw, 1)] * raw.shape[1]).transpose()
sim_test_results = {}
for test_image_ids in tqdm(test_data.img_ids_list):
test_vectors = []
for test_image in test_image_ids:
# Get test image embedding
test_vectors.append(self.image_to_vector(img2vec, test_image))
vector = vector_function(np.array(test_vectors))
# Compute cosine similarity with every train image
vector = vector / np.sum(vector)
# Clone to do efficient mat mul dot
test_mat = np.array([vector] * raw.shape[0])
sims = np.sum(test_mat * raw, 1)
top1 = np.argmax(sims)
# Assign the caption of the most similar train image
sim_test_results[test_image_ids] = train_images[ids[top1]]
# Save test results to tsv file
utils.save_results(sim_test_results, self.results_dir,
"onenn_results.json")
return sim_test_results
def _resnet_transform(self, zs):
from img2vec_pytorch import Img2Vec
img2vec = Img2Vec(cuda=False)
img_list = []; vecs = []
for z in zs:
if len(z.shape) < 3:
rgb_arr = 255*(np.array([z]*3).T)
else:
rgb_arr = z
img_list.append(Image.fromarray(np.uint8(rgb_arr)))
vecs += [img2vec.get_vec(img_list[-1], tensor=False)]
return np.vstack(vecs)
def obtenerVectorImagen(rutaImagen):
#Obtenemos la primera parte del vector a partir de Img2Vec
img2vec = Img2Vec(cuda=False)
img = Image.open(rutaImagen)
vec = img2vec.get_vec(resizeImagen(img)) #Vector img2vec
#Obtenemos la segunda parte del vector, los colores separados en RGB
image = resize(io.imread(rutaImagen), (224, 224))
image = maskTImagen(image)
#Canal rojo
vecHistograma = plt.hist(image[:, :, 0].ravel(), bins=32)
r = normalizarColores(np.array(vecHistograma[0])) #Vector del canal rojo
#Canal verde
vecHistograma = plt.hist(image[:, :, 1].ravel(), bins=32)
g = normalizarColores(np.array(vecHistograma[0])) #Vector del canal verde
#Canal azul
vecHistograma = plt.hist(image[:, :, 2].ravel(), bins=32)
b = normalizarColores(np.array(vecHistograma[0])) #Vector del canal azul
#Concatenamos y devolvemos el resultado
vecResultante = np.around(np.concatenate((vec, r, g, b)), 4)
return vecResultante
def load(directory, label):
embeds = []
labels = []
img2vec = Img2Vec(cuda=True)
for file in os.listdir(directory):
if not os.path.isfile(os.path.join(directory, file)):
continue
if not os.path.exists(directory + '/saves/' + file + '.npy'):
if not os.path.exists(directory + '/saves'):
os.makedirs(directory + '/saves')
image = Image.open(directory + '/' + file).convert('RGB')
embed = img2vec.get_vec(image)
np.save(directory + '/saves/' + file + '.npy',
embed,
allow_pickle=True)
embeds.append(embed)
labels.append(label)
print('cmpl: {}'.format(directory + '/' + file))
else:
embeds.append(np.load(directory + '/saves/' + file + '.npy'))
labels.append(label)
return embeds, labels
def get_representations(path, model = 'resnet-18'):
if(empty_folder(path) == True):
print("exiting the program as there was no files in the folder found")
return
list_dicts = []
img2vec = Img2Vec(cuda=False, model = model)
for filename in glob.glob(os.path.join(path, '*.jpg')):
im=Image.open(filename)
# Get a vector from img2vec, returned as a torch FloatTensor
vec = img2vec.get_vec(im, tensor=True)
dict_temp = {}
np_vec = vec.numpy().flatten()
dict_temp['representation'] = np_vec
dict_temp['patientid'] = filename.strip().split('_')[2]
dict_temp['sbp'] = filename.strip().split('_')[3]
dict_temp['dbp'] = filename.strip().split('_')[4]
list_dicts.append(dict_temp)
df_representation = pd.DataFrame(list_dicts)
return df_representation
# -*- coding: utf-8 -*-
"""
Created on Sat Oct 3 12:18:02 2020
@author: yuezh
"""
import os
import pandas as pd
import numpy as np
from img2vec_pytorch import Img2Vec
from PIL import Image
# Initialize Img2Vec with GPU
img2vec = Img2Vec(cuda=True)
#%%
# examples
img = Image.open('917791130590183424_0.jpg')
vec = img2vec.get_vec(img, tensor=True)
images = ['917791130590183424_0.jpg', '917791291823591425_0.jpg']
p_images = []
for i in images:
p_images.append(Image.open(i))
vectors = img2vec.get_vec(p_images)
#%%
# build image embeddings for train dataset
# %%
from img2vec_pytorch import Img2Vec
from PIL import Image
import glob
import numpy as np
import os
import pickle
from tqdm import tqdm
input_path = "images/UnsplashAPIdataset/"
img2vec = Img2Vec(cuda=False)
rebuild_vectors = False
# %%
def compute_vector(image_key):
img = Image.open(input_path + image_key + ".jpg")
vector = img2vec.get_vec(img)
return vector
def save_vectors(vectors, imagedict):
with open("data/images2.pickle", "wb") as filehandle:
pickle.dump(imagedict, filehandle, protocol=pickle.HIGHEST_PROTOCOL)
vectors = vectors.astype("float32")
np.save("data/vectors2.npy", vectors)
# %% Rebuild vectors and imagelist and write to disk
def build_vectors(input_path="images/UnsplashAPIdataset/", rebuild_vectors=False):
# %%
# load images dict from folder
from img2vec_pytorch import Img2Vec
from PIL import Image
import glob
import matplotlib.pyplot as plt
from matplotlib.image import BboxImage
from matplotlib.transforms import Bbox, TransformedBbox
from sklearn.manifold import TSNE
import sys
# we load in the files that we'd like to visualize
img_filepaths = glob.glob('../chronam-get-images/predicted/**/*.jpg',
recursive=True)
# we use img2vec (https://github.com/christiansafka/img2vec)
img2vec = Img2Vec(cuda=False, model='alexnet')
# a list to store the embeddings
img_vectors = []
# iterates through image and uses img2vec to generate vector from image (img2vec -> https://github.com/christiansafka/img2vec)
for filepath in img_filepaths:
# we need to convert to RGB to have appropriate number of channels (specifically, 3 for RGB)
img = Image.open(filepath).convert('RGB')
vec = img2vec.get_vec(img, tensor=False)
img_vectors.append(vec)
# we next compute T-SNE dimensionality reduction
embedded = TSNE(n_components=2, perplexity=20, init='random',
random_state=10).fit_transform(img_vectors)
def most_similar(train_path, test_path, images_path, results_path, cuda=False):
"""
Nearest Neighbor Baseline: Img2Vec library (https://github.com/christiansafka/img2vec/) is used to obtain
image embeddings, extracted from ResNet-18. For each test image the cosine similarity with all the training images
is computed in order to retrieve similar training images.
The caption of the most similar retrieved image is returned as the generated caption of the test image.
:param train_path: The path to the train data tsv file with the form: "image \t caption"
:param test_path: The path to the test data tsv file with the form: "image \t caption"
:param images_path: The path to the images folder
:param results_path: The folder in which to save the results file
:param cuda: Boolean value of whether to use cuda for image embeddings extraction. Default: False
If a GPU is available pass True
:return: Dictionary with the results
"""
img2vec = Img2Vec(cuda=cuda)
# Load train data
train_data = pd.read_csv(train_path, sep="\t", header=None)
train_data.columns = ["id", "caption"]
train_images = dict(zip(train_data.id, train_data.caption))
# Get embeddings of train images
print("Calculating visual embeddings from train images")
train_images_vec = {}
print("Extracting embeddings for all train images...")
for train_image in tqdm(train_data.id):
image = Image.open(os.path.join(images_path, train_image))
image = image.convert('RGB')
vec = img2vec.get_vec(image)
train_images_vec[train_image] = vec
print("Got embeddings for train images.")
# Load test data
test_data = pd.read_csv(test_path, sep="\t", header=None)
test_data.columns = ["id", "caption"]
# Save IDs and raw image vectors separately but aligned
ids = [i for i in train_images_vec]
raw = np.array([train_images_vec[i] for i in train_images_vec])
# Normalize image vectors to avoid normalized cosine and use dot
raw = raw / np.array([np.sum(raw, 1)] * raw.shape[1]).transpose()
sim_test_results = {}
for test_image in tqdm(test_data.id):
# Get test image embedding
image = Image.open(os.path.join(images_path, test_image))
image = image.convert('RGB')
vec = img2vec.get_vec(image)
# Compute cosine similarity with every train image
vec = vec / np.sum(vec)
# Clone to do efficient mat mul dot
test_mat = np.array([vec] * raw.shape[0])
sims = np.sum(test_mat * raw, 1)
top1 = np.argmax(sims)
# Assign the caption of the most similar train image
sim_test_results[test_image] = train_images[ids[top1]]
# Save test results to tsv file
df = pd.DataFrame.from_dict(sim_test_results, orient="index")
df.to_csv(os.path.join(results_path, "onenn_results.tsv"),
sep="\t",
header=False)
return sim_test_results
import os
import seaborn as sns
from img2vec_pytorch import Img2Vec
from sklearn.metrics.pairwise import cosine_similarity
from PIL import Image
from sklearn.preprocessing import minmax_scale
# Initialize Img2Vec with GPU
img2vec = Img2Vec()
import glob
import matplotlib.pyplot as plt
NumPics = 20
import pandas as pd
def image_our_option_similarity():
vector_list = []
cities = []
similarity_heatmap_data = pd.DataFrame()
# go through all cities pictures directories
'''change to directory of images in your computer'''
for filepath in glob.iglob(r'C:\Users\shova\Desktop\finalProject\im\*',
recursive=True):
city = str(filepath)
city = city[39:]
cities.append(city)
image_list = []
# Go through all pictures in directory
for f in os.listdir(filepath):
img = Image.open(os.path.join(filepath, f)).convert('RGB')
def __init__(self):
self._vectorizer = Img2Vec(cuda=False)