def __init__(self, update=False, compute_final_distances=False, layers=[]):
""" Computes the k-NN of all the images according to all the features. Creates
a similarity matrix (final_matrix) by computing the pairwise distances over all
images, according to all features. Then, normalizes those distances and sums them.
That sum of distances will be the similarity measure stored in the matrix, for each
pair of images. """
# Updates the image database if update=True
self.dm = DataManager(update)
# Extracts the features again if update=True
self.ip = ImageProcessor(update, layers)
num_imgs = self.dm.get_num_imgs()
if update:
self.final_matrix = np.zeros((num_imgs, num_imgs))
# Calculating k-NN of every image according to all features
self.calculate_neighbours(num_imgs, load_feature(HOC_MATRIX_FILE),
COLOR_NEIGH_FILE)
self.calculate_neighbours(num_imgs, load_feature(HOG_MATRIX_FILE),
GRADS_NEIGH_FILE)
for layer in layers:
self.calculate_neighbours(
num_imgs, load_feature(VGG_MATRIX_FILES[layer]),
VGG_NEIGH_FILES[layer])
# Calculating pairwise distance of every image according to all features
self.calculate_final_distance_matrix(num_imgs, layers)
elif compute_final_distances:
# Calculating pairwise distance of every image according to all features
self.calculate_final_distance_matrix(num_imgs, layers)
def show_full_graph(self, img_size=0.1, graph_name="graph.pdf"):
""" Creates a figure showing the full graph and saves it with a given name
(graph_name). """
# TODO clustered graph
pos = nx.spring_layout(self.graph, weight="distance")
fig = plt.figure(figsize=(35, 35))
ax = plt.subplot(111)
ax.set_aspect('equal')
nx.draw_networkx_edges(self.graph, pos, ax=ax)
plt.xlim(-1.5, 1.5)
plt.ylim(-1.5, 1.5)
trans = ax.transData.transform
trans2 = fig.transFigure.inverted().transform
p2 = img_size / 2.0
for n in self.graph:
xx, yy = trans(pos[n])
xa, ya = trans2((xx, yy))
a = plt.axes([xa - p2, ya - p2, img_size, img_size])
a.set_aspect('equal')
a.imshow(dm.get_single_img(n[1]))
a.axis('off')
ax.axis('off')
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
plt.savefig(RESULTS_DIR + graph_name, format="pdf")
def show_feature_neighbours(src,
graph_name="feature_neighbours.pdf",
k=10,
feat="colors"):
""" Creates a figure showing the k-NN of a given image (src), according to a
given feature (feat), side by side. Saves it with a given name (graph_name).
src is given as a tuple of (index, image_name), like so: (0, "img_00000000.jpg"). """
idx = dm.get_img_names()
fig = plt.figure(figsize=(25, 20))
columns = k
rows = 1
if feat == "colors":
neighbours = load_neigh(COLOR_NEIGH_FILE)
elif feat == "grads":
neighbours = load_neigh(GRADS_NEIGH_FILE)
elif feat == VGG16_BLOCK2_POOL_LAYER:
neighbours = load_neigh(VGG_BLOCK2_NEIGH_FILE)
elif feat == VGG16_BLOCK3_POOL_LAYER:
neighbours = load_neigh(VGG_BLOCK3_NEIGH_FILE)
elif feat == VGG16_BLOCK4_POOL_LAYER:
neighbours = load_neigh(VGG_BLOCK4_NEIGH_FILE)
elif feat == VGG16_BLOCK5_POOL_LAYER:
neighbours = load_neigh(VGG_BLOCK5_NEIGH_FILE)
else:
neighbours = load_neigh(COLOR_NEIGH_FILE)
for i in range(1, columns * rows + 1):
img = ip.center_crop_image(
dm.get_single_img(idx[neighbours[src[0]][i - 1]]))
fig.add_subplot(rows, columns, i)
plt.imshow(img)
if i == 1:
plt.title("Query Image", fontsize=20)
else:
plt.title("Neighbour " + str(i - 1), fontsize=20)
plt.axis("off")
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
fig.savefig(RESULTS_DIR + graph_name, format="pdf")
def show_node_neighbours(self,
node,
img_size=0.1,
graph_name="node_neighbours.pdf"):
""" Creates a figure showing the neighbours of a given image (node) in the graph.
Saves it with a given name (graph_name). Node is given as a tuple of (index, image_name),
like so: (0, "img_00000000.jpg"). """
ego = nx.ego_graph(self.graph, node, undirected=True)
for (u, v) in ego.edges():
if u != node and v != node:
ego.remove_edge(u, v)
pos = nx.spring_layout(ego)
labels = {}
for key in pos.keys():
labels[key] = key[1]
pos_attrs = {}
for node, coords in pos.items():
pos_attrs[node] = (coords[0], coords[1] - 0.25)
fig = plt.figure(figsize=(20, 20))
ax = plt.subplot(111)
ax.set_aspect('equal')
nx.draw_networkx_edges(ego, pos, ax=ax)
nx.draw_networkx_labels(ego,
pos=pos_attrs,
ax=ax,
labels=labels,
font_size=14,
font_color='r')
plt.xlim(-1.5, 1.5)
plt.ylim(-1.5, 1.5)
trans = ax.transData.transform
trans2 = fig.transFigure.inverted().transform
p2 = img_size / 2.0
for n in ego:
xx, yy = trans(pos[n])
xa, ya = trans2((xx, yy))
a = plt.axes([xa - p2, ya - p2, img_size, img_size])
a.set_aspect('equal')
a.imshow(dm.get_single_img(n[1]))
a.axis('off')
ax.axis('off')
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
plt.savefig(RESULTS_DIR + graph_name, format="pdf")
def extract_vgg_feature(img_names, npz_name, layer_name):
""" Given the image names (img_names) of all the images in the database,
extracts the features from a given VGG16 layer (layer_name) for each image.
Then, stores the features in an npz file with a given name (npz_name). """
features = []
for img_name in img_names:
img = dm.get_single_img(img_name)
img = center_crop_image(img, size=224)
features.append(fe.vgg16_layer(img, layer=layer_name))
features = np.array(features)
np.savez('{}.npz'.format(FILES_DIR + npz_name), features=features)
def extract_img_hog(img_name):
""" Given an image name (img_name), fetches the image, pre-processes it
and extracts its Histogram of Oriented Gradients. Returns this feature. """
# Fetching and pre-processing
img = dm.get_single_img(img_name)
img = center_crop_image(img, size=224)
img_gray = color.rgb2gray(img)
# Extracting feature
grad_hist = fe.my_hog(img_gray,
orientations=8,
pixels_per_cell=(32, 32))
grad_feat = np.squeeze(normalize(grad_hist.reshape(1, -1), norm="l2"))
return grad_feat
def extract_img_hoc(img_name):
""" Given an image name (img_name), fetches the image, pre-processes it
and extracts its Histogram of Colors. Returns this feature. """
# Fetching and pre-processing
img = dm.get_single_img(img_name)
img = center_crop_image(img, size=224)
img_hsv = color.rgb2hsv(img)
img_int = img_as_ubyte(img_hsv)
# Extracting feature
color_hist, bins = fe.hoc(img_int, bins=(4, 4, 4))
color_feat = np.squeeze(normalize(color_hist.reshape(1, -1),
norm="l2"))
return color_feat
def show_shortest_path(self,
src,
dst,
img_size=0.1,
graph_name="path.pdf"):
""" Creates a figure showing the shortest path between two images (src and dst)
and saves it with a given name (graph_name). The images are given as tuples of
(index, image_name), like so: (0, "img_00000000.jpg"). """
fig = plt.figure(figsize=(20, 10))
ax = plt.subplot(111)
ax.set_aspect('equal')
nodes = nx.shortest_path(self.graph, src, dst, weight="distance")
g = nx.Graph()
pos = {}
labels = {}
i = 0
for node in nodes:
pos[node] = [i, 0]
i += 1
labels[node] = str(node[1])
g.add_node(node)
edges = []
for i in range(1, len(nodes)):
edges.append((nodes[i - 1], nodes[i]))
i = 0
for edge in edges:
i += 1
g.add_edge(edge[0], edge[1])
pos_attrs = {}
if len(nodes) <= 3:
for node, coords in pos.items():
pos_attrs[node] = (coords[0], coords[1] - 0.3)
else:
for node, coords in pos.items():
pos_attrs[node] = (coords[0], coords[1] - 0.4)
nx.draw_networkx_edges(g, pos, ax=ax, edgelist=edges)
nx.draw_networkx_labels(g,
pos=pos_attrs,
ax=ax,
labels=labels,
font_color='r')
plt.xlim(-2, len(nodes) + 1)
plt.ylim(-1.5, 1.5)
trans = ax.transData.transform
trans2 = fig.transFigure.inverted().transform
p2 = img_size / 2.0
for n in nodes:
xx, yy = trans(pos[n])
xa, ya = trans2((xx, yy))
a = plt.axes([xa - p2, ya - p2, img_size, img_size])
a.set_aspect('equal')
a.imshow(dm.get_single_img(n[1]))
a.axis('off')
ax.axis('off')
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
plt.savefig(RESULTS_DIR + graph_name, format="pdf")
class SimilarityCalculator:
def __init__(self, update=False, compute_final_distances=False, layers=[]):
""" Computes the k-NN of all the images according to all the features. Creates
a similarity matrix (final_matrix) by computing the pairwise distances over all
images, according to all features. Then, normalizes those distances and sums them.
That sum of distances will be the similarity measure stored in the matrix, for each
pair of images. """
# Updates the image database if update=True
self.dm = DataManager(update)
# Extracts the features again if update=True
self.ip = ImageProcessor(update, layers)
num_imgs = self.dm.get_num_imgs()
if update:
self.final_matrix = np.zeros((num_imgs, num_imgs))
# Calculating k-NN of every image according to all features
self.calculate_neighbours(num_imgs, load_feature(HOC_MATRIX_FILE),
COLOR_NEIGH_FILE)
self.calculate_neighbours(num_imgs, load_feature(HOG_MATRIX_FILE),
GRADS_NEIGH_FILE)
for layer in layers:
self.calculate_neighbours(
num_imgs, load_feature(VGG_MATRIX_FILES[layer]),
VGG_NEIGH_FILES[layer])
# Calculating pairwise distance of every image according to all features
self.calculate_final_distance_matrix(num_imgs, layers)
elif compute_final_distances:
# Calculating pairwise distance of every image according to all features
self.calculate_final_distance_matrix(num_imgs, layers)
def calculate_neighbours(self, num_imgs, feat_matrix, npz_name):
""" Computes the k-NN of all the images according to a given feature matrix
(feat_matrix) and stores them in a file with a given name (npz_name). """
if not os.path.exists(FILES_DIR):
os.makedirs(FILES_DIR)
feature_neigh = []
for i in range(0, num_imgs):
# Calculating k-NN of image i
neighbours, _ = self.k_neighbours(feat_matrix[i].reshape(1, -1),
feat_matrix,
k=N_NEIGHBOURS)
feature_neigh.append(neighbours)
# Saving neighbours matrix in file
np.savez('{}.npz'.format(FILES_DIR + npz_name), knn=feature_neigh)
def calculate_final_distance_matrix(self, num_imgs, layers):
self.final_matrix = np.zeros((num_imgs, num_imgs))
# Calculating pairwise distance of every image according to color feature
self.calculate_distances(num_imgs, load_feature(HOC_MATRIX_FILE))
# Calculating pairwise distance of every image according to gradient feature
self.calculate_distances(num_imgs, load_feature(HOG_MATRIX_FILE))
# Calculating pairwise distance of every image according to the given VGG16 layers
for layer in layers:
self.calculate_distances(num_imgs,
load_feature(VGG_MATRIX_FILES[layer]))
# Saving the same values in opposite indexes because the final distance matrix is symmetric
for i in range(0, num_imgs):
for j in range(i + 1, num_imgs):
self.final_matrix[j, i] = self.final_matrix[i, j]
if not os.path.exists(FILES_DIR):
os.makedirs(FILES_DIR)
# Save the final distances matrix to a file
np.savez('{}.npz'.format(FILES_DIR + FINAL_DISTANCES_FILE),
dist=self.final_matrix)
def calculate_distances(self, num_imgs, feat_matrix):
""" Computes the pairwise distances over all images, according to the
feature matrix. Then, normalizes those distances and sums them to the
corresponding indexes of the final similarity matrix. """
# Calculating normalization value
normalizer = self.calc_sum_distances(self.dm, feat_matrix)
feature_neigh = []
for i in range(0, num_imgs):
for j in range(i + 1, num_imgs):
# Calculating the distances, normalizing and adding them to the final similarity matrix
dist = pairwise_distances(feat_matrix[i].reshape(1, -1),
feat_matrix[j].reshape(1, -1),
metric="euclidean")
norm_dist = np.squeeze(dist / normalizer)
self.final_matrix[i, j] += norm_dist
@staticmethod
def k_neighbours(query, matrix, metric="euclidean", k=10):
""" Computes the k-NN of a given image (query) by calculating the pairwise distance
of that image and all the images in the given matrix (matrix). Sorts the distances
and returns the sorted indexes of the neighbours and the sorted distances. """
dists = pairwise_distances(query, matrix, metric=metric)
dists = np.squeeze(dists)
sorted_indexes = np.argsort(dists)
return sorted_indexes[:k], dists[sorted_indexes[:k]]
@staticmethod
def calc_max_distances(dm, feat_matrix, metric="euclidean"):
""" Gets a sample of size SAMPLE_SET_SIZE from the image database. Calculates the
pairwise distance over all images according to the given feature matrix (feat_matrix).
Saves the maximum distance and returns it, so that it can then be used to normalize
the distances calculated later over the whole database. """
imgs = dm.get_rand_set(SAMPLE_SET_SIZE)
max_dist = -1
for img1 in imgs:
for img2 in imgs:
if img1 != img2:
d = pairwise_distances(feat_matrix[img1].reshape(1, -1),
feat_matrix[img2].reshape(1, -1),
metric=metric)
if d > max_dist:
max_dist = d
return max_dist
@staticmethod
def calc_sum_distances(dm, feat_matrix, metric="euclidean"):
""" Gets a sample of size SAMPLE_SET_SIZE from the image database. Calculates the
pairwise distance over all images according to the given feature matrix (feat_matrix).
Sums those distances and returns the sum, so that it can then be used to normalize
the distances calculated later over the whole database. """
imgs = dm.get_rand_set(SAMPLE_SET_SIZE)
sum_dist = 0
for img1 in imgs:
for img2 in imgs:
if img1 != img2:
sum_dist += pairwise_distances(
feat_matrix[img1].reshape(1, -1),
feat_matrix[img2].reshape(1, -1),
metric=metric)
return sum_dist