def visualizeVocabulary(newVocab=False):
# specific siftdir
siftdir = '100sift/'
# Get a list of all the .mat files
fnames = glob.glob(siftdir + '*.mat')
# only create new vocabs when needed
if newVocab:
# get all descriptors
allDescriptors = np.empty((0, 128))
for fname in fnames:
print fname
mat = scipy.io.loadmat(fname)
# keys
# ['numfeats', 'orients', 'scales', 'descriptors', '__header__',
# '__globals__', 'positions', '__version__', 'corners', 'imname']
allDescriptors = np.vstack((allDescriptors, mat['descriptors']))
# centroid = vocabularies, label = which vocabulary each one belongs to in the original list
centroid, label = vq.kmeans2(allDescriptors, 1500)
np.save('./100vocabularies.npy', centroid)
np.save('./labels.npy', label)
# load in dictionary
dictionary = np.load('./100vocabularies.npy')
# select two vocabularies
[vocab1, vocab2] = random.sample(dictionary, 2)
# iterate through sift to put (imageIndex, patchIndex) that belong to the chosen vocabulary
patches1 = []
patches2 = []
imageIndex = 0
while (len(patches1) < 25
or len(patches2) < 25) and imageIndex < len(fnames):
print fnames[imageIndex]
mat = scipy.io.loadmat(fnames[imageIndex])
# ['numfeats', 'orients', 'scales', 'descriptors', '__header__',
# '__globals__', 'positions', '__version__', 'corners', 'imname']
# iterate through all descriptors
for patchIndex in range(len(mat['descriptors'])):
if isPatchVocab(mat['descriptors'][patchIndex], dictionary,
vocab1) and len(patches1) < 25:
patches1.append((imageIndex, patchIndex))
print "patches 1: " + str(len(patches1))
elif isPatchVocab(mat['descriptors'][patchIndex], dictionary,
vocab2) and len(patches2) < 25:
patches2.append((imageIndex, patchIndex))
print "patches 2: " + str(len(patches2))
imageIndex += 1
# get all images
framedir = '100frames/'
framenames = glob.glob(framedir + '*.jpeg')
fig, axs = plt.subplots(5, 5)
# traverse through pathches1 to display
plotCounter = 0
for imageIndex, patchIndex in patches1:
currentFrame = misc.imread(framenames[imageIndex])
currentsift = scipy.io.loadmat(fnames[imageIndex])
img_patch = getPatchFromSIFTParameters(
currentsift['positions'][patchIndex, :],
currentsift['scales'][patchIndex],
currentsift['orients'][patchIndex], rgb2gray(currentFrame))
row = int(plotCounter / 5)
col = int(plotCounter % 5)
axs[row, col].imshow(img_patch, cmap="gray")
plotCounter += 1
plt.show()
fig, axs = plt.subplots(5, 5)
# traverse through pathches2 to display
plotCounter = 0
for imageIndex, patchIndex in patches2:
currentFrame = misc.imread(framenames[imageIndex])
currentsift = scipy.io.loadmat(fnames[imageIndex])
img_patch = getPatchFromSIFTParameters(
currentsift['positions'][patchIndex, :],
currentsift['scales'][patchIndex],
currentsift['orients'][patchIndex], rgb2gray(currentFrame))
row = int(plotCounter / 5)
col = int(plotCounter % 5)
axs[row, col].imshow(img_patch, cmap="gray")
plotCounter += 1
plt.show()
im = misc.imread(imname)
print 'imname = %s contains %d total features, each of dimension %d' %(imname, numfeats, mat['descriptors'].shape[1]);
fig=plt.figure()
ax=fig.add_subplot(111)
ax.imshow(im)
Ind = [750]
coners = displaySIFTPatches(mat['positions'][Ind,:], mat['scales'][Ind,:], mat['orients'][Ind,:])
print coners
for j in range(len(coners)):
ax.plot([coners[j][0][1], coners[j][1][1]], [coners[j][0][0], coners[j][1][0]], color='g', linestyle='-', linewidth=1)
ax.plot([coners[j][1][1], coners[j][2][1]], [coners[j][1][0], coners[j][2][0]], color='g', linestyle='-', linewidth=1)
ax.plot([coners[j][2][1], coners[j][3][1]], [coners[j][2][0], coners[j][3][0]], color='g', linestyle='-', linewidth=1)
ax.plot([coners[j][3][1], coners[j][0][1]], [coners[j][3][0], coners[j][0][0]], color='g', linestyle='-', linewidth=1)
ax.set_xlim(0, im.shape[1])
ax.set_ylim(0, im.shape[0])
plt.gca().invert_yaxis()
plt.show()
#%%
patch_num = 750
print imname
im = misc.imread(imname)
img_patch = getPatchFromSIFTParameters(allPos[patch_num,:], allScal[patch_num], allOri[patch_num], rgb2gray(im))
print img_patch
plt.imshow(img_patch, cmap = cm.Greys_r)
plt.show()
def visualizeVocabulary():
# specific frame dir and siftdir
framesdir = 'frames/'
siftdir = 'sift/'
# Get a list of all the .mat file in that directory.
# there is one .mat file per image.
fnames = glob.glob(siftdir + '*.mat')
fnames = [i[-27:] for i in fnames]
fnames = fnames[start:end]
print('reading %d total files...' % (len(fnames)))
N = 100 #to visualize a sparser set of the features
all_describtor = []
mapping = {}
#all_position = []
#all_orient = []
#all_scale = []
descriptors_count = 0
for i in range(len(fnames)):
print('reading frame %d of %d' % (i, len(fnames)))
# load that file
fname = siftdir + fnames[i]
mat = scipy.io.loadmat(fname)
#get parameters
describtors = mat['descriptors']
for j, des in enumerate(describtors):
all_describtor.append(des)
mapping[descriptors_count] = (fnames[i], j)
descriptors_count += 1
all_describtor = np.array(all_describtor)
#k-means
k = 1500
#whiten(all_describtor)
#center,label=kmeans2(all_describtor,k)
#save the dataset
if not is_clustered:
print("Start Kmeans")
cluster = KMeans(n_clusters=k).fit_predict(
all_describtor
) #Compute cluster centers and predict cluster index for each sample.
# to save it
with open('cluster.pkl', 'wb') as f:
pickle.dump(cluster, f)
print('Clustering finished')
else:
with open('cluster.pkl', 'rb') as f:
cluster = pickle.load(f)
plt.figure()
n = 1
for i in range(len(cluster)):
if cluster[i] == cluster_to_choose:
fname, index = mapping[i]
mat = scipy.io.loadmat(siftdir + fname)
imname = framesdir + fname[:-4]
im = imread(imname)
print(index)
#index=np.int(index)
a = mat['positions'][index, :]
b = mat['scales'][index]
c = mat['orients'][index]
img = getPatchFromSIFTParameters(a, b, c, rgb2gray(im))
#img = getPatchFromSIFTParameters(mat['positions'][index, :], mat['scales'][index],
# mat['orients'][index], rgb2gray(im))
pl.subplot(5, 5, n)
pl.imshow(img, cmap='gray')
n += 1
if n > 25:
break
plt.show()
def visualizeVocabulary(newVocab=False):
# specific siftdir
siftdir = '100sift/'
framedir = '100frames/'
# Get a list of all the .mat file in that directory.
# there is one .mat file per image.
fnames = glob.glob(siftdir + '*.mat')
framenames = glob.glob(framedir + '*.jpeg')
# only create new vocabs when needed
if newVocab:
# get all descrptors
allDescriptors = np.empty((0, 128))
for fname in fnames:
print fname
mat = scipy.io.loadmat(fname)
# ['numfeats', 'orients', 'scales', 'descriptors', '__header__',
# '__globals__', 'positions', '__version__', 'corners', 'imname']
allDescriptors = np.vstack((allDescriptors, mat['descriptors']))
print allDescriptors.shape
# centroid = vocabularies, label = which vocabulary each one belongs to in original
centroid, label = vq.kmeans2(allDescriptors, 1500)
np.save('./100vocabularies.npy', centroid)
np.save('./labels.npy', label)
dictionary = np.load('./100vocabularies.npy')
labels = np.load('./labels.npy') # same length as total sift
# select two vocabularies that are at least over 25 occurrences
occurencesList = np.bincount(labels)
indexList = np.where(occurencesList >= 25)[0]
[vocabIndex1, vocabIndex2] = random.sample(indexList, 2)
vocab1Location = np.where(labels == vocabIndex1)[0][0:25]
vocab2Location = np.where(labels == vocabIndex2)[0][0:25]
# contain (patchIndex, mat, img)
patches1 = []
patches2 = []
startPos = 0
# traverse through images
for index in range(len(fnames)):
currentFrame = scipy.io.loadmat(fnames[index])
currentImage = misc.imread(framenames[index])
# set frame
descriptors = currentFrame['descriptors']
endPos = startPos + len(descriptors)
# get indices
patches1Indices = vocab1Location[np.where(vocab1Location >= startPos)]
patches1Indices = patches1Indices[np.where(patches1Indices < endPos)]
patches2Indices = vocab2Location[np.where(vocab2Location >= startPos)]
patches2Indices = patches2Indices[np.where(patches2Indices < endPos)]
# put the indices in the patches list
for patchIndex in patches1Indices:
patches1.append(
(patchIndex - startPos, currentFrame, currentImage))
for patchIndex in patches2Indices:
patches2.append(
(patchIndex - startPos, currentFrame, currentImage))
startPos += len(descriptors)
fig, axs = plt.subplots(5, 5)
# traverse through pathches1 to display
plotCounter = 0
for (patchIndex, mat, img) in patches1:
img_patch = getPatchFromSIFTParameters(mat['positions'][patchIndex, :],
mat['scales'][patchIndex],
mat['orients'][patchIndex],
rgb2gray(img))
row = int(plotCounter / 5)
col = int(plotCounter % 5)
axs[row, col].imshow(img_patch, cmap="gray")
plotCounter += 1
plt.show()
fig, axs = plt.subplots(5, 5)
# traverse through pathches2 to display
plotCounter = 0
for (patchIndex, mat, img) in patches2:
img_patch = getPatchFromSIFTParameters(mat['positions'][patchIndex, :],
mat['scales'][patchIndex],
mat['orients'][patchIndex],
rgb2gray(img))
row = int(plotCounter / 5)
col = int(plotCounter % 5)
axs[row, col].imshow(img_patch, cmap="gray")
plotCounter += 1
plt.show()
#show the vocabular for feature 906 in 25 images
fig = plt.figure()
f = 0
for i in range(500):
if f > 25:
break
fname = siftdir + fnames[i]
mat = scipy.io.loadmat(fname)
idx, _ = vq(mat['descriptors'], centroids)
im = misc.imread(framesdir + fnames[i][:-4])
index = np.where(idx == 906)
for item in index[0]:
patch_num = item
img_patch = getPatchFromSIFTParameters(mat['positions'][patch_num, :],
mat['scales'][patch_num],
mat['orients'][patch_num],
rgb2gray(im))
# plt.imshow(img_patch, cmap = cm.Greys_r)
# name = 'patch_7_' + str(f) + '.jpg'
# misc.imsave(name,img_patch)
# plt.show()
f = f + 1
if f > 25:
break
a = fig.add_subplot(5, 5, f)
imgplot = plt.imshow(img_patch, cmap=cm.Greys_r)
a.set_title(str(f))
#for each image assign the samples to a cluster
#pics_feature_assignments = []
#for name in fnames:
coners = displaySIFTPatches(mat['positions'][Ind,:], mat['scales'][Ind,:], mat['orients'][Ind,:])
for j in range(len(coners)):
bx.plot([coners[j][0][1], coners[j][1][1]], [coners[j][0][0], coners[j][1][0]], color='g', linestyle='-', linewidth=1)
bx.plot([coners[j][1][1], coners[j][2][1]], [coners[j][1][0], coners[j][2][0]], color='g', linestyle='-', linewidth=1)
bx.plot([coners[j][2][1], coners[j][3][1]], [coners[j][2][0], coners[j][3][0]], color='g', linestyle='-', linewidth=1)
bx.plot([coners[j][3][1], coners[j][0][1]], [coners[j][3][0], coners[j][0][0]], color='g', linestyle='-', linewidth=1)
bx.set_xlim(0, im.shape[1])
bx.set_ylim(0, im.shape[0])
plt.gca().invert_yaxis()
plt.show()
# extract the image patch
patch_num = 1
img_patch = getPatchFromSIFTParameters(mat['positions'][patch_num,:], mat['scales'][patch_num], mat['orients'][patch_num], rgb2gray(im))
plt.imshow(img_patch, cmap = cm.Greys_r)
plt.show()
# In[ ]:
# In[ ]:
model = pkl.load(open("model.pkl", "rb"))
patches = []
for idx in indices:
imname = imnames[idx]
sift_fname = imname + ".mat"
gray_im = np.asarray(
Image.open(os.path.join(FRAME_DIR, imname)).convert('L'))
desc, _ = get_desc_imname(sift_fname)
pos, scale, orient = get_keypoints(sift_fname)
closest_patch_idx = get_closes_patch(
model.cluster_centers_[word_id], desc)
# print(closest_patch_idx)
patch = getPatchFromSIFTParameters(pos[closest_patch_idx],
scale[closest_patch_idx],
orient[closest_patch_idx],
gray_im)
patches.append(patch)
fig = plt.figure(figsize=(10, 10))
grid = ImageGrid(
fig,
111, # similar to subplot(111)
nrows_ncols=(5, 5),
share_all=True,
axes_pad=0.1, # pad between axes in inch.
)
count = 0
for ax in grid:
if count < len(patches):
patch = patches[count]
patches1 = []
patches2 = []
for filename in filenames:
mat = scipy.io.loadmat(filename, verify_compressed_data_integrity=False)
descriptors = mat['descriptors']
if(len(descriptors) == 0):
continue
imagename = framesdir + mat['imagename'][0]
im = misc.imread(imagename)
clusters = kmeans_model.predict(descriptors)
for i in range(len(clusters)):
cluster = clusters[i]
if cluster == 111:
image_patch = getPatchFromSIFTParameters(mat['positions'][i,:], mat['scales'][i], mat['orients'][i], rgb2gray(im))
patches1.append(image_patch)
elif cluster == 222:
image_patch = getPatchFromSIFTParameters(mat['positions'][i,:], mat['scales'][i], mat['orients'][i], rgb2gray(im))
patches2.append(image_patch)
if len(patches1) >= 25 and len(patches2) >= 25:
break
fig=plt.figure()
columns = 5
rows = 5
for i in range(1, columns * rows +1):
image = patches1[i - 1]
fig.add_subplot(rows, columns, i)
plt.imshow(image, cmap='gray')
plt.show()
distance = dist2(word0.reshape((-1, 1)), descriptor.reshape(
(-1, 1))).mean() # ***********************
words.append(((position, scale, orient, im), distance))
# if thisWord[0] in words:
# words[thisWord[0]] = words[thisWord[0]] + [(position, scale, orient, im)]
# else:
# words[thisWord[0]] = [(position, scale, orient, im)]
words.sort(key=operator.itemgetter(1))
wordPatches = []
for patch, _ in words:
im = misc.imread(patch[3])
wordPatches.append(
getPatchFromSIFTParameters(patch[0], patch[1], patch[2], rgb2gray(im)))
if len(wordPatches) > 25:
break
fig = plt.figure(figsize=(8, 8))
cols = 4
rows = 5
for j in range(1, cols * rows + 1):
img = wordPatches[j - 1]
fig.add_subplot(rows, cols, j)
plt.imshow(img, cmap=cm.Greys_r)
plt.show()
# for i in range(50):
# wordPatches = []
[coners[j][2][0], coners[j][3][0]],
color="g",
linestyle="-",
linewidth=1,
)
bx.plot(
[coners[j][3][1], coners[j][0][1]],
[coners[j][3][0], coners[j][0][0]],
color="g",
linestyle="-",
linewidth=1,
)
bx.set_xlim(0, im.shape[1])
bx.set_ylim(0, im.shape[0])
plt.gca().invert_yaxis()
plt.show()
# extract an image patch
print("displaying an image patch for one of the first 10 features", flush=True)
patch_num = random.choice(range(min(num_feats, 10)))
img_patch = getPatchFromSIFTParameters(
mat["positions"][patch_num, :],
mat["scales"][patch_num],
mat["orients"][patch_num],
rgb2gray(im),
)
plt.imshow(img_patch, cmap="gray")
plt.show()
