def get_bags_of_sifts(image_arrays, vocabulary, stride=5): """ You will want to construct SIFT features here in the same way you did in build_vocabulary() (except for possibly changing the sampling rate) and then assign each local feature to its nearest cluster center and build a histogram indicating how many times each cluster was used. Don't forget to normalize the histogram, or else a larger image with more SIFT features will look very different from a smaller version of the same image. Useful functions: - torch.from_numpy(img_array) for converting a numpy array to a torch tensor for siftnet. - torch.view() for reshaping the torch tensor - use torch.type() or np.array(img_array).astype() for typecasting - generate_sample_points() from utils.py for sampling interest points - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.histogram() : easy way to help you calculate for a particular image, how is the visual words span across the vocab. Check https://numpy.org/doc/stable/reference/generated/numpy.histogram.html for examples on how to use a histogram on an input array. - np.linalg.norm() for normalizing the histogram Useful note: - You will first need to convert each array in image_arrays into a float type array. Then convert each of these image arrays into a 4-D torch tensor by using torch.from_numpy(img_array) and then reshaping it to (1 x 1 x H x W) where H and W are the image height and width. You can use tensor views i.e torch.view to do the reshaping. Args: - image_arrays: A list of N PIL Image objects - vocabulary: A numpy array of dimensions: vocab_size x 128 where each row is a kmeans centroid or visual word. - stride: same functionality as the stride in build_vocabulary(). Returns: - image_feats: N x d matrix, where d is the dimensionality of the feature representation. In this case, d will equal the number of clusters or equivalently the number of entries in each image's histogram (vocab_size) below. """ # load vocabulary vocab = vocabulary vocab_size = len(vocab) num_images = len(image_arrays) feats = np.zeros((num_images, vocab_size)) ############################################################################# # TODO: YOUR CODE HERE ############################################################################# for i in range(len(image_arrays)): xv, yv = generate_sample_points(image_arrays[i].shape[0], image_arrays[i].shape[1], stride) img = torch.Tensor(image_arrays[i]).view( (1, 1, image_arrays[i].shape[0], image_arrays[i].shape[1])) img = img.type(torch.float32) sift = get_siftnet_features(img, xv, yv) # sift = np array indices = kmeans_quantize(sift, vocab) # counts, _ = np.histogram(indices, np.arange(vocab_size + 1)) # feats[i] = counts for idx in indices: feats[i, idx] += 1 feats[i, :] = feats[i, :] / np.linalg.norm(feats[i, :]) # feats = np.linalg.norm(feats) ############################################################################# # END OF YOUR CODE ############################################################################# return feats
def get_bags_of_sifts(image_arrays, vocabulary, step_size=10): """ This feature representation is described in the lecture materials, and Szeliski chapter 14. You will want to construct SIFT features here in the same way you did in build_vocabulary() (except for possibly changing the sampling rate) and then assign each local feature to its nearest cluster center and build a histogram indicating how many times each cluster was used. Don't forget to normalize the histogram, or else a larger image with more SIFT features will look very different from a smaller version of the same image. Useful functions: - np.array(img, dtype='float32'), torch.from_numpy(img_array), and img_tensor = img_tensor.reshape( (1, 1, img_array.shape[0], img_array.shape[1])) for converting a numpy array to a torch tensor for siftnet - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.histogram() or np.bincount(): easy way to help you calculate for a particular image, how is the visual words span across the vocab Args: - image_arrays: A list of input images in Numpy array, in grayscale - vocabulary: A numpy array of dimensions: vocab_size x 128 where each row is a kmeans centroid or visual word. - step_size: same functionality as the stride in build_vocabulary(). Feel free to experiment with different values, but the rationale is that you may want to set it smaller than stride in build_vocabulary() such that you collect more features from the image. Returns: - image_feats: N x d matrix, where d is the dimensionality of the feature representation. In this case, d will be equal to the number of clusters or equivalently the number of entries in each image's histogram (vocab_size) below. """ # load vocabulary vocab = vocabulary vocab_size = len(vocab) num_images = len(image_arrays) feats = np.zeros((num_images, vocab_size)) sift_vectors = np.zeros(128) for i, image in enumerate(image_arrays): img_h, img_w = image.shape image = np.array(image, dtype='float32') image_tensor = torch.from_numpy(image) image_tensor = image_tensor.reshape((1, 1, img_h, img_w)) h = np.arange(10, img_h - 10, step=step_size) w = np.arange(10, img_w - 10, step=step_size) grid_y, grid_x = np.meshgrid(h, w) dim2_idxs, dim3_idxs = grid_y.flatten(), grid_x.flatten() sift_feats = get_siftnet_features(image_tensor, dim3_idxs, dim2_idxs) # K, 128 indices = kmeans_quantize(sift_feats, vocab) hist, bin_edges = np.histogram(indices, bins=np.arange(vocab_size + 1)) feats[i] = hist return feats
def build_vocabulary(image_arrays, vocab_size=50, stride=20, max_iter=10): """ This function will generate the vocabulary which will be further used for bag of words classification. To generate the vocab you first randomly sample features from the training set. Get SIFT features for the images using get_siftnet_features() method. This method takes as input the image tensor and x and y coordinates as arguments. Now cluster sampled features from all images using kmeans method implemented by you previously and return the vocabulary of words i.e. cluster centers. Points to note: * To save computation time, you don't necessarily need to sample from all images, although it would be better to do so. * Sample the descriptors from each image to save memory and speed up the clustering. * For testing, you may experiment with larger stride so you just compute fewer points and check the result quickly. * The default vocab_size of 50 is sufficient for you to get a decent accuracy (>40%), but you are free to experiment with other values. Useful functions: - torch.from_numpy(img_array) for converting a numpy array to a torch tensor for siftnet - torch.view() for reshaping the torch tensor - use torch.type() or np.array(img_array).astype() for typecasting - generate_sample_points() from utils.py for sampling interest points Useful note: - You will first need to convert each array in image_arrays into a float type array. Then convert each of these image arrays into a 4-D torch tensor by using torch.from_numpy(img_array) and then reshaping it to (1 x 1 x H x W) where H and W are the image height and width. You can use tensor views i.e torch.view to do the reshaping. Args: - image_arrays: list of images in Numpy arrays, in grayscale - vocab_size: size of vocabulary - stride: the stride of your SIFT sampling Returns: - vocab: This is a (vocab_size, dim) Numpy array (vocabulary). Where dim is the length of your SIFT descriptor. Each row is a cluster center/visual word. """ dim = 128 # length of the SIFT descriptors that you are going to compute. vocab = None ############################################################################# # TODO: YOUR CODE HERE ############################################################################# vocab = np.zeros((vocab_size, dim)) total = np.zeros((8, dim)) # for _ in range(max_iter): for i in range(len(image_arrays)): xv, yv = generate_sample_points(image_arrays[i].shape[0], image_arrays[i].shape[1], stride) img = torch.Tensor(image_arrays[i]).view( (1, 1, image_arrays[i].shape[0], image_arrays[i].shape[1])) img = img.type(torch.float32) sift = get_siftnet_features(img, xv, yv) # sift = np array if i == 0: total = sift else: total = np.concatenate((total, sift), axis=0) vocab = kmeans(total, vocab_size) ############################################################################# # END OF YOUR CODE ############################################################################# return vocab
def build_vocabulary(image_arrays, vocab_size, stride=20): """ This function will sample SIFT descriptors from the training images, cluster them with kmeans, and then return the cluster centers. Load images from the training set. To save computation time, you don't necessarily need to sample from all images, although it would be better to do so. You can randomly sample the descriptors from each image to save memory and speed up the clustering. For testing, you may experiment with larger stride so you just compute fewer points and check the result quickly. In order to pass the unit test, leave out a 10-pixel margin in the image, that is, start your x and y from 10, and stop at len(image_width) - 10 and len(image_height) - 10. For each loaded image, get some SIFT features. You don't have to get as many SIFT features as you will in get_bags_of_sifts, because you're only trying to get a representative sample here. Once you have tens of thousands of SIFT features from many training images, cluster them with kmeans. The resulting centroids are now your visual word vocabulary. Note that the default vocab_size of 50 is sufficient for you to get a decent accuracy (>40%), but you are free to experiment with other values. Useful functions: - np.array(img, dtype='float32'), torch.from_numpy(img_array), and img_tensor = img_tensor.reshape( (1, 1, img_array.shape[0], img_array.shape[1])) for converting a numpy array to a torch tensor for siftnet - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.arange() and np.meshgrid(): for you to generate the sample x and y positions faster Args: - image_arrays: list of images in Numpy arrays, in grayscale - vocab_size: size of vocabulary - stride: the stride of your SIFT sampling Returns: - vocab: This is a (vocab_size, dim) Numpy array (vocabulary). Where dim is the length of your SIFT descriptor. Each row is a cluster center / visual word. """ dim = 128 # length of the SIFT descriptors that you are going to compute. vocab = np.zeros((vocab_size, dim)) sift_vectors = np.zeros(dim) for image in image_arrays: img_h, img_w = image.shape image = np.array(image, dtype='float32') image_tensor = torch.from_numpy(image) image_tensor = image_tensor.reshape((1, 1, img_h, img_w)) h = np.arange(10, img_h - 10, step=stride) w = np.arange(10, img_w - 10, step=stride) grid_y, grid_x = np.meshgrid(h, w) dim2_idxs, dim3_idxs = grid_y.flatten(), grid_x.flatten() feats = get_siftnet_features(image_tensor, dim3_idxs, dim2_idxs) sift_vectors = np.vstack((sift_vectors, feats)) vocab = kmeans(sift_vectors[1:], vocab_size) return vocab
def get_bags_of_sifts(image_arrays, vocabulary, step_size=10): """ This feature representation is described in the lecture materials, and Szeliski chapter 14. You will want to construct SIFT features here in the same way you did in build_vocabulary() (except for possibly changing the sampling rate) and then assign each local feature to its nearest cluster center and build a histogram indicating how many times each cluster was used. Don't forget to normalize the histogram, or else a larger image with more SIFT features will look very different from a smaller version of the same image. Useful functions: - np.array(img, dtype='float32'), torch.from_numpy(img_array), and img_tensor = img_tensor.reshape( (1, 1, img_array.shape[0], img_array.shape[1])) for converting a numpy array to a torch tensor for siftnet - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.histogram() or np.bincount(): easy way to help you calculate for a particular image, how is the visual words span across the vocab Args: - image_arrays: A list of input images in Numpy array, in grayscale - vocabulary: A numpy array of dimensions: vocab_size x 128 where each row is a kmeans centroid or visual word. - step_size: same functionality as the stride in build_vocabulary(). Feel free to experiment with different values, but the rationale is that you may want to set it smaller than stride in build_vocabulary() such that you collect more features from the image. Returns: - image_feats: N x d matrix, where d is the dimensionality of the feature representation. In this case, d will be equal to the number of clusters or equivalently the number of entries in each image's histogram (vocab_size) below. """ # load vocabulary vocab = vocabulary vocab_size = len(vocab) num_images = len(image_arrays) feats = np.zeros((num_images, vocab_size)) ########################################################################### # TODO: YOUR CODE HERE # ########################################################################### # print("START") # print(len(image_arrays)) # counter = 0 features = [None] * len(image_arrays) for i in range(len(image_arrays)): # print("img") img = image_arrays[i] # counter += 1 # if counter % 100 == 0: # print("PROGRESS") img_array = np.array(img, dtype='float32') img_tensor = torch.from_numpy(img_array) img_final = img_tensor.reshape(1, 1, img.shape[0], img.shape[1]) xv, yv = np.meshgrid(np.arange(10, img.shape[0] - 10, step_size), np.arange(10, img.shape[1] - 10, step_size)) sift_features = get_siftnet_features(img_final, yv.flatten(), xv.flatten()) indices = kmeans_quantize( sift_features, vocab) # For each feature, find the centroid it's closest to bins = np.bincount(indices, minlength=vocab_size) bins = bins / np.linalg.norm(bins) features[i] = bins feats = np.array(features) # print("DONE") ########################################################################### # END OF YOUR CODE # ########################################################################### return feats
def build_vocabulary(image_arrays, vocab_size, stride=20): """ This function will sample SIFT descriptors from the training images, cluster them with kmeans, and then return the cluster centers. Load images from the training set. To save computation time, you don't necessarily need to sample from all images, although it would be better to do so. You can randomly sample the descriptors from each image to save memory and speed up the clustering. For testing, you may experiment with larger stride so you just compute fewer points and check the result quickly. In order to pass the unit test, leave out a 10-pixel margin in the image, that is, start your x and y from 10, and stop at len(image_width) - 10 and len(image_height) - 10. For each loaded image, get some SIFT features. You don't have to get as many SIFT features as you will in get_bags_of_sifts, because you're only trying to get a representative sample here. Once you have tens of thousands of SIFT features from many training images, cluster them with kmeans. The resulting centroids are now your visual word vocabulary. Note that the default vocab_size of 50 is sufficient for you to get a decent accuracy (>40%), but you are free to experiment with other values. Useful functions: - np.array(img, dtype='float32'), torch.from_numpy(img_array), and img_tensor = img_tensor.reshape( (1, 1, img_array.shape[0], img_array.shape[1])) for converting a numpy array to a torch tensor for siftnet - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.arange() and np.meshgrid(): for you to generate the sample x and y positions faster Args: - image_arrays: list of images in Numpy arrays, in grayscale - vocab_size: size of vocabulary - stride: the stride of your SIFT sampling Returns: - vocab: This is a (vocab_size, dim) Numpy array (vocabulary). Where dim is the length of your SIFT descriptor. Each row is a cluster center / visual word. """ dim = 128 # length of the SIFT descriptors that you are going to compute. vocab = None ########################################################################### # TODO: YOUR CODE HERE # ########################################################################### # print(len(image_arrays)) features = np.array([]) for img in image_arrays: img_array = np.array(img, dtype='float32') img_tensor = torch.from_numpy(img_array) img_final = img_tensor.reshape(1, 1, img.shape[0], img.shape[1]) # sift_features = get_siftnet_features(img_final, np.arange(10, img.shape[0]-10, 20), np.arange(10, img.shape[1]-10, 20)) xv, yv = np.meshgrid(np.arange(10, img.shape[0] - 10, stride), np.arange(10, img.shape[1] - 10, stride)) sift_features = get_siftnet_features(img_final, yv.flatten(), xv.flatten()) if len(features) == 0: features = sift_features else: features = np.concatenate((features, sift_features)) # print("HALFWAY") vocab = kmeans(features, vocab_size) # print("vocab shape: ", vocab.shape) ########################################################################### # END OF YOUR CODE # ########################################################################### return vocab
def get_bags_of_sifts(image_arrays, vocabulary, step_size = 10): """ This feature representation is described in the lecture materials, and Szeliski chapter 14. You will want to construct SIFT features here in the same way you did in build_vocabulary() (except for possibly changing the sampling rate) and then assign each local feature to its nearest cluster center and build a histogram indicating how many times each cluster was used. Don't forget to normalize the histogram, or else a larger image with more SIFT features will look very different from a smaller version of the same image. Useful functions: - np.array(img, dtype='float32'), torch.from_numpy(img_array), and img_tensor = img_tensor.reshape( (1, 1, img_array.shape[0], img_array.shape[1])) for converting a numpy array to a torch tensor for siftnet - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.histogram() or np.bincount(): easy way to help you calculate for a particular image, how is the visual words span across the vocab Args: - image_arrays: A list of input images in Numpy array, in grayscale - vocabulary: A numpy array of dimensions: vocab_size x 128 where each row is a kmeans centroid or visual word. - step_size: same functionality as the stride in build_vocabulary(). Feel free to experiment with different values, but the rationale is that you may want to set it smaller than stride in build_vocabulary() such that you collect more features from the image. Returns: - image_feats: N x d matrix, where d is the dimensionality of the feature representation. In this case, d will be equal to the number of clusters or equivalently the number of entries in each image's histogram (vocab_size) below. """ # load vocabulary vocab = vocabulary vocab_size = len(vocab) num_images = len(image_arrays) feats = np.zeros((num_images, vocab_size)) ########################################################################### # TODO: YOUR CODE HERE # ########################################################################### import time start = time.time() img_feats = [] idx = 0 stride = step_size for img in image_arrays: #img_width = img.shape[0] #img_height = img.shape[1] img_array = np.array(img, dtype='float32') img_width = img.shape[1] img_height = img.shape[0] img_tensor = torch.from_numpy(img_array) img_tensor = img_tensor.reshape((1, 1, img.shape[0], img.shape[1])) x_s = np.arange(10, img_width - 10, stride) y_s = np.arange(10, img_height - 10, stride) x, y = np.meshgrid(x_s, y_s) x = x.flatten() y = y.flatten() x_length = x.shape[0] #print(x_length) #sift_feats.append(np.array( get_siftnet_features(img_tensor, x, y))) """if idx == 0: sift_feats= np.array( get_siftnet_features(img_tensor, x, y)) img_feats.append(sift_feats) else: new_feats = np.array( get_siftnet_features(img_tensor, x, y)) img_feats.append(new_feats) sift_feats = np.concatenate((sift_feats, new_feats)) idx = idx + 1""" centroids = vocabulary sift_feats = np.array(get_siftnet_features(img_tensor, x, y)) quantized = kmeans_quantize(sift_feats, centroids) bins = np.arange(len(vocab) + 1) hist = np.histogram(quantized, bins)[0] normed_hist = np.linalg.norm(hist) hist = np.divide(hist, normed_hist) feats[idx] = hist idx = idx + 1
def build_vocabulary(image_arrays, vocab_size, stride = 20): """ This function will sample SIFT descriptors from the training images, cluster them with kmeans, and then return the cluster centers. Load images from the training set. To save computation time, you don't necessarily need to sample from all images, although it would be better to do so. You can randomly sample the descriptors from each image to save memory and speed up the clustering. For testing, you may experiment with larger stride so you just compute fewer points and check the result quickly. In order to pass the unit test, leave out a 10-pixel margin in the image, that is, start your x and y from 10, and stop at len(image_width) - 10 and len(image_height) - 10. For each loaded image, get some SIFT features. You don't have to get as many SIFT features as you will in get_bags_of_sifts, because you're only trying to get a representative sample here. Once you have tens of thousands of SIFT features from many training images, cluster them with kmeans. The resulting centroids are now your visual word vocabulary. Note that the default vocab_size of 50 is sufficient for you to get a decent accuracy (>40%), but you are free to experiment with other values. Useful functions: - np.array(img, dtype='float32'), torch.from_numpy(img_array), and img_tensor = img_tensor.reshape( (1, 1, img_array.shape[0], img_array.shape[1])) for converting a numpy array to a torch tensor for siftnet - get_siftnet_features() from SIFTNet: you can pass in the image tensor in grayscale, together with the sampled x and y positions to obtain the SIFT features - np.arange() and np.meshgrid(): for you to generate the sample x and y positions faster Args: - image_arrays: list of images in Numpy arrays, in grayscale - vocab_size: size of vocabulary - stride: the stride of your SIFT sampling Returns: - vocab: This is a (vocab_size, dim) Numpy array (vocabulary). Where dim is the length of your SIFT descriptor. Each row is a cluster center / visual word. """ import time start = time.time() dim = 128 # length of the SIFT descriptors that you are going to compute. vocab = None ########################################################################### # TODO: YOUR CODE HERE # ########################################################################### #10, and stop at len(image_width) - 10 #len(image_height) - 10 # 1. For each image in image array, convert to tensor. Reshape # 2. Loop through image with correct bounds and stride, get_sift_features() # 3. Add all sift features to np.array called sift_feats # 4. Run K means on sift feats, the vocab words are the centroids sift_feats = [] x_length = 0 idx = 0 for img in image_arrays: #img_width = img.shape[0] #img_height = img.shape[1] img_array = np.array(img, dtype='float32') img_width = img.shape[1] img_height = img.shape[0] img_tensor = torch.from_numpy(img_array) img_tensor = img_tensor.reshape((1, 1, img.shape[0], img.shape[1])) x_s = np.arange(10, img_width - 10, stride) y_s = np.arange(10, img_height - 10, stride) x, y = np.meshgrid(x_s, y_s) x = x.flatten() y = y.flatten() x_length = x.shape[0] #sift_feats.append(np.array( get_siftnet_features(img_tensor, x, y))) if idx == 0: sift_feats= np.array( get_siftnet_features(img_tensor, x, y)) else: new_feats = np.array( get_siftnet_features(img_tensor, x, y)) sift_feats = np.concatenate((sift_feats, new_feats)) idx = idx + 1 #for f in sift_features: # sift_feats.append(f) #feat_array = np.array(sift_feats) #if (sift_feats.ndim > 2): #print(sift_feats.shape) #N = sift_feats.shape[0]*sift_feats.shape[1] #feat_array = sift_feats.reshape((N, dim)) #N = x_length * len(image_arrays) #feat_array = feat_array.reshape((N, dim)) #centroids = kmeans(sift_feats, len(image_arrays), max_iter = 1) centroids = kmeans(sift_feats, vocab_size, max_iter = 15) if len(centroids) > vocab_size: vocab = centroids[:vocab_size] else: vocab = centroids end = time.time() #print("Took: {} sec".format(end - start)) #vocab = centroids #vocab must be size vocab size, dim ########################################################################### # END OF YOUR CODE # ########################################################################### return vocab