def vlad_experiment(X, list_of_K, list_of_PC, label_map, frm_map, figure_name, list_of_layers=constants.alex_net_layers): assert len(list_of_K) != 0 assert len(list_of_PC) != 0 for K in list_of_K: for PC in list_of_PC: for layer in list_of_layers: print("[VLAD] K = %3d PC = %3d layer = %s " % (K, PC, layer)) X_layer = X[layer] X_vlad = encoding.encode_VLAD(X_layer, K, PC) X_vlad_pca = pca(X_vlad) X_vlad_tsne = tsne(X_vlad) name = figure_name + '_VLAD_' + 'K' + str(K) + '_PC' + str( PC) + '_' + layer plot_annotated_embedding(X_vlad_pca, label_map, frm_map, name + '_pca', title="PCA - " + name) plot_annotated_embedding(X_vlad_tsne, label_map, frm_map, name + '_tsne', title="t-SNE - " + name)
def process_batch(self, PATH_TO_DATA, annotations, list_of_layers, sampling_rate, batch_size, LCD): i = 0 label_map = {} frm_map = {} X = {} map_index_data = pickle.load(open(annotations, "rb")) for index in map_index_data: segments = map_index_data[index] # print "Processing images for label " + str(index) for seg in segments: # print str(seg) frm_num = seg[0] b = 1 #Running count of num frames in batch batch_data = {} while frm_num <= (seg[1] + batch_size): # Initialize Batch if b == 1: label_map[i] = index frm_map[i] = frm_num # Process frames and build up features in batches im = caffe.io.load_image(utils.get_full_image_path(PATH_TO_DATA, frm_num)) self.net.blobs['data'].data[...] = self.transformer.preprocess('data', im) out = self.net.forward() for layer in list_of_layers: if LCD: if layer == 'input': print "ERROR: Cannot do LCD on input layer" sys.exit() data = self.net.blobs[layer].data[0] data = lcd.LCD(data) utils.dict_insert(layer, data, batch_data, axis = 0) else: if layer == 'input': data = cv2.imread(full_image_path) else: data = self.net.blobs[layer].data[0] data = utils.flatten(data) utils.dict_insert(layer, data, batch_data, axis = 1) if b == batch_size: print("Batch %3d" % i) b = 0 i += 1 # Concatenate with main data dictionary for layer in list_of_layers: data = encoding.encode_VLAD(batch_data[layer] , 5) utils.dict_insert(layer, data, X) batch_data = {} b += 1 frm_num += sampling_rate return X, label_map, frm_map
def vlad_experiment(X, list_of_K, list_of_PC, label_map, frm_map, figure_name, list_of_layers = constants.alex_net_layers): assert len(list_of_K) != 0 assert len(list_of_PC) != 0 for K in list_of_K: for PC in list_of_PC: for layer in list_of_layers: print("[VLAD] K = %3d PC = %3d layer = %s " % (K, PC, layer)) X_layer = X[layer] X_vlad = encoding.encode_VLAD(X_layer, K, PC) X_vlad_pca = pca(X_vlad) X_vlad_tsne = tsne(X_vlad) name = figure_name + '_VLAD_' + 'K' + str(K) + '_PC'+ str(PC) + '_' + layer plot_annotated_embedding(X_vlad_pca, label_map, frm_map, name + '_pca', title = "PCA - " + name) plot_annotated_embedding(X_vlad_tsne, label_map, frm_map, name + '_tsne', title = "t-SNE - " + name)
def featurize_LCD_VLAD(list_of_demonstrations, kinematics, layer, net_name, folder, dimensions, batch_size, fname, config=[True, True, True]): M = dimensions[0] a = dimensions[1] print "Featurizing LCD + VLAD: ", layer, net_name, folder, M, a, batch_size BATCH_SIZE = batch_size if constants.SIMULATION: BATCH_SIZE = 5 data_X_PCA = {} data_X_CCA = {} data_X_GRP = {} size_sampled_matrices = [ utils.sample_matrix(kinematics[demo], sampling_rate=BATCH_SIZE).shape[0] for demo in list_of_demonstrations ] PC = min(100, min(size_sampled_matrices)) print "PC: ", PC for demonstration in list_of_demonstrations: print demonstration W = kinematics[demonstration] Z = load_cnn_features(demonstration, layer, folder, net_name) W_new = utils.sample_matrix(W, sampling_rate=BATCH_SIZE) Z_batch = None W_batch = None j = 1 Z_new = None IPython.embed() PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str( constants.CAMERA) + ".p" start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) Z = [] IPython.embed() for i in range(Z): vector_W = W[i] W_batch = utils.safe_concatenate(W_batch, vector_W) vector_Z = Z[i] vector_Z = vector_Z.reshape(M, a, a) vector_Z = lcd.LCD(vector_Z) Z_batch = utils.safe_concatenate(Z_batch, vector_Z) if (j == BATCH_SIZE): print "NEW BATCH", str(i) Z_batch_VLAD = encoding.encode_VLAD(Z_batch) Z_new = utils.safe_concatenate(Z_new, Z_batch_VLAD) # Re-initialize batch variables j = 0 Z_batch = None W_batch = None j += 1 # tail case if Z_batch is not None: print "TAIL CASE" print "NEW BATCH", str(i) Z_batch_VLAD = encoding.encode_VLAD(Z_batch) Z_new = utils.safe_concatenate(Z_new, Z_batch_VLAD) if config[0]: Z_new_pca = utils.pca_incremental(Z_new, PC=PC) print Z_new_pca.shape assert W_new.shape[0] == Z_new_pca.shape[0] X_PCA = np.concatenate((W_new, Z_new_pca), axis=1) data_X_PCA[demonstration] = X_PCA if config[1]: Z_new_cca = utils.cca(W_new, Z_new) print Z_new_cca.shape assert W_new.shape[0] == Z_new_cca.shape[0] X_CCA = np.concatenate((W_new, Z_new_cca), axis=1) data_X_CCA[demonstration] = X_CCA if config[2]: Z_new_grp = utils.grp(Z_new) print Z_new_grp.shape assert W_new.shape[0] == Z_new_grp.shape[0] X_GRP = np.concatenate((W_new, Z_new_grp), axis=1) data_X_GRP[demonstration] = X_GRP if config[0]: pickle.dump(data_X_PCA, open(PATH_TO_FEATURES + fname + "_PCA" + ".p", "wb")) if config[1]: pickle.dump(data_X_CCA, open(PATH_TO_FEATURES + fname + "_CCA" + ".p", "wb")) if config[2]: pickle.dump(data_X_GRP, open(PATH_TO_FEATURES + fname + "_GRP" + ".p", "wb"))
def featurize_LCD_VLAD(list_of_demonstrations, kinematics, layer, net_name, folder, dimensions, batch_size, fname, config = [True, True, True]): M = dimensions[0] a = dimensions[1] print "Featurizing LCD + VLAD: ", layer, net_name, folder, M, a, batch_size BATCH_SIZE = batch_size if constants.SIMULATION: BATCH_SIZE = 5 data_X_PCA = {} data_X_CCA = {} data_X_GRP = {} size_sampled_matrices = [utils.sample_matrix(kinematics[demo], sampling_rate = BATCH_SIZE).shape[0] for demo in list_of_demonstrations] PC = min(100, min(size_sampled_matrices)) print "PC: ", PC for demonstration in list_of_demonstrations: print demonstration W = kinematics[demonstration] Z = load_cnn_features(demonstration, layer, folder, net_name) W_new = utils.sample_matrix(W, sampling_rate = BATCH_SIZE) Z_batch = None W_batch = None j = 1 Z_new = None IPython.embed() PATH_TO_ANNOTATION = constants.PATH_TO_DATA + constants.ANNOTATIONS_FOLDER + demonstration + "_" + str(constants.CAMERA) + ".p" start, end = utils.get_start_end_annotations(PATH_TO_ANNOTATION) Z = [] IPython.embed() for i in range(Z): vector_W = W[i] W_batch = utils.safe_concatenate(W_batch, vector_W) vector_Z = Z[i] vector_Z = vector_Z.reshape(M, a, a) vector_Z = lcd.LCD(vector_Z) Z_batch = utils.safe_concatenate(Z_batch, vector_Z) if (j == BATCH_SIZE): print "NEW BATCH", str(i) Z_batch_VLAD = encoding.encode_VLAD(Z_batch) Z_new = utils.safe_concatenate(Z_new, Z_batch_VLAD) # Re-initialize batch variables j = 0 Z_batch = None W_batch = None j += 1 # tail case if Z_batch is not None: print "TAIL CASE" print "NEW BATCH", str(i) Z_batch_VLAD = encoding.encode_VLAD(Z_batch) Z_new = utils.safe_concatenate(Z_new, Z_batch_VLAD) if config[0]: Z_new_pca = utils.pca_incremental(Z_new, PC = PC) print Z_new_pca.shape assert W_new.shape[0] == Z_new_pca.shape[0] X_PCA = np.concatenate((W_new, Z_new_pca), axis = 1) data_X_PCA[demonstration] = X_PCA if config[1]: Z_new_cca = utils.cca(W_new, Z_new) print Z_new_cca.shape assert W_new.shape[0] == Z_new_cca.shape[0] X_CCA = np.concatenate((W_new, Z_new_cca), axis = 1) data_X_CCA[demonstration] = X_CCA if config[2]: Z_new_grp = utils.grp(Z_new) print Z_new_grp.shape assert W_new.shape[0] == Z_new_grp.shape[0] X_GRP = np.concatenate((W_new, Z_new_grp), axis = 1) data_X_GRP[demonstration] = X_GRP if config[0]: pickle.dump(data_X_PCA, open(PATH_TO_FEATURES + fname + "_PCA" + ".p", "wb")) if config[1]: pickle.dump(data_X_CCA, open(PATH_TO_FEATURES + fname + "_CCA" + ".p", "wb")) if config[2]: pickle.dump(data_X_GRP, open(PATH_TO_FEATURES + fname + "_GRP" + ".p", "wb"))