# layers = ['fc7', 'fc6', 'pool5', 'conv4', 'conv3', 'pool2', 'pool1'] layers = ['fc7', 'fc6', 'pool5'] #n_components = [64, 128, 256] n_components = [64, 128, 256] c_type = 'pca' #==== dimensions for tsne ========= dimensions = 10 # trainclass = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] # trainlabels = utils.load_train_class_labels() #========= t-sne ============== for layer in layers: #==== load original feature ====== X0, ids = utils.load_feature_layer(layer) # keep_idxs = [] # for i, img_file in enumerate(ids): # klass = trainlabels[img_file] # if klass in trainclass: # keep_idxs.append(i) # # keep_idxs = np.asarray(keep_idxs, dtype=np.int64) # ids = np.asarray(ids, dtype=np.object) # # X0 = X0[keep_idxs] # ids = ids[keep_idxs] #ids = ids[0:500] #===== Convert ids (filename) to labels(integer) =========
:param n_bits: :return: """ import lsh_compressor as li return li.generateHashes(X, str(layer) + "_" + str(n_bits), n_bits), None if __name__ == '__main__': import time for layer in utils.feature_layers: # layer = 'fc7' n_components = 128 X, ids = utils.load_feature_layer(layer) scalar = utils.load_scalar(layer) X = X[0:1000, :] ids = ids[0:1000] print 'PCA Num Components : ', n_components start_time = time.clock() X_prime, pca = generate_pca_compression(X, n_components) print 'PCA Compression time : ', (time.clock() - start_time) print 'PCA Compression time per sample : ', (time.clock() - start_time) / X.shape[0] compression = 'pca' compression_path = os.path.join(utils.compression_dir, compression, layer) file_name = compression + '_' + str(n_components) + '_gzip.hkl'
lsh = LSHash(128, np.shape(testSet[0])[0], matrices_filename='lsh_planes.data.npz', overwrite=True) for idx, input_point in enumerate(testSet): hastValue = lsh._hash(lsh.uniform_planes[0], input_point.tolist()) print hastValue lsh.index(input_point, idx) print lsh.query(testSet[3], 3) return None if __name__ == '__main__': import utils nBits = 4096 scalar = utils.load_scalar(layer='fc7') fileName = os.path.join(utils.lsh_planes_dir, 'randomPlanesBias' + str(nBits)) var = np.mean(scalar.std_) generatePlanesWithBias(fileName, nBits, 4096, var) # trueIds, testSet = utils.load_test_set('fc7', 'raw', 0) dataset, ids = utils.load_feature_layer('fc7') labels = utils.load_train_class_labels() hashes = generateHashes(dataset, scalar, 'randomPlanesBias' + str(nBits), nBits) storeHashesInDb(ids, labels, hashes, 'lsh_fc7', nBits)
def store_feature(layers, compression): conn = psycopg2.connect(dbname=utils.dbname, user=utils.user, password=utils.password, host=utils.host) cur = conn.cursor() start_time = time.clock() train_labels = utils.load_train_class_labels() for layer in layers: # CREATE THE TABLE table_name = create_table_name(compression, layer) cur.execute("DROP TABLE IF EXISTS " + table_name + ";") table_command = "CREATE TABLE " + table_name + " (id serial PRIMARY KEY, file text, class integer, " insert_command = "INSERT INTO " + table_name + " (file, class," values_sql = "VALUES(%s,%s," dimensions = utils.get_dimension_options(layer, compression) if len(dimensions) == 0: print 'POSSIBLE ERROR: No dimensions loaded for ', layer, ' with ', compression continue for dim in dimensions: table_command += create_feature_column(dim) insert_command += create_feature_name(dim) + "," values_sql += "%s," table_command = table_command[:-1] + ");" values_sql = values_sql[:-1] + ");" insert_command = insert_command[:-1] + ") " + values_sql print table_command print insert_command cur.execute(table_command) # INSERT DATA INTO TABLE # load the data X, imagenet_ids = utils.load_feature_layer(layer) scalar = utils.load_scalar(layer) X = scalar.transform(X) transforms = [] # apply the compression algorithm for dim in dimensions: compressor = utils.load_compressor(layer, dim, compression) transforms.append(compressor.transform(X)) value = [] for i in range(X.shape[0]): file_name = imagenet_ids[i] value = [file_name, train_labels[file_name]] for X_prime in transforms: value.append(X_prime[i, :].tolist()) cur.execute(insert_command, value) conn.commit() cur.close() conn.close() print 'Done Creating Tables' print 'Total Time : ', time.clock() - start_time
def store_tsne_feature(layers, compression, tsne_dim): conn = psycopg2.connect(dbname=utils.dbname, user=utils.user, password=utils.password, host=utils.host) cur = conn.cursor() start_time = time.clock() eng = matlab.engine.start_matlab() train_labels = utils.load_train_class_labels() for layer in layers: # CREATE THE TABLE table_name = create_table_name(compression, layer) cur.execute("DROP TABLE IF EXISTS " + table_name + ";") table_command = "CREATE TABLE " + table_name + " (id serial PRIMARY KEY, file text, class integer, " insert_command = "INSERT INTO " + table_name + " (file, class," values_sql = "VALUES(%s,%s," if compression == 'tsne': dimensions = [64] #[64, 128, 256] else: dimensions = utils.get_dimension_options(layer, compression) if len(dimensions) == 0: print 'POSSIBLE ERROR: No dimensions loaded for ', layer, ' with ', compression continue for dim in dimensions: table_command += create_feature_column(dim) insert_command += create_feature_name(dim) + "," values_sql += "%s," table_command = table_command[:-1] + ");" values_sql = values_sql[:-1] + ");" insert_command = insert_command[:-1] + ") " + values_sql #print table_command #print insert_command cur.execute(table_command) # INSERT DATA INTO TABLE # load the data X, imagenet_ids = utils.load_feature_layer(layer) scalar = utils.load_scalar(layer) X = scalar.transform(X) X = X[keep_idxs] imagenet_ids = np.asarray(imagenet_ids, dtype=np.object) imagenet_ids = imagenet_ids[keep_idxs] transforms = [] # apply the compression algorithm for dim in dimensions: if compression == 'tsne': print 'tsne' compressor = utils.load_compressor(layer, dim, 'pca') #utils.plot_tsne_features('fc7',64) comp_X = compressor.transform(X) comp_X = comp_X.tolist() comp_X = matlab.double(comp_X) comp_X = eng.tsne_testing_python(comp_X, tsne_dim, layer, dim, 'pca') comp_X = np.array(comp_X) print comp_X transforms.append(comp_X) else: compressor = utils.load_compressor(layer, dim, compression) transforms.append(compressor.transform(X)) value = [] for i in range(X.shape[0]): file_name = imagenet_ids[i] value = [file_name, train_labels[file_name]] for X_prime in transforms: value.append(X_prime[i, :].tolist()) cur.execute(insert_command, value) conn.commit() cur.close() conn.close() eng.exit() print 'Done Creating Tables' print 'Total Time : ', time.clock() - start_time
for idx, input_point in enumerate(testSet): hastValue = lsh._hash(lsh.uniform_planes[0], input_point.tolist()) print hastValue lsh.index(input_point, idx) print lsh.query(testSet[3], 3) return None if __name__ == '__main__': import utils nBits = 4096 scalar = utils.load_scalar(layer='fc7') fileName = os.path.join(utils.lsh_planes_dir, 'randomPlanesBias' + str(nBits)) var = np.mean(scalar.std_) generatePlanesWithBias(fileName, nBits, 4096, var) # trueIds, testSet = utils.load_test_set('fc7', 'raw', 0) dataset, ids = utils.load_feature_layer('fc7') labels = utils.load_train_class_labels() hashes = generateHashes(dataset, scalar, 'randomPlanesBias' + str(nBits), nBits) storeHashesInDb(ids, labels, hashes, 'lsh_fc7', nBits)