# layers = ['fc7', 'fc6', 'pool5', 'conv4', 'conv3', 'pool2', 'pool1']
layers = ['fc7', 'fc6', 'pool5']
#n_components = [64, 128, 256]
n_components = [64, 128, 256]
c_type = 'pca'
#==== dimensions for tsne =========
dimensions = 10
# trainclass = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
# trainlabels = utils.load_train_class_labels()
#========= t-sne ==============
for layer in layers:
#==== load original feature ======
X0, ids = utils.load_feature_layer(layer)
# keep_idxs = []
# for i, img_file in enumerate(ids):
# klass = trainlabels[img_file]
# if klass in trainclass:
# keep_idxs.append(i)
#
# keep_idxs = np.asarray(keep_idxs, dtype=np.int64)
# ids = np.asarray(ids, dtype=np.object)
#
# X0 = X0[keep_idxs]
# ids = ids[keep_idxs]
#ids = ids[0:500]
#===== Convert ids (filename) to labels(integer) =========
:param n_bits:
:return:
"""
import lsh_compressor as li
return li.generateHashes(X, str(layer) + "_" + str(n_bits), n_bits), None
if __name__ == '__main__':
import time
for layer in utils.feature_layers:
# layer = 'fc7'
n_components = 128
X, ids = utils.load_feature_layer(layer)
scalar = utils.load_scalar(layer)
X = X[0:1000, :]
ids = ids[0:1000]
print 'PCA Num Components : ', n_components
start_time = time.clock()
X_prime, pca = generate_pca_compression(X, n_components)
print 'PCA Compression time : ', (time.clock() - start_time)
print 'PCA Compression time per sample : ', (time.clock() - start_time) / X.shape[0]
compression = 'pca'
compression_path = os.path.join(utils.compression_dir, compression, layer)
file_name = compression + '_' + str(n_components) + '_gzip.hkl'
lsh = LSHash(128, np.shape(testSet[0])[0], matrices_filename='lsh_planes.data.npz', overwrite=True)
for idx, input_point in enumerate(testSet):
hastValue = lsh._hash(lsh.uniform_planes[0], input_point.tolist())
print hastValue
lsh.index(input_point, idx)
print lsh.query(testSet[3], 3)
return None
if __name__ == '__main__':
import utils
nBits = 4096
scalar = utils.load_scalar(layer='fc7')
fileName = os.path.join(utils.lsh_planes_dir, 'randomPlanesBias' + str(nBits))
var = np.mean(scalar.std_)
generatePlanesWithBias(fileName, nBits, 4096, var)
# trueIds, testSet = utils.load_test_set('fc7', 'raw', 0)
dataset, ids = utils.load_feature_layer('fc7')
labels = utils.load_train_class_labels()
hashes = generateHashes(dataset, scalar, 'randomPlanesBias' + str(nBits), nBits)
storeHashesInDb(ids, labels, hashes, 'lsh_fc7', nBits)
def store_feature(layers, compression):
conn = psycopg2.connect(dbname=utils.dbname, user=utils.user, password=utils.password, host=utils.host)
cur = conn.cursor()
start_time = time.clock()
train_labels = utils.load_train_class_labels()
for layer in layers:
# CREATE THE TABLE
table_name = create_table_name(compression, layer)
cur.execute("DROP TABLE IF EXISTS " + table_name + ";")
table_command = "CREATE TABLE " + table_name + " (id serial PRIMARY KEY, file text, class integer, "
insert_command = "INSERT INTO " + table_name + " (file, class,"
values_sql = "VALUES(%s,%s,"
dimensions = utils.get_dimension_options(layer, compression)
if len(dimensions) == 0:
print 'POSSIBLE ERROR: No dimensions loaded for ', layer, ' with ', compression
continue
for dim in dimensions:
table_command += create_feature_column(dim)
insert_command += create_feature_name(dim) + ","
values_sql += "%s,"
table_command = table_command[:-1] + ");"
values_sql = values_sql[:-1] + ");"
insert_command = insert_command[:-1] + ") " + values_sql
print table_command
print insert_command
cur.execute(table_command)
# INSERT DATA INTO TABLE
# load the data
X, imagenet_ids = utils.load_feature_layer(layer)
scalar = utils.load_scalar(layer)
X = scalar.transform(X)
transforms = []
# apply the compression algorithm
for dim in dimensions:
compressor = utils.load_compressor(layer, dim, compression)
transforms.append(compressor.transform(X))
value = []
for i in range(X.shape[0]):
file_name = imagenet_ids[i]
value = [file_name, train_labels[file_name]]
for X_prime in transforms:
value.append(X_prime[i, :].tolist())
cur.execute(insert_command, value)
conn.commit()
cur.close()
conn.close()
print 'Done Creating Tables'
print 'Total Time : ', time.clock() - start_time
def store_tsne_feature(layers, compression, tsne_dim):
conn = psycopg2.connect(dbname=utils.dbname,
user=utils.user,
password=utils.password,
host=utils.host)
cur = conn.cursor()
start_time = time.clock()
eng = matlab.engine.start_matlab()
train_labels = utils.load_train_class_labels()
for layer in layers:
# CREATE THE TABLE
table_name = create_table_name(compression, layer)
cur.execute("DROP TABLE IF EXISTS " + table_name + ";")
table_command = "CREATE TABLE " + table_name + " (id serial PRIMARY KEY, file text, class integer, "
insert_command = "INSERT INTO " + table_name + " (file, class,"
values_sql = "VALUES(%s,%s,"
if compression == 'tsne':
dimensions = [64] #[64, 128, 256]
else:
dimensions = utils.get_dimension_options(layer, compression)
if len(dimensions) == 0:
print 'POSSIBLE ERROR: No dimensions loaded for ', layer, ' with ', compression
continue
for dim in dimensions:
table_command += create_feature_column(dim)
insert_command += create_feature_name(dim) + ","
values_sql += "%s,"
table_command = table_command[:-1] + ");"
values_sql = values_sql[:-1] + ");"
insert_command = insert_command[:-1] + ") " + values_sql
#print table_command
#print insert_command
cur.execute(table_command)
# INSERT DATA INTO TABLE
# load the data
X, imagenet_ids = utils.load_feature_layer(layer)
scalar = utils.load_scalar(layer)
X = scalar.transform(X)
X = X[keep_idxs]
imagenet_ids = np.asarray(imagenet_ids, dtype=np.object)
imagenet_ids = imagenet_ids[keep_idxs]
transforms = []
# apply the compression algorithm
for dim in dimensions:
if compression == 'tsne':
print 'tsne'
compressor = utils.load_compressor(layer, dim, 'pca')
#utils.plot_tsne_features('fc7',64)
comp_X = compressor.transform(X)
comp_X = comp_X.tolist()
comp_X = matlab.double(comp_X)
comp_X = eng.tsne_testing_python(comp_X, tsne_dim, layer, dim,
'pca')
comp_X = np.array(comp_X)
print comp_X
transforms.append(comp_X)
else:
compressor = utils.load_compressor(layer, dim, compression)
transforms.append(compressor.transform(X))
value = []
for i in range(X.shape[0]):
file_name = imagenet_ids[i]
value = [file_name, train_labels[file_name]]
for X_prime in transforms:
value.append(X_prime[i, :].tolist())
cur.execute(insert_command, value)
conn.commit()
cur.close()
conn.close()
eng.exit()
print 'Done Creating Tables'
print 'Total Time : ', time.clock() - start_time
for idx, input_point in enumerate(testSet):
hastValue = lsh._hash(lsh.uniform_planes[0], input_point.tolist())
print hastValue
lsh.index(input_point, idx)
print lsh.query(testSet[3], 3)
return None
if __name__ == '__main__':
import utils
nBits = 4096
scalar = utils.load_scalar(layer='fc7')
fileName = os.path.join(utils.lsh_planes_dir,
'randomPlanesBias' + str(nBits))
var = np.mean(scalar.std_)
generatePlanesWithBias(fileName, nBits, 4096, var)
# trueIds, testSet = utils.load_test_set('fc7', 'raw', 0)
dataset, ids = utils.load_feature_layer('fc7')
labels = utils.load_train_class_labels()
hashes = generateHashes(dataset, scalar, 'randomPlanesBias' + str(nBits),
nBits)
storeHashesInDb(ids, labels, hashes, 'lsh_fc7', nBits)
def store_feature(layers, compression):
conn = psycopg2.connect(dbname=utils.dbname,
user=utils.user,
password=utils.password,
host=utils.host)
cur = conn.cursor()
start_time = time.clock()
train_labels = utils.load_train_class_labels()
for layer in layers:
# CREATE THE TABLE
table_name = create_table_name(compression, layer)
cur.execute("DROP TABLE IF EXISTS " + table_name + ";")
table_command = "CREATE TABLE " + table_name + " (id serial PRIMARY KEY, file text, class integer, "
insert_command = "INSERT INTO " + table_name + " (file, class,"
values_sql = "VALUES(%s,%s,"
dimensions = utils.get_dimension_options(layer, compression)
if len(dimensions) == 0:
print 'POSSIBLE ERROR: No dimensions loaded for ', layer, ' with ', compression
continue
for dim in dimensions:
table_command += create_feature_column(dim)
insert_command += create_feature_name(dim) + ","
values_sql += "%s,"
table_command = table_command[:-1] + ");"
values_sql = values_sql[:-1] + ");"
insert_command = insert_command[:-1] + ") " + values_sql
print table_command
print insert_command
cur.execute(table_command)
# INSERT DATA INTO TABLE
# load the data
X, imagenet_ids = utils.load_feature_layer(layer)
scalar = utils.load_scalar(layer)
X = scalar.transform(X)
transforms = []
# apply the compression algorithm
for dim in dimensions:
compressor = utils.load_compressor(layer, dim, compression)
transforms.append(compressor.transform(X))
value = []
for i in range(X.shape[0]):
file_name = imagenet_ids[i]
value = [file_name, train_labels[file_name]]
for X_prime in transforms:
value.append(X_prime[i, :].tolist())
cur.execute(insert_command, value)
conn.commit()
cur.close()
conn.close()
print 'Done Creating Tables'
print 'Total Time : ', time.clock() - start_time
def store_tsne_feature(layers, compression, tsne_dim):
conn = psycopg2.connect(dbname=utils.dbname, user=utils.user, password=utils.password, host=utils.host)
cur = conn.cursor()
start_time = time.clock()
eng = matlab.engine.start_matlab()
train_labels = utils.load_train_class_labels()
for layer in layers:
# CREATE THE TABLE
table_name = create_table_name(compression, layer)
cur.execute("DROP TABLE IF EXISTS " + table_name + ";")
table_command = "CREATE TABLE " + table_name + " (id serial PRIMARY KEY, file text, class integer, "
insert_command = "INSERT INTO " + table_name + " (file, class,"
values_sql = "VALUES(%s,%s,"
if compression == 'tsne':
dimensions = [64] #[64, 128, 256]
else:
dimensions = utils.get_dimension_options(layer, compression)
if len(dimensions) == 0:
print 'POSSIBLE ERROR: No dimensions loaded for ', layer, ' with ', compression
continue
for dim in dimensions:
table_command += create_feature_column(dim)
insert_command += create_feature_name(dim) + ","
values_sql += "%s,"
table_command = table_command[:-1] + ");"
values_sql = values_sql[:-1] + ");"
insert_command = insert_command[:-1] + ") " + values_sql
#print table_command
#print insert_command
cur.execute(table_command)
# INSERT DATA INTO TABLE
# load the data
X, imagenet_ids = utils.load_feature_layer(layer)
scalar = utils.load_scalar(layer)
X = scalar.transform(X)
X = X[keep_idxs]
imagenet_ids = np.asarray(imagenet_ids, dtype=np.object)
imagenet_ids = imagenet_ids[keep_idxs]
transforms = []
# apply the compression algorithm
for dim in dimensions:
if compression == 'tsne':
print 'tsne'
compressor = utils.load_compressor(layer, dim, 'pca')
#utils.plot_tsne_features('fc7',64)
comp_X = compressor.transform(X)
comp_X = comp_X.tolist()
comp_X = matlab.double(comp_X)
comp_X = eng.tsne_testing_python(comp_X, tsne_dim, layer, dim, 'pca')
comp_X = np.array(comp_X)
print comp_X
transforms.append(comp_X)
else:
compressor = utils.load_compressor(layer, dim, compression)
transforms.append(compressor.transform(X))
value = []
for i in range(X.shape[0]):
file_name = imagenet_ids[i]
value = [file_name, train_labels[file_name]]
for X_prime in transforms:
value.append(X_prime[i, :].tolist())
cur.execute(insert_command, value)
conn.commit()
cur.close()
conn.close()
eng.exit()
print 'Done Creating Tables'
print 'Total Time : ', time.clock() - start_time