def main(argv):
lmdb_name = sys.argv[1]
print "%s" % sys.argv[1]
batch_num = int(sys.argv[2])
batch_size = int(sys.argv[3])
window_num = batch_num * batch_size
start = time.time()
if 'db' not in locals().keys():
db = lmdb.open(lmdb_name)
txn = db.begin()
cursor = txn.cursor()
cursor.iternext()
datum = feat_helper_pb2.Datum()
keys = []
values = []
for key, value in enumerate(cursor.iternext_nodup()):
keys.append(key)
values.append(cursor.value())
ft = np.zeros((window_num, int(sys.argv[4])))
for im_idx in range(window_num):
datum.ParseFromString(values[im_idx])
ft[im_idx, :] = datum.float_data
print 'time 1: %f' % (time.time() - start)
sio.savemat(sys.argv[5], {'feats': ft})
print 'time 2: %f' % (time.time() - start)
print 'done!'
def main(argv):
leveldb_name = sys.argv[1]
print "%s" % sys.argv[1]
num_img = int(sys.argv[2])
dim_features = int(sys.argv[3])
file_target = sys.argv[4]
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
ft = np.zeros((num_img, dim_features))
for im_idx in range(num_img):
datum.ParseFromString(db.Get(str(im_idx).zfill(10)))
ft[im_idx, :] = datum.float_data
features_gallery = np.zeros((num_img / 2, dim_features * 2))
features_gallery = np.concatenate((ft[0::2, :], ft[1::2, :]), axis=1)
print 'Time (Assigning Values): %f' % (time.time() - start)
suffix = file_target[-3:]
if suffix == 'mat': # save in 'mat' format
sio.savemat(file_target, {'features_gallery': features_gallery})
else: # save in 'hdf5' format
h5f = h5py.File(file_target, 'w')
h5f.create_dataset('features_gallery', data=features_gallery)
h5f.close()
print 'Time (Writing to Disk): %f' % (time.time() - start)
print 'done!'
def main():
lmdb_name = "/media/dlw/work/featurefusion/bvlc_alexnet/feature/alexnet_myfc6_test21_4"
print("%s" % lmdb_name)
batch_num = int(1)
batch_size = int(2100)
window_num = batch_num * batch_size
start = time.time()
if 'db' not in locals().keys():
db = lmdb.open(lmdb_name)
txn = db.begin()
cursor = txn.cursor()
cursor.iternext()
datum = feat_helper_pb2.Datum()
keys = []
values = []
for key, value in enumerate(cursor.iternext_nodup()):
keys.append(key)
values.append(cursor.value())
print(len(values))
for i in range(batch_num):
ft = np.zeros((batch_size, int(4096)))
for im_idx in range(batch_size * i, batch_size * (i + 1)):
datum.ParseFromString(values[im_idx])
ft[im_idx - (batch_size) * i, :] = datum.float_data
print('deal with%s ' % str(i))
print('time 1: %f' % (time.time() - start))
sio.savemat(
"/media/dlw/work/featurefusion/bvlc_alexnet/feature/alexnet_myfc6_test21_4%s"
% str(i), {'feats': ft})
print('time 2: %f' % (time.time() - start))
print('done!')
def main(argv):
leveldb_name = sys.argv[1]
print "%s" % sys.argv[1]
batch_num = int(sys.argv[2])
batch_size = int(sys.argv[3])
window_num = batch_num * batch_size
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
ft = np.zeros((window_num, int(sys.argv[4])))
print "hello"
for it in db.RangeIter():
print "hello"
for im_idx in range(0, window_num):
print im_idx
print window_num
# print list(db.RangeIter())
db.Get('%d' % (im_idx))
datum.ParseFromString(db.Get('%d' % (im_idx)))
ft[im_idx, :] = datum.float_data
print 'time 1: %f' % (time.time() - start)
sio.savemat(sys.argv[5], {'feats': ft})
print 'time 2: %f' % (time.time() - start)
print 'done!'
def main(argv):
leveldb_name = sys.argv[1]
print "%s" % sys.argv[1]
# print "%s" % sys.argv[2]
# print "%s" % sys.argv[3]
# print "%s" % sys.argv[4]
# window_num = 1000;
# window_num = 12736;
# window_num = int(sys.argv[2]);
# window_num = 2845;
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
# iter = db.RangeIter(key_from ="%08d" %(0), include_value = False)
iter = db.RangeIter(include_value=False)
window_num = len(list(iter))
print window_num
# while (iter = iter.next())
datum.ParseFromString(db.Get("%08d" % (0)))
dim = len(datum.float_data)
print dim
ft = np.zeros((window_num, dim), dtype=np.float32)
for im_idx in range(window_num):
datum.ParseFromString(db.Get("%08d" % (im_idx)))
ft[im_idx, :] = datum.float_data
# print 'time 1: %f' %(time.time() - start)
sio.savemat(sys.argv[2], {'feats': ft}, oned_as='row')
print 'time 2: %f' % (time.time() - start)
print 'done!'
def main(argv):
leveldb_name = sys.argv[1]
print "%s" % sys.argv[1]
print "%s" % sys.argv[2]
print "%s" % sys.argv[3]
print "%s" % sys.argv[4]
# window_num = 1000;
# window_num = 12736;
window_num = int(sys.argv[2])
# window_num = 2845;
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
ft = np.zeros((window_num, int(sys.argv[3])))
for im_idx in range(window_num):
datum.ParseFromString(db.Get(str(im_idx)))
ft[im_idx, :] = datum.float_data
print 'time 1: %f' % (time.time() - start)
sio.savemat(sys.argv[4], {'feats': ft}, oned_as='row')
print 'time 2: %f' % (time.time() - start)
print 'done!'
def main(argv):
leveldb_name = sys.argv[1]
db_save_name = sys.argv[2]
print "%s" % sys.argv[1]
print "%s" % sys.argv[2]
# print "%s" % sys.argv[3]
# print "%s" % sys.argv[4]
# window_num = 1000;
# window_num = 12736;
# window_num = int(sys.argv[2]);
# window_num = 2845;
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
db_save = lmdb.open(db_save_name, map_size=int(1e12))
#batch = leveldb.WriteBatch()
datum_save = feat_helper_pb2.Datum()
iter = db.RangeIter(include_value=False)
window_num = len(list(iter))
print window_num
# while (iter = iter.next())
with db_save.begin(write=True) as in_save:
#ft = np.zeros((window_num, int(sys.argv[3])))
for im_idx in range(window_num):
key_str = '%08d' % (im_idx)
#datum.ParseFromString(db.Get(key_str))
#batch.Put(key_str, datum_save.SerializeToString())
in_save.put(key_str, db.Get(key_str))
if (im_idx % 10000 == 0):
print "%d images processed." % (im_idx)
#write leveldb
#db_save.Write(batch, sync = True)
db_save.close()
print 'time 1: %f' % (time.time() - start)
#sio.savemat(sys.argv[4], {'feats':ft},oned_as='row')
print 'time 2: %f' % (time.time() - start)
print 'done!'
def load_lmdb(lmdb_name, dim):
if 'db' not in locals().keys():
db = lmdb.open(lmdb_name)
txn = db.begin()
cursor = txn.cursor()
cursor.iternext()
datum = feat_helper_pb2.Datum()
keys = []
values = []
for key, value in enumerate(cursor.iternext_nodup()):
keys.append(key)
values.append(cursor.value())
num = len(values)
print 'number of samples: ', num
A = np.zeros((num, dim))
for im_idx in range(num):
datum.ParseFromString(values[im_idx])
A[im_idx, :] = datum.float_data
return A, num
def main(argv):
leveldb_name = sys.argv[1]
print "%s" % sys.argv[1]
batch_num = int(sys.argv[2]);
batch_size = int(sys.argv[3]);
window_num = batch_num*batch_size;
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
it = db.RangeIter()
#ft = np.zeros((int(sys.argv[6]), int(sys.argv[4])))
ft = np.zeros((window_num, int(sys.argv[4])))
#featureFile=sys.argv[5]
#output = open(featureFile, 'wb')
print "%s" %it
for key,value in it:
#print "%d" %int(sys.argv[6])
#if int(key)==int(sys.argv[6]):
# break
datum.ParseFromString(db.Get(key))
#if
print "%s," %key
print "%d" %len(datum.float_data)
#str = ' '.join(str(i) for i in datum.float_data)
#output.write(str)
#output.write("\n")
#print "%s" %value.size()
ft[int(key), :] = datum.float_data
#for im_idx in range(window_num):
#datum.ParseFromString(db.Get(im_idx))
#ft[im_idx, :] = datum.float_data
#sort_features = collections.OrderedDict(sorted(ft.items()))
#for k, arr in sort_features.iteritems():
print 'time 1: %f' %(time.time() - start)
sio.savemat(sys.argv[5], {'feats':ft})
print 'time 2: %f' %(time.time() - start)
print 'done!'
def main(argv):
leveldb_name = sys.argv[1]
print "%s" % sys.argv[1]
print "%s" % sys.argv[2]
# window_num = 1000;
# window_num = 12736;
# window_num = 2845;
iter = db.RangeIter(include_value = False)
window_num = len(list(iter))
print window_num
# while (iter = iter.next())
datum.ParseFromString(db.Get("%08d" %(0)))
dim = len(datum.float_data)
print dim
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
v_per_split = 1e8
split = np.ceil(float(window_num) * dim / v_per_split)
per_batch = int(np.ceil(float(window_num) / split))
for i in range(split):
start = i*per_batch
final = min((i+1)*per_batch,window_num)
ft = np.zeros((final-start, dim),dtype=np.float32)
for im_idx in range(start,final):
datum.ParseFromString(db.Get("%08d" %(im_idx)))
ft[im_idx-start, :] = datum.float_data
sio.savemat(sys.argv[2]+'_'+str(i+1), {'feats':ft},oned_as='row')
print 'done!'
def main(argv):
leveldb_name = sys.argv[1]
db_save_name = sys.argv[4]
print "%s" % sys.argv[1]
print "%s" % sys.argv[2]
print "%s" % sys.argv[3]
print "%s" % sys.argv[4]
print "%s" % sys.argv[5]
temp = int(sys.argv[5])
# window_num = 1000;
# window_num = 12736;
window_num = int(sys.argv[2])
# window_num = 2845;
start = time.time()
if 'db' not in locals().keys():
db = leveldb.LevelDB(leveldb_name)
datum = feat_helper_pb2.Datum()
db_save = lmdb.open(db_save_name, map_size=int(1e12))
#batch = leveldb.WriteBatch()
datum_save = feat_helper_pb2.Datum()
#ft = np.zeros((window_num, int(sys.argv[3])))
for im_idx in range(window_num):
key_str = '%08d' % (im_idx)
datum.ParseFromString(db.Get(key_str))
ft0 = np.array(datum.float_data)
#print ft0.shape
ft1 = ft0 / temp
m_ft1 = ft1.max()
ft1 = ft1 - m_ft1
exp_ft1 = np.exp(ft1)
exp_ft1_s = exp_ft1.sum()
p_ft1 = exp_ft1 / exp_ft1_s
del datum_save.float_data[:]
datum_save.float_data.extend(p_ft1.tolist())
#batch.Put(key_str, datum_save.SerializeToString())
db_save.begin(write=True).put(key_str, datum_save.SerializeToString())
if (im_idx % 10000 == 0):
print "%d images processed." % (im_idx)
# if (im_idx % 500 == 0 ):
# #print np.array(datum.float_data)
# print "sample %d" %(im_idx)
# print p_ft1.max()
# #calculate p when t=1
# m_ft0 = ft0.max()
# ft0 = ft0 - m_ft0
# exp_ft0 = np.exp(ft0)
# exp_ft0_s = exp_ft0.sum()
# p_ft0 = exp_ft0 / exp_ft0_s
# print p_ft0.max()
#write leveldb
#db_save.Write(batch, sync = True)
db_save.close()
print 'temperature is %d' % (temp)
print 'time 1: %f' % (time.time() - start)
#sio.savemat(sys.argv[4], {'feats':ft},oned_as='row')
print 'time 2: %f' % (time.time() - start)
print 'done!'
# -*- coding: utf-8 -*-
import scipy as sp
import numpy as np
import lmdb
import feat_helper_pb2
import time
start_time = time.time()
lmdb_name = 'data84_resnet_features'
if 'db' not in locals().keys():
db = lmdb.open(lmdb_name)
txn = db.begin()
cursor = txn.cursor()
cursor.iternext()
datum = feat_helper_pb2.Datum()
keys = []
values = []
for key, value in enumerate(cursor.iternext_nodup()):
keys.append(key)
values.append(cursor.value())
num = len(values)
dim = len(value[1])
print 'number_samples', num
print 'number_features', dim
A = np.zeros((num, dim))
for im_idx in range(num):
datum.ParseFromString(values[im_idx])
A[im_idx, :] = datum.float_data
np.savetxt('data84_resnet_features.txt', A)
