def test_bytesize(self):
assert dd.humanize_bytesize(1) == '1 B'
assert dd.humanize_bytesize(2 * 1024) == '2 KB'
assert dd.humanize_bytesize(3 * 1024**2) == '3 MB'
assert dd.humanize_bytesize(4 * 1024**3) == '4 GB'
assert dd.humanize_bytesize(5 * 1024**4) == '5 TB'
assert dd.bytesize(np.ones((5, 2), dtype=np.int16)) == 20
assert dd.memsize(np.ones((5, 2), dtype=np.int16)) == '20 B'
def test_bytesize(self):
assert dd.humanize_bytesize(1) == '1 B'
assert dd.humanize_bytesize(2 * 1024) == '2 KB'
assert dd.humanize_bytesize(3 * 1024**2) == '3 MB'
assert dd.humanize_bytesize(4 * 1024**3) == '4 GB'
assert dd.humanize_bytesize(5 * 1024**4) == '5 TB'
assert dd.bytesize(np.ones((5, 2), dtype=np.int16)) == 20
assert dd.memsize(np.ones((5, 2), dtype=np.int16)) == '20 B'
def classifyNN(self):
utils.mkdir_p(self.outDir)
self.readDataset()
# map_size = dd.bytesize(self.X_train) * 2
# env = lmdb.open('../computed/X_train_lmdb',
# map_size=map_size)
# for i in range(self.X_train.shape[0]):
# if i % 1000 == 0:
# print(i)
# datum = caffe.proto.caffe_pb2.Datum()
# datum.data = self.X_train[i, :].tostring()
# datum.label = int(self.y_train[i])
# with env.begin(write=True) as txn:
# txn.put(str(i), datum.SerializeToString())
# env.close()
map_size = dd.bytesize(self.X_test) * 2
env = lmdb.open('../computed/X_test_lmdb', map_size=map_size)
for i in range(self.X_test.shape[0]):
if i % 1000 == 0:
print(i)
datum = caffe.proto.caffe_pb2.Datum()
datum.data = self.X_test[i, :].tostring()
with env.begin(write=True) as txn:
txn.put(str(i), datum.SerializeToString())
env.close()
# lmdb_env = lmdb.open('../computed/X_train_lmdb/')
lmdb_env = lmdb.open('../computed/X_test_lmdb/')
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe_pb2.Datum()
i = 0
for key, value in lmdb_cursor:
datum.ParseFromString(value)
x = np.fromstring(datum.data)
y = datum.label
print(x)
print(y)
if i >= 10:
break
i = i + 1
lmdb_env.close()
def createLMDBLabel(Y, dbName, order):
N = Y.shape[0]
Y = np.array(Y, dtype=np.int64)
arr = order
map_size = dd.bytesize(Y) * 100
env = lmdb.open(dbName, map_size=map_size)
for i in range(N):
datum = caffe.proto.caffe_pb2.Datum()
datum.channels = 1
datum.height = 1
datum.width = 1
datum.data = np.zeros((1, 1, 1)).tobytes() # or .tostring() if numpy < 1.9
datum.label = int(Y[arr[i]])
str_id = "{:08}".format(i)
with env.begin(write=True) as txn:
# txn is a Transaction object
# The encode is only essential in Python 3
txn.put(str_id.encode("ascii"), datum.SerializeToString())
def createLMDBLabel(Y, dbName, order):
N = Y.shape[0]
Y = np.array(Y, dtype=np.int64)
arr = order
map_size = dd.bytesize(Y) * 100
env = lmdb.open(dbName, map_size=map_size)
for i in range(N):
datum = caffe.proto.caffe_pb2.Datum()
datum.channels = 1
datum.height = 1
datum.width = 1
datum.data = np.zeros(
(1, 1, 1)).tobytes() # or .tostring() if numpy < 1.9
datum.label = int(Y[arr[i]])
str_id = '{:08}'.format(i)
with env.begin(write=True) as txn:
# txn is a Transaction object
# The encode is only essential in Python 3
txn.put(str_id.encode('ascii'), datum.SerializeToString())
# !!! By changing the path to get the train/test set
datapath = '/Users/Desktop/Programming/Python/Conver_Data/TrainingSet/*.png'
# datapath = '/Users/Desktop/Programming/Python/Conver_Data/TestSet/*.png'
files = glob.glob(datapath)
sortedfiles = sorted(files)
nrfiles = np.size(sortedfiles)
N = 640 # training data (positive : negative = 1 : 1)!!!! can be changed
# Let's pretend this is interesting data
X = np.zeros((N, 3, 160, 160), dtype=np.uint8)
# We need to prepare the database for the size. If you don't have
# deepdish installed, just set this to something comfortably big
# (there is little drawback to settings this comfortably big).
map_size = dd.bytesize(X) * 2.5
db = plyvel.DB(
'testLDB_FGNet_640_aligned_cleaned/',
create_if_missing=True,
error_if_exists=True,
write_buffer_size=map_size) # trainLDB_40kids_128000_aligned_balanced_cv10
wb = db.write_batch()
count = 0
count_notsame = 0
count_same = 0
count_0 = 0 # the number of the negative pairs
count_1 = 0 # the number of the positive pairs
run = True
#Path to folder containing the training images.
train_path = proj_home + '/data/train/processed_images/'
level_db_path = proj_home + '/data/leveldb/train/'
train_files = glob.glob(train_path + '*.jpg')
sorted_files = sorted(train_files)
files_num = np.size(sorted_files)
# Let's pretend this is interesting data
X = np.zeros((N, 3, 160, 60), dtype=np.uint8)
# We need to prepare the database for the size. If you don't have
# deepdish installed, just set this to something comfortably big
# (there is little drawback to settings this comfortably big).
map_size = dd.bytesize(X) * 2.5
db = plyvel.DB(level_db_path,create_if_missing = True, error_if_exists=True, write_buffer_size=map_size)
wb=db.write_batch()
count = 0
for k in range(N):
i = random.randrange(files_num)
j = random.randrange(files_num)
label_i = 1 #label from image filename
label_j = 1
if label_i == label_j:
label = 1
else:
label = 0
