def test_bytesize(self): assert dd.humanize_bytesize(1) == '1 B' assert dd.humanize_bytesize(2 * 1024) == '2 KB' assert dd.humanize_bytesize(3 * 1024**2) == '3 MB' assert dd.humanize_bytesize(4 * 1024**3) == '4 GB' assert dd.humanize_bytesize(5 * 1024**4) == '5 TB' assert dd.bytesize(np.ones((5, 2), dtype=np.int16)) == 20 assert dd.memsize(np.ones((5, 2), dtype=np.int16)) == '20 B'
def classifyNN(self): utils.mkdir_p(self.outDir) self.readDataset() # map_size = dd.bytesize(self.X_train) * 2 # env = lmdb.open('../computed/X_train_lmdb', # map_size=map_size) # for i in range(self.X_train.shape[0]): # if i % 1000 == 0: # print(i) # datum = caffe.proto.caffe_pb2.Datum() # datum.data = self.X_train[i, :].tostring() # datum.label = int(self.y_train[i]) # with env.begin(write=True) as txn: # txn.put(str(i), datum.SerializeToString()) # env.close() map_size = dd.bytesize(self.X_test) * 2 env = lmdb.open('../computed/X_test_lmdb', map_size=map_size) for i in range(self.X_test.shape[0]): if i % 1000 == 0: print(i) datum = caffe.proto.caffe_pb2.Datum() datum.data = self.X_test[i, :].tostring() with env.begin(write=True) as txn: txn.put(str(i), datum.SerializeToString()) env.close() # lmdb_env = lmdb.open('../computed/X_train_lmdb/') lmdb_env = lmdb.open('../computed/X_test_lmdb/') lmdb_txn = lmdb_env.begin() lmdb_cursor = lmdb_txn.cursor() datum = caffe_pb2.Datum() i = 0 for key, value in lmdb_cursor: datum.ParseFromString(value) x = np.fromstring(datum.data) y = datum.label print(x) print(y) if i >= 10: break i = i + 1 lmdb_env.close()
def createLMDBLabel(Y, dbName, order): N = Y.shape[0] Y = np.array(Y, dtype=np.int64) arr = order map_size = dd.bytesize(Y) * 100 env = lmdb.open(dbName, map_size=map_size) for i in range(N): datum = caffe.proto.caffe_pb2.Datum() datum.channels = 1 datum.height = 1 datum.width = 1 datum.data = np.zeros((1, 1, 1)).tobytes() # or .tostring() if numpy < 1.9 datum.label = int(Y[arr[i]]) str_id = "{:08}".format(i) with env.begin(write=True) as txn: # txn is a Transaction object # The encode is only essential in Python 3 txn.put(str_id.encode("ascii"), datum.SerializeToString())
def createLMDBLabel(Y, dbName, order): N = Y.shape[0] Y = np.array(Y, dtype=np.int64) arr = order map_size = dd.bytesize(Y) * 100 env = lmdb.open(dbName, map_size=map_size) for i in range(N): datum = caffe.proto.caffe_pb2.Datum() datum.channels = 1 datum.height = 1 datum.width = 1 datum.data = np.zeros( (1, 1, 1)).tobytes() # or .tostring() if numpy < 1.9 datum.label = int(Y[arr[i]]) str_id = '{:08}'.format(i) with env.begin(write=True) as txn: # txn is a Transaction object # The encode is only essential in Python 3 txn.put(str_id.encode('ascii'), datum.SerializeToString())
# !!! By changing the path to get the train/test set datapath = '/Users/Desktop/Programming/Python/Conver_Data/TrainingSet/*.png' # datapath = '/Users/Desktop/Programming/Python/Conver_Data/TestSet/*.png' files = glob.glob(datapath) sortedfiles = sorted(files) nrfiles = np.size(sortedfiles) N = 640 # training data (positive : negative = 1 : 1)!!!! can be changed # Let's pretend this is interesting data X = np.zeros((N, 3, 160, 160), dtype=np.uint8) # We need to prepare the database for the size. If you don't have # deepdish installed, just set this to something comfortably big # (there is little drawback to settings this comfortably big). map_size = dd.bytesize(X) * 2.5 db = plyvel.DB( 'testLDB_FGNet_640_aligned_cleaned/', create_if_missing=True, error_if_exists=True, write_buffer_size=map_size) # trainLDB_40kids_128000_aligned_balanced_cv10 wb = db.write_batch() count = 0 count_notsame = 0 count_same = 0 count_0 = 0 # the number of the negative pairs count_1 = 0 # the number of the positive pairs run = True
#Path to folder containing the training images. train_path = proj_home + '/data/train/processed_images/' level_db_path = proj_home + '/data/leveldb/train/' train_files = glob.glob(train_path + '*.jpg') sorted_files = sorted(train_files) files_num = np.size(sorted_files) # Let's pretend this is interesting data X = np.zeros((N, 3, 160, 60), dtype=np.uint8) # We need to prepare the database for the size. If you don't have # deepdish installed, just set this to something comfortably big # (there is little drawback to settings this comfortably big). map_size = dd.bytesize(X) * 2.5 db = plyvel.DB(level_db_path,create_if_missing = True, error_if_exists=True, write_buffer_size=map_size) wb=db.write_batch() count = 0 for k in range(N): i = random.randrange(files_num) j = random.randrange(files_num) label_i = 1 #label from image filename label_j = 1 if label_i == label_j: label = 1 else: label = 0