def write_images_to_lmdb(img_dir, db_name, labels): for root, dirs, files in os.walk(img_dir, topdown = False): if root != img_dir: continue #multiply 2 to make the map_size large enough map_size = 2*IMAGE_WIDTH * IMAGE_HEIGHT *3*len(files) env = lmdb.Environment(db_name, map_size=map_size) txn = env.begin(write=True,buffers=True) for idx, name in enumerate(files): #print(img_dir + name) img = cv2.imread(img_dir + name, cv2.IMREAD_COLOR) img = transform_img(img, img_width=IMAGE_WIDTH, img_height=IMAGE_HEIGHT) #print(img.shape) #print(img) img=img.transpose(2,0,1) y = labels[name] #print(name) #print(y) datum = array_to_datum(img,y) #print(datum) #lala str_id = '{:08}'.format(idx) txn.put(str_id.encode('ascii'), datum.SerializeToString()) if idx % 1000 == 1: print("transforming" + str(idx) + "th image to sb") txn.commit() env.close() print " ".join(["Writing to", db_name, "done!"])
def createDB(name, contenido, funcion): db = lmdb.Environment(name, map_size=int(1e12)) tx = db.begin(write=True) for label, imagen in enumerate(contenido): im = Image.open(imagen) punt = im.fp im = im.resize((IMAGE_WIDTH, IMAGE_HEIGHT)) if 'cat' in os.path.split(imagen)[1]: y = 0 else: y = 1 x = np.array(im.getdata()).reshape(im.size[1], im.size[0], 3) datum = array_to_datum(np.transpose(x, (2, 0, 1)), y) if funcion(label, SCORE): print label tx.put('{:08}'.format(label), datum.SerializeToString()) if not punt.closed: punt.close() if (label + 1) % 2500 == 0: tx.commit() tx = db.begin(write=True) print '------- commit -------' tx.commit() db.close()
def create_lmdb(db_name, labels): map_size = SEQUENCE_LENGTH * ALPHA_LENGTH * 27 * len( labels) # 27x is just for safety env = lmdb.Environment(db_name, map_size=map_size) txn = env.begin(write=True, buffers=True) X_copy = np.ndarray((3, SEQUENCE_LENGTH, ALPHA_LENGTH)) for i, label in enumerate(labels): print('writing img-{}'.format(i)) X = mp.imread(IMG_BASE_NAME.format(i)) * 255 X_rev = np.transpose(X) X_copy[0, :, :] = X_rev X_copy[1, :, :] = X_rev X_copy[2, :, :] = X_rev if label < 5: class_label = 0 else: class_label = 1 datum = array_to_datum(X_copy.astype(np.uint8), class_label) str_id = '{:08}'.format(i) txn.put(str_id.encode('ascii'), datum.SerializeToString()) txn.commit() env.close() print('Done creating {}!'.format(db_name))
def write_images_to_db(self, db_name, images, callback=lambda a: 0): ''' Writes images to a lmdb database inside this model's folder. db_name is the name of the database. If a database by that name already exists, it will be overwritten. images is a list of 2-element tuples. The first elemnt of each tuple should be a path to an image, and the second element should be the label of that image. ''' random.shuffle(images) #Shuffle input data to improve training. p = os.path.join(self.get_folder(), db_name) s.call(['rm', '-r', p]) map_size = 256 * 256 * 3 * 2 * len(images) env = lmdb.Environment(p, map_size=map_size) write_to = env.begin(write=True, buffers=True) i = 0 num_images = len(images) update_interval = int(num_images / 100 + 1) for image in images: try: resize_image(image[0]) input = np.transpose( mp.imread('/tmp/resized.jpg'), (2, 1, 0)) #Caffe wants CxHxW, not the standard WxHxC. datum = array_to_datum(input, image[1]) write_to.put('{:08}'.format(i).encode('ascii'), datum.SerializeToString()) i += 1 except: pass if (i % update_interval == 0): callback([(i / num_images, '')]) write_to.commit() env.close()
def create_seqdb(db_path, root_dir, seqs, rels, resize=None): # create a lmdb of sequences stacked at the channel dimensions # therefore each blob is (SEQ_LEN*3) X H X W # suffle data seqs, rels = np.array(seqs), np.array(rels) sff = np.random.permutation(seqs.shape[0]) seqs, rels = seqs[sff], rels[sff] np.savez("{}_REL.npz".format(db_path), gt_rels=rels) in_db = lmdb.open(db_path, map_size=int(1e12)) with in_db.begin(write=True) as in_txn: for in_idx, seq in enumerate(seqs): im_seq = None for img_fname in seq: im = Image.open(osp.join(root_dir, img_fname)) im = im.convert('RGB') if im.mode == 'L' else im im = np.array(im.resize( resize, Image.BILINEAR)) if resize else np.array(im) im = im[:, :, ::-1] im = im.transpose((2, 0, 1)) im_seq = im if im_seq is None else np.append( im_seq, im, axis=0) im_dat = array_to_datum(im_seq, label=0) in_txn.put('{:0>10d}'.format(in_idx).encode('ascii'), im_dat.SerializeToString()) if in_idx % 1e3 == 0: print("{}/{} images saved..".format(in_idx, len(seqs))) in_db.close() print("Dataset created at {} with {} datums".format(db_path, len(seqs))) return db_path
def np2lmdb(data, lmdb_path, labels=None, shape=None): import matplotlib matplotlib.use("Agg") from caffe import io records = [] assert isinstance(data, np.ndarray) data = data.astype(np.float) if labels is not None: assert len(data) == len(labels) labels = np.asarray(labels, dtype=np.int) for i, each in enumerate(data): try: if shape is None: each.resize(each.shape[0], 1, 1) else: each.resize(shape) one = io.array_to_datum(each) if labels is not None: one.label = labels[i] records.append(one.SerializeToString()) except Exception, e: print each.shape raise e
def pair_gen(datum_1, datum_2): mat_1 = io.datum_to_array(datum_1) mat_2 = io.datum_to_array(datum_2) mat_3 = np.concatenate([mat_1, mat_2], axis=0) datum_3 = io.array_to_datum(mat_3) if datum_1.label == datum_2.label: datum_3.label = 1 else: datum_3.label = 0 return datum_3
def resize_lmdb(src_db, dest_db, width, height, length): ''' #resize_lmdb: create lmdb from existing lmdb with resizing # src_db: source lmdb name # dest_db: new lmdb name # width: new image width # height: new image height # length: limit of item count to process ''' print('[start] resize lmdb [{}]->[{}]'.format(src_db, dest_db)) src_env = lmdb.open(src_db, readonly=True) map_size = 100000000 * length #buffer size dest_env = lmdb.Environment(dest_db, map_size) dest_txn = dest_env.begin(write=True, buffers=True) with src_env.begin() as src_txn: cursor = src_txn.cursor() count = 0 before, after = None, None for key, value in cursor: datum = caffe_pb2.Datum() datum.ParseFromString(value) x = datum_to_array(datum) # (c,h,w) y = datum.label if datum.channels == CH_GRAY: img_array = x.transpose((1, 2, 0)) # (c,h,w)->(h,w,c) img_array = img_array.reshape( img_array.shape[0], img_array.shape[1]) # (h,w,1)->(h,w) img = Image.fromarray(np.uint8(img_array)) img = img.resize((width, height)) img_array = np.asarray(img) img_array = img_array.reshape(height, width, 1) # (h,w)->(h,w,1) elif datum.channels == CH_RGB: img_array = x.transpose((1, 2, 0)) # (c,h,w)->(h,w,c) img = Image.fromarray(np.uint8(img_array)) img = img.resize((width, height)) img_array = np.asarray(img) img_array = img_array.transpose((2, 0, 1)) # (h,w,c) -> (c,h,w) out_datum = array_to_datum(img_array, y) dest_txn.put(key.encode('ascii'), out_datum.SerializeToString()) if count == 0: before = x.shape after = img_array.shape #print ( '{0}: x.shape:{1} label:{2} -> x.shape{3} label:{4}'.format(key,x.shape,y,img.shape,y)) count += 1 if count >= length: break print('[finished] total count {}. shape {} -> {}'.format( count, before, after)) dest_txn.commit() dest_env.close() src_env.close()
def _create_lmdb_(lmdb_filename, X, Y): num = np.prod(X.shape) itemsize = np.dtype(X.dtype).itemsize # set a reasonable upper limit for database size map_size = 10240 * 1024 + num * itemsize * 2 print 'save {} instances...'.format(num) env = lmdb.open(lmdb_filename, map_size=map_size) for i, (x, y) in enumerate(zip(X, Y)): datum = array_to_datum(x, y) str_id = '{:08}'.format(i) with env.begin(write=True) as txn: txn.put(str_id, datum.SerializeToString())
def _create_lmdb_(lmdb_filename, X, Y): num = np.prod(X.shape) itemsize = np.dtype(X.dtype).itemsize # set a reasonable upper limit for database size map_size = 10240 * 1024 + num * itemsize * 2 print 'save {} instances...'.format(num) env = lmdb.open(lmdb_filename, map_size=map_size) for i, (x, y) in enumerate(zip(X, Y)): datum = array_to_datum(x, y) str_id = '{:08}'.format(i) with env.begin(write=True) as txn: txn.put(str_id, datum.SerializeToString())
def process_single_img(lmdb_txn, item_id, im, ptype): """convert single image according to different normalization methods""" datum = caffe_pb2.Datum() if ptype == SIMPLE_NORM: im = cv2.resize(im, (config.img_height, config.img_width)) datum = array_to_datum(im) keystr = '{:0>8d}'.format(item_id) lmdb_txn.put(keystr, datum.SerializeToString()) elif ptype == SQUARE_NORM: rim = np.transpose(im, axes=(1, 0, 2)) im = cv2.resize(im, (config.img_height, config.img_width)) rim = cv2.resize(rim, (config.img_height, config.img_width)) # save origin image datum = array_to_datum(im) keystr = '{:0>8d}'.format(item_id) lmdb_txn.put(keystr, datum.SerializeToString()) # save rotated image datum = array_to_datum(rim) keystr = '1{:0>7d}'.format(item_id) lmdb_txn.put(keystr, datum.SerializeToString())
def createDB(db, funcion): global label label = 0 for imagenes in getImagesAndClases(): #for imagenes in getImagesFile(): tx = db.begin(write=True) punt = None for imagen in imagenes: try: label += 1 ''' if not funcion(label,SCORE): label += 1 if True: ''' if funcion(label, SCORE): im = Image.open(imagen) punt = im.fp im = im.resize((IMAGE_WIDTH, IMAGE_HEIGHT)) if im.mode != 'RGB': im = im.convert('RGB') x = np.array(im.getdata()).reshape(im.size[1], im.size[0], 3) datum = array_to_datum(np.transpose(x, (2, 0, 1)), clase) print label, imagen tx.put('{:08}'.format(label), datum.SerializeToString()) if (label + 1) % COMMIT == 0: tx.commit() tx = db.begin(write=True) print '------- commit -------' except IOError as ioe: print 'Imagen:', imagen print ioe except: pass finally: if punt != None and not punt.closed: punt.close() tx.commit()
def write_images_to_db(db_folder, db_name, images): random.shuffle(images) #Shuffle input data to improve training. p = path.join(db_folder, db_name) s.call(['rm', '-r', p]) map_size = 256*256*3*2*len(images) env = lmdb.Environment(p, map_size = map_size) write_to = env.begin(write=True, buffers=True) for i, image in enumerate(images): resize_image(image[0]) input = np.transpose(mp.imread('/tmp/resized.jpg'), (2, 1, 0)) #Caffe wants CxHxW, not the standard WxHxC. datum = array_to_datum(input, image[1]) print('Finished image', i+1, 'which had label', image[1]) write_to.put('{:08}'.format(i).encode('ascii'), datum.SerializeToString()) write_to.commit() env.close()
def write_to_lmdb(image_list, db_path, h5_root): map_size = 150000000000 env = lmdb.Environment(db_path, map_size=map_size) with env.begin(write=True, buffers=True) as txn: for idx, image in enumerate(image_list): # 150 GB X = get_caffe_data.load_h5_file(os.path.join(h5_root, image[0])) X = X[np.newaxis, :, :] y = int(image[1]) datum = array_to_datum(X, y) str_id = '{:08}'.format(idx) txn.put(str_id.encode('ascii'), datum.SerializeToString()) env.close() print " ".join(["Writing to", db_path, "done!"])
def write_images_to_lmdb(img_dir, db_name): for root, dirs, files in os.walk(img_dir, topdown=False): if root != img_dir: continue map_size = 300 * 300 * 3 * 5 * len(files) env = lmdb.Environment(db_name, map_size=map_size) txn = env.begin(write=True, buffers=True) for idx, name in enumerate(files): X = cv2.imread(os.path.join(root, name)) y = 1 datum = array_to_datum(X, y) str_id = '{:08}'.format(idx) txn.put(str_id.encode('ascii'), datum.SerializeToString()) txn.commit() env.close() print " ".join(["Writing to", db_name, "done!"])
def imgs2lmdb(imgs, lmdb_path, is_verbose=False): import matplotlib matplotlib.use('Agg') from caffe import io values = [] for img in imgs: if img[0].ndim == 2: channels = int(1) else: channels = img[0].shape[2] height = img[0].shape[0] width= img[0].shape[1] img[0].resize(channels, height, width) datum = io.array_to_datum(img[0]) datum.label = int(img[1]) values.append(datum.SerializeToString()) if len(values) is not 0: dump_to_lmdb(values, lmdb_path, is_verbose, random_order=True)
def write_images_to_lmdb(img_dir, db_name, label): for root, dirs, files in os.walk(img_dir, topdown=False): if root != img_dir: continue map_size = 400 * 400 * 3 * 5 * len(files) env = lmdb.Environment(db_name, map_size=map_size) txn = env.begin(write=True, buffers=True) max_key = env.stat()["entries"] for idx, name in enumerate(files): X = cv2.imread(os.path.join(root, name)) width, height, ch = X.shape X = cv2.resize(X, (256, 256)) y = label datum = array_to_datum(X, y) str_id = '{:08}'.format(max_key + idx) txn.put(str_id.encode('ascii'), datum.SerializeToString()) txn.commit() env.close() print " ".join(["Writing to", db_name, "done!"])
def write_images_to_lmdb(img_dir, db_name): for root, dirs, files in os.walk(img_dir, topdown = False): if root != img_dir: continue map_size = 64*64*3*2*len(files) env = lmdb.Environment(db_name, map_size=map_size) txn = env.begin(write=True,buffers=True) for idx, name in enumerate(files): X = mp.imread(os.path.join(root, name)) y = 1 datum = array_to_datum(X,y) str_id = '{:08}'.format(idx) # The encode is only essential in Python 3 txn.put(str_id.encode('ascii'), datum.SerializeToString()) txn.commit() env.close() print " ".join(["Writing to", db_name, "done!"])
def create_lmdb_from_array(dest_db, imgs, labels): ''' # multi cahennl lmdb # imgs : list of img array # labels : list of label id # imgs[0].shape = (channel, height, width) # len(imgs) = item count # len(labels) = item count ''' #print ('[start] creating lmdb. ch:{}, w:{}, h:{}, item count:{}'.format(imgs[0].shape[0],imgs[0].shape[1],imgs[0].shape[2],len(imgs))) map_size = 100000000 * len(imgs[0]) #buffer size env = lmdb.Environment(dest_db, map_size) txn = env.begin(write=True, buffers=True) count = 0 for idx, img in enumerate(imgs): str_id = '{:08}'.format(idx) clsid = labels[idx] #print ( ' {0} shape:{1}, class:{2}'.format(str_id, img.shape , clsid)) datum = array_to_datum(img, clsid) txn.put(str_id.encode('ascii'), datum.SerializeToString()) count += 1 txn.commit() env.close() print('[finished] {} items have written to \'{}\''.format(count, dest_db))
def build_db(files,db_fn,with_images=True,with_labels=False, trf_func=np.array,pad=0): # Where from pvc_im_path = "~/data/VOCdevkit/VOC2012/JPEGImages/" pvc_sg_path = "~/data/VOCdevkit/VOC2012/SegmentationClass/" sds_sg_path = "~/data/SDS/cls" d = dict( pvc_im = pvc_im_path, pvc_sg = pvc_sg_path, sds_sg = sds_sg_path) #Don't know how to clear lmdb using api shutil.rmtree(db_fn) # Start env = lmdb.Environment(db_fn,map_size=10**12) for k,fn in d.items(): d[k] = os.path.expanduser(fn) files = sorted(list(files)) for i in range(len(files)): file_fn = files[i] print(file_fn,i,"/",len(files)) if with_images: im = Image.open(os.path.join(d["pvc_im"],file_fn + ".jpg")) #(366,500,3) -> (3,366,500) im = trf_func(im)[:,:,::-1].transpose(2,0,1) data = im if with_labels: sg_fn = os.path.join(d["pvc_sg"],file_fn + ".png") if os.path.isfile(sg_fn): sg = Image.open(sg_fn) #(366,500) -> (1,366,500) sg = trf_func(sg)[:,:,np.newaxis].transpose(2,0,1) else: sg_fn = os.path.join(d["sds_sg"],file_fn + ".mat") if os.path.isfile(sg_fn): data = loadmat(sg_fn) sg = Image.fromarray(data["GTcls"][0,0][1]) sg = trf_func(sg)[:,:,np.newaxis].transpose(2,0,1) if with_images: print("Check images here.") raise NotImplementedError data = np.dstack( (np.array(im)[:,:,::-1],np.array(sg))).transpose(2,0,1) else: data = sg key = str(i).zfill(8) + '_' + file_fn.replace(".png",".jpg") datum = array_to_datum(data) with env.begin(write=True) as txn: cursor = txn.cursor() cursor.put(key,datum.SerializeToString()) for j in range(pad): key = str(i+j).zfill(8) + '_' + "pad" print(j,"/",pad) # Should be able to put in empty images, but im afraid # that will raise errors, do this for now. if with_images: data = np.zeros( (3,384,384),dtype=np.uint8) if with_labels: data = np.zeros( (1,384,384),dtype=np.uint8 ) data[...] = ignore_label datum = array_to_datum(data) with env.begin(write=True) as txn: cursor = txn.cursor() cursor.put(key,datum.SerializeToString())
import caffe from caffe.io import array_to_datum parser = argparse.ArgumentParser() parser.add_argument('--dir', type=str, required=True) parser.add_argument('--output', type=str, required=True) args = parser.parse_args() if __name__ == '__main__': files = os.listdir(args.dir) new_db = lmdb.open(args.output, map_size=3 * 10**9) for f in files: print("processing: " + f) data = Dataset([args.dir + '/' + f]) X_hit, X_info, y = data.get_data(crop=True) with new_db.begin(write=True) as txn: # txn is a Transaction object N = X_hit.shape[0] for i in range(N): img_i = X_hit[i, :, :, :] img_i = img_i.transpose((2, 0, 1)) el = np.hstack([img_i.flatten(), X_info[i, :].flatten()]) el = np.reshape(el, (1, 1, -1)) datum = array_to_datum(el, int(y[i, 1])) str_id = '{}_ID{:08}'.format(f, i) # The encode is only essential in Python 3 txn.put(str_id.encode('ascii'), datum.SerializeToString()) new_db.close()
def export_to_lmdb(self, lmdb_foldername, ages_as_means=True, map_size=-1, splitters=None, apply_normalizers=False): """ Exports the current dataset to LMDB format. If the LMDB already exists, it will append to its content. :param lmdb_foldername: filename LMDB. :param ages_as_means: save the age_range in mean format. :param map_size: size of map of the LMDB database. If set to -1, it will attempt to calculate a map_size that fits this dataset. Remember however, that it won't allow to expand the LMDB database with new data and you'll need to create a new one in case you want to. :param splitters: a set of splitters to split the dataset into multiple lmdbs. The lmdb divided by each splitter will be stored with the splitter's name prepended to the lmdb name. This is useful if you want to extract a chunk of the dataset as a test or validation lmdbs. :param apply_normalizers: boolean flag to apply normalizers when the image is put into the dataset manually. """ iteration = 0 txn_index = 0 if splitters is None: splitters = [] keys = self.get_keys(shuffle=True) count = len(keys) datum_id_format = "{}:0>{}{}_dbuild_{}".format("{", len(str(count)), "}", "{}") if map_size == -1: #map_size = self.get_dataset_size() + count * 30000 map_size = 1e12 print("Map size is {} MBytes".format(round(map_size/1000/1000, 2))) if splitters: environments = [lmdb.Environment(lmdb_foldername + "_" + splitter.get_name(), map_size=map_size) for splitter in splitters] else: environments = [lmdb.Environment(lmdb_foldername, map_size=map_size)] txns = [env.begin(write=True, buffers=True) for env in environments] txn = env = None put_txns = {} for txn in txns: put_txns[txn] = 0 for key in keys: iteration += 1 for split_id in range(len(splitters)): splitter = splitters[split_id] if splitter.decide(iteration): txn = txns[split_id] env = environments[split_id] txn_index = split_id break if not txn or not env: txn = txns[0] env = environments[0] txn_index = 0 image = self.get_image(key) image.load_from_uri() age_range = image.get_metadata()[0] if ages_as_means: label = age_range.get_mean() else: label = age_range.get_range()[0] image_blob = image.get_blob() if apply_normalizers: for normalizer in self.normalizers: image_blob = normalizer.apply(image_blob) #HxWxC to CxHxW in caffe image_blob = np.transpose(image_blob, (2,0,1)) # Datum is the element map in LMDB. We associate image with label here. datum = array_to_datum(image_blob, label) # Now we encode the image id in ascii format inside the lmdb container that corresponds to this input. txn.put(datum_id_format.format(iteration, image.get_id()).encode("ascii"), datum.SerializeToString()) put_txns[txn] += 1 # write batch if put_txns[txn] % LMDB_BATCH_SIZE == 0: txn.commit() old_txn_count = put_txns[txn] del put_txns[txn] txn = env.begin(write=True) txns[txn_index] = txn put_txns[txn] = old_txn_count print("[{}%] Stored batch of {} image in LMDB".format(round(iteration/count * 100, 2), LMDB_BATCH_SIZE)) # There could be a last batch on each txn without being commited. [txn.commit() or print("[{}%] Stored batch of {} image in LMDB".format(round(iteration/len(keys) * 100, 2), count % LMDB_BATCH_SIZE)) for txn, count in put_txns.items() if count % LMDB_BATCH_SIZE != 0] [env.close() for env in environments]