def _load_validation_data(validation_leveldb, width, height): """ Loads all of our validation data from our leveldb database, producing unrolled numpy input vectors ready to test along with their correct, expected target values. """ print "\tLoading validation data..." input_vectors = [] expected_targets = [] db = plyvel.DB(validation_leveldb) for key, value in db: datum = Datum() datum.ParseFromString(value) data = np.fromstring(datum.data, dtype=np.uint8) data = np.reshape(data, (3, height, width)) # Move the color channel to the end to match what Caffe wants. data = np.swapaxes(data, 0, 2) # Swap channel with width. data = np.swapaxes( data, 0, 1) # Swap width with height, to yield final h x w x channel. input_vectors.append(data) expected_targets.append(datum.label) db.close() print "\t\tValidation data has %d images" % len(input_vectors) return { "input_vectors": np.asarray(input_vectors), "expected_targets": np.asarray(expected_targets) }
def _generate_leveldb(file_path, image_paths, targets, width, height): """ Caffe uses the LevelDB format to efficiently load its training and validation data; this method writes paired out faces in an efficient way into this format. """ print "\t\tGenerating LevelDB file at %s..." % file_path shutil.rmtree(file_path, ignore_errors=True) db = plyvel.DB(file_path, create_if_missing=True) wb = db.write_batch() commit_every = 10000 start_time = int(round(time.time() * 1000)) for idx in range(len(image_paths)): # Each image is a top level key with a keyname like 00000000011, in increasing # order starting from 00000000000. key = utils.get_key(idx) # Do common normalization that might happen across both testing and validation. try: image = _preprocess_data( _load_numpy_image(image_paths[idx], width, height)) except: print "\t\t\tWarning: Unable to process leveldb image %s" % image_paths[ idx] continue # Each entry in the leveldb is a Caffe protobuffer "Datum" object containing details. datum = Datum() datum.channels = 3 # RGB datum.height = height datum.width = width datum.data = image.tostring() datum.label = targets[idx] value = datum.SerializeToString() wb.put(key, value) if (idx + 1) % commit_every == 0: wb.write() del wb wb = db.write_batch() end_time = int(round(time.time() * 1000)) total_time = end_time - start_time print "\t\t\tWrote batch, key: %s, time for batch: %d ms" % ( key, total_time) start_time = int(round(time.time() * 1000)) end_time = int(round(time.time() * 1000)) total_time = end_time - start_time print "\t\t\tWriting final batch, time for batch: %d ms" % total_time wb.write() db.close()
def _generate_leveldb(self, file_path, pairs, target, single_data): """ Caffe uses the LevelDB format to efficiently load its training and validation data; this method writes paired out faces in an efficient way into this format. """ print "\tGenerating LevelDB file at %s..." % file_path shutil.rmtree(file_path, ignore_errors=True) db = plyvel.DB(file_path, create_if_missing=True) wb = db.write_batch() commit_every = 250000 start_time = int(round(time.time() * 1000)) for idx in range(len(pairs)): # Each image pair is a top level key with a keyname like 00000000011, in increasing # order starting from 00000000000. key = siamese_utils.get_key(idx) # Actually expand our images now, taking the index reference and turning it into real # image pairs; we delay doing this until now for efficiency reasons, as we will probably # have more pairs of images than actual computer memory. image_1 = single_data[pairs[idx][0]] image_2 = single_data[pairs[idx][1]] paired_image = np.concatenate([image_1, image_2]) # Do things like mean normalize, etc. that happen across both testing and validation. paired_image = self._preprocess_data(paired_image) # Each entry in the leveldb is a Caffe protobuffer "Datum" object containing details. datum = Datum() # One channel for each image in the pair. datum.channels = 2 # One channel for each image in the pair. datum.height = constants.HEIGHT datum.width = constants.WIDTH datum.data = paired_image.tostring() datum.label = target[idx] value = datum.SerializeToString() wb.put(key, value) if (idx + 1) % commit_every == 0: wb.write() del wb wb = db.write_batch() end_time = int(round(time.time() * 1000)) total_time = end_time - start_time print "Wrote batch, key: %s, time for batch: %d ms" % ( key, total_time) start_time = int(round(time.time() * 1000)) wb.write() db.close()
def __getitem__(self, index): if index > len(self.train_list): raise IndexError("index exceeds the max-length of the dataset.") lmdb_key1, lmdb_key2, label = self.train_list[index] datum = Datum() real_byte_buffer = self.txn.get(lmdb_key1.encode('utf-8')) raw_real_byte_buffer2 = self.txn.get(lmdb_key2.encode('utf-8')) datum.ParseFromString(real_byte_buffer) image1 = cv2.imdecode(np.fromstring(datum.data, dtype=np.uint8), -1) datum.ParseFromString(raw_real_byte_buffer2) image2 = cv2.imdecode(np.fromstring(datum.data, dtype=np.uint8), -1) image1 = transform(image1) image2 = transform(image2) img = torch.cat([image1,image2], dim=0) return img, label
def make_data(param): for phase in ['train', 'valid', 'test']: print 'Starting %s' % phase db_name = './examples/language_model/lm_%s_db' % phase subprocess.call(['rm', '-rf', db_name]) env = lmdb.open(db_name, map_size=2147483648 * 8) def vocab_transform(target_input): def t_foo(x): return x if x < param['unknown_symbol'] else param[ 'unknown_symbol'] target_line = [ t_foo(int(x)) for x in target_input.split(' ')[:param['maximum_length']] ] target_line = target_line[:param['maximum_length']] + \ [param['zero_symbol']] * (param['maximum_length'] - len(target_line[:param['maximum_length']])) assert len(target_line) == param['maximum_length'] return target_line allX = [] with open('./data/language_model/%s_indices.txt' % phase, 'r') as f1: for en in f1.readlines(): allX.append(vocab_transform(en)) print 'Writing %s sentences' % len(allX) with env.begin(write=True) as txn: for i, target_line in enumerate(allX): datum = Datum() datum.channels = 2 * param['maximum_length'] datum.width = 1 datum.height = 1 for j in range(param['maximum_length']): if j == 0: datum.float_data.append(param['start_symbol']) else: datum.float_data.append(target_line[j - 1]) for j in range(param['maximum_length']): datum.float_data.append(target_line[j]) key = str(i) txn.put(key, datum.SerializeToString())
def _generate_leveldb(self, file_path, image, target, single_data): """ Caffe uses the LevelDB format to efficiently load its training and validation data; this method writes paired out faces in an efficient way into this format. """ print "\tGenerating LevelDB file at %s..." % file_path shutil.rmtree(file_path, ignore_errors=True) db = plyvel.DB(file_path, create_if_missing=True) wb = db.write_batch() commit_every = 250000 start_time = int(round(time.time() * 1000)) for idx in range(len(pairs)): # Each image is a top level key with a keyname like 00000000011, in increasing # order starting from 00000000000. key = utils.get_key(idx) # Do things like mean normalize, etc. that happen across both testing and validation. paired_image = self._preprocess_data(paired_image) # Each entry in the leveldb is a Caffe protobuffer "Datum" object containing details. datum = Datum() # TODO(neuberg): Confirm that this is the correct way to setup RGB images for # Caffe for our dataset. datum.channels = 3 datum.height = constants.HEIGHT datum.width = constants.WIDTH datum.data = image.tostring() datum.label = target[idx] value = datum.SerializeToString() wb.put(key, value) if (idx + 1) % commit_every == 0: wb.write() del wb wb = db.write_batch() end_time = int(round(time.time() * 1000)) total_time = end_time - start_time print "Wrote batch, key: %s, time for batch: %d ms" % (key, total_time) start_time = int(round(time.time() * 1000)) wb.write() db.close()
def __getitem__(self, index): lmdb_key, label, db_id = self.train_list[index] datum = Datum() raw_byte_buffer = self.txns[db_id].get(lmdb_key.encode('utf-8')) datum.ParseFromString(raw_byte_buffer) cv_img = cv2.imdecode(np.frombuffer(datum.data, dtype=np.uint8), -1) if random.random() < 0.5: cv_img = cv2.flip(cv_img, 1) if cv_img.ndim == 2: rows = cv_img.shape[0] cols = cv_img.shape[1] buf = np.zeros((3, rows, cols), dtype=np.uint8) buf[0] = buf[1] = buf[2] = cv_img input_tensor = (torch.from_numpy(buf) - 127.5) * 0.0078125 else: assert cv_img.ndim == 3 cv_img = np.transpose(cv_img, (2, 0, 1)) input_tensor = (torch.from_numpy(cv_img) - 127.5) * 0.0078125 return input_tensor, label
def get_train_mb(self, mb_size, cropped_size=227): env = lmdb.open(self.train_db, readonly=True) # print env.stat() samples = np.zeros([mb_size, cropped_size**2 * 3], dtype=np.float32) labels = np.zeros([mb_size, 1000], dtype=np.float32) count = 0 with env.begin(write=False, buffers=False) as txn: cursor = txn.cursor() for key, value in cursor: d = Datum() d.ParseFromString(value) #print '#channels=', d.channels, 'height=', d.height, 'width=', d.width, 'label=', d.label im = np.fromstring(d.data, dtype=np.uint8).reshape( [3, 256, 256]) - self.mean_data [crop_h, crop_w] = np.random.randint(256 - cropped_size, size=2) im_cropped = im[:, crop_h:crop_h + cropped_size, crop_w:crop_w + cropped_size] ''' iim = np.transpose(im_cropped.reshape(cropped_size*cropped_size*3).reshape([3, cropped_size*cropped_size])).reshape([cropped_size, cropped_size, 3]) img = Image.fromarray(iim) img.save('cropimg.jpg', format='JPEG') exit(0) ''' samples[count, :] = im_cropped.reshape(cropped_size**2 * 3).astype(np.float32) labels[count, d.label] = 1 count = count + 1 if count == mb_size: yield (samples, labels) #samples = np.zeros([mb_size, cropped_size ** 2 * 3]) labels = np.zeros([mb_size, 1000], dtype=np.float32) count = 0 if count != mb_size: delete_idx = np.arange(count, mb_size) yield (np.delete(samples, delete_idx, 0), np.delete(labels, delete_idx, 0))
db = lmdb.open(args.save_db, max_dbs=2, map_size=1099511627776) txn = db.begin(write=True) maps = os.listdir(args.submap_dir) maps = sorted(maps) mean_blob = None n = 0 for cur_map in maps: # print "Proccessing {}".format(cur_map) key = os.path.splitext(cur_map)[0] # try: # value = txn.get(key) # except KeyError: # Make data blob datum = Datum() submap = sio.loadmat(os.path.join(args.submap_dir, cur_map)) submap = submap[args.variable].astype('float') if submap.ndim == 3: submap = submap.swapaxes(1,2).swapaxes(0,1).astype('float') datum.channels, datum.height, datum.width = submap.shape else: datum.height, datum.width = submap.shape datum.channels = 1 datum.float_data.extend(list(submap.flatten())) if mean_blob is None: mean_blob = BlobProto() mean_blob.height = datum.height mean_blob.width = datum.width mean_blob.channels = datum.channels