def _load_validation_data(validation_leveldb, width, height):
"""
Loads all of our validation data from our leveldb database, producing unrolled numpy input
vectors ready to test along with their correct, expected target values.
"""
print "\tLoading validation data..."
input_vectors = []
expected_targets = []
db = plyvel.DB(validation_leveldb)
for key, value in db:
datum = Datum()
datum.ParseFromString(value)
data = np.fromstring(datum.data, dtype=np.uint8)
data = np.reshape(data, (3, height, width))
# Move the color channel to the end to match what Caffe wants.
data = np.swapaxes(data, 0, 2) # Swap channel with width.
data = np.swapaxes(
data, 0,
1) # Swap width with height, to yield final h x w x channel.
input_vectors.append(data)
expected_targets.append(datum.label)
db.close()
print "\t\tValidation data has %d images" % len(input_vectors)
return {
"input_vectors": np.asarray(input_vectors),
"expected_targets": np.asarray(expected_targets)
}
def _generate_leveldb(file_path, image_paths, targets, width, height):
"""
Caffe uses the LevelDB format to efficiently load its training and validation data; this method
writes paired out faces in an efficient way into this format.
"""
print "\t\tGenerating LevelDB file at %s..." % file_path
shutil.rmtree(file_path, ignore_errors=True)
db = plyvel.DB(file_path, create_if_missing=True)
wb = db.write_batch()
commit_every = 10000
start_time = int(round(time.time() * 1000))
for idx in range(len(image_paths)):
# Each image is a top level key with a keyname like 00000000011, in increasing
# order starting from 00000000000.
key = utils.get_key(idx)
# Do common normalization that might happen across both testing and validation.
try:
image = _preprocess_data(
_load_numpy_image(image_paths[idx], width, height))
except:
print "\t\t\tWarning: Unable to process leveldb image %s" % image_paths[
idx]
continue
# Each entry in the leveldb is a Caffe protobuffer "Datum" object containing details.
datum = Datum()
datum.channels = 3 # RGB
datum.height = height
datum.width = width
datum.data = image.tostring()
datum.label = targets[idx]
value = datum.SerializeToString()
wb.put(key, value)
if (idx + 1) % commit_every == 0:
wb.write()
del wb
wb = db.write_batch()
end_time = int(round(time.time() * 1000))
total_time = end_time - start_time
print "\t\t\tWrote batch, key: %s, time for batch: %d ms" % (
key, total_time)
start_time = int(round(time.time() * 1000))
end_time = int(round(time.time() * 1000))
total_time = end_time - start_time
print "\t\t\tWriting final batch, time for batch: %d ms" % total_time
wb.write()
db.close()
def _generate_leveldb(self, file_path, pairs, target, single_data):
"""
Caffe uses the LevelDB format to efficiently load its training and validation data; this method
writes paired out faces in an efficient way into this format.
"""
print "\tGenerating LevelDB file at %s..." % file_path
shutil.rmtree(file_path, ignore_errors=True)
db = plyvel.DB(file_path, create_if_missing=True)
wb = db.write_batch()
commit_every = 250000
start_time = int(round(time.time() * 1000))
for idx in range(len(pairs)):
# Each image pair is a top level key with a keyname like 00000000011, in increasing
# order starting from 00000000000.
key = siamese_utils.get_key(idx)
# Actually expand our images now, taking the index reference and turning it into real
# image pairs; we delay doing this until now for efficiency reasons, as we will probably
# have more pairs of images than actual computer memory.
image_1 = single_data[pairs[idx][0]]
image_2 = single_data[pairs[idx][1]]
paired_image = np.concatenate([image_1, image_2])
# Do things like mean normalize, etc. that happen across both testing and validation.
paired_image = self._preprocess_data(paired_image)
# Each entry in the leveldb is a Caffe protobuffer "Datum" object containing details.
datum = Datum()
# One channel for each image in the pair.
datum.channels = 2 # One channel for each image in the pair.
datum.height = constants.HEIGHT
datum.width = constants.WIDTH
datum.data = paired_image.tostring()
datum.label = target[idx]
value = datum.SerializeToString()
wb.put(key, value)
if (idx + 1) % commit_every == 0:
wb.write()
del wb
wb = db.write_batch()
end_time = int(round(time.time() * 1000))
total_time = end_time - start_time
print "Wrote batch, key: %s, time for batch: %d ms" % (
key, total_time)
start_time = int(round(time.time() * 1000))
wb.write()
db.close()
def __getitem__(self, index):
if index > len(self.train_list):
raise IndexError("index exceeds the max-length of the dataset.")
lmdb_key1, lmdb_key2, label = self.train_list[index]
datum = Datum()
real_byte_buffer = self.txn.get(lmdb_key1.encode('utf-8'))
raw_real_byte_buffer2 = self.txn.get(lmdb_key2.encode('utf-8'))
datum.ParseFromString(real_byte_buffer)
image1 = cv2.imdecode(np.fromstring(datum.data, dtype=np.uint8), -1)
datum.ParseFromString(raw_real_byte_buffer2)
image2 = cv2.imdecode(np.fromstring(datum.data, dtype=np.uint8), -1)
image1 = transform(image1)
image2 = transform(image2)
img = torch.cat([image1,image2], dim=0)
return img, label
def make_data(param):
for phase in ['train', 'valid', 'test']:
print 'Starting %s' % phase
db_name = './examples/language_model/lm_%s_db' % phase
subprocess.call(['rm', '-rf', db_name])
env = lmdb.open(db_name, map_size=2147483648 * 8)
def vocab_transform(target_input):
def t_foo(x):
return x if x < param['unknown_symbol'] else param[
'unknown_symbol']
target_line = [
t_foo(int(x))
for x in target_input.split(' ')[:param['maximum_length']]
]
target_line = target_line[:param['maximum_length']] + \
[param['zero_symbol']] * (param['maximum_length'] - len(target_line[:param['maximum_length']]))
assert len(target_line) == param['maximum_length']
return target_line
allX = []
with open('./data/language_model/%s_indices.txt' % phase, 'r') as f1:
for en in f1.readlines():
allX.append(vocab_transform(en))
print 'Writing %s sentences' % len(allX)
with env.begin(write=True) as txn:
for i, target_line in enumerate(allX):
datum = Datum()
datum.channels = 2 * param['maximum_length']
datum.width = 1
datum.height = 1
for j in range(param['maximum_length']):
if j == 0:
datum.float_data.append(param['start_symbol'])
else:
datum.float_data.append(target_line[j - 1])
for j in range(param['maximum_length']):
datum.float_data.append(target_line[j])
key = str(i)
txn.put(key, datum.SerializeToString())
def _generate_leveldb(self, file_path, image, target, single_data):
"""
Caffe uses the LevelDB format to efficiently load its training and validation data; this method
writes paired out faces in an efficient way into this format.
"""
print "\tGenerating LevelDB file at %s..." % file_path
shutil.rmtree(file_path, ignore_errors=True)
db = plyvel.DB(file_path, create_if_missing=True)
wb = db.write_batch()
commit_every = 250000
start_time = int(round(time.time() * 1000))
for idx in range(len(pairs)):
# Each image is a top level key with a keyname like 00000000011, in increasing
# order starting from 00000000000.
key = utils.get_key(idx)
# Do things like mean normalize, etc. that happen across both testing and validation.
paired_image = self._preprocess_data(paired_image)
# Each entry in the leveldb is a Caffe protobuffer "Datum" object containing details.
datum = Datum()
# TODO(neuberg): Confirm that this is the correct way to setup RGB images for
# Caffe for our dataset.
datum.channels = 3
datum.height = constants.HEIGHT
datum.width = constants.WIDTH
datum.data = image.tostring()
datum.label = target[idx]
value = datum.SerializeToString()
wb.put(key, value)
if (idx + 1) % commit_every == 0:
wb.write()
del wb
wb = db.write_batch()
end_time = int(round(time.time() * 1000))
total_time = end_time - start_time
print "Wrote batch, key: %s, time for batch: %d ms" % (key, total_time)
start_time = int(round(time.time() * 1000))
wb.write()
db.close()
def __getitem__(self, index):
lmdb_key, label, db_id = self.train_list[index]
datum = Datum()
raw_byte_buffer = self.txns[db_id].get(lmdb_key.encode('utf-8'))
datum.ParseFromString(raw_byte_buffer)
cv_img = cv2.imdecode(np.frombuffer(datum.data, dtype=np.uint8), -1)
if random.random() < 0.5:
cv_img = cv2.flip(cv_img, 1)
if cv_img.ndim == 2:
rows = cv_img.shape[0]
cols = cv_img.shape[1]
buf = np.zeros((3, rows, cols), dtype=np.uint8)
buf[0] = buf[1] = buf[2] = cv_img
input_tensor = (torch.from_numpy(buf) - 127.5) * 0.0078125
else:
assert cv_img.ndim == 3
cv_img = np.transpose(cv_img, (2, 0, 1))
input_tensor = (torch.from_numpy(cv_img) - 127.5) * 0.0078125
return input_tensor, label
def get_train_mb(self, mb_size, cropped_size=227):
env = lmdb.open(self.train_db, readonly=True)
# print env.stat()
samples = np.zeros([mb_size, cropped_size**2 * 3], dtype=np.float32)
labels = np.zeros([mb_size, 1000], dtype=np.float32)
count = 0
with env.begin(write=False, buffers=False) as txn:
cursor = txn.cursor()
for key, value in cursor:
d = Datum()
d.ParseFromString(value)
#print '#channels=', d.channels, 'height=', d.height, 'width=', d.width, 'label=', d.label
im = np.fromstring(d.data, dtype=np.uint8).reshape(
[3, 256, 256]) - self.mean_data
[crop_h, crop_w] = np.random.randint(256 - cropped_size,
size=2)
im_cropped = im[:, crop_h:crop_h + cropped_size,
crop_w:crop_w + cropped_size]
'''
iim = np.transpose(im_cropped.reshape(cropped_size*cropped_size*3).reshape([3, cropped_size*cropped_size])).reshape([cropped_size, cropped_size, 3])
img = Image.fromarray(iim)
img.save('cropimg.jpg', format='JPEG')
exit(0)
'''
samples[count, :] = im_cropped.reshape(cropped_size**2 *
3).astype(np.float32)
labels[count, d.label] = 1
count = count + 1
if count == mb_size:
yield (samples, labels)
#samples = np.zeros([mb_size, cropped_size ** 2 * 3])
labels = np.zeros([mb_size, 1000], dtype=np.float32)
count = 0
if count != mb_size:
delete_idx = np.arange(count, mb_size)
yield (np.delete(samples, delete_idx,
0), np.delete(labels, delete_idx, 0))
db = lmdb.open(args.save_db, max_dbs=2, map_size=1099511627776)
txn = db.begin(write=True)
maps = os.listdir(args.submap_dir)
maps = sorted(maps)
mean_blob = None
n = 0
for cur_map in maps:
# print "Proccessing {}".format(cur_map)
key = os.path.splitext(cur_map)[0]
# try:
# value = txn.get(key)
# except KeyError:
# Make data blob
datum = Datum()
submap = sio.loadmat(os.path.join(args.submap_dir, cur_map))
submap = submap[args.variable].astype('float')
if submap.ndim == 3:
submap = submap.swapaxes(1,2).swapaxes(0,1).astype('float')
datum.channels, datum.height, datum.width = submap.shape
else:
datum.height, datum.width = submap.shape
datum.channels = 1
datum.float_data.extend(list(submap.flatten()))
if mean_blob is None:
mean_blob = BlobProto()
mean_blob.height = datum.height
mean_blob.width = datum.width
mean_blob.channels = datum.channels
