def ImageDatum(self, task_dict, fileQueue, datumQueue):
datum = Datum()
dims = task_dict['data'].shape
imageDatum = datum.imgdata
imageDatum.identifier = str(task_dict['key'])
imageDatum.channels = dims[2]
imageDatum.height = dims[0]
imageDatum.width = dims[1]
data = task_dict.pop('data')
data = np.asarray(data, dtype='float32')
imageDatum.data = data.tobytes()
task_dict['datum'] = datum
if 'label' in task_dict.keys():
labelDatum = datum.classs
labelDatum.identifier = str(task_dict['key'])
labelDatum.slabel = task_dict.pop('label')
task_dict['label'] = datum
datumQueue.put(task_dict)
logger.debug("ImageDatum: Pushed datum #{} into Datum Queue".format(
self.count))
fileQueue.task_done(
) # let the fileQueue know item has been processed and is safe to delete
def get_data(self):
idxs = np.arange(self.size())
if self.is_train:
idxs += CocoPoseLMDB.__valid_i
self.rng.shuffle(idxs)
else:
pass
for idx in idxs:
datum = Datum()
if self.only_idx < 0:
s = self.txn.get(('%07d' % idx).encode('utf-8'))
else:
s = self.txn.get(('%07d' % self.only_idx).encode('utf-8'))
datum.ParseFromString(s)
if isinstance(datum.data, bytes):
data = np.fromstring(datum.data, dtype=np.uint8).reshape(
datum.channels, datum.height, datum.width)
else:
data = np.fromstring(datum.data.tobytes(),
dtype=np.uint8).reshape(
datum.channels, datum.height,
datum.width)
if self.decode_img:
img = data[:3].transpose((1, 2, 0))
else:
img = None
meta = CocoMetadata(idx, img, data[3], sigma=8.0)
yield [meta]
def _get_datum_shapes(self, input_cursors, output_cursors):
input_shapes = []
output_shapes = []
for input_cursor in input_cursors:
input_cursor.first()
_, raw_datum = input_cursor.item()
datum = Datum()
datum.ParseFromString(raw_datum)
if datum.imgdata.data:
flat_x = np.fromstring(datum.imgdata.data, dtype='float32')
x = flat_x.reshape((datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels))
shape = (datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels)
logger.debug("input image shape: " + repr(shape))
input_shapes.append(shape)
if datum.numeric.data:
flat_x = np.fromstring(datum.numeric.data, dtype='float32')
shape = flat_x.shape
logger.debug("input numeric shape: " + repr(shape))
input_shapes.append(shape)
for output_cursor in output_cursors:
output_cursor.first()
_, raw_datum = output_cursor.item()
datum = Datum()
datum.ParseFromString(raw_datum)
if datum.imgdata.data:
flat_x = np.fromstring(datum.imgdata.data, dtype='float32')
x = flat_x.reshape((datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels))
shape = (datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels)
logger.debug("output image shape: " + repr(shape))
output_shapes.append(shape)
if datum.numeric.data:
flat_x = np.fromstring(datum.numeric.data, dtype='float32')
shape = flat_x.shape
logger.debug("output numeric shape: " + repr(shape))
output_shapes.append(shape)
return (input_shapes, output_shapes)
def get_data(self):
idxs = np.arange(self.size())
if self.is_train:
idxs += CocoPoseLMDB.__valid_i
self.rng.shuffle(idxs)
else:
pass
for idx in idxs:
datum = Datum()
s = self.txn.get(('%07d' % idx).encode('utf-8'))
datum.ParseFromString(s)
data = np.fromstring(datum.data.tobytes(), dtype=np.uint8).reshape(datum.channels, datum.height, datum.width)
img = data[:3].transpose((1, 2, 0))
meta = CocoMetadata(img, data[3], 4.0)
yield [meta]
def TextDatum(self, fileQueue, datumQueue):
datum = Datum()
textDatum = datum.numeric
textDatum.identifier = str(task_dict['key'])
textDatum.size.dim = 1
data = task_dict.pop('data')
data = np.asarray(data, dtype='float32')
textDatum.data = data.tobytes()
task_dict['datum'] = datum
if 'label' in task_dict.keys():
labelDatum = datum.classs
labelDatum.identifier = str(task_dict['key'])
labelDatum.nlabel = task_dict.pop('label')
task_dict['label'] = datum
datumQueue.put(task_dict)
logger.debug("TextDatum: Pushed datum #{} into Datum Queue".format(
self.count))
fileQueue.task_done()
def NumericDatum(self, task_dict, fileQueue, datumQueue):
datum = Datum()
numericDatum = datum.numeric
numericDatum.identifier = str(task_dict['key'])
# numericDatum.size.dim = label.shape[0]
numericDatum.size.dim = 1 # currently just for 1 label
data = task_dict.pop('data')
data = np.asarray(data, dtype='float32')
numericDatum.data = data.tobytes()
task_dict['datum'] = datum
if 'label' in task_dict.keys():
labelDatum = datum.classs
labelDatum.identifier = str(task_dict['key'])
labelDatum.nlabel = task_dict.pop('label')
task_dict['label'] = datum
datumQueue.put(task_dict)
logger.debug("NumericDatum: Pushed datum #{} into Datum Queue".format(
self.count))
fileQueue.task_done()
def batch_generator(self, input_cursors, output_cursors):
while 1:
for idx in xrange(self.n_batches):
inputs = []
outputs = []
for input_cursor in input_cursors:
n = self.batch_size
input_cursor.set_range(str(idx * self.batch_size + 1))
logger.debug("Current input key: " +
repr(input_cursor.key()))
batch_inputs = []
for _, raw_datum in input_cursor:
if n <= 0: break
datum = Datum()
datum.ParseFromString(raw_datum)
if datum.imgdata.data:
flat_x = np.fromstring(datum.imgdata.data,
dtype='float32')
x = flat_x.reshape(
(datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels))
shape = (datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels)
logger.debug("input image shape: " + repr(shape))
batch_inputs.append(x)
elif datum.numeric.data:
flat_x = np.fromstring(datum.numeric.data,
dtype='float32')
logger.debug("input numeric shape: " +
repr(flat_x.shape))
batch_inputs.append(flat_x)
n -= 1
batch_inputs = np.array(batch_inputs)
logger.debug("Batch input shape: " +
repr(batch_inputs.shape))
inputs.append(batch_inputs)
for output_cursor in output_cursors:
n = self.batch_size
output_cursor.set_range(str(idx * self.batch_size + 1))
logger.debug("Current output key: " +
repr(output_cursor.key()))
batch_outputs = []
for _, raw_datum in output_cursor:
if n <= 0: break
datum = Datum()
datum.ParseFromString(raw_datum)
if datum.imgdata.data:
flat_x = np.fromstring(datum.imgdata.data,
dtype='float32')
x = flat_x.reshape(
(datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels))
shape = (datum.imgdata.height, datum.imgdata.width,
datum.imgdata.channels)
logger.debug("output image shape: " + repr(shape))
batch_outputs.append(x)
elif datum.numeric.data:
flat_x = np.fromstring(datum.numeric.data,
dtype='float32')
logger.debug("output numeric shape: " +
repr(flat_x.shape))
batch_outputs.append(flat_x)
n -= 1
batch_outputs = np.array(batch_outputs)
logger.debug("Batch output shape: " +
repr(batch_outputs.shape))
outputs.append(batch_outputs)
yield (inputs, outputs)