def setup(self, bottom, top):
print "Setting up ShapePredictionDataLayer ..."
assert len(top) == 2, 'requires 2 tops: (image) data, gt_shape'
# params is expected as argparse style string
self.params = self.parse_param_str(self.param_str)
# ----- Reshape tops -----#
# data shape = B x C x H x W
# shape_params shape = B x 10
top[0].reshape(self.params.batch_size, 3, self.params.im_size[1],
self.params.im_size[0]) # Image Data
top[1].reshape(self.params.batch_size, 10) # ShapeTarget
# create mean bgr to directly operate on image data blob
self.mean_bgr = np.array(self.params.mean_bgr).reshape(1, 3, 1, 1)
# Create a loader to load the images.
self.image_loader = BatchImageLoader(self.params.im_size)
# Create placeholder for GT annotations
self.shape_params = []
print 'ShapePredictionDataLayer has been setup.'
def setup(self, bottom, top):
print "Setting up RCNNDataLayer ..."
# params is expected as argparse style string
params = self.parse_param_str(self.param_str)
# Store the ordered list of top_names
self.top_names = params.top_names
# Store batch size as member variable for use in other methods
self.batch_size = params.batch_size
# create mean bgr to directly operate on image data blob
self.mean_bgr = np.array(params.mean_bgr)
# create std bgr to directly operate on image data blob
self.std_bgr = np.array(params.std_bgr)
# set network input_image size
self.im_size = params.im_size
# set flip_ratio
self.flip_ratio = params.flip_ratio
# set crop_target
self.crop_target = params.crop_target
# set uniform_crop
self.uniform_crop = params.uniform_crop
assert len(top) == len(
self.top_names), 'Number of tops do not match specified top_names'
# set top shapes
top_shapes = {
"input_image":
(self.batch_size, 3, self.im_size[1], self.im_size[0]),
"viewpoint": (self.batch_size, 3),
"bbx_amodal": (self.batch_size, 4),
"bbx_crop": (self.batch_size, 4),
"center_proj": (self.batch_size, 2),
}
# Reshape tops
for top_index, top_name in enumerate(self.top_names):
top[top_index].reshape(*top_shapes[top_name])
# Create a loader to load the images.
self.image_loader = BatchImageLoader(transpose=False)
# Create placeholder for annotations
self.data_samples = []
print 'RCNNDataLayer has been setup.'
def setup(self, bottom, top):
print "Setting up ArticulatedObjectDataLayer ..."
# params is expected as argparse style string
params = self.parse_param_str(self.param_str)
# Store the ordered list of top_names
self.top_names = params.top_names
# Store batch size as member variable for use in other methods
self.batch_size = params.batch_size
# create mean bgr to directly operate on image data blob
self.mean_bgr = np.array(params.mean_bgr).reshape(1, 3, 1, 1)
self.shape_param_size = params.shape_param_size
self.pose_param_size = params.pose_param_size
assert 'data' in self.top_names, 'Requires atleast data layer'
assert len(top) == len(
self.top_names), 'Number of tops do not match specified top_names'
# Reshape image data top (B, C, H, W)
if 'data' in self.top_names:
top[self.top_names.index('data')].reshape(
self.batch_size, 3, params.im_size[1],
params.im_size[0]) # Image Data
# Reshape shape top (B, 10)
if 'shape' in self.top_names:
top[self.top_names.index('shape')].reshape(self.batch_size,
self.shape_param_size)
# Reshape pose top (B, 36)
if 'pose' in self.top_names:
top[self.top_names.index('pose')].reshape(self.batch_size,
self.pose_param_size)
# Create a loader to load the images.
self.image_loader = BatchImageLoader(params.im_size)
if 'shape' in self.top_names:
self.shape_params = []
if 'pose' in self.top_names:
self.pose_params = []
print 'ArticulatedObjectDataLayer has been setup.'
class ArticulatedObjectDataLayer(AbstractDataLayer):
def parse_param_str(self, param_str):
top_names_choices = ['data', 'shape', 'pose']
default_mean_bgr = [104.00698793, 116.66876762,
122.67891434] # CaffeNet
default_im_size = [227, 227] # CaffeNet
parser = argparse.ArgumentParser(
description='Articulated Object Data Layer')
parser.add_argument("-b",
"--batch_size",
default=50,
type=int,
help="Batch Size")
parser.add_argument("-wh",
"--im_size",
nargs=2,
default=default_im_size,
type=int,
metavar=('WIDTH', 'HEIGHT'),
help="Image Size [width, height]")
parser.add_argument("-m",
"--mean_bgr",
nargs=3,
default=default_mean_bgr,
type=float,
metavar=('B', 'G', 'R'),
help="Mean BGR color value")
parser.add_argument("-t",
"--top_names",
nargs='+',
choices=top_names_choices,
required=True,
type=str,
help="ordered list of top names e.g data shape")
parser.add_argument("-s",
"--shape_param_size",
default=10,
type=int,
help="Shape param Size")
parser.add_argument("-p",
"--pose_param_size",
default=36,
type=int,
help="Pose param Size")
params = parser.parse_args(param_str.split())
print "------------- ArticulatedObjectDataLayer Config -----------------"
for arg in vars(params):
print "\t{} \t= {}".format(arg, getattr(params, arg))
print "------------------------------------------------------------"
return params
def setup(self, bottom, top):
print "Setting up ArticulatedObjectDataLayer ..."
# params is expected as argparse style string
params = self.parse_param_str(self.param_str)
# Store the ordered list of top_names
self.top_names = params.top_names
# Store batch size as member variable for use in other methods
self.batch_size = params.batch_size
# create mean bgr to directly operate on image data blob
self.mean_bgr = np.array(params.mean_bgr).reshape(1, 3, 1, 1)
self.shape_param_size = params.shape_param_size
self.pose_param_size = params.pose_param_size
assert 'data' in self.top_names, 'Requires atleast data layer'
assert len(top) == len(
self.top_names), 'Number of tops do not match specified top_names'
# Reshape image data top (B, C, H, W)
if 'data' in self.top_names:
top[self.top_names.index('data')].reshape(
self.batch_size, 3, params.im_size[1],
params.im_size[0]) # Image Data
# Reshape shape top (B, 10)
if 'shape' in self.top_names:
top[self.top_names.index('shape')].reshape(self.batch_size,
self.shape_param_size)
# Reshape pose top (B, 36)
if 'pose' in self.top_names:
top[self.top_names.index('pose')].reshape(self.batch_size,
self.pose_param_size)
# Create a loader to load the images.
self.image_loader = BatchImageLoader(params.im_size)
if 'shape' in self.top_names:
self.shape_params = []
if 'pose' in self.top_names:
self.pose_params = []
print 'ArticulatedObjectDataLayer has been setup.'
def add_dataset(self, dataset):
print '---- Adding data from {} datatset -----'.format(dataset.name())
image_files = []
cropping_boxes = []
for i in xrange(dataset.num_of_images()):
annotation = dataset.image_infos()[i]
img_path = osp.join(dataset.rootdir(), annotation['image_file'])
image_files.append(img_path)
visible_bbx = np.array(
annotation['bbx_visible'],
dtype=np.float) # gt box (only visible path)
# We need to probably do jittering
cropping_boxes.append(visible_bbx)
if hasattr(self, 'shape_params'):
assert len(annotation['shape_param']) == self.shape_param_size
self.shape_params.append(
np.array(annotation['shape_param'], dtype=np.float))
if hasattr(self, 'pose_params'):
assert len(annotation['pose_param']) == self.pose_param_size
self.pose_params.append(
np.array(annotation['pose_param'], dtype=np.float))
self.image_loader.crop_and_preload_images(image_files, cropping_boxes)
print "--------------------------------------------------------------------"
def generate_datum_ids(self):
num_of_data_points = len(self.image_loader)
for attr in ['shape_params', 'pose_params']:
if hasattr(self, attr):
print attr
assert len(
getattr(self, attr)
) == num_of_data_points, 'attr {} has length {} instead of {}'.format(
attr, len(getattr(self, attr)), num_of_data_points)
# set of data indices in [0, num_of_data_points)
self.data_ids = range(num_of_data_points)
self.curr_data_ids_idx = 0
# Shuffle from the begining if in the train phase
if (self.phase == caffe.TRAIN):
shuffle(self.data_ids)
print 'Total number of data points (annotations) = {:,}'.format(
num_of_data_points)
return num_of_data_points
def forward(self, bottom, top):
"""
Load current batch of data and labels to caffe blobs
"""
assert hasattr(
self, 'data_ids'
), 'Most likely data has not been initialized before calling forward()'
assert len(
self.data_ids
) > self.batch_size, 'batch size cannot be smaller than total number of data points'
# For Debug
# print "{} -- {}".format(self.data_ids[self.curr_data_ids_idx],
# self.data_ids[self.curr_data_ids_idx + 100])
for i in xrange(self.batch_size):
# Did we finish an epoch?
if self.curr_data_ids_idx == len(self.data_ids):
self.curr_data_ids_idx = 0
shuffle(self.data_ids)
data_idx = self.data_ids[self.curr_data_ids_idx]
if 'data' in self.top_names:
top[self.top_names.index('data')].data[
i, ...] = self.image_loader[data_idx]
if hasattr(self, 'shape_params'):
top[self.top_names.index('shape')].data[
i, ...] = self.shape_params[data_idx]
if hasattr(self, 'pose_params'):
top[self.top_names.index('pose')].data[
i, ...] = self.pose_params[data_idx]
self.curr_data_ids_idx += 1
# subtarct mean from full image data blob
if 'data' in self.top_names:
top[self.top_names.index('data')].data[...] -= self.mean_bgr
class RCNNDataLayer(AbstractDataLayer):
"""Data Layer RCNN style. Inherits AbstractDataLayer"""
def parse_param_str(self, param_str):
top_names_choices = [
'input_image', 'viewpoint', 'bbx_amodal', 'bbx_crop', 'center_proj'
]
default_mean_bgr = [103.0626238, 115.90288257, 123.15163084] # ResNet
default_std_bgr = [1.0, 1.0, 1.0] # ResNet
default_im_size = [224, 224] # ResNet
parser = argparse.ArgumentParser(description='RCNN style Data Layer')
parser.add_argument("-b",
"--batch_size",
default=32,
type=int,
help="Batch Size")
parser.add_argument("-wh",
"--im_size",
nargs=2,
default=default_im_size,
type=int,
metavar=('WIDTH', 'HEIGHT'),
help="Image Size [width, height]")
parser.add_argument("-m",
"--mean_bgr",
nargs=3,
default=default_mean_bgr,
type=float,
metavar=('B', 'G', 'R'),
help="Mean BGR color value")
parser.add_argument("-s",
"--std_bgr",
nargs=3,
default=default_std_bgr,
type=float,
metavar=('B', 'G', 'R'),
help="Std BGR color value")
parser.add_argument(
"-t",
"--top_names",
nargs='+',
choices=top_names_choices,
required=True,
type=str,
help="ordered list of top names e.g input_image azimuth shape")
parser.add_argument(
"-f",
"--flip_ratio",
default=0.5,
type=float,
help="Flip ratio in range [0, 1] (Defaults to 0.5)")
parser.add_argument(
"-c",
"--crop_target",
default='bbx_visible',
choices=['bbx_amodal', 'bbx_visible'],
type=str,
help="bbx type used for cropping (Defaults to bbx_visible)")
parser.add_argument(
"-u",
"--uniform_crop",
action='store_true',
default=False,
help="If set we do not change the aspect ratio while cropping")
params = parser.parse_args(param_str.split())
print "-------------------- RCNNDataLayer Config ----------------------"
for arg in vars(params):
print "\t{} \t= {}".format(arg, getattr(params, arg))
print "------------------------------------------------------------"
return params
def setup(self, bottom, top):
print "Setting up RCNNDataLayer ..."
# params is expected as argparse style string
params = self.parse_param_str(self.param_str)
# Store the ordered list of top_names
self.top_names = params.top_names
# Store batch size as member variable for use in other methods
self.batch_size = params.batch_size
# create mean bgr to directly operate on image data blob
self.mean_bgr = np.array(params.mean_bgr)
# create std bgr to directly operate on image data blob
self.std_bgr = np.array(params.std_bgr)
# set network input_image size
self.im_size = params.im_size
# set flip_ratio
self.flip_ratio = params.flip_ratio
# set crop_target
self.crop_target = params.crop_target
# set uniform_crop
self.uniform_crop = params.uniform_crop
assert len(top) == len(
self.top_names), 'Number of tops do not match specified top_names'
# set top shapes
top_shapes = {
"input_image":
(self.batch_size, 3, self.im_size[1], self.im_size[0]),
"viewpoint": (self.batch_size, 3),
"bbx_amodal": (self.batch_size, 4),
"bbx_crop": (self.batch_size, 4),
"center_proj": (self.batch_size, 2),
}
# Reshape tops
for top_index, top_name in enumerate(self.top_names):
top[top_index].reshape(*top_shapes[top_name])
# Create a loader to load the images.
self.image_loader = BatchImageLoader(transpose=False)
# Create placeholder for annotations
self.data_samples = []
print 'RCNNDataLayer has been setup.'
def add_dataset(self, dataset):
"""Add annotations from a json dataset"""
print '---- Adding data from {} datatset -----'.format(dataset.name())
print 'Number of data points (annotations) = {:,}'.format(
len(self.data_samples))
prev_num_of_images = len(self.image_loader)
image_files = []
for i in xrange(dataset.num_of_images()):
image_info = dataset.image_infos()[i]
img_path = osp.join(dataset.rootdir(), image_info['image_file'])
image_files.append(img_path)
image_id = prev_num_of_images + i
for obj_info in image_info['object_infos']:
# TODO Use configurable params
if 'occlusion' in obj_info and obj_info['occlusion'] > 0.8:
continue
if 'truncation' in obj_info and obj_info['truncation'] > 0.8:
continue
data_sample = {}
data_sample['image_id'] = image_id
data_sample['id'] = obj_info['id']
data_sample['category'] = obj_info['category']
for field in [
'viewpoint', 'bbx_amodal', 'bbx_visible', 'center_proj'
]:
if field in obj_info:
data_sample[field] = np.array(obj_info[field])
if 'viewpoint' in data_sample:
vp = data_sample['viewpoint']
assert (vp >= -np.pi).all() and (
vp < np.pi).all(), "Bad viewpoint = {}".format(vp)
# Add data_sample
self.data_samples.append(data_sample)
self.image_loader.add_images(image_files)
print 'Number of data points (annotations) = {:,}'.format(
len(self.data_samples))
print "--------------------------------------------------------------------"
def generate_datum_ids(self):
num_of_data_points = len(self.data_samples)
# set of data indices in [0, num_of_data_points)
self.data_ids = range(num_of_data_points)
self.curr_data_ids_idx = 0
# Shuffle from the begining if in the train phase
if self.phase == caffe.TRAIN:
shuffle(self.data_ids)
assert len(
self.data_ids
) > self.batch_size, 'batch size ({})is smaller than number of data points ({}).'.format(
self.batch_size, len(self.data_ids))
print 'Total number of data points (annotations) = {:,}'.format(
num_of_data_points)
return num_of_data_points
def augment_data_sample(self, data_idx):
"""Returns augmented data_sample and image"""
# fetch the data sample (object)
original_data_sample = self.data_samples[data_idx]
full_image = self.image_loader[original_data_sample['image_id']]
# TODO Jitter
bbx_crop = original_data_sample[self.crop_target].copy()
data_sample = {}
data_sample['id'] = original_data_sample['id']
data_sample['category'] = original_data_sample['category']
data_sample['bbx_crop'] = bbx_crop
data_sample['bbx_amodal'] = original_data_sample['bbx_amodal'].copy()
data_sample['viewpoint'] = original_data_sample['viewpoint'].copy()
data_sample['center_proj'] = original_data_sample['center_proj'].copy()
if self.uniform_crop:
data_sample['input_image'] = uniform_crop_and_resize_image(
full_image, bbx_crop, self.im_size, self.mean_bgr)
else:
data_sample['input_image'] = crop_and_resize_image(
full_image, bbx_crop, self.im_size)
if random() < self.flip_ratio:
W = full_image.shape[1]
data_sample['bbx_crop'][[0,
2]] = W - data_sample['bbx_crop'][[2, 0]]
data_sample['bbx_amodal'][[
0, 2
]] = W - data_sample['bbx_amodal'][[2, 0]]
data_sample['center_proj'][0] = W - data_sample['center_proj'][0]
data_sample['viewpoint'][[0,
2]] = -data_sample['viewpoint'][[0, 2]]
data_sample['input_image'] = np.fliplr(data_sample['input_image'])
# Change image channel order
data_sample['input_image'] = data_sample['input_image'].transpose(
(2, 0, 1))
return data_sample
def forward(self, bottom, top):
"""
Load current batch of data and labels to caffe blobs
"""
assert hasattr(
self, 'data_ids'
), 'Most likely data has not been initialized before calling forward()'
for i in xrange(self.batch_size):
# Did we finish an epoch?
if self.curr_data_ids_idx == len(self.data_ids):
self.curr_data_ids_idx = 0
shuffle(self.data_ids)
# Current Data index
data_idx = self.data_ids[self.curr_data_ids_idx]
# fetch the data sample (object)
data_sample = self.augment_data_sample(data_idx)
# set to data
for top_index, top_name in enumerate(self.top_names):
top[top_index].data[i, ...] = data_sample[top_name]
self.curr_data_ids_idx += 1
# subtarct mean from image data blob
if 'input_image' in self.top_names:
top[self.top_names.index('input_image')].data[
...] -= self.mean_bgr.reshape(1, 3, 1, 1)
top[self.top_names.index('input_image')].data[
...] /= self.std_bgr.reshape(1, 3, 1, 1)
def get_predictions_on_image_croppings(img_files, cropping_boxes, net_file,
weights_file, requested_keys, mean,
gpu_id):
assert osp.exists(
net_file), 'Path to test net prototxt does not exist: {}'.format(
net_file)
assert osp.exists(
weights_file), 'Path to weights file does not exist: {}'.format(
weights_file)
assert len(mean) == 3, 'Expects mean as list of 3 numbers ([B, G, R])'
num_of_images = len(img_files)
caffe.set_device(gpu_id)
caffe.set_mode_gpu()
net = caffe.Net(net_file, weights_file, caffe.TEST)
data_blob_shape = net.blobs['data'].data.shape
assert len(data_blob_shape) == 4, 'Expects 4D data blob'
assert data_blob_shape[1] == 3, 'Expects 2nd channel to be 3 for BGR image'
batch_size = data_blob_shape[0]
im_size = [data_blob_shape[3],
data_blob_shape[2]] # caffe blob are (b,c,h,w)
image_loader = BatchImageLoader(im_size)
image_loader.crop_and_preload_images(img_files, cropping_boxes)
predictions = {}
for key in requested_keys:
if key in net.blobs:
blob_shape = net.blobs[key].data.shape
assert blob_shape[
0] == batch_size, 'Expects 1st channel to be batch_size'
pred_shape = list(blob_shape)
pred_shape[0] = num_of_images
predictions[key] = np.zeros(tuple(pred_shape))
else:
print 'Requested key {} not available in network'.format(key)
mean_bgr = np.array(mean).reshape(1, 3, 1, 1)
num_of_batches = int(math.ceil(num_of_images / float(batch_size)))
for b in xrange(num_of_batches):
start_idx = batch_size * b
end_idx = min(batch_size * (b + 1), num_of_images)
print 'Working on batch: %d/%d (Image# %d - %d)' % (b, num_of_batches,
start_idx, end_idx)
for i in xrange(start_idx, end_idx):
net.blobs['data'].data[i - start_idx, ...] = image_loader[i]
# subtarct mean from image data blob
net.blobs['data'].data[...] -= mean_bgr
output = net.forward()
for key in predictions:
predictions[key][start_idx:end_idx,
...] = output[key][0:end_idx - start_idx, ...]
return predictions
class ShapePredictionDataLayer(AbstractDataLayer):
def parse_param_str(self, param_str):
parser = argparse.ArgumentParser(
description='Shape Prediction Data Layer')
parser.add_argument("-b",
"--batch_size",
default=50,
type=int,
help="Batch Size")
parser.add_argument("-wh",
"--im_size",
nargs=2,
default=[227, 227],
type=int,
metavar=('WIDTH', 'HEIGHT'),
help="Image Size [width, height]")
parser.add_argument("-m",
"--mean_bgr",
nargs=3,
default=[104.00698793, 116.66876762, 122.67891434],
type=float,
metavar=('B', 'G', 'R'),
help="Mean BGR color value")
params = parser.parse_args(param_str.split())
print "------------- ShapePredictionDataLayer Config -----------------"
for arg in vars(params):
print "\t{} \t= {}".format(arg, getattr(params, arg))
print "------------------------------------------------------------"
return params
def setup(self, bottom, top):
print "Setting up ShapePredictionDataLayer ..."
assert len(top) == 2, 'requires 2 tops: (image) data, gt_shape'
# params is expected as argparse style string
self.params = self.parse_param_str(self.param_str)
# ----- Reshape tops -----#
# data shape = B x C x H x W
# shape_params shape = B x 10
top[0].reshape(self.params.batch_size, 3, self.params.im_size[1],
self.params.im_size[0]) # Image Data
top[1].reshape(self.params.batch_size, 10) # ShapeTarget
# create mean bgr to directly operate on image data blob
self.mean_bgr = np.array(self.params.mean_bgr).reshape(1, 3, 1, 1)
# Create a loader to load the images.
self.image_loader = BatchImageLoader(self.params.im_size)
# Create placeholder for GT annotations
self.shape_params = []
print 'ShapePredictionDataLayer has been setup.'
def add_dataset(self, dataset):
print '---- Adding data from {} datatset -----'.format(dataset.name())
image_files = []
for i in xrange(dataset.num_of_images()):
annotation = dataset.image_infos()[i]
img_path = osp.join(dataset.rootdir(), annotation['image_file'])
image_files.append(img_path)
self.shape_params.append(
np.array(annotation['shape_param'], dtype=np.float))
self.image_loader.add_images(image_files)
print "--------------------------------------------------------------------"
def generate_datum_ids(self):
assert len(self.shape_params) == len(self.image_loader)
num_of_data_points = len(self.image_loader)
# set of data indices in [0, num_of_data_points)
self.data_ids = range(num_of_data_points)
self.curr_data_ids_idx = 0
# Shuffle from the begining if in the train phase
if (self.phase == caffe.TRAIN):
shuffle(self.data_ids)
print 'Total number of data points (annotations) = {:,}'.format(
num_of_data_points)
return num_of_data_points
def forward(self, bottom, top):
"""
Load current batch of data and labels to caffe blobs
"""
assert hasattr(
self, 'data_ids'
), 'Most likely data has not been initialized before calling forward()'
assert len(
self.data_ids
) > self.params.batch_size, 'batch size cannot be smaller than total number of data points'
# For Debug
# print "{} -- {}".format(self.data_ids[self.curr_data_ids_idx],
# self.data_ids[self.curr_data_ids_idx + 100])
for i in xrange(self.params.batch_size):
# Did we finish an epoch?
if self.curr_data_ids_idx == len(self.data_ids):
self.curr_data_ids_idx = 0
shuffle(self.data_ids)
# Add directly to the caffe data layer
data_idx = self.data_ids[self.curr_data_ids_idx]
top[0].data[i, ...] = self.image_loader[data_idx]
top[1].data[i, ...] = self.shape_params[data_idx]
self.curr_data_ids_idx += 1
# subtarct mean from image data blob
top[0].data[...] -= self.mean_bgr