def __init__(self, dim_in, spatial_scales):
super().__init__()
self.dim_in = dim_in
self.spatial_scales = spatial_scales
self.dim_out = self.dim_in
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
if cfg.M_ANCHOR or cfg.RIGHT_ANCHOR:
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS) * 2
# Create conv ops shared by all FPN levels
self.FPN_RPN_conv = nn.Conv2d(dim_in, self.dim_out, 3, 1, 1)
dim_score = num_anchors * 2 if cfg.RPN.CLS_ACTIVATION == 'softmax' \
else num_anchors
self.FPN_RPN_cls_score = nn.Conv2d(self.dim_out, dim_score, 1, 1, 0)
self.FPN_RPN_bbox_pred = nn.Conv2d(self.dim_out, 4 * num_anchors, 1, 1,
0)
self.GenerateProposals_modules = nn.ModuleList()
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
for lvl in range(k_min, k_max + 1):
sc = self.spatial_scales[k_max - lvl] # in reversed order
if cfg.M_ANCHOR:
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(
cfg.FPN.RPN_ANCHOR_START_SIZE * 1.**(lvl - k_min),
cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min),
),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS)
elif cfg.RIGHT_ANCHOR:
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(
cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min - 1),
cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min),
),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS)
else:
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE *
2.**(lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS)
self.GenerateProposals_modules.append(
GenerateProposalsOp(lvl_anchors, sc))
self.CollectAndDistributeFpnRpnProposals = CollectAndDistributeFpnRpnProposalsOp(
)
self._init_weights()
def create_cell_anchors():
"""
Generate all types of anchors for all fpn levels/scales/aspect ratios.
This function is called only once at the beginning of inference.
"""
k_max, k_min = cfg.FPN.RPN_MAX_LEVEL, cfg.FPN.RPN_MIN_LEVEL
scales_per_octave = cfg.RETINANET.SCALES_PER_OCTAVE
aspect_ratios = cfg.RETINANET.ASPECT_RATIOS
anchor_scale = cfg.RETINANET.ANCHOR_SCALE
A = scales_per_octave * len(aspect_ratios)
anchors = {}
for lvl in range(k_min, k_max + 1):
# create cell anchors array
stride = 2. ** lvl
cell_anchors = np.zeros((A, 4))
a = 0
for octave in range(scales_per_octave):
octave_scale = 2 ** (octave / float(scales_per_octave))
for aspect in aspect_ratios:
anchor_sizes = (stride * octave_scale * anchor_scale, )
anchor_aspect_ratios = (aspect, )
cell_anchors[a, :] = generate_anchors(
stride=stride, sizes=anchor_sizes,
aspect_ratios=anchor_aspect_ratios)
a += 1
anchors[lvl] = cell_anchors
return anchors
def _create_cell_anchors():
"""
Generate all types of anchors for all fpn levels/scales/aspect ratios.
This function is called only once at the beginning of inference.
"""
k_max, k_min = cfg.FPN.RPN_MAX_LEVEL, cfg.FPN.RPN_MIN_LEVEL
scales_per_octave = cfg.RETINANET.SCALES_PER_OCTAVE
aspect_ratios = cfg.RETINANET.ASPECT_RATIOS
anchor_scale = cfg.RETINANET.ANCHOR_SCALE
A = scales_per_octave * len(aspect_ratios)
anchors = {}
for lvl in range(k_min, k_max + 1):
# create cell anchors array
stride = 2.**lvl
cell_anchors = np.zeros((A, 4))
a = 0
for octave in range(scales_per_octave):
octave_scale = 2**(octave / float(scales_per_octave))
for aspect in aspect_ratios:
anchor_sizes = (stride * octave_scale * anchor_scale, )
anchor_aspect_ratios = (aspect, )
cell_anchors[a, :] = generate_anchors(
stride=stride,
sizes=anchor_sizes,
aspect_ratios=anchor_aspect_ratios)
a += 1
anchors[lvl] = cell_anchors
return anchors
def test_res(self):
cls_probs, box_preds, im_info, im, im_name, root_dir, dataset = self._test_std()
max_level = cfg.FPN.RPN_MAX_LEVEL
min_level = cfg.FPN.RPN_MIN_LEVEL
# then build the anchors
anchors = []
for lvl in range(min_level, max_level+1):
field_stride = 2 ** lvl
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - min_level), )
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
anchor = generate_anchors(
stride=field_stride,
sizes=anchor_sizes,
aspect_ratios=anchor_aspect_ratios
)
anchors.append(anchor.astype(np.float32))
boxes = self._run_cpu_testing(cls_probs, box_preds, anchors, im_info)
utils.vis.vis_one_image(im[:, :, ::-1],
'{:s}-cd-output'.format(im_name),
osp.join(root_dir, 'tests'),
boxes,
segms=None,
keypoints=None,
thresh=0.,
box_alpha=0.8,
dataset=dataset,
show_class=False)
def __init__(self, dim_in, spatial_scales):
super().__init__()
self.dim_in = dim_in
self.spatial_scales = spatial_scales
self.dim_out = self.dim_in
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
# Create conv ops shared by all FPN levels
self.FPN_RPN_conv = nn.Conv2d(dim_in, self.dim_out, 3, 1, 1)
dim_score = num_anchors * 2 if cfg.RPN.CLS_ACTIVATION == 'softmax' \
else num_anchors
self.FPN_RPN_cls_score = nn.Conv2d(self.dim_out, dim_score, 1, 1, 0)
self.FPN_RPN_bbox_pred = nn.Conv2d(self.dim_out, 4 * num_anchors, 1, 1, 0)
self.GenerateProposals_modules = nn.ModuleList()
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
for lvl in range(k_min, k_max + 1):
sc = self.spatial_scales[k_max - lvl] # in reversed order
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS
)
self.GenerateProposals_modules.append(GenerateProposalsOp(lvl_anchors, sc))
self.CollectAndDistributeFpnRpnProposals = CollectAndDistributeFpnRpnProposalsOp()
self._init_weights()
def get_field_of_anchors(stride,
anchor_sizes,
anchor_aspect_ratios,
octave=None,
aspect=None):
global _threadlocal_foa
if not hasattr(_threadlocal_foa, 'cache'):
_threadlocal_foa.cache = {}
cache_key = str(stride) + str(anchor_sizes) + str(anchor_aspect_ratios)
if cache_key in _threadlocal_foa.cache:
return _threadlocal_foa.cache[cache_key]
# Anchors at a single feature cell
cell_anchors = generate_anchors(stride=stride,
sizes=anchor_sizes,
aspect_ratios=anchor_aspect_ratios)
num_cell_anchors = cell_anchors.shape[0]
# Generate canonical proposals from shifted anchors
# Enumerate all shifted positions on the (H, W) grid
# FPN max size
fpn_max_size = cfg.FPN.COARSEST_STRIDE * np.ceil(
cfg.TRAIN.MAX_SIZE / float(cfg.FPN.COARSEST_STRIDE))
# RefineNet max size
if cfg.MODEL.REFINE_ON:
refinenet_stride = (1. /
cfg.REFINENET.SPATIAL_SCALE) * cfg.REFINENET.STRIDE
refinenet_max_size = refinenet_stride * np.ceil(
cfg.TRAIN.MAX_SIZE / float(refinenet_stride))
fpn_max_size = max(fpn_max_size, refinenet_max_size)
field_size = int(np.ceil(fpn_max_size / float(stride)))
shifts = np.arange(0, field_size) * stride
shift_x, shift_y = np.meshgrid(shifts, shifts)
shift_x = shift_x.ravel()
shift_y = shift_y.ravel()
shifts = np.vstack((shift_x, shift_y, shift_x, shift_y)).transpose()
# Broacast anchors over shifts to enumerate all anchors at all positions
# in the (H, W) grid:
# - add A cell anchors of shape (1, A, 4) to
# - K shifts of shape (K, 1, 4) to get
# - all shifted anchors of shape (K, A, 4)
# - reshape to (K*A, 4) shifted anchors
A = num_cell_anchors
K = shifts.shape[0]
field_of_anchors = (cell_anchors.reshape((1, A, 4)) + shifts.reshape(
(1, K, 4)).transpose((1, 0, 2)))
field_of_anchors = field_of_anchors.reshape((K * A, 4))
foa = FieldOfAnchors(field_of_anchors=field_of_anchors.astype(np.float32),
num_cell_anchors=num_cell_anchors,
stride=stride,
field_size=field_size,
octave=octave,
aspect=aspect)
_threadlocal_foa.cache[cache_key] = foa
return foa
def add_single_scale_rpn_outputs(model, blob_in, dim_in, spatial_scale):
"""Add RPN outputs to a single scale model (i.e., no FPN)."""
anchors = generate_anchors(stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS)
num_anchors = anchors.shape[0]
dim_out = dim_in
# RPN hidden representation
model.Conv(blob_in,
'conv_rpn',
dim_in,
dim_out,
kernel=3,
pad=1,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0))
model.Relu('conv_rpn', 'conv_rpn')
# Proposal classification scores
model.Conv('conv_rpn',
'rpn_cls_logits',
dim_in,
num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0))
# Proposal bbox regression deltas
model.Conv('conv_rpn',
'rpn_bbox_pred',
dim_in,
4 * num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0))
if not model.train or cfg.MODEL.FASTER_RCNN:
# Proposals are needed during:
# 1) inference (== not model.train) for RPN only and Faster R-CNN
# OR
# 2) training for Faster R-CNN
# Otherwise (== training for RPN only), proposals are not needed
model.net.Sigmoid('rpn_cls_logits', 'rpn_cls_probs')
model.GenerateProposals(['rpn_cls_probs', 'rpn_bbox_pred', 'im_info'],
['rpn_rois', 'rpn_roi_probs'],
anchors=anchors,
spatial_scale=spatial_scale)
if cfg.MODEL.FASTER_RCNN:
if model.train:
# Add op that generates training labels for in-network RPN proposals
model.GenerateProposalLabels(['rpn_rois', 'roidb', 'im_info'])
else:
# Alias rois to rpn_rois for inference
model.net.Alias('rpn_rois', 'rois')
def get_field_of_anchors(
stride, anchor_sizes, anchor_aspect_ratios, octave=None, aspect=None
):
global _threadlocal_foa
if not hasattr(_threadlocal_foa, 'cache'):
_threadlocal_foa.cache = {}
cache_key = str(stride) + str(anchor_sizes) + str(anchor_aspect_ratios)
if cache_key in _threadlocal_foa.cache:
return _threadlocal_foa.cache[cache_key]
# Anchors at a single feature cell
cell_anchors = generate_anchors(
stride=stride, sizes=anchor_sizes, aspect_ratios=anchor_aspect_ratios
)
num_cell_anchors = cell_anchors.shape[0]
# Generate canonical proposals from shifted anchors
# Enumerate all shifted positions on the (H, W) grid
fpn_max_size = cfg.FPN.COARSEST_STRIDE * np.ceil(
cfg.TRAIN.MAX_SIZE / float(cfg.FPN.COARSEST_STRIDE)
)
field_size = int(np.ceil(fpn_max_size / float(stride)))
shifts = np.arange(0, field_size) * stride
shift_x, shift_y = np.meshgrid(shifts, shifts)
shift_x = shift_x.ravel()
shift_y = shift_y.ravel()
shifts = np.vstack((shift_x, shift_y, shift_x, shift_y)).transpose()
# Broacast anchors over shifts to enumerate all anchors at all positions
# in the (H, W) grid:
# - add A cell anchors of shape (1, A, 4) to
# - K shifts of shape (K, 1, 4) to get
# - all shifted anchors of shape (K, A, 4)
# - reshape to (K*A, 4) shifted anchors
A = num_cell_anchors
K = shifts.shape[0]
field_of_anchors = (
cell_anchors.reshape((1, A, 4)) +
shifts.reshape((1, K, 4)).transpose((1, 0, 2))
)
field_of_anchors = field_of_anchors.reshape((K * A, 4))
foa = FieldOfAnchors(
field_of_anchors=field_of_anchors.astype(np.float32),
num_cell_anchors=num_cell_anchors,
stride=stride,
field_size=field_size,
octave=octave,
aspect=aspect
)
_threadlocal_foa.cache[cache_key] = foa
return foa
def _get_field_of_anchors(stride, anchor_sizes, anchor_aspect_ratios,
time_dim):
global _threadlocal_foa
if not hasattr(_threadlocal_foa, 'cache'):
_threadlocal_foa.cache = {}
cache_key = str(stride) + str(anchor_sizes) + str(anchor_aspect_ratios)
if cache_key in _threadlocal_foa.cache:
return _threadlocal_foa.cache[cache_key]
# Anchors at a single feature cell
cell_anchors = generate_anchors(stride=stride,
sizes=anchor_sizes,
aspect_ratios=anchor_aspect_ratios,
time_dim=time_dim)
num_cell_anchors = cell_anchors.shape[0]
# Generate canonical proposals from shifted anchors
# Enumerate all shifted positions on the (H, W) grid
fpn_max_size = cfg.FPN.COARSEST_STRIDE * np.ceil(
cfg.TRAIN.MAX_SIZE / float(cfg.FPN.COARSEST_STRIDE))
field_size = int(np.ceil(fpn_max_size / float(stride)))
shifts = np.arange(0, field_size) * stride
shift_x, shift_y = np.meshgrid(shifts, shifts)
shift_x = shift_x.flatten()
shift_y = shift_y.flatten()
shifts = np.vstack((shift_x, shift_y, shift_x, shift_y)).transpose()
# For the time case, replicate the shifts for other boxes in the tube
shifts = time_extend_shifts(shifts, time_dim)
# Broacast anchors over shifts to enumerate all anchors at all positions
# in the (H, W) grid:
# - add A cell anchors of shape (1, A, 4) to
# - K shifts of shape (K, 1, 4) to get
# - all shifted anchors of shape (K, A, 4)
# - reshape to (K*A, 4) shifted anchors
A = num_cell_anchors
K = shifts.shape[0]
field_of_anchors = (cell_anchors.reshape(
(1, A, 4 * time_dim)) + shifts.reshape((1, K, 4 * time_dim)).transpose(
(1, 0, 2)))
field_of_anchors = field_of_anchors.reshape((K * A, 4 * time_dim))
foa = FieldOfAnchors(field_of_anchors=field_of_anchors.astype(np.float32),
num_cell_anchors=num_cell_anchors,
stride=stride,
field_size=field_size)
_threadlocal_foa.cache[cache_key] = foa
return foa
def __init__(self, dim_in, spatial_scales):
super().__init__()
self.dim_in = dim_in
self.spatial_scales = spatial_scales
self.dim_out = self.dim_in
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
# Create conv ops shared by all FPN levels
# 12/25,RPN 3D CONTEXT TEST
if cfg.LESION.CONCAT_BEFORE_RPN:
#self.cbam = CBAM(self.dim_in*cfg.LESION.NUM_IMAGES_3DCE, 16)
#self.FPN_RPN_conv_embedding = nn.Conv2d(self.dim_in*cfg.LESION.NUM_IMAGES_3DCE, self.dim_in, 1)
self.FPN_RPN_conv = nn.Conv2d(self.dim_in*cfg.LESION.NUM_IMAGES_3DCE, self.dim_in, 1)
else:
self.FPN_RPN_conv = nn.Conv2d(self.dim_in, self.dim_out, 3, 1, 1)
#elif cfg.LESION.SUM_BEFORE_RPN:
#self.FPN_RPN_conv_embedding = nn.Conv2d(self.dim_in, self.dim_in, 1)
#self.FPN_RPN_conv = nn.Conv2d(self.dim_in*cfg.LESION.NUM_IMAGES_3DCE, self.dim_out, 3, 1, 1)
dim_score = num_anchors * 2 if cfg.RPN.CLS_ACTIVATION == 'softmax' \
else num_anchors
self.FPN_RPN_cls_score = nn.Conv2d(self.dim_out, dim_score, 1, 1, 0)
self.FPN_RPN_bbox_pred = nn.Conv2d(self.dim_out, 4 * num_anchors, 1, 1, 0)
self.GenerateProposals_modules = nn.ModuleList()
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
#anchor_scale = [0,0,2,3,4,6,12]
#anchor_scale = [0,0,1,2,4,8,16]
#anchor_scale = [0,0,1,2,3,6,12]
for lvl in range(k_min, k_max + 1):
sc = self.spatial_scales[k_max - lvl] # in reversed order
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ),
#sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * anchor_scale[lvl], ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS
)
self.GenerateProposals_modules.append(GenerateProposalsOp(lvl_anchors, sc))
self.CollectAndDistributeFpnRpnProposals = CollectAndDistributeFpnRpnProposalsOp()
self._init_weights()
def __init__(self, dim_in, spatial_scale):
super().__init__()
self.dim_in = dim_in
self.dim_out = dim_in if cfg.RPN.OUT_DIM_AS_IN_DIM else cfg.RPN.OUT_DIM
anchors = generate_anchors(stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS)
num_anchors = anchors.shape[0]
# RPN hidden representation
self.RPN_conv = nn.Conv2d(self.dim_in, self.dim_out, 3, 1, 1)
# Proposal classification scores
self.n_score_out = num_anchors * 2 if cfg.RPN.CLS_ACTIVATION == 'softmax' \
else num_anchors
self.RPN_cls_score = nn.Conv2d(self.dim_out, self.n_score_out, 1, 1, 0)
# Proposal bbox regression deltas
self.RPN_bbox_pred = nn.Conv2d(self.dim_out, num_anchors * 4, 1, 1, 0)
self.RPN_GenerateProposals = GenerateProposalsOp(
anchors, spatial_scale)
self.RPN_GenerateProposalLabels = GenerateProposalLabelsOp()
self._init_weights()
def __init__(self, dim_in, spatial_scale):
super().__init__()
self.dim_in = dim_in
self.dim_out = dim_in if cfg.RPN.OUT_DIM_AS_IN_DIM else cfg.RPN.OUT_DIM
anchors = generate_anchors(
stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS)
num_anchors = anchors.shape[0]
# RPN hidden representation
self.RPN_conv = nn.Conv2d(self.dim_in, self.dim_out, 3, 1, 1)
# Proposal classification scores
self.n_score_out = num_anchors * 2 if cfg.RPN.CLS_ACTIVATION == 'softmax' \
else num_anchors
self.RPN_cls_score = nn.Conv2d(self.dim_out, self.n_score_out, 1, 1, 0)
# Proposal bbox regression deltas
self.RPN_bbox_pred = nn.Conv2d(self.dim_out, num_anchors * 4, 1, 1, 0)
self.RPN_GenerateProposals = GenerateProposalsOp(anchors, spatial_scale)
self.RPN_GenerateProposalLabels = GenerateProposalLabelsOp()
self._init_weights()
def __init__(self, dim_in, spatial_scales):
super().__init__()
self.dim_in = dim_in
self.spatial_scales = spatial_scales
self.dim_out = self.dim_in
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
# Create conv ops shared by all FPN levels
'''i think this is the head part, they go across a conv with equal-input-output-size, and then
go through the score and bbox subnet separately'''
self.FPN_RPN_conv = nn.Conv2d(dim_in, self.dim_out, 3, 1, 1)
dim_score = num_anchors * 2 if cfg.RPN.CLS_ACTIVATION == 'softmax' \
else num_anchors
self.FPN_RPN_cls_score = nn.Conv2d(self.dim_out, dim_score, 1, 1, 0)
self.FPN_RPN_bbox_pred = nn.Conv2d(self.dim_out, 4 * num_anchors, 1, 1,
0)
self.GenerateProposals_modules = nn.ModuleList()
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
for lvl in range(k_min, k_max + 1):
'''3..0'''
sc = self.spatial_scales[k_max - lvl] # in reversed order
'''FIXME the part to work with!!'''
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS)
self.GenerateProposals_modules.append(
GenerateProposalsOp(lvl_anchors, sc))
self.CollectAndDistributeFpnRpnProposals = CollectAndDistributeFpnRpnProposalsOp(
)
self._init_weights()
def add_rpn_outputs(model, blob_in, dim_in, spatial_scale, nd=False, time_dim=1):
anchors = generate_anchors(
stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS,
time_dim=time_dim)
num_anchors = anchors.shape[0]
dim_out = dim_in
# RPN hidden representation
if nd:
model.ConvNd(
blob_in, 'conv_rpn', dim_in, dim_out,
[cfg.VIDEO.TIME_KERNEL_DIM.HEAD_RPN, 3, 3],
pads=2 * [cfg.VIDEO.TIME_KERNEL_DIM.HEAD_RPN // 2, 1, 1], strides=[1, 1, 1],
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
else:
model.Conv(
blob_in, 'conv_rpn', dim_in, dim_out, 3, pad=1, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
model.Relu('conv_rpn', 'conv_rpn')
# Proposal classification scores
conv_rpn = 'conv_rpn'
# if nd:
# # Convert to 2D.
# model.MoveTimeToChannelDim('conv_rpn', 'conv_rpn_timepooled')
# # model.TimePool('conv_rpn', 'conv_rpn_timepooled', pool_type='avg')
# conv_rpn = 'conv_rpn_timepooled'
# final_dim_in = dim_out * time_dim
# # final_dim_in = dim_out
if nd:
model.ConvNd(
conv_rpn, 'rpn_cls_logits_1', dim_out, num_anchors,
[1, 1, 1], pads=2 * [0, 0, 0], strides=[1, 1, 1],
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
model.TimePool('rpn_cls_logits_1', 'rpn_cls_logits', pool_type='avg')
else:
model.Conv(
conv_rpn, 'rpn_cls_logits', dim_out, num_anchors, 1,
pad=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
# Proposal bbox regression deltas
if nd:
model.ConvNd(
conv_rpn, 'rpn_bbox_pred_1', dim_out,
4 * num_anchors, [1, 1, 1],
pads=2 * [0, 0, 0], strides=[1, 1, 1],
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
# Convert into the format RPN losses expect
model.ExpandDims('rpn_bbox_pred_1', 'rpn_bbox_pred_2', dims=[2])
model.Reshape(['rpn_bbox_pred_2'], ['rpn_bbox_pred_3', model.net.NextName()],
shape=(0, -1, 4, 0, 0, 0))
model.Transpose('rpn_bbox_pred_3', 'rpn_bbox_pred_4',
axes=(0, 1, 3, 2, 4, 5))
batch_size = model.GetShapeDimIdx(conv_rpn, 0)
ht = model.GetShapeDimIdx(conv_rpn, 3)
wd = model.GetShapeDimIdx(conv_rpn, 4)
final_shape = model.GetNewShape(batch_size, -1, ht, wd)
model.Reshape(['rpn_bbox_pred_4', final_shape],
['rpn_bbox_pred', model.net.NextName()])
else:
model.Conv(
conv_rpn, 'rpn_bbox_pred', dim_out,
4 * num_anchors, 1, pad=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
# Proposal visibility classification scores
# TODO: need to use in future
# model.Conv(
# conv_rpn, 'rpn_vis_logits', final_dim_in,
# num_anchors * time_dim, 1, pad=0, stride=1,
# weight_init=('GaussianFill', {'std': 0.01}),
# bias_init=('ConstantFill', {'value': 0.}))
if cfg.MODEL.FASTER_RCNN or (cfg.MODEL.RPN_ONLY and not model.train):
# Add op that generates RPN proposals
# The proposals are needed when generating proposals from an RPN-only
# model (but *not* when training it) or when training or testing a
# Faster R-CNN model
model.net.Sigmoid('rpn_cls_logits', 'rpn_cls_probs')
model.GenerateProposals(
['rpn_cls_probs', 'rpn_bbox_pred', 'im_info'],
['rpn_rois', 'rpn_roi_probs'],
anchors=anchors,
spatial_scale=spatial_scale)
if cfg.VIDEO.DEBUG_USE_RPN_GT:
model.net.SpatialNarrowAs(
['rpn_labels_int32_wide', 'rpn_cls_logits'], 'rpn_labels_int32')
for key in ('targets', 'inside_weights', 'outside_weights'):
model.net.SpatialNarrowAs(
['rpn_bbox_' + key + '_wide', 'rpn_bbox_pred'],
'rpn_bbox_' + key)
model.GenerateProposals(
['rpn_labels_int32', 'rpn_bbox_targets', 'im_info'],
['rpn_rois', 'rpn_roi_probs'],
anchors=anchors,
spatial_scale=spatial_scale)
if cfg.MODEL.FASTER_RCNN:
if model.train:
# Add op that generates training labels for in-network RPN proposals
model.GenerateProposalLabels(['rpn_rois', 'roidb', 'im_info'])
else:
# Alias rois to rpn_rois if not training
model.net.Alias('rpn_rois', 'rois')
def add_single_scale_rpn_outputs(model, blob_in, dim_in, spatial_scale):
"""Add RPN outputs to a single scale model (i.e., no FPN)."""
anchors = generate_anchors(
stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS
)
num_anchors = anchors.shape[0]
dim_out = dim_in
# RPN hidden representation
model.Conv(
blob_in,
'conv_rpn',
dim_in,
dim_out,
kernel=3,
pad=1,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
model.Relu('conv_rpn', 'conv_rpn')
# Proposal classification scores
model.Conv(
'conv_rpn',
'rpn_cls_logits',
dim_in,
num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
# Proposal bbox regression deltas
model.Conv(
'conv_rpn',
'rpn_bbox_pred',
dim_in,
4 * num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
if not model.train or cfg.MODEL.FASTER_RCNN:
# Proposals are needed during:
# 1) inference (== not model.train) for RPN only and Faster R-CNN
# OR
# 2) training for Faster R-CNN
# Otherwise (== training for RPN only), proposals are not needed
model.net.Sigmoid('rpn_cls_logits', 'rpn_cls_probs')
model.GenerateProposals(
['rpn_cls_probs', 'rpn_bbox_pred', 'im_info'],
['rpn_rois', 'rpn_roi_probs'],
anchors=anchors,
spatial_scale=spatial_scale
)
if cfg.MODEL.FASTER_RCNN:
if model.train:
# Add op that generates training labels for in-network RPN proposals
model.GenerateProposalLabels(['rpn_rois', 'roidb', 'im_info'])
else:
# Alias rois to rpn_rois for inference
model.net.Alias('rpn_rois', 'rois')
def add_fpn_rpn_outputs(model, blobs_in, dim_in, spatial_scales, time_dim=1):
# time_dim is only for consistency with 3D function, not used here.
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
dim_out = dim_in
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
assert len(blobs_in) == k_max - k_min + 1
for lvl in range(k_min, k_max + 1):
bl_in = blobs_in[k_max - lvl] # blobs_in is in reversed order
sc = spatial_scales[k_max - lvl] # in reversed order
slvl = str(lvl)
if lvl == k_min:
# Create conv ops with randomly initialized weights and
# zeroed biases for the first FPN level; these will be shared by
# all other FPN levels
# RPN hidden representation
conv_rpn_fpn = model.Conv(bl_in,
'conv_rpn_fpn' + slvl,
dim_in,
dim_out,
3,
pad=1,
stride=1,
weight_init=('GaussianFill', {
'std': 0.01
}),
bias_init=('ConstantFill', {
'value': 0.
}))
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.Conv(conv_rpn_fpn,
'rpn_cls_logits_fpn' + slvl,
dim_in,
num_anchors,
1,
pad=0,
stride=1,
weight_init=('GaussianFill', {
'std': 0.01
}),
bias_init=('ConstantFill', {
'value': 0.
}))
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.Conv(conv_rpn_fpn,
'rpn_bbox_pred_fpn' + slvl,
dim_in,
4 * num_anchors,
1,
pad=0,
stride=1,
weight_init=('GaussianFill', {
'std': 0.01
}),
bias_init=('ConstantFill', {
'value': 0.
}))
else:
# Share weights and biases
sk_min = str(k_min)
# RPN hidden representation
conv_rpn_fpn = model.ConvShared(
bl_in,
'conv_rpn_fpn' + slvl,
dim_in,
dim_out,
3,
pad=1,
stride=1,
weight='conv_rpn_fpn' + sk_min + '_w',
bias='conv_rpn_fpn' + sk_min + '_b')
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.ConvShared(
conv_rpn_fpn,
'rpn_cls_logits_fpn' + slvl,
dim_in,
num_anchors,
1,
pad=0,
stride=1,
weight='rpn_cls_logits_fpn' + sk_min + '_w',
bias='rpn_cls_logits_fpn' + sk_min + '_b')
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.ConvShared(
conv_rpn_fpn,
'rpn_bbox_pred_fpn' + slvl,
dim_in,
4 * num_anchors,
1,
pad=0,
stride=1,
weight='rpn_bbox_pred_fpn' + sk_min + '_w',
bias='rpn_bbox_pred_fpn' + sk_min + '_b')
if not model.train or cfg.MODEL.FASTER_RCNN:
# Add op that generates RPN proposals
# The proposals are needed when generating proposals from an
# RPN-only model at inference time, but *not* when training it
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS,
time_dim=1)
rpn_cls_probs_fpn = model.net.Sigmoid(rpn_cls_logits_fpn,
'rpn_cls_probs_fpn' + slvl)
model.GenerateProposals(
[rpn_cls_probs_fpn, rpn_bbox_pred_fpn, 'im_info'],
['rpn_rois_fpn' + slvl, 'rpn_roi_probs_fpn' + slvl],
anchors=lvl_anchors,
spatial_scale=sc)
def add_fpn_rpn_outputs(model, blobs_in, dim_in, spatial_scales):
"""Add RPN on FPN specific outputs."""
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
dim_out = dim_in
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
assert len(blobs_in) == k_max - k_min + 1
for lvl in range(k_min, k_max + 1):
bl_in = blobs_in[k_max - lvl] # blobs_in is in reversed order
sc = spatial_scales[k_max - lvl] # in reversed order
slvl = str(lvl)
if lvl == k_min:
# Create conv ops with randomly initialized weights and
# zeroed biases for the first FPN level; these will be shared by
# all other FPN levels
# RPN hidden representation
conv_rpn_fpn = model.Conv(
bl_in,
'conv_rpn_fpn' + slvl,
dim_in,
dim_out,
kernel=3,
pad=1,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.Conv(
conv_rpn_fpn,
'rpn_cls_logits_fpn' + slvl,
dim_in,
num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.Conv(
conv_rpn_fpn,
'rpn_bbox_pred_fpn' + slvl,
dim_in,
4 * num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
else:
# Share weights and biases
sk_min = str(k_min)
# RPN hidden representation
conv_rpn_fpn = model.ConvShared(
bl_in,
'conv_rpn_fpn' + slvl,
dim_in,
dim_out,
kernel=3,
pad=1,
stride=1,
weight='conv_rpn_fpn' + sk_min + '_w',
bias='conv_rpn_fpn' + sk_min + '_b'
)
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.ConvShared(
conv_rpn_fpn,
'rpn_cls_logits_fpn' + slvl,
dim_in,
num_anchors,
kernel=1,
pad=0,
stride=1,
weight='rpn_cls_logits_fpn' + sk_min + '_w',
bias='rpn_cls_logits_fpn' + sk_min + '_b'
)
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.ConvShared(
conv_rpn_fpn,
'rpn_bbox_pred_fpn' + slvl,
dim_in,
4 * num_anchors,
kernel=1,
pad=0,
stride=1,
weight='rpn_bbox_pred_fpn' + sk_min + '_w',
bias='rpn_bbox_pred_fpn' + sk_min + '_b'
)
if not model.train or cfg.MODEL.FASTER_RCNN:
# Proposals are needed during:
# 1) inference (== not model.train) for RPN only and Faster R-CNN
# OR
# 2) training for Faster R-CNN
# Otherwise (== training for RPN only), proposals are not needed
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS
)
rpn_cls_probs_fpn = model.net.Sigmoid(
rpn_cls_logits_fpn, 'rpn_cls_probs_fpn' + slvl
)
model.GenerateProposals(
[UnscopeGPUName(rpn_cls_probs_fpn._name), UnscopeGPUName(rpn_bbox_pred_fpn._name), 'im_info'],
['rpn_rois_fpn' + slvl, 'rpn_roi_probs_fpn' + slvl],
anchors=lvl_anchors,
spatial_scale=sc
)
def add_fpn_rpn_outputs(model, blobs_in, dim_in, spatial_scales):
"""Add RPN on FPN specific outputs."""
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
dim_out = dim_in
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
assert len(blobs_in) == k_max - k_min + 1
for lvl in range(k_min, k_max + 1):
bl_in = blobs_in[k_max - lvl] # blobs_in is in reversed order
sc = spatial_scales[k_max - lvl] # in reversed order
slvl = str(lvl)
if lvl == k_min:
# Create conv ops with randomly initialized weights and
# zeroed biases for the first FPN level; these will be shared by
# all other FPN levels
# RPN hidden representation
conv_rpn_fpn = model.Conv(
bl_in,
'conv_rpn_fpn' + slvl,
dim_in,
dim_out,
kernel=3,
pad=1,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.Conv(
conv_rpn_fpn,
'rpn_cls_logits_fpn' + slvl,
dim_in,
num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.Conv(
conv_rpn_fpn,
'rpn_bbox_pred_fpn' + slvl,
dim_in,
4 * num_anchors,
kernel=1,
pad=0,
stride=1,
weight_init=gauss_fill(0.01),
bias_init=const_fill(0.0)
)
else:
# Share weights and biases
sk_min = str(k_min)
# RPN hidden representation
conv_rpn_fpn = model.ConvShared(
bl_in,
'conv_rpn_fpn' + slvl,
dim_in,
dim_out,
kernel=3,
pad=1,
stride=1,
weight='conv_rpn_fpn' + sk_min + '_w',
bias='conv_rpn_fpn' + sk_min + '_b'
)
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.ConvShared(
conv_rpn_fpn,
'rpn_cls_logits_fpn' + slvl,
dim_in,
num_anchors,
kernel=1,
pad=0,
stride=1,
weight='rpn_cls_logits_fpn' + sk_min + '_w',
bias='rpn_cls_logits_fpn' + sk_min + '_b'
)
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.ConvShared(
conv_rpn_fpn,
'rpn_bbox_pred_fpn' + slvl,
dim_in,
4 * num_anchors,
kernel=1,
pad=0,
stride=1,
weight='rpn_bbox_pred_fpn' + sk_min + '_w',
bias='rpn_bbox_pred_fpn' + sk_min + '_b'
)
if not model.train or cfg.MODEL.FASTER_RCNN:
# Proposals are needed during:
# 1) inference (== not model.train) for RPN only and Faster R-CNN
# OR
# 2) training for Faster R-CNN
# Otherwise (== training for RPN only), proposals are not needed
lvl_anchors = generate_anchors(
stride=2.**lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS
)
rpn_cls_probs_fpn = model.net.Sigmoid(
rpn_cls_logits_fpn, 'rpn_cls_probs_fpn' + slvl
)
model.GenerateProposals(
[rpn_cls_probs_fpn, rpn_bbox_pred_fpn, 'im_info'],
['rpn_rois_fpn' + slvl, 'rpn_roi_probs_fpn' + slvl],
anchors=lvl_anchors,
spatial_scale=sc
)
def get_anchors(spatial_scale):
anchors = generate_anchors.generate_anchors(
stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS).astype(np.float32)
return anchors
def get_anchors(spatial_scale):
anchors = generate_anchors.generate_anchors(
stride=1. / spatial_scale,
sizes=cfg.RPN.SIZES,
aspect_ratios=cfg.RPN.ASPECT_RATIOS).astype(np.float32)
return anchors
def add_fpn_rpn_outputs(model, blobs_in, dim_in, spatial_scales, time_dim):
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
dim_out = dim_in
raise NotImplementedError('Redo bbox_targets like in model_builder.py')
if cfg.VIDEO.DEBUG_USE_RPN_GT:
raise NotImplementedError('Need to implement this similar to non-FPN')
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
assert len(blobs_in) == k_max - k_min + 1
for lvl in range(k_min, k_max + 1):
bl_in = blobs_in[k_max - lvl] # blobs_in is in reversed order
sc = spatial_scales[k_max - lvl] # in reversed order
slvl = str(lvl)
if lvl == k_min:
# Create conv ops with randomly initialized weights and
# zeroed biases for the first FPN level; these will be shared by
# all other FPN levels
# RPN hidden representation
conv_rpn_fpn = model.ConvNd(
bl_in, 'conv_rpn_fpn' + slvl,
dim_in, dim_out, [cfg.VIDEO.TIME_KERNEL_DIM.HEAD_RPN, 3, 3],
pads=2 * [cfg.VIDEO.TIME_KERNEL_DIM.HEAD_RPN // 2, 1, 1],
strides=[1, 1, 1],
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Convert to 2D. Earlier was averaging in time, now moving to channel
conv_rpn_fpn_timepool = model.MoveTimeToChannelDim(
conv_rpn_fpn, 'conv_rpn_timepooled_fpn' + slvl)
# Proposal classification scores
rpn_cls_logits_fpn = model.Conv(
conv_rpn_fpn_timepool, 'rpn_cls_logits_fpn' + slvl,
dim_out * time_dim, num_anchors, 1, pads=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.Conv(
conv_rpn_fpn_timepool, 'rpn_bbox_pred_fpn' + slvl,
dim_out * time_dim, 4 * time_dim * num_anchors, 1, pad=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
# Proposal visibility classification scores
# TODO(rgirdhar): Need to use this in future
# rpn_vis_cls_logits_fpn =
model.Conv(
conv_rpn_fpn_timepool, 'rpn_vis_cls_logits_fpn' + slvl,
dim_out * time_dim, num_anchors * time_dim, 1, pads=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
else:
# Share weights and biases
sk_min = str(k_min)
# RPN hidden representation
conv_rpn_fpn = model.ConvShared(
bl_in, 'conv_rpn_fpn' + slvl,
dim_in, dim_out, [3, 3, 3],
pads=2 * [1, 1, 1], strides=[1, 1, 1],
nd=True,
weight='conv_rpn_fpn' + sk_min + '_w',
bias='conv_rpn_fpn' + sk_min + '_b')
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Convert to 2D. Earlier was averaging in time, now moving to channel
conv_rpn_fpn_timepool = model.MoveTimeToChannelDim(
conv_rpn_fpn, 'conv_rpn_timepooled_fpn' + slvl)
# Proposal classification scores
rpn_cls_logits_fpn = model.ConvShared(
conv_rpn_fpn_timepool, 'rpn_cls_logits_fpn' + slvl,
dim_out * time_dim, num_anchors, 1, pad=0, stride=1,
weight='rpn_cls_logits_fpn' + sk_min + '_w',
bias='rpn_cls_logits_fpn' + sk_min + '_b')
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.ConvShared(
conv_rpn_fpn_timepool, 'rpn_bbox_pred_fpn' + slvl,
dim_out * time_dim, 4 * time_dim * num_anchors, 1, pad=0, stride=1,
weight='rpn_bbox_pred_fpn' + sk_min + '_w',
bias='rpn_bbox_pred_fpn' + sk_min + '_b')
# Proposal visibility classification scores
# TODO(rgirdhar): Need to use this in future
# rpn_vis_cls_logits_fpn =
model.ConvShared(
conv_rpn_fpn_timepool, 'rpn_vis_cls_logits_fpn' + slvl,
dim_out * time_dim, num_anchors * time_dim, 1, pad=0, stride=1,
weight='rpn_vis_cls_logits_fpn' + sk_min + '_w',
bias='rpn_vis_cls_logits_fpn' + sk_min + '_b')
if not model.train or cfg.MODEL.FASTER_RCNN:
# Add op that generates RPN proposals
# The proposals are needed when generating proposals from an
# RPN-only model at inference time, but *not* when training it
lvl_anchors = generate_anchors(
stride=2. ** lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2. ** (lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS,
time_dim=time_dim)
rpn_cls_probs_fpn = model.net.Sigmoid(
rpn_cls_logits_fpn, 'rpn_cls_probs_fpn' + slvl)
# Need to use this in future
# rpn_vis_cls_probs_fpn = model.net.Sigmoid(
# rpn_cls_logits_fpn, 'rpn_vis_cls_probs_fpn' + slvl)
model.GenerateProposals(
[rpn_cls_probs_fpn, rpn_bbox_pred_fpn, 'im_info'],
['rpn_rois_fpn' + slvl, 'rpn_roi_probs_fpn' + slvl],
anchors=lvl_anchors,
spatial_scale=sc)
def add_fpn_rpn_outputs(model, blobs_in, dim_in, spatial_scales, time_dim=1):
# time_dim is only for consistency with 3D function, not used here.
num_anchors = len(cfg.FPN.RPN_ASPECT_RATIOS)
dim_out = dim_in
k_max = cfg.FPN.RPN_MAX_LEVEL # coarsest level of pyramid
k_min = cfg.FPN.RPN_MIN_LEVEL # finest level of pyramid
assert len(blobs_in) == k_max - k_min + 1
for lvl in range(k_min, k_max + 1):
bl_in = blobs_in[k_max - lvl] # blobs_in is in reversed order
sc = spatial_scales[k_max - lvl] # in reversed order
slvl = str(lvl)
if lvl == k_min:
# Create conv ops with randomly initialized weights and
# zeroed biases for the first FPN level; these will be shared by
# all other FPN levels
# RPN hidden representation
conv_rpn_fpn = model.Conv(
bl_in, 'conv_rpn_fpn' + slvl,
dim_in, dim_out, 3, pad=1, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.Conv(
conv_rpn_fpn, 'rpn_cls_logits_fpn' + slvl,
dim_in, num_anchors, 1, pad=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.Conv(
conv_rpn_fpn, 'rpn_bbox_pred_fpn' + slvl,
dim_in, 4 * num_anchors, 1, pad=0, stride=1,
weight_init=('GaussianFill', {'std': 0.01}),
bias_init=('ConstantFill', {'value': 0.}))
else:
# Share weights and biases
sk_min = str(k_min)
# RPN hidden representation
conv_rpn_fpn = model.ConvShared(
bl_in, 'conv_rpn_fpn' + slvl,
dim_in, dim_out, 3, pad=1, stride=1,
weight='conv_rpn_fpn' + sk_min + '_w',
bias='conv_rpn_fpn' + sk_min + '_b')
model.Relu(conv_rpn_fpn, conv_rpn_fpn)
# Proposal classification scores
rpn_cls_logits_fpn = model.ConvShared(
conv_rpn_fpn, 'rpn_cls_logits_fpn' + slvl,
dim_in, num_anchors, 1, pad=0, stride=1,
weight='rpn_cls_logits_fpn' + sk_min + '_w',
bias='rpn_cls_logits_fpn' + sk_min + '_b')
# Proposal bbox regression deltas
rpn_bbox_pred_fpn = model.ConvShared(
conv_rpn_fpn, 'rpn_bbox_pred_fpn' + slvl,
dim_in, 4 * num_anchors, 1, pad=0, stride=1,
weight='rpn_bbox_pred_fpn' + sk_min + '_w',
bias='rpn_bbox_pred_fpn' + sk_min + '_b')
if not model.train or cfg.MODEL.FASTER_RCNN:
# Add op that generates RPN proposals
# The proposals are needed when generating proposals from an
# RPN-only model at inference time, but *not* when training it
lvl_anchors = generate_anchors(
stride=2. ** lvl,
sizes=(cfg.FPN.RPN_ANCHOR_START_SIZE * 2. ** (lvl - k_min), ),
aspect_ratios=cfg.FPN.RPN_ASPECT_RATIOS,
time_dim=1)
rpn_cls_probs_fpn = model.net.Sigmoid(
rpn_cls_logits_fpn, 'rpn_cls_probs_fpn' + slvl)
model.GenerateProposals(
[rpn_cls_probs_fpn, rpn_bbox_pred_fpn, 'im_info'],
['rpn_rois_fpn' + slvl, 'rpn_roi_probs_fpn' + slvl],
anchors=lvl_anchors,
spatial_scale=sc)
