def add_rpn_blobs(blobs, im_scales, roidb):
"""Add blobs needed training RPN-only and end-to-end Faster R-CNN models."""
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
foas = []
for lvl in range(k_min, k_max + 1):
field_stride = 2.**lvl
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE *
2.**(lvl - k_min), )
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
foa = data_utils.get_field_of_anchors(field_stride, anchor_sizes,
anchor_aspect_ratios)
foas.append(foa)
all_anchors = np.concatenate([f.field_of_anchors for f in foas])
else:
foa = data_utils.get_field_of_anchors(cfg.RPN.STRIDE, cfg.RPN.SIZES,
cfg.RPN.ASPECT_RATIOS)
all_anchors = foa.field_of_anchors
for im_i, entry in enumerate(roidb):
scale = im_scales[im_i]
im_height = np.round(entry['height'] * scale)
im_width = np.round(entry['width'] * scale)
gt_inds = np.where((entry['gt_classes'] > 0)
& (entry['is_crowd'] == 0))[0]
gt_rois = entry['boxes'][gt_inds, :] * scale
im_info = np.array([[im_height, im_width, scale]], dtype=np.float32)
blobs['im_info'].append(im_info)
# Add RPN targets
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
rpn_blobs = _get_rpn_blobs(im_height, im_width, foas, all_anchors,
gt_rois)
for i, lvl in enumerate(range(k_min, k_max + 1)):
for k, v in rpn_blobs[i].items():
blobs[k + '_fpn' + str(lvl)].append(v)
else:
# Classical RPN, applied to a single feature level
rpn_blobs = _get_rpn_blobs(im_height, im_width, [foa], all_anchors,
gt_rois)
for k, v in rpn_blobs.items():
blobs[k].append(v)
for k, v in blobs.items():
if isinstance(v, list) and len(v) > 0:
blobs[k] = np.concatenate(v)
valid_keys = [
'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes',
'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints'
]
minimal_roidb = [{} for _ in range(len(roidb))]
for i, e in enumerate(roidb):
for k in valid_keys:
if k in e:
minimal_roidb[i][k] = e[k]
blobs['roidb'] = blob_utils.serialize(minimal_roidb)
# Always return valid=True, since RPN minibatches are valid by design
return True
def add_rpn_blobs(blobs, im_tr_matrix, roidb, im_shapes, im_scales):
"""Add blobs needed training RPN-only and end-to-end Faster R-CNN models."""
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
foas = []
for lvl in range(k_min, k_max + 1):
field_stride = 2.**lvl
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE *
2.**(lvl - k_min), )
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
foa = data_utils.get_field_of_anchors(field_stride, anchor_sizes,
anchor_aspect_ratios)
foas.append(foa)
all_anchors = np.concatenate([f.field_of_anchors for f in foas])
else:
foa = data_utils.get_field_of_anchors(cfg.RPN.STRIDE, cfg.RPN.SIZES,
cfg.RPN.ASPECT_RATIOS)
all_anchors = foa.field_of_anchors
for im_i, entry in enumerate(roidb):
transformation_matrix = im_tr_matrix[im_i]
scale = im_scales[im_i]
im_height = im_shapes[im_i][0]
im_width = im_shapes[im_i][1]
# zoom = zooms[im_i]
# im_height = np.round(entry['height'] * scale)
# im_width = np.round(entry['width'] * scale)
gt_inds = np.where((entry['gt_classes'] > 0)
& (entry['is_crowd'] == 0))[0]
# gt_rois = entry['boxes'][gt_inds, :]
gt_rois = []
for gt_roi in roidb[im_i]['boxes']:
w, h = gt_roi[2] - gt_roi[0], gt_roi[3] - gt_roi[1]
nw, nh = int(w * scale), int(h * scale)
center_x, center_y = gt_roi[0] + w / 2, gt_roi[1] + h / 2
new_center = np.dot(transformation_matrix,
[[center_x], [center_y], [1.0]]).astype('int')
new_center_x = int(new_center[0][0])
new_center_y = int(new_center[1][0])
nbx = int(new_center_x - nw / 2)
nby = int(new_center_y - nh / 2)
nbx2 = int(nbx + nw)
nby2 = int(nby + nh)
gt_rois.append([nbx, nby, nbx2, nby2])
im_info = np.array([[im_height, im_width, scale]], dtype=np.float32)
matrix = np.array(transformation_matrix, dtype=np.float32)
blobs['im_info'].append(im_info)
blobs['im_tr_matrix'].append(matrix)
gt_rois = np.asarray(gt_rois, dtype=np.float32)
# Add RPN targets
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
rpn_blobs = _get_rpn_blobs(im_height, im_width, foas, all_anchors,
gt_rois)
for i, lvl in enumerate(range(k_min, k_max + 1)):
for k, v in rpn_blobs[i].items():
blobs[k + '_fpn' + str(lvl)].append(v)
else:
# Classical RPN, applied to a single feature level
rpn_blobs = _get_rpn_blobs(im_height, im_width, [foa], all_anchors,
gt_rois)
for k, v in rpn_blobs.items():
blobs[k].append(v)
for k, v in blobs.items():
if isinstance(v, list) and len(v) > 0:
blobs[k] = np.concatenate(v)
valid_keys = [
'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes',
'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints'
]
minimal_roidb = [{} for _ in range(len(roidb))]
for i, e in enumerate(roidb):
for k in valid_keys:
if k in e:
minimal_roidb[i][k] = e[k]
blobs['roidb'] = blob_utils.serialize(minimal_roidb)
# Always return valid=True, since RPN minibatches are valid by design
return True
def add_rpn_blobs(blobs, im_scales, roidb):
"""Add blobs needed training RPN-only and end-to-end Faster R-CNN models."""
"""
添加训练faster rcnn需要的blobs
"""
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
foas = []
for lvl in range(k_min, k_max + 1):
# lvl = 2 => 4.0
field_stride = 2.**lvl
# 32.0
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE *
2.**(lvl - k_min), )
# [0.5, 1.0, 2.0]
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
# 对于一个特征图，获得特征图上每一个cell所对应的anchor，
# 该anchor对应于网络输入的大小
foa = data_utils.get_field_of_anchors(field_stride, anchor_sizes,
anchor_aspect_ratios)
foas.append(foa)
all_anchors = np.concatenate([f.field_of_anchors for f in foas])
else:
foa = data_utils.get_field_of_anchors(cfg.RPN.STRIDE, cfg.RPN.SIZES,
cfg.RPN.ASPECT_RATIOS)
all_anchors = foa.field_of_anchors
for im_i, entry in enumerate(roidb):
scale = im_scales[im_i]
# * scale获得相对于网络输入的信息
im_height = np.round(entry['height'] * scale)
im_width = np.round(entry['width'] * scale)
gt_inds = np.where((entry['gt_classes'] > 0)
& (entry['is_crowd'] == 0))[0]
# gt box
gt_rois = entry['boxes'][gt_inds, :] * scale
# TODO(rbg): gt_boxes is poorly named;
# should be something like 'gt_rois_info'
gt_boxes = blob_utils.zeros((len(gt_inds), 6))
# 属于哪个图片
gt_boxes[:, 0] = im_i # batch inds
# box
gt_boxes[:, 1:5] = gt_rois
# 类别信息
gt_boxes[:, 5] = entry['gt_classes'][gt_inds]
# 写入blob
im_info = np.array([[im_height, im_width, scale]], dtype=np.float32)
blobs['im_info'].append(im_info)
# Add RPN targets
# 添加RPN的目标值
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
rpn_blobs = _get_rpn_blobs(im_height, im_width, foas, all_anchors,
gt_rois)
for i, lvl in enumerate(range(k_min, k_max + 1)):
for k, v in rpn_blobs[i].items():
blobs[k + '_fpn' + str(lvl)].append(v)
else:
# Classical RPN, applied to a single feature level
rpn_blobs = _get_rpn_blobs(im_height, im_width, [foa], all_anchors,
gt_rois)
for k, v in rpn_blobs.items():
blobs[k].append(v)
for k, v in blobs.items():
if isinstance(v, list) and len(v) > 0:
blobs[k] = np.concatenate(v)
valid_keys = [
'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes',
'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints'
]
minimal_roidb = [{} for _ in range(len(roidb))]
for i, e in enumerate(roidb):
for k in valid_keys:
if k in e:
minimal_roidb[i][k] = e[k]
blobs['roidb'] = blob_utils.serialize(minimal_roidb)
# Always return valid=True, since RPN minibatches are valid by design
return True
def add_rpn_blobs(blobs, im_scales, roidb):
"""Add blobs needed training RPN-only and end-to-end Faster R-CNN models."""
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
foas = []
for lvl in range(k_min, k_max + 1):
field_stride = 2.**lvl
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), )
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
foa = data_utils.get_field_of_anchors(
field_stride, anchor_sizes, anchor_aspect_ratios
)
foas.append(foa)
all_anchors = np.concatenate([f.field_of_anchors for f in foas])
else:
foa = data_utils.get_field_of_anchors(
cfg.RPN.STRIDE, cfg.RPN.SIZES, cfg.RPN.ASPECT_RATIOS
)
all_anchors = foa.field_of_anchors
for im_i, entry in enumerate(roidb):
scale = im_scales[im_i]
im_height = np.round(entry['height'] * scale)
im_width = np.round(entry['width'] * scale)
gt_inds = np.where(
(entry['gt_classes'] > 0) & (entry['is_crowd'] == 0)
)[0]
gt_rois = entry['boxes'][gt_inds, :] * scale
im_info = np.array([[im_height, im_width, scale]], dtype=np.float32)
blobs['im_info'].append(im_info)
# Add RPN targets
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
rpn_blobs = _get_rpn_blobs(
im_height, im_width, foas, all_anchors, gt_rois
)
for i, lvl in enumerate(range(k_min, k_max + 1)):
for k, v in rpn_blobs[i].items():
blobs[k + '_fpn' + str(lvl)].append(v)
else:
# Classical RPN, applied to a single feature level
rpn_blobs = _get_rpn_blobs(
im_height, im_width, [foa], all_anchors, gt_rois
)
for k, v in rpn_blobs.items():
blobs[k].append(v)
for k, v in blobs.items():
if isinstance(v, list) and len(v) > 0:
blobs[k] = np.concatenate(v)
valid_keys = [
'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes',
'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints'
]
minimal_roidb = [{} for _ in range(len(roidb))]
for i, e in enumerate(roidb):
for k in valid_keys:
if k in e:
minimal_roidb[i][k] = e[k]
blobs['roidb'] = blob_utils.serialize(minimal_roidb)
# Always return valid=True, since RPN minibatches are valid by design
return True
def add_rpn_blobs(blobs, im_scales, roidb):
"""Add blobs needed training RPN-only and end-to-end Faster R-CNN models."""
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
k_max = cfg.FPN.RPN_MAX_LEVEL
k_min = cfg.FPN.RPN_MIN_LEVEL
foas = []
# fetch the spatial scales for every FPN level except fpn6
#fpn_spatial_scales = globals().get('fpn_level_info_' + cfg.MODEL.BACKBONE_NAME + '_conv5')().spatial_scales
fpn_spatial_scales = getattr(FPN, 'fpn_level_info_' + cfg.MODEL.BACKBONE_NAME + '_conv5')().spatial_scales
for lvl in range(k_min, k_max):
field_stride = 1. / fpn_spatial_scales[k_max-lvl-1]
#field_stride = 2.**(lvl - int(math.log(cfg.FPN.FINEST_LEVEL_SCALE,2))-2)
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(lvl - k_min), )
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
foa = data_utils.get_field_of_anchors(
field_stride, anchor_sizes, anchor_aspect_ratios
)
foas.append(foa)
# for p6, the scale should be the corest level divided by 2
if k_max == 6:
field_stride = 2 * (1. / fpn_spatial_scales[0])
anchor_sizes = (cfg.FPN.RPN_ANCHOR_START_SIZE * 2.**(k_max - k_min), )
anchor_aspect_ratios = cfg.FPN.RPN_ASPECT_RATIOS
foa = data_utils.get_field_of_anchors(
field_stride, anchor_sizes, anchor_aspect_ratios
)
foas.append(foa)
all_anchors = np.concatenate([f.field_of_anchors for f in foas])
else:
foa = data_utils.get_field_of_anchors(
cfg.RPN.STRIDE, cfg.RPN.SIZES, cfg.RPN.ASPECT_RATIOS
)
all_anchors = foa.field_of_anchors
for im_i, entry in enumerate(roidb):
scale = im_scales[im_i]
im_height = np.round(entry['height'] * scale)
im_width = np.round(entry['width'] * scale)
gt_inds = np.where(
(entry['gt_classes'] > 0) & (entry['is_crowd'] == 0)
)[0]
gt_rois = entry['boxes'][gt_inds, :] * scale
# TODO(rbg): gt_boxes is poorly named;
# should be something like 'gt_rois_info'
gt_boxes = blob_utils.zeros((len(gt_inds), 6))
gt_boxes[:, 0] = im_i # batch inds
gt_boxes[:, 1:5] = gt_rois
gt_boxes[:, 5] = entry['gt_classes'][gt_inds]
im_info = np.array([[im_height, im_width, scale]], dtype=np.float32)
blobs['im_info'].append(im_info)
# Add RPN targets
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_RPN:
# RPN applied to many feature levels, as in the FPN paper
rpn_blobs = _get_rpn_blobs(
im_height, im_width, foas, all_anchors, gt_rois
)
for i, lvl in enumerate(range(k_min, k_max + 1)):
for k, v in rpn_blobs[i].items():
blobs[k + '_fpn' + str(lvl)].append(v)
else:
# Classical RPN, applied to a single feature level
rpn_blobs = _get_rpn_blobs(
im_height, im_width, [foa], all_anchors, gt_rois
)
for k, v in rpn_blobs.items():
blobs[k].append(v)
for k, v in blobs.items():
if isinstance(v, list) and len(v) > 0:
blobs[k] = np.concatenate(v)
valid_keys = [
'has_visible_keypoints', 'boxes', 'segms', 'seg_areas', 'gt_classes',
'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map', 'gt_keypoints'
]
minimal_roidb = [{} for _ in range(len(roidb))]
for i, e in enumerate(roidb):
for k in valid_keys:
if k in e:
minimal_roidb[i][k] = e[k]
blobs['roidb'] = blob_utils.serialize(minimal_roidb)
# Always return valid=True, since RPN minibatches are valid by design
return True
