def combine_heatmaps_size_dep(hms_ts, ds_ts, us_ts, boxes, heur_f):
"""Combines heatmaps while taking object sizes into account."""
assert len(hms_ts) == len(ds_ts) and len(ds_ts) == len(us_ts), \
'All sets of hms must be tagged with downscaling and upscaling flags'
# Classify objects into small+medium and large based on their box areas
areas = box_utils.boxes_area(boxes)
sm_objs = areas < cfg.TEST.KPS_AUG.AREA_TH
l_objs = areas >= cfg.TEST.KPS_AUG.AREA_TH
# Combine heatmaps computed under different transformations for each object
hms_c = np.zeros_like(hms_ts[0])
for i in range(hms_c.shape[0]):
hms_to_combine = []
for hms_t, ds_t, us_t in zip(hms_ts, ds_ts, us_ts):
# Discard downscaling predictions for small and medium objects
if sm_objs[i] and ds_t:
continue
# Discard upscaling predictions for large objects
if l_objs[i] and us_t:
continue
hms_to_combine.append(hms_t[i])
hms_c[i] = heur_f(hms_to_combine)
return hms_c
def combine_heatmaps_size_dep(hms_ts, ds_ts, us_ts, boxes, heur_f):
"""Combines heatmaps while taking object sizes into account."""
assert len(hms_ts) == len(ds_ts) and len(ds_ts) == len(us_ts), \
'All sets of hms must be tagged with downscaling and upscaling flags'
# Classify objects into small+medium and large based on their box areas
areas = box_utils.boxes_area(boxes)
sm_objs = areas < cfg.TEST.KPS_AUG.AREA_TH
l_objs = areas >= cfg.TEST.KPS_AUG.AREA_TH
# Combine heatmaps computed under different transformations for each object
hms_c = np.zeros_like(hms_ts[0])
for i in range(hms_c.shape[0]):
hms_to_combine = []
for hms_t, ds_t, us_t in zip(hms_ts, ds_ts, us_ts):
# Discard downscaling predictions for small and medium objects
if sm_objs[i] and ds_t:
continue
# Discard upscaling predictions for large objects
if l_objs[i] and us_t:
continue
hms_to_combine.append(hms_t[i])
hms_c[i] = heur_f(hms_to_combine)
return hms_c
def map_rois_to_fpn_levels(rois, k_min, k_max):
# Compute level ids
s = np.sqrt(box_utils.boxes_area(rois))
s0 = cfg.FPN.ROI_CANONICAL_SCALE # default: 224
lvl0 = cfg.FPN.ROI_CANONICAL_LEVEL # default: 4
lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6)) # Eqn.(1) in FPN paper
lvls = np.clip(lvls, k_min, k_max)
# lvls is a list of length len(rois) with a ID from k_min to k_max, as to
# where it maps to. This might lead to some levels from k_min to k_max not
# getting any rois mapped to them.
return lvls
def map_rois_to_fpn_levels(rois, k_min, k_max):
# Compute level ids
s = np.sqrt(box_utils.boxes_area(rois))
s0 = cfg.FPN.ROI_CANONICAL_SCALE # default: 224
lvl0 = cfg.FPN.ROI_CANONICAL_LEVEL # default: 4
lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6)) # Eqn.(1) in FPN paper
lvls = np.clip(lvls, k_min, k_max)
# lvls is a list of length len(rois) with a ID from k_min to k_max, as to
# where it maps to. This might lead to some levels from k_min to k_max not
# getting any rois mapped to them.
return lvls
def map_rois_to_fpn_levels(rois, k_min, k_max):
"""Determine which FPN level each RoI in a set of RoIs should map to based
on the heuristic in the FPN paper.
"""
# Compute level ids
s = np.sqrt(box_utils.boxes_area(rois))
s0 = cfg.FPN.ROI_CANONICAL_SCALE # default: 224
lvl0 = cfg.FPN.ROI_CANONICAL_LEVEL # default: 4
# Eqn.(1) in FPN paper
target_lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6))
target_lvls = np.clip(target_lvls, k_min, k_max)
return target_lvls
def map_rois_to_fpn_levels(rois, k_min, k_max):
"""Determine which FPN level each RoI in a set of RoIs should map to based
on the heuristic in the FPN paper.
"""
# Compute level ids
s = np.sqrt(box_utils.boxes_area(rois))
s0 = cfg.FPN.ROI_CANONICAL_SCALE # default: 224
lvl0 = cfg.FPN.ROI_CANONICAL_LEVEL # default: 4
# Eqn.(1) in FPN paper
target_lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6))
target_lvls = np.clip(target_lvls, k_min, k_max)
return target_lvls
def map_rois_to_fpn_levels(rois, k_min, k_max, roi_canonical_scale=224, roi_canonical_level=4):
"""Determine which FPN level each RoI in a set of RoIs should map to based
on the heuristic in the FPN paper.
"""
# Compute level ids
s = np.sqrt(box_utils.boxes_area(rois))
s0 = roi_canonical_scale # default: 224
lvl0 = roi_canonical_level # default: 4
# Eqn.(1) in FPN paper
target_lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6))
target_lvls = np.clip(target_lvls, k_min, k_max)
return target_lvls
def map_rois_to_fpn_levels(rois, k_min, k_max):
"""Determine which FPN level each RoI in a set of RoIs should map to based
on the heuristic in the FPN paper.
"""
# Compute level ids
areas, neg_idx = box_utils.boxes_area(rois)
areas[neg_idx] = 0 # np.sqrt will remove the entries with negative value
s = np.sqrt(areas)
s0 = cfg.FPN.ROI_CANONICAL_SCALE # default: 224
lvl0 = cfg.FPN.ROI_CANONICAL_LEVEL # default: 4
# Eqn.(1) in FPN paper
target_lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6))
target_lvls = np.clip(target_lvls, k_min, k_max)
# Mark to discard negative area roi. See utils.fpn.add_multilevel_roi_blobs
# target_lvls[neg_idx] = -1
return target_lvls
def map_rois_to_fpn_levels(rois, k_min, k_max):
"""Determine which FPN level each RoI in a set of RoIs should map to based
on the heuristic in the FPN paper.
"""
if cfg.FPN.FPN_DISTRIBUTE:
length = rois.shape[0]
lvls = []
import random
for i in range(length):
lvl = k_min
if cfg.FPN.RANDOM_DISTRIBUTE:
lvl = random.randint(k_min, k_max)
if cfg.FPN.TWO_DISTRIBUTE:
lvl = k_min
if cfg.FPN.THREE_DISTRIBUTE:
lvl = k_min + 1
if cfg.FPN.FOUR_DISTRIBUTE:
lvl = k_min + 2
if cfg.FPN.FIVE_DISTRIBUTE:
lvl = k_min + 3
lvls.append(lvl)
lvls = np.array(lvls, dtype=np.float32)
return lvls
else:
# Compute level ids
areas, neg_idx = box_utils.boxes_area(rois)
areas[
neg_idx] = 0 # np.sqrt will remove the entries with negative value
s = np.sqrt(areas)
s0 = cfg.FPN.ROI_CANONICAL_SCALE # default: 224
lvl0 = cfg.FPN.ROI_CANONICAL_LEVEL # default: 4
# Eqn.(1) in FPN paper
target_lvls = np.floor(lvl0 + np.log2(s / s0 + 1e-6))
target_lvls = np.clip(target_lvls, k_min, k_max)
# Mark to discard negative area roi. See utils.fpn.add_multilevel_roi_blobs
# target_lvls[neg_idx] = -1
return target_lvls
def _sample_rois_gan(roidb, im_scale, batch_idx, flags):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
assert isinstance(flags, ModeFlags) is True
# gt_boxes and sample such that they fulfill threshold criterion
gt_inds = np.where(roidb['gt_classes'] > 0)[0]
gt_boxes = roidb['boxes'][gt_inds, :]
if cfg.DEBUG:
logger.info("sample from {} gt boxes".format(len(gt_boxes)))
areas_gt, _ = box_utils.boxes_area(gt_boxes)
areas_gt = np.sqrt(areas_gt)
print("gt-boxes: area_thres: {} vs areas: {}".format(
cfg.GAN.AREA_THRESHOLD, areas_gt))
gt_keep_inds = []
if cfg.GAN.AREA_THRESHOLD > 0:
area_thres = 1.0 * cfg.GAN.AREA_THRESHOLD * cfg.GAN.AREA_THRESHOLD # no scaling, as rois are scaled latter
if flags.fake_mode:
# for fake samples: keep only samples with area < area-threshold
gt_keep_inds = gt_inds[box_utils.filter_large_boxes_area(
gt_boxes, max_area=area_thres)]
elif flags.real_mode:
# for real samples: keep only samples with area >= area-threshold
gt_keep_inds = gt_inds[box_utils.filter_small_boxes_area(
gt_boxes, min_area=area_thres)]
elif flags.real_fake_mode:
gt_keep_inds = gt_inds
if flags.train_generator:
rois_per_image = int(cfg.GAN.TRAIN.BATCH_SIZE_PER_IM_G)
fg_rois_per_image = int(
np.round(cfg.GAN.TRAIN.FG_FRACTION_G * rois_per_image))
elif flags.train_discriminator: # discriminator
rois_per_image = int(cfg.GAN.TRAIN.BATCH_SIZE_PER_IM_D)
fg_rois_per_image = int(
np.round(cfg.GAN.TRAIN.FG_FRACTION_D * rois_per_image))
elif flags.train_pre:
rois_per_image = int(cfg.GAN.TRAIN.BATCH_SIZE_PER_IM_PRE)
fg_rois_per_image = int(
np.round(cfg.GAN.TRAIN.FG_FRACTION_PRE * rois_per_image))
max_overlaps = roidb['max_overlaps']
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= cfg.TRAIN.FG_THRESH)[0]
# with area-threshold, only select indices of boxes, whose corresponding ground-truth-box fulfills criterion
# i.e. whose corresponding index to gt-box is in gt_keep_inds
if cfg.GAN.AREA_THRESHOLD > 0:
#if cfg.DEBUG:
# fg_boxes = gt_boxes[gt_inds[roidb['box_to_gt_ind_map'][fg_inds]], :]
# areas_fg, _ = box_utils.boxes_area(fg_boxes)
# areas_fg = np.sqrt(areas_fg)
# print("fg-before: area_thres: {} vs areas: {}".format(cfg.GAN.AREA_THRESHOLD, areas_fg))
fg_inds = np.asarray([
x for x in fg_inds
if gt_inds[roidb['box_to_gt_ind_map'][x]] in gt_keep_inds
]).astype(int)
if cfg.DEBUG:
fg_boxes = gt_boxes[
gt_inds[roidb['box_to_gt_ind_map'][fg_inds]], :]
areas_fg, _ = box_utils.boxes_area(fg_boxes)
areas_fg = np.sqrt(areas_fg)
print("fg-after: area_thres: {} vs areas: {}".format(
cfg.GAN.AREA_THRESHOLD, areas_fg))
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, int(fg_inds.size))
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds = np.where((max_overlaps < cfg.TRAIN.BG_THRESH_HI)
& (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_inds.size)
# Sample foreground regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Label is the class each RoI has max overlap with
sampled_labels = roidb['max_classes'][keep_inds]
sampled_labels[fg_rois_per_this_image:] = 0 # Label bg RoIs with class 0
sampled_boxes = roidb['boxes'][keep_inds]
if 'bbox_targets' not in roidb:
gt_assignments = gt_inds[roidb['box_to_gt_ind_map'][keep_inds]]
bbox_targets = _compute_targets(sampled_boxes,
gt_boxes[gt_assignments, :],
sampled_labels)
bbox_targets, bbox_inside_weights = _expand_bbox_targets(bbox_targets)
else:
bbox_targets, bbox_inside_weights = _expand_bbox_targets(
roidb['bbox_targets'][keep_inds, :])
bbox_outside_weights = np.array(bbox_inside_weights > 0,
dtype=bbox_inside_weights.dtype)
# Scale rois and format as (batch_idx, x1, y1, x2, y2)
sampled_rois = sampled_boxes * im_scale
repeated_batch_idx = batch_idx * blob_utils.ones(
(sampled_rois.shape[0], 1))
sampled_rois = np.hstack((repeated_batch_idx, sampled_rois))
if not cfg.RPN.RPN_ON: # FAST-RCNN training
# need to unsqueeze things for functionality in loader / minibatch.py ...
sampled_rois = np.expand_dims(sampled_rois, axis=0)
sampled_labels = np.expand_dims(sampled_labels, axis=0)
bbox_targets = np.expand_dims(bbox_targets, axis=0)
bbox_outside_weights = np.expand_dims(bbox_outside_weights, axis=0)
bbox_inside_weights = np.expand_dims(bbox_inside_weights, axis=0)
# Base Fast R-CNN blobs
blob_dict = dict(labels_int32=sampled_labels.astype(np.int32, copy=False),
rois=sampled_rois,
bbox_targets=bbox_targets,
bbox_inside_weights=bbox_inside_weights,
bbox_outside_weights=bbox_outside_weights)
return blob_dict