def generate_anchors_opr(self,
fm_3x3,
fm_stride,
anchor_scales=(8, 16, 32, 64, 128),
anchor_ratios=(1, 2, 3),
base_size=4):
np_anchors = generate_anchors(base_size=base_size,
ratios=np.array(anchor_ratios),
scales=np.array(anchor_scales))
device = fm_3x3.device
anchors = mge.tensor(np_anchors).to(device)
height, width = fm_3x3.shape[2], fm_3x3.shape[3]
shift_x = F.linspace(0, width - 1, width).to(device) * fm_stride
shift_y = F.linspace(0, height - 1, height).to(device) * fm_stride
broad_shift_x = F.broadcast_to(shift_x.reshape(1, -1),
(height, width)).flatten()
broad_shift_y = F.broadcast_to(shift_y.reshape(-1, 1),
(height, width)).flatten()
shifts = F.stack(
[broad_shift_x, broad_shift_y, broad_shift_x, broad_shift_y],
axis=1)
c = anchors.shape[1]
all_anchors = F.expand_dims(anchors, axis=0) + F.expand_dims(shifts,
axis=1)
all_anchors = all_anchors.reshape(-1, c).detach()
return all_anchors
def get_center_offsets(self, featmap, stride):
f_shp = featmap.shape
fm_height, fm_width = f_shp[-2], f_shp[-1]
shift_x = F.linspace(0, fm_width - 1, fm_width) * stride
shift_y = F.linspace(0, fm_height - 1, fm_height) * stride
# make the mesh grid of shift_x and shift_y
mesh_shape = (fm_height, fm_width)
broad_shift_x = shift_x.reshape(
-1, shift_x.shape[0]).broadcast(*mesh_shape)
broad_shift_y = shift_y.reshape(shift_y.shape[0],
-1).broadcast(*mesh_shape)
flatten_shift_x = F.add_axis(broad_shift_x.reshape(-1), 1)
flatten_shift_y = F.add_axis(broad_shift_y.reshape(-1), 1)
centers = F.concat(
[
flatten_shift_x,
flatten_shift_y,
flatten_shift_x,
flatten_shift_y,
],
axis=1,
)
centers = centers + self.offset * self.base_size
return centers
def get_center_offsets(self, featmap, stride):
# f_shp = featmap.shape
# fm_height, fm_width = f_shp[-2], f_shp[-1]
fm_height, fm_width = featmap.shape[2:]
shift_x = F.linspace(0, fm_width - 1, fm_width) * stride
shift_y = F.linspace(0, fm_height - 1, fm_height) * stride
# make the mesh grid of shift_x and shift_y
mesh_shape = (fm_height, fm_width)
broad_shift_x = F.broadcast_to(shift_x.reshape(1, -1), mesh_shape)
broad_shift_y = F.broadcast_to(shift_y.reshape(-1, 1), mesh_shape)
# broad_shift_x = shift_x.reshape(-1, shift_x.shape[0]).broadcast_to(*mesh_shape)
# broad_shift_y = shift_y.reshape(shift_y.shape[0], -1).broadcast_to(*mesh_shape)
flatten_shift_x = broad_shift_x.flatten()
flatten_shift_y = broad_shift_y.flatten()
shifts = F.stack([
flatten_shift_x, flatten_shift_y, flatten_shift_x, flatten_shift_y
],
axis=1)
# flatten_shift_x = F.add_axis(broad_shift_x.reshape(-1), 1)
# flatten_shift_y = F.add_axis(broad_shift_y.reshape(-1), 1)
# shifts = F.concat(
# [flatten_shift_x, flatten_shift_y, flatten_shift_x, flatten_shift_y,],
# axis=1)
return shifts
def fpn_anchor_target_opr_core_impl(gt_boxes,
im_info,
anchors,
allow_low_quality_matches=True):
ignore_label = config.ignore_label
# get the gt boxes
gtboxes = gt_boxes[:im_info[5].astype(np.int32)]
ignore_mask = F.equal(gtboxes[:, 4], config.ignore_label)
# find the valid gtboxes
_, index = F.cond_take(1 - ignore_mask > 0, ignore_mask)
valid_gt_boxes = gtboxes[index.astype(np.int32)]
# compute the iou matrix
overlaps = box_overlap_opr(anchors, valid_gt_boxes[:, :4])
# match the dtboxes
a_shp0 = anchors.shape[0]
argmax_overlaps = F.argmax(overlaps, axis=1)
max_overlaps = F.nn.indexing_one_hot(overlaps,
argmax_overlaps.astype(np.int32), 1)
labels = F.ones(a_shp0).astype(np.int32) * ignore_label
# set negative ones
labels = labels * (max_overlaps >= config.rpn_negative_overlap).astype(
np.float32)
# set positive ones
fg_mask = (max_overlaps >= config.rpn_positive_overlap)
const_one = mge.tensor(1.0)
if allow_low_quality_matches:
# match the max gt
gt_max_overlaps = F.max(overlaps, axis=0)
gt_argmax_overlaps = F.argmax(overlaps, axis=0)
gt_argmax_overlaps = gt_argmax_overlaps.astype(np.int32)
max_overlaps[gt_argmax_overlaps] = 1.
m = gt_max_overlaps.shape[0]
argmax_overlaps[gt_argmax_overlaps] = F.linspace(0, m - 1,
m).astype(np.int32)
fg_mask = (max_overlaps >= config.rpn_positive_overlap)
labels[fg_mask] = 1
# compute the bbox targets
bbox_targets = bbox_transform_opr(anchors,
valid_gt_boxes[argmax_overlaps, :4])
if config.rpn_bbox_normalize_targets:
std_opr = mge.tensor(config.bbox_normalize_stds[None, :]).to(
anchors.device)
mean_opr = mge.tensor(config.bbox_normalize_means[None, :]).to(
anchors.device)
minus_opr = mean_opr / std_opr
bbox_targets = bbox_targets / std_opr - minus_opr
return labels, bbox_targets
def get_ground_truth(self, anchors, batched_gt_boxes, batched_valid_gt_box_number):
total_anchors = anchors.shape[0]
labels_cat_list = []
bbox_targets_list = []
for b_id in range(self.batch_size):
gt_boxes = batched_gt_boxes[b_id, : batched_valid_gt_box_number[b_id]]
overlaps = layers.get_iou(anchors, gt_boxes[:, :4])
argmax_overlaps = F.argmax(overlaps, axis=1)
max_overlaps = overlaps.ai[
F.linspace(0, total_anchors - 1, total_anchors).astype(np.int32),
argmax_overlaps,
]
labels = mge.tensor([-1]).broadcast(total_anchors)
labels = labels * (max_overlaps >= self.cfg.negative_thresh)
labels = labels * (max_overlaps < self.cfg.positive_thresh) + (
max_overlaps >= self.cfg.positive_thresh
)
bbox_targets = self.box_coder.encode(
anchors, gt_boxes.ai[argmax_overlaps, :4]
)
labels_cat = gt_boxes.ai[argmax_overlaps, 4]
labels_cat = labels_cat * (1.0 - F.less_equal(F.abs(labels), 1e-5))
ignore_mask = F.less_equal(F.abs(labels + 1), 1e-5)
labels_cat = labels_cat * (1 - ignore_mask) - ignore_mask
# assign low_quality boxes
if self.cfg.allow_low_quality:
gt_argmax_overlaps = F.argmax(overlaps, axis=0)
labels_cat = labels_cat.set_ai(gt_boxes[:, 4])[gt_argmax_overlaps]
matched_low_bbox_targets = self.box_coder.encode(
anchors.ai[gt_argmax_overlaps, :], gt_boxes[:, :4]
)
bbox_targets = bbox_targets.set_ai(matched_low_bbox_targets)[
gt_argmax_overlaps, :
]
labels_cat_list.append(F.add_axis(labels_cat, 0))
bbox_targets_list.append(F.add_axis(bbox_targets, 0))
return (
F.zero_grad(F.concat(labels_cat_list, axis=0)),
F.zero_grad(F.concat(bbox_targets_list, axis=0)),
)
def forward(self, input_ids, token_type_ids=None):
seq_length = input_ids.shape[1]
if token_type_ids is None:
token_type_ids = F.zeros_like(input_ids)
position_ids = F.linspace(0, seq_length - 1, seq_length).astype(np.int32)
position_ids = F.broadcast_to(F.expand_dims(position_ids, 0), input_ids.shape)
words_embeddings = self.word_embeddings(input_ids)
position_embeddings = self.position_embeddings(position_ids)
token_type_embeddings = self.token_type_embeddings(token_type_ids)
embeddings = words_embeddings + position_embeddings + token_type_embeddings
embeddings = self.LayerNorm(embeddings)
embeddings = self.dropout(embeddings)
return embeddings
def fpn_anchor_target_opr_core_impl(gt_boxes,
im_info,
anchors,
allow_low_quality_matches=True):
ignore_label = config.ignore_label
# get the gt boxes
valid_gt_boxes = gt_boxes[:im_info[5], :]
non_ignore_mask = valid_gt_boxes[:, -1] > 0
non_ignore_inds = mask_to_inds(non_ignore_mask)
valid_gt_boxes = valid_gt_boxes.ai[non_ignore_inds]
# compute the iou matrix
overlaps = box_overlap_opr(anchors, valid_gt_boxes[:, :4])
# match the dtboxes
a_shp0 = anchors.shape[0]
max_overlaps = F.max(overlaps, axis=1)
argmax_overlaps = F.argmax(overlaps, axis=1)
# all ignore
labels = mge.ones(a_shp0).astype(np.int32) * ignore_label
# set negative ones
labels = labels * (max_overlaps >= config.rpn_negative_overlap)
# set positive ones
fg_mask = (max_overlaps >= config.rpn_positive_overlap)
const_one = mge.tensor(1.0)
if allow_low_quality_matches:
# match the max gt
gt_max_overlaps = F.max(overlaps, axis=0)
gt_argmax_overlaps = F.argmax(overlaps, axis=0)
g_shp0 = valid_gt_boxes.shapeof()[0]
gt_id = F.linspace(0, g_shp0 - 1, g_shp0).astype(np.int32)
argmax_overlaps = argmax_overlaps.set_ai(gt_id)[gt_argmax_overlaps]
max_overlaps = max_overlaps.set_ai(
const_one.broadcast(g_shp0))[gt_argmax_overlaps]
fg_mask = (max_overlaps >= config.rpn_positive_overlap)
# set positive ones
fg_mask_ind = mask_to_inds(fg_mask)
labels = labels.set_ai(const_one.broadcast(
fg_mask_ind.shapeof()))[fg_mask_ind]
# compute the targets
bbox_targets = bbox_transform_opr(anchors,
valid_gt_boxes.ai[argmax_overlaps, :4])
if config.rpn_bbox_normalize_targets:
std_opr = mge.tensor(config.bbox_normalize_stds[None, :])
mean_opr = mge.tensor(config.bbox_normalize_means[None, :])
minus_opr = mean_opr / std_opr
bbox_targets = bbox_targets / std_opr - minus_opr
return labels, bbox_targets
def per_level_gt(self,
gt_boxes,
im_info,
anchors,
allow_low_quality_matches=True):
ignore_label = self.cfg.ignore_label
# get the gt boxes
valid_gt_boxes = gt_boxes[:im_info[4], :]
# compute the iou matrix
overlaps = layers.get_iou(anchors, valid_gt_boxes[:, :4])
# match the dtboxes
a_shp0 = anchors.shape[0]
max_overlaps = F.max(overlaps, axis=1)
argmax_overlaps = F.argmax(overlaps, axis=1)
# all ignore
labels = mge.ones(a_shp0).astype("int32") * ignore_label
# set negative ones
labels = labels * (max_overlaps >= self.cfg.rpn_negative_overlap)
# set positive ones
fg_mask = max_overlaps >= self.cfg.rpn_positive_overlap
const_one = mge.tensor(1.0)
if allow_low_quality_matches:
# make sure that max iou of gt matched
gt_argmax_overlaps = F.argmax(overlaps, axis=0)
num_valid_boxes = valid_gt_boxes.shapeof(0)
gt_id = F.linspace(0, num_valid_boxes - 1,
num_valid_boxes).astype("int32")
argmax_overlaps = argmax_overlaps.set_ai(gt_id)[gt_argmax_overlaps]
max_overlaps = max_overlaps.set_ai(
const_one.broadcast(num_valid_boxes))[gt_argmax_overlaps]
fg_mask = max_overlaps >= self.cfg.rpn_positive_overlap
# set positive ones
_, fg_mask_ind = F.cond_take(fg_mask == 1, fg_mask)
labels = labels.set_ai(const_one.broadcast(
fg_mask_ind.shapeof(0)))[fg_mask_ind]
# compute the targets
bbox_targets = self.box_coder.encode(
anchors, valid_gt_boxes.ai[argmax_overlaps, :4])
return labels, bbox_targets
def fwd():
a = F.linspace(3, 10, 3, device=Device("xpux").to_c())
b = F.eye(3, device=Device("xpux").to_c())
return a, b
