def forward_for_single_feature_map(self, anchors, box_cls, box_regression):
"""
Arguments:
anchors: list[BoxList]
box_cls: tensor of size N, A * C, H, W
box_regression: tensor of size N, A * 4, H, W
"""
N, _, H, W = box_cls.shape
A = box_regression.shape[1] // 4
C = box_cls.shape[1] // A
# put in the same format as anchors
box_cls = permute_and_flatten(box_cls, N, A, C, H, W)
box_cls = box_cls.sigmoid()
box_regression = permute_and_flatten(box_regression, N, A, 4, H, W)
num_anchors = A * H * W
candidate_inds = box_cls > self.pre_nms_thresh
pre_nms_top_n = candidate_inds.reshape(N, -1).sum(1)
pre_nms_top_n = pre_nms_top_n.clamp(max_v=self.pre_nms_top_n)
results = []
for i in range(box_cls.shape[0]):
per_box_cls, per_box_regression, per_pre_nms_top_n,per_candidate_inds, per_anchors = \
box_cls[i],box_regression[i],pre_nms_top_n[i],candidate_inds[i],anchors[i]
# Sort and select TopN
# TODO most of this can be made out of the loop for
# all images.
# TODO:Yang: Not easy to do. Because the numbers of detections are
# different in each image. Therefore, this part needs to be done
# per image.
per_box_cls = per_box_cls[per_candidate_inds]
per_box_cls, top_k_indices = \
per_box_cls.topk(per_pre_nms_top_n, sorted=False)
per_candidate_nonzeros = \
per_candidate_inds.nonzero()[top_k_indices, :]
per_box_loc = per_candidate_nonzeros[:, 0]
per_class = per_candidate_nonzeros[:, 1]
if per_class.numel() > 0:
per_class += 1
detections = self.box_coder.decode(
per_box_regression[per_box_loc, :].view(-1, 4),
per_anchors.bbox[per_box_loc, :].view(-1, 4))
boxlist = BoxList(detections, per_anchors.size, mode="xyxy")
boxlist.add_field("labels", per_class)
boxlist.add_field("scores", per_box_cls)
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
results.append(boxlist)
return results
def __getitem__(self, idx):
img, anno = super(COCODataset, self).__getitem__(idx)
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh").convert("xyxy")
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size, mode='poly')
target.add_field("masks", masks)
target = target.clip_to_image(remove_empty=True)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, idx
def execute(self, x, boxes):
"""
Arguments:
x (Tensor): the mask logits
boxes (list[BoxList]): bounding boxes that are used as
reference, one for each image
Returns:
results (list[BoxList]): one BoxList for each image, containing
the extra field mask
"""
mask_prob = x.sigmoid()
# select masks coresponding to the predicted classes
num_masks = x.shape[0]
labels = [bbox.get_field("labels") for bbox in boxes]
labels = jt.contrib.concat(labels, dim=0)
index = jt.arange(num_masks)
mask_prob = mask_prob[index, labels].unsqueeze(1)
boxes_per_image = [len(box) for box in boxes]
mask_prob = mask_prob.split(boxes_per_image, dim=0)
if self.masker:
mask_prob = self.masker(mask_prob, boxes)
results = []
for prob, box in zip(mask_prob, boxes):
bbox = BoxList(box.bbox, box.size, mode="xyxy")
for field in box.fields():
bbox.add_field(field, box.get_field(field))
bbox.add_field("mask", prob)
results.append(bbox)
return results
def get_groundtruth(self, index):
img_id = self.ids[index]
anno = ET.parse(self._annopath % img_id).getroot()
anno = self._preprocess_annotation(anno)
height, width = anno["im_info"]
target = BoxList(anno["boxes"], (width, height), mode="xyxy")
target.add_field("labels", anno["labels"])
target.add_field("difficult", anno["difficult"])
return target
def forward_for_single_feature_map(self, anchors, objectness,
box_regression):
"""
Arguments:
anchors: list[BoxList]
objectness: tensor of size N, A, H, W
box_regression: tensor of size N, A * 4, H, W
"""
device = objectness.device
N, A, H, W = objectness.shape
# put in the same format as anchors
objectness = permute_and_flatten(objectness, N, A, 1, H, W).view(N, -1)
objectness = objectness.sigmoid()
box_regression = permute_and_flatten(box_regression, N, A, 4, H, W)
num_anchors = A * H * W
pre_nms_top_n = min(self.pre_nms_top_n, num_anchors)
objectness, topk_idx = objectness.topk(pre_nms_top_n,
dim=1,
sorted=True)
batch_idx = torch.arange(N, device=device)[:, None]
box_regression = box_regression[batch_idx, topk_idx]
image_shapes = [box.size for box in anchors]
concat_anchors = torch.cat([a.bbox for a in anchors], dim=0)
concat_anchors = concat_anchors.reshape(N, -1, 4)[batch_idx, topk_idx]
proposals = self.box_coder.decode(box_regression.view(-1, 4),
concat_anchors.view(-1, 4))
proposals = proposals.view(N, -1, 4)
result = []
for proposal, score, im_shape in zip(proposals, objectness,
image_shapes):
boxlist = BoxList(proposal, im_shape, mode="xyxy")
boxlist.add_field("objectness", score)
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
boxlist = boxlist_nms(
boxlist,
self.nms_thresh,
max_proposals=self.post_nms_top_n,
score_field="objectness",
)
result.append(boxlist)
return result
def filter_results_v2(self, boxlist, num_classes):
"""Returns bounding-box detection results by thresholding on scores and
applying non-maximum suppression (NMS).
"""
# unwrap the boxlist to avoid additional overhead.
# if we had multi-class NMS, we could perform this directly on the boxlist
boxes = boxlist.bbox.reshape(-1, num_classes,4)
scores = boxlist.get_field("scores").reshape(-1, num_classes)
result = []
# Apply threshold on detection probabilities and apply NMS
# Skip j = 0, because it's the background class
# inds_all = (scores > self.score_thresh).int()
scores = scores[:,1:]
inds_all = scores > self.score_thresh
# print(inds_all.shape)
# inds_all = inds_all.transpose(1,0)
inds_all = inds_all.nonzero()
labels = inds_all[:,1]+1
ind_scores = scores[inds_all[:,0],inds_all[:,1]]
ind_boxes = boxes[inds_all[:,0],inds_all[:,1],:]
ind_boxes = ind_boxes.reshape(-1,4)
result = BoxList(ind_boxes, boxlist.size, mode="xyxy")
result.add_field("scores", ind_scores)
result.add_field("labels", labels)
result = boxlist_ml_nms(result, self.nms)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
# if number_of_detections > self.detections_per_img > 0:
# cls_scores = result.get_field("scores")
# image_thresh, _ = jt.kthvalue(
# cls_scores, number_of_detections - self.detections_per_img + 1
# )
# keep = cls_scores >= image_thresh
# keep = jt.nonzero(keep).squeeze(1)
# result = result[keep]
# Absolute limit detection imgs
if number_of_detections > self.detections_per_img > 0:
cls_scores = result.get_field("scores")
scores, indices = jt.topk(
cls_scores, self.detections_per_img
)
result = result[indices]
return result
def prepare_boxlist(self, boxes, scores, image_shape):
"""
Returns BoxList from `boxes` and adds probability scores information
as an extra field
`boxes` has shape (#detections, 4 * #classes), where each row represents
a list of predicted bounding boxes for each of the object classes in the
dataset (including the background class). The detections in each row
originate from the same object proposal.
`scores` has shape (#detection, #classes), where each row represents a list
of object detection confidence scores for each of the object classes in the
dataset (including the background class). `scores[i, j]`` corresponds to the
box at `boxes[i, j * 4:(j + 1) * 4]`.
"""
boxes = boxes.reshape(-1, 4)
scores = scores.reshape(-1)
boxlist = BoxList(boxes, image_shape, mode="xyxy")
boxlist.add_field("scores", scores)
return boxlist
def filter_results(self, boxlist, num_classes):
"""Returns bounding-box detection results by thresholding on scores and
applying non-maximum suppression (NMS).
"""
# unwrap the boxlist to avoid additional overhead.
# if we had multi-class NMS, we could perform this directly on the boxlist
boxes = boxlist.bbox.reshape(-1, num_classes * 4)
scores = boxlist.get_field("scores").reshape(-1, num_classes)
device = scores.device
result = []
# Apply threshold on detection probabilities and apply NMS
# Skip j = 0, because it's the background class
inds_all = scores > self.score_thresh
for j in range(1, num_classes):
inds = inds_all[:, j].nonzero().squeeze(1)
scores_j = scores[inds, j]
boxes_j = boxes[inds, j * 4 : (j + 1) * 4]
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class = boxlist_nms(
boxlist_for_class, self.nms
)
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field(
"labels", torch.full((num_labels,), j, dtype=torch.int64, device=device)
)
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > self.detections_per_img > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = torch.kthvalue(
cls_scores.cpu(), number_of_detections - self.detections_per_img + 1
)
keep = cls_scores >= image_thresh.item()
keep = torch.nonzero(keep).squeeze(1)
result = result[keep]
return result
def select_over_all_levels(self, boxlists):
num_images = len(boxlists)
results = []
for i in range(num_images):
scores = boxlists[i].get_field("scores")
labels = boxlists[i].get_field("labels")
boxes = boxlists[i].bbox
boxlist = boxlists[i]
result = []
# skip the background
for j in range(1, self.num_classes):
inds = (labels == j).nonzero().view(-1)
scores_j = scores[inds]
boxes_j = boxes[inds, :].view(-1, 4)
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class = boxlist_nms(boxlist_for_class,
self.nms_thresh,
score_field="scores")
num_labels = len(boxlist_for_class)
boxlist_for_class.add_field("labels",
jt.full((num_labels, ), j).int32())
result.append(boxlist_for_class)
result = cat_boxlist(result)
number_of_detections = len(result)
# Limit to max_per_image detections **over all classes**
if number_of_detections > self.fpn_post_nms_top_n > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = jt.kthvalue(
cls_scores,
number_of_detections - self.fpn_post_nms_top_n + 1)
keep = cls_scores >= image_thresh
keep = jt.nonzero(keep).squeeze(1)
result = result[keep]
results.append(result)
return results
def execute(self, x, boxes):
mask_prob = x
scores = None
if self.keypointer:
mask_prob, scores = self.keypointer(x, boxes)
assert len(boxes) == 1, "Only non-batched inference supported for now"
boxes_per_image = [box.bbox.size(0) for box in boxes]
mask_prob = mask_prob.split(boxes_per_image, dim=0)
scores = scores.split(boxes_per_image, dim=0)
results = []
for prob, box, score in zip(mask_prob, boxes, scores):
bbox = BoxList(box.bbox, box.size, mode="xyxy")
for field in box.fields():
bbox.add_field(field, box.get_field(field))
prob = PersonKeypoints(prob, box.size)
prob.add_field("logits", score)
bbox.add_field("keypoints", prob)
results.append(bbox)
return results
def __getitem__(self, idx):
img_path = self.img_paths[idx]
ann_path = self.ann_paths[idx]
if self.mode == "mask":
ann = jt.array(np.asarray(Image.open(ann_path)))
# masks are represented with tensors
boxes, segmentations, labels = self._processBinayMasks(ann)
else:
with open(ann_path, "r") as ann_file:
ann = json.load(ann_file)
# masks are represented with polygons
boxes, segmentations, labels = self._processPolygons(ann)
boxes, segmentations, labels = self._filterGT(boxes, segmentations,
labels)
if len(segmentations) == 0:
empty_ann_path = self.get_img_info(idx)["ann_path"]
print("EMPTY ENTRY:", empty_ann_path)
# self.img_paths.pop(idx)
# self.ann_paths.pop(idx)
img, target, _ = self[(idx + 1) % len(self)]
# just override this image with the next
return img, target, idx
img = Image.open(img_path)
# Compose all into a BoxList instance
target = BoxList(boxes, img.size, mode="xyxy")
target.add_field("labels", jt.array(labels))
masks = SegmentationMask(segmentations, img.size, mode=self.mode)
target.add_field("masks", masks)
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target, idx
def __getitem__(self, idx):
img, anno = super(COCODataset, self).__getitem__(idx)
if not self.is_train:
if self._transforms is not None:
img, target = self._transforms(img, None)
return img, target, idx
# filter crowd annotations
# TODO might be better to add an extra field
anno = [obj for obj in anno if obj["iscrowd"] == 0]
boxes = np.array([obj["bbox"] for obj in anno])
boxes = boxes.reshape(-1, 4)
# boxes = jt.array(boxes).reshape(-1, 4) # guard against no boxes
target = BoxList(boxes, img.size, mode="xywh",
to_jittor=False) #.convert("xyxy")
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
# classes = jt.array(classes)
target.add_field("labels", classes)
if self.with_masks and anno and "segmentation" in anno[0]:
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks,
img.size,
mode='poly',
to_jittor=False)
target.add_field("masks", masks)
if self.with_masks and anno and "keypoints" in anno[0]:
keypoints = [obj["keypoints"] for obj in anno]
keypoints = PersonKeypoints(keypoints, img.size)
target.add_field("keypoints", keypoints)
#target = target.clip_to_image(remove_empty=True)
if self._transforms is not None:
img, target = self._transforms(img, target)
return img, target, idx
def forward_for_mask(self, boxlists, pixel_embed):
N, dim, m_h, m_w = pixel_embed.shape
new_boxlists = []
stride = self.fpn_strides[0] / self.mask_scale_factor
for im in range(N):
boxlist = boxlists[im]
boxes = boxlist.bbox
input_w, input_h = boxlist.size
proposal_embed = boxlist.get_field('proposal_embed')
if proposal_embed.shape[0] == 0:
new_boxlist = BoxList(boxes, boxlist.size, mode="xyxy")
new_boxlist.add_field("labels", boxlist.get_field("labels"))
new_boxlist.add_field("scores", boxlist.get_field("scores"))
new_boxlist.add_field('mask', jt.array([]))
if self.post_process_masks:
new_boxlist.add_field('stride', jt.array([1]))
new_boxlist.add_field('mask_th', jt.array([0.0]))
else:
new_boxlist.add_field('stride', jt.array([stride]))
new_boxlist.add_field('mask_th', jt.array([self.mask_th]))
new_boxlists.append(new_boxlist)
continue
mask_boxes = boxes / stride
box_masks = boxes_to_masks(mask_boxes, m_h, m_w)
proposal_margin = boxlist.get_field('proposal_margin')
mask_prob = self.compute_mask_prob(pixel_embed[im], proposal_embed,
proposal_margin, mask_boxes)
masks = mask_prob * box_masks.float()
if self.post_process_masks:
masks = nn.interpolate(X=masks.unsqueeze(1).float(),
scale_factor=stride,
mode="bilinear",
align_corners=False) > self.mask_th
masks = masks[:, 0, :input_h, :input_w]
new_boxlist = BoxList(boxes, boxlist.size, mode="xyxy")
new_boxlist.add_field('mask', masks)
new_boxlist.add_field("labels", boxlist.get_field("labels"))
new_boxlist.add_field("scores", boxlist.get_field("scores"))
if self.post_process_masks:
new_boxlist.add_field('stride', jt.array([1]))
new_boxlist.add_field('mask_th', jt.array([0.0]))
else:
new_boxlist.add_field('stride', jt.array([stride]))
new_boxlist.add_field('mask_th', jt.array([self.mask_th]))
new_boxlists.append(new_boxlist)
return new_boxlists
def im_detect_bbox_aug(model, images):
# Collect detections computed under different transformations
boxlists_ts = []
for _ in range(len(images)):
boxlists_ts.append([])
def add_preds_t(boxlists_t):
for i, boxlist_t in enumerate(boxlists_t):
if len(boxlists_ts[i]) == 0:
# The first one is identity transform, no need to resize the boxlist
boxlists_ts[i].append(boxlist_t)
else:
# Resize the boxlist as the first one
boxlists_ts[i].append(boxlist_t.resize(boxlists_ts[i][0].size))
# Compute detections for the original image (identity transform)
boxlists_i = im_detect_bbox(
model,
images,
cfg.INPUT.MIN_SIZE_TEST,
cfg.INPUT.MAX_SIZE_TEST,
)
add_preds_t(boxlists_i)
# Perform detection on the horizontally flipped image
if cfg.TEST.BBOX_AUG.H_FLIP:
boxlists_hf = im_detect_bbox_hflip(
model,
images,
cfg.INPUT.MIN_SIZE_TEST,
cfg.INPUT.MAX_SIZE_TEST,
)
add_preds_t(boxlists_hf)
# Compute detections at different scales
for scale in cfg.TEST.BBOX_AUG.SCALES:
max_size = cfg.TEST.BBOX_AUG.MAX_SIZE
boxlists_scl = im_detect_bbox_scale(
model,
images,
scale,
max_size,
)
add_preds_t(boxlists_scl)
if cfg.TEST.BBOX_AUG.SCALE_H_FLIP:
boxlists_scl_hf = im_detect_bbox_scale(model,
images,
scale,
max_size,
hflip=True)
add_preds_t(boxlists_scl_hf)
# Merge boxlists detected by different bbox aug params
boxlists = []
for i, boxlist_ts in enumerate(boxlists_ts):
bbox = jt.contrib.concat([boxlist_t.bbox for boxlist_t in boxlist_ts])
scores = jt.contrib.concat(
[boxlist_t.get_field('scores') for boxlist_t in boxlist_ts])
boxlist = BoxList(bbox, boxlist_ts[0].size, boxlist_ts[0].mode)
boxlist.add_field('scores', scores)
boxlists.append(boxlist)
# Apply NMS and limit the final detections
results = []
post_processor = make_roi_box_post_processor(cfg)
for boxlist in boxlists:
results.append(
post_processor.filter_results(boxlist,
cfg.MODEL.ROI_BOX_HEAD.NUM_CLASSES))
return results
def forward_for_single_feature_map(self, locations, box_cls,
box_regression, centerness,
image_sizes):
"""
Arguments:
anchors: list[BoxList]
box_cls: tensor of size N, A * C, H, W
box_regression: tensor of size N, A * 4, H, W
"""
N, C, H, W = box_cls.shape
# put in the same format as locations
box_cls = box_cls.view(N, C, H, W).permute(0, 2, 3, 1)
box_cls = box_cls.reshape(N, -1, self.num_classes - 1).sigmoid()
box_regression = box_regression.view(N, self.dense_points * 4, H,
W).permute(0, 2, 3, 1)
box_regression = box_regression.reshape(N, -1, 4)
centerness = centerness.view(N, self.dense_points, H,
W).permute(0, 2, 3, 1)
centerness = centerness.reshape(N, -1).sigmoid()
candidate_inds = box_cls > self.pre_nms_thresh
pre_nms_top_n = candidate_inds.view(N, -1).sum(1)
pre_nms_top_n = pre_nms_top_n.clamp(max_v=self.pre_nms_top_n)
# multiply the classification scores with centerness scores
box_cls = box_cls * centerness[:, :].unsqueeze(2)
results = []
#print('forward_for_single_feature_map start',N)
for i in range(N):
#print(i)
per_box_cls = box_cls[i]
per_candidate_inds = candidate_inds[i]
#print(per_candidate_inds.shape,per_box_cls.shape)
# if per_candidate_inds.sum().item()>0:
# per_box_cls = per_box_cls[per_candidate_inds]
# else:
# per_box_cls = jt.zeros((0,),dtype=per_box_cls.dtype)
#print(per_candidate_inds.shape,jt.sum(per_candidate_inds))
per_box_cls = per_box_cls[per_candidate_inds]
per_candidate_nonzeros = per_candidate_inds.nonzero()
per_box_loc = per_candidate_nonzeros[:, 0]
per_class = per_candidate_nonzeros[:, 1]
# if per_candidate_nonzeros.numel()>0:
# per_class = per_candidate_nonzeros[:, 1] + 1
per_class = per_candidate_nonzeros[:, 1] + 1
#print(per_candidate_nonzeros.shape)
per_box_regression = box_regression[i]
#print('GG',per_box_loc.numel(),per_box_loc.shape)
# if per_box_loc.numel()>0:
# per_box_regression = per_box_regression[per_box_loc]
# per_locations = locations[per_box_loc]
# else:
# shape = list(per_box_regression.shape)
# shape[0]=0
# per_box_regression = jt.zeros(shape,dtype=per_box_regression.dtype)
# shape = list(locations.shape)
# shape[0]=0
# per_locations = jt.zeros(shape,dtype=locations.dtype)
per_box_regression = per_box_regression[per_box_loc]
per_locations = locations[per_box_loc]
#print('??')
#print('per_box_cls1',per_box_cls.mean())
per_pre_nms_top_n = pre_nms_top_n[i]
#print('per_locations',jt.mean(per_locations))
#print('per_box_regressions',jt.mean(per_box_regression))
#print(per_pre_nms_top_n.item(),per_candidate_inds.sum().item())
if per_candidate_inds.sum().item() > per_pre_nms_top_n.item():
per_box_cls, top_k_indices = \
per_box_cls.topk(per_pre_nms_top_n.item(), sorted=False)
per_class = per_class[top_k_indices]
per_box_regression = per_box_regression[top_k_indices]
per_locations = per_locations[top_k_indices]
#print('per_box_cls',per_box_cls.mean())
#print('emmm',jt.mean(per_locations))
#print('hhh',jt.mean(per_box_regression))
# if per_box_loc.numel()>0:
# detections = jt.stack([
# per_locations[:, 0] - per_box_regression[:, 0],
# per_locations[:, 1] - per_box_regression[:, 1],
# per_locations[:, 0] + per_box_regression[:, 2],
# per_locations[:, 1] + per_box_regression[:, 3],
# ], dim=1)
# else:
# detections = jt.zeros((0,4),dtype=per_locations.dtype)
detections = jt.stack([
per_locations[:, 0] - per_box_regression[:, 0],
per_locations[:, 1] - per_box_regression[:, 1],
per_locations[:, 0] + per_box_regression[:, 2],
per_locations[:, 1] + per_box_regression[:, 3],
],
dim=1)
#print('detections',jt.mean(detections),detections.shape)
h, w = image_sizes[i]
boxlist = BoxList(detections, (int(w), int(h)), mode="xyxy")
boxlist.add_field("labels", per_class)
if self.is_sqrt:
boxlist.add_field("scores", per_box_cls.sqrt())
else:
boxlist.add_field("scores", per_box_cls)
#print('??',boxlist.get_field('scores'))
if boxlist.bbox.numel() > 0:
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
results.append(boxlist)
#print('Good')
return results
def forward_for_single_feature_map(self, locations, box_cls,
box_regression, centerness,
proposal_embed, proposal_margin,
image_sizes, level):
"""
Arguments:
anchors: list[BoxList]
box_cls: tensor of size N, A * C, H, W
box_regression: tensor of size N, A * 4, H, W
"""
N, C, H, W = box_cls.shape
# put in the same format as locations
box_cls = box_cls.view(N, C, H, W).transpose(0, 2, 3, 1)
box_cls = box_cls.reshape(N, -1, C).sigmoid()
box_regression = box_regression.view(N, 4, H, W).transpose(0, 2, 3, 1)
box_regression = box_regression.reshape(N, -1, 4)
centerness = centerness.view(N, 1, H, W).transpose(0, 2, 3, 1)
centerness = centerness.reshape(N, -1).sigmoid()
proposal_embed = proposal_embed.view(N, -1, H, W).transpose(0, 2, 3, 1)
proposal_embed = proposal_embed.reshape(N, H * W, -1)
proposal_margin = proposal_margin.view(N, 1, H,
W).transpose(0, 2, 3, 1)
proposal_margin = proposal_margin.reshape(N, -1)
candidate_inds = box_cls > self.pre_nms_thresh
pre_nms_top_n = candidate_inds.view(N, -1).sum(1)
pre_nms_top_n = pre_nms_top_n.clamp(max_v=self.pre_nms_top_n)
# multiply the classification scores with centerness scores
box_cls = box_cls * centerness[:, :].unsqueeze(2)
results = []
for i in range(N):
per_box_cls = box_cls[i]
per_candidate_inds = candidate_inds[i]
per_box_cls = per_box_cls[per_candidate_inds]
per_candidate_nonzeros = per_candidate_inds.nonzero()
per_box_loc = per_candidate_nonzeros[:, 0]
per_class = per_candidate_nonzeros[:, 1]
if per_candidate_nonzeros.numel() > 0:
per_class = per_candidate_nonzeros[:, 1] + 1
per_box_regression = box_regression[i]
per_box_regression = per_box_regression[per_box_loc]
per_locations = locations[per_box_loc]
per_proposal_embed = proposal_embed[i]
per_proposal_embed = per_proposal_embed[per_box_loc, :]
per_proposal_margin = proposal_margin[i][per_box_loc]
per_pre_nms_top_n = pre_nms_top_n[i]
if per_candidate_inds.sum().item() > per_pre_nms_top_n.item():
per_box_cls, top_k_indices = \
per_box_cls.topk(per_pre_nms_top_n.item(), sorted=False)
per_class = per_class[top_k_indices]
per_box_regression = per_box_regression[top_k_indices]
per_locations = per_locations[top_k_indices]
per_proposal_embed = per_proposal_embed[top_k_indices]
per_proposal_margin = per_proposal_margin[top_k_indices]
detections = jt.stack([
per_locations[:, 0] - per_box_regression[:, 0],
per_locations[:, 1] - per_box_regression[:, 1],
per_locations[:, 0] + per_box_regression[:, 2],
per_locations[:, 1] + per_box_regression[:, 3],
],
dim=1)
h, w = image_sizes[i]
boxlist = BoxList(detections, (int(w), int(h)), mode="xyxy")
boxlist.add_field("labels", per_class)
boxlist.add_field("scores", per_box_cls)
boxlist.add_field("proposal_embed", per_proposal_embed)
boxlist.add_field("proposal_margin", per_proposal_margin)
if boxlist.bbox.numel() > 0:
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
results.append(boxlist)
return results
def filter_results(self, boxlist, num_classes):
"""Returns bounding-box detection results by thresholding on scores and
applying non-maximum suppression (NMS).
"""
# unwrap the boxlist to avoid additional overhead.
# if we had multi-class NMS, we could perform this directly on the boxlist
boxes = boxlist.bbox.reshape(-1, num_classes * 4)
scores = boxlist.get_field("scores").reshape(-1, num_classes)
result = []
# Apply threshold on detection probabilities and apply NMS
# Skip j = 0, because it's the background class
# inds_all = (scores > self.score_thresh).int()
inds_all = scores > self.score_thresh
# print(self.score_thresh,num_classes)
# print(inds_all.shape)
# inds_all = inds_all.transpose(1,0)
inds_nonzeros = [ inds_all[:,j].nonzero() for j in range(1, num_classes) ]
jt.sync(inds_nonzeros)
for j in range(1, num_classes):
# with nvtx_scope("aa"):
# inds = inds_all[:,j].nonzero().squeeze(1)
# with nvtx_scope("bb"):
# scores_j = scores[inds, j]
# boxes_j = boxes[inds, j * 4 : (j + 1) * 4]
# with nvtx_scope("cc"):
# boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
# with nvtx_scope("cc2"):
# boxlist_for_class.add_field("scores", scores_j)
# with nvtx_scope("cc3"):
# boxlist_for_class = boxlist_nms(
# boxlist_for_class, self.nms
# )
# with nvtx_scope("dd"):
# num_labels = len(boxlist_for_class)
# with nvtx_scope("dd2"):
# boxlist_for_class.add_field(
# "labels", jt.full((num_labels,), j).int32()
# )
# result.append(boxlist_for_class)
# inds = inds_all[:,j].nonzero().squeeze(1)
inds = inds_nonzeros[j-1]
if inds.shape[0] == 0:
continue
inds = inds.squeeze(1)
scores_j = scores[inds, j]
boxes_j = boxes[inds, j * 4 : (j + 1) * 4]
boxlist_for_class = BoxList(boxes_j, boxlist.size, mode="xyxy")
boxlist_for_class.add_field("scores", scores_j)
boxlist_for_class = boxlist_nms(
boxlist_for_class, self.nms
)
num_labels = len(boxlist_for_class)
# print(j,num_labels)
boxlist_for_class.add_field(
"labels", jt.full((num_labels,), j).int32()
)
result.append(boxlist_for_class)
result = cat_boxlist(result)
if not result.has_field('labels'):
result.add_field('labels',jt.empty((0,)))
if not result.has_field('scores'):
result.add_field('scores',jt.empty((0,)))
number_of_detections = len(result)
#Limit to max_per_image detections **over all classes**
if number_of_detections > self.detections_per_img > 0:
cls_scores = result.get_field("scores")
image_thresh, _ = jt.kthvalue(
cls_scores, number_of_detections - self.detections_per_img + 1
)
keep = cls_scores >= image_thresh
keep = jt.nonzero(keep).squeeze(1)
result = result[keep]
# # Absolute limit detection imgs
# if number_of_detections > self.detections_per_img > 0:
# cls_scores = result.get_field("scores")
# scores, indices = jt.topk(
# cls_scores, self.detections_per_img
# )
# result = result[indices]
return result
def forward_for_single_feature_map(self, anchors, objectness,
box_regression):
"""
Arguments:
anchors: list[BoxList]
objectness: tensor of size N, A, H, W
box_regression: tensor of size N, A * 4, H, W
"""
# global II
# import pickle
N, A, H, W = objectness.shape
# put in the same format as anchors
objectness = permute_and_flatten(objectness, N, A, 1, H,
W).reshape(N, -1)
# print('objectness',objectness.mean())
objectness = objectness.sigmoid()
box_regression = permute_and_flatten(box_regression, N, A, 4, H, W)
# print('regression',box_regression.mean())
num_anchors = A * H * W
pre_nms_top_n = min(self.pre_nms_top_n, num_anchors)
# print(pre_nms_top_n)
#print('objectness',objectness)
# objectness = jt.array(pickle.load(open(f'/home/lxl/objectness_0_{II}.pkl','rb')))
# print(objectness.shape)
objectness, topk_idx = objectness.topk(pre_nms_top_n,
dim=1,
sorted=True)
# print(II,'topk',topk_idx.sum(),topk_idx.shape)
batch_idx = jt.arange(N).unsqueeze(1)
# pickle.dump(topk_idx.numpy(),open(f'/home/lxl/topk_idx_{II}_jt.pkl','wb'))
# topk_idx_tmp = topk_idx.numpy()
# batch_idx = jt.array(pickle.load(open(f'/home/lxl/batch_idx_{II}.pkl','rb')))
# topk_idx = jt.array(pickle.load(open(f'/home/lxl/topk_idx_{II}.pkl','rb')))
# err = np.abs(topk_idx_tmp-topk_idx.numpy())
# print('Error!!!!!!!!!!!!!!!!',err.sum())
# print(err.nonzero())
#print('box_regression0',box_regression)
#batch_idx = jt.index(topk_idx.shape,dim=0)
box_regression = box_regression[batch_idx, topk_idx]
#print('box_regression1',box_regression)
image_shapes = [box.size for box in anchors]
concat_anchors = jt.contrib.concat([a.bbox for a in anchors], dim=0)
concat_anchors = concat_anchors.reshape(N, -1, 4)[batch_idx, topk_idx]
# box_regression = jt.array(pickle.load(open(f'/home/lxl/box_regression_{II}.pkl','rb')))
# concat_anchors = jt.array(pickle.load(open(f'/home/lxl/concat_anchors_{II}.pkl','rb')))
proposals = self.box_coder.decode(box_regression.reshape(-1, 4),
concat_anchors.reshape(-1, 4))
proposals = proposals.reshape(N, -1, 4)
# proposals = jt.array(pickle.load(open(f'/home/lxl/proposal_{II}.pkl','rb')))
# objectness = jt.array(pickle.load(open(f'/home/lxl/objectness_{II}.pkl','rb')))
# II+=1
result = []
for i in range(len(image_shapes)):
proposal, score, im_shape = proposals[i], objectness[
i], image_shapes[i]
boxlist = BoxList(proposal, im_shape, mode="xyxy")
boxlist.add_field("objectness", score)
boxlist = boxlist.clip_to_image(remove_empty=False)
boxlist = remove_small_boxes(boxlist, self.min_size)
boxlist = boxlist_nms(
boxlist,
self.nms_thresh,
max_proposals=self.post_nms_top_n,
score_field="objectness",
)
result.append(boxlist)
return result