def forward(self, images, gts=None, counts=None,get_features=False):
sources = self.backbone_net(images)
features = list()
# pdb.set_trace()
if self.shared_heads>0:
for x in sources:
features.append(self.features_layers(x))
else:
features = sources
grid_sizes = [feature_map.shape[-2:] for feature_map in features]
ancohor_boxes = self.anchors(grid_sizes)
loc = list()
conf = list()
for x in features:
loc.append(self.reg_heads(x).permute(0, 2, 3, 1).contiguous())
conf.append(self.cls_heads(x).permute(0, 2, 3, 1).contiguous())
loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1)
conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1)
flat_loc = loc.view(loc.size(0), -1, 4)
flat_conf = conf.view(conf.size(0), -1, self.num_classes)
# pdb.set_trace()
if get_features: # testing mode with feature return
return torch.stack([decode(flat_loc[b], ancohor_boxes) for b in range(flat_loc.shape[0])], 0), flat_conf, features
elif gts is not None: # training mode
return self.criterion(flat_conf, flat_loc, gts, counts, ancohor_boxes)
else: # otherwise testing mode
return torch.stack([decode(flat_loc[b], ancohor_boxes) for b in range(flat_loc.shape[0])], 0), flat_conf
def get_ids_n_boxes(loc, sf_conf, anchors, num_nodes, last_id, empty_box,
empty_score):
ids = []
boxes = []
all_scores = []
scores = []
for b in range(loc.size(0)):
decoded_boxes = decode(loc[b], anchors, [0.1, 0.2])
obj_boxes, obj_ids, obj_scores, bin_scores = apply_nms(
decoded_boxes, 1 - sf_conf[b, :, 0], sf_conf[b], num_nodes,
args.conf_thresh, args.nms_thresh)
# pdb.set_trace()
num_obj = obj_ids.size(0)
if num_nodes > num_obj:
# print(obj_ids.size(), obj_boxes.size(), self.last_id.size(), self.empty_box.size(), self.last_id)
for _ in range(num_nodes - num_obj):
obj_ids = torch.cat((obj_ids, last_id), 0)
obj_boxes = torch.cat((obj_boxes, empty_box), 0)
obj_scores = torch.cat((obj_scores, empty_score), 0)
bin_scores = torch.cat((obj_scores, empty_score[0]), 0)
else:
obj_ids = obj_ids[:num_nodes]
obj_boxes = obj_boxes[:num_nodes]
obj_scores = obj_scores[:num_nodes]
bin_scores = bin_scores[:num_nodes]
# print(obj_ids.size(), obj_boxes.size())
ids.append(obj_ids)
boxes.append(obj_boxes)
all_scores.append(obj_scores)
scores.append(bin_scores)
return ids, torch.stack(boxes, 0), torch.stack(all_scores,
0), torch.stack(scores, 0)
def extract_boxes(args, images, net, softmax, anchors):
images = images.cuda(0, non_blocking=True)
loc_data, sf_conf = net(images)
sf_conf = torch.softmax(sf_conf, 2)
b = 0
decoded_boxes = decode(loc_data[b], anchors, [0.1, 0.2])
obj_boxes, obj_ids, cls_scores, bin_scores = apply_nms(
decoded_boxes, 1 - sf_conf[b, :, 0], sf_conf[b], 10, args.conf_thresh,
args.nms_thresh)
return obj_boxes, cls_scores, bin_scores, obj_ids
def forward(self,
images,
gt_boxes=None,
gt_labels=None,
ego_labels=None,
counts=None,
img_indexs=None,
get_features=False):
sources, ego_feat = self.backbone(images)
ego_preds = self.ego_head(ego_feat).squeeze(-1).squeeze(-1).permute(
0, 2, 1).contiguous()
grid_sizes = [feature_map.shape[-2:] for feature_map in sources]
ancohor_boxes = self.anchors(grid_sizes)
loc = list()
conf = list()
for x in sources:
loc.append(self.reg_heads(x).permute(0, 2, 3, 4, 1).contiguous())
conf.append(self.cls_heads(x).permute(0, 2, 3, 4, 1).contiguous())
loc = torch.cat([o.view(o.size(0), o.size(1), -1) for o in loc], 2)
conf = torch.cat([o.view(o.size(0), o.size(1), -1) for o in conf], 2)
flat_loc = loc.view(loc.size(0), loc.size(1), -1, 4)
flat_conf = conf.view(conf.size(0), conf.size(1), -1, self.num_classes)
# pdb.set_trace()
if get_features: # testing mode with feature return
return flat_conf, features
elif gt_boxes is not None: # training mode
return self.criterion(flat_conf, flat_loc, gt_boxes, gt_labels,
counts, ancohor_boxes, ego_preds, ego_labels)
else: # otherwise testing mode
decoded_boxes = []
for b in range(flat_loc.shape[0]):
temp_l = []
for s in range(flat_loc.shape[1]):
# torch.stack([decode(flat_loc[b], ancohor_boxes) for b in range(flat_loc.shape[0])], 0),
temp_l.append(decode(flat_loc[b, s], ancohor_boxes))
decoded_boxes.append(torch.stack(temp_l, 0))
return torch.stack(decoded_boxes, 0), flat_conf, ego_preds
def validate(args,
net,
anchors,
val_data_loader,
val_dataset,
iteration_num,
iou_thresh=0.5):
"""Test a FPN network on an image database."""
print('Validating at ', iteration_num)
num_images = len(val_dataset)
num_classes = args.num_classes
det_boxes = [[] for _ in range(num_classes - 1)]
gt_boxes = []
print_time = True
val_step = 20
count = 0
torch.cuda.synchronize()
ts = time.perf_counter()
softmax = nn.Softmax(dim=2).cuda()
with torch.no_grad():
for val_itr, (images, targets,
img_indexs) in enumerate(val_data_loader):
torch.cuda.synchronize()
t1 = time.perf_counter()
batch_size = images.size(0)
height, width = images.size(2), images.size(3)
images = images.cuda(0, non_blocking=True)
loc_data, conf_data = net(images)
conf_scores_all = softmax(conf_data).clone()
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
tf = time.perf_counter()
print('Forward Time {:0.3f}'.format(tf - t1))
for b in range(batch_size):
gt = targets[b].numpy()
gt[:, 0] *= width
gt[:, 2] *= width
gt[:, 1] *= height
gt[:, 3] *= height
gt_boxes.append(gt)
decoded_boxes = decode(loc_data[b], anchors,
[0.1, 0.2]).clone()
conf_scores = conf_scores_all[b].clone()
#Apply nms per class and obtain the results
for cl_ind in range(1, num_classes):
# pdb.set_trace()
scores = conf_scores[:, cl_ind].squeeze()
c_mask = scores.gt(
args.conf_thresh) # greater than minmum threshold
scores = scores[c_mask].squeeze()
# print('scores size',scores.size())
if scores.dim() == 0:
# print(len(''), ' dim ==0 ')
det_boxes[cl_ind - 1].append(np.asarray([]))
continue
boxes = decoded_boxes.clone()
l_mask = c_mask.unsqueeze(1).expand_as(boxes)
boxes = boxes[l_mask].view(-1, 4)
# idx of highest scoring and non-overlapping boxes per class
ids, counts = nms(boxes, scores, args.nms_thresh,
args.topk) # idsn - ids after nms
scores = scores[ids[:counts]].cpu().numpy()
boxes = boxes[ids[:counts]].cpu().numpy()
# print('boxes sahpe',boxes.shape)
boxes[:, 0] *= width
boxes[:, 2] *= width
boxes[:, 1] *= height
boxes[:, 3] *= height
for ik in range(boxes.shape[0]):
boxes[ik, 0] = max(0, boxes[ik, 0])
boxes[ik, 2] = min(width, boxes[ik, 2])
boxes[ik, 1] = max(0, boxes[ik, 1])
boxes[ik, 3] = min(height, boxes[ik, 3])
cls_dets = np.hstack(
(boxes, scores[:, np.newaxis])).astype(np.float32,
copy=True)
det_boxes[cl_ind - 1].append(cls_dets)
count += 1
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
print('im_detect: {:d}/{:d} time taken {:0.3f}'.format(
count, num_images, te - ts))
torch.cuda.synchronize()
ts = time.perf_counter()
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
print('NMS stuff Time {:0.3f}'.format(te - tf))
print('Evaluating detections for itration number ', iteration_num)
return evaluate_detections(gt_boxes,
det_boxes,
val_dataset.classes,
iou_thresh=iou_thresh)
def validate_coco(args,
net,
anchors,
val_data_loader,
val_dataset,
iteration_num,
iou_thresh=0.5):
"""Test a FPN network on an image database."""
print('Validating at ', iteration_num)
annFile = '{}/instances_{}.json'.format(args.data_dir, args.val_sets[0])
cocoGT = COCO(annFile)
coco_dets = []
resFile = args.save_root + 'detections-{:05d}.json'.format(args.det_itr)
resFile_txt = open(
args.save_root + 'detections-{:05d}.txt'.format(args.det_itr), 'w')
num_images = len(val_dataset)
num_classes = args.num_classes
det_boxes = [[] for _ in range(num_classes - 1)]
gt_boxes = []
print_time = True
val_step = 20
count = 0
torch.cuda.synchronize()
ts = time.perf_counter()
softmax = nn.Softmax(dim=2).cuda()
idlist = val_dataset.idlist
all_ids = val_dataset.ids
with torch.no_grad():
for val_itr, (images, targets, img_indexs,
wh) in enumerate(val_data_loader):
# if val_itr>1:
# break
torch.cuda.synchronize()
t1 = time.perf_counter()
batch_size = images.size(0)
height, width = images.size(2), images.size(3)
images = images.cuda(0, non_blocking=True)
loc_data, conf_data = net(images)
conf_scores_all = softmax(conf_data).clone()
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
tf = time.perf_counter()
print('Forward Time {:0.3f}'.format(tf - t1))
for b in range(batch_size):
coco_image_id = int(all_ids[img_indexs[b]][1][8:])
width, height = wh[b][0], wh[b][1]
gt = targets[b].numpy()
gt[:, 0] *= width
gt[:, 2] *= width
gt[:, 1] *= height
gt[:, 3] *= height
gt_boxes.append(gt)
decoded_boxes = decode(loc_data[b], anchors,
[0.1, 0.2]).clone()
conf_scores = conf_scores_all[b].clone()
#Apply nms per class and obtain the results
for cl_ind in range(1, num_classes):
# pdb.set_trace()
scores = conf_scores[:, cl_ind].squeeze()
c_mask = scores.gt(
args.conf_thresh) # greater than minmum threshold
scores = scores[c_mask].squeeze()
# print('scores size',scores.size())
if scores.dim() == 0:
# print(len(''), ' dim ==0 ')
det_boxes[cl_ind - 1].append(np.asarray([]))
continue
boxes = decoded_boxes.clone()
l_mask = c_mask.unsqueeze(1).expand_as(boxes)
boxes = boxes[l_mask].view(-1, 4)
# idx of highest scoring and non-overlapping boxes per class
ids, counts = nms(boxes, scores, args.nms_thresh,
args.topk) # idsn - ids after nms
scores = scores[ids[:counts]].cpu().numpy()
pick = min(scores.shape[0], 20)
scores = scores[:pick]
boxes = boxes[ids[:counts]].cpu().numpy()
boxes = boxes[:pick, :]
# print('boxes sahpe',boxes.shape)
boxes[:, 0] *= width
boxes[:, 2] *= width
boxes[:, 1] *= height
boxes[:, 3] *= height
cls_id = cl_ind - 1
if len(idlist) > 0:
cls_id = idlist[cl_ind - 1]
# pdb.set_trace()
for ik in range(boxes.shape[0]):
boxes[ik, 0] = max(0, boxes[ik, 0])
boxes[ik, 2] = min(width, boxes[ik, 2])
boxes[ik, 1] = max(0, boxes[ik, 1])
boxes[ik, 3] = min(height, boxes[ik, 3])
box_ = [
round(boxes[ik, 0], 1),
round(boxes[ik, 1], 1),
round(boxes[ik, 2], 1),
round(boxes[ik, 3], 1)
]
box_[2] = round(box_[2] - box_[0], 1)
box_[3] = round(box_[3] - box_[1], 1)
box_ = [float(b) for b in box_]
coco_dets.append({
"image_id": int(coco_image_id),
"category_id": int(cls_id),
"bbox": box_,
"score": float(scores[ik]),
})
cls_dets = np.hstack(
(boxes, scores[:, np.newaxis])).astype(np.float32,
copy=True)
det_boxes[cl_ind - 1].append(cls_dets)
count += 1
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
print('im_detect: {:d}/{:d} time taken {:0.3f}'.format(
count, num_images, te - ts))
torch.cuda.synchronize()
ts = time.perf_counter()
if print_time and val_itr % val_step == 0:
torch.cuda.synchronize()
te = time.perf_counter()
print('NMS stuff Time {:0.3f}'.format(te - tf))
# print('Evaluating detections for itration number ', iteration_num)
mAP, ap_all, ap_strs, det_boxes = evaluate_detections(
gt_boxes, det_boxes, val_dataset.classes, iou_thresh=iou_thresh)
for ap_str in ap_strs:
print(ap_str)
resFile_txt.write(ap_str + '\n')
ptr_str = '\nMEANAP:::=>' + str(mAP) + '\n'
print(ptr_str)
resFile_txt.write(ptr_str)
print('saving results :::::')
with open(resFile, 'w') as f:
json.dump(coco_dets, f)
cocoDt = cocoGT.loadRes(resFile)
# running evaluation
cocoEval = COCOeval(cocoGT, cocoDt, 'bbox')
# cocoEval.params.imgIds = imgIds
cocoEval.evaluate()
cocoEval.accumulate()
cocoEval.summarize()
resFile_txt.write(ptr_str)
# pdb.set_trace()
ptr_str = ''
for s in cocoEval.stats:
ptr_str += str(s) + '\n'
print('\n\nPrintning COCOeval Generated results\n\n ')
print(ptr_str)
resFile_txt.write(ptr_str)
return mAP, ap_all, ap_strs, det_boxes