def get_node_index(classname,
bbox,
det_classes,
det_boxes,
node_num,
labeled=True):
bbox = np.array(bbox, dtype=np.float32)
max_iou = 0.5 # Use 0.5 as a threshold for evaluation
max_iou_index = -1
for i_node in range(node_num):
if labeled:
if classname == metadata.coco_classes[det_classes[i_node]]:
# check bbox overlap
# intersection_area = compute_area(get_intersection(bbox, det_boxes[i_node, :]))
# iou = intersection_area/(compute_area(bbox)+compute_area(det_boxes[i_node, :])-intersection_area)
iou = compute_iou(bbox, det_boxes[i_node, :])
if iou > max_iou:
max_iou = iou
max_iou_index = i_node
else:
iou = compute_iou(bbox, det_boxes[i_node, :])
if iou > max_iou:
max_iou = iou
max_iou_index = i_node
return max_iou_index
def match_hoi(pred_det,gt_dets):
is_match = False
for gt_det in gt_dets:
human_iou = compute_iou(pred_det['human_box'],gt_det['human_box'])
if human_iou > 0.5:
object_iou = compute_iou(pred_det['object_box'],gt_det['object_box'])
if object_iou > 0.5:
is_match = True
break
return is_match
def match_hoi(pred_det, gt_dets):
is_match = False
remaining_gt_dets = [gt_det for gt_det in gt_dets]
for i, gt_det in enumerate(gt_dets):
human_iou = compute_iou(pred_det['human_box'], gt_det['human_box'])
if human_iou > 0.5:
object_iou = compute_iou(pred_det['object_box'],
gt_det['object_box'])
if object_iou > 0.5:
is_match = True
del remaining_gt_dets[i]
break
#remaining_gt_dets.append(gt_det)
return is_match, remaining_gt_dets
def compute_overlap_matrix(pdf_bboxes, iou_thresh):
nb_tables = len(pdf_bboxes)
overlap = np.zeros((nb_tables, nb_tables))
for i, bb1 in enumerate(pdf_bboxes):
for j, bb2 in enumerate(pdf_bboxes):
if i != j and bb1[0] == bb2[0] and doOverlap(bb1[-4:], bb2[-4:]):
iou = compute_iou(bb1[-4:], bb2[-4:])
if iou > iou_thresh or isContained(
bb1[-4:], bb2[-4:]) or isContained(bb2[-4:], bb1[-4:]):
overlap[i, j] = 1.
return overlap
def get_node_index(bbox, det_boxes, index_list):
bbox = np.array(bbox, dtype=np.float32)
max_iou = 0.3 # Use 0.5 as a threshold for evaluation
max_iou_index = -1
for i_node in index_list:
# check bbox overlap
iou = compute_iou(bbox, det_boxes[i_node, :])
if iou > max_iou:
max_iou = iou
max_iou_index = i_node
return max_iou_index
def match_hoi(pred_det, gt_dets, detected):
flag2 = 1 # bck
flag3 = 1 # human misloc
flag4 = 1 # obj misloc
flag5 = 0 # duplicate detection
flag6 = 1 # mis grouping
is_match = False
remaining_gt_dets = [gt_det for gt_det in gt_dets]
for i, gt_det in enumerate(gt_dets):
human_iou = compute_iou(pred_det['human_box'], gt_det['human_box'])
if human_iou > 0.1:
flag2 = 0
if human_iou > 0.5:
flag3 = 0
object_iou = compute_iou(pred_det['object_box'],
gt_det['object_box'])
if object_iou > 0.1:
flag6 = 0
if object_iou > 0.5:
flag4 = 0
is_match = True
detected.append(gt_det)
del remaining_gt_dets[i]
break
# remaining_gt_dets.append(gt_det)
if not is_match:
for i, gt_det in enumerate(detected):
human_iou = compute_iou(pred_det['human_box'], gt_det['human_box'])
if human_iou > 0.5:
object_iou = compute_iou(pred_det['object_box'],
gt_det['object_box'])
if object_iou > 0.5:
flag2, flag3, flag4, flag6 = 0, 0, 0, 0
flag5 = 1
break
return is_match, remaining_gt_dets, detected, (flag2, flag3, flag4, flag5,
flag6)
def assign_pose(human_box, pose_boxes, pose_keypoints, num_keypoints):
max_idx = -1
max_frac_inside = 0.7
found_match = False
for i, pose_box in enumerate(pose_boxes):
iou, intersection, union = compute_iou(human_box, pose_box,True)
pose_area = compute_area(pose_box)
frac_inside = intersection / pose_area
if frac_inside > max_frac_inside:
max_frac_inside = frac_inside
max_idx = i
found_match = True
if max_idx == -1:
keypoints = np.zeros([num_keypoints,3])
else:
keypoints = pose_keypoints[max_idx]
return keypoints, found_match
def compute_recall(pred_boxes, gt_boxes, k=1):
recalled = [0] * k
pred_box = [None] * k
gt_box = [None] * k
for i, pred_box_ in enumerate(pred_boxes):
if i >= k:
break
for gt_box_ in gt_boxes:
iou = compute_iou(pred_box_, gt_box_)
if iou >= 0.5:
recalled[i] = 1
pred_box[i] = pred_box_
gt_box[i] = gt_box_
break
max_recall = 0
for i in range(k):
max_recall = max(recalled[i], max_recall)
recalled[i] = max_recall
return recalled, pred_box, gt_box
def assign_pose(det_human_boxes, det_pose_boxes, det_pose, data_const):
global MISS_POSE_NUM
num_keypoints = det_pose.shape[1]
assigned_pose = []
for human_box in det_human_boxes:
max_iou_index = -1
max_iou = data_const.pose_bbox_iou_thresh # 0.5
keypoints = np.zeros([num_keypoints, 2])
for idx in range(det_pose.shape[0]):
iou = compute_iou(human_box, det_pose_boxes[idx, :])
if iou > max_iou:
max_iou = iou
max_iou_index = idx
if max_iou_index != -1:
keypoints = det_pose[max_iou_index,:,:2]
det_pose_boxes = np.delete(det_pose_boxes, max_iou_index, axis=0)
det_pose = np.delete(det_pose, max_iou_index, axis=0)
else:
MISS_POSE_NUM += 1
# ipdb.set_trace()
assigned_pose.append(keypoints)
return np.array(assigned_pose)
def get_labels(self, gt_tables):
"""
:param gt_tables: dict, keys are page number and values are list of tables bbox within that page
:return:
"""
labels = np.zeros(len(self.candidates))
for i, candidate in enumerate(self.candidates):
page_num = candidate[0]
try:
tables = gt_tables[page_num]
for gt_table in tables:
page_width, page_height, y0, x0, y1, x1 = gt_table
w_ratio = float(candidate[1]) / page_width
h_ratio = float(candidate[2]) / page_height
rescaled_gt_table = (y0 * h_ratio, x0 * w_ratio,
y1 * h_ratio, x1 * w_ratio)
iou = compute_iou(candidate[-4:], rescaled_gt_table)
if iou > self.iou_thresh:
# candidate region is a table
labels[i] = 1
except KeyError:
# any of the candidates is a true table, all zero labels
pass
return labels
def box_recall(gt_hois, human_boxes, object_boxes, iou_thresh):
num_pred_human_boxes = len(human_boxes)
num_pred_object_boxes = len(object_boxes)
num_pred_connections = num_pred_human_boxes * num_pred_object_boxes
num_gt_connections_recalled = 0
num_gt_connections = 0
num_gt_human_boxes_recalled = 0
num_gt_human_boxes = 0
num_gt_object_boxes_recalled = 0
num_gt_object_boxes = 0
for hois_per_type in gt_hois:
gt_connections = hois_per_type['connections']
gt_human_boxes = hois_per_type['human_bboxes']
gt_object_boxes = hois_per_type['object_bboxes']
invis = hois_per_type['invis']
gt_human_boxes_recalled = [False] * len(gt_human_boxes)
for i, gt_box in enumerate(gt_human_boxes):
for box in human_boxes:
try:
iou = compute_iou(box, gt_box)
except:
import pdb
pdb.set_trace()
if iou >= iou_thresh:
gt_human_boxes_recalled[i] = True
break
gt_object_boxes_recalled = [False] * len(gt_object_boxes)
for i, gt_box in enumerate(gt_object_boxes):
for box in object_boxes:
try:
iou = compute_iou(box, gt_box)
except:
import pdb
pdb.set_trace()
if iou >= iou_thresh:
gt_object_boxes_recalled[i] = True
break
gt_connections_recalled = [False] * len(gt_connections)
for k, (i, j) in enumerate(gt_connections):
if gt_human_boxes_recalled[i] and gt_object_boxes_recalled[j]:
gt_connections_recalled[k] = True
num_gt_connections += len(gt_connections)
num_gt_connections_recalled += gt_connections_recalled.count(True)
num_gt_human_boxes += len(gt_human_boxes)
num_gt_human_boxes_recalled += gt_human_boxes_recalled.count(True)
num_gt_object_boxes += len(gt_object_boxes)
num_gt_object_boxes_recalled += gt_object_boxes_recalled.count(True)
try:
connection_recall = num_gt_connections_recalled / num_gt_connections
except ZeroDivisionError:
connection_recall = None
try:
human_recall = num_gt_human_boxes_recalled / num_gt_human_boxes
except ZeroDivisionError:
human_recall = None
try:
object_recall = num_gt_object_boxes_recalled / num_gt_object_boxes
except ZeroDivisionError:
object_recall = None
stats = {
'connection_recall': connection_recall,
'human_recall': human_recall,
'object_recall': object_recall,
'num_gt_connections_recalled': num_gt_connections_recalled,
'num_gt_connections': num_gt_connections,
'num_gt_human_boxes_recalled': num_gt_human_boxes_recalled,
'num_gt_human_boxes': num_gt_human_boxes,
'num_gt_object_boxes_recalled': num_gt_object_boxes_recalled,
'num_gt_object_boxes': num_gt_object_boxes,
'num_connection_proposals': num_pred_connections,
'num_human_proposals': num_pred_human_boxes,
'num_object_proposals': num_pred_object_boxes,
}
return stats
