def get_rel_counts(ds_name, must_overlap=True):
"""
Get counts of all of the relations. Used for modeling directly P(rel | o1, o2)
:param train_data:
:param must_overlap:
:return:
"""
if ds_name.find('vg') >= 0:
with open(DATASETS['vg_train'][ANN_FN2]) as f:
train_data = json.load(f)
elif ds_name.find('gqa') >= 0:
with open(DATASETS['gqa_train'][ANN_FN2]) as f:
train_data = json.load(f)
elif ds_name.find('oi') >= 0:
with open(DATASETS['oi_rel_train'][ANN_FN2]) as f:
train_data = json.load(f)
elif ds_name.find('vrd') >= 0:
with open(DATASETS['vrd_train'][ANN_FN2]) as f:
train_data = json.load(f)
else:
raise NotImplementedError
fg_matrix = np.zeros(
(
cfg.MODEL.NUM_CLASSES - 1, # not include background
cfg.MODEL.NUM_CLASSES - 1, # not include background
cfg.MODEL.NUM_PRD_CLASSES + 1, # include background
),
dtype=np.int64)
bg_matrix = np.zeros(
(
cfg.MODEL.NUM_CLASSES - 1, # not include background
cfg.MODEL.NUM_CLASSES - 1, # not include background
),
dtype=np.int64)
for _, im_rels in train_data.items():
# get all object boxes
gt_box_to_label = {}
for i, rel in enumerate(im_rels):
sbj_box = box_utils_rel.y1y2x1x2_to_x1y1x2y2(
rel['subject']['bbox'])
obj_box = box_utils_rel.y1y2x1x2_to_x1y1x2y2(rel['object']['bbox'])
sbj_lbl = rel['subject']['category'] # not include background
obj_lbl = rel['object']['category'] # not include background
prd_lbl = rel['predicate'] # not include background
if tuple(sbj_box) not in gt_box_to_label:
gt_box_to_label[tuple(sbj_box)] = sbj_lbl
if tuple(obj_box) not in gt_box_to_label:
gt_box_to_label[tuple(obj_box)] = obj_lbl
fg_matrix[sbj_lbl, obj_lbl, prd_lbl + 1] += 1
if cfg.MODEL.USE_OVLP_FILTER:
if len(gt_box_to_label):
gt_boxes = np.array(list(gt_box_to_label.keys()),
dtype=np.int32)
gt_classes = np.array(list(gt_box_to_label.values()),
dtype=np.int32)
o1o2_total = gt_classes[np.array(box_filter(
gt_boxes, must_overlap=must_overlap),
dtype=int)]
for (o1, o2) in o1o2_total:
bg_matrix[o1, o2] += 1
else:
# consider all pairs of boxes, overlapped or non-overlapped
for b1, l1 in gt_box_to_label.items():
for b2, l2 in gt_box_to_label.items():
if b1 == b2:
continue
bg_matrix[l1, l2] += 1
return fg_matrix, bg_matrix
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
# valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros(
(num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype
)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype
)
if self.keypoints is not None:
gt_keypoints = np.zeros(
(num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype
)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0
)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map
)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(
entry['gt_keypoints'], gt_keypoints, axis=0
)
entry['has_visible_keypoints'] = im_has_visible_keypoints
entry['dataset_name'] = self.name
# add relationship annotations
im_rels = self.rel_anns[entry['file_name']]
sbj_gt_boxes = np.zeros((len(im_rels), 4), dtype=entry['sbj_gt_boxes'].dtype)
obj_gt_boxes = np.zeros((len(im_rels), 4), dtype=entry['obj_gt_boxes'].dtype)
sbj_gt_classes = np.zeros(len(im_rels), dtype=entry['sbj_gt_classes'].dtype)
obj_gt_classes = np.zeros(len(im_rels), dtype=entry['obj_gt_classes'].dtype)
prd_gt_classes = np.zeros(len(im_rels), dtype=entry['prd_gt_classes'].dtype)
for ix, rel in enumerate(im_rels):
# sbj
sbj_gt_box = box_utils_rel.y1y2x1x2_to_x1y1x2y2(rel['subject']['bbox'])
sbj_gt_boxes[ix] = sbj_gt_box
sbj_gt_classes[ix] = rel['subject']['category'] # excludes background
# obj
obj_gt_box = box_utils_rel.y1y2x1x2_to_x1y1x2y2(rel['object']['bbox'])
obj_gt_boxes[ix] = obj_gt_box
obj_gt_classes[ix] = rel['object']['category'] # excludes background
# prd
prd_gt_classes[ix] = rel['predicate'] # exclude background
entry['sbj_gt_boxes'] = np.append(entry['sbj_gt_boxes'], sbj_gt_boxes, axis=0)
entry['obj_gt_boxes'] = np.append(entry['obj_gt_boxes'], obj_gt_boxes, axis=0)
entry['sbj_gt_classes'] = np.append(entry['sbj_gt_classes'], sbj_gt_classes)
entry['obj_gt_classes'] = np.append(entry['obj_gt_classes'], obj_gt_classes)
entry['prd_gt_classes'] = np.append(entry['prd_gt_classes'], prd_gt_classes)
# misc
sbj_gt_overlaps = np.zeros(
(len(im_rels), self.num_obj_classes), dtype=entry['sbj_gt_overlaps'].dtype)
for ix in range(len(im_rels)):
sbj_cls = sbj_gt_classes[ix]
sbj_gt_overlaps[ix, sbj_cls] = 1.0
entry['sbj_gt_overlaps'] = np.append(
entry['sbj_gt_overlaps'].toarray(), sbj_gt_overlaps, axis=0)
entry['sbj_gt_overlaps'] = scipy.sparse.csr_matrix(entry['sbj_gt_overlaps'])
obj_gt_overlaps = np.zeros(
(len(im_rels), self.num_obj_classes), dtype=entry['obj_gt_overlaps'].dtype)
for ix in range(len(im_rels)):
obj_cls = obj_gt_classes[ix]
obj_gt_overlaps[ix, obj_cls] = 1.0
entry['obj_gt_overlaps'] = np.append(
entry['obj_gt_overlaps'].toarray(), obj_gt_overlaps, axis=0)
entry['obj_gt_overlaps'] = scipy.sparse.csr_matrix(entry['obj_gt_overlaps'])
prd_gt_overlaps = np.zeros(
(len(im_rels), self.num_prd_classes), dtype=entry['prd_gt_overlaps'].dtype)
pair_to_gt_ind_map = np.zeros(
(len(im_rels)), dtype=entry['pair_to_gt_ind_map'].dtype)
for ix in range(len(im_rels)):
prd_cls = prd_gt_classes[ix]
prd_gt_overlaps[ix, prd_cls] = 1.0
pair_to_gt_ind_map[ix] = ix
entry['prd_gt_overlaps'] = np.append(
entry['prd_gt_overlaps'].toarray(), prd_gt_overlaps, axis=0)
entry['prd_gt_overlaps'] = scipy.sparse.csr_matrix(entry['prd_gt_overlaps'])
entry['pair_to_gt_ind_map'] = np.append(
entry['pair_to_gt_ind_map'], pair_to_gt_ind_map)
for k in ['file_name']:
if k in entry:
del entry[k]