def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
# crowd regions are RLE encoded and stored as dicts
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
num_valid_objs = len(valid_objs)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
gt_classes[ix] = cls
for cls in gt_classes:
entry['gt_classes'][0, cls] = 1
def _add_gt_boxes(self, entry):
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
valid_objs = []
width = entry['width']
height = entry['height']
for obj in objs:
# Convert from (x1, y1, w, h) to (x_center/img_w, y_center/img_h, w/img_w, h/img_h)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width)
x_yolo, y_yolo, w_yolo, h_yolo = box_utils.xyxy_to_xywh_yolo(
[x1, y1, x2, y2], height, width)
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x_yolo, y_yolo, w_yolo, h_yolo]
valid_objs.append(obj)
num_valid_objs = len(valid_objs)
gt_boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs),
dtype=entry['gt_classes'].dtype)
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
gt_boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
entry['boxes'] = np.append(entry['boxes'], gt_boxes, axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
def _add_gt_annotations_wad(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
label_image_name = entry['entry_id'][:-4] + '_instanceIds.png'
# color_img = Image.open(os.path.join(self.WAD_CVPR2018.train_image_dir, entry['entry_id']))
label_image = os.path.join(self.WAD_CVPR2018.train_label_dir, label_image_name)
assert os.path.exists(label_image), 'Label \'{}\' not found'.format(label_image)
l_img = Image.open(label_image)
l_img = np.asarray(l_img)
entry['height'] = self.WAD_CVPR2018.image_shape[0]
entry['width'] = self.WAD_CVPR2018.image_shape[1]
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
for label in np.unique(l_img):
class_id = label // 1000
if class_id in self.WAD_CVPR2018.eval_class:
area = np.sum(l_img == label)
if area < cfg.TRAIN.GT_MIN_AREA:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
mask = l_img == label
mask_f = np.array(mask, order='F', dtype=np.uint8)
rle = COCOmask.encode(mask_f)
valid_segms.append(rle)
xd, yd = np.where(mask)
x1, y1, x2, y2 = yd.min(), xd.min(), yd.max(), xd.max()
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(x1, y1, x2, y2, entry['height'], entry['width'])
# Require non-zero seg area and more than 1x1 box size\
obj = {'area': area, 'clean_bbox': [x1, y1, x2, y2], 'category_id': class_id}
valid_objs.append(obj)
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=np.float32)
gt_classes = np.zeros((num_valid_objs), dtype=np.int32)
gt_overlaps = np.zeros((num_valid_objs, self.num_classes), dtype=np.float32)
seg_areas = np.zeros((num_valid_objs), dtype=np.float32)
is_crowd = np.zeros((num_valid_objs), dtype=np.bool)
box_to_gt_ind_map = np.zeros((num_valid_objs), dtype=np.int32)
for ix, obj in enumerate(valid_objs):
cls = self.WAD_CVPR2018.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = False #TODO: What's this flag for?
box_to_gt_ind_map[ix] = ix
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(), gt_overlaps, axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'], box_to_gt_ind_map)
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
objs = self.scenes[entry['id']]['objects']
# Sanitize bboxes -- some are invalid
valid_objs = []
width = entry['width']
height = entry['height']
for obj in objs.values():
x, y, w, h = obj['x'], obj['y'], obj['w'], obj['h']
x1, y1, x2, y2 = x, y, x + w, y + h
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
if x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros(
(num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype
)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype
)
for ix, obj in enumerate(valid_objs):
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = self.category_to_id_map[obj['name']]
# is_crowd[ix] = obj['iscrowd']
is_crowd[ix] = 0
box_to_gt_ind_map[ix] = ix
if is_crowd[ix]:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, gt_classes[ix]] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0
)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map
)
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
# valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros(
(num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype
)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype
)
if self.keypoints is not None:
gt_keypoints = np.zeros(
(num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype
)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0
)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map
)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(
entry['gt_keypoints'], gt_keypoints, axis=0
)
entry['has_visible_keypoints'] = im_has_visible_keypoints
entry['dataset_name'] = self.name
# add relationship annotations
im_rels = self.rel_anns[entry['file_name']]
sbj_gt_boxes = np.zeros((len(im_rels), 4), dtype=entry['sbj_gt_boxes'].dtype)
obj_gt_boxes = np.zeros((len(im_rels), 4), dtype=entry['obj_gt_boxes'].dtype)
sbj_gt_classes = np.zeros(len(im_rels), dtype=entry['sbj_gt_classes'].dtype)
obj_gt_classes = np.zeros(len(im_rels), dtype=entry['obj_gt_classes'].dtype)
prd_gt_classes = np.zeros(len(im_rels), dtype=entry['prd_gt_classes'].dtype)
for ix, rel in enumerate(im_rels):
# sbj
sbj_gt_box = box_utils_rel.y1y2x1x2_to_x1y1x2y2(rel['subject']['bbox'])
sbj_gt_boxes[ix] = sbj_gt_box
sbj_gt_classes[ix] = rel['subject']['category'] # excludes background
# obj
obj_gt_box = box_utils_rel.y1y2x1x2_to_x1y1x2y2(rel['object']['bbox'])
obj_gt_boxes[ix] = obj_gt_box
obj_gt_classes[ix] = rel['object']['category'] # excludes background
# prd
prd_gt_classes[ix] = rel['predicate'] # exclude background
entry['sbj_gt_boxes'] = np.append(entry['sbj_gt_boxes'], sbj_gt_boxes, axis=0)
entry['obj_gt_boxes'] = np.append(entry['obj_gt_boxes'], obj_gt_boxes, axis=0)
entry['sbj_gt_classes'] = np.append(entry['sbj_gt_classes'], sbj_gt_classes)
entry['obj_gt_classes'] = np.append(entry['obj_gt_classes'], obj_gt_classes)
entry['prd_gt_classes'] = np.append(entry['prd_gt_classes'], prd_gt_classes)
# misc
sbj_gt_overlaps = np.zeros(
(len(im_rels), self.num_obj_classes), dtype=entry['sbj_gt_overlaps'].dtype)
for ix in range(len(im_rels)):
sbj_cls = sbj_gt_classes[ix]
sbj_gt_overlaps[ix, sbj_cls] = 1.0
entry['sbj_gt_overlaps'] = np.append(
entry['sbj_gt_overlaps'].toarray(), sbj_gt_overlaps, axis=0)
entry['sbj_gt_overlaps'] = scipy.sparse.csr_matrix(entry['sbj_gt_overlaps'])
obj_gt_overlaps = np.zeros(
(len(im_rels), self.num_obj_classes), dtype=entry['obj_gt_overlaps'].dtype)
for ix in range(len(im_rels)):
obj_cls = obj_gt_classes[ix]
obj_gt_overlaps[ix, obj_cls] = 1.0
entry['obj_gt_overlaps'] = np.append(
entry['obj_gt_overlaps'].toarray(), obj_gt_overlaps, axis=0)
entry['obj_gt_overlaps'] = scipy.sparse.csr_matrix(entry['obj_gt_overlaps'])
prd_gt_overlaps = np.zeros(
(len(im_rels), self.num_prd_classes), dtype=entry['prd_gt_overlaps'].dtype)
pair_to_gt_ind_map = np.zeros(
(len(im_rels)), dtype=entry['pair_to_gt_ind_map'].dtype)
for ix in range(len(im_rels)):
prd_cls = prd_gt_classes[ix]
prd_gt_overlaps[ix, prd_cls] = 1.0
pair_to_gt_ind_map[ix] = ix
entry['prd_gt_overlaps'] = np.append(
entry['prd_gt_overlaps'].toarray(), prd_gt_overlaps, axis=0)
entry['prd_gt_overlaps'] = scipy.sparse.csr_matrix(entry['prd_gt_overlaps'])
entry['pair_to_gt_ind_map'] = np.append(
entry['pair_to_gt_ind_map'], pair_to_gt_ind_map)
for k in ['file_name']:
if k in entry:
del entry[k]
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry.
"""
seq_idx = entry['seq_idx']
idx = entry['idx']
self.set_to_sequence(seq_idx)
#get gt image.
gt = self.get_gt(idx)
vals = np.unique(gt)
objs = []
for val in vals:
#it is background when val==0
if val != 0:
obj = {}
mask = np.array(gt == val, dtype=np.uint8)
#make sure gt==val is converted to value in 0 and 1.
assert (len(set(mask.reshape(-1)) - {0, 1}) == 0)
x, y, w, h = cv2.boundingRect(mask)
x, y, w, h = self._expand_box(x, y, w, h, rate=0.05)
#obj['segmentation'] = binary_mask_to_rle(mask)
obj['segmentation'] = mask_util.encode(
np.array(mask, order='F', dtype=np.uint8))
obj['area'] = np.sum(mask)
obj['iscrowd'] = 0
obj['bbox'] = x, y, w, h
if self.cls_mapper is not None:
#set category id by cls_mapper.
obj['category_id'] = self.cls_mapper[val]
else:
if not self.use_local_id:
obj['category_id'] = self.global_instance_id_start_of_seq[
seq_idx] + val - 1
else:
obj['category_id'] = val
obj['instance_id'] = val
assert (self.global_instance_id_start_of_seq[seq_idx] != 0)
# val-1 to remove background.
obj['global_instance_id'] = self.global_instance_id_start_of_seq[
seq_idx] + val - 1
objs.append(obj)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
# crowd regions are RLE encoded and stored as dicts
assert (isinstance(obj['segmentation'], dict))
if isinstance(obj['segmentation'], list):
# Valid polygons have >= 3 points, so require >= 6 coordinates
obj['segmentation'] = [
p for p in obj['segmentation'] if len(p) >= 6
]
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs),
dtype=entry['gt_classes'].dtype)
instance_id = np.zeros((num_valid_objs),
dtype=entry['instance_id'].dtype)
global_instance_id = np.zeros((num_valid_objs),
dtype=entry['global_instance_id'].dtype)
gt_overlaps = np.zeros((num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype)
gt_overlaps_id = np.zeros(
(num_valid_objs, self.number_of_instance_ids),
dtype=entry['gt_overlaps_id'].dtype)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros((num_valid_objs),
dtype=entry['box_to_gt_ind_map'].dtype)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
if obj['category_id'] is not None:
#cls = self.json_category_id_to_contiguous_id[obj['category_id']]
cls = obj['category_id']
else:
#if no category_id specified, use background instead. index is 'self.num_classes-1'
cls = self.num_classes - 1
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
instance_id[ix] = obj['instance_id']
global_instance_id[ix] = obj['global_instance_id']
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
gt_overlaps_id[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
gt_overlaps_id[ix, global_instance_id[ix]] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['instance_id'] = np.append(entry['instance_id'], instance_id)
entry['global_instance_id'] = np.append(entry['global_instance_id'],
global_instance_id)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(),
gt_overlaps,
axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['gt_overlaps_id'] = np.append(entry['gt_overlaps_id'].toarray(),
gt_overlaps_id,
axis=0)
entry['gt_overlaps_id'] = scipy.sparse.csr_matrix(
entry['gt_overlaps_id'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'],
box_to_gt_ind_map)
assert (
entry['gt_overlaps_id'].shape[0] == entry['gt_overlaps'].shape[0])
def _add_gt_annotations_Car3d(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
entry_id = entry['entry_id']
# Make file_name an abs path
car_pose_file = os.path.join(self.Car3D.data_dir, 'car_poses',
entry_id + '.json')
assert os.path.exists(car_pose_file), 'Label \'{}\' not found'.format(
car_pose_file)
with open(car_pose_file) as f:
car_poses = json.load(f)
entry['height'] = self.Car3D.image_shape[0]
entry['width'] = self.Car3D.image_shape[1]
intrinsic_mat = self.Car3D.get_intrinsic_mat()
# Sanitize bboxes -- some are invalid
valid_objs = []
for i, car_pose in enumerate(car_poses):
car_name = self.Car3D.car_id2name[car_pose['car_id']].name
car = self.car_models[car_name]
pose = np.array(car_pose['pose'])
# project 3D points to 2d image plane
rot_mat = euler_angles_to_rotation_matrix(pose[:3])
rvect, _ = cv2.Rodrigues(rot_mat)
imgpts, jac = cv2.projectPoints(np.float32(car['vertices']),
rvect,
pose[3:],
intrinsic_mat,
distCoeffs=None)
imgpts = np.int32(imgpts).reshape(-1, 2)
x1, y1, x2, y2 = imgpts[:, 0].min(), imgpts[:, 1].min(
), imgpts[:, 0].max(), imgpts[:, 1].max()
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, entry['height'], entry['width'])
# Require non-zero seg area and more than 1x1 box size\
obj = {
'area': car_pose['area'],
'clean_bbox': [x1, y1, x2, y2],
'category_id': 33,
'car_id': car_pose['car_id'],
'visible_rate': car_pose['visible_rate'],
'pose': car_pose['pose']
}
valid_objs.append(obj)
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=np.float32)
# this is a legacy network from WAD Mask-RCNN
car_class = 4
gt_overlaps = np.zeros((num_valid_objs, 8), dtype=np.float32)
seg_areas = np.zeros((num_valid_objs), dtype=np.float32)
is_crowd = np.zeros((num_valid_objs), dtype=np.bool)
box_to_gt_ind_map = np.zeros((num_valid_objs), dtype=np.int32)
# newly added for 3d car
visible_rate = np.zeros((num_valid_objs), dtype=np.float32)
poses = np.zeros((num_valid_objs, 6), dtype=np.float32)
quaternions = np.zeros((num_valid_objs, 4), dtype=np.float32)
car_cat_classes = np.zeros((num_valid_objs), dtype=np.int32)
for ix, obj in enumerate(valid_objs):
cls = np.where(self.Car3D.unique_car_models == obj['car_id'])[0][0]
boxes[ix, :] = obj['clean_bbox']
car_cat_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = False # TODO: What's this flag for?
box_to_gt_ind_map[ix] = ix
gt_overlaps[ix, car_class] = 1.0
visible_rate[ix] = obj['visible_rate']
poses[ix] = obj['pose']
quaternions[ix] = euler_angles_to_quaternions(
np.array([obj['pose'][:3]]))
# ensure the quaternion is upper hemispher
quaternions[ix] = quaternion_upper_hemispher(quaternions[ix])
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(),
gt_overlaps,
axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'],
box_to_gt_ind_map)
# newly added for 3d car
entry['visible_rate'] = np.append(entry['visible_rate'], visible_rate)
entry['poses'] = np.append(entry['poses'], poses, axis=0)
entry['car_cat_classes'] = np.append(entry['car_cat_classes'],
car_cat_classes)
entry['quaternions'] = np.append(entry['quaternions'],
quaternions,
axis=0)
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
objs = self.scenes[entry['id']]['objects']
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
rle = self._preprocess_rle(obj['mask'])
mask = mask_utils.decode(rle)
bbox, _ = segm_utils.rle_masks_to_boxes([rle])
x1, y1, x2, y2 = bbox[0][0], bbox[0][1], bbox[0][2], bbox[0][3]
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
obj['area'] = mask.sum()
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
valid_segms.append(rle)
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros(
(num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype
)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype
)
for ix, obj in enumerate(valid_objs):
if cfg.CLEVR.COMP_CAT:
cls = self.category_to_id_map[' '.join([obj['color'], obj['material'], obj['shape']])]
else:
cls = self.category_to_id_map[obj['shape']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
# is_crowd[ix] = obj['iscrowd']
is_crowd[ix] = 0
box_to_gt_ind_map[ix] = ix
if is_crowd[ix]:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0
)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map
)
def _add_gt_annotations(self, entry, entry_id, annots):
if len(annots) > 0:
objs = self.convert_raw_predictions_to_objs(
annots[entry_id], entry['id'])
else:
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
# crowd regions are RLE encoded and stored as dicts
if isinstance(obj['segmentation'], list):
# Valid polygons have >= 3 points, so require >= 6 coordinates
obj['segmentation'] = [
p for p in obj['segmentation'] if len(p) >= 6
]
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form x1, y1, w, h to x1, y1, x2, y2
x1 = obj['bbox'][0]
y1 = obj['bbox'][1]
x2 = x1 + np.maximum(0., obj['bbox'][2] - 1.)
y2 = y1 + np.maximum(0., obj['bbox'][3] - 1.)
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
tracks = -np.ones((num_valid_objs, 1), dtype=entry['tracks'].dtype)
head_boxes = -np.ones((num_valid_objs, 4),
dtype=entry['head_boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros(
(num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype)
if self.keypoints is not None:
gt_keypoints = np.zeros(
(num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
if 'track_id' in obj:
tracks[ix, 0] = obj['track_id']
if 'head_box' in obj:
# NOTE: This box has NOT BEEN CLEANED, and NOT BEEN converted
# to (xmin, ymin, xmax, ymax). This is only here to be used
# in MPII evaluations
head_boxes[ix, :] = obj['head_box']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['tracks'] = np.append(entry['tracks'], tracks, axis=0)
entry['head_boxes'] = np.append(entry['head_boxes'], head_boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(
entry['gt_keypoints'], gt_keypoints, axis=0)
entry['has_visible_keypoints'] = im_has_visible_keypoints
def _load_annotation_to_roidb(self, anno_file):
gt_roidb = []
entries = json.load(open(anno_file))
# for imageId, entry in entries.items(): # for each sample in all the samples
# if dir_name == '_part':
# pdb.set_trace()
small_cnt = 0
for entry in entries: # for each sample in all the samples
if entry['fold'] == '309':
continue
syms = entry['syms']
polygons = entry['polygon']
if 'rows' in entry:
rows = entry['rows']
cols = entry['cols']
elif 'bottom_row' in entry:
rows = entry['bottom_row']
cols = entry['right_col']
file_name = entry['filename'] + '.jpg'
evaId = entry['evaId']
image_dir = self.image_directory
docName = entry['docName']
fold = entry['fold']
if 'offset_x' in entry and 'offset_y' in entry:
offset_x, offset_y = entry['offset_x'], entry['offset_y']
else:
offset_x, offset_y = 0, 0
npoly = len(polygons)
is_valid_polygon = [True for _ in range(npoly)]
# count number of valid syms
for idx, (sym, polygon) in enumerate(zip(syms, polygons)):
if not self._valid_polygon(polygon): # exclude non-valid polygons
is_valid_polygon[idx] = False
continue
cls_list, gt_boxes = [], []
filtered_polygons = []
for idx, (sym, polygon) in enumerate(zip(syms, polygons)): # for each sym or region
if not is_valid_polygon[idx]:
continue
for s in sym: # for each sub-sym
if s == '膈面膨隆' or s == '膈面抬高': # awkward ...
s = '膈面异常'
if s == '盘状肺不张':
s = '纤维化表现'
if s in self._class_to_ind: # if in the given classes
polygon = [tuple(point) for point in polygon]
polygon_np = np.array(polygon)
x1, y1, x2, y2 = polygon_np[:, 0].min(), polygon_np[:, 1].min(), \
polygon_np[:, 0].max(), polygon_np[:, 1].max(),
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, rows, cols
)
# if nodule is too large, then assign it to diffusive nodules
if s == '结节' and (x2 - x1 > 300 or y2 - y1 > 300):
s = '弥漫性结节'
cls = self._class_to_ind[s]
cls_list.append(cls)
# expand too-small boxes
if x2 - x1 < 20:
cx = (x1 + x2) * 0.5
x1 = cx - 10
x2 = cx + 10
small_cnt += 1
if y2 - y1 < 20:
cy = (y1 + y2) * 0.5
y1 = cy - 10
y2 = cy + 10
small_cnt += 1
gt_boxes.append([x1, y1, x2, y2])
tmp = [list(chain.from_iterable(polygon))] # [[x1, y1], [x2, y2]] -> [[x1, y1, x2, y2]]
filtered_polygons.append(tmp)
else:
# print(s)
pass
assert len(cls_list) == len(gt_boxes) == len(filtered_polygons)
new_entry = {'file_name': file_name, 'cls_list': cls_list, 'height': rows, 'width': cols,
'polygons': filtered_polygons, 'gt_boxes': gt_boxes, 'eva_id': evaId, 'doc_name': docName,
'image_dir': image_dir, 'offset_x': offset_x, 'offset_y': offset_y, 'fold': fold}
gt_roidb.append(new_entry)
print('small boxes: ', small_cnt)
return gt_roidb
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'])
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
im_path = self.image_path_from_index(obj['image_id'])
# crowd regions are RLE encoded and stored as dicts
if isinstance(obj['segmentation'], list):
# Valid polygons have >= 3 points, so require >= 6 coordinates
obj['segmentation'] = [
p for p in obj['segmentation'] if len(p) >= 6
]
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
# Compress the rle
if 'counts' in obj['segmentation'] and type(obj['segmentation']['counts']) == list:
# Magic RLE format handling painfully discovered by looking at the
# COCO API showAnns function.
rle = mask_util.frPyObjects(obj['segmentation'], height, width)
valid_segms.append(rle)
else:
binary_mask = segm_utils.polys_to_mask(obj['segmentation'], height, width)
if len(np.unique(binary_mask))==1:
continue
valid_segms.append(obj['segmentation'])
valid_objs.append(obj)
query = {
'boxes': obj['clean_bbox'],
'segms': valid_segms,
'image_path': im_path,
'area': obj['area'],
}
self.cat_data[obj['category_id']].append(query)
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_cats = np.zeros((num_valid_objs), dtype=entry['gt_cats'].dtype)
#gt_overlaps = np.zeros(
# (num_valid_objs, self.num_classes),
# dtype=entry['gt_overlaps'].dtype
#)
gt_overlaps = np.zeros(
(num_valid_objs, cfg.MODEL.NUM_CLASSES),
dtype=entry['gt_overlaps'].dtype
)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype
)
if self.keypoints is not None:
gt_keypoints = np.zeros(
(num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype
)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = obj['category_id']
boxes[ix, :] = obj['clean_bbox']
#gt_classes[ix] = cls
gt_classes[ix] = 1
gt_cats[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
#gt_overlaps[ix, cls] = 1.0
gt_overlaps[ix, 1] = 1.0
box_utils.validate_boxes(boxes, width=width, height=height)
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['gt_cats'] = np.append(entry['gt_cats'], gt_cats)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0
)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map
)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(
entry['gt_keypoints'], gt_keypoints, axis=0
)
entry['has_visible_keypoints'] = im_has_visible_keypoints
def _load_annotation_to_roidb(self, anno_file):
gt_roidb = []
entries = json.load(open(anno_file))
# for imageId, entry in entries.items(): # for each sample in all the samples
small_cnt = 0
for entry in entries: # for each sample in all the samples
# pdb.set_trace()
syms = entry['syms']
polygons = entry['polygon']
if 'boxes' in entry:
boxes = entry['boxes']
else:
boxes = []
if 'rows' in entry:
rows = entry['rows']
cols = entry['cols']
elif 'bottom_row' in entry:
rows = entry['bottom_row'] - entry['top_row']
cols = entry['right_col'] - entry['left_col']
# 新版json 关键字变化 file_name eva_id doc_name
file_name = entry['file_name']
# if file_name in ['58695.png', '57569.png', '45795.png', '59788.png', '60191.png',
# '60795.png', '69838.png', '70454.png', '69845.png']:
# continue
image_dir = self.image_directory
evaId = entry['eva_id']
docName = entry['doc_name']
fold = entry['fold']
if 'offset_x' in entry and 'offset_y' in entry:
offset_x, offset_y = entry['offset_x'], entry['offset_y']
else:
offset_x, offset_y = 0, 0
has_mask = False
if len(polygons) > 0 and len(boxes) == 0:
has_mask = True
elif len(boxes) > 0 and len(polygons) == 0:
has_mask = False
for box in boxes:
x1, y1, x2, y2 = box
polygon = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
polygons.append(polygon)
npoly = len(polygons)
is_valid_polygon = [True for _ in range(npoly)]
# count number of valid syms
for idx, (sym, polygon) in enumerate(zip(syms, polygons)):
if not self._valid_polygon(
polygon): # exclude non-valid polygons
is_valid_polygon[idx] = False
continue
cls_list, gt_boxes = [], []
filtered_polygons = []
remove_flag = False
for idx, (sym, polygon) in enumerate(zip(
syms, polygons)): # for each sym or region
if not is_valid_polygon[idx]:
continue
if '膈面异常' in sym and entry['doc_name'] == 'fj6311':
continue
# if '胸腔积液' in sym and entry['file_name'] == '39420.png':
# continue
if '主动脉异常' in sym and '钙化' in sym:
sym = ['主动脉钙化', '主动脉异常']
if '结节' in sym and '乳头影' in sym: # 费主任标了好多这种，结节和乳头影都在，我们认为是乳头影
sym = ['乳头影']
if '结节' in sym and '弥漫性结节' in sym:
sym.remove('结节')
if '结节' in sym and '多发结节' in sym:
sym.remove('结节')
if '结核结节' in sym and '弥漫性结节' in sym:
sym.remove('结核结节')
if '结核结节' in sym and '多发结节' in sym:
sym.remove('结核结节')
if '结核球' in sym and '弥漫性结节' in sym:
sym.remove('结核球')
if '结核球' in sym and '多发结节' in sym:
sym.remove('结核球')
for s in sym: # for each sub-sym
if s == '膈面膨隆' or s == '膈面抬高': # awkward ...
s = '膈面异常'
# if s == '盘状肺不张':
# s = '纤维化表现'
# if s == '肺结核': # ignore 肺结核
# s = '肺实变'
if s == '肺门影浓' or s == '肺门影大':
s = '肺门增浓'
if s == '主动脉异常':
s = '主动脉结增宽'
# 以下是肺结核的征象
if s == '三均匀粟粒样结节' or s == '非三均匀粟粒样结节':
s = '弥漫性结节'
if s == '结核球' or s == '结核结节':
s = '结节'
if s == '索条影':
s = '纤维化表现'
# cvpr code
if s == '骨折' or s == '肋骨缺失':
s = '骨折'
if s == '弥漫性结节' or s == '多发结节':
s = '弥漫性结节'
if s in self._class_to_ind: # if in the given classes
polygon = [tuple(point) for point in polygon]
polygon_np = np.array(polygon)
x1, y1, x2, y2 = polygon_np[:, 0].min(), polygon_np[:, 1].min(), \
polygon_np[:, 0].max(), polygon_np[:, 1].max(),
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, rows, cols)
# # if nodule is too large, then assign it to diffusive nodules
if s == '结节' and (x2 - x1 > 300 or y2 - y1 > 300):
s = '弥漫性结节'
# remove_flag = True
# break
cls = self._class_to_ind[s]
cls_list.append(cls)
# expand too-small boxes (width or height < 20)
if x2 - x1 < 20:
cx = (x1 + x2) * 0.5
x1 = cx - 10
x2 = cx + 10
small_cnt += 1
if y2 - y1 < 20:
cy = (y1 + y2) * 0.5
y1 = cy - 10
y2 = cy + 10
small_cnt += 1
gt_boxes.append([x1, y1, x2, y2])
tmp = [list(chain.from_iterable(polygon))
] # [[x1, y1], [x2, y2]] -> [[x1, y1, x2, y2]]
filtered_polygons.append(tmp)
else:
# print(s)
pass
if not remove_flag:
assert len(cls_list) == len(gt_boxes) == len(filtered_polygons)
# LJ add parts info
parts_list = []
parts = entry['parts']
part_boxes = entry['part_boxes']
for part_name in parts:
parts_list.append(self._class_to_parts[part_name])
part_boxes = np.array(part_boxes, dtype=np.float32)
parts_list = np.array(parts_list, dtype=np.int32)
new_entry = {
'file_name': file_name,
'cls_list': cls_list,
'height': rows,
'width': cols,
'polygons': filtered_polygons,
'gt_boxes': gt_boxes,
'eva_id': evaId,
'doc_name': docName,
'image_dir': image_dir,
'offset_x': offset_x,
'offset_y': offset_y,
'fold': fold,
'has_mask': has_mask,
'parts_list': parts_list,
'part_boxes': part_boxes
}
gt_roidb.append(new_entry)
return gt_roidb
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
if len(objs) > 0:
has_scores = ('score' in objs[0].keys())
else:
has_scores = False
if len(objs) > 0:
has_source = ('source' in objs[0].keys())
else:
has_source = False
# EDIT: detection "soft"-scores
# if has_scores:
if not 'gt_scores' in entry.keys():
entry['gt_scores'] = np.empty((0), dtype=np.float32)
# EDIT: dataset name
if len(objs) > 0:
has_dataset = ('dataset' in objs[0].keys())
else:
has_dataset = True # TODO: Check. This assumes that the "dataset" field is given
# if has_dataset:
if not 'dataset_id' in entry.keys():
entry['dataset_id'] = np.empty((0), dtype=np.int32)
if not 'gt_source' in entry.keys():
entry['gt_source'] = np.empty((0), dtype=np.int32)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
"""input('HEREEEE1111 '+str(has_dataset)+': '+str(entry['dataset'].name)+' __ '+str(entry))
if not has_dataset:
print(entry['dataset'].name)
input(str(entry))
"""
for obj in objs:
# crowd regions are RLE encoded and stored as dicts
if isinstance(obj['segmentation'], list):
# Valid polygons have >= 3 points, so require >= 6 coordinates
obj['segmentation'] = [
p for p in obj['segmentation'] if len(p) >= 6
]
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
#print('>>>',num_valid_objs)
#input('checkpoint here')
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs),
dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros((num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype)
# if has_scores:
gt_scores = np.zeros((num_valid_objs),
dtype=entry['gt_scores'].dtype) # EDIT: scores
gt_source = np.zeros((num_valid_objs), dtype=entry['gt_source'].dtype)
# if has_dataset:
dataset_annot = np.zeros(
(num_valid_objs), dtype=entry['dataset_id'].dtype) # EDIT: dataset
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros((num_valid_objs),
dtype=entry['box_to_gt_ind_map'].dtype)
if self.keypoints is not None:
gt_keypoints = np.zeros((num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
if has_scores:
gt_scores[ix] = obj['score'] # EDIT: scores
if has_source:
gt_source[ix] = obj[
'source'] # EDIT: annot source (det or track)
if has_dataset:
dataset_annot[ix] = cfg.TRAIN.DATASETS.index(
obj['dataset']) # EDIT: dataset id
#else:
# input('>>>'+str(obj['dataset']))
# input(':::'+str(has_dataset)+'__'+str(dataset_annot))
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
if has_scores:
entry['gt_scores'] = np.append(entry['gt_scores'],
gt_scores) # EDIT: scores
if has_source:
entry['gt_source'] = np.append(entry['gt_source'], gt_source)
if has_dataset:
entry['dataset_id'] = np.append(entry['dataset_id'],
dataset_annot) # EDIT: dataset
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(),
gt_overlaps,
axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'],
box_to_gt_ind_map)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(entry['gt_keypoints'],
gt_keypoints,
axis=0)
entry['has_visible_keypoints'] = im_has_visible_keypoints
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
# crowd regions are RLE encoded and stored as dicts
if isinstance(obj['segmentation'], list):
# Valid polygons have >= 3 points, so require >= 6 coordinates
obj['segmentation'] = [
p for p in obj['segmentation'] if len(p) >= 6
]
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width
)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs), dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros(
(num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype
)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros(
(num_valid_objs), dtype=entry['box_to_gt_ind_map'].dtype
)
if self.keypoints is not None:
gt_keypoints = np.zeros(
(num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype
)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(
entry['gt_overlaps'].toarray(), gt_overlaps, axis=0
)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(
entry['box_to_gt_ind_map'], box_to_gt_ind_map
)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(
entry['gt_keypoints'], gt_keypoints, axis=0
)
entry['has_visible_keypoints'] = im_has_visible_keypoints
def _load_annotation_to_roidb(self, anno_file):
gt_roidb = []
entries = json.load(open(anno_file))
# for imageId, entry in entries.items(): # for each sample in all the samples
# if dir_name == '_part':
# pdb.set_trace()
small_cnt = 0
for entry in entries: # for each sample in all the samples
# if entry['fold'] == '309':
# continue
syms = entry['syms']
polygons = entry['polygon']
if 'boxes' in entry:
boxes = entry['boxes']
else:
boxes = []
if 'rows' in entry:
rows = entry['rows']
cols = entry['cols']
elif 'bottom_row' in entry:
rows = entry['bottom_row'] - entry['top_row']
cols = entry['right_col'] - entry['left_col']
# if 'filename' not in entry.keys():
# pdb.set_trace()
if entry['filename'].endswith('.jpg'):
file_name = entry['filename']
else:
file_name = entry['filename'] + '.jpg'
evaId = entry['evaId']
image_dir = self.image_directory
docName = entry['docName']
fold = entry['fold']
manufacturer = None if 'manufacturer' not in entry or entry[
'manufacturer'] is None else entry['manufacturer'].strip('\"')
manufacturer_model = None if 'manufacturer_model' not in entry or entry[
'manufacturer_model'] is None else entry[
'manufacturer_model'].strip('\"')
# manufacturer = entry['manufacturer'].strip('\"') if 'manufacturer' in entry else None
# manufacturer_model = entry['manufacturer_model'].strip('\"') if 'manufacturer_model' in entry else None
if 'offset_x' in entry and 'offset_y' in entry:
offset_x, offset_y = entry['offset_x'], entry['offset_y']
else:
offset_x, offset_y = 0, 0
has_mask = False
if len(polygons) > 0 and len(boxes) == 0:
has_mask = True
elif len(boxes) > 0 and len(polygons) == 0:
has_mask = False
for box in boxes:
x1, y1, x2, y2 = box
polygon = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
polygons.append(polygon)
npoly = len(polygons)
is_valid_polygon = [True for _ in range(npoly)]
# count number of valid syms
for idx, (sym, polygon) in enumerate(zip(syms, polygons)):
if not self._valid_polygon(
polygon): # exclude non-valid polygons
is_valid_polygon[idx] = False
continue
cls_list, gt_boxes = [], []
filtered_polygons = []
# pdb.set_trace()
for idx, (sym, polygon) in enumerate(zip(
syms, polygons)): # for each sym or region
if not is_valid_polygon[idx]:
continue
if '膈面异常' in sym and entry['docName'] == 'fj6311':
continue
if '主动脉异常' in sym and '钙化' in sym:
sym = ['主动脉钙化', '主动脉异常']
if '结节' in sym and '乳头影' in sym: # 费主任标了好多这种，结节和乳头影都在，我们认为是乳头影
sym = ['乳头影']
if '结节' in sym and '弥漫性结节' in sym:
sym.remove('结节')
if '结节' in sym and '多发结节' in sym:
sym.remove('结节')
if '结核结节' in sym and '弥漫性结节' in sym:
sym.remove('结核结节')
if '结核结节' in sym and '多发结节' in sym:
sym.remove('结核结节')
if '结核球' in sym and '弥漫性结节' in sym:
sym.remove('结核球')
if '结核球' in sym and '多发结节' in sym:
sym.remove('结核球')
for s in sym: # for each sub-sym
if s == '膈面膨隆' or s == '膈面抬高': # awkward ...
s = '膈面异常'
# if s == '盘状肺不张':
# s = '纤维化表现'
# if s == '肺结核': # ignore 肺结核
# s = '肺实变'
if s == '肺门影浓' or s == '肺门影大':
s = '肺门增浓'
if s == '主动脉异常':
s = '主动脉结增宽'
# 以下是肺结核的征象
if s == '三均匀粟粒样结节' or s == '非三均匀粟粒样结节':
s = '弥漫性结节'
if s == '结核球' or s == '结核结节':
s = '结节'
if s == '索条影':
s = '纤维化表现'
if s in self._class_to_ind: # if in the given classes
polygon = [tuple(point) for point in polygon]
polygon_np = np.array(polygon)
x1, y1, x2, y2 = polygon_np[:, 0].min(), polygon_np[:, 1].min(), \
polygon_np[:, 0].max(), polygon_np[:, 1].max(),
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, rows, cols)
# if nodule is too large, then assign it to diffusive nodules
if s == '结节' and (x2 - x1 > 300 or y2 - y1 > 300):
# if fold == '2019.06.14':
# pdb.set_trace()
s = '多发结节'
print('fold ', fold, 'evaId ', evaId, idx + 1,
'多发结节')
cls = self._class_to_ind[s]
cls_list.append(cls)
# expand too-small boxes (width or height < 20)
if x2 - x1 < 20:
cx = (x1 + x2) * 0.5
x1 = cx - 10
x2 = cx + 10
small_cnt += 1
if y2 - y1 < 20:
cy = (y1 + y2) * 0.5
y1 = cy - 10
y2 = cy + 10
small_cnt += 1
gt_boxes.append([x1, y1, x2, y2])
tmp = [list(chain.from_iterable(polygon))
] # [[x1, y1], [x2, y2]] -> [[x1, y1, x2, y2]]
filtered_polygons.append(tmp)
else:
# print(s)
pass
assert len(cls_list) == len(gt_boxes) == len(filtered_polygons)
new_entry = {
'file_name': file_name,
'cls_list': cls_list,
'height': rows,
'width': cols,
'polygons': filtered_polygons,
'gt_boxes': gt_boxes,
'eva_id': evaId,
'doc_name': docName,
'image_dir': image_dir,
'offset_x': offset_x,
'offset_y': offset_y,
'fold': fold,
'manufacturer': manufacturer,
'manufacturer_model': manufacturer_model,
'has_mask': has_mask
}
gt_roidb.append(new_entry)
print('small boxes: ', small_cnt)
# pdb.set_trace()
return gt_roidb
def _add_gt_annotations(self, entry):
"""Add ground truth annotation metadata to an roidb entry."""
ann_ids = self.COCO.getAnnIds(imgIds=entry['id'], iscrowd=None)
objs = self.COCO.loadAnns(ann_ids)
# Sanitize bboxes -- some are invalid
valid_objs = []
valid_segms = []
width = entry['width']
height = entry['height']
for obj in objs:
# crowd regions are RLE encoded and stored as dicts
if isinstance(obj['segmentation'], list):
# Valid polygons have >= 3 points, so require >= 6 coordinates
obj['segmentation'] = [
p for p in obj['segmentation'] if len(p) >= 6
]
if obj['area'] < cfg.TRAIN.GT_MIN_AREA:
continue
if 'ignore' in obj and obj['ignore'] == 1:
continue
# Convert form (x1, y1, w, h) to (x1, y1, x2, y2)
x1, y1, x2, y2 = box_utils.xywh_to_xyxy(obj['bbox'])
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, height, width)
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 0 and x2 > x1 and y2 > y1:
obj['clean_bbox'] = [x1, y1, x2, y2]
valid_objs.append(obj)
valid_segms.append(obj['segmentation'])
num_valid_objs = len(valid_objs)
boxes = np.zeros((num_valid_objs, 4), dtype=entry['boxes'].dtype)
gt_classes = np.zeros((num_valid_objs),
dtype=entry['gt_classes'].dtype)
gt_overlaps = np.zeros((num_valid_objs, self.num_classes),
dtype=entry['gt_overlaps'].dtype)
seg_areas = np.zeros((num_valid_objs), dtype=entry['seg_areas'].dtype)
is_crowd = np.zeros((num_valid_objs), dtype=entry['is_crowd'].dtype)
box_to_gt_ind_map = np.zeros((num_valid_objs),
dtype=entry['box_to_gt_ind_map'].dtype)
if self.keypoints is not None:
gt_keypoints = np.zeros((num_valid_objs, 3, self.num_keypoints),
dtype=entry['gt_keypoints'].dtype)
im_has_visible_keypoints = False
for ix, obj in enumerate(valid_objs):
cls = self.json_category_id_to_contiguous_id[obj['category_id']]
boxes[ix, :] = obj['clean_bbox']
gt_classes[ix] = cls
seg_areas[ix] = obj['area']
is_crowd[ix] = obj['iscrowd']
box_to_gt_ind_map[ix] = ix
if self.keypoints is not None:
gt_keypoints[ix, :, :] = self._get_gt_keypoints(obj)
if np.sum(gt_keypoints[ix, 2, :]) > 0:
im_has_visible_keypoints = True
if obj['iscrowd']:
# Set overlap to -1 for all classes for crowd objects
# so they will be excluded during training
gt_overlaps[ix, :] = -1.0
else:
gt_overlaps[ix, cls] = 1.0
entry['boxes'] = np.append(entry['boxes'], boxes, axis=0)
entry['segms'].extend(valid_segms)
# To match the original implementation:
# entry['boxes'] = np.append(
# entry['boxes'], boxes.astype(np.int).astype(np.float), axis=0)
entry['gt_classes'] = np.append(entry['gt_classes'], gt_classes)
entry['seg_areas'] = np.append(entry['seg_areas'], seg_areas)
entry['gt_overlaps'] = np.append(entry['gt_overlaps'].toarray(),
gt_overlaps,
axis=0)
entry['gt_overlaps'] = scipy.sparse.csr_matrix(entry['gt_overlaps'])
entry['is_crowd'] = np.append(entry['is_crowd'], is_crowd)
entry['box_to_gt_ind_map'] = np.append(entry['box_to_gt_ind_map'],
box_to_gt_ind_map)
if self.keypoints is not None:
entry['gt_keypoints'] = np.append(entry['gt_keypoints'],
gt_keypoints,
axis=0)
entry['has_visible_keypoints'] = im_has_visible_keypoints
def _load_annotation_to_roidb(self, anno_file):
gt_roidb = []
entries = json.load(open(anno_file))
small_cnt = 0
for entry in entries: # for each sample in all the samples
syms = entry['syms']
polygons = entry['polygon']
boxes = entry['boxes']
if 'rows' in entry:
rows = entry['rows']
cols = entry['cols']
elif 'bottom_row' in entry:
rows = entry['bottom_row'] - entry['top_row']
cols = entry['right_col'] - entry['left_col']
file_name = entry['filename'] + '.jpg'
evaId = entry['evaId']
image_dir = self.image_directory
docName = entry['docName']
fold = entry['fold']
manufacturer = None if 'manufacturer' not in entry or entry['manufacturer'] is None \
else entry['manufacturer'].strip('\"')
manufacturer_model = None if 'manufacturer_model' not in entry or entry['manufacturer_model'] is None \
else entry['manufacturer_model'].strip('\"')
if 'offset_x' in entry and 'offset_y' in entry:
offset_x, offset_y = entry['offset_x'], entry['offset_y']
else:
offset_x, offset_y = 0, 0
has_mask = False
if len(polygons) > 0 and len(boxes) == 0:
has_mask = True
elif len(boxes) > 0 and len(polygons) == 0:
has_mask = False
for box in boxes:
x1, y1, x2, y2 = box
polygon = [[x1, y1], [x2, y1], [x2, y2], [x1, y2]]
polygons.append(polygon)
npoly = len(polygons)
is_valid_polygon = [True for _ in range(npoly)]
# count number of valid syms
for idx, (sym, polygon) in enumerate(zip(syms, polygons)):
if not self._valid_polygon(polygon): # exclude non-valid polygons
is_valid_polygon[idx] = False
continue
cls_list, gt_boxes = [], []
filtered_polygons = []
for idx, (sym, polygon) in enumerate(zip(syms, polygons)): # for each sym or region
if not is_valid_polygon[idx]:
continue
for s in sym: # for each sub-sym
if s in self._class_to_ind: # if in the given classes
polygon = [tuple(point) for point in polygon]
polygon_np = np.array(polygon)
x1, y1, x2, y2 = polygon_np[:, 0].min(), polygon_np[:, 1].min(), \
polygon_np[:, 0].max(), polygon_np[:, 1].max(),
x1, y1, x2, y2 = box_utils.clip_xyxy_to_image(
x1, y1, x2, y2, rows, cols
)
cls = self._class_to_ind[s]
cls_list.append(cls)
# expand too-small boxes (width or height < 20)
if x2 - x1 < 20:
cx = (x1 + x2) * 0.5
x1 = cx - 10
x2 = cx + 10
small_cnt += 1
if y2 - y1 < 20:
cy = (y1 + y2) * 0.5
y1 = cy - 10
y2 = cy + 10
small_cnt += 1
gt_boxes.append([x1, y1, x2, y2])
tmp = [list(chain.from_iterable(polygon))] # [[x1, y1], [x2, y2]] -> [[x1, y1, x2, y2]]
filtered_polygons.append(tmp)
else:
# print(s)
pass
assert len(cls_list) == len(gt_boxes) == len(filtered_polygons)
new_entry = {'file_name': file_name, 'cls_list': cls_list, 'height': rows, 'width': cols,
'polygons': filtered_polygons, 'gt_boxes': gt_boxes, 'eva_id': evaId, 'doc_name': docName,
'image_dir': image_dir, 'offset_x': offset_x, 'offset_y': offset_y, 'fold': fold,
'manufacturer': manufacturer, 'manufacturer_model': manufacturer_model,
'has_mask': has_mask}
gt_roidb.append(new_entry)
print('small boxes: ', small_cnt)
return gt_roidb
