def load_query(self):
imgs = []
with open(pjoin(self._basedir, 'query_info.txt'), 'r') as f:
for line in f:
img = {}
line_list = line.split()
self._use_absolute_file_name(img, line_list[5])
if line_list[5][1] == '6':
img['height'] = 576
img['width'] = 720
else:
img['height'] = 1080
img['width'] = 1920
x1 = float(line_list[1])
y1 = float(line_list[2])
w = float(line_list[3])
h = float(line_list[4])
box = FloatBox(x1, y1, x1 + w, y1 + h)
box.clip_by_shape([img['height'], img['width']])
img['boxes'] = []
img['boxes'].append([box.x1, box.y1, box.x2, box.y2])
img['boxes'] = np.asarray(img['boxes'], dtype='float32')
img['re_id_class'] = []
img['re_id_class'].append(line_list[0])
img['re_id_class'] = np.asarray(img['re_id_class'],
dtype='int32') + 1
# we can remove this since it's only checked in dataflow processing
# img['is_crowd'] = np.zeros(len(img['re_id_class']), dtype='int8')
imgs.append(img)
return imgs
def _add_detection_gt(self, img, add_mask):
"""
Add 'boxes', 'class', 'is_crowd' of this image to the dict, used by detection.
If add_mask is True, also add 'segmentation' in coco poly format.
"""
# ann_ids = self.coco.getAnnIds(imgIds=img['id'])
# objs = self.coco.loadAnns(ann_ids)
objs = self.coco.imgToAnns[img['id']] # equivalent but faster than the above two lines
# clean-up boxes
valid_objs = []
width = img['width']
height = img['height']
for obj in objs:
if obj.get('ignore', 0) == 1:
continue
x1, y1, w, h = obj['bbox']
# bbox is originally in float
# x1/y1 means upper-left corner and w/h means true w/h. This can be verified by segmentation pixels.
# But we do assume that (0.0, 0.0) is upper-left corner of the first pixel
box = FloatBox(float(x1), float(y1),
float(x1 + w), float(y1 + h))
box.clip_by_shape([height, width])
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 1 and box.is_box() and box.area() >= 4:
obj['bbox'] = [box.x1, box.y1, box.x2, box.y2]
valid_objs.append(obj)
if add_mask:
segs = obj['segmentation']
if not isinstance(segs, list):
assert obj['iscrowd'] == 1
obj['segmentation'] = None
else:
valid_segs = [np.asarray(p).reshape(-1, 2).astype('float32') for p in segs if len(p) >= 6]
if len(valid_segs) < len(segs):
log_once("Image {} has invalid polygons!".format(img['file_name']), 'warn')
obj['segmentation'] = valid_segs
# all geometrically-valid boxes are returned
boxes = np.asarray([obj['bbox'] for obj in valid_objs], dtype='float32') # (n, 4)
cls = np.asarray([
COCOMeta.category_id_to_class_id[obj['category_id']]
for obj in valid_objs], dtype='int32') # (n,)
is_crowd = np.asarray([obj['iscrowd'] for obj in valid_objs], dtype='int8')
# add the keys
img['boxes'] = boxes # nx4
img['class'] = cls # n, always >0
img['is_crowd'] = is_crowd # n,
if add_mask:
# also required to be float32
img['segmentation'] = [
obj['segmentation'] for obj in valid_objs]
def _add_detection_gt(self, img, add_mask):
"""
Add 'boxes', 'class', 'is_crowd' of this image to the dict, used by detection.
If add_mask is True, also add 'segmentation' in coco poly format.
"""
ann_ids = self.coco.getAnnIds(imgIds=img['id'], iscrowd=None)
objs = self.coco.loadAnns(ann_ids)
# clean-up boxes
valid_objs = []
width = img['width']
height = img['height']
for obj in objs:
if obj.get('ignore', 0) == 1:
continue
x1, y1, w, h = obj['bbox']
# bbox is originally in float
# x1/y1 means upper-left corner and w/h means true w/h. This can be verified by segmentation pixels.
# But we do assume that (0.0, 0.0) is upper-left corner of the first pixel
box = FloatBox(float(x1), float(y1), float(x1 + w), float(y1 + h))
box.clip_by_shape([height, width])
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 1 and box.is_box() and box.area() >= 4:
obj['bbox'] = [box.x1, box.y1, box.x2, box.y2]
valid_objs.append(obj)
if add_mask:
segs = obj['segmentation']
if not isinstance(segs, list):
assert obj['iscrowd'] == 1
obj['segmentation'] = None
else:
valid_segs = [
np.asarray(p).reshape(-1, 2) for p in segs
if len(p) >= 6
]
if len(valid_segs) < len(segs):
log_once(
"Image {} has invalid polygons!".format(
img['file_name']), 'warn')
obj['segmentation'] = valid_segs
# all geometrically-valid boxes are returned
boxes = np.asarray([obj['bbox'] for obj in valid_objs],
dtype='float32') # (n, 4)
cls = np.asarray([
COCOMeta.category_id_to_class_id[obj['category_id']]
for obj in valid_objs
],
dtype='int32') # (n,)
is_crowd = np.asarray([obj['iscrowd'] for obj in valid_objs],
dtype='int8')
# add the keys
img['boxes'] = boxes # nx4
img['class'] = cls # n, always >0
img['is_crowd'] = is_crowd # n,
if add_mask:
img['segmentation'] = [obj['segmentation'] for obj in valid_objs]
def load(self, add_gt=True, add_mask=False):
"""
Args:
add_gt: whether to add ground truth bounding box annotations to the dicts
add_mask: whether to also add ground truth mask
Returns:
a list of dict, each has keys including:
height, width, id, file_name,
and (if add_gt is True) boxes, class, is_crowd
"""
filenames = glob.glob(self.basedir + '/**/*.pkl')
ret = []
for index, filename in tqdm(enumerate(filenames)):
basename = os.path.basename(filename)
basename, _ = os.path.splitext(basename)
patid, idx = basename.split('_')
patid, idx = int(patid), int(idx)
with open(filename, 'rb') as f:
_file = pickle.load(f, encoding='bytes')
image_data = _file[b'image']
xoffset = (image_data.shape[0] - 200) // 2
yoffset = (image_data.shape[1] - 200) // 2
assert xoffset == yoffset
offset = xoffset
assert image_data.shape == (200, 200, 3)
image_data = image_data.astype(np.uint8)
data = {}
ret.append(data)
data['height'] = 200
data['width'] = 200
data['id'] = index
data['file_name'] = os.path.join(config.IMAGEDIR,
basename) + '.png'
num_instances = _file[b'num_instance']
# if (num_instances == 0):
# continue
gt_masks = _file[b'masks']
gt_masks = [
mask[offset:offset + 200, offset:offset + 200]
for mask in gt_masks
]
# list of numpy arrays
data['masks'] = gt_masks
assert len(data['masks']) == num_instances
labels = _file[b'label']
bboxes = _file[b'bboxes']
boxes = []
for gt in bboxes:
xmin = float(gt[0]) - offset
ymin = float(gt[1]) - offset
xmax = float(gt[2]) - offset
ymax = float(gt[3]) - offset
# box = FloatBox(xmin, ymin, xmax, ymax)
box = FloatBox(ymin, xmin, ymax, xmax)
boxes.append([box.x1, box.y1, box.x2, box.y2])
#classes = [ORGANS_MAPPING[la] for la in labels]
classes = labels
data['class'] = np.asarray(classes)
data['boxes'] = np.asarray(boxes).astype(np.float32)
#print(np.asarray(boxes).dtype)
data['is_crowd'] = np.asarray([0] * data['class'].shape[0])
return ret
def _add_detection_gt(
self, img, add_mask
): # 디텍션을 위해 박스와 클래스와 is_crowd를 만든다. 이것인 ground truth인가?@@@@@
"""
Add 'boxes', 'class', 'is_crowd' of this image to the dict, used by detection.
If add_mask is True, also add 'segmentation' in coco poly format.
"""
# ann_ids = self.coco.getAnnIds(imgIds=img['id'])
# objs = self.coco.loadAnns(ann_ids)
objs = self.coco.imgToAnns[img[
'id']] # equivalent but faster than the above two lines # id 값을 통해 이미지 객체 만든다.
# clean-up boxes
valid_objs = [] # 리스트 하나 만들어서
width = img['width'] # 이미지 객체에 대한 width 값 초기화
height = img['height'] # height 값 초기화
for obj in objs: # 객체들 중에서
if obj.get('ignore',
0) == 1: # ignore 값이 있는 딕셔너리가 있으면 뛰어넘는다. 보지 않는다.
continue
x1, y1, w, h = obj['bbox'] # 객체에서 bbox 정보를 초기화한다.
# bbox is originally in float
# x1/y1 means upper-left corner and w/h means true w/h. This can be verified by segmentation pixels.
# But we do assume that (0.0, 0.0) is upper-left corner of the first pixel
box = FloatBox(
float(x1),
float(y1), # float 박스를 만든다.(네모난 박스를 만든다.)
float(x1 + w),
float(y1 + h))
box.clip_by_shape([height, width]) # clip_by_shape함수가 뭐지?@@@@@
# Require non-zero seg area and more than 1x1 box size
if obj['area'] > 1 and box.is_box(
) and box.area() >= 4: # 객체의 너비가 1보다 크고 박스가 있고 박스의 너비가 4이상이면
obj['bbox'] = [box.x1, box.y1, box.x2,
box.y2] # 객체의 bbox는 x1,x2,y1,y2로 지정해준다.
valid_objs.append(obj) # 그리고 객체를 유효한 객체들의 리스트에 넣는다.
if add_mask: # 그리고 여기서 마스크가 있으면(mask r cnn 일때를 말한다.)
segs = obj[
'segmentation'] # 객체의 segmentation 부분을 가지고 segs 라는 변수를 초기화한다.
if not isinstance(segs, list): # segs 라는 변수가 리스트가 아닐때,
assert obj['iscrowd'] == 1 # 객체에 iscrowd가 1이면 예외처리해준다.
obj['segmentation'] = None # 객체의 segmentation이 없다? @@@@@
else:
valid_segs = [
np.asarray(p).reshape(-1, 2).astype('float32')
for p in segs if len(p) >= 6
] # segs라는 리스트에서 유효한 것들만 뽑는다.
if len(valid_segs) < len(
segs): # 근데 segs들에서 유효한 segs들이 별로 없다면
log_once("Image {} has invalid polygons!".format(
img['file_name']),
'warn') # 로그를 띄운다. 별로 없어서 warning이라고
obj['segmentation'] = valid_segs # 유효한 segs들은 객체의 segmentation에 다시 넣어준다.
# all geometrically-valid boxes are returned
boxes = np.asarray(
[obj['bbox'] for obj in valid_objs],
dtype='float32') # (n, 4) # 유효한 객체들의 bbox를 np를 통해 만든어 준다.
cls = np.asarray(
[ # 유효한 객체들로 클래스를 만든어 cls라는 변수를 만들어 준다.
COCOMeta.category_id_to_class_id[obj['category_id']]
for obj in valid_objs
],
dtype='int32') # (n,)
is_crowd = np.asarray(
[obj['iscrowd'] for obj in valid_objs], dtype='int8'
) # 유효한 객체에서 각 객체의 is_crowd 를 가지고 is_crowd라는 똑같은 이름의 변수를 초기화해준다.
# add the keys
img['boxes'] = boxes # nx4 # 박스들을 이미지 객체의 박스에 넣는다. 여기서 boxes는 아마 유효한 것들의 boxes 일것이다.
img['class'] = cls # n, always >0 # 클래스들을 이미지 객체의 클래스에 넣는다.
img['is_crowd'] = is_crowd # n, # is_crowd를 이미지 객체의 그것에 넣는다.
if add_mask: # 만약 마스크 rcnn 이라면
# also required to be float32
img['segmentation'] = [ # 세그멘테이션도 이와 동일한 맥락이다.
obj['segmentation'] for obj in valid_objs
]
def load(self, split_set='train'):
"""
Args:
split_set: ['train', 'val']
Returns:
a list of dict, each has keys including:
'height', 'width', 'id', 'file_name',
and (if split_set is 'train') 'boxes', 'class', 'is_crowd'.
"""
with timed_operation('Load Groundtruth Boxes...'):
frame_list_mat = scipy.io.loadmat(
pjoin(self._basedir, 'frame_' + split_set + '.mat'))
frame_list = frame_list_mat['img_index_' + split_set]
imgs = []
imgs_without_fg = 0
# each iteration only reads one file so it's faster
for idx, frame in enumerate(frame_list):
img = {}
self._use_absolute_file_name(img, frame[0][0])
if split_set == 'train':
if frame[0][0][1] == '6':
img['height'] = 576
img['width'] = 720
else:
img['height'] = 1080
img['width'] = 1920
anno_data = scipy.io.loadmat(
pjoin(self._annodir, frame[0][0] + '.jpg.mat'))
if 'box_new' in anno_data:
gt_bb_array = anno_data['box_new']
elif 'anno_file' in anno_data:
gt_bb_array = anno_data['anno_file']
elif 'anno_previous' in anno_data:
gt_bb_array = anno_data['anno_previous']
else:
raise Exception(frame[0][0] +
' bounding boxes info missing!')
# if true, include gts that are bg as well
# todo: since this option will rarely be used, re-id class not added yet
include_all = cfg.DATA.INCLUDE_ALL
if include_all:
img['boxes'] = []
for bb in gt_bb_array:
box = FloatBox(bb[1], bb[2], bb[1] + bb[3],
bb[2] + bb[4])
box.clip_by_shape([img['height'], img['width']])
img['boxes'].append(
[box.x1, box.y1, box.x2, box.y2])
img['boxes'] = np.asarray(img['boxes'],
dtype='float32')
img['class'] = np.ones(len(gt_bb_array))
img['re_id_class'] = np.asarray(gt_bb_array[:, 0] + 1,
dtype='int32')
img['re_id_class'][img['re_id_class'] == -1] = 1
else:
img['boxes'] = []
# the 2-class detection class, pedestrian/bg
img['class'] = []
img['re_id_class'] = []
for bb in gt_bb_array:
if bb[0] != -2:
box = FloatBox(bb[1], bb[2], bb[1] + bb[3],
bb[2] + bb[4])
box.clip_by_shape(
[img['height'], img['width']])
img['boxes'].append(
[box.x1, box.y1, box.x2, box.y2])
img['class'].append(1)
img['re_id_class'].append(bb[0])
else:
continue
if len(img['boxes']) == 0:
imgs_without_fg += 1
continue
img['boxes'] = np.asarray(img['boxes'],
dtype='float32')
img['class'] = np.asarray(img['class'], dtype='int32')
img['re_id_class'] = np.asarray(img['re_id_class'],
dtype='int32')
img['is_crowd'] = np.zeros(len(img['re_id_class']),
dtype='int8')
imgs.append(img)
print(
'Number of images without identified pedestrians: {}.'.format(
imgs_without_fg))
return imgs
