def gen_anchors(self, search_boxes, bound):
stride = cfg.STRIDE
rpn_conv_size = cfg.DET_ROI_SIZE - cfg.TEMP_ROI_SIZE + 1
K = len(cfg.RATIOS) * len(cfg.SCALES)
box_anchors = G.gen_region_anchors(self.raw_anchors,
search_boxes,
bound,
stride=stride,
K=K,
rpn_conv_size=rpn_conv_size)
return box_anchors
def best_search_box_train(temp_box, det_box, templates, raw_anchors, bound, K,
size, fg_thresh):
tmpl_sz = templates[:, 0]
x1, y1, x2, y2 = temp_box[:]
tw, th = x2 - x1 + 1, y2 - y1 + 1
cx, cy = x1 + 0.5 * tw, y1 + 0.5 * th
ind1 = np.where(np.bitwise_and(tmpl_sz >= tw, tmpl_sz >= th) == 1)[0]
_templates = templates[ind1]
overlaps_list = []
tmpls = []
shift_tmpls = []
anchors_list = []
for tmpl in _templates:
sz_half = tmpl[0] * 0.5
cx = min(max(cx, sz_half), bound[0] - sz_half)
cy = min(max(cy, sz_half), bound[1] - sz_half)
shift_tmpl = np.array(
[cx - sz_half, cy - sz_half, cx + sz_half, cy + sz_half],
dtype=np.float32)
_anchors = gen_region_anchors(raw_anchors, shift_tmpl.reshape(1, -1),
bound, K, size)[0]
overlaps = bbox_overlaps_per_image(_anchors,
det_box.reshape(1, -1)).ravel()
fg_inds = np.where(overlaps >= fg_thresh)[0]
overlaps_list.append(fg_inds.size)
tmpls.append(tmpl)
shift_tmpls.append(shift_tmpl)
anchors_list.append(_anchors)
overlaps_list = np.array(overlaps_list)
best_ind = np.argmax(overlaps_list)
max_overlaps_num = overlaps_list[best_ind]
replicas = np.where(overlaps_list == max_overlaps_num)[0]
if replicas.size == 1:
best_tmpl = shift_tmpls[best_ind]
return best_tmpl, max_overlaps_num, anchors_list[best_ind]
else:
if replicas.size == 0:
print(overlaps_list)
print(max_overlaps_num)
print(replicas)
assert 0, 'Weird error, replicas is zero'
tmpls = np.array(tmpls)
raw_tmpl_inds = np.arange(tmpls.shape[0])
tmpls = tmpls[replicas]
best_ind, best_tmpl = best_search_box_test(tmpls, temp_box, bound)
return best_tmpl, max_overlaps_num, anchors_list[
raw_tmpl_inds[replicas][best_ind]]
def __init__(self, im_width, im_height, batch_size=8):
self.im_w = im_width
self.im_h = im_height
self.batch_size = batch_size
self.stride = cfg.STRIDE
self.bound = (im_width, im_height)
self.out_size = (self.im_w // self.stride, self.im_h // self.stride)
self.fetch_config()
self.raw_anchors = G.generate_anchors(self.basic_size, self.ratios,
self.scales)
dummy_search_box = np.array([[0, 0, self.im_w - 1, self.im_h - 1]])
self.anchors=G.gen_region_anchors(self.raw_anchors, \
dummy_search_box, self.bound, K=self.K, size=self.out_size)[0]
cfg.TEST.IMS_PER_BATCH = 1
cfg.TEST.NMS_THRESH = 0.5
cfg.TEST.RPN_NMS_THRESH = 0.7
# cfg.TEST.RPN_POST_NMS_TOP_N=300
K = len(ratios) * len(scales)
raw_anchors = G.generate_anchors(basic_size, ratios, scales)
bound = (im_width, im_height)
out_size = (im_width // stride, im_height // stride)
dummy_search_box = np.array([0, 0, bound[0], bound[1]]).reshape(1, -1)
anchors = G.gen_region_anchors(raw_anchors,
dummy_search_box,
bound,
K=K,
size=out_size)[0]
print(anchors.shape)
img_files = [
'img00337.jpg', 'img00832.jpg', 'img00995.jpg', 'img01879.jpg',
'road.jpg'
]
# img_files=['road.jpg']
model_path = './ckpt/model_660000.pkl'
model = FasterRCNN(im_width, im_height, pretrained=False)
model.load_weights(model_path=model_path)
model.cuda()
def __init__(self, im_width, im_height, batch_size=8):
self.data_dir = op.join(cfg.DATA_DIR, 'Insight-MVT_Annotation_Train')
self.anno_dir = op.join(cfg.DATA_DIR, 'DETRAC-Train-Annotations-XML')
self.stride = cfg.STRIDE
self.basic_size = cfg.BASIC_SIZE
self.ratios = cfg.RATIOS
self.scales = cfg.SCALES
self.track_basic_size = cfg.TRACK_BASIC_SIZE
self.track_ratios = cfg.TRACK_RATIOS
self.track_scales = cfg.TRACK_SCALES
self.K = len(self.ratios) * len(self.scales)
self.TK = len(self.track_ratios) * len(self.track_scales)
self.rpn_conv_size = cfg.RPN_CONV_SIZE
self.img_dirs = sorted(os.listdir(self.data_dir))
self.anno_files = sorted(os.listdir(self.anno_dir))
for ext_seq in EXTRA_SEQS:
ext_anno = '{}.xml'.format(ext_seq)
# assert ext_anno in self.anno_files, '{} not exists'.format(ext_anno)
self.anno_files.remove(ext_anno)
self.img_dirs.remove(ext_seq)
self.index = 0
self.vis_dir = './vis_vid'
self.vis_index = 0
self.margin_gain = 0.2
self.im_w = im_width
self.im_h = im_height
self.bound = (im_width, im_height)
self.out_size = (im_width // self.stride, im_height // self.stride)
self.batch_size = batch_size
self.roi_size = cfg.DET_ROI_SIZE - cfg.TEMP_ROI_SIZE + 1
self.num_sequences = len(self.anno_files)
self.num_images = 0
self.num_visualize = 100
self.permute_inds = np.random.permutation(np.arange(
self.num_sequences))
self.max_interval = 4 if cfg.PHASE == 'TRAIN' else 1
self.iter_stop = False
self.enum_sequences()
self.templates = get_template(min_size=cfg.TEMP_MIN_SIZE,
max_size=cfg.TEMP_MAX_SIZE,
num_templates=cfg.TEMP_NUM)
self.raw_anchors = G.generate_anchors(self.basic_size, self.ratios,
self.scales)
dummy_search_box = np.array([[0, 0, self.im_w - 1, self.im_h - 1]])
self.det_anchors = G.gen_region_anchors(self.raw_anchors,
dummy_search_box,
self.bound,
K=self.K,
size=self.out_size)[0]
self.track_raw_anchors = G.generate_anchors(self.track_basic_size,
self.track_ratios,
self.track_scales)
def get_track_output(output_dict, configs):
K = configs['K']
temp_boxes = configs['temp_boxes']
search_boxes = configs['search_boxes']
rpn_conv_size = configs['rpn_conv_size']
raw_anchors = configs['raw_anchors']
bound = configs['bound']
track_rpn_logits = output_dict['track_rpn_logits']
track_rpn_bbox = output_dict['track_rpn_bbox']
num_targets = track_rpn_logits.shape[0]
track_rpn_cls = F.softmax(track_rpn_logits, dim=1).cpu().data.numpy()
track_rpn_cls = track_rpn_cls[:, 1, :, :].reshape(
num_targets, K, rpn_conv_size,
rpn_conv_size).transpose(0, 2, 3, 1).reshape(num_targets, -1)
track_rpn_bbox = track_rpn_bbox.cpu().data.numpy().transpose(
0, 2, 3, 1).reshape(num_targets, -1, 4)
bboxes_list = []
anchors_list = []
for i in range(num_targets):
temp_box = temp_boxes[i]
temp_cx = 0.5 * (temp_box[0] + temp_box[2])
temp_cy = 0.5 * (temp_box[1] + temp_box[3])
rpn_cls = track_rpn_cls[i]
rpn_bbox = track_rpn_bbox[i]
# print(rpn_cls.size)
# print(rpn_bbox.shape[0])
order = np.argsort(rpn_cls)[::-1]
_anchors = G.gen_region_anchors(raw_anchors,
search_boxes[i].reshape(1, -1),
bound,
K,
size=(rpn_conv_size, rpn_conv_size))[0]
top = 1
# print(rpn_cls[order[:top]])
fg_anchors = _anchors[order[:top]]
fg_rpn_bbox = rpn_bbox[order[:top]]
if cfg.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
fg_rpn_bbox *= cfg.BBOX_STD_DEV
bboxes = bbox_transform_inv(fg_anchors, fg_rpn_bbox)
'''
best_id=select_proposals_cosine(temp_boxes[i], bboxes, rpn_cls[order[:top]], order[:top], rpn_conv_size, K)
anchors_list.append(fg_anchors[best_id][np.newaxis,:])
bboxes_list.append(bboxes[best_id][np.newaxis,:])
'''
if np.max(rpn_cls[order[:top]]) < cfg.TRACK_SCORE_THRESH:
bboxes_list.append([])
else:
cx = 0.5 * (bboxes[:, 0] + bboxes[:, 2])
cy = 0.5 * (bboxes[:, 1] + bboxes[:, 3])
dist_cx = np.abs(cx - temp_cx)
dist_cy = np.abs(cy - temp_cy)
fg_inds = np.where(
np.bitwise_and(dist_cx < cfg.TRACK_MAX_DIST,
dist_cy < cfg.TRACK_MAX_DIST) == 1)[0]
bboxes_list.append(bboxes[fg_inds])
# bboxes_list.append(bboxes)
anchors_list.append(fg_anchors)
ret = {}
ret['bboxes_list'] = bboxes_list
ret['anchors_list'] = anchors_list
return ret
def get_minibatch_inter_seq(self):
if self.iter_stop:
self.iter_stop = False
return None
roidbs = []
index_res = self.index_per_seq - self.upper_bound_per_seq
'''permute'''
index_res = index_res[self.permute_inds]
valid_seq_inds = np.where(index_res <= 0)[0]
num_valid_seqs = len(valid_seq_inds)
subdir_inds = [
self.permute_inds[valid_seq_inds[i % num_valid_seqs]]
for i in range(self.index, self.index + self.batch_size)
]
chosen_samples_per_seq = {}
for i in subdir_inds:
if i not in chosen_samples_per_seq.keys():
chosen_samples_per_seq[i] = 0
chosen_samples_per_seq[i] += 1
img_dirs = []
anno_dirs = []
img_files = []
anno_files = []
cur_image_inds = []
for i in subdir_inds:
'''already permuted'''
# img_dirs.append(op.join(self.vid_dir, self.img_dirs[i]))
img_dirs.append(op.join(self.vid_dir, self.anno_dirs[i]))
anno_dirs.append(op.join(self.annot_dir, self.anno_dirs[i]))
img_files.append(sorted(os.listdir(img_dirs[-1])))
anno_files.append(sorted(os.listdir(anno_dirs[-1])))
cur_image_inds.append(self.index_per_seq[i])
ref_inds = np.asarray(cur_image_inds)
tmp = copy.deepcopy(chosen_samples_per_seq)
for i in range(len(ref_inds)):
seq_ind = subdir_inds[i]
ref_inds[i] += (tmp[seq_ind] - chosen_samples_per_seq[seq_ind])
chosen_samples_per_seq[seq_ind] -= 1
real_num_samples = ref_inds.size
interval = np.random.randint(1,
self.max_interval + 1,
size=real_num_samples)
det_inds = ref_inds + interval
det_inds = np.minimum(det_inds, self.images_per_seq[subdir_inds] - 1)
non_empty_batch = False
for batch_index, inds in enumerate(zip(ref_inds, det_inds)):
if len(img_files[batch_index]) == 0:
continue
roidb = {}
ref_boxes = {}
det_boxes = {}
temp_image = None
det_image = None
for ind in inds:
img_file = img_files[batch_index][ind]
anno_file = anno_files[batch_index][ind]
img_index = img_file[:img_file.rfind('.')]
anno_index = anno_file[:anno_file.rfind('.')]
assert img_index == anno_index, 'Index not uniformed'
image = cv2.imread(
op.join(img_dirs[batch_index],
img_files[batch_index][ind]))
image, (xstart, ystart), scale = util.resize_and_pad_image(
image, self.im_w, self.im_h)
xml_file = op.join(anno_dirs[batch_index],
anno_files[batch_index][ind])
tree = ET.parse(xml_file)
root = tree.getroot()
objects = root.findall('object')
if len(objects) == 0:
# print('{} has no targets'.format(op.join(img_dirs[batch_index], img_files[batch_index][ind])))
if DEBUG and self.vis_index < self.num_visualize:
cv2.imwrite(
op.join('mot_no_target',
img_files[batch_index][ind]), image)
for obj in objects:
obj_id = int(obj.find('trackid').text)
bndbox = obj.find('bndbox')
bbox = np.zeros(4, dtype=np.float32)
bbox[0] = float(bndbox.find('xmin').text)
bbox[1] = float(bndbox.find('ymin').text)
bbox[2] = float(bndbox.find('xmax').text)
bbox[3] = float(bndbox.find('ymax').text)
bbox *= scale
bbox[[0, 2]] += xstart
bbox[[1, 3]] += ystart
if ind == inds[0]:
ref_boxes[obj_id] = bbox
else:
det_boxes[obj_id] = bbox
if ind == inds[0] and len(ref_boxes.keys()) > 0:
temp_image = image[np.newaxis, :, :, :].astype(np.float32)
non_empty_batch = True
elif ind == inds[1] and len(ref_boxes.keys()) > 0:
det_image = image[np.newaxis, :, :, :].astype(np.float32)
ref_boxes_align, det_boxes_align = self.align_boxes(
ref_boxes, det_boxes)
if len(ref_boxes_align) > 0:
roidb['raw_temp_boxes'] = ref_boxes_align
temp_boxes = util.compute_template_boxes(
ref_boxes_align,
(temp_image.shape[2], temp_image.shape[1]),
gain=self.margin_gain,
shape='same')
roidb['temp_image'] = temp_image
roidb['det_image'] = det_image
roidb['temp_boxes'] = temp_boxes
roidb['det_boxes'] = det_boxes_align
'''NHWC'''
bound = (det_image.shape[2], det_image.shape[1])
search_boxes = util.calc_search_boxes(temp_boxes, bound)
roidb['search_boxes'] = search_boxes
roidb['bound'] = bound
'''
Next
roidb['bbox_overlaps']
'''
# anchors=self.gen_anchors(search_boxes, bound)
anchors = G.gen_region_anchors(self.raw_anchors,
search_boxes,
bound,
stride=self.stride,
K=self.K,
rpn_conv_size=self.roi_size)
bbox_overlaps = U.bbox_overlaps_per_image(
np.vstack(anchors), det_boxes_align)
roidb['anchors'] = anchors
roidb['bbox_overlaps'] = bbox_overlaps
roidbs.append(roidb)
'''
[NHWC,NHWC]
'''
# assert len(ref_images)==len(box_sizes), 'Images and size array must have the same length: {} v.s. {}'.format(len(ref_images), len(box_sizes))
for ind, v in tmp.items():
self.index_per_seq[ind] += v
index_res = self.index_per_seq - self.upper_bound_per_seq
index_res = index_res[self.permute_inds[[valid_seq_inds]]]
invalid_seq_inds = np.where(index_res > 0)[0]
num_sequences = num_valid_seqs - len(invalid_seq_inds)
if num_sequences == 0:
self.index_per_seq = np.zeros(self.num_sequences, dtype=np.int32)
self.round_per_seq = np.zeros(self.num_sequences, dtype=np.int32)
self.inds = np.random.permutation(np.arange(self.num_sequences))
self.index = 0
self.iter_stop = True
else:
self.index += self.batch_size
if self.index >= num_sequences:
self.index = 0
else:
subtract_inds = np.where(invalid_seq_inds <= self.index)[0]
self.index -= len(subtract_inds)
return roidbs, non_empty_batch
def get_minibatch_inter_img(self):
if self.iter_stop:
self.iter_stop = False
return None
roidbs = []
index = self.permute_inds[self.index]
while self.index_per_seq[index] > self.upper_bound_per_seq[index]:
self.index += 1
if self.index == self.num_sequences:
self.index = 0
index = self.permute_inds[self.index]
anno_dir = op.join(self.annot_dir, self.anno_dirs[index])
img_dir = op.join(self.vid_dir, self.anno_dirs[index])
img_files = sorted(os.listdir(img_dir))
anno_files = sorted(os.listdir(anno_dir))
cur_image_index = self.index_per_seq[index]
ref_inds = np.arange(cur_image_index,
cur_image_index + self.batch_size)
real_num_samples = ref_inds.size
interval = np.random.randint(1,
self.max_interval + 1,
size=real_num_samples)
det_inds = ref_inds + interval
det_inds = np.minimum(det_inds, self.images_per_seq[index] - 1)
for _, inds in enumerate(zip(ref_inds, det_inds)):
roidb = {}
ref_boxes = {}
det_boxes = {}
temp_image = None
det_image = None
for ind in inds:
img_file = img_files[ind]
anno_file = anno_files[ind]
img_index = img_file[:img_file.rfind('.')]
anno_index = anno_file[:anno_file.rfind('.')]
assert img_index == anno_index, 'Index not uniformed'
image = cv2.imread(op.join(img_dir, img_file))
# image, (xstart, ystart), scale=util.resize_and_pad_image(image, self.im_w, self.im_h)
h, w = image.shape[0:2]
image = cv2.resize(image, (self.im_w, self.im_h),
interpolation=cv2.INTER_LINEAR)
nh, nw = image.shape[0:2]
yscale = 1.0 * nh / h
xscale = 1.0 * nw / w
xml_file = op.join(anno_dir, anno_files[ind])
tree = ET.parse(xml_file)
root = tree.getroot()
objects = root.findall('object')
if len(objects) == 0:
# print('{} has no targets'.format(op.join(img_dirs[batch_index], img_files[batch_index][ind])))
if DEBUG and self.vis_index < self.num_visualize:
cv2.imwrite(op.join('mot_no_target', img_files[ind]),
image)
for obj in objects:
obj_id = int(obj.find('trackid').text)
bndbox = obj.find('bndbox')
bbox = np.zeros(4, dtype=np.float32)
bbox[0] = float(bndbox.find('xmin').text)
bbox[1] = float(bndbox.find('ymin').text)
bbox[2] = float(bndbox.find('xmax').text)
bbox[3] = float(bndbox.find('ymax').text)
'''
bbox*=scale
bbox[[0,2]]+=xstart
bbox[[1,3]]+=ystart
'''
bbox[[0, 2]] *= xscale
bbox[[1, 3]] *= yscale
if ind == inds[0]:
ref_boxes[obj_id] = bbox
else:
det_boxes[obj_id] = bbox
if ind == inds[0] and len(ref_boxes.keys()) > 0:
temp_image = image[np.newaxis, :, :, :].astype(np.float32)
elif ind == inds[1] and len(ref_boxes.keys()) > 0:
det_image = image[np.newaxis, :, :, :].astype(np.float32)
ref_boxes_align, det_boxes_align = self.align_boxes(
ref_boxes, det_boxes)
if len(ref_boxes_align) > 0:
roidb['raw_temp_boxes'] = ref_boxes_align
# temp_boxes=util.compute_template_boxes(ref_boxes_align, (temp_image.shape[2], temp_image.shape[1]), gain=self.margin_gain, shape='same')
roidb['temp_image'] = temp_image
roidb['det_image'] = det_image
# roidb['temp_boxes']=temp_boxes
roidb['temp_boxes'] = ref_boxes_align
roidb['det_boxes'] = det_boxes_align
'''NHWC'''
bound = (det_image.shape[2], det_image.shape[1])
search_boxes = util.calc_search_boxes(ref_boxes_align, bound)
roidb['search_boxes'] = search_boxes
roidb['bound'] = bound
'''
Next
roidb['bbox_overlaps']
'''
# anchors=self.gen_anchors(search_boxes, bound)
anchors = G.gen_region_anchors(self.raw_anchors,
search_boxes,
bound,
stride=self.stride,
K=self.K,
rpn_conv_size=self.roi_size)
bbox_overlaps = U.bbox_overlaps_per_image(
np.vstack(anchors), det_boxes_align)
roidb['anchors'] = anchors
roidb['bbox_overlaps'] = bbox_overlaps
roidbs.append(roidb)
'''
[NHWC,NHWC]
'''
self.index_per_seq[index] += self.batch_size
index_res = self.index_per_seq - self.upper_bound_per_seq
index_res = index_res[self.permute_inds]
valid_seq_inds = np.where(index_res <= 0)[0]
if valid_seq_inds.size == 0:
self.index_per_seq = np.zeros(self.num_sequences, dtype=np.int32)
self.round_per_seq = np.zeros(self.num_sequences, dtype=np.int32)
self.permute_inds = np.random.permutation(
np.arange(self.num_sequences))
self.index = 0
self.iter_stop = True
else:
self.index += 1
if self.index == self.num_sequences:
self.index = 0
return roidbs
def main(dataset_obj, model=None):
loader = DataLoader(dataset_obj)
temp_boxes = None
temp_image = None
det_image = None
started = False
track_started = False
num_instances = 0
track_ids = None
VIS_DATASET = False
TRACK_LAST_FRAME = False
# writer=cv2.VideoWriter('./track.avi', cv2.VideoWriter_fourcc('M','J','P','G'), 30.0, (im_width, im_height))
frame_ind = 0
for _, (image, gt_boxes) in enumerate(loader):
canvas = image.copy()
if VIS_DATASET:
if gt_boxes is not None:
temp_boxes = np.asarray(gt_boxes[:4]).reshape(-1, 4)
for i, box in enumerate(temp_boxes):
box = box.astype(np.int32)
cv2.rectangle(canvas, (box[0], box[1]), (box[2], box[3]),
tuple(colors[i]), 2)
else:
image = image.astype(np.float32)
if len(gt_boxes) == 0 and not started:
print('Waiting for a frame with gt_box')
continue
started = True
if not track_started:
temp_boxes = np.asarray(gt_boxes[:, :4]).reshape(-1, 4).astype(
np.float32)
temp_image = image.copy()
for i, box in enumerate(gt_boxes):
box = box.astype(np.int32)
cv2.rectangle(canvas, (box[0], box[1]), (box[2], box[3]),
tuple(colors[i]), 2)
track_started = True
num_instances = temp_boxes.shape[0]
track_ids = np.ones(num_instances, dtype=np.int32)
else:
'''replace with detection'''
if not TRACK_LAST_FRAME and len(gt_boxes) > num_instances:
track_ids, temp_boxes = add_new_targets(
track_ids, temp_boxes,
np.arange(num_instances, len(gt_boxes)),
gt_boxes[num_instances:])
print('add {} new targets'.format(
len(gt_boxes) - num_instances))
num_instances = len(track_ids)
if TRACK_LAST_FRAME:
num_instances = len(temp_boxes)
det_image = image.copy()
roidb = dict()
bound = (temp_image.shape[1], temp_image.shape[0])
roidb['bound'] = bound
roidb['temp_image'] = temp_image[np.newaxis, :, :, :].astype(
np.float32)
roidb['det_image'] = det_image[np.newaxis, :, :, :].astype(
np.float32)
roidb['good_inds'] = check_boxes(temp_boxes)
roidb['temp_boxes'] = temp_boxes
search_boxes = []
for temp_box in temp_boxes:
_, best_template = butil.best_search_box_test(
templates, temp_box, bound)
search_boxes.append(best_template)
search_boxes = np.array(search_boxes)
# print(search_boxes)
roidb['det_boxes'] = None
roidb['det_classes'] = None
roidb['temp_classes'] = None
roidb['search_boxes'] = search_boxes
# anchors=G.gen_region_anchors(track_raw_anchors, search_boxes, bound, K=TK, size=(rpn_conv_size, rpn_conv_size))[0]
# roidb['track_anchors']=anchors
dummy_search_box = np.array(
[[0, 0, im_width - 1, im_height - 1]])
anchors = G.gen_region_anchors(det_raw_anchors,
dummy_search_box,
bound,
K=K,
size=out_size)[0]
roidb['anchors'] = anchors
bboxes_list, _ = inference_track(model, roidb)
last_temp_boxes = temp_boxes.copy()
if not TRACK_LAST_FRAME and len(bboxes_list) > 0:
valid_boxes = np.zeros((0, 4), dtype=np.float32)
for i, bboxes in enumerate(bboxes_list):
if len(bboxes) > 0:
if len(bboxes.shape) > 0:
bboxes = bboxes.squeeze()
bbox = bboxes[:4]
valid_boxes = np.append(valid_boxes,
bbox.reshape(1, -1), 0)
if valid_boxes.shape[0] > 0:
temp_boxes = valid_boxes.copy()
index = 0
for i in range(num_instances):
if not track_ids[i]:
continue
boxes_one_target = bboxes_list[index]
if len(boxes_one_target) == 0:
track_ids[i] = 0
print('target {} disappears'.format(i))
else:
box_inst = bboxes_list[index][0].astype(np.int32)
cv2.rectangle(canvas, (box_inst[0], box_inst[1]),
(box_inst[2], box_inst[3]),
tuple(colors[i % len(colors)]), 2)
index += 1
num_exist_instances = len(np.where(track_ids != 0)[0])
for i in range(num_exist_instances):
temp_box = last_temp_boxes[i].astype(np.int32)
cv2.rectangle(canvas, (temp_box[0], temp_box[1]),
(temp_box[2], temp_box[3]), (0, 0, 255), 1)
if TRACK_LAST_FRAME:
temp_boxes = np.asarray(gt_boxes[:, :4]).reshape(
-1, 4).astype(np.float32)
temp_image = image.copy()
frame_ind += 1
print('Frame {}'.format(frame_ind))
cv2.imshow('benchmark', canvas)
# writer.write(canvas)
if cv2.waitKey(1) == 27:
break
def get_minibatch(self):
if self.iter_stop:
self.iter_stop = False
return None
roidbs = []
index = self.inds[self.index]
while self.index_per_seq[index] > self.upper_bound_per_seq[index]:
self.index += 1
if self.index == self.num_sequences:
self.index = 0
index = self.inds[self.index]
img_dir = op.join(self.mot_dir, MOT_SUBDIRS[index], 'img1')
annotation = self.gt_annotations[MOT_SUBDIRS[index]]
img_files = sorted(os.listdir(img_dir))
'''
if MAX_SAMPLES>0:
max_samples=min(MAX_SAMPLES, len(img_files))
img_files=img_files[:max_samples]
'''
cur_image_index = self.index_per_seq[index]
temp_inds = np.arange(cur_image_index,
cur_image_index + self.batch_size)
real_num_samples = temp_inds.size
interval = np.random.randint(1,
self.max_interval + 1,
size=real_num_samples)
det_inds = temp_inds + interval
det_inds = np.minimum(det_inds, self.images_per_seq[index] - 1)
# non_empty_batch=False
for _, inds in enumerate(zip(temp_inds, det_inds)):
roidb = {}
temp_boxes = {}
det_boxes = {}
temp_image = None
det_image = None
temp_gt_classes = {}
det_gt_classes = {}
for ind in inds:
image = cv2.imread(op.join(img_dir, img_files[ind]))
h, w = image.shape[0:2]
image = cv2.resize(image, (self.im_w, self.im_h),
interpolation=cv2.INTER_LINEAR)
nh, nw = image.shape[0:2]
yscale = 1.0 * nh / h
xscale = 1.0 * nw / w
objects = annotation[ind]
# obj_boxes=np.zeros((0,4),dtype=np.float32)
if len(objects) == 0:
print('{} has no targets'.format(
op.join(img_dir, img_files[ind])))
if DEBUG and self.vis_index < self.num_visualize:
cv2.imwrite(op.join('mot_no_target', img_files[ind]),
image)
for obj in objects:
obj_id = obj['id']
bbox = obj['bbox'].copy()
bbox[[0, 2]] *= xscale
bbox[[1, 3]] *= yscale
self.clip_boxes(bbox)
bbox = bbox.astype(np.int32)
# obj_boxes=np.append(obj_boxes, bbox[np.newaxis,:], 0)
if ind == inds[0]:
temp_boxes[obj_id] = bbox
temp_gt_classes[obj_id] = 1
else:
det_boxes[obj_id] = bbox
det_gt_classes[obj_id] = 1
# if self.vis_index < self.num_visualize:
# cv2.rectangle(image_cpy, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 0, 255), 2)
if ind == inds[0] and len(temp_boxes.keys()) > 0:
temp_image = image[np.newaxis, :, :, :].astype(np.float32)
# non_empty_batch=True
elif ind == inds[1] and len(temp_boxes.keys()) > 0:
det_image = image[np.newaxis, :, :, :].astype(np.float32)
ref_boxes_align,det_boxes_align,ref_classes_align,det_classes_align=butil.align_boxes(temp_boxes, \
det_boxes, temp_gt_classes, det_gt_classes)
# print(ref_boxes_align)
# print(det_boxes_align)
good_inds = []
if len(ref_boxes_align) > 0:
roidb['raw_temp_boxes'] = ref_boxes_align
roidb['temp_image'] = temp_image
roidb['det_image'] = det_image
roidb['temp_boxes'] = ref_boxes_align
roidb['det_boxes'] = det_boxes_align
roidb['temp_classes'] = ref_classes_align
roidb['det_classes'] = det_classes_align
'''NHWC'''
bound = (det_image.shape[2], det_image.shape[1])
search_boxes = np.zeros((0, 4), dtype=np.float32)
num_fg_anchors = -1
best_track_anchors = None
best_ind = 0
for i, box in enumerate(ref_boxes_align):
if cfg.PHASE == 'TEST':
_, search_box = butil.best_search_box_test(
self.templates, box, bound)
else:
search_box, max_overlap_num, best_anchors = butil.best_search_box_train(
box, det_boxes_align[i], self.templates,
self.track_raw_anchors, bound, self.TK,
(self.rpn_conv_size, self.rpn_conv_size),
cfg.TRAIN.TRACK_RPN_POSITIVE_THRESH)
if max_overlap_num > num_fg_anchors:
num_fg_anchors = max_overlap_num
best_track_anchors = best_anchors
best_ind = i
search_boxes = np.append(search_boxes,
search_box.reshape(1, -1), 0)
good_inds.append(i)
roidb['search_boxes'] = search_boxes
roidb['bound'] = bound
roidb['good_inds'] = good_inds
if cfg.PHASE == 'TRAIN':
roidb['best_anchors'] = best_track_anchors
roidb['best_ind'] = best_ind
dummy_search_box = np.array(
[[0, 0, self.im_w - 1, self.im_h - 1]])
anchors = G.gen_region_anchors(self.raw_anchors,
dummy_search_box,
bound,
K=self.K,
size=self.out_size)
'''detectio anchors'''
roidb['anchors'] = anchors[0]
roidbs.append(roidb)
'''
[NHWC,NHWC]
'''
self.index_per_seq[index] += self.batch_size
index_res = self.index_per_seq[self.inds] - self.upper_bound_per_seq[
self.inds]
self.valid_seq_inds = np.nonzero(index_res <= 0)[0]
if self.valid_seq_inds.size == 0:
self.index_per_seq = np.zeros(self.num_sequences, dtype=np.int32)
self.round_per_seq = np.zeros(self.num_sequences, dtype=np.int32)
self.inds = np.random.permutation(np.arange(self.num_sequences))
self.index = 0
self.iter_stop = True
else:
self.index += 1
if self.index == self.num_sequences:
self.index = 0
if self.batch_size == 1:
if len(roidbs) == 0:
return {}
else:
return roidbs[0]
else:
return roidbs