def get_label(self, track_res, thr_iou=0.2, need_iou=False):
anchors = self.state['p'].anchor
anchor_num = anchors.shape[0]
clss = np.zeros((anchor_num, ), dtype=np.int64)
delta = np.zeros((4, anchor_num), dtype=np.float32)
tcx, tcy, tw, th = track_res
cx, cy, w, h = anchors[:,
0] + 127, anchors[:,
1] + 127, anchors[:,
2], anchors[:,
3]
x1, y1, x2, y2 = center2corner(np.array((cx, cy, w, h)))
# delta
delta[0] = (tcx - cx) / w
delta[1] = (tcy - cy) / h
delta[2] = np.log(tw / w)
delta[3] = np.log(th / h)
# IoU
overlap = IoU([x1, y1, x2, y2], center2corner(track_res))
pos = np.where(overlap > thr_iou)
clss[pos] = 1
if not need_iou:
return clss, delta
else:
return clss, delta, overlap
def show_pscore_delta(self, pscore, delta, track_bbox, fig_num='pscore_delta'):
if torch.is_tensor(delta):
delta = delta.detach().cpu().numpy()
if not len(delta.shape) == 3:
delta = delta.reshape((-1, 4, 3125))
anchor = self.model.all_anchors.detach().cpu().numpy()
cx = delta[:, 0, :] * anchor[:, 2] + anchor[:, 0]
cy = delta[:, 1, :] * anchor[:, 3] + anchor[:, 1]
w = np.exp(delta[:, 2, :]) * anchor[:, 2]
h = np.exp(delta[:, 3, :]) * anchor[:, 3]
iou_list = list()
for i in range(track_bbox.shape[0]):
tx, ty, tw, th = track_bbox[i]
tcx, tcy = tx+tw/2, ty+th/2
overlap = IoU(center2corner(np.array((cx[i]+127,cy[i]+127,w[i],h[i]))), center2corner(np.array((tcx,tcy,tw,th))))
iou_list.append(overlap)
ious = np.array(iou_list) # (B, 3125)
ious_img = ious.mean(axis=0).reshape(-1, 25)# (B*5, 25)
fig, axes = plt.subplots(1,2,num=fig_num)
ax = axes[0]
ax.set_title('pscore')
ax.imshow(pscore.detach().reshape(-1, 3125).mean(dim=0).reshape(-1, 25).cpu().numpy(),
vmin=0, vmax=1)
ax = axes[1]
ax.set_title('iou: {:.2f}'.format(ious.max()))
ax.imshow(ious_img, vmin=0, vmax=1)
plt.pause(0.001)
return ious, ious_img
def get_label(self, track_bbox, thr_iou=0.2, need_iou=False):
'''Input track_bbox: np.array of size (B, 4)
Return np.array type '''
anchors = self.model.anchor.all_anchors[1].reshape(4, -1).transpose(1, 0)
anchor_num = anchors.shape[0]
cls_list, delta_list, iou_list = list(), list(), list()
for i in range(track_bbox.shape[0]):
tx, ty, tw, th = track_bbox[i]
tcx, tcy = tx+tw/2, ty+th/2
cx, cy, w, h = anchors[:,0]+127, anchors[:,1]+127, anchors[:,2], anchors[:,3]
clss = np.zeros((anchor_num,), dtype=np.int64)
delta = np.zeros((4, anchor_num), dtype=np.float32)
# delta
delta[0] = (tcx - cx) / w
delta[1] = (tcy - cy) / h
delta[2] = np.log(tw / w)
delta[3] = np.log(th / h)
# IoU
overlap = IoU(center2corner(np.array((cx,cy,w,h))), center2corner(np.array((tcx,tcy,tw,th))))
pos = np.where(overlap > thr_iou)
clss[pos] = 1
cls_list.append(clss)
delta_list.append(delta)
iou_list.append(overlap)
#
# fig = plt.figure('Label')
# for i in range(track_bbox.shape[0]):
# ax = fig.add_subplot(1, track_bbox.shape[0], i+1)
# tx, ty, tw, th = track_bbox[i]
# ax.imshow(kornia.tensor_to_image(self.model.search_cropped[i].byte()))
# rect = patches.Rectangle((tx, ty), tw, th, linewidth=2, edgecolor='r', facecolor='none')
# ax.add_patch(rect)
# for i in range(anchors.shape[0]):
# cx, cy, w, h = anchors[i,0], anchors[i,1], anchors[i,2], anchors[i,3]
# bb_center = patches.Circle((cx+127, cy+127), color='b', radius=0.2)
# ax.add_patch(bb_center)
# for i in range(anchors.shape[0]):
# if clss[i]==1:
# cx, cy, w, h = anchors[i,0], anchors[i,1], anchors[i,2], anchors[i,3]
# bb_center = patches.Circle((cx+127, cy+127), color='r', radius=0.2)
# ax.add_patch(bb_center)
# plt.show()
if not need_iou:
return np.array(cls_list), np.array(delta_list)
else:
return np.array(cls_list), np.array(delta_list), np.array(iou_list)
def generate_all_anchors(self, im_c, size):
if self.image_center == im_c and self.size == size:
return False
self.image_center = im_c
self.size = size
a0x = im_c - size // 2 * self.stride
ori = np.array([a0x] * 4, dtype=np.float32)
zero_anchors = self.anchors + ori
x1 = zero_anchors[:, 0]
y1 = zero_anchors[:, 1]
x2 = zero_anchors[:, 2]
y2 = zero_anchors[:, 3]
x1, y1, x2, y2 = map(lambda x: x.reshape(self.anchor_num, 1, 1),
[x1, y1, x2, y2])
cx, cy, w, h = corner2center([x1, y1, x2, y2])
disp_x = np.arange(0, size).reshape(1, 1, -1) * self.stride
disp_y = np.arange(0, size).reshape(1, -1, 1) * self.stride
cx = cx + disp_x
cy = cy + disp_y
# broadcast
zero = np.zeros((self.anchor_num, size, size), dtype=np.float32)
cx, cy, w, h = map(lambda x: x + zero, [cx, cy, w, h])
x1, y1, x2, y2 = center2corner([cx, cy, w, h])
self.all_anchors = np.stack([x1, y1, x2, y2]), np.stack([cx, cy, w, h])
return True
def generate_all_anchors(self, im_c, size):
"""
生成整幅图像的anchors
:param im_c:图像的中心点
:param size:图像的尺寸
:return:
"""
if self.image_center == im_c and self.size == size:
return False
# 更新config中的内容
self.image_center = im_c
self.size = size
# anchor0 的xy 坐标，即 x 和 y 对称。
a0x = im_c - size // 2 * self.stride
# 生成anchor0的坐标
ori = np.array([a0x] * 4, dtype=np.float32)
# 以图像中心点为中心点的anchor
zero_anchors = self.anchors + ori
# 获取anchor0的坐标
x1 = zero_anchors[:, 0]
y1 = zero_anchors[:, 1]
x2 = zero_anchors[:, 2]
y2 = zero_anchors[:, 3]
x1, y1, x2, y2 = map(lambda x: x.reshape(self.anchor_num, 1, 1),
[x1, y1, x2, y2])
cx, cy, w, h = corner2center([x1, y1, x2, y2])
# disp_x是[1, 1, size]，disp_y是[1, size, 1]
disp_x = np.arange(0, size).reshape(1, 1, -1) * self.stride
disp_y = np.arange(0, size).reshape(1, -1, 1) * self.stride
# 得到整幅图像中anchor中心点的坐标
cx = cx + disp_x
cy = cy + disp_y
# 通过广播生成整幅图像的anchor broadcast
zero = np.zeros((self.anchor_num, size, size), dtype=np.float32)
cx, cy, w, h = map(lambda x: x + zero, [cx, cy, w, h])
x1, y1, x2, y2 = center2corner([cx, cy, w, h])
# 以中心点坐标,宽高和左上角、右下角坐标两种方式存储anchors
self.all_anchors = np.stack([x1, y1, x2, y2]), np.stack([cx, cy, w, h])
return True
def generate_all_anchors(self, im_c, size):
if self.image_center == im_c and self.size == size:
return False
self.image_center = im_c
self.size = size # size到底代表啥
# 下面的一行是说：size是search image经过孪生网络后、再经过depth-wise的xcorr后的feature map的大小(17x17)，从论文中可以看出stride=16(相当于经过四次下采样)
# 而feature map左上角 第一个点 对应的在输入孪生网络前的search image里的横/纵坐标值就是下面的公式计算的值
# 而计算出来的a0x就是一个锚点，一个锚点就可以产生一系列的anchors
a0x = im_c - size // 2 * self.stride
ori = np.array([a0x] * 4, dtype=np.float32)
zero_anchors = self.anchors + ori # 这里的加法存在广播机制; generate_anchors里面产生的是相对坐标，作为中心是0；
# 这里我们要给出锚点的绝对中心，才能得到绝对坐标
x1 = zero_anchors[:, 0]
y1 = zero_anchors[:, 1]
x2 = zero_anchors[:, 2]
y2 = zero_anchors[:, 3]
x1, y1, x2, y2 = map(lambda x: x.reshape(self.anchor_num, 1, 1),
[x1, y1, x2, y2])
cx, cy, w, h = corner2center([x1, y1, x2, y2])
disp_x = np.arange(0, size).reshape(1, 1, -1) * self.stride
disp_y = np.arange(0, size).reshape(1, -1, 1) * self.stride
cx = cx + disp_x
cy = cy + disp_y
# broadcast
zero = np.zeros((self.anchor_num, size, size), dtype=np.float32)
# 从这里我们可以看到，zero的第零个维度方向代表的是anchor的种类(种类包括不同的横纵比，尺度scale，密度)
# 第一、二个维度方向代表的是每个锚点(即feature map点，要区别于anchor)针对于第零维度上某一类anchor的具体的横纵坐标值
cx, cy, w, h = map(
lambda x: x + zero,
[cx, cy, w, h
]) # 这四个值的每一个都需要一个shape=(self.anchor_num, size, size)的ndarray来描述
x1, y1, x2, y2 = center2corner([cx, cy, w, h])
self.all_anchors = np.stack([x1, y1, x2, y2]), np.stack(
[cx, cy, w, h]) # np.stack就是摞起来了，多了一个维度方向；注意和np.cat的区别;
return True # 返回来的是tuple，第一个元素是两坐标表示法的anchor信息；第二个是中心做标加h,w