def test_dataloader(dataloader, opt='first_batch'):
if opt == 'first_batch':
mx_imgs, mx_labels = next(iter(dataloader))
for i in range(len(mx_imgs)):
plt.imshow(
myutils.denormalize(mx_imgs[i].transpose((1, 2, 0)).asnumpy()))
myutils.visualize_boxes(mx_labels[i, :, 1:].asnumpy(),
'blue',
plt.gcf(),
is_rltv_cor=True,
img_size=mx_imgs[i].shape[-2:])
if not plt.gca().yaxis_inverted():
plt.gca().invert_yaxis()
plt.show()
if opt == 'all_batch':
for mx_imgs, mx_labels in dataloader:
for i in range(len(mx_imgs)):
plt.imshow(
myutils.denormalize(mx_imgs[i].transpose(
(1, 2, 0)).asnumpy()))
myutils.visualize_boxes(mx_labels[i, :, 1:].asnumpy(),
'blue',
plt.gcf(),
is_rltv_cor=True,
img_size=mx_imgs[i].shape[-2:])
if not plt.gca().yaxis_inverted():
plt.gca().invert_yaxis()
plt.show()
return
def test_and_save():
# load model
backbone_root_path = 'd:/Documents/Data_Files/Parameters/'
ctx = mx.gpu()
net = SSD_rank_matching.SSD(backbone_root_path=backbone_root_path, ctx=ctx)
model_param_path = 'd:/Documents/Git/pascal_voc_benchmark/training_result/rank_matching_net_param'
net.load_parameters(model_param_path)
# prepare test dataset
root_path = './input_images/'
dataset = dataset_utils.VideoFrameDataset(root_path, net.model_img_size)
dataloader = mx.gluon.data.DataLoader(dataset.transform(
dataset.transform_val_fn),
batch_size=5,
shuffle=False,
last_batch='keep')
print(next(iter(dataloader)).shape)
frame_idx = 0
fig = plt.figure()
for idx, mx_imgs in enumerate(dataloader):
mx_imgs = mx_imgs.as_in_context(net.ctx)
tensor_preds = net(mx_imgs)
scr_cls_boxes = SSD_rank_matching.get_pred_scores_classes_and_boxes(
tensor_preds,
mx.nd.array(net.get_anchors(), ctx=net.ctx).expand_dims(axis=0))
for i, scr_cls_box in enumerate(scr_cls_boxes):
fig.clf()
img = myutils.denormalize(mx_imgs[i].transpose(
(1, 2, 0)).asnumpy())
plt.imshow(img)
myutils.visualize_boxes(scr_cls_box[:, -4:],
color='blue',
fig=fig,
is_rltv_cor=True,
img_size=(300, 300))
myutils.visualize_pred(
img,
None,
scr_cls_box,
class_names_list=dataset_utils.Dataset.class_names,
fig=plt.gcf(),
show_label=True)
if not fig.gca().yaxis_inverted():
fig.gca().invert_yaxis()
fig.set_figwidth(10)
fig.set_figheight(fig.get_figwidth())
fig.set_tight_layout(True)
fig.savefig(
os.path.abspath(
r'D:\Documents\Git\pascal_voc_benchmark\test_output' +
'/' + str(frame_idx) + '.png'))
frame_idx += 1
return
def test_get_pred_scores_classes_and_boxes(mx_imgs, mx_labels, net, anchors):
"""
anchors: (1, N, 4)
"""
tensor_preds = net(mx_imgs) # (b, N, 25)
for i in range(len(mx_imgs)):
scr_cls_box = ssd_utils.get_pred_scores_classes_and_boxes(
tensor_preds[i].expand_dims(axis=0), None, anchors)
plt.imshow(
myutils.denormalize(mx_imgs[i].transpose((1, 2, 0)).asnumpy()))
myutils.visualize_boxes(scr_cls_box[:, -4:],
'blue',
plt.gcf(),
is_rltv_cor=True,
img_size=[300, 300])
if not plt.gca().yaxis_inverted():
plt.gca().invert_yaxis()
plt.show()
print(scr_cls_box.shape)
print(scr_cls_box[:10])
print('label:',
mx_labels[i].asnumpy()[mx_labels[i].asnumpy()[:, 0] >= 0])
return
mx_img, mx_label = ssd.get_transform_fn_val()(img, label)
mx_label = mx_label.expand_dims(axis=0) # (1, M, 5)
tensor_preds = ssd(mx_img.expand_dims(axis=0))
cls_targets, box_targets, pos_neg_samples = generate_target(mx.nd.array(ssd.get_anchors()).expand_dims(axis=0), tensor_preds[:, :, 21:],
mx_label[:, :, -4:],
mx_label[:, :, 0])
pos_idx = (np.where(pos_neg_samples[0].asnumpy() > 0))[0] # (P, )
fig = plt.figure()
plt.imshow(img)
anchors = ssd.get_anchors() # (N, 4)
boxes = []
for i, idx in enumerate(pos_idx):
idx = int(idx)
myutils.visualize_boxes(anchors[idx:idx+1], 'blue', fig=fig, is_rltv_cor=True, img_size=img.shape[:2])
myutils.visualize_boxes(label[:, -4:], 'red', fig=fig, is_rltv_cor=False)
class_names_list = dataset.class_names
font_size = img.shape[1] * 0.03
for box in label:
cls_name = class_names_list[int(box[0])]
plt.gca().text(box[1], box[2], cls_name, fontname='monospace', fontsize=font_size,
horizontalalignment='left', verticalalignment='bottom',
bbox={'boxstyle': 'square', 'pad': 0.2, 'alpha': 0.6,
'facecolor': plt.get_cmap('tab20').colors[int(box[0])],
'edgecolor': plt.get_cmap('tab20').colors[int(box[0])]})
if not fig.gca().yaxis_inverted():
fig.gca().invert_yaxis()
coor_y2 = y + h/2
anchors.append([coor_x1, coor_y1, coor_x2, coor_y2])
else:
w = s_k * s
h = s_k * s
coor_x1 = x - w/2
coor_y1 = y - h/2
coor_x2 = x + w/2
coor_y2 = y + h/2
anchors.append([coor_x1, coor_y1, coor_x2, coor_y2])
anchors = np.array(anchors)
return anchors
if __name__ == '__main__':
# 产生一组 ssd style 的 anchor
anchors = generate_anchor([1, 0.45, 0.2], [1, 0.5, 2], s_k=0.4, style='ssd')
import dataset_utils
root_path = r'D:\Documents\Data_Files\Datasets\Pascal\VOC2012'
dataset = dataset_utils.Dataset(root_path)
img, label = dataset[0]
plt.imshow(img)
myutils.visualize_boxes(anchors[:3], 'red', fig=plt.gcf(), is_rltv_cor=True, img_size=img.shape[:2])
myutils.visualize_boxes(anchors[3:4], 'blue', fig=plt.gcf(), is_rltv_cor=True, img_size=img.shape[:2])
myutils.visualize_boxes(anchors[4:5], 'green', fig=plt.gcf(), is_rltv_cor=True, img_size=img.shape[:2])
if not plt.gca().yaxis_inverted():
plt.gca().invert_yaxis()
plt.show()
def test_generate_target(mx_imgs, mx_labels, net):
"""
对传入的数据一张一张地计算 target，并统计结果
net: attr: ctx, get_anchors(), get_map_shapes_list()
"""
mx_imgs = mx_imgs.as_in_context(net.ctx) # (b, c, h, w)
mx_labels = mx_labels.as_in_context(net.ctx) # (b, M, 5)
anchors = net.get_anchors() # (N, 4)
anchors = mx.nd.array(anchors).expand_dims(axis=0).as_in_context(
net.ctx) # (1, N, 4)
tensor_preds = net(mx_imgs)
tensor_preds = tensor_preds.reshape((0, -1, 25))
tensor_targs = ssd_utils.generate_target(anchors,
tensor_preds,
mx_labels[:, :, -4:],
mx_labels[:, :, 0:1],
iou_thresh=0.5,
neg_thresh=0.5,
negative_mining_ratio=3,
dynamic_sampling=True)
cls_targs, box_targs, pos_neg_samples = tensor_targs
for i, mx_img in enumerate(mx_imgs):
print('img ' + str(i) + ':', 'positives:',
str(mx.nd.sum(pos_neg_samples[i] > 0).asscalar()) + ',',
'negatives:',
mx.nd.sum(pos_neg_samples[i] < 0).asscalar())
# visualize original image and label boxes
fig = plt.figure()
plt.imshow(myutils.denormalize(mx_img.transpose((1, 2, 0)).asnumpy()))
myutils.visualize_boxes(mx_labels[i][:, -4:].asnumpy(),
'blue',
fig,
is_rltv_cor=True,
img_size=mx_imgs.shape[-2:])
if not plt.gca().yaxis_inverted():
plt.gca().invert_yaxis()
plt.show()
# visualize samples
# positives
print('positive samples on each feature map:')
for anchor_idx in range(net.anchor_num_per_position):
print('anchor index:', anchor_idx)
pos_on_fmaps = unreval_anchors(pos_neg_samples[i] > 0,
net.get_feature_map_shapes_list(),
net.anchor_num_per_position,
ith_anchor=anchor_idx)
fig = plt.figure()
axes = fig.subplots(nrows=1,
ncols=len(net.get_feature_map_shapes_list()))
for j, fmap_shape in enumerate(net.get_feature_map_shapes_list()):
axes[j].imshow(pos_on_fmaps[j])
axes[j].set_title(
'*'.join([str(elm) for elm in fmap_shape[-2:]]) +
', num: ' + str(np.sum(pos_on_fmaps[j] > 0)))
fig.set_figwidth(16)
fig.set_figheight(fig.get_figwidth())
plt.show()
# negatives
print('negative samples on each feature map:')
def test_generate_target2(mx_imgs, mx_labels, net, check_opt=None):
"""
对传入的数据一张一张地计算 target，并统计结果
net: attr: ctx, get_anchors(), get_map_shapes_list()
"""
mx_imgs = mx_imgs.as_in_context(net.ctx) # (b, c, h, w)
mx_labels = mx_labels.as_in_context(net.ctx) # (b, M, 5)
anchors = net.get_anchors() # (N, 4)
anchors = mx.nd.array(anchors).expand_dims(axis=0).as_in_context(
net.ctx) # (1, N, 4)
tensor_preds = net(mx_imgs)
tensor_targs = ssd_utils.generate_target2(anchors,
tensor_preds,
mx_labels,
iou_thresh=0.5,
neg_thresh=0.3,
negative_mining_ratio=3,
dynamic_sampling=True)
con_targs, cls_targs, box_targs, pos_neg_samples = tensor_targs
for i, mx_img in enumerate(mx_imgs):
print('img ' + str(i) + ':', 'positives:',
str(mx.nd.sum(pos_neg_samples[i] > 0).asscalar()) + ',',
'negatives:',
mx.nd.sum(pos_neg_samples[i] < 0).asscalar())
# visualize original image and label boxes
fig = plt.figure()
plt.imshow(myutils.denormalize(mx_img.transpose((1, 2, 0)).asnumpy()))
myutils.visualize_boxes(mx_labels[i][:, -4:].asnumpy(),
'blue',
fig,
is_rltv_cor=True,
img_size=mx_imgs.shape[-2:])
if not plt.gca().yaxis_inverted():
plt.gca().invert_yaxis()
plt.show()
if 'samples' in check_opt:
# visualize samples
# positives
print('positive samples on each feature map:')
for anchor_idx in range(net.anchor_num_per_position):
print('anchor index:', anchor_idx)
pos_on_fmaps = unreval_anchors(
pos_neg_samples[i] > 0,
net.get_feature_map_shapes_list(),
net.anchor_num_per_position,
ith_anchor=anchor_idx)
fig = plt.figure()
axes = fig.subplots(nrows=1,
ncols=len(
net.get_feature_map_shapes_list()))
for j, fmap_shape in enumerate(
net.get_feature_map_shapes_list()):
axes[j].imshow(pos_on_fmaps[j],
cmap=plt.get_cmap('tab20'),
vmin=0,
vmax=1)
axes[j].set_title(
'*'.join([str(elm) for elm in fmap_shape[-2:]]) +
', num: ' + str(np.sum(pos_on_fmaps[j])))
fig.set_figwidth(16)
fig.set_figheight(fig.get_figwidth())
plt.show()
# negatives
print('negative samples on each feature map:')
for anchor_idx in range(net.anchor_num_per_position):
print('anchor index:', anchor_idx)
neg_on_fmaps = unreval_anchors(
pos_neg_samples[i] < 0,
net.get_feature_map_shapes_list(),
net.anchor_num_per_position,
ith_anchor=anchor_idx)
fig = plt.figure()
axes = fig.subplots(nrows=1,
ncols=len(
net.get_feature_map_shapes_list()))
for j, fmap_shape in enumerate(
net.get_feature_map_shapes_list()):
axes[j].imshow(neg_on_fmaps[j])
axes[j].set_title(
'*'.join([str(elm) for elm in fmap_shape[-2:]]) +
', num: ' + str(np.sum(neg_on_fmaps[j])))
fig.set_figwidth(16)
fig.set_figheight(fig.get_figwidth())
plt.show()
if 'ohem' in check_opt:
# check out if thre prediction of negatives is 1.
pred_con = tensor_preds[i, :, :2].argmax(axis=-1) # (N, )
pred_con = mx.nd.where(con_targs[i] == 0, pred_con,
-mx.nd.ones_like(pred_con)) # (N, )
correct_ratio = mx.nd.sum(pred_con == 0) / mx.nd.sum(pred_con >= 0)
print('correct prediction in negatives:', correct_ratio.asscalar())
if 'confidence' in check_opt:
# visualize object confidence
print('object confidence on each feature map:')
for anchor_idx in range(net.anchor_num_per_position):
print('anchor index:', anchor_idx)
con_on_fmaps = unreval_anchors(
con_targs[i],
net.get_feature_map_shapes_list(),
net.anchor_num_per_position,
ith_anchor=anchor_idx)
fig = plt.figure()
axes = fig.subplots(nrows=1,
ncols=len(
net.get_feature_map_shapes_list()))
for j, fmap_shape in enumerate(
net.get_feature_map_shapes_list()):
axes[j].imshow(con_on_fmaps[j],
cmap=plt.get_cmap('tab20'),
vmin=-1,
vmax=1)
axes[j].set_title(
'*'.join([str(elm) for elm in fmap_shape[-2:]]) +
', num: ' + str(np.sum(con_on_fmaps[j] > 0)))
fig.set_figwidth(16)
fig.set_figheight(fig.get_figwidth())
plt.show()
if 'class' in check_opt:
# visualize class targets
print('class targets on each feature map:')
for anchor_idx in range(net.anchor_num_per_position):
print('anchor index:', anchor_idx)
cls_on_fmaps = unreval_anchors(
cls_targs[i],
net.get_feature_map_shapes_list(),
net.anchor_num_per_position,
ith_anchor=anchor_idx)
fig = plt.figure()
axes = fig.subplots(nrows=1,
ncols=len(
net.get_feature_map_shapes_list()))
for j, fmap_shape in enumerate(
net.get_feature_map_shapes_list()):
axes[j].imshow(cls_on_fmaps[j],
cmap=plt.get_cmap('terrain'),
vmin=-1,
vmax=20)
axes[j].set_title(
'*'.join([str(elm) for elm in fmap_shape[-2:]]) +
', num: ' + str(np.sum(cls_on_fmaps[j] >= 0)))
fig.set_figwidth(16)
fig.set_figheight(fig.get_figwidth())
plt.show()
if 'box' in check_opt:
# visualize box targets
print('box targets on each feature map:')
# vmin = box_targs.min().asscalar()
# vmax = box_targs.max().asscalar()
for anchor_idx in range(net.anchor_num_per_position):
print('anchor index:', anchor_idx)
box_on_fmaps = unreval_anchors(
box_targs[i],
net.get_feature_map_shapes_list(),
net.anchor_num_per_position,
ith_anchor=anchor_idx)
fig = plt.figure()
axes = fig.subplots(nrows=1,
ncols=len(
net.get_feature_map_shapes_list()))
for j, fmap_shape in enumerate(
net.get_feature_map_shapes_list()):
axes[j].imshow(box_on_fmaps[j][:, :, 0] != 0,
cmap=plt.get_cmap('terrain'),
vmin=0,
vmax=1)
axes[j].set_title(
'*'.join([str(elm) for elm in fmap_shape[-2:]]) +
', num: ' + str(np.sum(box_on_fmaps[j][:, :, 0] != 0)))
fig.set_figwidth(16)
fig.set_figheight(fig.get_figwidth())
plt.show()
return