def forward_det(self):
anno_img = self._anno[self._cur]
boxes_img = self._boxes[self._cur]
pred_cls_img = self._pred_cls[self._cur]
pred_confs_img = self._pred_confs[self._cur]
if (boxes_img.shape[0] < 2):
self._cur += 1
if (self._cur >= len(self._anno)):
self._cur = 0
return None
im_path = anno_img['img_path']
im = cv2.imread(im_path)
ih = im.shape[0]
iw = im.shape[1]
PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])
image_blob, im_scale = prep_im_for_blob(im, PIXEL_MEANS)
blob = np.zeros((1, ) + image_blob.shape, dtype=np.float32)
blob[0] = image_blob
# Reshape net's input blobs
boxes = np.zeros((boxes_img.shape[0], 5))
boxes[:, 1:5] = boxes_img * im_scale
classes = pred_cls_img
ix1 = []
ix2 = []
n_rel_inst = len(pred_cls_img) * (len(pred_cls_img) - 1)
rel_boxes = np.zeros((n_rel_inst, 5))
SpatialFea = np.zeros((n_rel_inst, 2, 32, 32))
# SpatialFea = np.zeros((n_rel_inst, 8))
rel_so_prior = np.zeros((n_rel_inst, self._num_relations))
i_rel_inst = 0
for s_idx in range(len(pred_cls_img)):
for o_idx in range(len(pred_cls_img)):
if (s_idx == o_idx):
continue
ix1.append(s_idx)
ix2.append(o_idx)
sBBox = boxes_img[s_idx]
oBBox = boxes_img[o_idx]
rBBox = self._getUnionBBox(sBBox, oBBox, ih, iw)
soMask = [self._getDualMask(ih, iw, sBBox), \
self._getDualMask(ih, iw, oBBox)]
rel_boxes[i_rel_inst, 1:5] = np.array(rBBox) * im_scale
SpatialFea[i_rel_inst] = soMask
# SpatialFea[i_rel_inst] = self._getRelativeLoc(sBBox, oBBox)
rel_so_prior[i_rel_inst] = self._so_prior[classes[s_idx],
classes[o_idx]]
i_rel_inst += 1
image_blob = image_blob.astype(np.float32, copy=False)
boxes = boxes.astype(np.float32, copy=False)
classes = classes.astype(np.float32, copy=False)
ix1 = np.array(ix1)
ix2 = np.array(ix2)
self._cur += 1
if (self._cur >= len(self._anno)):
self._cur = 0
return blob, boxes, rel_boxes, SpatialFea, classes, ix1, ix2, boxes_img, pred_confs_img, rel_so_prior
def forward_test(self):
"""Get blobs and copy them into this layer's top blob vector."""
anno_img = self._anno[self._cur]
if (anno_img is None):
self._cur += 1
if (self._cur >= len(self._anno)):
self._cur = 0
return None
im_path = anno_img['img_path']
if im_path[-3:] == 'png':
im_path = im_path[:-3] + 'jpg'
im = cv2.imread(im_path)
ih = im.shape[0]
iw = im.shape[1]
PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])
image_blob, im_scale = prep_im_for_blob(im, PIXEL_MEANS)
blob = np.zeros((1, ) + image_blob.shape, dtype=np.float32)
blob[0] = image_blob
# Reshape net's input blobs
boxes = np.zeros((anno_img['boxes'].shape[0], 5))
boxes[:, 1:5] = anno_img['boxes'] * im_scale
classes = np.array(anno_img['classes'])
ix1 = np.array(anno_img['ix1'])
ix2 = np.array(anno_img['ix2'])
rel_classes = anno_img['rel_classes']
n_rel_inst = len(rel_classes)
rel_boxes = np.zeros((n_rel_inst, 5))
SpatialFea = np.zeros((n_rel_inst, 2, 32, 32))
# SpatialFea = np.zeros((n_rel_inst, 8))
for ii in range(n_rel_inst):
sBBox = anno_img['boxes'][ix1[ii]]
oBBox = anno_img['boxes'][ix2[ii]]
rBBox = self._getUnionBBox(sBBox, oBBox, ih, iw)
soMask = [self._getDualMask(ih, iw, sBBox), \
self._getDualMask(ih, iw, oBBox)]
rel_boxes[ii, 1:5] = np.array(rBBox) * im_scale
SpatialFea[ii] = soMask
# SpatialFea[ii] = self._getRelativeLoc(sBBox, oBBox)
image_blob = image_blob.astype(np.float32, copy=False)
boxes = boxes.astype(np.float32, copy=False)
classes = classes.astype(np.float32, copy=False)
self._cur += 1
if (self._cur >= len(self._anno)):
self._cur = 0
return blob, boxes, rel_boxes, SpatialFea, classes, ix1, ix2, anno_img[
'boxes']
def forward_train_rank_im(self):
"""Get blobs and copy them into this layer's top blob vector."""
anno_img = self._anno[self._cur]
im_path = anno_img['img_path']
im = cv2.imread(im_path)
ih = im.shape[0]
iw = im.shape[1]
PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])
image_blob, im_scale = prep_im_for_blob(im, PIXEL_MEANS)
blob = np.zeros((1, ) + image_blob.shape, dtype=np.float32)
blob[0] = image_blob
boxes = np.zeros((anno_img['boxes'].shape[0], 5))
boxes[:, 1:5] = anno_img['boxes'] * im_scale
classes = np.array(anno_img['classes'])
ix1 = np.array(anno_img['ix1'])
ix2 = np.array(anno_img['ix2'])
rel_classes = anno_img['rel_classes']
n_rel_inst = len(rel_classes)
rel_boxes = np.zeros((n_rel_inst, 5))
rel_labels = -1 * np.ones((1, n_rel_inst * self._num_relations))
SpatialFea = np.zeros((n_rel_inst, 2, 32, 32))
rel_so_prior = np.zeros((n_rel_inst, self._num_relations))
pos_idx = 0
for ii in range(len(rel_classes)):
sBBox = anno_img['boxes'][ix1[ii]]
oBBox = anno_img['boxes'][ix2[ii]]
rBBox = self._getUnionBBox(sBBox, oBBox, ih, iw)
rel_boxes[ii, 1:5] = np.array(rBBox) * im_scale
SpatialFea[ii] = [self._getDualMask(ih, iw, sBBox), \
self._getDualMask(ih, iw, oBBox)]
rel_so_prior[ii] = self._so_prior[classes[ix1[ii]],
classes[ix2[ii]]]
for r in rel_classes[ii]:
rel_labels[0, pos_idx] = ii * self._num_relations + r
pos_idx += 1
image_blob = image_blob.astype(np.float32, copy=False)
boxes = boxes.astype(np.float32, copy=False)
classes = classes.astype(np.float32, copy=False)
self._cur += 1
if (self._cur >= len(self._anno)):
self._cur = 0
return blob, boxes, rel_boxes, SpatialFea, classes, ix1, ix2, rel_labels, rel_so_prior
def relation_im(self, im_path, res):
boxes_img = res['box']
pred_cls_img = np.array(res['cls'])
pred_confs = np.array(res['confs'])
time1 = time.time()
im = cv2.imread(im_path)
ih = im.shape[0]
iw = im.shape[1]
PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])
image_blob, im_scale = prep_im_for_blob(im, PIXEL_MEANS)
blob = np.zeros((1, ) + image_blob.shape, dtype=np.float32)
blob[0] = image_blob
# Reshape net's input blobs
boxes = np.zeros((boxes_img.shape[0], 5))
boxes[:, 1:5] = boxes_img * im_scale
classes = pred_cls_img
ix1 = []
ix2 = []
n_rel_inst = len(pred_cls_img) * (len(pred_cls_img) - 1)
rel_boxes = np.zeros((n_rel_inst, 5))
SpatialFea = np.zeros((n_rel_inst, 8))
rel_so_prior = np.zeros((n_rel_inst, 70))
i_rel_inst = 0
for s_idx in range(len(pred_cls_img)):
for o_idx in range(len(pred_cls_img)):
if (s_idx == o_idx):
continue
ix1.append(s_idx)
ix2.append(o_idx)
sBBox = boxes_img[s_idx]
oBBox = boxes_img[o_idx]
rBBox = self.getUnionBBox(sBBox, oBBox, ih, iw)
rel_boxes[i_rel_inst, 1:5] = np.array(rBBox) * im_scale
SpatialFea[i_rel_inst] = self.getRelativeLoc(sBBox, oBBox)
rel_so_prior[i_rel_inst] = self.so_prior[classes[s_idx],
classes[o_idx]]
i_rel_inst += 1
boxes = boxes.astype(np.float32, copy=False)
classes = classes.astype(np.float32, copy=False)
ix1 = np.array(ix1)
ix2 = np.array(ix2)
obj_score, rel_score = self.net(blob, boxes, rel_boxes, SpatialFea,
classes, ix1, ix2, self.args)
rel_prob = rel_score.data.cpu().numpy()
rel_prob += np.log(0.5 *
(rel_so_prior + 1.0 / self.args.num_relations))
rlp_labels_im = np.zeros((rel_prob.shape[0] * rel_prob.shape[1], 5),
dtype=np.int)
tuple_confs_im = []
n_idx = 0
for tuple_idx in range(rel_prob.shape[0]):
sub = ix1[tuple_idx]
obj = ix2[tuple_idx]
for rel in range(rel_prob.shape[1]):
conf = rel_prob[tuple_idx, rel]
rlp_labels_im[n_idx] = [
classes[sub], sub, rel, classes[obj], obj
]
tuple_confs_im.append(conf)
n_idx += 1
tuple_confs_im = np.array(tuple_confs_im)
idx_order = tuple_confs_im.argsort()[::-1][:20]
rlp_labels_im = rlp_labels_im[idx_order, :]
tuple_confs_im = tuple_confs_im[idx_order]
vrd_res = []
for tuple_idx in range(rlp_labels_im.shape[0]):
label_tuple = rlp_labels_im[tuple_idx]
sub_cls = self.vrd_classes[label_tuple[0]]
obj_cls = self.vrd_classes[label_tuple[3]]
rel_cls = self.vrd_rels[label_tuple[2]]
vrd_res.append(
('%s%d-%s-%s%d' %
(sub_cls, label_tuple[1], rel_cls, obj_cls, label_tuple[4]),
tuple_confs_im[tuple_idx]))
print vrd_res
time2 = time.time()
print "TEST Time:%s" % (time.strftime('%H:%M:%S',
time.gmtime(int(time2 - time1))))
return vrd_res
def forward_det(self):
anno_img = self._anno[self._cur]
# detection box
boxes_img = self._boxes[self._cur]
# detection class
pred_cls_img = self._pred_cls[self._cur]
# detection conf
pred_confs_img = self._pred_confs[self._cur]
if (boxes_img.shape[0] < 2):
# 没有物体，或只有一个物体，无法构成relationship
# 跳过当前img
self._cur += 1
if (self._cur >= len(self._anno)):
# 数据指针到尽头，归零
self._cur = 0
return None
im_path = anno_img['img_path']
# by sunx
if im_path[-3:] == 'png':
im_path = im_path[:-3] + 'jpg'
# 加载图片
im = cv2.imread(im_path)
ih = im.shape[0]
iw = im.shape[1]
# 三通道均值
PIXEL_MEANS = np.array([[[102.9801, 115.9465, 122.7717]]])
# 图像归一化
# 图像resize
image_blob, im_scale = prep_im_for_blob(im, PIXEL_MEANS)
blob = np.zeros((1, ) + image_blob.shape, dtype=np.float32)
blob[0] = image_blob
# Reshape net's input blobs
# detection resize
boxes = np.zeros((boxes_img.shape[0], 5))
boxes[:, 1:5] = boxes_img * im_scale
classes = pred_cls_img
# sbj index
ix1 = []
# obj index
ix2 = []
# relationship 最大数量
n_rel_inst = len(pred_cls_img) * (len(pred_cls_img) - 1)
# relationship boxes(union box)
rel_boxes = np.zeros((n_rel_inst, 5))
# 空间特征2
# 两通道 32*32，分别对应sbj和obj
SpatialFea = np.zeros((n_rel_inst, 2, 32, 32))
# SpatialFea = np.zeros((n_rel_inst, 8))
# relationship 先验概率
rel_so_prior = np.zeros((n_rel_inst, self._num_relations))
i_rel_inst = 0
# 全组合det, 产生relationship candidates
for s_idx in range(len(pred_cls_img)):
for o_idx in range(len(pred_cls_img)):
if (s_idx == o_idx):
continue
ix1.append(s_idx)
ix2.append(o_idx)
sBBox = boxes_img[s_idx]
oBBox = boxes_img[o_idx]
rBBox = self._getUnionBBox(sBBox, oBBox, ih, iw)
# [sbj, obj] 空间特征
soMask = [self._getDualMask(ih, iw, sBBox), \
self._getDualMask(ih, iw, oBBox)]
# resize union box
rel_boxes[i_rel_inst, 1:5] = np.array(rBBox) * im_scale
SpatialFea[i_rel_inst] = soMask
# SpatialFea[i_rel_inst] = self._getRelativeLoc(sBBox, oBBox)
# 先验概率
rel_so_prior[i_rel_inst] = self._so_prior[classes[s_idx],
classes[o_idx]]
i_rel_inst += 1
# img data
image_blob = image_blob.astype(np.float32, copy=False)
# detection boxes
boxes = boxes.astype(np.float32, copy=False)
# detection classes
classes = classes.astype(np.float32, copy=False)
ix1 = np.array(ix1)
ix2 = np.array(ix2)
self._cur += 1
if (self._cur >= len(self._anno)):
self._cur = 0
return blob, boxes, rel_boxes, SpatialFea, classes, ix1, ix2, boxes_img, pred_confs_img, rel_so_prior