def get_detections_from_im(net,
im,
image_id,
bbox=None,
num_bbox=None,
conf_thresh=0.2):
if bbox is not None:
scores, boxes, attr_scores, rel_scores = im_detect(net,
im,
bbox[:num_bbox, 1:],
force_boxes=True)
else:
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes': len(rois),
'boxes': base64.b64encode(cls_boxes),
'features': base64.b64encode(pool5),
'cls_prob': base64.b64encode(cls_prob)
}
def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2):
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep], cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes': len(keep_boxes),
'boxes': base64.b64encode(cls_boxes[keep_boxes]),
'features': base64.b64encode(pool5[keep_boxes])
}
def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2):
"""
:param net:
:param im_file: full path to an image
:param image_id:
:param conf_thresh:
:return: all information from detection and attr prediction
"""
im = cv2.imread(im_file)
if im is None or min(im.shape[:2]) < 200 or im.shape[2] != 3:
return None
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
print(len(keep_boxes))
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
objects_conf = np.max(cls_prob[keep_boxes][:, 1:], axis=1)
attrs = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attrs_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
return {
"img_id": image_id,
"img_h": np.size(im, 0),
"img_w": np.size(im, 1),
"objects_id": base64.b64encode(objects), # int64
"objects_conf": base64.b64encode(objects_conf), # float32
"attrs_id": base64.b64encode(attrs), # int64
"attrs_conf": base64.b64encode(attrs_conf), # float32
"num_boxes": len(keep_boxes),
"boxes": base64.b64encode(cls_boxes[keep_boxes]), # float32
"features": base64.b64encode(pool5[keep_boxes]), # float32
"cls_prob": base64.b64encode(cls_prob[keep_boxes]),
"classes": base64.b64encode(scores[keep_boxes]),
"attrs": base64.b64encode(attr_scores[keep_boxes])
}
def getFeaturesByImagePath(self, imagePath):
image = self.loadImageByFileName(imagePath)
scores, boxes, attr_scores, rel_scores = im_detect(self.net, image)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = self.net.blobs['rois'].data.copy()
# unscale back to raw image space
_, im_scales = _get_blobs(image, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = self.net.blobs['cls_prob'].data
pool5 = self.net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= self.conf_thresh)[0]
if len(keep_boxes) < self.MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:self.MIN_BOXES]
elif len(keep_boxes) > self.MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:self.MAX_BOXES]
features = []
for box in pool5[keep_boxes]:
features.append(box.tolist())
return features
def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2):
print "get_detections_from_im: im_file = {}, image_id = {}".format(im_file, image_id)
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
roipool5 = net.blobs['roipool5'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1,cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep], cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
print "feature dim = {}".format(roipool5[keep_boxes].shape)
return roipool5[keep_boxes]
def im_detect(net, im, boxes, layer='fc7'):
"""Detect object classes in an image given object proposals.
Arguments:
net (caffe.Net): Fast R-CNN network to use
im (ndarray): color image to test (in BGR order)
boxes (ndarray): R x 4 array of object proposals
Returns:
scores (ndarray): R x K array of object class scores (K includes
background as object category 0)
boxes (ndarray): R x (4*K) array of predicted bounding boxes
"""
blobs, unused_im_scale_factors = _get_blobs(im, boxes)
# When mapping from image ROIs to feature map ROIs, there's some aliasing
# (some distinct image ROIs get mapped to the same feature ROI).
# Here, we identify duplicate feature ROIs, so we only compute features
# on the unique subset.
if cfg.DEDUP_BOXES > 0:
v = np.array([1, 1e3, 1e6, 1e9, 1e12])
hashes = np.round(blobs['rois'] * cfg.DEDUP_BOXES).dot(v)
_, index, inv_index = np.unique(hashes, return_index=True,
return_inverse=True)
blobs['rois'] = blobs['rois'][index, :]
boxes = boxes[index, :]
# reshape network inputs
net.blobs['data'].reshape(*(blobs['data'].shape))
net.blobs['rois'].reshape(*(blobs['rois'].shape))
blobs_out = net.forward(data=blobs['data'].astype(np.float32, copy=False),
rois=blobs['rois'].astype(np.float32, copy=False))
data = net.blobs[layer].data
return data[inv_index, :]
def get_feats_from_im(net, im, min_max_bboxes, feat_name, conf_thresh=0.2):
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
feat = net.blobs[feat_name].data # extract feature from layer with name 'feat_name'
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1,cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
#print keep.shape
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep], cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_max_bboxes[0]:
keep_boxes = np.argsort(max_conf)[::-1][:min_max_bboxes[0]]
elif len(keep_boxes) > min_max_bboxes[1]:
keep_boxes = np.argsort(max_conf)[::-1][:min_max_bboxes[1]]
image_h = np.size(im, 0)
image_w = np.size(im, 1)
return feat[keep_boxes], cls_boxes[keep_boxes], image_h, image_w, len(keep_boxes)
def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2):
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
boxes = cls_boxes[keep_boxes]
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
cls_all = []
for i in range(len(keep_boxes)):
bbox = boxes[i]
if bbox[0] == 0:
bbox[0] = 1
if bbox[1] == 0:
bbox[1] = 1
cls = classes[objects[i] + 1]
if attr_conf[i] > attr_thresh:
cls = attributes[attr[i] + 1] + " " + cls
cls_all.append(cls)
assert len(cls_all) == 36
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes': len(keep_boxes),
'boxes': base64.b64encode(cls_boxes[keep_boxes]),
'features': base64.b64encode(pool5[keep_boxes]),
'cls': cls_all
}
def extract_features(params,net, im_file, boxes):
im = cv2.imread(im_file)
blobs, unused_im_scale_factors = test_ops._get_blobs(im, boxes)
# reshape network inputs
net.blobs['data'].reshape(*(blobs['data'].shape))
net.blobs['rois'].reshape(*(blobs['rois'].shape))
blobs_out = net.forward(data=blobs['data'].astype(np.float32, copy=False),
rois=blobs['rois'].astype(np.float32, copy=False))
if 'fc' in params['layer'] or 'score' in params['layer']:
scores = net.blobs[params['layer']].data
else:
scores = blobs_out[params['layer']]
# Save regressed windows
box_deltas = blobs_out['bbox_pred_trecvid']
pred_boxes = test_ops._bbox_pred(boxes, box_deltas)
pred_boxes = test_ops._clip_boxes(pred_boxes, im.shape)
boxes = pred_boxes
return scores, boxes
def get_detections_from_im(net,
im_file,
conf_thresh=0.2,
min_num_boxes=36,
max_num_boxes=36):
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# keep the original boxes, don't worry about the regression bounding box outputs
rois = net.blobs["rois"].data.copy()
# unscale back to the raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs["cls_prob"].data
pool5 = net.blobs["pool5_flat"].data
# keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_num_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:min_num_boxes]
elif len(keep_boxes) > max_num_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:max_num_boxes]
attr_prob = net.blobs["attr_prob"].data
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
annotated_boxes = []
for i in range(len(keep_boxes)):
cls = classes[objects[i] + 1]
if attr_conf[i] > attr_thresh:
cls = attributes[attr[i] + 1] + " " + cls
annotated_boxes.append({
"annotation": cls,
"coordinates": cls_boxes[keep_boxes][i]
})
return {
"image_h": np.size(im, 0),
"image_w": np.size(im, 1),
"num_boxes": len(keep_boxes),
"boxes": annotated_boxes,
"features": pool5[keep_boxes]
}
def get_det_feats_from_im(net, im, boxes, feat_name):
scores, _, _, _ = im_detect(net, im, boxes=boxes, force_boxes=True)
# Keep the original boxes, don't worry about the regresssion bbox outputs
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
feat = net.blobs[feat_name].data
image_h = np.size(im, 0)
image_w = np.size(im, 1)
return feat, image_h, image_w, boxes.shape[0]
def get_detections_from_im(net,
im_file,
image_id,
bbox=None,
num_bbox=None,
conf_thresh=0.2):
"""
:param net:
:param im_file: full path to an image
:param image_id:
:param conf_thresh:
:return: all information from detection and attr prediction
"""
im = cv2.imread(im_file)
if bbox is not None:
scores, boxes, attr_scores, rel_scores = im_detect(net,
im,
bbox[:num_bbox, 1:],
force_boxes=True)
else:
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
objects = np.argmax(cls_prob[:, 1:], axis=1)
objects_conf = np.max(cls_prob[:, 1:], axis=1)
attrs = np.argmax(attr_prob[:, 1:], axis=1)
attrs_conf = np.max(attr_prob[:, 1:], axis=1)
return {
"img_id": image_id,
"img_h": np.size(im, 0),
"img_w": np.size(im, 1),
"objects_id": base64.b64encode(objects), # int64
"objects_conf": base64.b64encode(objects_conf), # float32
"attrs_id": base64.b64encode(attrs), # int64
"attrs_conf": base64.b64encode(attrs_conf), # float32
"num_boxes": len(rois),
"boxes": base64.b64encode(cls_boxes), # float32
"features": base64.b64encode(pool5), # float32
"cls_prob": base64.b64encode(cls_prob),
"classes": base64.b64encode(scores),
"attrs": base64.b64encode(attr_scores)
}
def get_detections_from_im(net,
im_file,
image_id,
ziphelper,
data_root,
conf_thresh=0.5):
zip_image = ziphelper.imread(str(os.path.join(data_root, im_file)))
im = cv2.cvtColor(np.array(zip_image), cv2.COLOR_RGB2BGR)
if np.max(im.shape) < 20:
print("image too small: ", image_id)
return None
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes': len(keep_boxes),
'boxes': base64.b64encode(cls_boxes[keep_boxes]),
'classes': base64.b64encode(scores[keep_boxes]),
'attrs': base64.b64encode(attr_scores[keep_boxes]),
'features': base64.b64encode(pool5[keep_boxes])
}
def get_detections_from_im(split, net, im_file, image_id, minboxes, maxboxes, conf_thresh):
im = cv2.imread(im_file)
if im is None: # video stream/video file
_, im = cv2.VideoCapture(im_file).read()
if 'referit' in split:
im = validate_referit_image(image_id, im_file, im)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
cls_idx = net.blobs['predicted_cls'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1,cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep], cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < minboxes:
keep_boxes = np.argsort(max_conf)[::-1][:minboxes]
elif len(keep_boxes) > maxboxes:
keep_boxes = np.argsort(max_conf)[::-1][:maxboxes]
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes' : len(keep_boxes),
'boxes': base64.b64encode(cls_boxes[keep_boxes]),
'features': base64.b64encode(pool5[keep_boxes]),
'confidence': base64.b64encode(max_conf[keep_boxes]),
'class': base64.b64encode(cls_idx[keep_boxes])
}
def extract_im(net, img_path):
img = cv2.imread(img_path)
st2_scores, st2_boxes, st2_attr_scores, st2_rel_scores = im_detect(net, img)
pool5 = net.blobs['pool5_flat'].data
# unscale back to raw image space
blobs, im_scales = _get_blobs(img, None)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
st1_scores = rois[:, 0]
st1_boxes = rois[:, 1:5] / im_scales[0]
# Keep only the best class_box of each row in st2_boxes has 1601 boxes for 1 ROI
max_cls_scores, max_cls_indices = get_nms_boxes(st2_scores, st1_boxes)
# For each threshold of boxes,
# save (keep_box_indices, keep_box_cls_indices)
keep_ind = []
for (min_boxes, max_boxes) in NUM_BOXES:
keep_box_indices = np.where(max_cls_scores >= conf_thresh)[0]
if len(keep_box_indices) < min_boxes:
keep_box_indices = np.argsort(max_cls_scores)[::-1][:min_boxes]
elif len(keep_box_indices) > max_boxes:
keep_box_indices = np.argsort(max_cls_scores)[::-1][:max_boxes]
# print("keep_box_indices len", len(keep_box_indices))
keep_box_cls_indices = max_cls_indices[keep_box_indices]
keep_ind.append((keep_box_indices, keep_box_cls_indices))
return {
"image_id": image_id_from_path(img_path),
"image_h": np.size(img, 0),
"image_w": np.size(img, 1),
"keep_ind": keep_ind,
"st2_scores": st2_scores,
"st2_boxes": st2_boxes,
"st2_attr_scores": st2_attr_scores,
"pool5": pool5,
"st1_boxes": st1_boxes
}
def extract_fea(net, im_file, conf_thresh=0.4, min_boxes=36, max_boxes=36):
im = cv2.imread(im_file)
scores, boxes, _, _ = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
# attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:min_boxes]
elif len(keep_boxes) > max_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:max_boxes]
############################
boxes = cls_boxes[keep_boxes]
# objects = np.argmax(cls_prob[keep_boxes][:,1:], axis=1)
# attr_thresh = 0.1
# attr = np.argmax(attr_prob[keep_boxes][:,1:], axis=1)
# attr_conf = np.max(attr_prob[keep_boxes][:,1:], axis=1)
vfeat = np.asarray(pool5[keep_boxes])
sfeat = _boxes2sfeat(boxes, im)
return vfeat, sfeat
def extract_from_rois(self, im, rois):
"""
Parameters
------------
im: np.array 图片数据
rois: ROI列表, [(x1, y1, x2, y2)]
"""
# 设置网络输入
# 默认test scale为600(短边)
from fast_rcnn.test import _get_blobs
blobs, unused_im_scale_factors = _get_blobs(im, rois)
# When mapping from image ROIs to feature map ROIs, there's some aliasing
# (some distinct image ROIs get mapped to the same feature ROI).
# Here, we identify duplicate feature ROIs, so we only compute features
# on the unique subset.
# dedup_boxes默认为1/16, 对于CaffeNet正确
if self.cfg["dedup_boxes"] > 0:
v = np.array([1, 1e3, 1e6, 1e9, 1e12])
hashes = np.round(blobs['rois'] * self.cfg["dedup_boxes"]).dot(v)
_, index, inv_index = np.unique(hashes,
return_index=True,
return_inverse=True)
blobs['rois'] = blobs['rois'][index, :]
#rois = rois[index, :] # ? 这里还有什么用...用于debug?
# reshape network inputs
self.net.blobs['data'].reshape(*(blobs['data'].shape))
self.net.blobs['rois'].reshape(*(blobs['rois'].shape))
features = self.net.forward(data=blobs['data'].astype(np.float32,
copy=False),
rois=blobs['rois'].astype(
np.float32, copy=False))["pool5"]
# import ipdb
# ipdb.set_trace()
if self.cfg["dedup_boxes"] > 0:
# Map scores and predictions back to the original set of boxes
features = features[inv_index, :, :, :].reshape(rois.shape[0], -1)
return features
def extract_from_image(image, conf_thresh=0.2):
caffe.set_mode_cpu()
im = image
image_h, image_w, _ = im.shape
scores, boxes, rois = im_detect(net, im)
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
boxes, features = cls_boxes[keep_boxes], pool5[keep_boxes]
boxes = boxes.reshape(36, -1)
box_width = boxes[:, 2] - boxes[:, 0]
box_height = boxes[:, 3] - boxes[:, 1]
scaled_width = box_width / image_w
scaled_height = box_height / image_h
scaled_x = boxes[:, 0] / image_w
scaled_y = boxes[:, 1] / image_h
scaled_width = scaled_width[..., np.newaxis]
scaled_height = scaled_height[..., np.newaxis]
scaled_x = scaled_x[..., np.newaxis]
scaled_y = scaled_y[..., np.newaxis]
spatial_features = np.concatenate(
(scaled_x, scaled_y, scaled_x + scaled_width,
scaled_y + scaled_height, scaled_width, scaled_height),
axis=1)
return spatial_features, features
def extract_features(params, net, im_file, boxes):
im = cv2.imread(im_file)
blobs, unused_im_scale_factors = test_ops._get_blobs(im, boxes)
# reshape network inputs
net.blobs['data'].reshape(*(blobs['data'].shape))
net.blobs['rois'].reshape(*(blobs['rois'].shape))
blobs_out = net.forward(data=blobs['data'].astype(np.float32, copy=False),
rois=blobs['rois'].astype(np.float32, copy=False))
if 'fc' in params['layer'] or 'score' in params['layer']:
scores = net.blobs[params['layer']].data
else:
scores = blobs_out[params['layer']]
# Save regressed windows
box_deltas = blobs_out['bbox_pred_trecvid']
pred_boxes = test_ops._bbox_pred(boxes, box_deltas)
pred_boxes = test_ops._clip_boxes(pred_boxes, im.shape)
boxes = pred_boxes
return scores, boxes
def extract_hypercolumns(net, im, boxes):
blobs, unused_im_scale_factors = _get_blobs(im, boxes)
# # When mapping from image ROIs to feature map ROIs, there's some aliasing
# # (some distinct image ROIs get mapped to the same feature ROI).
# # Here, we identify duplicate feature ROIs, so we only compute features
# # on the unique subset.
# if cfg.DEDUP_BOXES > 0:
# v = np.array([1, 1e3, 1e6, 1e9, 1e12])
# hashes = np.round(blobs['rois'] * cfg.DEDUP_BOXES).dot(v)
# _, index, inv_index = np.unique(hashes, return_index=True,
# return_inverse=True)
# blobs['rois'] = blobs['rois'][index, :]
# boxes = boxes[index, :]
# reshape network inputs
net.blobs['data'].reshape(*(blobs['data'].shape))
net.blobs['rois'].reshape(*(blobs['rois'].shape))
blobs_out = net.forward(data=blobs['data'].astype(np.float32, copy=False),
rois=blobs['rois'].astype(np.float32, copy=False))
print dir(net.blobs), net.blobs.keys(), net.blobs['conv1'].data.shape
hypercolumns = []
# layers = ['conv2', 'conv3', 'conv4', 'conv5']
layers = ['norm1', 'norm2']
layers = ['pool1', 'pool2', 'pool5']
# layers = ['fc6', 'fc7']
for layer in layers:
print layer, net.blobs[layer].data.shape
convmap = net.blobs[layer].data
for fmap in convmap[0]:
# print 'fmap', fmap.shape
upscaled = sp.misc.imresize(fmap, size=(im.shape[0], im.shape[1]),
mode="F", interp='bilinear')
hypercolumns.append(upscaled)
return np.asarray(hypercolumns)
def extract_features_boxes(folders):
features = []
num_im = 0
num_image = 0
temp = 1
npboxes = []
for folder in folders:
for root, dirs, files in os.walk(folder):
for file in files:
filepath = root + "/" + file
if 'png' in file and not 'depth' in file:
im = cv2.imread(filepath)
npboxes = selective_boxes(filepath)
print "our boxes are", npboxes
in_blobs, _ = _get_blobs(im, np.asarray(npboxes))
net.blobs['data'].reshape(*(in_blobs["data"].shape))
net.blobs['rois'].reshape(*(in_blobs["rois"].shape))
blobs_out = net.forward(
data=in_blobs["data"].astype(np.float32, copy=False),
rois=in_blobs["rois"].astype(np.float32, copy=False))
# print "OUR: ", transformer.preprocess("data", im).shape
# print "DEMO: ", in_blobs["data"].shape
print file
print blobs_out.keys()
print blobs_out["cls_prob"]
# net.blobs['rois'].reshape(len(npboxes),5)
# net.reshape()
# net.blobs['data'].data[...] = transformer.preprocess('data', im)
# net.blobs['rois'].data[...] = np.array(npboxes)
# out=net.forward()
fc7 = deepcopy(net.blobs['fc7'].data)
num_im += 1
#print str(num_im), "\b"*(len(str(num_im))+2),
sys.stdout.flush()
features.append((filepath, npboxes, deepcopy(fc7)))
print features
return features
im_file = 'data/demo/Adv101.jpg'
###########################
# Similar to get_detections_from_im
conf_thresh = 0.4
min_boxes = 10
max_boxes = 20
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
import caffe
import cv2
import h5py
file = tempfile.NamedTemporaryFile(dir = '.', prefix = 'vgg16_test_force_backward', suffix = '.prototxt', delete = False)
lines = list(open(os.path.join(cfg.ROOT_DIR, 'models', 'VGG16', 'test.prototxt')))
file.writelines([lines[0], 'force_backward: true\n', 'debug_info: true\n'] + lines[1:])
file.close()
caffe.set_mode_gpu()
net = caffe.Net(file.name, os.path.join(cfg.ROOT_DIR, 'data', 'fast_rcnn_models', 'vgg16_fast_rcnn_iter_40000.caffemodel'), caffe.TRAIN)
image_name = '000004'
obj_proposals = sio.loadmat(os.path.join(cfg.ROOT_DIR, 'data', 'demo', image_name + '_boxes.mat'))['boxes']
im = cv2.imread(os.path.join(cfg.ROOT_DIR, 'data', 'demo', image_name + '.jpg'))
blobs, unused_im_scale_factors = _get_blobs(im, obj_proposals)
blobs['rois'] = blobs['rois'][:1]
net.blobs['data'].reshape(*(blobs['data'].shape))
net.blobs['rois'].reshape(*(blobs['rois'].shape))
blobs_out = net.forward(data = blobs['data'].astype(np.float32, copy=False), rois = blobs['rois'].astype(np.float32, copy=False))
pool5_diff = np.ones_like(net.blobs['pool5'].diff, dtype = np.float32)
net.blobs['pool5'].diff[...] = pool5_diff
layerIdx = list(net._layer_names).index('roi_pool5')
net._backward(layerIdx, layerIdx)
conv5_3_diff = net.blobs['conv5_3'].diff.astype(np.float32).copy()
with h5py.File('test_fastrcnn_python.h5', 'w') as file:
file['rois'] = blobs['rois'].astype(np.float32)
file['im'] = blobs['data'].astype(np.float32)
def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2):
print(im_file)
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
attr_prob = net.blobs['attr_prob'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
data_path = './data/genome/1600-400-20'
# Load classes
classes = ['__background__']
with open(os.path.join(data_path, 'objects_vocab.txt')) as f:
for object in f.readlines():
classes.append(object.split(',')[0].lower().strip())
print(os.getcwd())
# Load attributes
attributes = ['__no_attribute__']
with open(os.path.join(data_path, 'attributes_vocab.txt')) as f:
for att in f.readlines():
attributes.append(att.split(',')[0].lower().strip())
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
attr_gt_thresh = np.greater(attr_conf, attr_thresh)
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes': len(keep_boxes),
'boxes': base64.b64encode(cls_boxes[keep_boxes]).decode('ascii'),
'features': base64.b64encode(pool5[keep_boxes]).decode('ascii'),
'classes': base64.b64encode(objects).decode('ascii'),
'attributes': base64.b64encode(attr).decode('ascii'),
#'attr_conf': base64.b64encode(attr_conf).decode('ascii')
'attr_gt_thresh': base64.b64encode(attr_gt_thresh).decode('ascii')
}
def demo(image_name, image_no, image_index, net):
colors = [
"blue", "green", "red", "cyan", "magenta", "yellow", "black", "white",
"darkblue", "orchid", "springgreen", "lime", "deepskyblue",
"mediumvioletred", "maroon", "orangered", "blue", "green", "red",
"cyan", "magenta", "yellow", "black", "white", "darkblue", "orchid",
"springgreen", "lime", "deepskyblue", "mediumvioletred", "maroon",
"orangered", "orangered", "orangered", "yellow", "black", "white",
"darkblue", "orchid", "springgreen", "orangered", "blue", "green",
"red", "cyan", "magenta", "yellow", "black", "white", "darkblue",
"orchid", "springgreen", "lime", "deepskyblue", "mediumvioletred",
"maroon", "orangered", "orangered", "orangered", "yellow", "black",
"white", "darkblue", "orchid", "springgreen", "orangered", "blue",
"green", "red", "cyan", "magenta", "yellow", "black", "white",
"darkblue", "orchid", "springgreen", "lime", "deepskyblue",
"mediumvioletred", "maroon", "orangered", "orangered", "orangered",
"yellow", "black", "white", "darkblue", "orchid", "springgreen",
"orangered", "blue", "green", "red", "cyan", "magenta", "yellow",
"black", "white", "darkblue", "orchid", "springgreen", "lime",
"deepskyblue", "mediumvioletred", "maroon", "orangered", "orangered",
"orangered", "yellow", "black", "white", "darkblue", "orchid",
"springgreen"
]
conf_thresh = 0.4
min_boxes = 15
max_boxes = 15
indexes = []
cfg.TEST.NMS = 0.6
im = cv2.imread(
os.path.join(
"/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/AWA_data/Animals_with_Attributes2/clean_images",
image_name))
cls_append = []
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
print(len(cls_boxes))
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:min_boxes]
elif len(keep_boxes) > max_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:max_boxes]
############################
att_unique = np.unique(att_names[image_index * scale:(image_index * scale +
scale)])
att_unique_adv = np.unique(
att_names_adv[image_index * scale:(image_index * scale + scale)])
# cls_unique=np.unique(att_cls[image_index*scale:(image_index*scale+scale)])
# cls_unique_adv=np.unique(att_cls_adv[image_index*scale:(image_index*scale+scale)])
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
fig.set_size_inches(width / height, 1, forward=False)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
plt.imshow(im)
boxes = cls_boxes[keep_boxes]
#print (boxes)
#print (keep_boxes)
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
count_box = 0
print("image #", image_index)
for i in range(len(keep_boxes)):
bbox = boxes[i]
if bbox[0] == 0:
bbox[0] = 1
if bbox[1] == 0:
bbox[1] = 1
#cls = classes[objects[i]+1]
if attr_conf[i] > attr_thresh:
#for k in range (len(att_unique)):
# for l in range (len(cls_unique)):
#if attributes[attr[i]+1]==att_unique[k]:
# if classes[objects[i]+1] == cls_unique[l]:
#if attributes[attr[i]+1] not in att_unique_adv:
#if classes[objects[i]+1] not in cls_unique_adv:
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"longleg", "leg")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"longneck", "neck")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"patches", "patch")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"bulbous", "round")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"lean", "leaning")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"chewteeth", "teeth")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"meatteeth", "teeth")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"buckteeth", "teeth")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"strainteeth", "teeth")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"flys", "flying")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"swims", "swimming")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"tunnels", "tunnel")
attributes[attr[i] + 1] = attributes[attr[i] + 1].replace(
"walks", "walking")
if attributes[attr[i] + 1] in att_unique:
# if (attributes[attr[i]+1]=="patch"):
cls = attributes[attr[i] + 1]
cls = cls.replace("patch", "have patches")
# cls = attributes[attr[i]+1]
# elif (attributes[attr[i]+1]=="spots"):
cls = cls.replace("spots", "have spots")
# cls = attributes[attr[i]+1]
# elif (attributes[attr[i]+1]=="stripes"):
cls = cls.replace("stripes", "have stripes")
# cls = attributes[attr[i]+1]
# elif (attributes[attr[i]+1]=="furry"):
cls = cls.replace("furry", "have fur")
# cls = attributes[attr[i]+1]
# elif (cls=="hands"):
# elif (cls=="hands"):
cls = cls.replace("hands", "have hands")
# cls = cls
# elif (cls=="pads"):
cls = cls.replace("pads", "have pads")
# cls = cls
# elif (cls=="paws"):
cls = cls.replace("paws", "have paws")
# cls = cls
# elif (cls=="leg"):
cls = cls.replace("leg", "have leg")
# cls = cls
# elif (cls=="neck"):
cls = cls.replace("neck", "have neck")
# cls = cls
# elif (cls=="tail"):
cls = cls.replace("tail", "have tail")
# cls = cls
# elif (cls=="teeth"):
cls = cls.replace("teeth", "have teeth")
# cls = cls
# elif (cls=="horns"):
cls = cls.replace("horns", "have horns")
# cls = cls
# elif (cls=="claws"):
cls = cls.replace("claws", "have claws")
# cls = cls
# elif (cls=="tusks"):
cls = cls.replace("tusks", "have tusks")
# cls = cls
# elif (cls=="flying"):
cls = cls.replace("flying", "is flying")
# cls = cls
# elif (cls=="swimming"):
cls = cls.replace("swimming", "is swimming")
# cls = cls
# elif (cls=="tunnel"):
cls = cls.replace("tunnel", "in tunnel")
# cls = cls
# elif (cls=="walking"):
cls = cls.replace("walking", "is walking")
# cls = cls
# elif (cls=="fish"):
cls = cls.replace("fish", "eats fish")
# cls = cls
# elif (cls=="meat"):
cls = cls.replace("meat", "eats meat")
# cls = cls
# elif (cls=="desert"):
cls = cls.replace("desert", "lives in desert")
# cls = cls
# elif (cls=="bush"):
cls = cls.replace("bush", "lives in bush")
# cls = cls
# elif (cls=="plains"):
cls = cls.replace("plains", "lives in plains")
# cls = cls
# elif (cls=="forest"):
cls = cls.replace("forest", "lives in forest")
# cls = cls
# elif (cls=="fields"):
cls = cls.replace("fields", "lives in fields")
# cls = cls
# elif (cls=="mountains"):
cls = cls.replace("mountains", "lives in mountains")
# cls = cls
# elif (cls=="ocean"):
cls = cls.replace("ocean", "lives in ocean")
# cls = cls
# elif (cls=="ground"):
cls = cls.replace("ground", "lives in ground")
# cls = cls
# elif (cls=="water"):
cls = cls.replace("water", "lives in water")
# cls = cls
# elif (cls=="tree"):
cls = cls.replace("tree", "lives in tree")
# cls = cls
# elif (cls=="group"):
cls = cls.replace("group", "lives in group")
cls = cls.replace("black", "is black")
# cls = cls
# elif (str(cls)=="white"):
cls = cls.replace("white", "is white")
# cls = cls
# elif (str(cls)=="blue"):
cls = cls.replace("blue", "is blue")
# cls = cls
# elif (str(cls)=="brown"):
cls = cls.replace("brown", "is brown")
# cls = cls
# elif (str(cls)=="gray"):
cls = cls.replace("gray", "is gray")
# cls = cls
# elif (str(cls)=="orange"):
cls = cls.replace("orange", "is orange")
# cls = cls
# elif (str(cls)=="yellow"):
cls = cls.replace("yellow", "is yellow")
# cls = cls
# elif (str(cls)=="green"):
cls = cls.replace("green", "is green")
# cls = cls
# elif (str(cls)=="red"):
cls = cls.replace("red", "is red")
# cls = cls
# elif (cls=="furry"):
cls = cls.replace("furry", "is furry")
cls = cls.replace("spots", "have spots")
# cls = cls
# elif (cls=="stripes"):
cls = cls.replace("stripes", "have stripes")
cls = cls.replace("big", "is big")
cls = cls.replace("small", "is small")
# else:
#cls = attributes[attr[i]+1] + " " + classes[objects[i]+1]
# cls = attributes[attr[i]+1] + " " + correct_cls[image_index]
#cls = attributes[attr[i]+1]
# cls = cls.replace('brown','brown '+wrong_cls[image_no])
# cls = cls.replace('black','black '+wrong_cls[image_no])
# cls = cls.replace('white','white '+wrong_cls[image_no])
# cls = cls.replace('blue','blue '+wrong_cls[image_no])
# cls = cls.replace('gray','gray '+wrong_cls[image_no])
# cls = cls.replace('orange','orange '+wrong_cls[image_no])
# cls = cls.replace('red','red '+wrong_cls[image_no])
# cls = cls.replace('yellow','yellow '+wrong_cls[image_no])
# cls = cls.replace('patch','have patches')
# cls = cls.replace('spots','have spots')
cls_append.append(cls)
count = cls_append.count(cls)
if count == 1:
count_box = count_box + 1
plt.gca().add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False,
edgecolor=colors[i],
linewidth=0.3,
alpha=0.5))
plt.gca().text(bbox[0],
bbox[1] - 2,
'%s' % (cls),
bbox=dict(facecolor='blue',
alpha=0,
linewidth=0.2),
fontsize=2.5,
color=colors[i])
plt.savefig(
'/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/AWA_data/Animals_with_Attributes2/clean_bb1/clean_bb{}.jpg'
.format(image_no),
dpi=1500)
#plt.savefig('/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/pytorch-adversarial_box/plots_AT_NoAT/adv_bb_AT/adv_bb_AT{}_25.jpg'.format(image_no), dpi = 1500)
plt.close()
def demo(image_name, image_no, image_index, net):
colors = [
"blue", "green", "red", "cyan", "magenta", "yellow", "black", "white",
"darkblue", "orchid", "springgreen", "lime", "deepskyblue",
"mediumvioletred", "maroon", "orangered", "blue", "green", "red",
"cyan", "magenta", "yellow", "black", "white", "darkblue", "orchid",
"springgreen", "lime", "deepskyblue", "mediumvioletred", "maroon",
"orangered", "blue", "green", "red", "cyan", "magenta", "yellow",
"black", "white", "darkblue", "orchid", "springgreen", "lime",
"deepskyblue", "mediumvioletred", "maroon", "orangered"
]
conf_thresh = 0.4
min_boxes = 15
max_boxes = 15
indexes = []
cfg.TEST.NMS = 0.6
im = cv2.imread(
os.path.join(
"/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/LAD_experiments/Analysis/Analysis_new/adv_images",
image_name))
cls_append = []
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
print(len(cls_boxes))
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:min_boxes]
elif len(keep_boxes) > max_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:max_boxes]
############################
att_unique = np.unique(att_names[image_index * scale:(image_index * scale +
scale)])
att_unique_adv = np.unique(
att_names_adv[image_index * scale:(image_index * scale + scale)])
# cls_unique=np.unique(att_cls[image_index*scale:(image_index*scale+scale)])
# cls_unique_adv=np.unique(att_cls_adv[image_index*scale:(image_index*scale+scale)])
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
fig.set_size_inches(width / height, 1, forward=False)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
plt.imshow(im)
boxes = cls_boxes[keep_boxes]
#print (boxes)
#print (keep_boxes)
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
count_box = 0
print("image #", image_index)
for i in range(len(keep_boxes)):
bbox = boxes[i]
if bbox[0] == 0:
bbox[0] = 1
if bbox[1] == 0:
bbox[1] = 1
#cls = classes[objects[i]+1]
if attr_conf[i] > attr_thresh:
#for k in range (len(att_unique)):
# for l in range (len(cls_unique)):
#if attributes[attr[i]+1]==att_unique[k]:
# if classes[objects[i]+1] == cls_unique[l]:
#if attributes[attr[i]+1] not in att_unique_adv:
#if classes[objects[i]+1] not in cls_unique_adv:
if attributes[attr[i] + 1] in att_unique_adv:
cls = attributes[attr[i] + 1]
# if (str(cls)=="black"):
cls = cls.replace("black", "is black")
# cls = cls
# elif (str(cls)=="white"):
cls = cls.replace("white", "is white")
# cls = cls
# elif (str(cls)=="blue"):
cls = cls.replace("blue", "is blue")
# cls = cls
# elif (str(cls)=="brown"):
cls = cls.replace("brown", "is brown")
# cls = cls
# elif (str(cls)=="gray"):
cls = cls.replace("gray", "is gray")
# cls = cls
# elif (str(cls)=="orange"):
cls = cls.replace("orange", "is orange")
# cls = cls
# elif (str(cls)=="yellow"):
cls = cls.replace("yellow", "is yellow")
# cls = cls
# elif (str(cls)=="green"):
cls = cls.replace("green", "is green")
# cls = cls
# elif (str(cls)=="red"):
cls = cls.replace("red", "is red")
# cls = cls
# elif (cls=="furry"):
cls = cls.replace("furry", "is furry")
# cls = cls
# elif (cls=="feather"):
cls = cls.replace("feather", "has feathers")
# cls = cls
# elif (cls=="whiskers"):
cls = cls.replace("whiskers", "has whiskers")
# cls = cls
# elif (cls=="patch"):
cls = cls.replace("patch", "has patches")
# cls = cls
# elif (cls=="spots"):
cls = cls.replace("spots", "have spots")
# cls = cls
# elif (cls=="stripes"):
cls = cls.replace("stripes", "have stripes")
# cls = cls
# elif (cls=="small"):
cls = cls.replace("small", "is small")
# cls = cls
# elif (cls=="big"):
cls = cls.replace("big", "is big")
# cls = cls
# elif (cls=="legs"):
cls = cls.replace("legs", "has legs")
# cls = cls
# elif (cls=="arms"):
cls = cls.replace("arms", "has arms")
# cls = cls
# elif (cls=="wings"):
cls = cls.replace("wings", "has wings")
# cls = cls
# elif (cls=="paws"):
cls = cls.replace("paws", "has paws")
# cls = cls
# elif (cls=="neck"):
cls = cls.replace("neck", "has neck")
# cls = cls
# elif (cls=="teeth"):
cls = cls.replace("teeth", "has teeth")
# cls = cls
# elif (cls=="tusks"):
cls = cls.replace("tusks", "has tusks")
# cls = cls
# elif (cls=="tails"):
cls = cls.replace("tails", "has tails")
# cls = cls
# elif (cls=="horns"):
cls = cls.replace("horns", "has horns")
# cls = cls
# elif (cls=="horn"):
cls = cls.replace("horn", "has horn")
# cls = cls
# elif (cls=="beak"):
cls = cls.replace("beak", "has beak")
# cls = cls
# elif (cls=="tongue"):
cls = cls.replace("tongue", "have tongue")
# cls = cls
# elif (cls=="eyes"):
cls = cls.replace("eyes", "have eyes")
# cls = cls
# elif (cls=="fin"):
cls = cls.replace("fin", "have fin")
# cls = cls
# elif (cls=="ears"):
cls = cls.replace("ears", "have ears")
# cls = cls
# elif (cls=="nose"):
cls = cls.replace("nose", "have nose")
# cls = cls
# elif (cls=="flying"):
cls = cls.replace("flying", "can fly")
# cls = cls
# elif (cls=="walking"):
cls = cls.replace("walking", "can walk")
# cls = cls
# elif (cls=="eggs"):
cls = cls.replace("eggs", "lays eggs")
# cls = cls
# elif (cls=="jumping"):
cls = cls.replace("jumping", "can jump")
# cls = cls
# elif (cls=="plants"):
cls = cls.replace("plants", "eats plants")
# cls = cls
# elif (cls=="fish"):
cls = cls.replace("fish", "eats fish")
# cls = cls
# elif (cls=="leaves"):
cls = cls.replace("leaves", "eats leaves")
# cls = cls
# elif (cls=="meat"):
cls = cls.replace("meat", "eats meat")
# cls = cls
# elif (cls=="seeds"):
cls = cls.replace("seeds", "eats seeds")
# cls = cls
# elif (cls=="moving"):
cls = cls.replace("moving", "moves")
# cls = cls
# elif (cls=="desert"):
cls = cls.replace("desert", "lives in desert")
# cls = cls
# elif (cls=="bush"):
cls = cls.replace("bush", "lives in bush")
# cls = cls
# elif (cls=="plain"):
cls = cls.replace("plain", "lives in plains")
# cls = cls
# elif (cls=="forest"):
cls = cls.replace("forest", "lives in forest")
# cls = cls
# elif (cls=="field"):
cls = cls.replace("field", "lives in fields")
# cls = cls
# elif (cls=="mountains"):
cls = cls.replace("mountains", "lives in mountains")
# cls = cls
# elif (cls=="ocean"):
cls = cls.replace("ocean", "lives in ocean")
# cls = cls
# elif (cls=="ground"):
cls = cls.replace("ground", "lives in ground")
# cls = cls
# elif (cls=="water"):
cls = cls.replace("water", "lives in water")
# cls = cls
# elif (cls=="trees"):
cls = cls.replace("trees", "lives in trees")
# cls = cls
# elif (cls=="globe"):
cls = cls.replace("globe", "is globular")
# cls = cls
# elif (cls=="star"):
cls = cls.replace("star", "is star shaped")
# cls = cls
# elif (cls=="long"):
cls = cls.replace("long", "is long")
# cls = cls
# elif (cls=="hairy"):
cls = cls.replace("hairy", "is hairy")
# cls = cls
# elif (cls=="rough"):
cls = cls.replace("rough", "is rough")
# cls = cls
# elif (cls=="peeled"):
cls = cls.replace("peeled", "has peel")
# cls = cls
# elif (cls=="crust"):
cls = cls.replace("crust", "has crust")
# cls = cls
# elif (cls=="eating"):
cls = cls.replace("eating", "can be eaten")
# cls = cls
# elif (cls=="doors"):
cls = cls.replace("doors", "has doors")
# cls = cls
# elif (cls=="windows"):
cls = cls.replace("windows", "has windows")
# cls = cls
# elif (cls=="seats"):
cls = cls.replace("seats", "has seats")
# cls = cls
# elif (cls=="jet engine"):
cls = cls.replace("jet engine", "has jet engine")
# cls = cls
# elif (cls=="engine"):
cls = cls.replace("engine", "has engine")
# cls = cls
# elif (cls=="sail"):
cls = cls.replace("sail", "has sail")
# cls = cls
# elif (cls=="mast"):
cls = cls.replace("mast", "has mast")
# cls = cls
# elif (cls=="steering wheel"):
cls = cls.replace("steering wheel", "has steering wheel")
# cls = cls
# elif (cls=="track"):
cls = cls.replace("track", "has track")
# cls = cls
# elif (cls=="brake"):
cls = cls.replace("brake", "has brake")
# cls = cls
# elif (cls=="number"):
cls = cls.replace("number", "has number plate")
# cls = cls
# elif (cls=="propeller"):
cls = cls.replace("propeller", "has propeller")
# cls = cls
# elif (cls=="cable"):
cls = cls.replace("cable", "has cable")
# cls = cls
# elif (cls=="wheels"):
cls = cls.replace("wheels", "has wheels")
# cls = cls
# elif (cls=="wheel"):
cls = cls.replace("wheel", "has wheel")
# cls = cls
# elif (cls=="lights"):
cls = cls.replace("lights", "has lights")
# cls = cls
# elif (cls=="driving"):
cls = cls.replace("driving", "can be driven")
# cls = cls
# elif (cls=="clean"):
cls = cls.replace("clean", "for cleaning")
# cls = cls
# elif (cls=="family"):
cls = cls.replace("family", "for family")
# cls = cls
# elif (cls=="power"):
cls = cls.replace("power", "consumes power")
# cls = cls
# elif (cls=="road"):
cls = cls.replace("road", "used on road")
# cls = cls
# elif (cls=="river"):
cls = cls.replace("river", "used in river")
# cls = cls
# elif (cls=="lake"):
cls = cls.replace("lake", "used in lake")
# cls = cls
# elif (cls=="sea"):
cls = cls.replace("sea", "used in sea")
# cls = cls
# elif (cls=="sky"):
cls = cls.replace("sky", "used in sky")
# cls = cls
# elif (cls=="safety"):
cls = cls.replace("safety", "is safe")
# cls = cls
# elif (cls=="plastic"):
cls = cls.replace("plastic", "made of plastic")
# cls = cls
# elif (cls=="wood"):
cls = cls.replace("wood", "made of wood")
# cls = cls
# elif (cls=="cloth"):
cls = cls.replace("cloth", "made of cloth")
# cls = cls
# elif (cls=="flat"):
cls = cls.replace("flat", "is flat")
# cls = cls
# elif (cls=="screen"):
cls = cls.replace("screen", "has screen")
# cls = cls
# elif (cls=="wristband"):
cls = cls.replace("wristband", "has wristband")
# cls = cls
# elif (cls=="motor"):
cls = cls.replace("motor", "has motor")
# cls = cls
# elif (cls=="fan"):
cls = cls.replace("fan", "has fan")
# cls = cls
# elif (cls=="camera"):
cls = cls.replace("camera", "has camera")
# cls = cls
# elif (cls=="keys"):
cls = cls.replace("keys", "has keys")
# cls = cls
# elif (cls=="headphones"):
cls = cls.replace("headphones", "has headphones")
# cls = cls
# elif (cls=="attena"):
cls = cls.replace("attena", "has attena")
# cls = cls
# elif (cls=="plug"):
cls = cls.replace("plug", "has plug")
# cls = cls
# elif (cls=="refrigerator"):
cls = cls.replace("refrigerator", "can refrigerate")
# cls = cls
# elif (cls=="heater"):
cls = cls.replace("heater", "can heat")
# cls = cls
# elif (cls=="signal"):
cls = cls.replace("signal", "can send signal")
# else:
#
# # cls = attributes[attr[i]+1] + " " + correct_cls[image_index]
# cls = attributes[attr[i]+1]
cls_append.append(cls)
count = cls_append.count(cls)
if count == 1:
count_box = count_box + 1
plt.gca().add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False,
edgecolor=colors[i],
linewidth=0.3,
alpha=0.5))
plt.gca().text(bbox[0],
bbox[1] - 2,
'%s' % (cls),
bbox=dict(facecolor='blue',
alpha=0,
linewidth=0.2),
fontsize=2.5,
color=colors[i])
plt.savefig(
'/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/LAD_experiments/Analysis/Analysis_new/adv_bb/adv_bb{}.jpg'
.format(image_no),
dpi=1500)
#plt.savefig('/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/pytorch-adversarial_box/plots_AT_NoAT/adv_bb_AT/adv_bb_AT{}_25.jpg'.format(image_no), dpi = 1500)
plt.close()
def demo(image_name, image_no, net):
colors = [
"blue", "green", "red", "cyan", "magenta", "yellow", "black", "white",
"darkblue", "orchid", "springgreen", "lime", "deepskyblue",
"mediumvioletred", "maroon", "orangered", "navy", "olive", "orange",
"orangered", "orchid", "pink", "plum", "purple", "salmon", "sienna",
"silver", "tan", "teal", "tomato", "violet", "wheat", "yellow",
"yellowgreen", "lavender", "palevioletred"
]
conf_thresh = 0.3
min_boxes = 36
max_boxes = 36
indexes = []
cfg.TEST.NMS = 0.6
im = cv2.imread(
os.path.join(
"/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/Analysis/clean_images",
image_name))
cls_append = []
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
print(len(cls_boxes))
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:min_boxes]
elif len(keep_boxes) > max_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:max_boxes]
############################
att_unique = np.unique(att_names[image_no * scale:(image_no * scale +
scale)])
print(att_unique)
att_unique_adv = np.unique(att_names_adv[image_no *
scale:(image_no * scale + scale)])
cls_unique = np.unique(att_cls[image_no * scale:(image_no * scale +
scale)])
print(cls_unique)
cls_unique_adv = np.unique(att_cls_adv[image_no * scale:(image_no * scale +
scale)])
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
fig.set_size_inches(width / height, 1, forward=False)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
plt.imshow(im)
#colors=["blue","green","red","cyan","magenta","yellow","black","white","darkblue","orchid","springgreen","lime","deepskyblue","mediumvioletred","maroon","orangered"]
boxes = cls_boxes[keep_boxes]
#print (boxes)
#print (keep_boxes)
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
count_box = 0
for i in range(len(keep_boxes)):
bbox = boxes[i]
if bbox[0] == 0:
bbox[0] = 1
if bbox[1] == 0:
bbox[1] = 1
#cls = classes[objects[i]+1]
if attr_conf[i] > attr_thresh:
#for k in range (len(att_unique)):
# for l in range (len(cls_unique)):
#if attributes[attr[i]+1]==att_unique[k]:
# if classes[objects[i]+1] == cls_unique[l]:
#if attributes[attr[i]+1] not in att_unique_adv:
#if classes[objects[i]+1] not in cls_unique_adv:
if attributes[attr[i] + 1] in att_unique_adv:
if classes[objects[i] + 1] in cls_unique_adv:
cls = attributes[attr[i] + 1] + " " + classes[objects[i] +
1]
cls_append.append(cls)
count = cls_append.count(cls)
if count == 1:
count_box = count_box + 1
plt.gca().add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False,
edgecolor=colors[i],
linewidth=0.3,
alpha=0.5))
plt.gca().text(bbox[0],
bbox[1] - 2,
'%s' % (cls),
bbox=dict(facecolor='blue',
alpha=0,
linewidth=0.2),
fontsize=2.5,
color=colors[i])
#plt.suptitle((correct_cls[int(image_no)])+ " "+(wrong_cls[int(image_no)]),fontsize=2)
plt.savefig(
'/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/Analysis/adv_bb1_ground_clean_/adv_bb_clean{}.jpg'
.format(image_no),
dpi=1500)
#plt.savefig('/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/Analysis/clean_bb/clean_bb{}_50.jpg'.format(image_no), dpi = 1500)
#plt.tight_layout()
plt.close()
def get_detections_from_im(net, im_file, image_id, conf_thresh=0.2, visualize=False):
"""Load im_file and extract bottom-up features using Faster RCNN"""
MIN_BOXES, MAX_BOXES=36,36
NMS_THRESH = 0.05
CONF_THRESH = 0.1
ATTR_THRESH = 0.1
im = cv2.imread(im_file)
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1,cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep], cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
#Normalize scores of best detections, sum their features to
#obtain bottom-up attention features of im
best_scores = max_conf[keep_boxes]
best_feats = pool5[keep_boxes]
scores_norm = np.expand_dims(np.exp(best_scores)/np.sum(np.exp(best_scores))+eps,axis=1)
cumulative_feats = scores_norm.T.dot(best_feats)
sum_feats = np.sum(best_feats,axis=0)
if visualize:
#To visualize the top scoring bounding boxes overlaid on the image im
im = im[:, :, (2, 1, 0)]
fig, ax = plt.subplots(figsize=(12, 12))
ax.imshow(im, aspect='equal')
for cls_ind, cls in enumerate(CLASSES[1:]):
cls_ind += 1 # because we skipped background
cls_boxes = boxes[:, 4*cls_ind:4*(cls_ind + 1)]
cls_scores = scores[:, cls_ind]
dets = np.hstack((cls_boxes,
cls_scores[:, np.newaxis])).astype(np.float32)
keep = nms(dets, NMS_THRESH)
dets = dets[keep, :]
if attr_scores is not None:
attributes = attr_scores[keep]
else:
attributes = None
if rel_scores is not None:
rel_argmax_c = rel_argmax[keep]
rel_score_c = rel_score[keep]
else:
rel_argmax_c = None
rel_score_c = None
vis_detections(ax, cls, dets, attributes, rel_argmax_c, rel_score_c, thresh=CONF_THRESH)
plt.savefig('./'+im_file.split('/')[-1].replace(".jpg", "_demo.png"))
return {
'image_id': image_id,
'image_h': np.size(im, 0),
'image_w': np.size(im, 1),
'num_boxes' : len(keep_boxes),
'boxes': base64.b64encode(cls_boxes[keep_boxes]),
'features': base64.b64encode(pool5[keep_boxes]),
'cumulative_feats':cumulative_feats, #softmax normalized features of best roi's
'sum_feats':sum_feats # features of best roi's summed
}
cfg.TEST.SCALE = (600, )
model_name = 'residual_pos_neg_attention_12'
#model_name = 'rfcn_alt_opt_5step_ohem'
im = cv2.imread('/net/wujial/py-R-FCN/data/demo/006177.jpg')
rpn_proto = "/net/wujial/py-R-FCN/models/pascal_voc/ResNet-50/rfcn_alt_opt_5step_ohem/rpn_test.pt"
rpn_model = "/net/wujial/py-R-FCN/output/rfcn_alt_opt_5step_ohem/voc_2007_trainval/stage1_rpn_final.caffemodel"
prototxt = '/net/wujial/py-R-FCN/models/pascal_voc/ResNet-50/' + model_name + '/soft_rfcn_test.pt'
model = '/net/wujial/py-R-FCN/models/pascal_voc/ResNet-50/' + model_name + '/resnet50_rfcn_mask_ohem_iter_75000.caffemodel'
#model = '/net/wujial/py-R-FCN/models/pascal_voc/ResNet-50/' + model_name + '/stage1_mask_rfcn_final.caffemodel'
#model = "/net/wujial/py-R-FCN/models/pascal_voc/ResNet-50/rfcn_alt_opt_5step_ohem/resnet50_rfcn_ohem_iter_120000.caffemodel"
caffe.set_mode_gpu()
caffe.set_device(2)
rfcn_net = caffe.Net(rpn_proto, rpn_model, caffe.TEST)
blobs, im_scales = _get_blobs(im, None)
im_blob = blobs['data']
blobs['im_info'] = np.array(
[[im_blob.shape[2], im_blob.shape[3], im_scales[0]]], dtype=np.float32)
rfcn_net.blobs['data'].reshape(*(blobs['data'].shape))
rfcn_net.blobs['im_info'].reshape(*(blobs['im_info'].shape))
forward_kwargs = {'data': blobs['data'].astype(np.float32, copy=False)}
forward_kwargs['im_info'] = blobs['im_info'].astype(np.float32, copy=False)
blobs_out = rfcn_net.forward(**forward_kwargs)
rois = rfcn_net.blobs['rois'].data.copy()
boxes = rois[10:30, 1:5] / im_scales[0]
cfg.TEST.HAS_RPN = False
blobs, im_scales = _get_blobs(im, boxes)
rfcn_net = caffe.Net(prototxt, model, caffe.TEST)
def get_detections_from_im(net,
msg_q,
outfile,
num_videos,
conf_thresh=0.2,
attr_thresh=0.1,
part=None):
if part is not None:
attr_p = '../data/video_attr' + str(part) + '.json'
I_idx_p = '../data/I_idx' + str(part) + '.json'
else:
attr_p = '../data/video_attr.json'
I_idx_p = '../data/I_idx.json'
if os.path.exists(attr_p):
with open(attr_p, 'r') as f:
videos_attr_data = json.loads(f.read())
else:
videos_attr_data = {}
if os.path.exists(I_idx_p):
with open(I_idx_p, 'r') as f:
I_idx_data = json.loads(f.read())
else:
I_idx_data = {}
for num_v in tqdm(range(num_videos)):
frames, v_id, I_idx = msg_q.get()
if v_id in videos_attr_data:
continue
I_idx_data[v_id] = I_idx
features = []
video_attr = []
print(v_id)
for i, im in tqdm(enumerate(frames), total=len(frames)):
frame_attr = []
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regresssion bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
cls_prob = net.blobs['cls_prob'].data
pool5 = net.blobs['pool5_flat'].data
attr_prob = net.blobs['attr_prob'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < MIN_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MIN_BOXES]
elif len(keep_boxes) > MAX_BOXES:
keep_boxes = np.argsort(max_conf)[::-1][:MAX_BOXES]
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
for i in range(len(keep_boxes)):
cls = classes[objects[i] + 1]
if attr_conf[i] > attr_thresh:
cls = attributes[attr[i] + 1] + " " + cls
frame_attr.append(cls)
video_attr.append(frame_attr)
features.append(pool5[keep_boxes])
videos_attr_data[v_id] = video_attr
if num_v % 100 == 0:
with open(attr_p, 'w') as f:
f.write(
json.dumps(videos_attr_data,
indent=4,
separators=(',', ': ')))
with open(I_idx_p, 'w') as f:
f.write(
json.dumps(I_idx_data, indent=4, separators=(',', ': ')))
np.save(outfile + v_id, np.array(features))
with open(attr_p, 'w') as f:
f.write(json.dumps(videos_attr_data, indent=4, separators=(',', ': ')))
with open(I_idx_p, 'w') as f:
f.write(json.dumps(I_idx_data, indent=4, separators=(',', ': ')))
def demo(image_name, image_no, net):
conf_thresh = 0.4
min_boxes = 36
max_boxes = 36
indexes = []
cfg.TEST.NMS = 0.6
im = cv2.imread(
os.path.join(
"/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/Analysis/clean_images/",
image_name))
scores, boxes, attr_scores, rel_scores = im_detect(net, im)
# Keep the original boxes, don't worry about the regression bbox outputs
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
blobs, im_scales = _get_blobs(im, None)
cls_boxes = rois[:, 1:5] / im_scales[0]
print(len(cls_boxes))
cls_prob = net.blobs['cls_prob'].data
attr_prob = net.blobs['attr_prob'].data
pool5 = net.blobs['pool5_flat'].data
# Keep only the best detections
max_conf = np.zeros((rois.shape[0]))
for cls_ind in range(1, cls_prob.shape[1]):
cls_scores = scores[:, cls_ind]
dets = np.hstack(
(cls_boxes, cls_scores[:, np.newaxis])).astype(np.float32)
keep = np.array(nms(dets, cfg.TEST.NMS))
max_conf[keep] = np.where(cls_scores[keep] > max_conf[keep],
cls_scores[keep], max_conf[keep])
keep_boxes = np.where(max_conf >= conf_thresh)[0]
if len(keep_boxes) < min_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:min_boxes]
elif len(keep_boxes) > max_boxes:
keep_boxes = np.argsort(max_conf)[::-1][:max_boxes]
############################
att_unique = np.unique(att_names[image_no * scale:(image_no * scale +
scale)])
print(att_unique)
att_unique_adv = np.unique(att_names_adv[image_no *
scale:(image_no * scale + scale)])
cls_unique = np.unique(att_cls[image_no * scale:(image_no * scale +
scale)])
print(cls_unique)
cls_unique_adv = np.unique(att_cls_adv[image_no * scale:(image_no * scale +
scale)])
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
fig.set_size_inches(width / height, 1, forward=False)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
plt.imshow(im)
boxes = cls_boxes[keep_boxes]
#print (boxes)
#print (keep_boxes)
objects = np.argmax(cls_prob[keep_boxes][:, 1:], axis=1)
attr_thresh = 0.1
attr = np.argmax(attr_prob[keep_boxes][:, 1:], axis=1)
attr_conf = np.max(attr_prob[keep_boxes][:, 1:], axis=1)
count_box = 0
for i in range(len(keep_boxes)):
bbox = boxes[i]
if bbox[0] == 0:
bbox[0] = 1
if bbox[1] == 0:
bbox[1] = 1
#cls = classes[objects[i]+1]
if attr_conf[i] > attr_thresh:
#for k in range (len(att_unique)):
# for l in range (len(cls_unique)):
#if attributes[attr[i]+1]==att_unique[k]:
# if classes[objects[i]+1] == cls_unique[l]:
#if attributes[attr[i]+1] not in att_unique_adv:
#if classes[objects[i]+1] not in cls_unique_adv:
if attributes[attr[i] + 1] in att_unique:
if classes[objects[i] + 1] in cls_unique:
cls = attributes[attr[i] + 1] + " " + classes[objects[i] +
1]
count_box = count_box + 1
plt.gca().add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False,
edgecolor='red',
linewidth=0.3,
alpha=0.5))
plt.gca().text(bbox[0],
bbox[1] - 2,
'%s' % (cls),
bbox=dict(facecolor='blue',
alpha=0,
linewidth=0.2),
fontsize=1.5,
color='red')
plt.savefig(
'/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/Analysis/clean_bb/clean_bb{}.jpg'
.format(image_no),
dpi=1500)
#plt.savefig('/media/sadaf/e4da0f25-29be-4c9e-a432-3193ff5f5baf/Code/Pytorch_Code/transfer_learn/pytorch-adversarial_box/plots_AT_NoAT/adv_bb_AT/adv_bb_AT{}_25.jpg'.format(image_no), dpi = 1500)
plt.close()
def im_detect(net, im, boxes=None):
"""Detect object classes in an image given object proposals.
Arguments:
net (caffe.Net): Fast R-CNN network to use
im (ndarray): color image to test (in BGR order)
boxes (ndarray): R x 4 array of object proposals or None (for RPN)
Returns:
scores (ndarray): R x K array of object class scores (K includes
background as object category 0)
boxes (ndarray): R x (4*K) array of predicted bounding boxes
"""
blobs, im_scales = _get_blobs(im, boxes)
# When mapping from image ROIs to feature map ROIs, there's some aliasing
# (some distinct image ROIs get mapped to the same feature ROI).
# Here, we identify duplicate feature ROIs, so we only compute features
# on the unique subset.
if cfg.DEDUP_BOXES > 0 and not cfg.TEST.HAS_RPN:
v = np.array([1, 1e3, 1e6, 1e9, 1e12])
hashes = np.round(blobs['rois'] * cfg.DEDUP_BOXES).dot(v)
_, index, inv_index = np.unique(hashes,
return_index=True,
return_inverse=True)
blobs['rois'] = blobs['rois'][index, :]
boxes = boxes[index, :]
if cfg.TEST.HAS_RPN:
im_blob = blobs['data']
blobs['im_info'] = np.array(
[[im_blob.shape[2], im_blob.shape[3], im_scales[0]]],
dtype=np.float32)
# reshape network inputs
net.blobs['data'].reshape(*(blobs['data'].shape))
if cfg.TEST.HAS_RPN:
net.blobs['im_info'].reshape(*(blobs['im_info'].shape))
else:
net.blobs['rois'].reshape(*(blobs['rois'].shape))
# do forward
forward_kwargs = {'data': blobs['data'].astype(np.float32, copy=False)}
if cfg.TEST.HAS_RPN:
forward_kwargs['im_info'] = blobs['im_info'].astype(np.float32,
copy=False)
else:
forward_kwargs['rois'] = blobs['rois'].astype(np.float32, copy=False)
blobs_out = net.forward(**forward_kwargs)
if cfg.TEST.HAS_RPN:
assert len(im_scales) == 1, "Only single-image batch implemented"
rois = net.blobs['rois'].data.copy()
# unscale back to raw image space
boxes = rois[:, 1:5] / im_scales[0]
if cfg.TEST.SVM:
# use the raw scores before softmax under the assumption they
# were trained as linear SVMs
scores = net.blobs['cls_score'].data
else:
# use softmax estimated probabilities
scores = blobs_out['cls_prob']
if cfg.TEST.BBOX_REG:
# Apply bounding-box regression deltas
box_deltas = blobs_out['bbox_pred']
pred_boxes = bbox_transform_inv(boxes, box_deltas)
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, scores.shape[1]))
if cfg.DEDUP_BOXES > 0 and not cfg.TEST.HAS_RPN:
# Map scores and predictions back to the original set of boxes
scores = scores[inv_index, :]
pred_boxes = pred_boxes[inv_index, :]
return scores, pred_boxes
