def generate_mean_pixel_file():
C = Config()
all_imgs, _, _ = get_data(ROI_BBOX_FILE)
avg = [0, 0, 0]
for img_data in all_imgs:
print(img_data['filepath'])
img_data_aug, x_img = augment(img_data, C, augment=False)
(width, height) = (img_data_aug['width'], img_data_aug['height'])
(rows, cols, _) = x_img.shape
# get image dimensions for resizing
(resized_width,
resized_height) = get_new_img_size(width, height, C.im_size)
# resize the image so that smalles side is length = 600px
x_img = cv2.resize(x_img, (resized_width, resized_height),
interpolation=cv2.INTER_CUBIC)
pixels = (resized_width * resized_height)
avg[0] += np.sum(x_img[:, :, 0]) / pixels
avg[1] += np.sum(x_img[:, :, 1]) / pixels
avg[2] += np.sum(x_img[:, :, 2]) / pixels
avg = [a / len(all_imgs) for a in list(avg)]
np.savetxt(MEAN_PIXEL_FILE, avg, delimiter=',')
def format_img_size(img, C): """ formats the image size based on config """ img_min_side = float(C.im_size) (height,width,_) = img.shape (resized_width, resized_height, ratio) = data_generators.get_new_img_size(width, height, C.im_size) img = cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_CUBIC) return img, ratio
def preprocess_image(image, config):
# Deep-copy the image, changing data type to float
x_img = np.array(image.data, copy=True, dtype=np.float32)
rows, cols, _ = x_img.shape
width, height = cols, rows
# get image dimensions for resizing
resized_width, resized_height = data_generators.get_new_img_size(
width, height, config.im_size)
# resize the image so that smalles side is length = 600px
x_img = cv2.resize(x_img, (resized_width, resized_height),
interpolation=cv2.INTER_CUBIC)
# Build the metadata including bounding boxes in the format keras_frcnn expects
# We filter out bounding boxes for classes we don't care about
bboxes = []
for obj in image.metadata.labelled_objects:
classes = set(obj.class_names) & set(config.class_mapping.keys())
for cls in classes:
bboxes.append({
'class': cls,
'x1': obj.bounding_box[0],
'y1': obj.bounding_box[1],
'x2': obj.bounding_box[0] + obj.bounding_box[2],
'y2': obj.bounding_box[1] + obj.bounding_box[3]
})
metadata = {'width': width, 'height': height, 'bboxes': bboxes}
try:
y_rpn_cls, y_rpn_regr = data_generators.calc_rpn(
config, metadata, width, height, resized_width, resized_height,
nn.get_img_output_length)
except Exception:
return None, None, None
# Zero-center by mean pixel, and preprocess image
x_img[:, :, 0] -= config.img_channel_mean[0]
x_img[:, :, 1] -= config.img_channel_mean[1]
x_img[:, :, 2] -= config.img_channel_mean[2]
x_img /= config.img_scaling_factor
x_img = np.transpose(x_img, (2, 0, 1))
x_img = np.expand_dims(x_img, axis=0)
y_rpn_regr[:, y_rpn_regr.shape[1] // 2:, :, :] *= config.std_scaling
if keras_backend.image_dim_ordering() == 'tf':
x_img = np.transpose(x_img, (0, 2, 3, 1))
y_rpn_cls = np.transpose(y_rpn_cls, (0, 2, 3, 1))
y_rpn_regr = np.transpose(y_rpn_regr, (0, 2, 3, 1))
return np.copy(x_img), [np.copy(y_rpn_cls), np.copy(y_rpn_regr)], metadata
def plot_bbox(img_data, C, bbox):
img_original = cv2.imread(img_data['filepath'])
(width, height) = (img_data['width'], img_data['height'])
(rows, cols, _) = img_original.shape
path_old = img_data['filepath']
assert cols == width
assert rows == height
(resized_width, resized_height) = data_generators.get_new_img_size(
width, height, C.im_size)
if bbox is not None:
for i in range(bbox.shape[1]):
x_c = bbox[0][i][0]
y_c = bbox[0][i][1]
w = bbox[0][i][2]
h = bbox[0][i][3]
x1_resize = x_c - w / 2.0
y1_resize = y_c - w / 2.0
x2_resize = x_c + h / 2.0
y2_resize = y_c + h / 2.0
x1_original = x1_resize * (float(width) / resized_width)
x2_original = x2_resize * (float(width) / resized_width)
y1_original = y1_resize * (float(height) / resized_height)
y2_original = y2_resize * (float(height) / resized_height)
x1 = int(round(x1_original * C.rpn_stride))
x2 = int(round(x2_original * C.rpn_stride))
y1 = int(round(y1_original * C.rpn_stride))
y2 = int(round(y2_original * C.rpn_stride))
cv2.rectangle(img_original, (x1, y1), (x2, y2), (255, 0, 0), 2)
img_ID_start = path_old.find('/JPEGImages/')
img_ID = path_old[img_ID_start + 12:]
img_addr = '/home/xuele/rpn_bf/faster_rcnn/keras-frcnn/bbox_plot/' + img_ID
cv2.imwrite(img_addr, img_original)
def calc_iou(R, img_data, C, class_mapping):
"""
本函数读入图片数据和经过非极大值抑制的rpn预测的坐标数据，对每个提出的predict box进行label的制作
label包括GT 类别， GT坐标
一个box四坐标只能对应一个类别，所以即使一张图里面有很多GT，只能选择iou最大的用于匹配
:param R:
:param img_data:
:param C:
:param class_mapping:
:return:
"""
## img_data_aug 是哈希表，存放各种和当前图象有关的信息
bboxes = img_data['bboxes']
(width, height) = (img_data['width'], img_data['height'])
# get image dimensions for resizing
(resized_width, resized_height) = data_generators.get_new_img_size(
width, height, C.im_size)
## 有多少标注的ground truth boxes 就有多少gta
gta = np.zeros((len(bboxes), 4))
for bbox_num, bbox in enumerate(bboxes):
# get the GT box coordinates, and resize to account for image resizing
## 计算调整后且映射到输出图大小上的ground truth boxes
gta[bbox_num, 0] = int(
round(bbox['x1'] * (resized_width / float(width)) / C.rpn_stride))
gta[bbox_num, 1] = int(
round(bbox['x2'] * (resized_width / float(width)) / C.rpn_stride))
gta[bbox_num, 2] = int(
round(bbox['y1'] * (resized_height / float(height)) /
C.rpn_stride))
gta[bbox_num, 3] = int(
round(bbox['y2'] * (resized_height / float(height)) /
C.rpn_stride))
x_roi = []
y_class_num = []
y_class_regr_coords = []
y_class_regr_label = []
IoUs = [] # for debugging only
## R 只有 [bbox, prob] 中的 bbox 对应坐标值
## A: layerkinds*w*h, coor， 这里[0]会小于layerkinds*w*h 因为最多300
for ix in range(R.shape[0]):
(x1, y1, x2, y2) = R[ix, :]
x1 = int(round(x1))
y1 = int(round(y1))
x2 = int(round(x2))
y2 = int(round(y2))
best_iou = 0.0
best_bbox = -1
for bbox_num in range(len(bboxes)):
curr_iou = data_generators.iou([
gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1],
gta[bbox_num, 3]
], [x1, y1, x2, y2])
if curr_iou > best_iou:
best_iou = curr_iou
best_bbox = bbox_num
## 选择可以用来最终分类的predict box
if best_iou < C.classifier_min_overlap: #if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:
continue
else:
w = x2 - x1
h = y2 - y1
x_roi.append([x1, y1, w, h])
IoUs.append(best_iou)
## 难例挖掘，与目标有部分重合但不够判断，分类成背景
## 一张图里面有再多的类
if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap: #if best_iou < C.classifier_min_overlap:
# hard negative example
cls_name = 'bg'
elif C.classifier_max_overlap <= best_iou:
cls_name = bboxes[best_bbox]['class']
## ground truth box的中心点
cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0
cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0
## predict box的中心值
cx = x1 + w / 2.0
cy = y1 + h / 2.0
tx = (cxg - cx) / float(w)
ty = (cyg - cy) / float(h)
tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))
th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))
else:
print('roi = {}'.format(best_iou))
raise RuntimeError
class_num = class_mapping[cls_name]
class_label = len(class_mapping) * [0]
class_label[class_num] = 1
y_class_num.append(copy.deepcopy(class_label))
## -1 是因为背景
coords = [0] * 4 * (len(class_mapping) - 1)
labels = [0] * 4 * (len(class_mapping) - 1)
if cls_name != 'bg':
label_pos = 4 * class_num
sx, sy, sw, sh = C.classifier_regr_std
coords[label_pos:4 +
label_pos] = [sx * tx, sy * ty, sw * tw, sh * th]
labels[label_pos:4 + label_pos] = [1, 1, 1, 1]
y_class_regr_coords.append(copy.deepcopy(coords))
y_class_regr_label.append(copy.deepcopy(labels))
else:
y_class_regr_coords.append(copy.deepcopy(coords))
y_class_regr_label.append(copy.deepcopy(labels))
if len(x_roi) == 0:
return None, None, None, None
## 感觉像创建label
X = np.array(x_roi)
Y1 = np.array(y_class_num)
## axis= 1 这样同属于同一个像素的位置的同一个格式anchorbox的label coord 标签在同一行
Y2 = np.concatenate(
[np.array(y_class_regr_label),
np.array(y_class_regr_coords)], axis=1)
return np.expand_dims(X, axis=0), np.expand_dims(
Y1, axis=0), np.expand_dims(Y2, axis=0), IoUs
landmarks.append(np.zeros(42)) [new_bboxes,new_bboxes_rpn, new_probs,new_poses, new_genders,new_vizs, new_landmarks] = roi_helpers.non_max_suppression_fast_classifier( np.array(bboxes),np.array(bboxes_rpn),np.array(probs),np.array(poses), np.array(genders),np.array(vizs), np.array(landmarks), overlap_thresh=0.2) base_threshold = 0.5 true_bboxes = img_data['bboxes'] gta = np.zeros((len(true_bboxes), 4)) (width, height) = (img_data['width'], img_data['height']) (resized_width, resized_height) = data_generators.get_new_img_size(width, height, C.im_size) # bp() for bbox_num, bbox in enumerate(true_bboxes): # get the GT box coordinates, and resize to account for image resizing # bbox_gt2pred[bbox_num] = (-1,0) gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width)))) gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width)))) gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height)))) gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height)))) for jk in range(new_bboxes.shape[0]): # bbox_pred2tr[jk] = False ################################################ best_iou = 0.0 best_bbox = 0 best_score = 0
def test_view_func(C, model_rpn, model_classifier):
base_dir = os.getcwd()
test_cls_all = ['aeroplane', 'bus', 'motorbike']
class_mapping = C.class_mapping
inv_class_mapping = {v: k for k, v in class_mapping.iteritems()}
backend = K.image_dim_ordering()
filename = '/home/gilad/bar/real7.p'
video_filename = "/home/gilad/ssd/keras-frcnn-master/a.mp4"
write_flag = False
class_to_color = {
class_mapping[v]: np.random.randint(0, 255, 3)
for v in class_mapping
}
# turn off any data augmentation at test time
save_flag = False
visualise = False
count = 0
good_img = 0
not_good = 0
mAP = 0
def format_img_size(img, C):
""" formats the image size based on config """
img_min_side = float(C.im_size)
(height, width, _) = img.shape
if width <= height:
ratio = img_min_side / width
new_height = int(ratio * height)
new_width = int(img_min_side)
else:
ratio = img_min_side / height
new_width = int(ratio * width)
new_height = int(img_min_side)
img = cv2.resize(img, (new_width, new_height),
interpolation=cv2.INTER_CUBIC)
return img, ratio
def format_img_channels(img, C):
""" formats the image channels based on config """
img = img[:, :, (2, 1, 0)]
img = img.astype(np.float32)
img[:, :, 0] -= C.img_channel_mean[0]
img[:, :, 1] -= C.img_channel_mean[1]
img[:, :, 2] -= C.img_channel_mean[2]
img /= C.img_scaling_factor
img = np.transpose(img, (2, 0, 1))
img = np.expand_dims(img, axis=0)
return img
def draw_bbox(img, bbox, prob, azimuth, ratio):
# new_boxes, new_probs, new_az = roi_helpers.non_max_suppression_fast(bbox, prob, azimuth, overlap_thresh=0.3,use_az=True)
new_boxes = bbox
new_az = azimuth
new_probs = prob
for jk in range(new_boxes.shape[0]):
(x1, y1, x2, y2) = new_boxes[jk, :]
(real_x1, real_y1, real_x2,
real_y2) = get_real_coordinates(ratio, x1, y1, x2, y2)
cv2.rectangle(
img, (real_x1, real_y1), (real_x2, real_y2),
(int(class_to_color[key][0]), int(
class_to_color[key][1]), int(class_to_color[key][2])), 2)
# cv2.rectangle(img,(bbox_gt['x1'], bbox_gt['y1']), (bbox_gt['x2'], bbox_gt['y2']), (int(class_to_color[key][0]), int(class_to_color[key][1]), int(class_to_color[key][2])),2)
# textLabel = '{}: {},azimuth : {}'.format(key,int(100*new_probs[jk]),new_az[jk])
textLabel = 'azimuth : {}'.format(new_az[jk])
all_dets.append((key, 100 * new_probs[jk]))
(retval, baseLine) = cv2.getTextSize(textLabel,
cv2.FONT_HERSHEY_COMPLEX, 1,
1)
textOrg = (real_x1, real_y1 + 15)
cv2.rectangle(
img, (textOrg[0] - 5, textOrg[1] + baseLine - 5),
(textOrg[0] + retval[0] + 5, textOrg[1] - retval[1] - 5),
(0, 0, 0), 2)
cv2.rectangle(
img, (textOrg[0] - 5, textOrg[1] + baseLine - 5),
(textOrg[0] + retval[0] + 5, textOrg[1] - retval[1] - 5),
(255, 255, 255), -1)
cv2.putText(img, textLabel, textOrg, cv2.FONT_HERSHEY_DUPLEX, 1,
(0, 0, 0), 1)
return img
def format_img(img, C):
""" formats an image for model prediction based on config """
img, ratio = format_img_size(img, C)
img = format_img_channels(img, C)
return img, ratio
def display_image(img):
img1 = img[:, :, (2, 1, 0)]
# img1=img
im = Image.fromarray(img1.astype('uint8'), 'RGB')
im.show()
# Method to transform the coordinates of the bounding box to its original size
def get_real_coordinates(ratio, x1, y1, x2, y2):
## read the training data from pickle file or from annotations
real_x1 = int(round(x1 // ratio))
real_y1 = int(round(y1 // ratio))
real_x2 = int(round(x2 // ratio))
real_y2 = int(round(y2 // ratio))
return (real_x1, real_y1, real_x2, real_y2)
vnum_test = 24
azimuth_vec = np.concatenate(
([0],
np.linspace((360. / (vnum_test * 2)), 360. -
(360. / (vnum_test * 2)), vnum_test)),
axis=0)
def find_interval(azimuth, azimuth_vec):
for i in range(len(azimuth_vec)):
if azimuth < azimuth_vec[i]:
break
ind = i
if azimuth > azimuth_vec[-1]:
ind = 1
return ind
# print(rep)
obj_num = 0
bbox_threshold_orig = 0.6
th_bbox = 0.3
#### open images from folder
# for idx, img_name in enumerate(sorted(os.listdir(img_path))):
# if not img_name.lower().endswith(('.bmp', '.jpeg', '.jpg', '.png', '.tif', '.tiff')):
# continue
# print(img_name)
# filepath = os.path.join(img_path,img_name)
# img = cv2.imread(filepath)caricycle
#### open images from file
## read the training data from pickle file or from annotations
# class_mapping = C.class_mapping
succ = []
for test_cls in test_cls_all:
good_img = 0
not_good = 0
count = 0
obj_num = 0
gt_cls_num = class_mapping[test_cls]
print('work on class {}'.format(test_cls))
test_pickle = os.path.join(
base_dir, 'pickle_data/test_data_{}.pickle'.format(test_cls))
if os.path.exists(test_pickle):
with open(test_pickle) as f:
all_imgs, classes_count, _ = pickle.load(f)
for im_file in all_imgs:
filepath = im_file['filepath']
img = cv2.imread(filepath)
img_gt = np.copy(img)
if img is None:
not_good += 1
continue
else:
good_img += 1
# print ('im num {}'.format(good_img))
if good_img % 50 == 0:
print("worked on {} images".format(good_img))
X, ratio = format_img(img, C)
if backend == 'tf':
X = np.transpose(X, (0, 2, 3, 1))
# get the feature maps and output from the RPN
[Y1, Y2] = model_rpn.predict(X)
R = roi_helpers.rpn_to_roi(Y1,
Y2,
C,
K.image_dim_ordering(),
overlap_thresh=0.7)
# # convert from (x1,y1,x2,y2) to (x,y,w,h)
R[:, 2] -= R[:, 0]
R[:, 3] -= R[:, 1]
width, height = int(im_file["width"]), int(im_file["height"])
resized_width, resized_height = data_generators.get_new_img_size(
width, height, C.im_size)
# [_,_, F] = model_rpn.predict(X)
ROIs = []
## pass on all the labels in the image, some of them are not equal to test_cls
for bbox_gt in im_file['bboxes']:
if not bbox_gt['class'] == test_cls:
continue
no_bbox_flag = 1
bbox_threshold = bbox_threshold_orig
while no_bbox_flag and bbox_threshold > th_bbox:
cls_gt = bbox_gt['class']
az_gt = bbox_gt['azimuth']
el_gt = bbox_gt['elevation']
t_gt = bbox_gt['tilt']
if bbox_gt[
'class'] == test_cls and bbox_threshold == bbox_threshold_orig:
obj_num += 1
if len(ROIs) == 0:
# apply the spatial pyramid pooling to the proposed regions
bboxes = {}
probs = {}
azimuths = {}
# print ('obj num {}'.format(obj_num))
for jk in range(R.shape[0] // C.num_rois + 1):
ROIs = np.expand_dims(
R[C.num_rois * jk:C.num_rois * (jk + 1), :],
axis=0)
if ROIs.shape[1] == 0:
break
if jk == R.shape[0] // C.num_rois:
#pad R
curr_shape = ROIs.shape
target_shape = (curr_shape[0], C.num_rois,
curr_shape[2])
ROIs_padded = np.zeros(target_shape).astype(
ROIs.dtype)
ROIs_padded[:, :curr_shape[1], :] = ROIs
ROIs_padded[0, curr_shape[1]:, :] = ROIs[0,
0, :]
ROIs = ROIs_padded
[P_cls, P_regr,
P_view] = model_classifier.predict([X, ROIs])
for ii in range(P_cls.shape[1]):
if np.max(P_cls[0, ii, :]
) < bbox_threshold or np.argmax(
P_cls[0, ii, :]) == (
P_cls.shape[2] - 1):
continue
## get class from the net
# cls_num = np.argmax(P_cls[0, ii, :])
## use gt class
cls_num = gt_cls_num
cls_name = inv_class_mapping[cls_num]
cls_view = P_view[0, ii, 360 * cls_num:360 *
(cls_num + 1)]
# cls_name_gt = cls_nimg = draw_bbox(img,bbox, prob, azimuth, ratio)ame
# if cls_name == cls_name_gt:
# print(np.argmax(cls_view,axis=0))
if cls_name not in bboxes:
bboxes[cls_name] = []
probs[cls_name] = []
azimuths[cls_name] = []
(x, y, w, h) = ROIs[0, ii, :]
try:
(tx, ty, tw, th) = P_regr[0, ii,
4 * cls_num:4 *
(cls_num + 1)]
tx /= C.classifier_regr_std[0]
ty /= C.classifier_regr_std[1]
tw /= C.classifier_regr_std[2]
th /= C.classifier_regr_std[3]
x, y, w, h = roi_helpers.apply_regr(
x, y, w, h, tx, ty, tw, th)
except:
pass
bboxes[cls_name].append([
C.rpn_stride * x, C.rpn_stride * y,
C.rpn_stride * (x + w),
C.rpn_stride * (y + h)
])
probs[cls_name].append(np.max(P_cls[0, ii, :]))
azimuths[cls_name].append(
np.argmax(cls_view, axis=0))
all_dets = []
if len(bboxes) == 0:
bbox_threshold -= 0.1
for key in bboxes:
# if 1:
if key == test_cls and bbox_gt['class'] == test_cls:
bbox = np.array(bboxes[key])
prob = np.array(probs[key])
azimuth = np.array(azimuths[key])
## get the azimuth from bbox that have more than 'overlap_thresh' overlap with gt_bbox
az = []
overlap_thresh = 0.5
try:
while np.size(
az) == 0 and overlap_thresh > 0.3:
_, prob_bbox, az = roi_helpers.overlap_with_gt(
bbox,
prob,
azimuth,
bbox_gt,
ratio=ratio,
overlap_thresh=overlap_thresh,
max_boxes=300,
use_az=True)
if np.size(
az) != 0 and overlap_thresh == 0.5:
mAP += 1
overlap_thresh -= 0.1
if overlap_thresh == 0:
print("No good Bbox was found")
counts = np.bincount(az)
except:
az = []
counts = []
try:
az_fin = np.argmax(counts)
true_bin = find_interval(az_gt, azimuth_vec)
prob_bin = find_interval(az_fin, azimuth_vec)
no_bbox_flag = 0
if true_bin == prob_bin:
count += 1
break
except:
# print('here')
no_bbox_flag = 1
bbox_threshold -= 0.1
## azimuth calculations
## display
# if visualise:
# display_image(img)
# # cv2.imshow('img', img)
# # cv2.waitKey(0)
# if save_flag:
# cv2.imwrite('./results_imgs/{}'.format(img_name),img)
# # img = img[:, :, (2, 1, 0)]
# # cv2.imwrite('./results_imgs/video/{}.png'.format(num),img)
# # print('save')
bbox_threshold -= 0.1
# if visualise:
# display_image(img)
succ.append(float(count) / float(obj_num) * 100.)
string = 'for class {} -true count is {} out of {} from {} images . {} success'.format(
test_cls, count, obj_num, good_img,
float(count) / float(obj_num) * 100.)
print(string)
mAP = float(mAP) / float(obj_num) * 100.
print("MAP is {}".format(mAP))
# if write_flag:
# f = open('{}_results.txt'.format(weight_name),'a')
# f.write(string+'\n')
# f.close()
return succ, mAP
def test_view_func_NN(model_classifier, model_rpn, model_inner, C):
test_cls = 'aeroplane'
input_train_file = 'pickle_data/train_data_Wflip_all.pickle'
## read the training data from pickle file or from annotations
test_pickle = 'pickle_data/test_data_{}.pickle'.format(test_cls)
if os.path.exists(test_pickle):
with open(test_pickle) as f:
all_imgs, classes_count, _ = pickle.load(f)
class_mapping = C.class_mapping
inv_class_mapping = {v: k for k, v in class_mapping.iteritems()}
backend = K.image_dim_ordering()
gt_cls_num = class_mapping[test_cls]
print('work on class {}'.format(test_cls))
base_path = os.getcwd()
# turn off any data augmentation at test time
C.use_horizontal_flips = False
C.use_vertical_flips = False
C.rot_90 = False
count = 0
good_img = 0
not_good = 0
def format_img_size(img, C):
""" formats the image size based on config """
img_min_side = float(C.im_size)
(height, width, _) = img.shape
if width <= height:
ratio = img_min_side / width
new_height = int(ratio * height)
new_width = int(img_min_side)
else:
ratio = img_min_side / height
new_width = int(ratio * width)
new_height = int(img_min_side)
img = cv2.resize(img, (new_width, new_height),
interpolation=cv2.INTER_CUBIC)
return img, ratio
def format_img_channels(img, C):
""" formats the image channels based on config """
img = img[:, :, (2, 1, 0)]
img = img.astype(np.float32)
img[:, :, 0] -= C.img_channel_mean[0]
img[:, :, 1] -= C.img_channel_mean[1]
img[:, :, 2] -= C.img_channel_mean[2]
img /= C.img_scaling_factor
img = np.transpose(img, (2, 0, 1))
img = np.expand_dims(img, axis=0)
return img
def format_img(img, C):
""" formats an image for model prediction based on config """
img, ratio = format_img_size(img, C)
img = format_img_channels(img, C)
return img, ratio
def display_image(img):
img1 = img[:, :, (2, 1, 0)]
# img1=img
im = Image.fromarray(img1.astype('uint8'), 'RGB')
im.show()
# Method to transform the coordinates of the bounding box to its original size
def get_real_coordinates(ratio, x1, y1, x2, y2):
## read the training data from pickle file or from annotations
real_x1 = int(round(x1 // ratio))
real_y1 = int(round(y1 // ratio))
real_x2 = int(round(x2 // ratio))
real_y2 = int(round(y2 // ratio))
return (real_x1, real_y1, real_x2, real_y2)
vnum_test = 24
azimuth_vec = np.concatenate(
([0],
np.linspace((360. / (vnum_test * 2)), 360. -
(360. / (vnum_test * 2)), vnum_test)),
axis=0)
def find_interval(azimuth, azimuth_vec):
for i in range(len(azimuth_vec)):
if azimuth < azimuth_vec[i]:
break
ind = i
if azimuth > azimuth_vec[-1]:
ind = 1
return ind
class_mapping = C.class_mapping
if 'bg' not in class_mapping:
class_mapping['bg'] = len(class_mapping)
class_mapping = {v: k for k, v in class_mapping.items()}
# print(class_mapping)
class_to_color = {
class_mapping[v]: np.random.randint(0, 255, 3)
for v in class_mapping
}
C.num_rois = 32
obj_num = 0
bbox_threshold_orig = 0.6
th_bbox = 0.4
## get GT for all az for single cls
feature_az = []
sorted_path = input_train_file
tmp_ind = sorted_path.index('.pickle')
sorted_path = sorted_path[:tmp_ind] + "_sorted_Angles" + sorted_path[
tmp_ind:]
if os.path.exists(sorted_path):
print("loading sorted data")
with open(sorted_path) as f:
trip_data = pickle.load(f)
im_file = []
ind = []
for ii in range(360):
for jj in range(3):
try:
im_file.append(trip_data[test_cls][ii][jj])
ind.append(ii)
except:
if jj == 0:
print('no azimuth {}'.format(ii))
data_gen_train = data_generators.get_anchor_gt(im_file, [],
C,
K.image_dim_ordering(),
mode='test')
azimuth_dict = []
inner_NN = []
azimuths = []
for tt in range(len(ind)):
try:
if tt % 100 == 0:
print('worked on {}/{}'.format(tt, len(ind)))
# print ('im num {}'.format(good_img))
X, Y, img_data = next(data_gen_train)
P_rpn = model_rpn.predict_on_batch(X)
R = roi_helpers.rpn_to_roi(P_rpn[0],
P_rpn[1],
C,
K.image_dim_ordering(),
use_regr=True,
overlap_thresh=0.7,
max_boxes=300)
X2, Y1, Y2, Y_view = roi_helpers.calc_iou_new(
R, img_data, C, C.class_mapping)
pos_samples = np.where(Y1[0, :, -1] == 0)
sel_samples = pos_samples[0].tolist()
R = X2[0, sel_samples, :]
for jk in range(R.shape[0] // C.num_rois + 1):
ROIs = np.expand_dims(R[C.num_rois * jk:C.num_rois *
(jk + 1), :],
axis=0)
if ROIs.shape[1] == 0:
break
if jk == R.shape[0] // C.num_rois:
# pad R
curr_shape = ROIs.shape
target_shape = (curr_shape[0], C.num_rois, curr_shape[2])
ROIs_padded = np.zeros(target_shape).astype(ROIs.dtype)
ROIs_padded[:, :curr_shape[1], :] = ROIs
ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :]
ROIs = ROIs_padded
[P_cls, P_regr, P_view] = model_classifier.predict([X, ROIs])
iner_f = model_inner.predict([X, ROIs])
# oo = model_classifier_only.predict([F, ROIs])
for ii in range(len(sel_samples)):
if np.max(P_cls[0,
ii, :]) < bbox_threshold_orig or np.argmax(
P_cls[0,
ii, :]) == (P_cls.shape[2] - 1):
continue
## get class from the net
# cls_num = np.argmax(P_cls[0, ii, :])
## use gt class
cls_num = gt_cls_num
cls_name = inv_class_mapping[cls_num]
cls_view = P_view[0, ii, 360 * cls_num:360 * (cls_num + 1)]
# azimuths[cls_name].append(np.argmax(cls_view, axis=0))
inner_NN.append(iner_f[0, ii, :])
azimuth_dict.append(img_data['bboxes'][0]['azimuth'])
except:
print('failed on az {}'.format(img_data['bboxes'][0]['azimuth']))
## calculating some mean feature map for every az
with open('pickle_data/{}_NN.pickle'.format(C.weight_name), 'w') as f:
pickle.dump([inner_NN, azimuth_dict], f)
print('saved PICKLE')
with open('pickle_data/{}_NN.pickle'.format(C.weight_name)) as f:
inner_NN, azimuth_dict = pickle.load(f)
neigh = KNeighborsClassifier(n_neighbors=1)
neigh.fit(inner_NN, azimuth_dict)
jj = 0
for im_file in all_imgs:
jj += 1
if jj % 50 == 0:
print(jj)
filepath = im_file['filepath']
img = cv2.imread(filepath)
img_gt = np.copy(img)
if img is None:
not_good += 1
continue
else:
good_img += 1
# print ('im num {}'.format(good_img))
X, ratio = format_img(img, C)
if backend == 'tf':
X = np.transpose(X, (0, 2, 3, 1))
# get the feature maps and output from the RPN
Y1, Y2 = model_rpn.predict(X)
R = roi_helpers.rpn_to_roi(Y1,
Y2,
C,
K.image_dim_ordering(),
overlap_thresh=0.7)
# # convert from (x1,y1,x2,y2) to (x,y,w,h)
R[:, 2] -= R[:, 0]
R[:, 3] -= R[:, 1]
width, height = int(im_file["width"]), int(im_file["height"])
resized_width, resized_height = data_generators.get_new_img_size(
width, height, C.im_size)
# [_,_, F] = model_rpn.predict(X)
ROIs = []
## pass on all the labels in the image, some of them are not equal to test_cls
for bbox_gt in im_file['bboxes']:
no_bbox_flag = 1
bbox_threshold = bbox_threshold_orig
if not bbox_gt['class'] == test_cls:
continue
if bbox_gt[
'class'] == test_cls and bbox_threshold == bbox_threshold_orig:
obj_num += 1
while no_bbox_flag and bbox_threshold > th_bbox:
cls_gt = bbox_gt['class']
az_gt = bbox_gt['azimuth']
el_gt = bbox_gt['elevation']
t_gt = bbox_gt['tilt']
if len(ROIs) == 0:
# apply the spatial pyramid pooling to the proposed regions
bboxes = {}
probs = {}
azimuths = {}
inner_res = {}
# print ('obj num {}'.format(obj_num))
for jk in range(R.shape[0] // C.num_rois + 1):
ROIs = np.expand_dims(R[C.num_rois * jk:C.num_rois *
(jk + 1), :],
axis=0)
if ROIs.shape[1] == 0:
break
if jk == R.shape[0] // C.num_rois:
#pad R
curr_shape = ROIs.shape
target_shape = (curr_shape[0], C.num_rois,
curr_shape[2])
ROIs_padded = np.zeros(target_shape).astype(
ROIs.dtype)
ROIs_padded[:, :curr_shape[1], :] = ROIs
ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :]
ROIs = ROIs_padded
[P_cls, P_regr,
P_view] = model_classifier.predict([X, ROIs])
inner_out = model_inner.predict([X, ROIs])
# oo = model_classifier_only.predict([F, ROIs])
for ii in range(P_cls.shape[1]):
if np.max(P_cls[
0, ii, :]) < bbox_threshold or np.argmax(
P_cls[0,
ii, :]) == (P_cls.shape[2] - 1):
continue
## get class from the net
# cls_num = np.argmax(P_cls[0, ii, :])
## use gt class
cls_num = gt_cls_num
cls_name = inv_class_mapping[cls_num]
cls_view = P_view[0, ii, 360 * cls_num:360 *
(cls_num + 1)]
if cls_name not in bboxes:
bboxes[cls_name] = []
probs[cls_name] = []
azimuths[cls_name] = []
inner_res[cls_name] = []
(x, y, w, h) = ROIs[0, ii, :]
try:
(tx, ty, tw,
th) = P_regr[0, ii,
4 * cls_num:4 * (cls_num + 1)]
tx /= C.classifier_regr_std[0]
ty /= C.classifier_regr_std[1]
tw /= C.classifier_regr_std[2]
th /= C.classifier_regr_std[3]
x, y, w, h = roi_helpers.apply_regr(
x, y, w, h, tx, ty, tw, th)
except:
pass
bboxes[cls_name].append([
C.rpn_stride * x, C.rpn_stride * y,
C.rpn_stride * (x + w), C.rpn_stride * (y + h)
])
probs[cls_name].append(np.max(P_cls[0, ii, :]))
azimuths[cls_name].append(
np.argmax(cls_view, axis=0))
inner_res[cls_name].append(inner_out[0, ii, :])
# cv2.rectangle(img_gt, (bbox_gt['x1'], bbox_gt['y1']), (bbox_gt['x2'], bbox_gt['y2']), (int(class_to_color[test_cls][0]), int(class_to_color[test_cls][1]), int(class_to_color[test_cls][2])), 2)
for key in bboxes:
# if 1:
if key == test_cls and bbox_gt['class'] == test_cls:
bbox = np.array(bboxes[key])
prob = np.array(probs[key])
azimuth = np.array(azimuths[key])
inner_result = np.array(inner_res[key])
# img = draw_bbox(img,bbox, prob, azimuth, ratio)
azimuth = neigh.predict(inner_result)
## get the azimuth from bbox that have more than 'overlap_thresh' overlap with gt_bbox
az = []
overlap_thresh = 0.5
try:
while np.size(az) == 0 and overlap_thresh > 0:
_, prob_bbox, az = roi_helpers.overlap_with_gt(
bbox,
prob,
azimuth,
bbox_gt,
ratio=ratio,
overlap_thresh=overlap_thresh,
max_boxes=300,
use_az=True)
overlap_thresh -= 0.1
if overlap_thresh == 0:
print("No good Bbox was found")
counts = np.bincount(az)
except:
az = []
counts = []
try:
az_fin = np.argmax(counts)
true_bin = find_interval(az_gt, azimuth_vec)
prob_bin = find_interval(az_fin, azimuth_vec)
no_bbox_flag = 0
if true_bin == prob_bin:
count += 1
break
except:
# print('here')
no_bbox_flag = 1
bbox_threshold -= 0.1
## azimuth calculations
## display
bbox_threshold -= 0.1
succ = float(count) / float(obj_num) * 100.
print(
'for class {} -true count is {} out of {} from {} images . {} success'.
format(test_cls, count, obj_num, good_img, succ))
return succ
def calc_iou(R, img_data, C, class_mapping):
bboxes = img_data['bboxes']
(width, height) = (img_data['width'], img_data['height'])
# get image dimensions for resizing
(resized_width, resized_height) = data_generators.get_new_img_size(width, height, C.im_size)
gta = np.zeros((len(bboxes), 4))
for bbox_num, bbox in enumerate(bboxes):
# get the GT box coordinates, and resize to account for image resizing
gta[bbox_num, 0] = int(round(bbox['x1'] * (resized_width / float(width))/C.rpn_stride))
gta[bbox_num, 1] = int(round(bbox['x2'] * (resized_width / float(width))/C.rpn_stride))
gta[bbox_num, 2] = int(round(bbox['y1'] * (resized_height / float(height))/C.rpn_stride))
gta[bbox_num, 3] = int(round(bbox['y2'] * (resized_height / float(height))/C.rpn_stride))
x_roi = []
y_class_num = []
y_class_regr_coords = []
y_class_regr_label = []
for ix in range(R.shape[0]):
(x1, y1, x2, y2) = R[ix, :]
x1 = int(round(x1))
y1 = int(round(y1))
x2 = int(round(x2))
y2 = int(round(y2))
best_iou = 0.0
best_bbox = -1
for bbox_num in range(len(bboxes)):
curr_iou = data_generators.iou([gta[bbox_num, 0], gta[bbox_num, 2], gta[bbox_num, 1], gta[bbox_num, 3]], [x1, y1, x2, y2])
if curr_iou > best_iou:
best_iou = curr_iou
best_bbox = bbox_num
if best_iou < C.classifier_min_overlap:
continue
else:
w = x2 - x1
h = y2 - y1
x_roi.append([x1, y1, w, h])
if C.classifier_min_overlap <= best_iou < C.classifier_max_overlap:
# hard negative example
cls_name = 'bg'
elif C.classifier_max_overlap <= best_iou:
cls_name = bboxes[best_bbox]['class']
cxg = (gta[best_bbox, 0] + gta[best_bbox, 1]) / 2.0
cyg = (gta[best_bbox, 2] + gta[best_bbox, 3]) / 2.0
cx = x1 + w / 2.0
cy = y1 + h / 2.0
tx = (cxg - cx) / float(w)
ty = (cyg - cy) / float(h)
tw = np.log((gta[best_bbox, 1] - gta[best_bbox, 0]) / float(w))
th = np.log((gta[best_bbox, 3] - gta[best_bbox, 2]) / float(h))
else:
print('roi = {}'.format(best_iou))
raise RuntimeError
class_num = class_mapping[cls_name]
class_label = len(class_mapping) * [0]
class_label[class_num] = 1
y_class_num.append(copy.deepcopy(class_label))
coords = [0] * 4 * (len(class_mapping) - 1)
labels = [0] * 4 * (len(class_mapping) - 1)
if cls_name != 'bg':
label_pos = 4 * class_num
sx, sy, sw, sh = C.classifier_regr_std
coords[label_pos:4+label_pos] = [sx*tx, sy*ty, sw*tw, sh*th]
labels[label_pos:4+label_pos] = [1, 1, 1, 1]
y_class_regr_coords.append(copy.deepcopy(coords))
y_class_regr_label.append(copy.deepcopy(labels))
else:
y_class_regr_coords.append(copy.deepcopy(coords))
y_class_regr_label.append(copy.deepcopy(labels))
if len(x_roi) == 0:
return None, None, None
X = np.array(x_roi)
Y1 = np.array(y_class_num)
Y2 = np.concatenate([np.array(y_class_regr_label),np.array(y_class_regr_coords)],axis=1)
return np.expand_dims(X, axis=0), np.expand_dims(Y1, axis=0), np.expand_dims(Y2, axis=0)