def generate_data_trip(img_data, C, backend,draw_flag = False): X, Y, img_data= data_generators.get_anchor_gt_trip(img_data, C, backend) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=0.7, max_boxes=300) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, Y_view = roi_helpers.calc_iou_new(R, img_data, C, class_mapping) pos_samples = np.where(Y_view[0, :, -1] != 20) if C.num_rois > 1: if len(pos_samples) < C.num_rois: selected_pos_samples = pos_samples[0].tolist() else: selected_pos_samples = np.random.choice(pos_samples[0], C.num_rois, replace=False).tolist() R,Y_view = roi_helpers.prep_flip(X2[:,selected_pos_samples,:],Y_view[:,selected_pos_samples,:],C) if draw_flag: Im = cv2.imread(img_data['filepath']) key = img_data['bboxes'][0]['class'] azimuth = img_data['bboxes'][0]['azimuth'] bbox = np.array([[img_data['bboxes'][0]['x1'],img_data['bboxes'][0]['y1'],img_data['bboxes'][0]['x2'],img_data['bboxes'][0]['y2']]]) img = img_helpers.draw_bbox(Im, bbox, 0, [azimuth], 1, class_mapping_inv, key) img_helpers.display_image(img,0) return X,R,Y_view
# loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=0.7, max_boxes=300) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format # X2, Y1, Y2 = roi_helpers.calc_iou(R, img_data, C, class_mapping) X2, Y1, Y2, Y_view = roi_helpers.calc_iou_new( R, img_data, C, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y_view[0, :, -1] == 20) pos_samples = np.where(Y_view[0, :, -1] != 20) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0:
def test_view_func_NN(model_classifier, model_rpn, model_inner, C): test_cls = 'aeroplane' input_train_file = 'pickle_data/train_data_Wflip_all.pickle' ## read the training data from pickle file or from annotations test_pickle = 'pickle_data/test_data_{}.pickle'.format(test_cls) if os.path.exists(test_pickle): with open(test_pickle) as f: all_imgs, classes_count, _ = pickle.load(f) class_mapping = C.class_mapping inv_class_mapping = {v: k for k, v in class_mapping.iteritems()} backend = K.image_dim_ordering() gt_cls_num = class_mapping[test_cls] print('work on class {}'.format(test_cls)) base_path = os.getcwd() # turn off any data augmentation at test time C.use_horizontal_flips = False C.use_vertical_flips = False C.rot_90 = False count = 0 good_img = 0 not_good = 0 def format_img_size(img, C): """ formats the image size based on config """ img_min_side = float(C.im_size) (height, width, _) = img.shape if width <= height: ratio = img_min_side / width new_height = int(ratio * height) new_width = int(img_min_side) else: ratio = img_min_side / height new_width = int(ratio * width) new_height = int(img_min_side) img = cv2.resize(img, (new_width, new_height), interpolation=cv2.INTER_CUBIC) return img, ratio def format_img_channels(img, C): """ formats the image channels based on config """ img = img[:, :, (2, 1, 0)] img = img.astype(np.float32) img[:, :, 0] -= C.img_channel_mean[0] img[:, :, 1] -= C.img_channel_mean[1] img[:, :, 2] -= C.img_channel_mean[2] img /= C.img_scaling_factor img = np.transpose(img, (2, 0, 1)) img = np.expand_dims(img, axis=0) return img def format_img(img, C): """ formats an image for model prediction based on config """ img, ratio = format_img_size(img, C) img = format_img_channels(img, C) return img, ratio def display_image(img): img1 = img[:, :, (2, 1, 0)] # img1=img im = Image.fromarray(img1.astype('uint8'), 'RGB') im.show() # Method to transform the coordinates of the bounding box to its original size def get_real_coordinates(ratio, x1, y1, x2, y2): ## read the training data from pickle file or from annotations real_x1 = int(round(x1 // ratio)) real_y1 = int(round(y1 // ratio)) real_x2 = int(round(x2 // ratio)) real_y2 = int(round(y2 // ratio)) return (real_x1, real_y1, real_x2, real_y2) vnum_test = 24 azimuth_vec = np.concatenate( ([0], np.linspace((360. / (vnum_test * 2)), 360. - (360. / (vnum_test * 2)), vnum_test)), axis=0) def find_interval(azimuth, azimuth_vec): for i in range(len(azimuth_vec)): if azimuth < azimuth_vec[i]: break ind = i if azimuth > azimuth_vec[-1]: ind = 1 return ind class_mapping = C.class_mapping if 'bg' not in class_mapping: class_mapping['bg'] = len(class_mapping) class_mapping = {v: k for k, v in class_mapping.items()} # print(class_mapping) class_to_color = { class_mapping[v]: np.random.randint(0, 255, 3) for v in class_mapping } C.num_rois = 32 obj_num = 0 bbox_threshold_orig = 0.6 th_bbox = 0.4 ## get GT for all az for single cls feature_az = [] sorted_path = input_train_file tmp_ind = sorted_path.index('.pickle') sorted_path = sorted_path[:tmp_ind] + "_sorted_Angles" + sorted_path[ tmp_ind:] if os.path.exists(sorted_path): print("loading sorted data") with open(sorted_path) as f: trip_data = pickle.load(f) im_file = [] ind = [] for ii in range(360): for jj in range(3): try: im_file.append(trip_data[test_cls][ii][jj]) ind.append(ii) except: if jj == 0: print('no azimuth {}'.format(ii)) data_gen_train = data_generators.get_anchor_gt(im_file, [], C, K.image_dim_ordering(), mode='test') azimuth_dict = [] inner_NN = [] azimuths = [] for tt in range(len(ind)): try: if tt % 100 == 0: print('worked on {}/{}'.format(tt, len(ind))) # print ('im num {}'.format(good_img)) X, Y, img_data = next(data_gen_train) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], C, K.image_dim_ordering(), use_regr=True, overlap_thresh=0.7, max_boxes=300) X2, Y1, Y2, Y_view = roi_helpers.calc_iou_new( R, img_data, C, C.class_mapping) pos_samples = np.where(Y1[0, :, -1] == 0) sel_samples = pos_samples[0].tolist() R = X2[0, sel_samples, :] for jk in range(R.shape[0] // C.num_rois + 1): ROIs = np.expand_dims(R[C.num_rois * jk:C.num_rois * (jk + 1), :], axis=0) if ROIs.shape[1] == 0: break if jk == R.shape[0] // C.num_rois: # pad R curr_shape = ROIs.shape target_shape = (curr_shape[0], C.num_rois, curr_shape[2]) ROIs_padded = np.zeros(target_shape).astype(ROIs.dtype) ROIs_padded[:, :curr_shape[1], :] = ROIs ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :] ROIs = ROIs_padded [P_cls, P_regr, P_view] = model_classifier.predict([X, ROIs]) iner_f = model_inner.predict([X, ROIs]) # oo = model_classifier_only.predict([F, ROIs]) for ii in range(len(sel_samples)): if np.max(P_cls[0, ii, :]) < bbox_threshold_orig or np.argmax( P_cls[0, ii, :]) == (P_cls.shape[2] - 1): continue ## get class from the net # cls_num = np.argmax(P_cls[0, ii, :]) ## use gt class cls_num = gt_cls_num cls_name = inv_class_mapping[cls_num] cls_view = P_view[0, ii, 360 * cls_num:360 * (cls_num + 1)] # azimuths[cls_name].append(np.argmax(cls_view, axis=0)) inner_NN.append(iner_f[0, ii, :]) azimuth_dict.append(img_data['bboxes'][0]['azimuth']) except: print('failed on az {}'.format(img_data['bboxes'][0]['azimuth'])) ## calculating some mean feature map for every az with open('pickle_data/{}_NN.pickle'.format(C.weight_name), 'w') as f: pickle.dump([inner_NN, azimuth_dict], f) print('saved PICKLE') with open('pickle_data/{}_NN.pickle'.format(C.weight_name)) as f: inner_NN, azimuth_dict = pickle.load(f) neigh = KNeighborsClassifier(n_neighbors=1) neigh.fit(inner_NN, azimuth_dict) jj = 0 for im_file in all_imgs: jj += 1 if jj % 50 == 0: print(jj) filepath = im_file['filepath'] img = cv2.imread(filepath) img_gt = np.copy(img) if img is None: not_good += 1 continue else: good_img += 1 # print ('im num {}'.format(good_img)) X, ratio = format_img(img, C) if backend == 'tf': X = np.transpose(X, (0, 2, 3, 1)) # get the feature maps and output from the RPN Y1, Y2 = model_rpn.predict(X) R = roi_helpers.rpn_to_roi(Y1, Y2, C, K.image_dim_ordering(), overlap_thresh=0.7) # # convert from (x1,y1,x2,y2) to (x,y,w,h) R[:, 2] -= R[:, 0] R[:, 3] -= R[:, 1] width, height = int(im_file["width"]), int(im_file["height"]) resized_width, resized_height = data_generators.get_new_img_size( width, height, C.im_size) # [_,_, F] = model_rpn.predict(X) ROIs = [] ## pass on all the labels in the image, some of them are not equal to test_cls for bbox_gt in im_file['bboxes']: no_bbox_flag = 1 bbox_threshold = bbox_threshold_orig if not bbox_gt['class'] == test_cls: continue if bbox_gt[ 'class'] == test_cls and bbox_threshold == bbox_threshold_orig: obj_num += 1 while no_bbox_flag and bbox_threshold > th_bbox: cls_gt = bbox_gt['class'] az_gt = bbox_gt['azimuth'] el_gt = bbox_gt['elevation'] t_gt = bbox_gt['tilt'] if len(ROIs) == 0: # apply the spatial pyramid pooling to the proposed regions bboxes = {} probs = {} azimuths = {} inner_res = {} # print ('obj num {}'.format(obj_num)) for jk in range(R.shape[0] // C.num_rois + 1): ROIs = np.expand_dims(R[C.num_rois * jk:C.num_rois * (jk + 1), :], axis=0) if ROIs.shape[1] == 0: break if jk == R.shape[0] // C.num_rois: #pad R curr_shape = ROIs.shape target_shape = (curr_shape[0], C.num_rois, curr_shape[2]) ROIs_padded = np.zeros(target_shape).astype( ROIs.dtype) ROIs_padded[:, :curr_shape[1], :] = ROIs ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :] ROIs = ROIs_padded [P_cls, P_regr, P_view] = model_classifier.predict([X, ROIs]) inner_out = model_inner.predict([X, ROIs]) # oo = model_classifier_only.predict([F, ROIs]) for ii in range(P_cls.shape[1]): if np.max(P_cls[ 0, ii, :]) < bbox_threshold or np.argmax( P_cls[0, ii, :]) == (P_cls.shape[2] - 1): continue ## get class from the net # cls_num = np.argmax(P_cls[0, ii, :]) ## use gt class cls_num = gt_cls_num cls_name = inv_class_mapping[cls_num] cls_view = P_view[0, ii, 360 * cls_num:360 * (cls_num + 1)] if cls_name not in bboxes: bboxes[cls_name] = [] probs[cls_name] = [] azimuths[cls_name] = [] inner_res[cls_name] = [] (x, y, w, h) = ROIs[0, ii, :] try: (tx, ty, tw, th) = P_regr[0, ii, 4 * cls_num:4 * (cls_num + 1)] tx /= C.classifier_regr_std[0] ty /= C.classifier_regr_std[1] tw /= C.classifier_regr_std[2] th /= C.classifier_regr_std[3] x, y, w, h = roi_helpers.apply_regr( x, y, w, h, tx, ty, tw, th) except: pass bboxes[cls_name].append([ C.rpn_stride * x, C.rpn_stride * y, C.rpn_stride * (x + w), C.rpn_stride * (y + h) ]) probs[cls_name].append(np.max(P_cls[0, ii, :])) azimuths[cls_name].append( np.argmax(cls_view, axis=0)) inner_res[cls_name].append(inner_out[0, ii, :]) # cv2.rectangle(img_gt, (bbox_gt['x1'], bbox_gt['y1']), (bbox_gt['x2'], bbox_gt['y2']), (int(class_to_color[test_cls][0]), int(class_to_color[test_cls][1]), int(class_to_color[test_cls][2])), 2) for key in bboxes: # if 1: if key == test_cls and bbox_gt['class'] == test_cls: bbox = np.array(bboxes[key]) prob = np.array(probs[key]) azimuth = np.array(azimuths[key]) inner_result = np.array(inner_res[key]) # img = draw_bbox(img,bbox, prob, azimuth, ratio) azimuth = neigh.predict(inner_result) ## get the azimuth from bbox that have more than 'overlap_thresh' overlap with gt_bbox az = [] overlap_thresh = 0.5 try: while np.size(az) == 0 and overlap_thresh > 0: _, prob_bbox, az = roi_helpers.overlap_with_gt( bbox, prob, azimuth, bbox_gt, ratio=ratio, overlap_thresh=overlap_thresh, max_boxes=300, use_az=True) overlap_thresh -= 0.1 if overlap_thresh == 0: print("No good Bbox was found") counts = np.bincount(az) except: az = [] counts = [] try: az_fin = np.argmax(counts) true_bin = find_interval(az_gt, azimuth_vec) prob_bin = find_interval(az_fin, azimuth_vec) no_bbox_flag = 0 if true_bin == prob_bin: count += 1 break except: # print('here') no_bbox_flag = 1 bbox_threshold -= 0.1 ## azimuth calculations ## display bbox_threshold -= 0.1 succ = float(count) / float(obj_num) * 100. print( 'for class {} -true count is {} out of {} from {} images . {} success'. format(test_cls, count, obj_num, good_img, succ)) return succ