def split_labels_XY(self, pred_labels, eps_std_div=25.0): """ Method which splits labels into X labels, and Y labels. Inputs: pred_labels: (Annotationlist) List of annotations of the predicted labels. eps_std_div (float): Parameters which controls how far appart the bounding boxes are allowed to be on an axis. Outputs: pred_labels_X, pred_labels_Y (Annotationlist, Annotationlist): Annotation lists for the predicted labels on the X and Y axis. """ pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList() for j in range(len(pred_labels)): pred_X, pred_Y = al.Annotation(), al.Annotation() pred_X.imageName, pred_Y.imageName = pred_labels[ j].imageName, pred_labels[j].imageName X, Y = [], [] for rect in pred_labels[j].rects: X.append((rect.x2 + rect.x1) / 2.0) Y.append((rect.y2 + rect.y1) / 2.0) X, Y = np.reshape(np.array(X), (len(X), 1)), np.reshape(np.array(Y), (len(Y), 1)) std_x = np.std(X) std_y = np.std(Y) if std_x > 0 and std_y > 0: db_scan_x = DBSCAN(eps=std_y / eps_std_div, min_samples=2).fit(Y) db_scan_y = DBSCAN(eps=std_x / eps_std_div, min_samples=2).fit(X) cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_) cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_) if len(cluster_list_x) > 0 and len(cluster_list_y) > 0: cluster_label_x = max(set(cluster_list_x), key=cluster_list_x.count) cluster_label_y = max(set(cluster_list_y), key=cluster_list_y.count) rect_x, rect_y = [], [] for k in range(len(db_scan_x.labels_)): if db_scan_x.labels_[ k] == cluster_label_x and db_scan_y.labels_[ k] != cluster_label_y: rect_x.append(pred_labels[j].rects[k]) elif db_scan_y.labels_[ k] == cluster_label_y and db_scan_x.labels_[ k] != cluster_label_x: rect_y.append(pred_labels[j].rects[k]) pred_X.rects, pred_Y.rects = rect_x, rect_y pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y) return pred_labels_X, pred_labels_Y
def split_labels_XY(self, pred_labels, eps_std_div=25.0): #function to cluster the (labels, tick value) to either X or Y axis pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList() for j in range(len(pred_labels)): pred_X, pred_Y = al.Annotation(), al.Annotation() pred_X.imageName, pred_Y.imageName = pred_labels[ j].imageName, pred_labels[j].imageName X, Y = [], [] for rect in pred_labels[j].rects: X.append((rect.x2 + rect.x1) / 2.0) Y.append((rect.y2 + rect.y1) / 2.0) X, Y = np.reshape(np.array(X), (len(X), 1)), np.reshape(np.array(Y), (len(Y), 1)) std_x = np.std(X) std_y = np.std(Y) if std_x > 0 and std_y > 0: db_scan_x = DBSCAN(eps=std_y / eps_std_div, min_samples=2).fit(Y) db_scan_y = DBSCAN(eps=std_x / eps_std_div, min_samples=2).fit(X) cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_) cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_) if len(cluster_list_x) > 0 and len(cluster_list_y) > 0: cluster_label_x = max(set(cluster_list_x), key=cluster_list_x.count) cluster_label_y = max(set(cluster_list_y), key=cluster_list_y.count) rect_x, rect_y = [], [] for k in range(len(db_scan_x.labels_)): if db_scan_x.labels_[ k] == cluster_label_x and db_scan_y.labels_[ k] != cluster_label_y: rect_x.append(pred_labels[j].rects[k]) elif db_scan_y.labels_[ k] == cluster_label_y and db_scan_x.labels_[ k] != cluster_label_x: rect_y.append(pred_labels[j].rects[k]) pred_X.rects, pred_Y.rects = rect_x, rect_y pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y) return pred_labels_X, pred_labels_Y
def predict_model(self, model, H, true_annos, image_dir, conf_threshold): """ Method uses to get the prediction from the object detection models. Inputs: model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model H (dictionary): Loaded json hype which describes the hyperparameters of the models. true_annos (Annotationlist): List of annotations which contain the ground truth bounding boxes. image_dir: (string): Path to the location of the images Outputs: annolist (Annotationlist): List of annotations which contain the predicted bounding boxes. """ annolist = al.AnnoList() with tf.Session(graph=model['graph']) as sess: sess.run(tf.global_variables_initializer()) model['saver'].restore( sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration)) for i in range(len(true_annos)): true_anno = true_annos[i] img = self.open_image(image_dir + true_anno.imageName) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') (np_pred_boxes, np_pred_confidences) = sess.run( [model['pred_boxes'], model['pred_confidences']], feed_dict={model['x_in']: img}) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) return annolist
def get_closest_ticks(self, pred_labels, pred_ticks): """ Method that matches ticks with their closest labels Inputs: pred_labels (Annotationlist): List of annotations (bounding boxes) for the predicted labels pred_ticks (Annotationlist): List of annotations (bounding boxes) for the predicted ticks Outputs annolist (Annotationlist): List of annotations which has the bounding boxes of the ticks, but the label values of the labels. """ annolist = al.AnnoList() for j in range(len(pred_labels)): new_annot = al.Annotation() new_annot.imageName = pred_labels[j].imageName if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0: distances = np.zeros( (len(pred_labels[j].rects), len(pred_ticks[j].rects))) for k in range(len(pred_labels[j].rects)): for i in range(len(pred_ticks[j].rects)): distances[k, i] = pred_labels[j].rects[k].distance( pred_ticks[j].rects[i]) min_index_labels = np.argmin(distances, axis=1) min_index_ticks = np.argmin(distances, axis=0) new_rects = [] for k in range(len(pred_labels[j].rects)): min_index = min_index_labels[k] if min_index_ticks[min_index] == k: temp_rect = pred_ticks[j].rects[min_index] temp_rect.classID = pred_labels[j].rects[k].classID new_rects.append(temp_rect) new_annot.rects = new_rects annolist.append(new_annot) else: new_annot.rects = [] annolist.append(new_annot) return annolist
def get_closest_ticks(self, pred_labels, pred_ticks): # calculating the closest distance between the (tick value,tick mark) annolist = al.AnnoList() for j in range(len(pred_labels)): new_annot = al.Annotation() new_annot.imageName = pred_labels[j].imageName if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0: distances = np.zeros( (len(pred_labels[j].rects), len(pred_ticks[j].rects))) for k in range(len(pred_labels[j].rects)): for i in range(len(pred_ticks[j].rects)): distances[k, i] = pred_labels[j].rects[k].distance( pred_ticks[j].rects[i]) min_index_labels = np.argmin(distances, axis=1) min_index_ticks = np.argmin(distances, axis=0) new_rects = [] for k in range(len(pred_labels[j].rects)): min_index = min_index_labels[k] if min_index_ticks[min_index] == k: temp_rect = pred_ticks[j].rects[min_index] temp_rect.classID = pred_labels[j].rects[k].classID new_rects.append(temp_rect) new_annot.rects = new_rects annolist.append(new_annot) else: new_annot.rects = [] annolist.append(new_annot) return annolist
def predict_model(self, model, H, true_annos, image_dir, conf_threshold): #function to get the predicted bounding boxes annolist = al.AnnoList() with tf.Session(graph=model['graph']) as sess: sess.run(tf.initialize_all_variables()) for i in range(len(true_annos)): true_anno = true_annos[i] img = self.open_image(image_dir + true_anno.imageName) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') (np_pred_boxes, np_pred_confidences) = sess.run( [model['pred_boxes'], model['pred_confidences']], feed_dict={model['x_in']: img}) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) return annolist
def main(model_dir, image_dir, true_idl, iteration, iou_threshold, conf_threshold): hypes_file = '{}/hypes.json'.format(model_dir) with open(hypes_file, 'r') as f: H = json.load(f) model_name = model_dir.split("/")[1] pred_idl = './output/%s_%d_val_%s.idl' % ( model_name, iteration, os.path.basename(hypes_file).replace( '.json', '')) true_annos = al.parse(true_idl) tf.reset_default_graph() googlenet = googlenet_load.init(H) x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3]) if H['use_rezoom']: pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward( H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None) grid_area = H['grid_height'] * H['grid_width'] pred_confidences = tf.reshape( tf.nn.softmax( tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])), [grid_area, H['rnn_len'], 2]) if H['reregress']: pred_boxes = pred_boxes + pred_boxes_deltas else: pred_boxes, pred_logits, pred_confidences = build_forward( H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None) saver = tf.train.Saver() with tf.Session() as sess: #sess.run(tf.initialize_all_variables()) print(saver) saver.restore(sess, '{}/save.ckpt-{}'.format(model_dir, iteration)) annolist = al.AnnoList() t = time.time() for i in range(len(true_annos)): true_anno = true_annos[i] img = Image.open(image_dir + "/" + true_anno.imageName) bg = Image.new("RGB", img.size, (255, 255, 255)) bg.paste(img, img) img = np.array(bg) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: true_anno = rescale_boxes(img.shape, true_anno, H["image_height"], H["image_width"]) img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') feed = {x_in: img} (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold) new_img_true = np.copy(img) new_img_pred = np.copy(img) for rect_true in true_anno.rects: cv2.rectangle(new_img_true, (int(rect_true.x1), int(rect_true.y1)), (int(rect_true.x2), int(rect_true.y2)), (0, 255, 0), 2) for rect_pred in rects: cv2.rectangle(new_img_pred, (int(rect_pred.x1), int(rect_pred.y1)), (int(rect_pred.x2), int(rect_pred.y2)), (0, 0, 255), 2) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) if i % 10 == 0 and i < 400: imsave( "{}/".format(model_dir) + pred_anno.imageName.split('/')[-1][:-4] + '_pred.bmp', new_img_pred) imsave( "{}/".format(model_dir) + pred_anno.imageName.split('/')[-1][:-4] + '_true.bmp', new_img_true) if (i + 1) % 200 == 0 or i == 0: mylogger.debug("Number of images analyzed: {}".format(i + 1)) avg_time = (time.time() - t) / (i + 1) mylogger.debug('%f images/sec' % (1. / avg_time)) annolist.save(pred_idl) rpc_cmd = './tensorbox/utils/annolist/doRPC.py --minOverlap %f %s %s' % ( iou_threshold, true_idl, pred_idl) mylogger.debug('$ %s' % rpc_cmd) rpc_output = subprocess.check_output(rpc_cmd, shell=True) mylogger.debug(rpc_output) txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1] output_png = 'output/{}_{}_results.png'.format(model_name, iteration) plot_cmd = './tensorbox/utils/annolist/plotSimple.py %s --output %s' % ( txt_file, output_png) mylogger.debug('$ %s' % plot_cmd) plot_output = subprocess.check_output(plot_cmd, shell=True) Image2(filename=output_png) df = pd.read_csv( "output/rpc-{}_{}_val_hypes_overlap{}.txt".format( model_name, iteration, iou_threshold), sep=" ", names=['precision', 'recall', 'fpii', 'score', 'accuracy']) auc = metrics.auc(df['recall'], df['precision']) mylogger.info("Average Precision: {}".format(auc))
def predict_model_no_truth(self, model, H, image_dir, conf_threshold, image_output_dir=None): """ Method uses to get the prediction from the object detection models. Inputs: model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model H (dictionary): Loaded json hype which describes the hyperparameters of the models. image_dir: (string): Path to the location of the images Outputs: annolist (Annotationlist): List of annotations which contain the predicted bounding boxes. """ if image_output_dir == None: image_output_dir = image_dir annolist = al.AnnoList() img_list = [] for fmt in ["jpg", "jpeg", "png", "tiff", "tif", "gif", "bmp"]: img_list.extend(glob(os.path.join(image_dir, "*.{}".format(fmt)))) with tf.Session(graph=model['graph']) as sess: sess.run(tf.global_variables_initializer()) model['saver'].restore( sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration)) for img_file in img_list: img = self.open_image(img_file) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') (np_pred_boxes, np_pred_confidences) = sess.run( [model['pred_boxes'], model['pred_confidences']], feed_dict={model['x_in']: img}) pred_anno = al.Annotation() pred_anno.imageName = img_file new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold, show_suppressed=False) imsave(os.path.join(image_output_dir, img_file + "_pred.png"), new_img) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) return annolist