def split_labels_XY(self, pred_labels, eps_std_div=25.0): """ Method which splits labels into X labels, and Y labels. Inputs: pred_labels: (Annotationlist) List of annotations of the predicted labels. eps_std_div (float): Parameters which controls how far appart the bounding boxes are allowed to be on an axis. Outputs: pred_labels_X, pred_labels_Y (Annotationlist, Annotationlist): Annotation lists for the predicted labels on the X and Y axis. """ pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList() for j in range(len(pred_labels)): pred_X, pred_Y = al.Annotation(), al.Annotation() pred_X.imageName, pred_Y.imageName = pred_labels[ j].imageName, pred_labels[j].imageName X, Y = [], [] for rect in pred_labels[j].rects: X.append((rect.x2 + rect.x1) / 2.0) Y.append((rect.y2 + rect.y1) / 2.0) X, Y = np.reshape(np.array(X), (len(X), 1)), np.reshape(np.array(Y), (len(Y), 1)) std_x = np.std(X) std_y = np.std(Y) if std_x > 0 and std_y > 0: db_scan_x = DBSCAN(eps=std_y / eps_std_div, min_samples=2).fit(Y) db_scan_y = DBSCAN(eps=std_x / eps_std_div, min_samples=2).fit(X) cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_) cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_) if len(cluster_list_x) > 0 and len(cluster_list_y) > 0: cluster_label_x = max(set(cluster_list_x), key=cluster_list_x.count) cluster_label_y = max(set(cluster_list_y), key=cluster_list_y.count) rect_x, rect_y = [], [] for k in range(len(db_scan_x.labels_)): if db_scan_x.labels_[ k] == cluster_label_x and db_scan_y.labels_[ k] != cluster_label_y: rect_x.append(pred_labels[j].rects[k]) elif db_scan_y.labels_[ k] == cluster_label_y and db_scan_x.labels_[ k] != cluster_label_x: rect_y.append(pred_labels[j].rects[k]) pred_X.rects, pred_Y.rects = rect_x, rect_y pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y) return pred_labels_X, pred_labels_Y
def test(self,image_dir, image_output_dir, csv_output_dir , true_idl_dict = None, coord_idl = None, predict_idl = None, quick = False, conf_threshold = 0.3, max_dist_perc = 2.0): """ Method used to test the object on a set of scatter plots. Inputs: true_idl_dict (dictionary): Dictionary where the key is the object name (ticks, points, labels) and the values are the path to their corresponding idl file which may or may not contain the ground truth. image_dir: (string): Path to the location of the images image_output_dir (string): Path to the location where we want to save new images with marked bounding boxes. csv_output_dir (string): Path to the location where we want to save the output csv for each images. coord_idl: (string) Optional, ground truth idl for the real coordinate values. Outputs: pred_dict: (dictionary) Dictionary of dataframes which contains the predicted coordinate values. """ if not os.path.exists(image_output_dir): os.makedirs(image_output_dir) if not os.path.exists(csv_output_dir): os.makedirs(csv_output_dir) if true_idl_dict is not None: true_annos_dict = {key:al.parse(true_idl_dict[key]) for key in true_idl_dict} else: true_annos_dict = {key:al.parse(predict_idl) for key in self.my_dir_dict} start_time = time.time() pred_dict = {} for key in self.models: pred_dict[key] = self.predict_model(self.models[key], self.H[key], true_annos_dict[key], image_dir, conf_threshold) pred_dict[key].save('{}/model_pred_{}.idl'.format(self.models[key]['dir'],self.iteration)) self.draw_images(image_dir, true_annos_dict, pred_dict, image_output_dir, true_idl_dict) pred_dict['labels'] = self.get_ocr(pred_dict['labels'], image_dir) pred_dict['labels'] = self.get_closest_ticks(pred_dict['labels'], pred_dict['ticks']) pred_labels_X, pred_labels_Y = self.split_labels_XY(pred_dict['labels']) self.regressor_x_dict = self.get_conversion(pred_labels_X, cat = 'x') self.regressor_y_dict = self.get_conversion(pred_labels_Y, cat = 'y') df_dict = self.predict_points(pred_dict['points'],self.regressor_x_dict, self.regressor_y_dict, csv_output_dir) mylogger.debug("{} images/sec".format(float(len(pred_dict['labels']))/(time.time()-start_time))) if coord_idl is not None: mylogger.debug("Now using ground truth to test ...") self.get_metrics(df_dict, coord_idl, csv_output_dir, quick = quick, max_dist_perc = max_dist_perc) return pred_dict
def predict_model(self, model, H, true_annos, image_dir, conf_threshold): """ Method uses to get the prediction from the object detection models. Inputs: model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model H (dictionary): Loaded json hype which describes the hyperparameters of the models. true_annos (Annotationlist): List of annotations which contain the ground truth bounding boxes. image_dir: (string): Path to the location of the images Outputs: annolist (Annotationlist): List of annotations which contain the predicted bounding boxes. """ annolist = al.AnnoList() with tf.Session(graph=model['graph']) as sess: sess.run(tf.global_variables_initializer()) model['saver'].restore( sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration)) for i in range(len(true_annos)): true_anno = true_annos[i] img = self.open_image(image_dir + true_anno.imageName) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') (np_pred_boxes, np_pred_confidences) = sess.run( [model['pred_boxes'], model['pred_confidences']], feed_dict={model['x_in']: img}) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) return annolist
def split_labels_XY(self, pred_labels, eps_std_div=25.0): #function to cluster the (labels, tick value) to either X or Y axis pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList() for j in range(len(pred_labels)): pred_X, pred_Y = al.Annotation(), al.Annotation() pred_X.imageName, pred_Y.imageName = pred_labels[ j].imageName, pred_labels[j].imageName X, Y = [], [] for rect in pred_labels[j].rects: X.append((rect.x2 + rect.x1) / 2.0) Y.append((rect.y2 + rect.y1) / 2.0) X, Y = np.reshape(np.array(X), (len(X), 1)), np.reshape(np.array(Y), (len(Y), 1)) std_x = np.std(X) std_y = np.std(Y) if std_x > 0 and std_y > 0: db_scan_x = DBSCAN(eps=std_y / eps_std_div, min_samples=2).fit(Y) db_scan_y = DBSCAN(eps=std_x / eps_std_div, min_samples=2).fit(X) cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_) cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_) if len(cluster_list_x) > 0 and len(cluster_list_y) > 0: cluster_label_x = max(set(cluster_list_x), key=cluster_list_x.count) cluster_label_y = max(set(cluster_list_y), key=cluster_list_y.count) rect_x, rect_y = [], [] for k in range(len(db_scan_x.labels_)): if db_scan_x.labels_[ k] == cluster_label_x and db_scan_y.labels_[ k] != cluster_label_y: rect_x.append(pred_labels[j].rects[k]) elif db_scan_y.labels_[ k] == cluster_label_y and db_scan_x.labels_[ k] != cluster_label_x: rect_y.append(pred_labels[j].rects[k]) pred_X.rects, pred_Y.rects = rect_x, rect_y pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y) return pred_labels_X, pred_labels_Y
def main(directory, labels_idl): if not os.path.exists( "{}/label_images".format(directory) ): #creates a directory where the label images will be stored os.makedirs("{}/label_images".format(directory)) with open("{}/".format(directory) + "label_image_values.tsv", 'w') as f_label_values: labels_annos = al.parse( labels_idl) #writes the label values to .tsv file for j in range(len(labels_annos)): labels_anno = labels_annos[j] img = imread(directory + "/" + labels_anno.imageName) index = 1 for rect in labels_anno.rects: curr_label_image = np.copy(img[int(rect.y1):int(rect.y2), int(rect.x1):int(rect.x2), :]) curr_label_value = rect.score if np.shape(curr_label_image)[0] > 0 and np.shape( curr_label_image)[1] > 0: curr_file_name = ( labels_anno.imageName.split('/')[-1][:-4] + '_{}.png'.format(index)) imsave( "{}/label_images/".format(directory) + curr_file_name, curr_label_image) write_labels(f_label_values, curr_file_name, curr_label_value) index += 1
def get_closest_ticks(self, pred_labels, pred_ticks): """ Method that matches ticks with their closest labels Inputs: pred_labels (Annotationlist): List of annotations (bounding boxes) for the predicted labels pred_ticks (Annotationlist): List of annotations (bounding boxes) for the predicted ticks Outputs annolist (Annotationlist): List of annotations which has the bounding boxes of the ticks, but the label values of the labels. """ annolist = al.AnnoList() for j in range(len(pred_labels)): new_annot = al.Annotation() new_annot.imageName = pred_labels[j].imageName if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0: distances = np.zeros( (len(pred_labels[j].rects), len(pred_ticks[j].rects))) for k in range(len(pred_labels[j].rects)): for i in range(len(pred_ticks[j].rects)): distances[k, i] = pred_labels[j].rects[k].distance( pred_ticks[j].rects[i]) min_index_labels = np.argmin(distances, axis=1) min_index_ticks = np.argmin(distances, axis=0) new_rects = [] for k in range(len(pred_labels[j].rects)): min_index = min_index_labels[k] if min_index_ticks[min_index] == k: temp_rect = pred_ticks[j].rects[min_index] temp_rect.classID = pred_labels[j].rects[k].classID new_rects.append(temp_rect) new_annot.rects = new_rects annolist.append(new_annot) else: new_annot.rects = [] annolist.append(new_annot) return annolist
def get_closest_ticks(self, pred_labels, pred_ticks): # calculating the closest distance between the (tick value,tick mark) annolist = al.AnnoList() for j in range(len(pred_labels)): new_annot = al.Annotation() new_annot.imageName = pred_labels[j].imageName if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0: distances = np.zeros( (len(pred_labels[j].rects), len(pred_ticks[j].rects))) for k in range(len(pred_labels[j].rects)): for i in range(len(pred_ticks[j].rects)): distances[k, i] = pred_labels[j].rects[k].distance( pred_ticks[j].rects[i]) min_index_labels = np.argmin(distances, axis=1) min_index_ticks = np.argmin(distances, axis=0) new_rects = [] for k in range(len(pred_labels[j].rects)): min_index = min_index_labels[k] if min_index_ticks[min_index] == k: temp_rect = pred_ticks[j].rects[min_index] temp_rect.classID = pred_labels[j].rects[k].classID new_rects.append(temp_rect) new_annot.rects = new_rects annolist.append(new_annot) else: new_annot.rects = [] annolist.append(new_annot) return annolist
def predict_model(self, model, H, true_annos, image_dir, conf_threshold): #function to get the predicted bounding boxes annolist = al.AnnoList() with tf.Session(graph=model['graph']) as sess: sess.run(tf.initialize_all_variables()) for i in range(len(true_annos)): true_anno = true_annos[i] img = self.open_image(image_dir + true_anno.imageName) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') (np_pred_boxes, np_pred_confidences) = sess.run( [model['pred_boxes'], model['pred_confidences']], feed_dict={model['x_in']: img}) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) return annolist
def test(self, image_dir, image_output_dir, csv_output_dir, true_idl_dict=None, coord_idl=None, predict_idl=None, quick=False, conf_threshold=0.3, max_dist_perc=2.0): #method used to get the predicted coordinate values if not os.path.exists(image_output_dir): os.makedirs(image_output_dir) if not os.path.exists(csv_output_dir): os.makedirs(csv_output_dir) if true_idl_dict is not None: true_annos_dict = { key: al.parse(true_idl_dict[key]) for key in true_idl_dict } else: true_annos_dict = { key: al.parse(predict_idl) for key in self.my_dir_dict } start_time = time.time() pred_dict = {} for key in self.models: pred_dict[key] = self.predict_model(self.models[key], self.H[key], true_annos_dict[key], image_dir, conf_threshold) pred_dict[key].save('{}/model_pred_{}.idl'.format( self.models[key]['dir'], self.iteration)) self.draw_images(image_dir, true_annos_dict, pred_dict, image_output_dir, true_idl_dict) pred_dict['labels'] = self.get_ocr(pred_dict['labels'], image_dir) pred_dict['labels'] = self.get_closest_ticks(pred_dict['labels'], pred_dict['ticks']) pred_labels_X, pred_labels_Y = self.split_labels_XY( pred_dict['labels']) self.regressor_x_dict = self.get_conversion(pred_labels_X, cat='x') self.regressor_y_dict = self.get_conversion(pred_labels_Y, cat='y') dictionary = self.predict_points(pred_dict['points'], self.regressor_x_dict, self.regressor_y_dict, csv_output_dir) mylogger.debug("{} images/sec".format( float(len(pred_dict['labels'])) / (time.time() - start_time))) if coord_idl is not None: mylogger.debug("comparing with ground truth") self.get_metrics(dictionary, coord_idl, csv_output_dir, quick=quick, max_dist_perc=max_dist_perc) return pred_dict
def main(model_dir, image_dir, true_idl, iteration, iou_threshold, conf_threshold): hypes_file = '{}/hypes.json'.format(model_dir) with open(hypes_file, 'r') as f: H = json.load(f) model_name = model_dir.split("/")[1] pred_idl = './output/%s_%d_val_%s.idl' % ( model_name, iteration, os.path.basename(hypes_file).replace( '.json', '')) true_annos = al.parse(true_idl) tf.reset_default_graph() googlenet = googlenet_load.init(H) x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3]) if H['use_rezoom']: pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward( H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None) grid_area = H['grid_height'] * H['grid_width'] pred_confidences = tf.reshape( tf.nn.softmax( tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])), [grid_area, H['rnn_len'], 2]) if H['reregress']: pred_boxes = pred_boxes + pred_boxes_deltas else: pred_boxes, pred_logits, pred_confidences = build_forward( H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None) saver = tf.train.Saver() with tf.Session() as sess: #sess.run(tf.initialize_all_variables()) print(saver) saver.restore(sess, '{}/save.ckpt-{}'.format(model_dir, iteration)) annolist = al.AnnoList() t = time.time() for i in range(len(true_annos)): true_anno = true_annos[i] img = Image.open(image_dir + "/" + true_anno.imageName) bg = Image.new("RGB", img.size, (255, 255, 255)) bg.paste(img, img) img = np.array(bg) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: true_anno = rescale_boxes(img.shape, true_anno, H["image_height"], H["image_width"]) img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') feed = {x_in: img} (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold) new_img_true = np.copy(img) new_img_pred = np.copy(img) for rect_true in true_anno.rects: cv2.rectangle(new_img_true, (int(rect_true.x1), int(rect_true.y1)), (int(rect_true.x2), int(rect_true.y2)), (0, 255, 0), 2) for rect_pred in rects: cv2.rectangle(new_img_pred, (int(rect_pred.x1), int(rect_pred.y1)), (int(rect_pred.x2), int(rect_pred.y2)), (0, 0, 255), 2) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) if i % 10 == 0 and i < 400: imsave( "{}/".format(model_dir) + pred_anno.imageName.split('/')[-1][:-4] + '_pred.bmp', new_img_pred) imsave( "{}/".format(model_dir) + pred_anno.imageName.split('/')[-1][:-4] + '_true.bmp', new_img_true) if (i + 1) % 200 == 0 or i == 0: mylogger.debug("Number of images analyzed: {}".format(i + 1)) avg_time = (time.time() - t) / (i + 1) mylogger.debug('%f images/sec' % (1. / avg_time)) annolist.save(pred_idl) rpc_cmd = './tensorbox/utils/annolist/doRPC.py --minOverlap %f %s %s' % ( iou_threshold, true_idl, pred_idl) mylogger.debug('$ %s' % rpc_cmd) rpc_output = subprocess.check_output(rpc_cmd, shell=True) mylogger.debug(rpc_output) txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1] output_png = 'output/{}_{}_results.png'.format(model_name, iteration) plot_cmd = './tensorbox/utils/annolist/plotSimple.py %s --output %s' % ( txt_file, output_png) mylogger.debug('$ %s' % plot_cmd) plot_output = subprocess.check_output(plot_cmd, shell=True) Image2(filename=output_png) df = pd.read_csv( "output/rpc-{}_{}_val_hypes_overlap{}.txt".format( model_name, iteration, iou_threshold), sep=" ", names=['precision', 'recall', 'fpii', 'score', 'accuracy']) auc = metrics.auc(df['recall'], df['precision']) mylogger.info("Average Precision: {}".format(auc))
def predict_model_no_truth(self, model, H, image_dir, conf_threshold, image_output_dir=None): """ Method uses to get the prediction from the object detection models. Inputs: model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model H (dictionary): Loaded json hype which describes the hyperparameters of the models. image_dir: (string): Path to the location of the images Outputs: annolist (Annotationlist): List of annotations which contain the predicted bounding boxes. """ if image_output_dir == None: image_output_dir = image_dir annolist = al.AnnoList() img_list = [] for fmt in ["jpg", "jpeg", "png", "tiff", "tif", "gif", "bmp"]: img_list.extend(glob(os.path.join(image_dir, "*.{}".format(fmt)))) with tf.Session(graph=model['graph']) as sess: sess.run(tf.global_variables_initializer()) model['saver'].restore( sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration)) for img_file in img_list: img = self.open_image(img_file) img_orig = np.copy(img) if img.shape[0] != H["image_height"] or img.shape[1] != H[ "image_width"]: img = imresize(img, (H["image_height"], H["image_width"]), interp='cubic') (np_pred_boxes, np_pred_confidences) = sess.run( [model['pred_boxes'], model['pred_confidences']], feed_dict={model['x_in']: img}) pred_anno = al.Annotation() pred_anno.imageName = img_file new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=conf_threshold, show_suppressed=False) imsave(os.path.join(image_output_dir, img_file + "_pred.png"), new_img) pred_anno.rects = rects pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0], img_orig.shape[1]) annolist.append(pred_anno) return annolist