def split_labels_XY(self, pred_labels, eps_std_div=25.0):
"""
Method which splits labels into X labels, and Y labels.
Inputs:
pred_labels: (Annotationlist) List of annotations of the predicted labels.
eps_std_div (float): Parameters which controls how far appart the bounding boxes are allowed to be on an axis.
Outputs:
pred_labels_X, pred_labels_Y (Annotationlist, Annotationlist): Annotation lists for the predicted labels on the X and Y axis.
"""
pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList()
for j in range(len(pred_labels)):
pred_X, pred_Y = al.Annotation(), al.Annotation()
pred_X.imageName, pred_Y.imageName = pred_labels[
j].imageName, pred_labels[j].imageName
X, Y = [], []
for rect in pred_labels[j].rects:
X.append((rect.x2 + rect.x1) / 2.0)
Y.append((rect.y2 + rect.y1) / 2.0)
X, Y = np.reshape(np.array(X),
(len(X), 1)), np.reshape(np.array(Y),
(len(Y), 1))
std_x = np.std(X)
std_y = np.std(Y)
if std_x > 0 and std_y > 0:
db_scan_x = DBSCAN(eps=std_y / eps_std_div,
min_samples=2).fit(Y)
db_scan_y = DBSCAN(eps=std_x / eps_std_div,
min_samples=2).fit(X)
cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_)
cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_)
if len(cluster_list_x) > 0 and len(cluster_list_y) > 0:
cluster_label_x = max(set(cluster_list_x),
key=cluster_list_x.count)
cluster_label_y = max(set(cluster_list_y),
key=cluster_list_y.count)
rect_x, rect_y = [], []
for k in range(len(db_scan_x.labels_)):
if db_scan_x.labels_[
k] == cluster_label_x and db_scan_y.labels_[
k] != cluster_label_y:
rect_x.append(pred_labels[j].rects[k])
elif db_scan_y.labels_[
k] == cluster_label_y and db_scan_x.labels_[
k] != cluster_label_x:
rect_y.append(pred_labels[j].rects[k])
pred_X.rects, pred_Y.rects = rect_x, rect_y
pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y)
return pred_labels_X, pred_labels_Y
def test(self,image_dir, image_output_dir, csv_output_dir , true_idl_dict = None, coord_idl = None, predict_idl = None,
quick = False, conf_threshold = 0.3, max_dist_perc = 2.0):
"""
Method used to test the object on a set of scatter plots.
Inputs:
true_idl_dict (dictionary): Dictionary where the key is the object name (ticks, points, labels) and
the values are the path to their corresponding idl file which may or may not contain the ground truth.
image_dir: (string): Path to the location of the images
image_output_dir (string): Path to the location where we want to save new images with marked bounding boxes.
csv_output_dir (string): Path to the location where we want to save the output csv for each images.
coord_idl: (string) Optional, ground truth idl for the real coordinate values.
Outputs:
pred_dict: (dictionary) Dictionary of dataframes which contains the predicted coordinate values.
"""
if not os.path.exists(image_output_dir):
os.makedirs(image_output_dir)
if not os.path.exists(csv_output_dir):
os.makedirs(csv_output_dir)
if true_idl_dict is not None:
true_annos_dict = {key:al.parse(true_idl_dict[key]) for key in true_idl_dict}
else:
true_annos_dict = {key:al.parse(predict_idl) for key in self.my_dir_dict}
start_time = time.time()
pred_dict = {}
for key in self.models:
pred_dict[key] = self.predict_model(self.models[key], self.H[key], true_annos_dict[key], image_dir, conf_threshold)
pred_dict[key].save('{}/model_pred_{}.idl'.format(self.models[key]['dir'],self.iteration))
self.draw_images(image_dir, true_annos_dict, pred_dict, image_output_dir, true_idl_dict)
pred_dict['labels'] = self.get_ocr(pred_dict['labels'], image_dir)
pred_dict['labels'] = self.get_closest_ticks(pred_dict['labels'], pred_dict['ticks'])
pred_labels_X, pred_labels_Y = self.split_labels_XY(pred_dict['labels'])
self.regressor_x_dict = self.get_conversion(pred_labels_X, cat = 'x')
self.regressor_y_dict = self.get_conversion(pred_labels_Y, cat = 'y')
df_dict = self.predict_points(pred_dict['points'],self.regressor_x_dict, self.regressor_y_dict, csv_output_dir)
mylogger.debug("{} images/sec".format(float(len(pred_dict['labels']))/(time.time()-start_time)))
if coord_idl is not None:
mylogger.debug("Now using ground truth to test ...")
self.get_metrics(df_dict, coord_idl, csv_output_dir, quick = quick, max_dist_perc = max_dist_perc)
return pred_dict
def predict_model(self, model, H, true_annos, image_dir, conf_threshold):
"""
Method uses to get the prediction from the object detection models.
Inputs:
model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model
H (dictionary): Loaded json hype which describes the hyperparameters of the models.
true_annos (Annotationlist): List of annotations which contain the ground truth bounding boxes.
image_dir: (string): Path to the location of the images
Outputs:
annolist (Annotationlist): List of annotations which contain the predicted bounding boxes.
"""
annolist = al.AnnoList()
with tf.Session(graph=model['graph']) as sess:
sess.run(tf.global_variables_initializer())
model['saver'].restore(
sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration))
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = self.open_image(image_dir + true_anno.imageName)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
(np_pred_boxes, np_pred_confidences) = sess.run(
[model['pred_boxes'], model['pred_confidences']],
feed_dict={model['x_in']: img})
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno,
img_orig.shape[0], img_orig.shape[1])
annolist.append(pred_anno)
return annolist
def split_labels_XY(self, pred_labels, eps_std_div=25.0):
#function to cluster the (labels, tick value) to either X or Y axis
pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList()
for j in range(len(pred_labels)):
pred_X, pred_Y = al.Annotation(), al.Annotation()
pred_X.imageName, pred_Y.imageName = pred_labels[
j].imageName, pred_labels[j].imageName
X, Y = [], []
for rect in pred_labels[j].rects:
X.append((rect.x2 + rect.x1) / 2.0)
Y.append((rect.y2 + rect.y1) / 2.0)
X, Y = np.reshape(np.array(X),
(len(X), 1)), np.reshape(np.array(Y),
(len(Y), 1))
std_x = np.std(X)
std_y = np.std(Y)
if std_x > 0 and std_y > 0:
db_scan_x = DBSCAN(eps=std_y / eps_std_div,
min_samples=2).fit(Y)
db_scan_y = DBSCAN(eps=std_x / eps_std_div,
min_samples=2).fit(X)
cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_)
cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_)
if len(cluster_list_x) > 0 and len(cluster_list_y) > 0:
cluster_label_x = max(set(cluster_list_x),
key=cluster_list_x.count)
cluster_label_y = max(set(cluster_list_y),
key=cluster_list_y.count)
rect_x, rect_y = [], []
for k in range(len(db_scan_x.labels_)):
if db_scan_x.labels_[
k] == cluster_label_x and db_scan_y.labels_[
k] != cluster_label_y:
rect_x.append(pred_labels[j].rects[k])
elif db_scan_y.labels_[
k] == cluster_label_y and db_scan_x.labels_[
k] != cluster_label_x:
rect_y.append(pred_labels[j].rects[k])
pred_X.rects, pred_Y.rects = rect_x, rect_y
pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y)
return pred_labels_X, pred_labels_Y
def main(directory, labels_idl):
if not os.path.exists(
"{}/label_images".format(directory)
): #creates a directory where the label images will be stored
os.makedirs("{}/label_images".format(directory))
with open("{}/".format(directory) + "label_image_values.tsv",
'w') as f_label_values:
labels_annos = al.parse(
labels_idl) #writes the label values to .tsv file
for j in range(len(labels_annos)):
labels_anno = labels_annos[j]
img = imread(directory + "/" + labels_anno.imageName)
index = 1
for rect in labels_anno.rects:
curr_label_image = np.copy(img[int(rect.y1):int(rect.y2),
int(rect.x1):int(rect.x2), :])
curr_label_value = rect.score
if np.shape(curr_label_image)[0] > 0 and np.shape(
curr_label_image)[1] > 0:
curr_file_name = (
labels_anno.imageName.split('/')[-1][:-4] +
'_{}.png'.format(index))
imsave(
"{}/label_images/".format(directory) + curr_file_name,
curr_label_image)
write_labels(f_label_values, curr_file_name,
curr_label_value)
index += 1
def get_closest_ticks(self, pred_labels, pred_ticks):
"""
Method that matches ticks with their closest labels
Inputs:
pred_labels (Annotationlist): List of annotations (bounding boxes) for the predicted labels
pred_ticks (Annotationlist): List of annotations (bounding boxes) for the predicted ticks
Outputs
annolist (Annotationlist): List of annotations which has the bounding boxes of the ticks, but the label values of the labels.
"""
annolist = al.AnnoList()
for j in range(len(pred_labels)):
new_annot = al.Annotation()
new_annot.imageName = pred_labels[j].imageName
if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0:
distances = np.zeros(
(len(pred_labels[j].rects), len(pred_ticks[j].rects)))
for k in range(len(pred_labels[j].rects)):
for i in range(len(pred_ticks[j].rects)):
distances[k, i] = pred_labels[j].rects[k].distance(
pred_ticks[j].rects[i])
min_index_labels = np.argmin(distances, axis=1)
min_index_ticks = np.argmin(distances, axis=0)
new_rects = []
for k in range(len(pred_labels[j].rects)):
min_index = min_index_labels[k]
if min_index_ticks[min_index] == k:
temp_rect = pred_ticks[j].rects[min_index]
temp_rect.classID = pred_labels[j].rects[k].classID
new_rects.append(temp_rect)
new_annot.rects = new_rects
annolist.append(new_annot)
else:
new_annot.rects = []
annolist.append(new_annot)
return annolist
def get_closest_ticks(self, pred_labels, pred_ticks):
# calculating the closest distance between the (tick value,tick mark)
annolist = al.AnnoList()
for j in range(len(pred_labels)):
new_annot = al.Annotation()
new_annot.imageName = pred_labels[j].imageName
if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0:
distances = np.zeros(
(len(pred_labels[j].rects), len(pred_ticks[j].rects)))
for k in range(len(pred_labels[j].rects)):
for i in range(len(pred_ticks[j].rects)):
distances[k, i] = pred_labels[j].rects[k].distance(
pred_ticks[j].rects[i])
min_index_labels = np.argmin(distances, axis=1)
min_index_ticks = np.argmin(distances, axis=0)
new_rects = []
for k in range(len(pred_labels[j].rects)):
min_index = min_index_labels[k]
if min_index_ticks[min_index] == k:
temp_rect = pred_ticks[j].rects[min_index]
temp_rect.classID = pred_labels[j].rects[k].classID
new_rects.append(temp_rect)
new_annot.rects = new_rects
annolist.append(new_annot)
else:
new_annot.rects = []
annolist.append(new_annot)
return annolist
def predict_model(self, model, H, true_annos, image_dir, conf_threshold):
#function to get the predicted bounding boxes
annolist = al.AnnoList()
with tf.Session(graph=model['graph']) as sess:
sess.run(tf.initialize_all_variables())
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = self.open_image(image_dir + true_anno.imageName)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
(np_pred_boxes, np_pred_confidences) = sess.run(
[model['pred_boxes'], model['pred_confidences']],
feed_dict={model['x_in']: img})
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno,
img_orig.shape[0], img_orig.shape[1])
annolist.append(pred_anno)
return annolist
def test(self,
image_dir,
image_output_dir,
csv_output_dir,
true_idl_dict=None,
coord_idl=None,
predict_idl=None,
quick=False,
conf_threshold=0.3,
max_dist_perc=2.0):
#method used to get the predicted coordinate values
if not os.path.exists(image_output_dir):
os.makedirs(image_output_dir)
if not os.path.exists(csv_output_dir):
os.makedirs(csv_output_dir)
if true_idl_dict is not None:
true_annos_dict = {
key: al.parse(true_idl_dict[key])
for key in true_idl_dict
}
else:
true_annos_dict = {
key: al.parse(predict_idl)
for key in self.my_dir_dict
}
start_time = time.time()
pred_dict = {}
for key in self.models:
pred_dict[key] = self.predict_model(self.models[key], self.H[key],
true_annos_dict[key],
image_dir, conf_threshold)
pred_dict[key].save('{}/model_pred_{}.idl'.format(
self.models[key]['dir'], self.iteration))
self.draw_images(image_dir, true_annos_dict, pred_dict,
image_output_dir, true_idl_dict)
pred_dict['labels'] = self.get_ocr(pred_dict['labels'], image_dir)
pred_dict['labels'] = self.get_closest_ticks(pred_dict['labels'],
pred_dict['ticks'])
pred_labels_X, pred_labels_Y = self.split_labels_XY(
pred_dict['labels'])
self.regressor_x_dict = self.get_conversion(pred_labels_X, cat='x')
self.regressor_y_dict = self.get_conversion(pred_labels_Y, cat='y')
dictionary = self.predict_points(pred_dict['points'],
self.regressor_x_dict,
self.regressor_y_dict, csv_output_dir)
mylogger.debug("{} images/sec".format(
float(len(pred_dict['labels'])) / (time.time() - start_time)))
if coord_idl is not None:
mylogger.debug("comparing with ground truth")
self.get_metrics(dictionary,
coord_idl,
csv_output_dir,
quick=quick,
max_dist_perc=max_dist_perc)
return pred_dict
def main(model_dir, image_dir, true_idl, iteration, iou_threshold,
conf_threshold):
hypes_file = '{}/hypes.json'.format(model_dir)
with open(hypes_file, 'r') as f:
H = json.load(f)
model_name = model_dir.split("/")[1]
pred_idl = './output/%s_%d_val_%s.idl' % (
model_name, iteration, os.path.basename(hypes_file).replace(
'.json', ''))
true_annos = al.parse(true_idl)
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32,
name='x_in',
shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])),
[grid_area, H['rnn_len'], 2])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
#sess.run(tf.initialize_all_variables())
print(saver)
saver.restore(sess, '{}/save.ckpt-{}'.format(model_dir, iteration))
annolist = al.AnnoList()
t = time.time()
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = Image.open(image_dir + "/" + true_anno.imageName)
bg = Image.new("RGB", img.size, (255, 255, 255))
bg.paste(img, img)
img = np.array(bg)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
true_anno = rescale_boxes(img.shape, true_anno,
H["image_height"], H["image_width"])
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
feed = {x_in: img}
(np_pred_boxes,
np_pred_confidences) = sess.run([pred_boxes, pred_confidences],
feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold)
new_img_true = np.copy(img)
new_img_pred = np.copy(img)
for rect_true in true_anno.rects:
cv2.rectangle(new_img_true,
(int(rect_true.x1), int(rect_true.y1)),
(int(rect_true.x2), int(rect_true.y2)),
(0, 255, 0), 2)
for rect_pred in rects:
cv2.rectangle(new_img_pred,
(int(rect_pred.x1), int(rect_pred.y1)),
(int(rect_pred.x2), int(rect_pred.y2)),
(0, 0, 255), 2)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0],
img_orig.shape[1])
annolist.append(pred_anno)
if i % 10 == 0 and i < 400:
imsave(
"{}/".format(model_dir) +
pred_anno.imageName.split('/')[-1][:-4] + '_pred.bmp',
new_img_pred)
imsave(
"{}/".format(model_dir) +
pred_anno.imageName.split('/')[-1][:-4] + '_true.bmp',
new_img_true)
if (i + 1) % 200 == 0 or i == 0:
mylogger.debug("Number of images analyzed: {}".format(i + 1))
avg_time = (time.time() - t) / (i + 1)
mylogger.debug('%f images/sec' % (1. / avg_time))
annolist.save(pred_idl)
rpc_cmd = './tensorbox/utils/annolist/doRPC.py --minOverlap %f %s %s' % (
iou_threshold, true_idl, pred_idl)
mylogger.debug('$ %s' % rpc_cmd)
rpc_output = subprocess.check_output(rpc_cmd, shell=True)
mylogger.debug(rpc_output)
txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1]
output_png = 'output/{}_{}_results.png'.format(model_name, iteration)
plot_cmd = './tensorbox/utils/annolist/plotSimple.py %s --output %s' % (
txt_file, output_png)
mylogger.debug('$ %s' % plot_cmd)
plot_output = subprocess.check_output(plot_cmd, shell=True)
Image2(filename=output_png)
df = pd.read_csv(
"output/rpc-{}_{}_val_hypes_overlap{}.txt".format(
model_name, iteration, iou_threshold),
sep=" ",
names=['precision', 'recall', 'fpii', 'score', 'accuracy'])
auc = metrics.auc(df['recall'], df['precision'])
mylogger.info("Average Precision: {}".format(auc))
def predict_model_no_truth(self,
model,
H,
image_dir,
conf_threshold,
image_output_dir=None):
"""
Method uses to get the prediction from the object detection models.
Inputs:
model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model
H (dictionary): Loaded json hype which describes the hyperparameters of the models.
image_dir: (string): Path to the location of the images
Outputs:
annolist (Annotationlist): List of annotations which contain the predicted bounding boxes.
"""
if image_output_dir == None:
image_output_dir = image_dir
annolist = al.AnnoList()
img_list = []
for fmt in ["jpg", "jpeg", "png", "tiff", "tif", "gif", "bmp"]:
img_list.extend(glob(os.path.join(image_dir, "*.{}".format(fmt))))
with tf.Session(graph=model['graph']) as sess:
sess.run(tf.global_variables_initializer())
model['saver'].restore(
sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration))
for img_file in img_list:
img = self.open_image(img_file)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
(np_pred_boxes, np_pred_confidences) = sess.run(
[model['pred_boxes'], model['pred_confidences']],
feed_dict={model['x_in']: img})
pred_anno = al.Annotation()
pred_anno.imageName = img_file
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold,
show_suppressed=False)
imsave(os.path.join(image_output_dir, img_file + "_pred.png"),
new_img)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno,
img_orig.shape[0], img_orig.shape[1])
annolist.append(pred_anno)
return annolist