def split_labels_XY(self, pred_labels, eps_std_div=25.0):
"""
Method which splits labels into X labels, and Y labels.
Inputs:
pred_labels: (Annotationlist) List of annotations of the predicted labels.
eps_std_div (float): Parameters which controls how far appart the bounding boxes are allowed to be on an axis.
Outputs:
pred_labels_X, pred_labels_Y (Annotationlist, Annotationlist): Annotation lists for the predicted labels on the X and Y axis.
"""
pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList()
for j in range(len(pred_labels)):
pred_X, pred_Y = al.Annotation(), al.Annotation()
pred_X.imageName, pred_Y.imageName = pred_labels[
j].imageName, pred_labels[j].imageName
X, Y = [], []
for rect in pred_labels[j].rects:
X.append((rect.x2 + rect.x1) / 2.0)
Y.append((rect.y2 + rect.y1) / 2.0)
X, Y = np.reshape(np.array(X),
(len(X), 1)), np.reshape(np.array(Y),
(len(Y), 1))
std_x = np.std(X)
std_y = np.std(Y)
if std_x > 0 and std_y > 0:
db_scan_x = DBSCAN(eps=std_y / eps_std_div,
min_samples=2).fit(Y)
db_scan_y = DBSCAN(eps=std_x / eps_std_div,
min_samples=2).fit(X)
cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_)
cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_)
if len(cluster_list_x) > 0 and len(cluster_list_y) > 0:
cluster_label_x = max(set(cluster_list_x),
key=cluster_list_x.count)
cluster_label_y = max(set(cluster_list_y),
key=cluster_list_y.count)
rect_x, rect_y = [], []
for k in range(len(db_scan_x.labels_)):
if db_scan_x.labels_[
k] == cluster_label_x and db_scan_y.labels_[
k] != cluster_label_y:
rect_x.append(pred_labels[j].rects[k])
elif db_scan_y.labels_[
k] == cluster_label_y and db_scan_x.labels_[
k] != cluster_label_x:
rect_y.append(pred_labels[j].rects[k])
pred_X.rects, pred_Y.rects = rect_x, rect_y
pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y)
return pred_labels_X, pred_labels_Y
def split_labels_XY(self, pred_labels, eps_std_div=25.0):
#function to cluster the (labels, tick value) to either X or Y axis
pred_labels_X, pred_labels_Y = al.AnnoList(), al.AnnoList()
for j in range(len(pred_labels)):
pred_X, pred_Y = al.Annotation(), al.Annotation()
pred_X.imageName, pred_Y.imageName = pred_labels[
j].imageName, pred_labels[j].imageName
X, Y = [], []
for rect in pred_labels[j].rects:
X.append((rect.x2 + rect.x1) / 2.0)
Y.append((rect.y2 + rect.y1) / 2.0)
X, Y = np.reshape(np.array(X),
(len(X), 1)), np.reshape(np.array(Y),
(len(Y), 1))
std_x = np.std(X)
std_y = np.std(Y)
if std_x > 0 and std_y > 0:
db_scan_x = DBSCAN(eps=std_y / eps_std_div,
min_samples=2).fit(Y)
db_scan_y = DBSCAN(eps=std_x / eps_std_div,
min_samples=2).fit(X)
cluster_list_x = filter(lambda x: x != -1, db_scan_x.labels_)
cluster_list_y = filter(lambda x: x != -1, db_scan_y.labels_)
if len(cluster_list_x) > 0 and len(cluster_list_y) > 0:
cluster_label_x = max(set(cluster_list_x),
key=cluster_list_x.count)
cluster_label_y = max(set(cluster_list_y),
key=cluster_list_y.count)
rect_x, rect_y = [], []
for k in range(len(db_scan_x.labels_)):
if db_scan_x.labels_[
k] == cluster_label_x and db_scan_y.labels_[
k] != cluster_label_y:
rect_x.append(pred_labels[j].rects[k])
elif db_scan_y.labels_[
k] == cluster_label_y and db_scan_x.labels_[
k] != cluster_label_x:
rect_y.append(pred_labels[j].rects[k])
pred_X.rects, pred_Y.rects = rect_x, rect_y
pred_labels_X.append(pred_X), pred_labels_Y.append(pred_Y)
return pred_labels_X, pred_labels_Y
def predict_model(self, model, H, true_annos, image_dir, conf_threshold):
"""
Method uses to get the prediction from the object detection models.
Inputs:
model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model
H (dictionary): Loaded json hype which describes the hyperparameters of the models.
true_annos (Annotationlist): List of annotations which contain the ground truth bounding boxes.
image_dir: (string): Path to the location of the images
Outputs:
annolist (Annotationlist): List of annotations which contain the predicted bounding boxes.
"""
annolist = al.AnnoList()
with tf.Session(graph=model['graph']) as sess:
sess.run(tf.global_variables_initializer())
model['saver'].restore(
sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration))
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = self.open_image(image_dir + true_anno.imageName)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
(np_pred_boxes, np_pred_confidences) = sess.run(
[model['pred_boxes'], model['pred_confidences']],
feed_dict={model['x_in']: img})
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno,
img_orig.shape[0], img_orig.shape[1])
annolist.append(pred_anno)
return annolist
def get_closest_ticks(self, pred_labels, pred_ticks):
"""
Method that matches ticks with their closest labels
Inputs:
pred_labels (Annotationlist): List of annotations (bounding boxes) for the predicted labels
pred_ticks (Annotationlist): List of annotations (bounding boxes) for the predicted ticks
Outputs
annolist (Annotationlist): List of annotations which has the bounding boxes of the ticks, but the label values of the labels.
"""
annolist = al.AnnoList()
for j in range(len(pred_labels)):
new_annot = al.Annotation()
new_annot.imageName = pred_labels[j].imageName
if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0:
distances = np.zeros(
(len(pred_labels[j].rects), len(pred_ticks[j].rects)))
for k in range(len(pred_labels[j].rects)):
for i in range(len(pred_ticks[j].rects)):
distances[k, i] = pred_labels[j].rects[k].distance(
pred_ticks[j].rects[i])
min_index_labels = np.argmin(distances, axis=1)
min_index_ticks = np.argmin(distances, axis=0)
new_rects = []
for k in range(len(pred_labels[j].rects)):
min_index = min_index_labels[k]
if min_index_ticks[min_index] == k:
temp_rect = pred_ticks[j].rects[min_index]
temp_rect.classID = pred_labels[j].rects[k].classID
new_rects.append(temp_rect)
new_annot.rects = new_rects
annolist.append(new_annot)
else:
new_annot.rects = []
annolist.append(new_annot)
return annolist
def get_closest_ticks(self, pred_labels, pred_ticks):
# calculating the closest distance between the (tick value,tick mark)
annolist = al.AnnoList()
for j in range(len(pred_labels)):
new_annot = al.Annotation()
new_annot.imageName = pred_labels[j].imageName
if len(pred_labels[j].rects) > 0 and len(pred_ticks[j].rects) > 0:
distances = np.zeros(
(len(pred_labels[j].rects), len(pred_ticks[j].rects)))
for k in range(len(pred_labels[j].rects)):
for i in range(len(pred_ticks[j].rects)):
distances[k, i] = pred_labels[j].rects[k].distance(
pred_ticks[j].rects[i])
min_index_labels = np.argmin(distances, axis=1)
min_index_ticks = np.argmin(distances, axis=0)
new_rects = []
for k in range(len(pred_labels[j].rects)):
min_index = min_index_labels[k]
if min_index_ticks[min_index] == k:
temp_rect = pred_ticks[j].rects[min_index]
temp_rect.classID = pred_labels[j].rects[k].classID
new_rects.append(temp_rect)
new_annot.rects = new_rects
annolist.append(new_annot)
else:
new_annot.rects = []
annolist.append(new_annot)
return annolist
def predict_model(self, model, H, true_annos, image_dir, conf_threshold):
#function to get the predicted bounding boxes
annolist = al.AnnoList()
with tf.Session(graph=model['graph']) as sess:
sess.run(tf.initialize_all_variables())
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = self.open_image(image_dir + true_anno.imageName)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
(np_pred_boxes, np_pred_confidences) = sess.run(
[model['pred_boxes'], model['pred_confidences']],
feed_dict={model['x_in']: img})
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno,
img_orig.shape[0], img_orig.shape[1])
annolist.append(pred_anno)
return annolist
def main(model_dir, image_dir, true_idl, iteration, iou_threshold,
conf_threshold):
hypes_file = '{}/hypes.json'.format(model_dir)
with open(hypes_file, 'r') as f:
H = json.load(f)
model_name = model_dir.split("/")[1]
pred_idl = './output/%s_%d_val_%s.idl' % (
model_name, iteration, os.path.basename(hypes_file).replace(
'.json', ''))
true_annos = al.parse(true_idl)
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32,
name='x_in',
shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])),
[grid_area, H['rnn_len'], 2])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
#sess.run(tf.initialize_all_variables())
print(saver)
saver.restore(sess, '{}/save.ckpt-{}'.format(model_dir, iteration))
annolist = al.AnnoList()
t = time.time()
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = Image.open(image_dir + "/" + true_anno.imageName)
bg = Image.new("RGB", img.size, (255, 255, 255))
bg.paste(img, img)
img = np.array(bg)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
true_anno = rescale_boxes(img.shape, true_anno,
H["image_height"], H["image_width"])
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
feed = {x_in: img}
(np_pred_boxes,
np_pred_confidences) = sess.run([pred_boxes, pred_confidences],
feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold)
new_img_true = np.copy(img)
new_img_pred = np.copy(img)
for rect_true in true_anno.rects:
cv2.rectangle(new_img_true,
(int(rect_true.x1), int(rect_true.y1)),
(int(rect_true.x2), int(rect_true.y2)),
(0, 255, 0), 2)
for rect_pred in rects:
cv2.rectangle(new_img_pred,
(int(rect_pred.x1), int(rect_pred.y1)),
(int(rect_pred.x2), int(rect_pred.y2)),
(0, 0, 255), 2)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno, img_orig.shape[0],
img_orig.shape[1])
annolist.append(pred_anno)
if i % 10 == 0 and i < 400:
imsave(
"{}/".format(model_dir) +
pred_anno.imageName.split('/')[-1][:-4] + '_pred.bmp',
new_img_pred)
imsave(
"{}/".format(model_dir) +
pred_anno.imageName.split('/')[-1][:-4] + '_true.bmp',
new_img_true)
if (i + 1) % 200 == 0 or i == 0:
mylogger.debug("Number of images analyzed: {}".format(i + 1))
avg_time = (time.time() - t) / (i + 1)
mylogger.debug('%f images/sec' % (1. / avg_time))
annolist.save(pred_idl)
rpc_cmd = './tensorbox/utils/annolist/doRPC.py --minOverlap %f %s %s' % (
iou_threshold, true_idl, pred_idl)
mylogger.debug('$ %s' % rpc_cmd)
rpc_output = subprocess.check_output(rpc_cmd, shell=True)
mylogger.debug(rpc_output)
txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1]
output_png = 'output/{}_{}_results.png'.format(model_name, iteration)
plot_cmd = './tensorbox/utils/annolist/plotSimple.py %s --output %s' % (
txt_file, output_png)
mylogger.debug('$ %s' % plot_cmd)
plot_output = subprocess.check_output(plot_cmd, shell=True)
Image2(filename=output_png)
df = pd.read_csv(
"output/rpc-{}_{}_val_hypes_overlap{}.txt".format(
model_name, iteration, iou_threshold),
sep=" ",
names=['precision', 'recall', 'fpii', 'score', 'accuracy'])
auc = metrics.auc(df['recall'], df['precision'])
mylogger.info("Average Precision: {}".format(auc))
def predict_model_no_truth(self,
model,
H,
image_dir,
conf_threshold,
image_output_dir=None):
"""
Method uses to get the prediction from the object detection models.
Inputs:
model (dictionary) : Dictionary which contains all the Tensorflow object required to run the model
H (dictionary): Loaded json hype which describes the hyperparameters of the models.
image_dir: (string): Path to the location of the images
Outputs:
annolist (Annotationlist): List of annotations which contain the predicted bounding boxes.
"""
if image_output_dir == None:
image_output_dir = image_dir
annolist = al.AnnoList()
img_list = []
for fmt in ["jpg", "jpeg", "png", "tiff", "tif", "gif", "bmp"]:
img_list.extend(glob(os.path.join(image_dir, "*.{}".format(fmt))))
with tf.Session(graph=model['graph']) as sess:
sess.run(tf.global_variables_initializer())
model['saver'].restore(
sess, '{}/save.ckpt-{}'.format(model['dir'], self.iteration))
for img_file in img_list:
img = self.open_image(img_file)
img_orig = np.copy(img)
if img.shape[0] != H["image_height"] or img.shape[1] != H[
"image_width"]:
img = imresize(img, (H["image_height"], H["image_width"]),
interp='cubic')
(np_pred_boxes, np_pred_confidences) = sess.run(
[model['pred_boxes'], model['pred_confidences']],
feed_dict={model['x_in']: img})
pred_anno = al.Annotation()
pred_anno.imageName = img_file
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=conf_threshold,
show_suppressed=False)
imsave(os.path.join(image_output_dir, img_file + "_pred.png"),
new_img)
pred_anno.rects = rects
pred_anno = rescale_boxes(img.shape, pred_anno,
img_orig.shape[0], img_orig.shape[1])
annolist.append(pred_anno)
return annolist