def get_results(args, H):
tf.reset_default_graph()
x_in = tf.placeholder(tf.float32, name="x_in", shape=[H["image_height"], H["image_width"], 3])
googlenet = googlenet_load.init(H)
if H["use_rezoom"]:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, "test", reuse=None
)
grid_area = H["grid_height"] * H["grid_width"]
pred_confidences = tf.reshape(
tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H["rnn_len"], 2])), [grid_area, H["rnn_len"], 2]
)
if H["reregress"]:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, "test", reuse=None
)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, args.weights)
pred_annolist = al.AnnoList()
true_annolist = al.parse(args.test_idl)
data_dir = os.path.dirname(args.test_idl)
image_dir = get_image_dir(args)
subprocess.call("mkdir -p %s" % image_dir, shell=True)
for i in range(len(true_annolist)):
true_anno = true_annolist[i]
orig_img = imread("%s/%s" % (data_dir, true_anno.imageName))[:, :, :3]
img = imresize(orig_img, (H["image_height"], H["image_width"]), interp="cubic")
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(
H,
[img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H["rnn_len"],
min_conf=0.2,
tau=args.tau,
)
pred_anno.rects = rects
pred_anno.imagePath = os.path.abspath(data_dir)
pred_anno = rescale_boxes(
(H["image_height"], H["image_width"]), pred_anno, orig_img.shape[0], orig_img.shape[1]
)
pred_annolist.append(pred_anno)
imname = "%s/%s" % (image_dir, os.path.basename(true_anno.imageName))
misc.imsave(imname, new_img)
if i % 25 == 0:
print(i)
return pred_annolist, true_annolist
def inference(hypes, images, phase):
# Load googlenet and returns the cnn_codes
if phase == 'train':
encoder_net.append(googlenet_load.init(hypes))
input_mean = 117.
images -= input_mean
cnn, early_feat, _ = googlenet_load.model(images, encoder_net[0], hypes)
return cnn, early_feat, _
def bbox_det_TENSORBOX_multiclass(frames_list, path_video_folder, hypes_file,
weights_file, pred_idl):
from train import build_forward
print("Starting DET Phase")
#### START TENSORBOX CODE ###
lenght = int(len(frames_list))
video_info = []
### Opening Hypes file for parameters
with open(hypes_file, 'r') as f:
H = json.load(f)
### Building Network
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32,
name='x_in',
shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_confs_deltas,
[grid_area * H['rnn_len'], H['num_classes']])),
[grid_area, H['rnn_len'], H['num_classes']])
pred_logits = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_logits,
[grid_area * H['rnn_len'], H['num_classes']])),
[grid_area, H['rnn_len'], H['num_classes']])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(
sess, weights_file
) ##### Restore a Session of the Model to get weights and everything working
#### Starting Evaluating the images
print(("%d Frames to DET" % len(frames_list)))
progress = progressbar.ProgressBar(widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage(), ' ',
progressbar.ETA()
])
frameNr = 0
skipped = 0
for i in progress(list(range(0, len(frames_list)))):
current_frame = frame.Frame_Info()
current_frame.frame = frameNr
current_frame.filename = frames_list[i]
if utils_image.isnotBlack(
frames_list[i]) & utils_image.check_image_with_pil(
frames_list[i]):
img = imread(frames_list[i])
# test(frames_list[i])
feed = {x_in: img}
(np_pred_boxes, np_pred_logits,
np_pred_confidences) = sess.run(
[pred_boxes, pred_logits, pred_confidences],
feed_dict=feed)
_, rects = get_multiclass_rectangles(H,
np_pred_confidences,
np_pred_boxes,
rnn_len=H['rnn_len'])
if len(rects) > 0:
# pick = NMS(rects)
pick = rects
print((len(rects), len(pick)))
current_frame.rects = pick
frameNr = frameNr + 1
video_info.insert(len(video_info), current_frame)
print((len(current_frame.rects)))
else:
skipped = skipped + 1
else:
skipped = skipped + 1
print(("Skipped %d Black Frames" % skipped))
#### END TENSORBOX CODE ###
return video_info
def build(H, q):
'''
Build full model for training, including forward / backward passes,
optimizers, and summary statistics.
'''
arch = H
solver = H["solver"]
os.environ['CUDA_VISIBLE_DEVICES'] = str(solver['gpu'])
gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=0.8)
#gpu_options = tf.compat.v1.GPUOptions(allow_growth=True)
config = tf.compat.v1.ConfigProto(gpu_options=gpu_options)
encoder_net = googlenet_load.init(H, config)
learning_rate = tf.compat.v1.placeholder(tf.float32)
if solver['opt'] == 'RMS':
opt = tf.compat.v1.train.RMSPropOptimizer(learning_rate=learning_rate,
decay=0.9, epsilon=solver['epsilon'])
elif solver['opt'] == 'Adam':
opt = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=solver['epsilon'])
elif solver['opt'] == 'SGD':
opt = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=learning_rate)
else:
raise ValueError('Unrecognized opt type')
loss, accuracy, confidences_loss, boxes_loss = {}, {}, {}, {}
for phase in ['train', 'test']:
# generate predictions and losses from forward pass
x, confidences, boxes = q[phase].dequeue_many(arch['batch_size'])
flags = tf.argmax(confidences, 3)
grid_size = H['grid_width'] * H['grid_height']
(pred_boxes, pred_confidences,
loss[phase], confidences_loss[phase],
boxes_loss[phase]) = build_forward_backward(H, x, encoder_net, phase, boxes, flags)
pred_confidences_r = tf.reshape(pred_confidences, [H['batch_size'], grid_size, H['rnn_len'], arch['num_classes']])
pred_boxes_r = tf.reshape(pred_boxes, [H['batch_size'], grid_size, H['rnn_len'], 4])
# Set up summary operations for tensorboard
a = tf.equal(tf.argmax(confidences[:, :, 0, :], 2), tf.argmax(pred_confidences_r[:, :, 0, :], 2))
accuracy[phase] = tf.reduce_mean(tf.cast(a, 'float32'), name=phase+'/accuracy')
if phase == 'train':
global_step = tf.Variable(0, trainable=False)
tvars = tf.compat.v1.trainable_variables()
if H['clip_norm'] <= 0:
grads = tf.gradients(loss['train'], tvars)
else:
grads, norm = tf.clip_by_global_norm(tf.gradients(loss['train'], tvars), H['clip_norm'])
train_op = opt.apply_gradients(zip(grads, tvars), global_step=global_step)
elif phase == 'test':
moving_avg = tf.train.ExponentialMovingAverage(0.95)
smooth_op = moving_avg.apply([accuracy['train'], accuracy['test'],
confidences_loss['train'], boxes_loss['train'],
confidences_loss['test'], boxes_loss['test'],
])
for p in ['train', 'test']:
tf.compat.v1.summary.scalar('%s/accuracy' % p, accuracy[p])
tf.compat.v1.summary.scalar('%s/accuracy/smooth' % p, moving_avg.average(accuracy[p]))
tf.compat.v1.summary.scalar("%s/confidences_loss" % p, confidences_loss[p])
tf.compat.v1.summary.scalar("%s/confidences_loss/smooth" % p,
moving_avg.average(confidences_loss[p]))
tf.compat.v1.summary.scalar("%s/regression_loss" % p, boxes_loss[p])
tf.compat.v1.summary.scalar("%s/regression_loss/smooth" % p,
moving_avg.average(boxes_loss[p]))
if phase == 'test':
test_image = x
# show ground truth to verify labels are correct
test_true_confidences = confidences[0, :, :, :]
test_true_boxes = boxes[0, :, :, :]
# show predictions to visualize training progress
test_pred_confidences = pred_confidences_r[0, :, :, :]
test_pred_boxes = pred_boxes_r[0, :, :, :]
def log_image(np_img, np_confidences, np_boxes, np_global_step, pred_or_true):
merged = train_utils.add_rectangles(H, np_img, np_confidences, np_boxes,
use_stitching=True,
rnn_len=H['rnn_len'])[0]
num_images = 10
img_path = os.path.join(H['save_dir'], '%s_%s.jpg' % ((np_global_step / H['logging']['display_iter']) % num_images, pred_or_true))
misc.imsave(img_path, merged)
return merged
pred_log_img = tf.py_func(log_image,
[test_image, test_pred_confidences, test_pred_boxes, global_step, 'pred'],
[tf.float32])
true_log_img = tf.py_func(log_image,
[test_image, test_true_confidences, test_true_boxes, global_step, 'true'],
[tf.float32])
#tf.image_summary(phase + '/pred_boxes', tf.pack(pred_log_img),max_images=10)
#tf.image_summary(phase + '/true_boxes', tf.pack(true_log_img),max_images=10)
tf.compat.v1.summary.image(phase + '/pred_boxes', tf.stack(pred_log_img),max_outputs=10)
tf.compat.v1.summary.image(phase + '/true_boxes', tf.stack(true_log_img),max_outputs=10)
summary_op = tf.compat.v1.summary.merge_all()
return (config, loss, accuracy, summary_op, train_op,
smooth_op, global_step, learning_rate, encoder_net)
def build(H, q):
'''
Build full model for training, including forward / backward passes,
optimizers, and summary statistics.
'''
arch = H
solver = H["solver"]
os.environ['CUDA_VISIBLE_DEVICES'] = str(solver['gpu'])
#gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
gpu_options = tf.GPUOptions()
config = tf.ConfigProto(gpu_options=gpu_options)
encoder_net = googlenet_load.init(H, config)
learning_rate = tf.placeholder(tf.float32)
if solver['opt'] == 'RMS':
opt = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
decay=0.9, epsilon=solver['epsilon'])
elif solver['opt'] == 'Adam':
opt = tf.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=solver['epsilon'])
elif solver['opt'] == 'SGD':
opt = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
else:
raise ValueError('Unrecognized opt type')
loss, accuracy, confidences_loss, boxes_loss = {}, {}, {}, {}
for phase in ['train', 'test']:
# generate predictions and losses from forward pass
x, confidences, boxes = q[phase].dequeue_many(arch['batch_size'])
flags = tf.argmax(confidences, 3)
grid_size = H['grid_width'] * H['grid_height']
(pred_boxes, pred_confidences,
loss[phase], confidences_loss[phase],
boxes_loss[phase]) = build_forward_backward(H, x, encoder_net, phase, boxes, flags)
pred_confidences_r = tf.reshape(pred_confidences, [H['batch_size'], grid_size, H['rnn_len'], arch['num_classes']])
pred_boxes_r = tf.reshape(pred_boxes, [H['batch_size'], grid_size, H['rnn_len'], 4])
# Set up summary operations for tensorboard
a = tf.equal(tf.argmax(confidences[:, :, 0, :], 2), tf.argmax(pred_confidences_r[:, :, 0, :], 2))
accuracy[phase] = tf.reduce_mean(tf.cast(a, 'float32'), name=phase+'/accuracy')
if phase == 'train':
global_step = tf.Variable(0, trainable=False)
tvars = tf.trainable_variables()
if H['clip_norm'] <= 0:
grads = tf.gradients(loss['train'], tvars)
else:
grads, norm = tf.clip_by_global_norm(tf.gradients(loss['train'], tvars), H['clip_norm'])
train_op = opt.apply_gradients(zip(grads, tvars), global_step=global_step)
elif phase == 'test':
moving_avg = tf.train.ExponentialMovingAverage(0.95)
smooth_op = moving_avg.apply([accuracy['train'], accuracy['test'],
confidences_loss['train'], boxes_loss['train'],
confidences_loss['test'], boxes_loss['test'],
])
for p in ['train', 'test']:
tf.scalar_summary('%s/accuracy' % p, accuracy[p])
tf.scalar_summary('%s/accuracy/smooth' % p, moving_avg.average(accuracy[p]))
tf.scalar_summary("%s/confidences_loss" % p, confidences_loss[p])
tf.scalar_summary("%s/confidences_loss/smooth" % p,
moving_avg.average(confidences_loss[p]))
tf.scalar_summary("%s/regression_loss" % p, boxes_loss[p])
tf.scalar_summary("%s/regression_loss/smooth" % p,
moving_avg.average(boxes_loss[p]))
if phase == 'test':
test_image = x
# show ground truth to verify labels are correct
test_true_confidences = confidences[0, :, :, :]
test_true_boxes = boxes[0, :, :, :]
# show predictions to visualize training progress
test_pred_confidences = pred_confidences_r[0, :, :, :]
test_pred_boxes = pred_boxes_r[0, :, :, :]
def log_image(np_img, np_confidences, np_boxes, np_global_step, pred_or_true):
merged = train_utils.add_rectangles(H, np_img, np_confidences, np_boxes,
use_stitching=True,
rnn_len=H['rnn_len'])[0]
num_images = 10
img_path = os.path.join(H['save_dir'], '%s_%s.jpg' % ((np_global_step / H['logging']['display_iter']) % num_images, pred_or_true))
misc.imsave(img_path, merged)
return merged
pred_log_img = tf.py_func(log_image,
[test_image, test_pred_confidences, test_pred_boxes, global_step, 'pred'],
[tf.float32])
true_log_img = tf.py_func(log_image,
[test_image, test_true_confidences, test_true_boxes, global_step, 'true'],
[tf.float32])
tf.image_summary(phase + '/pred_boxes', tf.pack(pred_log_img),max_images=10)
tf.image_summary(phase + '/true_boxes', tf.pack(true_log_img),max_images=10)
summary_op = tf.merge_all_summaries()
return (config, loss, accuracy, summary_op, train_op,
smooth_op, global_step, learning_rate, encoder_net)
def build(H, q):
'''
Build full model for training, including forward / backward passes,
optimizers, and summary statistics.
'''
arch = H['arch']
solver = H["solver"]
os.environ['CUDA_VISIBLE_DEVICES'] = str(solver['gpu'])
#gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
gpu_options = tf.GPUOptions()
config = tf.ConfigProto(gpu_options=gpu_options)
googlenet = googlenet_load.init(H, config)
learning_rate = tf.placeholder(tf.float32)
if solver['opt'] == 'RMS':
opt = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
decay=0.9, epsilon=solver['epsilon'])
elif solver['opt'] == 'SGD':
opt = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
else:
raise ValueError('Unrecognized opt type')
loss, accuracy, confidences_loss, boxes_loss = {}, {}, {}, {}
for phase in ['train', 'test']:
# generate predictions and losses from forward pass
x, confidences, boxes = q[phase].dequeue_many(arch['batch_size'])
flags = tf.argmax(confidences, 3)
grid_size = H['arch']['grid_width'] * H['arch']['grid_height']
confidences_r = tf.cast(
tf.reshape(confidences[:, :, 0, :],
[H['arch']['batch_size'] * grid_size, arch['num_classes']]), 'float32')
if arch['use_lstm']:
(pred_boxes, pred_confidences,
loss[phase], confidences_loss[phase],
boxes_loss[phase]) = build_lstm(H, x, googlenet, phase, boxes, flags)
pred_confidences = pred_confidences[:, 0, :]
else:
(pred_boxes, pred_confidences,
loss[phase], confidences_loss[phase],
boxes_loss[phase]) = build_overfeat(H, x, googlenet, phase, boxes, confidences_r)
# Set up summary operations for tensorboard
a = tf.equal(tf.argmax(confidences_r, 1), tf.argmax(pred_confidences, 1))
accuracy[phase] = tf.reduce_mean(tf.cast(a, 'float32'), name=phase+'/accuracy')
if phase == 'train':
global_step = tf.Variable(0, trainable=False)
train_op = opt.minimize(loss['train'], global_step=global_step)
elif phase == 'test':
test_image = x
moving_avg = tf.train.ExponentialMovingAverage(0.99)
smooth_op = moving_avg.apply([accuracy['train'], accuracy['test'],
confidences_loss['train'], boxes_loss['train'],
confidences_loss['test'], boxes_loss['test'],
])
for p in ['train', 'test']:
tf.scalar_summary('%s/accuracy' % p, accuracy[p])
tf.scalar_summary('%s/accuracy/smooth' % p, moving_avg.average(accuracy[p]))
tf.scalar_summary("%s/confidences_loss" % p, confidences_loss[p])
tf.scalar_summary("%s/confidences_loss/smooth" % p,
moving_avg.average(confidences_loss[p]))
tf.scalar_summary("%s/regression_loss" % p, boxes_loss[p])
tf.scalar_summary("%s/regression_loss/smooth" % p,
moving_avg.average(boxes_loss[p]))
# show ground truth to verify labels are correct
test_true_confidences = confidences_r
test_true_boxes = boxes[0, :, 0, :]
# show predictions to visualize training progress
test_pred_confidences = pred_confidences
test_pred_boxes = pred_boxes[:, 0, :]
summary_op = tf.merge_all_summaries()
return (config, loss, accuracy, summary_op, train_op, googlenet['W_norm'],
test_image, test_pred_boxes, test_pred_confidences,
test_true_boxes, test_true_confidences, smooth_op,
global_step, learning_rate)
def build(H, q):
'''
Build full model for training, including forward / backward passes,
optimizers, and summary statistics.
'''
arch = H['arch']
solver = H["solver"]
os.environ['CUDA_VISIBLE_DEVICES'] = str(solver['gpu'])
#gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
gpu_options = tf.GPUOptions()
config = tf.ConfigProto(gpu_options=gpu_options)
googlenet = googlenet_load.init(H, config)
learning_rate = tf.placeholder(tf.float32)
if solver['opt'] == 'RMS':
opt = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
decay=0.9,
epsilon=solver['epsilon'])
elif solver['opt'] == 'SGD':
opt = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)
else:
raise ValueError('Unrecognized opt type')
loss, accuracy, confidences_loss, boxes_loss = {}, {}, {}, {}
for phase in ['train', 'test']:
# generate predictions and losses from forward pass
x, confidences, boxes = q[phase].dequeue_many(arch['batch_size'])
flags = tf.argmax(confidences, 3)
grid_size = H['arch']['grid_width'] * H['arch']['grid_height']
confidences_r = tf.cast(
tf.reshape(
confidences[:, :, 0, :],
[H['arch']['batch_size'] * grid_size, arch['num_classes']]),
'float32')
(pred_boxes, pred_confidences, loss[phase], confidences_loss[phase],
boxes_loss[phase]) = build_overfeat(H, x, googlenet, phase, boxes,
confidences_r)
# Set up summary operations for tensorboard
a = tf.equal(tf.argmax(confidences_r, 1),
tf.argmax(pred_confidences, 1))
accuracy[phase] = tf.reduce_mean(tf.cast(a, 'float32'),
name=phase + '/accuracy')
if phase == 'train':
global_step = tf.Variable(0, trainable=False)
train_op = opt.minimize(loss['train'], global_step=global_step)
elif phase == 'test':
test_image = x
moving_avg = tf.train.ExponentialMovingAverage(0.99)
smooth_op = moving_avg.apply([
accuracy['train'],
accuracy['test'],
confidences_loss['train'],
boxes_loss['train'],
confidences_loss['test'],
boxes_loss['test'],
])
for p in ['train', 'test']:
tf.scalar_summary('%s/accuracy' % p, accuracy[p])
tf.scalar_summary('%s/accuracy/smooth' % p,
moving_avg.average(accuracy[p]))
tf.scalar_summary("%s/confidences_loss" % p,
confidences_loss[p])
tf.scalar_summary("%s/confidences_loss/smooth" % p,
moving_avg.average(confidences_loss[p]))
tf.scalar_summary("%s/regression_loss" % p, boxes_loss[p])
tf.scalar_summary("%s/regression_loss/smooth" % p,
moving_avg.average(boxes_loss[p]))
# show ground truth to verify labels are correct
test_true_confidences = confidences_r
test_true_boxes = boxes[0, :, 0, :]
# show predictions to visualize training progress
test_pred_confidences = pred_confidences
test_pred_boxes = pred_boxes[:, 0, :]
summary_op = tf.merge_all_summaries()
return (config, loss, accuracy, summary_op, train_op, googlenet['W_norm'],
test_image, test_pred_boxes, test_pred_confidences,
test_true_boxes, test_true_confidences, smooth_op, global_step,
learning_rate)
def still_image_TENSORBOX_multiclass(frames_list,path_video_folder,hypes_file,weights_file,pred_idl):
from train import build_forward
print("Starting DET Phase")
if not os.path.exists(path_video_folder+'/'+folder_path_det_frames):
os.makedirs(path_video_folder+'/'+folder_path_det_frames)
print("Created Folder: %s"%path_video_folder+'/'+folder_path_det_frames)
if not os.path.exists(path_video_folder+'/'+folder_path_det_result):
os.makedirs(path_video_folder+'/'+folder_path_det_result)
print("Created Folder: %s"% path_video_folder+'/'+folder_path_det_result)
det_frames_list=[]
#### START TENSORBOX CODE ###
idl_filename=path_video_folder+'/'+path_video_folder+'.idl'
### Opening Hypes file for parameters
with open(hypes_file, 'r') as f:
H = json.load(f)
### Building Network
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], H['num_classes']])), [grid_area, H['rnn_len'], H['num_classes']])
pred_logits = tf.reshape(tf.nn.softmax(tf.reshape(pred_logits, [grid_area * H['rnn_len'], H['num_classes']])), [grid_area, H['rnn_len'], H['num_classes']])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, weights_file )##### Restore a Session of the Model to get weights and everything working
annolist = al.AnnoList()
#### Starting Evaluating the images
lenght=int(len(frames_list))
print("%d Frames to DET"%len(frames_list))
progress = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ',progressbar.Percentage(), ' ',progressbar.ETA()])
frameNr=0
skipped=0
for i in progress(range(0, len(frames_list))):
if Utils_Image.isnotBlack(frames_list[i]) & Utils_Image.check_image_with_pil(frames_list[i]):
img = imread(frames_list[i])
feed = {x_in: img}
(np_pred_boxes,np_pred_logits, np_pred_confidences) = sess.run([pred_boxes,pred_logits, pred_confidences], feed_dict=feed)
# print_logits(np_pred_confidences)
pred_anno = al.Annotation()
#pred_anno.imageName = test_anno.imageName
# print "np_pred_confidences shape" + str(np_pred_confidences.shape)
# print "np_pred_boxes shape" + str(np_pred_boxes.shape)
# for i in range(0, np_pred_confidences.shape[0]):
# print np_pred_confidences[i]
# for j in range(0, np_pred_confidences.shape[2]):
# print np_pred_confidences[i][0][j]
rects = get_multiclass_rectangles(H, np_pred_confidences, np_pred_boxes, rnn_len=H['rnn_len'])
pred_anno.rects = rects
pred_anno.imageName = frames_list[i]
pred_anno.frameNr = frameNr
frameNr=frameNr+1
det_frames_list.append(frames_list[i])
pick = NMS(rects)
draw_rectangles(frames_list[i],frames_list[i], pick)
annolist.append(pred_anno)
else: skipped=skipped+1
saveTextResults(idl_filename,annolist)
annolist.save(pred_idl)
print("Skipped %d Black Frames"%skipped)
#### END TENSORBOX CODE ###
return det_frames_list
def run_eval(H, checkpoint_dir , hypes_file, output_path):
"""Do Evaluation with full epoche of data.
Args:
H: Hypes
checkpoint_dir: directory with checkpoint files
output_path: path to save results
"""
#Load GT
true_idl = H['data']['test_idl']
true_annos = al.parse(true_idl)
# define output files
pred_file = 'val_%s.idl' % os.path.basename(hypes_file).replace('.json', '')
pred_idl = os.path.join(output_path, pred_file)
true_file = 'true_%s.idl' % os.path.basename(hypes_file).replace('.json', '')
true_idl_scaled = os.path.join(output_path, true_file)
data_folder = os.path.dirname(os.path.realpath(true_idl))
#Load Graph Model
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in')
if H['arch']['use_lstm']:
lstm_forward = build_lstm_forward(H, tf.expand_dims(x_in, 0),
googlenet, 'test', reuse=None)
pred_boxes, pred_logits, pred_confidences = lstm_forward
else:
overfeat_forward = build_overfeat_forward(H, tf.expand_dims(x_in, 0),
googlenet, 'test')
pred_boxes, pred_logits, pred_confidences = overfeat_forward
start_time = time.time()
saver = tf.train.Saver()
with tf.Session() as sess:
logging.info("Starting Evaluation")
sess.run(tf.initialize_all_variables())
# Restore Checkpoints
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
logging.info(ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
annolist = al.AnnoList()
trueanno = al.AnnoList()
#shuffle true_annos to randomize plottet Images
shuffle(true_annos)
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = imread( os.path.join(data_folder, true_anno.imageName))
# Rescale Boxes
trueanno.append(rescale_boxes(img.shape, true_annos[i],
H["arch"]["image_height"],
H["arch"]["image_width"]))
# Rescale Images
img = imresize(img, (H["arch"]["image_height"],
H["arch"]["image_width"]), interp='cubic')
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes,
pred_confidences],
feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles([img], np_pred_confidences,
np_pred_boxes, H["arch"],
use_stitching=True,
rnn_len=H['arch']['rnn_len'],
min_conf=0.3)
pred_anno.rects = rects
annolist.append(pred_anno)
if i % 20 == 0:
# Draw every 20th Image;
# plotted Image is randomized due to shuffling
duration = time.time() - start_time
duration = float(duration)*1000/20
out_img = os.path.join(output_path, 'test_%i.png'%i)
scp.misc.imsave(out_img, new_img)
logging.info('Step %d: Duration %.3f ms'
% (i, duration))
start_time = time.time()
annolist.save(pred_idl)
trueanno.save(true_idl_scaled)
# write results to disk
iou_threshold = 0.5
rpc_cmd = './utils/annolist/doRPC.py --minOverlap %f %s %s' % (iou_threshold, true_idl_scaled,
pred_idl)
rpc_output = subprocess.check_output(rpc_cmd, shell=True)
txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1]
output_png = os.path.join(output_path, "roc.png")
plot_cmd = './utils/annolist/plotSimple.py %s --output %s' % (txt_file, output_png)
plot_output = subprocess.check_output(plot_cmd, shell=True)
def run_eval(H, checkpoint_dir, hypes_file, output_path):
"""Do Evaluation with full epoche of data.
Args:
H: Hypes
checkpoint_dir: directory with checkpoint files
output_path: path to save results
"""
#Load GT
true_idl = H['data']['test_idl']
true_annos = al.parse(true_idl)
# define output files
pred_file = 'val_%s.idl' % os.path.basename(hypes_file).replace(
'.json', '')
pred_idl = os.path.join(output_path, pred_file)
true_file = 'true_%s.idl' % os.path.basename(hypes_file).replace(
'.json', '')
true_idl_scaled = os.path.join(output_path, true_file)
data_folder = os.path.dirname(os.path.realpath(true_idl))
#Load Graph Model
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in')
if H['arch']['use_lstm']:
lstm_forward = build_lstm_forward(H,
tf.expand_dims(x_in, 0),
googlenet,
'test',
reuse=None)
pred_boxes, pred_logits, pred_confidences = lstm_forward
else:
overfeat_forward = build_overfeat_forward(H, tf.expand_dims(x_in, 0),
googlenet, 'test')
pred_boxes, pred_logits, pred_confidences = overfeat_forward
start_time = time.time()
saver = tf.train.Saver()
with tf.Session() as sess:
logging.info("Starting Evaluation")
sess.run(tf.initialize_all_variables())
# Restore Checkpoints
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
logging.info(ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
annolist = al.AnnoList()
trueanno = al.AnnoList()
#shuffle true_annos to randomize plottet Images
shuffle(true_annos)
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = imread(os.path.join(data_folder, true_anno.imageName))
# Rescale Boxes
trueanno.append(
rescale_boxes(img.shape, true_annos[i],
H["arch"]["image_height"],
H["arch"]["image_width"]))
# Rescale Images
img = imresize(
img, (H["arch"]["image_height"], H["arch"]["image_width"]),
interp='cubic')
feed = {x_in: img}
(np_pred_boxes,
np_pred_confidences) = sess.run([pred_boxes, pred_confidences],
feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles([img],
np_pred_confidences,
np_pred_boxes,
H["arch"],
use_stitching=True,
rnn_len=H['arch']['rnn_len'],
min_conf=0.3)
pred_anno.rects = rects
annolist.append(pred_anno)
if i % 20 == 0:
# Draw every 20th Image;
# plotted Image is randomized due to shuffling
duration = time.time() - start_time
duration = float(duration) * 1000 / 20
out_img = os.path.join(output_path, 'test_%i.png' % i)
scp.misc.imsave(out_img, new_img)
logging.info('Step %d: Duration %.3f ms' % (i, duration))
start_time = time.time()
annolist.save(pred_idl)
trueanno.save(true_idl_scaled)
# write results to disk
iou_threshold = 0.5
rpc_cmd = './utils/annolist/doRPC.py --minOverlap %f %s %s' % (
iou_threshold, true_idl_scaled, pred_idl)
rpc_output = subprocess.check_output(rpc_cmd, shell=True)
txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1]
output_png = os.path.join(output_path, "roc.png")
plot_cmd = './utils/annolist/plotSimple.py %s --output %s' % (txt_file,
output_png)
plot_output = subprocess.check_output(plot_cmd, shell=True)
def bbox_det_TENSORBOX_multiclass(frames_list,path_video_folder,hypes_file,weights_file,pred_idl):
from train import build_forward
print("Starting DET Phase")
#### START TENSORBOX CODE ###
lenght=int(len(frames_list))
video_info = []
### Opening Hypes file for parameters
with open(hypes_file, 'r') as f:
H = json.load(f)
### Building Network
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], H['num_classes']])), [grid_area, H['rnn_len'], H['num_classes']])
pred_logits = tf.reshape(tf.nn.softmax(tf.reshape(pred_logits, [grid_area * H['rnn_len'], H['num_classes']])), [grid_area, H['rnn_len'], H['num_classes']])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, weights_file )##### Restore a Session of the Model to get weights and everything working
#### Starting Evaluating the images
print("%d Frames to DET"%len(frames_list))
progress = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ',progressbar.Percentage(), ' ',progressbar.ETA()])
frameNr=0
skipped=0
for i in progress(range(0, len(frames_list))):
current_frame = frame.Frame_Info()
current_frame.frame=frameNr
current_frame.filename=frames_list[i]
if utils_image.isnotBlack(frames_list[i]) & utils_image.check_image_with_pil(frames_list[i]):
img = imread(frames_list[i])
# test(frames_list[i])
feed = {x_in: img}
(np_pred_boxes,np_pred_logits, np_pred_confidences) = sess.run([pred_boxes,pred_logits, pred_confidences], feed_dict=feed)
_,rects = get_multiclass_rectangles(H, np_pred_confidences, np_pred_boxes, rnn_len=H['rnn_len'])
if len(rects)>0:
# pick = NMS(rects)
pick = rects
print len(rects),len(pick)
current_frame.rects=pick
frameNr=frameNr+1
video_info.insert(len(video_info), current_frame)
print len(current_frame.rects)
else: skipped=skipped+1
else: skipped=skipped+1
print("Skipped %d Black Frames"%skipped)
#### END TENSORBOX CODE ###
return video_info
def get_results(args, H):
tf.reset_default_graph()
x_in = tf.placeholder(
tf.float32,
name='x_in',
shape=[H['arch']['image_height'], H['arch']['image_width'], 3])
googlenet = googlenet_load.init(H)
if H['arch']['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward_backward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['arch']['grid_height'] * H['arch']['grid_width']
pred_confidences = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_confs_deltas,
[grid_area * H['arch']['rnn_len'], 2])),
[grid_area, H['arch']['rnn_len'], 2])
if H['arch']['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward_backward(
H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, args.weights)
pred_annolist = al.AnnoList()
true_annolist = al.parse(args.test_idl)
data_dir = os.path.dirname(args.test_idl)
image_dir = get_image_dir(args)
subprocess.call('mkdir -p %s' % image_dir, shell=True)
for i in range(len(true_annolist)):
true_anno = true_annolist[i]
orig_img = imread('%s/%s' %
(data_dir, true_anno.imageName))[:, :, :3]
img = imresize(
orig_img,
(H["arch"]["image_height"], H["arch"]["image_width"]),
interp='cubic')
feed = {x_in: img}
(np_pred_boxes,
np_pred_confidences) = sess.run([pred_boxes, pred_confidences],
feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
H["arch"],
use_stitching=True,
rnn_len=H['arch']['rnn_len'],
min_conf=0.2,
tau=args.tau)
pred_anno.rects = rects
pred_anno.imagePath = os.path.abspath(data_dir)
pred_anno = rescale_boxes(
(H["arch"]["image_height"], H["arch"]["image_width"]),
pred_anno, orig_img.shape[0], orig_img.shape[1])
pred_annolist.append(pred_anno)
imname = '%s/%s' % (image_dir, os.path.basename(
true_anno.imageName))
misc.imsave(imname, new_img)
if i % 25 == 0:
print(i)
return pred_annolist, true_annolist
def still_image_TENSORBOX(idl_filename, frames_list,folder_path_det_frames,folder_path_det_result,folder_path_frames,path_video_folder,hypes_file,weights_file,pred_idl):
print("Starting DET Phase")
if not os.path.exists(path_video_folder+'/'+folder_path_det_frames):
os.makedirs(path_video_folder+'/'+folder_path_det_frames)
print("Created Folder: %s"%path_video_folder+'/'+folder_path_det_frames)
if not os.path.exists(path_video_folder+'/'+folder_path_det_result):
os.makedirs(path_video_folder+'/'+folder_path_det_result)
print("Created Folder: %s"% path_video_folder+'/'+folder_path_det_result)
det_frames_list=[]
#### START TENSORBOX CODE ###
### Opening Hypes file for parameters
with open(hypes_file, 'r') as f:
H = json.load(f)
### Get Annotation List of all the image to test
test_annos = al.parse(idl_filename)
### Building Network
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['arch']['image_height'], H['arch']['image_width'], 3])
if H['arch']['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
grid_area = H['arch']['grid_height'] * H['arch']['grid_width']
pred_confidences = tf.reshape(tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H['arch']['rnn_len'], 2])), [grid_area, H['arch']['rnn_len'], 2])
if H['arch']['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, weights_file )##### Restore a Session of the Model to get weights and everything working
annolist = al.AnnoList()
import time; t = time.time()
#### Starting Evaluating the images
lenght=int(len(frames_list))
print("%d Frames to DET"%len(frames_list))
progress = progressbar.ProgressBar(widgets=[progressbar.Bar('=', '[', ']'), ' ',progressbar.Percentage(), ' ',progressbar.ETA()])
for i in progress(range(0, len(frames_list)-1)):
img = imread(frames_list[i])
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed)
pred_anno = al.Annotation()
#pred_anno.imageName = test_anno.imageName
new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes,H["arch"], use_stitching=True, rnn_len=H['arch']['rnn_len'], min_conf=0.5)
pred_anno.rects = rects
bb_img = Image.open(frames_list[i])
for bb_rect in rects:
################ Adding Rectangle ###################
dr = ImageDraw.Draw(bb_img)
cor = (bb_rect.x1,bb_rect.y1,bb_rect.x2 ,bb_rect.y2) # DA VERIFICARE Try_2 (x1,y1, x2,y2) cor = (bb_rect.left() ,bb_rect.right(),bb_rect.bottom(),bb_rect.top()) Try_1
dr.rectangle(cor, outline="red")
bb_img_det_name = frames_list[i].replace(folder_path_frames,folder_path_det_frames)
bb_img.save(bb_img_det_name)
det_frames_list.append(bb_img_det_name)
annolist.append(pred_anno)
annolist.save(pred_idl)
#### END TENSORBOX CODE ###
return det_frames_list
