def get_results(args, H):
tf.reset_default_graph()
x_in = tf.placeholder(tf.float32, name="x_in", shape=[H["image_height"], H["image_width"], 3])
googlenet = googlenet_load.init(H)
if H["use_rezoom"]:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, "test", reuse=None
)
grid_area = H["grid_height"] * H["grid_width"]
pred_confidences = tf.reshape(
tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H["rnn_len"], 2])), [grid_area, H["rnn_len"], 2]
)
if H["reregress"]:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), googlenet, "test", reuse=None
)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, args.weights)
pred_annolist = al.AnnoList()
true_annolist = al.parse(args.test_idl)
data_dir = os.path.dirname(args.test_idl)
image_dir = get_image_dir(args)
subprocess.call("mkdir -p %s" % image_dir, shell=True)
for i in range(len(true_annolist)):
true_anno = true_annolist[i]
orig_img = imread("%s/%s" % (data_dir, true_anno.imageName))[:, :, :3]
img = imresize(orig_img, (H["image_height"], H["image_width"]), interp="cubic")
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles(
H,
[img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H["rnn_len"],
min_conf=0.2,
tau=args.tau,
)
pred_anno.rects = rects
pred_anno.imagePath = os.path.abspath(data_dir)
pred_anno = rescale_boxes(
(H["image_height"], H["image_width"]), pred_anno, orig_img.shape[0], orig_img.shape[1]
)
pred_annolist.append(pred_anno)
imname = "%s/%s" % (image_dir, os.path.basename(true_anno.imageName))
misc.imsave(imname, new_img)
if i % 25 == 0:
print(i)
return pred_annolist, true_annolist
def log_image(np_img, np_confidences, np_boxes, np_global_step, pred_or_true):
merged = train_utils.add_rectangles(H, np_img, np_confidences, np_boxes,
use_stitching=True,
rnn_len=H['rnn_len'])[0]
num_images = 10
img_path = os.path.join(H['save_dir'], '%s_%s.jpg' % ((np_global_step / H['logging']['display_iter']) % num_images, pred_or_true))
misc.imsave(img_path, merged)
return merged
def get_results(args, H):
tf.reset_default_graph()
H["grid_width"] = H["image_width"] / H["region_size"]
H["grid_height"] = H["image_height"] / H["region_size"]
x_in = tf.placeholder(tf.float32,
name='x_in',
shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(
H, tf.expand_dims(x_in, 0), 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(
tf.nn.softmax(
tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])),
[grid_area, H['rnn_len'], 2])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(
H, tf.expand_dims(x_in, 0), 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess, args.weights)
orig_img = imread('data/tshirtslayer/tss-images/wearing.jpg')
img = imresize(orig_img, (H["image_height"], H["image_width"]),
interp='cubic')
feed = {x_in: img}
now = time.time()
(np_pred_boxes,
np_pred_confidences) = sess.run([pred_boxes, pred_confidences],
feed_dict=feed)
print(time.time() - now)
pred_anno = al.Annotation()
pred_anno.imageName = "image.jpg"
new_img, rects = add_rectangles(H, [img],
np_pred_confidences,
np_pred_boxes,
use_stitching=True,
rnn_len=H['rnn_len'],
min_conf=args.min_conf,
tau=args.tau,
show_suppressed=args.show_suppressed)
misc.imsave("data/xxx.jpg", new_img)
def eval(H, ckpt_file, in_dir, out_dir, conf):
"""
Re-constructs a TF based on the input checkpoint file (and the default googlenet graph) and applies it to the set
of images in the input directory. If the output directory is supplied, annotated images are saved there, with detect
boxes shown for any detect whose computed condifence exceeds 'conf'
Note: the re
"""
# load graph
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in')
# add the TF ops necessary for the reinspect algorihtm, based on the architecture defined in the hypes file
if H['arch']['use_lstm']:
pred_boxes, pred_logits, pred_confidences = build_lstm_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None)
else:
pred_boxes, pred_logits, pred_confidences = build_overfeat_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test')
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, ckpt_file)
# process the files
save_output = False
if out_dir != None and os.path.isdir(out_dir):
save_output = True
file_list = os.listdir(in_dir)
filenames = next(os.walk(in_dir))[2]
num_files = len(filenames)
for i in range(1,num_files,1):
f = filenames[i]
img_raw = cv2.imread(os.path.join(in_dir,f))
# Rudin images are 480 x 704, need them to be 480 x 640, so crop or resize ...
img = img_raw[:,1:641,:]
print 'Processing file:', f
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed)
new_img, rects = add_rectangles([img], np_pred_confidences, np_pred_boxes,
H["arch"], use_stitching=True, rnn_len=H['arch']['rnn_len'], min_conf=conf)
if True:
cv2.imshow('Image with overlay',new_img)
cv2.waitKey(1000)
if save_output:
cv2.imwrite(os.path.join(out_dir,f), new_img)
def postprocess_regular(image_info, np_pred_boxes, np_pred_confidences, H, options):
pred_anno = al.Annotation()
pred_anno.imageName = image_info['path']
pred_anno.imagePath = os.path.abspath(image_info['path'])
_, rects = add_rectangles(H, [image_info['transformed']], np_pred_confidences, np_pred_boxes, use_stitching=True,
rnn_len=H['rnn_len'], min_conf=options['min_conf'], tau=options['tau'],
show_suppressed=False)
h, w = image_info['original_shape']
if 'rotate90' in H['data'] and H['data']['rotate90']:
# original image height is a width for rotated one
rects = Rotate90.invert(h, rects)
rects = [r for r in rects if r.x1 < r.x2 and r.y1 < r.y2 and r.score > options['min_conf']]
pred_anno.rects = rects
pred_anno = rescale_boxes((H['image_height'], H['image_width']), pred_anno, h, w)
return pred_anno
def train(H, test_images):
if not os.path.exists(H['save_dir']): os.makedirs(H['save_dir'])
ckpt_file = H['save_dir'] + '/save.ckpt'
with open(H['save_dir'] + '/hypes.json', 'w') as f:
json.dump(H, f, indent=4)
x_in = tf.placeholder(tf.float32)
confs_in = tf.placeholder(tf.float32)
boxes_in = tf.placeholder(tf.float32)
q = {}
enqueue_op = {}
for phase in ['train', 'test']:
dtypes = [tf.float32, tf.float32, tf.float32]
grid_size = H['arch']['grid_width'] * H['arch']['grid_height']
shapes = (
[H['arch']['image_height'], H['arch']['image_width'], 3],
[grid_size, H['arch']['rnn_len'], H['arch']['num_classes']],
[grid_size, H['arch']['rnn_len'], 4],
)
q[phase] = tf.FIFOQueue(capacity=30, dtypes=dtypes, shapes=shapes)
enqueue_op[phase] = q[phase].enqueue((x_in, confs_in, boxes_in))
def make_feed(d):
return {x_in: d['image'], confs_in: d['confs'], boxes_in: d['boxes'],
learning_rate: H['solver']['learning_rate']}
def MyLoop(sess, enqueue_op, phase, gen):
for d in gen:
sess.run(enqueue_op[phase], feed_dict=make_feed(d))
(config, loss, accuracy, summary_op, train_op, W_norm,
test_image, test_pred_boxes, test_pred_confidences,
test_true_boxes, test_true_confidences,
smooth_op, global_step, learning_rate) = build(H, q)
saver = tf.train.Saver(max_to_keep=None)
writer = tf.train.SummaryWriter(
logdir=H['save_dir'],
flush_secs=10
)
test_image_to_log = tf.placeholder(tf.uint8,
[H['arch']['image_height'], H['arch']['image_width'], 3])
log_image_name = tf.placeholder(tf.string)
log_image = tf.image_summary(log_image_name, tf.expand_dims(test_image_to_log, 0))
with tf.Session(config=config) as sess:
threads = []
for phase in ['train', 'test']:
# enqueue once manually to avoid thread start delay
gen = train_utils.load_data_gen(H, phase, jitter=H['solver']['use_jitter'])
d = gen.next()
sess.run(enqueue_op[phase], feed_dict=make_feed(d))
threads.append(tf.train.threading.Thread(target=MyLoop,
args=(sess, enqueue_op, phase, gen)))
threads[-1].start()
tf.set_random_seed(H['solver']['rnd_seed'])
sess.run(tf.initialize_all_variables())
weights_str = H['solver']['weights']
if len(weights_str) > 0:
print('Restoring from: %s' % weights_str)
saver.restore(sess, weights_str)
# train model for N iterations
for i in xrange(10000000):
display_iter = 10
adjusted_lr = (H['solver']['learning_rate'] *
0.5 ** max(0, (i / H['solver']['learning_rate_step']) - 2))
lr_feed = {learning_rate: adjusted_lr}
if i % display_iter == 0:
if i > 0:
dt = (time.time() - start) / (H['arch']['batch_size'] * display_iter)
start = time.time()
(batch_loss_train, test_accuracy, weights_norm,
summary_str, np_test_image, np_test_pred_boxes,
np_test_pred_confidences, np_test_true_boxes,
np_test_true_confidences, _, _) = sess.run([
loss['train'], accuracy['test'], W_norm,
summary_op, test_image, test_pred_boxes,
test_pred_confidences, test_true_boxes, test_true_confidences,
train_op, smooth_op,
], feed_dict=lr_feed)
pred_true = [("%d_pred_output" % (i % 3), np_test_pred_boxes, np_test_pred_confidences),
("%d_true_output" % (i % 3), np_test_true_boxes, np_test_true_confidences)]
for name, boxes, confidences in pred_true:
test_output_to_log = train_utils.add_rectangles(np_test_image,
confidences,
boxes,
H["arch"])[0]
assert test_output_to_log.shape == (H['arch']['image_height'],
H['arch']['image_width'], 3)
feed = {test_image_to_log: test_output_to_log, log_image_name: name}
test_image_summary_str = sess.run(log_image, feed_dict=feed)
writer.add_summary(test_image_summary_str, global_step=global_step.eval())
writer.add_summary(summary_str, global_step=global_step.eval())
print_str = string.join([
'Step: %d',
'lr: %f',
'Train Loss: %.2f',
'Test Accuracy: %.1f%%',
'Time/image (ms): %.1f'
], ', ')
print(print_str %
(i, adjusted_lr, batch_loss_train,
test_accuracy * 100, dt * 1000 if i > 0 else 0))
else:
batch_loss_train, _ = sess.run([loss['train'], train_op], feed_dict=lr_feed)
if global_step.eval() % 1000 == 0:
saver.save(sess, ckpt_file, global_step=global_step)
def run_eval(H, checkpoint_dir , hypes_file, output_path):
"""Do Evaluation with full epoche of data.
Args:
H: Hypes
checkpoint_dir: directory with checkpoint files
output_path: path to save results
"""
#Load GT
true_idl = H['data']['test_idl']
true_annos = al.parse(true_idl)
# define output files
pred_file = 'val_%s.idl' % os.path.basename(hypes_file).replace('.json', '')
pred_idl = os.path.join(output_path, pred_file)
true_file = 'true_%s.idl' % os.path.basename(hypes_file).replace('.json', '')
true_idl_scaled = os.path.join(output_path, true_file)
data_folder = os.path.dirname(os.path.realpath(true_idl))
#Load Graph Model
tf.reset_default_graph()
googlenet = googlenet_load.init(H)
x_in = tf.placeholder(tf.float32, name='x_in')
if H['arch']['use_lstm']:
lstm_forward = build_lstm_forward(H, tf.expand_dims(x_in, 0),
googlenet, 'test', reuse=None)
pred_boxes, pred_logits, pred_confidences = lstm_forward
else:
overfeat_forward = build_overfeat_forward(H, tf.expand_dims(x_in, 0),
googlenet, 'test')
pred_boxes, pred_logits, pred_confidences = overfeat_forward
start_time = time.time()
saver = tf.train.Saver()
with tf.Session() as sess:
logging.info("Starting Evaluation")
sess.run(tf.initialize_all_variables())
# Restore Checkpoints
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
logging.info(ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
annolist = al.AnnoList()
trueanno = al.AnnoList()
#shuffle true_annos to randomize plottet Images
shuffle(true_annos)
for i in range(len(true_annos)):
true_anno = true_annos[i]
img = imread( os.path.join(data_folder, true_anno.imageName))
# Rescale Boxes
trueanno.append(rescale_boxes(img.shape, true_annos[i],
H["arch"]["image_height"],
H["arch"]["image_width"]))
# Rescale Images
img = imresize(img, (H["arch"]["image_height"],
H["arch"]["image_width"]), interp='cubic')
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes,
pred_confidences],
feed_dict=feed)
pred_anno = al.Annotation()
pred_anno.imageName = true_anno.imageName
new_img, rects = add_rectangles([img], np_pred_confidences,
np_pred_boxes, H["arch"],
use_stitching=True,
rnn_len=H['arch']['rnn_len'],
min_conf=0.3)
pred_anno.rects = rects
annolist.append(pred_anno)
if i % 20 == 0:
# Draw every 20th Image;
# plotted Image is randomized due to shuffling
duration = time.time() - start_time
duration = float(duration)*1000/20
out_img = os.path.join(output_path, 'test_%i.png'%i)
scp.misc.imsave(out_img, new_img)
logging.info('Step %d: Duration %.3f ms'
% (i, duration))
start_time = time.time()
annolist.save(pred_idl)
trueanno.save(true_idl_scaled)
# write results to disk
iou_threshold = 0.5
rpc_cmd = './utils/annolist/doRPC.py --minOverlap %f %s %s' % (iou_threshold, true_idl_scaled,
pred_idl)
rpc_output = subprocess.check_output(rpc_cmd, shell=True)
txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1]
output_png = os.path.join(output_path, "roc.png")
plot_cmd = './utils/annolist/plotSimple.py %s --output %s' % (txt_file, output_png)
plot_output = subprocess.check_output(plot_cmd, shell=True)
def get_results(args, H):
tf.reset_default_graph()
x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3])
if H['use_rezoom']:
pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(H, tf.expand_dims(x_in, 0), 'test', reuse=None)
grid_area = H['grid_height'] * H['grid_width']
pred_confidences = tf.reshape(tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])), [grid_area, H['rnn_len'], 2])
if H['reregress']:
pred_boxes = pred_boxes + pred_boxes_deltas
else:
pred_boxes, pred_logits, pred_confidences = build_forward(H, tf.expand_dims(x_in, 0), 'test', reuse=None)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
saver.restore(sess, args.weights)
pred_annolist = al.AnnoList()
data_dir = os.path.dirname(args.test_boxes)
image_dir = get_image_dir(args)
subprocess.call('mkdir -p %s' % image_dir, shell=True)
memory = sysv_ipc.SharedMemory(123463)
memory2 = sysv_ipc.SharedMemory(123464)
size = 768, 1024, 3
pedal = PyMouse()
pedal.press(1)
road_center = 320
while True:
cv2.waitKey(1)
frameCount = bytearray(memory.read())
curve = bytearray(memory2.read())
curve = str(struct.unpack('i',curve)[0])
m = np.array(frameCount, dtype=np.uint8)
orig_img = m.reshape(size)
img = imresize(orig_img, (H["image_height"], H["image_width"]), interp='cubic')
feed = {x_in: img}
(np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed)
pred_anno = al.Annotation()
new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes,
use_stitching=True, rnn_len=H['rnn_len'], min_conf=args.min_conf, tau=args.tau, show_suppressed=args.show_suppressed)
flag = 0
road_center = 320 + int(curve)
print(road_center)
for rect in rects:
print(rect.x1, rect.x2, rect.y2)
if (rect.x1 < road_center and rect.x2 > road_center and rect.y2 > 200) and (rect.x2 - rect.x1 > 30):
flag = 1
if flag is 1:
pedal.press(2)
print("break!")
else:
pedal.release(2)
pedal.press(1)
print("acceleration!")
pred_anno.rects = rects
pred_anno.imagePath = os.path.abspath(data_dir)
pred_anno = rescale_boxes((H["image_height"], H["image_width"]), pred_anno, orig_img.shape[0], orig_img.shape[1])
pred_annolist.append(pred_anno)
cv2.imshow('.jpg', new_img)
return none;
