def get_results(args, H): tf.reset_default_graph() x_in = tf.placeholder(tf.float32, name="x_in", shape=[H["image_height"], H["image_width"], 3]) googlenet = googlenet_load.init(H) if H["use_rezoom"]: pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward( H, tf.expand_dims(x_in, 0), googlenet, "test", reuse=None ) grid_area = H["grid_height"] * H["grid_width"] pred_confidences = tf.reshape( tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H["rnn_len"], 2])), [grid_area, H["rnn_len"], 2] ) if H["reregress"]: pred_boxes = pred_boxes + pred_boxes_deltas else: pred_boxes, pred_logits, pred_confidences = build_forward( H, tf.expand_dims(x_in, 0), googlenet, "test", reuse=None ) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) saver.restore(sess, args.weights) pred_annolist = al.AnnoList() true_annolist = al.parse(args.test_idl) data_dir = os.path.dirname(args.test_idl) image_dir = get_image_dir(args) subprocess.call("mkdir -p %s" % image_dir, shell=True) for i in range(len(true_annolist)): true_anno = true_annolist[i] orig_img = imread("%s/%s" % (data_dir, true_anno.imageName))[:, :, :3] img = imresize(orig_img, (H["image_height"], H["image_width"]), interp="cubic") feed = {x_in: img} (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles( H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H["rnn_len"], min_conf=0.2, tau=args.tau, ) pred_anno.rects = rects pred_anno.imagePath = os.path.abspath(data_dir) pred_anno = rescale_boxes( (H["image_height"], H["image_width"]), pred_anno, orig_img.shape[0], orig_img.shape[1] ) pred_annolist.append(pred_anno) imname = "%s/%s" % (image_dir, os.path.basename(true_anno.imageName)) misc.imsave(imname, new_img) if i % 25 == 0: print(i) return pred_annolist, true_annolist
def log_image(np_img, np_confidences, np_boxes, np_global_step, pred_or_true): merged = train_utils.add_rectangles(H, np_img, np_confidences, np_boxes, use_stitching=True, rnn_len=H['rnn_len'])[0] num_images = 10 img_path = os.path.join(H['save_dir'], '%s_%s.jpg' % ((np_global_step / H['logging']['display_iter']) % num_images, pred_or_true)) misc.imsave(img_path, merged) return merged
def get_results(args, H): tf.reset_default_graph() H["grid_width"] = H["image_width"] / H["region_size"] H["grid_height"] = H["image_height"] / H["region_size"] x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3]) if H['use_rezoom']: pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward( H, tf.expand_dims(x_in, 0), 'test', reuse=None) grid_area = H['grid_height'] * H['grid_width'] pred_confidences = tf.reshape( tf.nn.softmax( tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])), [grid_area, H['rnn_len'], 2]) if H['reregress']: pred_boxes = pred_boxes + pred_boxes_deltas else: pred_boxes, pred_logits, pred_confidences = build_forward( H, tf.expand_dims(x_in, 0), 'test', reuse=None) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, args.weights) orig_img = imread('data/tshirtslayer/tss-images/wearing.jpg') img = imresize(orig_img, (H["image_height"], H["image_width"]), interp='cubic') feed = {x_in: img} now = time.time() (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) print(time.time() - now) pred_anno = al.Annotation() pred_anno.imageName = "image.jpg" new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=args.min_conf, tau=args.tau, show_suppressed=args.show_suppressed) misc.imsave("data/xxx.jpg", new_img)
def eval(H, ckpt_file, in_dir, out_dir, conf): """ Re-constructs a TF based on the input checkpoint file (and the default googlenet graph) and applies it to the set of images in the input directory. If the output directory is supplied, annotated images are saved there, with detect boxes shown for any detect whose computed condifence exceeds 'conf' Note: the re """ # load graph tf.reset_default_graph() googlenet = googlenet_load.init(H) x_in = tf.placeholder(tf.float32, name='x_in') # add the TF ops necessary for the reinspect algorihtm, based on the architecture defined in the hypes file if H['arch']['use_lstm']: pred_boxes, pred_logits, pred_confidences = build_lstm_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None) else: pred_boxes, pred_logits, pred_confidences = build_overfeat_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test') saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) saver.restore(sess, ckpt_file) # process the files save_output = False if out_dir != None and os.path.isdir(out_dir): save_output = True file_list = os.listdir(in_dir) filenames = next(os.walk(in_dir))[2] num_files = len(filenames) for i in range(1,num_files,1): f = filenames[i] img_raw = cv2.imread(os.path.join(in_dir,f)) # Rudin images are 480 x 704, need them to be 480 x 640, so crop or resize ... img = img_raw[:,1:641,:] print 'Processing file:', f feed = {x_in: img} (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) new_img, rects = add_rectangles([img], np_pred_confidences, np_pred_boxes, H["arch"], use_stitching=True, rnn_len=H['arch']['rnn_len'], min_conf=conf) if True: cv2.imshow('Image with overlay',new_img) cv2.waitKey(1000) if save_output: cv2.imwrite(os.path.join(out_dir,f), new_img)
def postprocess_regular(image_info, np_pred_boxes, np_pred_confidences, H, options): pred_anno = al.Annotation() pred_anno.imageName = image_info['path'] pred_anno.imagePath = os.path.abspath(image_info['path']) _, rects = add_rectangles(H, [image_info['transformed']], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=options['min_conf'], tau=options['tau'], show_suppressed=False) h, w = image_info['original_shape'] if 'rotate90' in H['data'] and H['data']['rotate90']: # original image height is a width for rotated one rects = Rotate90.invert(h, rects) rects = [r for r in rects if r.x1 < r.x2 and r.y1 < r.y2 and r.score > options['min_conf']] pred_anno.rects = rects pred_anno = rescale_boxes((H['image_height'], H['image_width']), pred_anno, h, w) return pred_anno
def train(H, test_images): if not os.path.exists(H['save_dir']): os.makedirs(H['save_dir']) ckpt_file = H['save_dir'] + '/save.ckpt' with open(H['save_dir'] + '/hypes.json', 'w') as f: json.dump(H, f, indent=4) x_in = tf.placeholder(tf.float32) confs_in = tf.placeholder(tf.float32) boxes_in = tf.placeholder(tf.float32) q = {} enqueue_op = {} for phase in ['train', 'test']: dtypes = [tf.float32, tf.float32, tf.float32] grid_size = H['arch']['grid_width'] * H['arch']['grid_height'] shapes = ( [H['arch']['image_height'], H['arch']['image_width'], 3], [grid_size, H['arch']['rnn_len'], H['arch']['num_classes']], [grid_size, H['arch']['rnn_len'], 4], ) q[phase] = tf.FIFOQueue(capacity=30, dtypes=dtypes, shapes=shapes) enqueue_op[phase] = q[phase].enqueue((x_in, confs_in, boxes_in)) def make_feed(d): return {x_in: d['image'], confs_in: d['confs'], boxes_in: d['boxes'], learning_rate: H['solver']['learning_rate']} def MyLoop(sess, enqueue_op, phase, gen): for d in gen: sess.run(enqueue_op[phase], feed_dict=make_feed(d)) (config, loss, accuracy, summary_op, train_op, W_norm, test_image, test_pred_boxes, test_pred_confidences, test_true_boxes, test_true_confidences, smooth_op, global_step, learning_rate) = build(H, q) saver = tf.train.Saver(max_to_keep=None) writer = tf.train.SummaryWriter( logdir=H['save_dir'], flush_secs=10 ) test_image_to_log = tf.placeholder(tf.uint8, [H['arch']['image_height'], H['arch']['image_width'], 3]) log_image_name = tf.placeholder(tf.string) log_image = tf.image_summary(log_image_name, tf.expand_dims(test_image_to_log, 0)) with tf.Session(config=config) as sess: threads = [] for phase in ['train', 'test']: # enqueue once manually to avoid thread start delay gen = train_utils.load_data_gen(H, phase, jitter=H['solver']['use_jitter']) d = gen.next() sess.run(enqueue_op[phase], feed_dict=make_feed(d)) threads.append(tf.train.threading.Thread(target=MyLoop, args=(sess, enqueue_op, phase, gen))) threads[-1].start() tf.set_random_seed(H['solver']['rnd_seed']) sess.run(tf.initialize_all_variables()) weights_str = H['solver']['weights'] if len(weights_str) > 0: print('Restoring from: %s' % weights_str) saver.restore(sess, weights_str) # train model for N iterations for i in xrange(10000000): display_iter = 10 adjusted_lr = (H['solver']['learning_rate'] * 0.5 ** max(0, (i / H['solver']['learning_rate_step']) - 2)) lr_feed = {learning_rate: adjusted_lr} if i % display_iter == 0: if i > 0: dt = (time.time() - start) / (H['arch']['batch_size'] * display_iter) start = time.time() (batch_loss_train, test_accuracy, weights_norm, summary_str, np_test_image, np_test_pred_boxes, np_test_pred_confidences, np_test_true_boxes, np_test_true_confidences, _, _) = sess.run([ loss['train'], accuracy['test'], W_norm, summary_op, test_image, test_pred_boxes, test_pred_confidences, test_true_boxes, test_true_confidences, train_op, smooth_op, ], feed_dict=lr_feed) pred_true = [("%d_pred_output" % (i % 3), np_test_pred_boxes, np_test_pred_confidences), ("%d_true_output" % (i % 3), np_test_true_boxes, np_test_true_confidences)] for name, boxes, confidences in pred_true: test_output_to_log = train_utils.add_rectangles(np_test_image, confidences, boxes, H["arch"])[0] assert test_output_to_log.shape == (H['arch']['image_height'], H['arch']['image_width'], 3) feed = {test_image_to_log: test_output_to_log, log_image_name: name} test_image_summary_str = sess.run(log_image, feed_dict=feed) writer.add_summary(test_image_summary_str, global_step=global_step.eval()) writer.add_summary(summary_str, global_step=global_step.eval()) print_str = string.join([ 'Step: %d', 'lr: %f', 'Train Loss: %.2f', 'Test Accuracy: %.1f%%', 'Time/image (ms): %.1f' ], ', ') print(print_str % (i, adjusted_lr, batch_loss_train, test_accuracy * 100, dt * 1000 if i > 0 else 0)) else: batch_loss_train, _ = sess.run([loss['train'], train_op], feed_dict=lr_feed) if global_step.eval() % 1000 == 0: saver.save(sess, ckpt_file, global_step=global_step)
def run_eval(H, checkpoint_dir , hypes_file, output_path): """Do Evaluation with full epoche of data. Args: H: Hypes checkpoint_dir: directory with checkpoint files output_path: path to save results """ #Load GT true_idl = H['data']['test_idl'] true_annos = al.parse(true_idl) # define output files pred_file = 'val_%s.idl' % os.path.basename(hypes_file).replace('.json', '') pred_idl = os.path.join(output_path, pred_file) true_file = 'true_%s.idl' % os.path.basename(hypes_file).replace('.json', '') true_idl_scaled = os.path.join(output_path, true_file) data_folder = os.path.dirname(os.path.realpath(true_idl)) #Load Graph Model tf.reset_default_graph() googlenet = googlenet_load.init(H) x_in = tf.placeholder(tf.float32, name='x_in') if H['arch']['use_lstm']: lstm_forward = build_lstm_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test', reuse=None) pred_boxes, pred_logits, pred_confidences = lstm_forward else: overfeat_forward = build_overfeat_forward(H, tf.expand_dims(x_in, 0), googlenet, 'test') pred_boxes, pred_logits, pred_confidences = overfeat_forward start_time = time.time() saver = tf.train.Saver() with tf.Session() as sess: logging.info("Starting Evaluation") sess.run(tf.initialize_all_variables()) # Restore Checkpoints ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: logging.info(ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) annolist = al.AnnoList() trueanno = al.AnnoList() #shuffle true_annos to randomize plottet Images shuffle(true_annos) for i in range(len(true_annos)): true_anno = true_annos[i] img = imread( os.path.join(data_folder, true_anno.imageName)) # Rescale Boxes trueanno.append(rescale_boxes(img.shape, true_annos[i], H["arch"]["image_height"], H["arch"]["image_width"])) # Rescale Images img = imresize(img, (H["arch"]["image_height"], H["arch"]["image_width"]), interp='cubic') feed = {x_in: img} (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) pred_anno = al.Annotation() pred_anno.imageName = true_anno.imageName new_img, rects = add_rectangles([img], np_pred_confidences, np_pred_boxes, H["arch"], use_stitching=True, rnn_len=H['arch']['rnn_len'], min_conf=0.3) pred_anno.rects = rects annolist.append(pred_anno) if i % 20 == 0: # Draw every 20th Image; # plotted Image is randomized due to shuffling duration = time.time() - start_time duration = float(duration)*1000/20 out_img = os.path.join(output_path, 'test_%i.png'%i) scp.misc.imsave(out_img, new_img) logging.info('Step %d: Duration %.3f ms' % (i, duration)) start_time = time.time() annolist.save(pred_idl) trueanno.save(true_idl_scaled) # write results to disk iou_threshold = 0.5 rpc_cmd = './utils/annolist/doRPC.py --minOverlap %f %s %s' % (iou_threshold, true_idl_scaled, pred_idl) rpc_output = subprocess.check_output(rpc_cmd, shell=True) txt_file = [line for line in rpc_output.split('\n') if line.strip()][-1] output_png = os.path.join(output_path, "roc.png") plot_cmd = './utils/annolist/plotSimple.py %s --output %s' % (txt_file, output_png) plot_output = subprocess.check_output(plot_cmd, shell=True)
def get_results(args, H): tf.reset_default_graph() x_in = tf.placeholder(tf.float32, name='x_in', shape=[H['image_height'], H['image_width'], 3]) if H['use_rezoom']: pred_boxes, pred_logits, pred_confidences, pred_confs_deltas, pred_boxes_deltas = build_forward(H, tf.expand_dims(x_in, 0), 'test', reuse=None) grid_area = H['grid_height'] * H['grid_width'] pred_confidences = tf.reshape(tf.nn.softmax(tf.reshape(pred_confs_deltas, [grid_area * H['rnn_len'], 2])), [grid_area, H['rnn_len'], 2]) if H['reregress']: pred_boxes = pred_boxes + pred_boxes_deltas else: pred_boxes, pred_logits, pred_confidences = build_forward(H, tf.expand_dims(x_in, 0), 'test', reuse=None) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.initialize_all_variables()) saver.restore(sess, args.weights) pred_annolist = al.AnnoList() data_dir = os.path.dirname(args.test_boxes) image_dir = get_image_dir(args) subprocess.call('mkdir -p %s' % image_dir, shell=True) memory = sysv_ipc.SharedMemory(123463) memory2 = sysv_ipc.SharedMemory(123464) size = 768, 1024, 3 pedal = PyMouse() pedal.press(1) road_center = 320 while True: cv2.waitKey(1) frameCount = bytearray(memory.read()) curve = bytearray(memory2.read()) curve = str(struct.unpack('i',curve)[0]) m = np.array(frameCount, dtype=np.uint8) orig_img = m.reshape(size) img = imresize(orig_img, (H["image_height"], H["image_width"]), interp='cubic') feed = {x_in: img} (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) pred_anno = al.Annotation() new_img, rects = add_rectangles(H, [img], np_pred_confidences, np_pred_boxes, use_stitching=True, rnn_len=H['rnn_len'], min_conf=args.min_conf, tau=args.tau, show_suppressed=args.show_suppressed) flag = 0 road_center = 320 + int(curve) print(road_center) for rect in rects: print(rect.x1, rect.x2, rect.y2) if (rect.x1 < road_center and rect.x2 > road_center and rect.y2 > 200) and (rect.x2 - rect.x1 > 30): flag = 1 if flag is 1: pedal.press(2) print("break!") else: pedal.release(2) pedal.press(1) print("acceleration!") pred_anno.rects = rects pred_anno.imagePath = os.path.abspath(data_dir) pred_anno = rescale_boxes((H["image_height"], H["image_width"]), pred_anno, orig_img.shape[0], orig_img.shape[1]) pred_annolist.append(pred_anno) cv2.imshow('.jpg', new_img) return none;