def do_inference(logdir): """ Analyze a trained model. This will load model files and weights found in logdir and run a basic analysis. Parameters ---------- logdir : string Directory with logs. """ hypes = utils.load_hypes_from_logdir(logdir) modules = utils.load_modules_from_logdir(logdir) # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # prepaire the tv session with tf.name_scope('Validation'): image_pl, label_pl = _create_input_placeholder() image = tf.expand_dims(image_pl, 0) softmax = core.build_inference_graph(hypes, modules, image=image) sess = tf.Session() saver = tf.train.Saver() core.load_weights(logdir, sess, saver) create_test_output(hypes, sess, image_pl, softmax) return
def continue_training(logdir): """ Continues training of a model. This will load model files and weights found in logdir and continues an aborted training. Parameters ---------- logdir : string Directory with logs. """ hypes = utils.load_hypes_from_logdir(logdir) modules = utils.load_modules_from_logdir(logdir) data_input, arch, objective, solver = modules # append output to output.log logging_file = os.path.join(logdir, 'output.log') utils.create_filewrite_handler(logging_file, mode='a') # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default() as graph: # build the graph based on the loaded modules graph_ops = build_training_graph(hypes, modules) q = graph_ops[0] # prepaire the tv session sess_coll = core.start_tv_session(hypes) sess, saver, summary_op, summary_writer, coord, threads = sess_coll if hasattr(objective, 'evaluate'): with tf.name_scope('Validation'): image_pl, label_pl = _create_input_placeholder() image = tf.expand_dims(image_pl, 0) softmax = core.build_inference_graph(hypes, modules, image=image, label=label_pl) # Load weights from logdir cur_step = core.load_weights(logdir, sess, saver) # Start the data load _start_enqueuing_threads(hypes, q, sess, data_input) # And then after everything is built, start the training loop. start_time = time.time() for step in xrange(cur_step+1, hypes['solver']['max_steps']): start_time = run_training_step(hypes, step, start_time, graph_ops, sess_coll, objective, image_pl, softmax) # stopping input Threads coord.request_stop() coord.join(threads)
def do_training(hypes): """ Train model for a number of steps. This trains the model for at most hypes['solver']['max_steps']. It shows an update every utils.cfg.step_show steps and writes the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval steps. Paramters --------- hypes : dict Hyperparameters """ # Get the sets of images and labels for training, validation, and # test on MNIST. modules = utils.load_modules_from_hypes(hypes) data_input, arch, objective, solver = modules # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # build the graph based on the loaded modules graph_ops = build_training_graph(hypes, modules) q = graph_ops[0] # prepaire the tv session sess_coll = core.start_tv_session(hypes) sess, saver, summary_op, summary_writer, coord, threads = sess_coll with tf.name_scope('Validation'): image_pl, label_pl = _create_input_placeholder() image = tf.expand_dims(image_pl, 0) softmax = core.build_inference_graph(hypes, modules, image=image, label=label_pl) # Start the data load _start_enqueuing_threads(hypes, q, sess, data_input) # And then after everything is built, start the training loop. start_time = time.time() for step in xrange(hypes['solver']['max_steps']): start_time = run_training_step(hypes, step, start_time, graph_ops, sess_coll, objective, image_pl, softmax) if hasattr(solver, 'update_learning_rate'): solver.update_learning_rate(hypes, step) # stopping input Threads coord.request_stop() coord.join(threads)
def do_analyze(logdir, base_path=None): """ Analyze a trained model. This will load model files and weights found in logdir and run a basic analysis. Parameters ---------- logdir : string Directory with logs. """ hypes = utils.load_hypes_from_logdir(logdir) modules = utils.load_modules_from_logdir(logdir) if base_path is not None: hypes['dirs']['base_path'] = base_path # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # prepaire the tv session image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() core.load_weights(logdir, sess, saver) logging.info("Graph loaded succesfully. Starting evaluation.") output_dir = os.path.join(logdir, 'analyse') logging.info("Output Images will be written to: {}".format( os.path.join(output_dir, "images/"))) logging_file = os.path.join(logdir, "analyse/output.log") utils.create_filewrite_handler(logging_file) eval_dict, images = modules['eval'].evaluate( hypes, sess, image_pl, inf_out) logging.info("Evaluation Succesfull. Results:") utils.print_eval_dict(eval_dict) _write_images_to_logdir(images, output_dir)
def do_training(hypes): """ Train model for a number of steps. This trains the model for at most hypes['solver']['max_steps']. It shows an update every utils.cfg.step_show steps and writes the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval steps. Paramters --------- hypes : dict Hyperparameters """ # Get the sets of images and labels for training, validation, and # test on MNIST. modules = utils.load_modules_from_hypes(hypes) # Tell TensorFlow that the model will be built into the default Graph. with tf.Session() as sess: # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = core.build_training_graph(hypes, queue, modules) # prepaire the tv session tv_sess = core.start_tv_session(hypes) with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess) # And then after everything is built, start the training loop. run_training(hypes, modules, tv_graph, tv_sess) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def init(): tv_utils.set_gpus_to_use() if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiBox') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") return hypes, image_pl, prediction, sess
def main(_): tv_utils.set_gpus_to_use() # Download and use weights from the MultiNet Paper runs_dir = '.' logdir = os.path.join(runs_dir, default_run) # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32, name='input_image') image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) tf.identity(prediction['logits'], name='output_logits') tf.identity(prediction['softmax'], name='output_softmax') logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") # Save model again with renamed tensors saver.save(sess, 'renamed/KittiSeg_pretrained') logging.info("Model saved successfully.")
def do_analyze(logdir): """ Analyze a trained model. This will load model files and weights found in logdir and run a basic analysis. Parameters ---------- logdir : string Directory with logs. """ hypes = utils.load_hypes_from_logdir(logdir) modules = utils.load_modules_from_logdir(logdir) data_input, arch, objective, solver = modules # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # prepaire the tv session with tf.name_scope('Validation'): image_pl, label_pl = _create_input_placeholder() image = tf.expand_dims(image_pl, 0) softmax = core.build_inference_graph(hypes, modules, image=image, label=label_pl) sess_coll = core.start_tv_session(hypes) sess, saver, summary_op, summary_writer, coord, threads = sess_coll core.load_weights(logdir, sess, saver) eval_dict, images = objective.tensor_eval(hypes, sess, image_pl, softmax) logging_file = os.path.join(logdir, "eval/analysis.log") utils.create_filewrite_handler(logging_file) utils.print_eval_dict(eval_dict) _write_images_to_logdir(images, logdir) return
def load(logdir): import tensorflow as tf import tensorvision.utils as tv_utils import tensorvision.core as core tv_utils.set_gpus_to_use() # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32, shape=(hypes["image_height"], hypes["image_width"], 3)) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") return image_pl, prediction, sess, hypes
def main(_): tv_utils.set_gpus_to_use() if (FLAGS.input_image is None) and (FLAGS.output_image is None): logging.error("No input_image or output_image were given.") logging.info( "Usage: python demo_modified.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") # Load and resize input image input_image = FLAGS.input_image input_images = open(input_image, 'r').read().splitlines() # Run KittiSeg model on image n = 0 for list_images in input_images: if (n % 3) == 0: image = scp.misc.imread(list_images) name_split = list_images.split('/') image_name = name_split[len(name_split) - 1].split('.')[0] if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') feed = {image_pl: image} softmax = prediction['softmax'] #time of net foward start_time = timeit.default_timer() output = sess.run([softmax], feed_dict=feed) elapsed = timeit.default_timer() - start_time print(elapsed) #exit(1) #feed = {image_pl: image} #softmax = prediction['softmax'] #output = sess.run([softmax], feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) # Plot confidences as red-blue overlay rb_image = seg.make_overlay(image, output_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.01 street_prediction = output_image > threshold # Plot the hard prediction as green overlay green_image = tv_utils.fast_overlay(image, street_prediction) # Save output images to disk. output_base_name = FLAGS.output_image raw_image_name = output_base_name + image_name + '_raw.png' #rb_image_name = output_base_name + image_name + '_rb.png' green_image_name = output_base_name + image_name + '_green.png' scp.misc.imsave(raw_image_name, output_image) #scp.misc.imsave(rb_image_name, rb_image) scp.misc.imsave(green_image_name, green_image) logging.info("") logging.info("Raw output image has been saved to: {}".format( os.path.realpath(raw_image_name))) #logging.info("Red-Blue overlay of confs have been saved to: {}".format( # os.path.realpath(rb_image_name))) logging.info( "Green plot of predictions have been saved to: {}".format( os.path.realpath(green_image_name))) n = n + 1
def do_evaling(hypes): """ eval model for a number of steps. This trains the model for at most hypes['solver']['max_steps']. It shows an update every utils.cfg.step_show steps and writes the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval steps. Paramters --------- hypes : dict Hyperparameters """ # Get the sets of images and labels for training, validation, and # test on MNIST. modules = utils.load_modules_from_hypes(hypes) # Tell TensorFlow that the model will be built into the default Graph. with tf.Session() as sess: # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = core.build_training_graph(hypes, queue, modules) # prepaire the tv session tv_sess = core.start_tv_session(hypes) with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) calib = tf.placeholder( tf.float32, shape=[1, hypes['grid_height'], hypes['grid_width'], 3, 4]) xy_scale = tf.placeholder( tf.float32, shape=[1, hypes['grid_height'], hypes['grid_width'], 2]) image = tf.expand_dims(image_pl, 0) image.set_shape([1, 384, 1248, 3]) inf_out = core.build_inference_graph(hypes, modules, image, calib, xy_scale) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out tv_graph['calib_pl'] = calib tv_graph['xy_scale_pl'] = xy_scale all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES) sess.run(tf.variables_initializer(all_variables)) # Start the data load modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess) saver = tf.train.Saver() print("Reading checkpoints...") ckpt = tf.train.get_checkpoint_state(hypes["dirs"]["ckpt_dir"]) if ckpt and ckpt.model_checkpoint_path: global_step = ckpt.model_checkpoint_path.split('/')[-1].split( '-')[-1] saver.restore(sess, ckpt.model_checkpoint_path) print('Loading success, global_step is %s' % global_step) # And then after everything is built, start the training loop. run_evaling(hypes, modules, tv_graph, tv_sess) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def build_united_model(meta_hypes): logging.info("Initialize training folder") subhypes = {} subgraph = {} submodules = {} subqueues = {} subgraph['debug_ops'] = {} base_path = meta_hypes['dirs']['base_path'] first_iter = True for model in meta_hypes['model_list']: subhypes_file = os.path.join(base_path, meta_hypes['models'][model]) with open(subhypes_file, 'r') as f: logging.info("f: %s", f) subhypes[model] = json.load(f) hypes = subhypes[model] utils.set_dirs(hypes, subhypes_file) #output directories hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir'] #data directories hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir'] #initialize the training folders train.initialize_training_folder(hypes, files_dir=model, logging=first_iter) #get the image directory from hypes meta_hypes['dirs']['image_dir'] = hypes['dirs']['image_dir'] #load the modules from hypes submodules[model] = utils.load_modules_from_hypes( hypes, postfix="_%s" % model) modules = submodules[model] logging.info("Build %s computation Graph.", model) with tf.name_scope("Queues_%s" % model): subqueues[model] = modules['input'].create_queues(hypes, 'train') logging.info('Building Model: %s' % model) subgraph[model] = build_training_graph(hypes, subqueues[model], modules, first_iter) first_iter = False #if model list is having detection and segmentation if len(meta_hypes['models']) == 2: #calculate the losses _recombine_2_losses(meta_hypes, subgraph, subhypes, submodules) #if model list is having detection, segmentation and classification else: #calculate combined losses _recombine_3_losses(meta_hypes, subgraph, subhypes, submodules) hypes = subhypes[meta_hypes['model_list'][0]] tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] #for each of the models in model list expand the image dimension for model in meta_hypes['model_list']: hypes = subhypes[model] modules = submodules[model] optimizer = modules['solver'] with tf.name_scope('Validation_%s' % model): tf.get_variable_scope().reuse_variables() #returns the tensor that may be used as handle for feeding a value image_pl = tf.placeholder(tf.float32) #expand the shape of the array by inserting new axes in 0th positon image = tf.expand_dims(image_pl, 0) inf_out = core.build_inference_graph(hypes, modules, image=image) subgraph[model]['image_pl'] = image_pl subgraph[model]['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, subqueues[model], 'train', sess) #join paths for the output result target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json') with open(target_file, 'w') as outfile: json.dump(meta_hypes, outfile, indent=2, sort_keys=True) return subhypes, submodules, subgraph, tv_sess
def main(_): blank_image = scp.misc.imread('blank_image_google.png') output_dir_raw = '' output_dir_raw = output_dir_raw + FLAGS.logdir output_dir_raw = output_dir_raw + dirResName print(output_dir_raw) os.makedirs(output_dir_raw, exist_ok=True) tv_utils.set_gpus_to_use() # if FLAGS.input_image is None: # logging.error("No input_image was given.") # logging.info( # "Usage: python demo.py --input_image data/test.png " # "[--output_image output_image] [--logdir /path/to/weights] " # "[--gpus GPUs_to_use] ") # exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") for streetname in os.listdir(inputdir): ##trocar linha comentada abaixo pelas posteriores streetpath = os.path.join(inputdir, streetname) #streetpath = inputdir #streetname = 'unica' output_dir_raw_currentstreet = os.path.join(output_dir_raw, streetname) print("output_dir_raw_currentstreet: " + output_dir_raw_currentstreet) #input('parada:') os.makedirs(output_dir_raw_currentstreet, exist_ok=True) for filename in os.listdir(streetpath): input_image = os.path.join(streetpath, filename) logging.info( "Starting inference using {} as input".format(input_image)) # Load and resize input image image = scp.misc.imread(input_image) width, height, z = blank_image.shape isBlank = True for i in range(width): for j in range(height): if blank_image[i, j, 0] != image[i, j, 0] or blank_image[ i, j, 1] != image[i, j, 1] or blank_image[ i, j, 2] != image[i, j, 2]: isBlank = False break if isBlank == False: break if isBlank: continue if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') #image = convertToHSV(image) # classes classes_colors = [ hypes['data']['background_color'], hypes['data']['paved_road_color'], hypes['data']['nonpaved_road_color'], hypes['data']['rocks_road_color'] ] #classes_colors = [ hypes['data']['background_color'], hypes['data']['paved_road_color'], hypes['data']['nonpaved_road_color'] ] # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run(softmax, feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output.reshape(shape[0], shape[1], -1) x = np.argmax(output_image, axis=2) im = np.zeros((shape[0], shape[1], 3), dtype=np.uint8) for i, _ in enumerate(x): for j, _ in enumerate(x[i]): value = x[i][j] color_code = classes_colors[value] im[i][j] = color_code # Save output images to disk. if FLAGS.output_image is None: output_base_name = input_image else: output_base_name = FLAGS.output_image raw_image_name = filename.split('.png')[0] + '_raw.png' #print('============= '+output_dir_raw_currentstreet) #print('============= '+raw_image_name) #print('============= '+os.path.join(output_dir_raw_currentstreet, raw_image_name)) #input('parada:') scp.misc.imsave( os.path.join(output_dir_raw_currentstreet, raw_image_name), im)
def do_training(hypes): """ Train model for a number of steps. This trains the model for at most hypes['solver']['max_steps']. It shows an update every utils.cfg.step_show steps and writes the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval steps. Paramters --------- hypes : dict Hyperparameters """ # Get the sets of images and labels for training, validation, and # test on MNIST. modules = utils.load_modules_from_hypes(hypes) # Tell TensorFlow that the model will be built into the default Graph. with tf.Session() as sess: # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') regression_weights = tf.placeholder(dtype=tf.float32, shape=(3,)) hypes['solver']['regression_weights'] = regression_weights tv_graph = core.build_training_graph(hypes, queue, modules) # prepaire the tv session tv_sess = core.start_tv_session(hypes) with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) calib = tf.placeholder(tf.float32, shape=[1, hypes['grid_height'], hypes['grid_width'], 3, 4]) xy_scale = tf.placeholder(tf.float32, shape=[1, hypes['grid_height'], hypes['grid_width'], 2]) image = tf.expand_dims(image_pl, 0) image.set_shape([1, 384, 1248, 3]) inf_out, encoder_out = core.build_inference_graph(hypes, modules, image, calib, xy_scale) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out tv_graph['calib_pl'] = calib tv_graph['xy_scale_pl'] = xy_scale tv_graph['encoder_out'] = encoder_out all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES) sess.run(tf.variables_initializer(all_variables)) var_list = [var for var in all_variables if "beta" not in var.name and 'Adam' not in var.name] saver = tf.train.Saver(var_list=var_list) saver.restore(sess, hypes['pretrained']) # Start the data load modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess) # And then after everything is built, start the training loop. run_training(hypes, modules, tv_graph, tv_sess) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) # Load and resize input image image = scp.misc.imread(input_image) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run([softmax], feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) print("pixe_value") print(output_image[0][1]) # Plot confidences as red-blue overlay rb_image = seg.make_overlay(image, output_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.5 street_prediction = output_image > threshold print("predic_val") suoyin = np.where(street_prediction == True) chang = len(suoyin[0]) test = np.zeros((chang, 2), dtype=np.int) for tmp0 in range(chang): test[tmp0][0] = suoyin[0][tmp0] test[tmp0][1] = suoyin[1][tmp0] print(test[0].shape) print(suoyin[0].shape) print(len(suoyin[0])) # Plot the hard prediction as green overlay green_image = tv_utils.fast_overlay(image, street_prediction) # Save output images to disk. if FLAGS.output_image is None: output_base_name = input_image else: output_base_name = FLAGS.output_image raw_image_name = output_base_name.split('.')[0] + '_raw.png' rb_image_name = output_base_name.split('.')[0] + '_rb.png' green_image_name = output_base_name.split('.')[0] + '_green.png' #scp.misc.imsave(raw_image_name, output_image) #scp.misc.imsave(rb_image_name, rb_image) scp.misc.imsave(green_image_name, green_image) logging.info("") logging.info("Raw output image has been saved to: {}".format( os.path.realpath(raw_image_name))) logging.info("Red-Blue overlay of confs have been saved to: {}".format( os.path.realpath(rb_image_name))) logging.info("Green plot of predictions have been saved to: {}".format( os.path.realpath(green_image_name)))
def main(_): tv_utils.set_gpus_to_use() # if FLAGS.input_image is None: # logging.error("No input_image was given.") # logging.info( # "Usage: python demo.py --input_image data/test.png " # "[--output_image output_image] [--logdir /path/to/weights] " # "[--gpus GPUs_to_use] ") # exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) # Load and resize input image count = 1 duration = 0 video = cv2.VideoCapture( "/home/zzhang52/Data/south_farm/front/VID_20170426_144800.mp4") fourcc = cv2.VideoWriter_fourcc(*'XVID') writer = cv2.VideoWriter("/home/zzhang52/Videos/visual_nav.avi", fourcc, 12.0, (960, 540)) while video.grab(): start = time() _, image = video.retrieve() image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='bilinear') # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run([softmax], feed_dict=feed) duration += time() - start count += 1 # Reshape output from flat vector to 2D Image shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) # Plot confidences as red-blue overlay # rb_image = seg.make_overlay(image, output_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.5 street_prediction = output_image > threshold thresh = (255 * street_prediction).astype(np.uint8) _, thresh = cv2.threshold(thresh, 100, 255, 0) _, contours, _ = cv2.findContours(thresh, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_NONE) contours.sort(key=cv2.contourArea, reverse=True) mask = np.zeros(shape, np.uint8) cv2.drawContours(mask, contours, 0, (0, 255, 0), -1) cnt = np.asarray(contours[0]) cnt = np.squeeze(cnt) cnt = cnt[np.argsort(cnt[:, 1])] cnt = np.expand_dims(cnt, axis=1) points = [] idx = 0 x_sum = 0 x_count = 0 for i in range(shape[0]): while idx < cnt.shape[0] and cnt[idx][0][1] == i: x_sum += float(cnt[idx][0][0]) x_count += 1 idx += 1 if x_count != 0: avg = int(x_sum / x_count) points.append(np.array([[avg, i]])) x_sum = 0 x_count = 0 points = np.asarray(points) # p1, p2, [vx,vy,x,y] = fitLine(points, shape[0], shape[1]) # [vx,vy,x,y] = cv2.fitLine(points, cv2.DIST_L2,0,0.01,0.01) # lefty = int((-x*vy/vx) + y) # righty = int(((shape[1]-x)*vy/vx)+y) p1 = Point(points[0][0][0], points[0][0][1]) p2 = Point(points[-1][0][0], points[-1][0][1]) dx = float(p1.x - p2.x) dy = float(p1.y - p2.y) r = np.arctan(dx / dy) * 180 / np.pi t = float(shape[1] / 2 - p2.x) / shape[1] # r = np.sign(r) * (90 - abs(r)) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) alpha = 0.3 overlay = cv2.addWeighted(image, 1 - alpha, mask, alpha, 0) # Plot the hard prediction as green overlay # green_image = tv_utils.fast_overlay(image, street_prediction) # green_image = cv2.cvtColor(green_image, cv2.COLOR_RGB2BGR) cv2.line(overlay, (p1.x, p1.y), (p2.x, p2.y), (0, 0, 200), 2) cv2.putText(overlay, "Heading: {:.2f}".format(r), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2) cv2.putText(overlay, "Offset: {:.3e}".format(t), (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2) cv2.imshow("show", overlay) writer.write(overlay) if cv2.waitKey(15) == 27: break cv2.destroyAllWindows() video.release() writer.release() # Save output images to disk. # if FLAGS.output_image is None: # output_base_name = input_image # else: # output_base_name = FLAGS.output_image # raw_image_name = output_base_name.split('.')[0] + '_raw.png' # rb_image_name = output_base_name.split('.')[0] + '_rb.png' # green_image_name = output_base_name.split('.')[0] + '_green.png' # scp.misc.imsave(raw_image_name, street_prediction) # scp.misc.imsave(rb_image_name, rb_image) # scp.misc.imsave(green_image_name, green_image) # logging.info("") # logging.info("Raw output image has been saved to: {}".format( # os.path.realpath(raw_image_name))) # logging.info("Red-Blue overlay of confs have been saved to: {}".format( # os.path.realpath(rb_image_name))) # logging.info("Green plot of predictions have been saved to: {}".format( # os.path.realpath(green_image_name))) print("FPS: {}".format(count / duration))
def model_path(): tv_utils.set_gpus_to_use() # if FLAGS.input_image is None: # logging.error("No input_image was given.") # logging.info( # "Usage: python demo.py --input_image data/test.png " # "[--output_image output_image] [--logdir /path/to/weights] " # "[--gpus GPUs_to_use] ") # exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") cv2.namedWindow("visual_image", flags=cv2.WINDOW_FREERATIO) while True: frame = queue.get() cv2.imshow("image", frame) #cv2.waitKey(1) image = frame # logging.info("Starting inference using {} as input".format(input_image)) # Load and resize input image #image = scp.misc.imread(input_image) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run([softmax], feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) # Plot confidences as red-blue overlay rb_image = seg.make_overlay(image, output_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.5 street_prediction = output_image > threshold # Plot the hard prediction as green overlay green_image = tv_utils.fast_overlay(image, street_prediction) cv2.imshow("visual_image", green_image) cv2.waitKey(1)
def continue_training(logdir): """ Continues training of a model. This will load model files and weights found in logdir and continues an aborted training. Parameters ---------- logdir : string Directory with logs. """ hypes = utils.load_hypes_from_logdir(logdir) modules = utils.load_modules_from_logdir(logdir) # Tell TensorFlow that the model will be built into the default Graph. with tf.Session() as sess: # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = core.build_training_graph(hypes, queue, modules) # prepaire the tv session tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] saver = tv_sess['saver'] logging_file = os.path.join(logdir, 'output.log') utils.create_filewrite_handler(logging_file, mode='a') logging.info("Continue training.") cur_step = core.load_weights(logdir, sess, saver) if cur_step is None: logging.warning("Loaded global_step is None.") logging.warning("This could mean," " that no weights have been loaded.") logging.warning("Starting Training with step 0.") cur_step = 0 with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess) # And then after everything is built, start the training loop. run_training(hypes, modules, tv_graph, tv_sess, cur_step) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def main(): tv_utils.set_gpus_to_use() args = parser.parse_args() logdir = args.logdir # http://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python try: os.makedirs(args.save) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(args.save): pass else: logging.error('--save flag must be a directory.') # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32, shape=(hypes["image_height"], hypes["image_width"], 3)) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") test_path = args.test_path image_names = sorted(os.listdir(test_path)) start = time.time() for i, image_name in enumerate(image_names): input_image = os.path.join(test_path, image_name) # Load and resize input image image = scp.misc.imread(input_image) image = scp.misc.imresize(image, (hypes["image_height"], hypes["image_width"]), interp='cubic') feed = {image_pl: image} # Run KittiBox model on image pred_boxes = prediction['pred_boxes_new'] pred_confidences = prediction['pred_confidences'] (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) # Apply non-maximal suppression # and draw predictions on the image output_image, rects = kittibox_utils.add_rectangles( hypes, [image], np_pred_confidences, np_pred_boxes, show_removed=False, use_stitching=True, rnn_len=1, min_conf=args.threshold, tau=hypes['tau'], color_acc=(0, 255, 0)) test_file_name = image_name.split('.')[0] + '.txt' test_file = os.path.join(args.save, test_file_name) write_rects(rects, test_file) time_taken = time.time() - start logging.info('Number of images: %d' % len(image_names)) logging.info('Time takes: %.2f s' % (time_taken)) logging.info('Frequency: %.2f fps' % (len(image_names) / time_taken))
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) maybe_download_and_extract(runs_dir) logging.info("Trimming weights.") logdir = os.path.join(runs_dir, FLAGS.RUN) modules = utils.load_modules_from_hypes(hypes) with tf.Graph().as_default(): # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = core.build_training_graph(hypes, queue, modules) # prepare the tv session with tf.Session().as_default(): tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] saver = tv_sess['saver'] cur_step = core.load_weights(logdir, sess, saver) if cur_step is None: logging.warning("Loaded global_step is None.") logging.warning("This could mean," " that no weights have been loaded.") logging.warning("Starting Training with step 0.") cur_step = 0 with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out # prepaire the tv session image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) # Create a session for running Ops on the Graph. trim_dir = 'RUNS/trimmed' shutil.copytree(logdir, trim_dir) shutil.copy(tf.app.flags.FLAGS.hypes, os.path.join(trim_dir, 'model_files', 'hypes.json')) sess = tf.Session() saver = tf.train.Saver() core.load_weights(trim_dir, sess, saver) for weight in tf.contrib.model_pruning.get_masks(): if any([ layer in weight.name for layer in hypes['layer_pruning']['layers'] ]): weight_value = tv_sess['sess'].run(weight) kernel_count = int(weight_value.shape[3] * hypes['layer_pruning']['layer_sparsity']) l1_values = np.sum(np.abs(weight_value), axis=(0, 1, 2)) toss_kernels = l1_values.argsort()[:kernel_count] weight_value[:, :, :, toss_kernels] = 0 assign_op = tf.assign(weight, tf.constant(weight_value)) tv_sess['sess'].run(assign_op) checkpoint_path = os.path.join(trim_dir, 'model.ckpt') tv_sess['saver'].save(sess, checkpoint_path, global_step=cur_step) train.continue_training(trim_dir)
def do_training(hypes): """ Train model for a number of steps. This trains the model for at most hypes['solver']['max_steps']. It shows an update every utils.cfg.step_show steps and writes the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval steps. Paramters --------- hypes : dict Hyperparameters """ # Get the sets of images and labels for training, validation, and # test on MNIST. modules = utils.load_modules_from_hypes(hypes) # set to allocate memory on GPU as needed # For more details, look at # https://stackoverflow.com/questions/36927607/how-can-i-solve-ran-out-of-gpu-memory-in-tensorflow config = tf.ConfigProto() config.gpu_options.allow_growth = True # Tell TensorFlow that the model will be built into the default Graph. # with tf.Session(config=config) as sess: with tf.Session() as sess: # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = build_training_graph(hypes, queue, modules) # prepaire the tv session tv_sess = core.start_tv_session(hypes) with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() num_channels = hypes['arch']['num_channels'] image_pl = tf.placeholder(tf.float32, [None, None, num_channels]) image = tf.expand_dims(image_pl, 0) if hypes['jitter']['resize_image']: height = hypes['jitter']['image_height'] width = hypes['jitter']['image_width'] # set the pre-defined image size here image.set_shape([1, height, width, num_channels]) inf_out = core.build_inference_graph(hypes, modules, image=image) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess) # And then after everything is built, start the training loop. run_training(hypes, modules, tv_graph, tv_sess) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def do_finetuning(hypes): """ Finetune model for a number of steps. This finetunes the model for at most hypes['solver']['max_steps']. It shows an update every utils.cfg.step_show steps and writes the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval steps. Paramters --------- hypes : dict Hyperparameters """ # Get the sets of images and labels for training, validation, and # test on MNIST. try: import tensorvision.core as core except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) modules = utils.load_modules_from_hypes(hypes) # set to allocate memory on GPU as needed config = tf.ConfigProto() config.gpu_options.allow_growth = True # Tell TensorFlow that the model will be built into the default Graph. with tf.Session(config=config) as sess: # with tf.Session() as sess: # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = build_training_graph(hypes, queue, modules) # restoring vars vars_to_restore = restoring_vars(hypes) restorer = tf.train.Saver(vars_to_restore) # load pre-trained model of hand segmentation logging.info("Loading pretrained model's weights") model_dir = hypes['transfer']['model_folder'] model_file = hypes['transfer']['model_name'] # DEBUG: check the model file # check_model(os.path.join(model_dir, model_file)) """ # Get a list of vars to restore vars_to_restore = restoring_vars(sess) print("vars to restore:", vars_to_restore) # Create another Saver for restoring pre-trained vars saver = tf.train.Saver(vars_to_restore) """ core.load_weights(model_dir, sess, restorer) # load_trained_model(sess, hypes) saver = tf.train.Saver(max_to_keep=int(utils.cfg.max_to_keep)) # prepaire the tv session tv_sess = prepare_tv_session(hypes, sess, saver) # DEBUG: print weights # check_weights(tv_sess['sess']) # check_graph(tv_sess['sess']) with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() num_channels = hypes['arch']['num_channels'] image_pl = tf.placeholder(tf.float32, [None, None, num_channels]) image = tf.expand_dims(image_pl, 0) if hypes['jitter']['resize_image']: height = hypes['jitter']['image_height'] width = hypes['jitter']['image_width'] # set the pre-defined image size here image.set_shape([1, height, width, num_channels]) inf_out = core.build_inference_graph(hypes, modules, image=image) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess) # And then after everything is built, start the training loop. run_training(hypes, modules, tv_graph, tv_sess) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_folder is None: logging.error("No input_image was given.") logging.info( "Usage: python segment.py [--input_folder /path/to/data/] " "[--output_folder /path/to/result/] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from FacadeSeg if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'FacadeSeg') else: runs_dir = 'RUNS' logdir = os.path.join(runs_dir, "FacadeSegPreTrained") else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") # classes classes_colors = [[0, 0, 255], [128, 255, 255], [255, 255, 0], [128, 0, 255], [255, 0, 0], [0, 0, 0], [255, 128, 0], [0, 255, 0]] input_folder = FLAGS.input_folder output_folder = FLAGS.output_folder # If the folder is not empty if (not os.listdir(input_folder) == ""): for root, dirs, files in os.walk(input_folder): for file in files: input_image = os.path.join(input_folder, file) logging.info( "Starting inference using {} as input".format(input_image)) # Load and resize input image image = scp.misc.imread(input_image) if (extension.lower() == ".tif" or extension.lower() == ".tiff"): logging.info( ">>>> TIFF file format. Applying convertion...") image = tiff.imread(input_image) if (image.dtype == 'uint16'): image = cv2.convertScaleAbs(image) image = image.astype(float) else: image = scp.misc.imread(input_image) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') # Run FacadeSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] logits = prediction['logits'] output, lll = sess.run([softmax, logits], feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output.reshape(shape[0], shape[1], -1) x = np.argmax(output_image, axis=2) im = np.zeros((shape[0], shape[1], 3), dtype=np.uint8) for i, _ in enumerate(x): for j, _ in enumerate(x[i]): value = x[i][j] color_code = classes_colors[value] im[i][j] = color_code # Save output images to disk. if (not os.path.isdir(output_folder)): logging.info( "Output directory does not exist. Creating in: " + output_folder) os.makedirs(output_folder) raw_image_name = file.split('.')[0] + '.png' full_raw_image_name = os.path.join(output_folder, raw_image_name) scp.misc.imsave(full_raw_image_name, im) logging.info(">>>> Labelled image " + raw_image_name + " saved!\n") else: logging.info("Input folder is empty. Check it again")
def main(_): config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.7 tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") num_classes = 5 # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session(config=config) saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) #Dealing with video segmentation with frame by frame #TODO: build a commandline loaded_video = skvideo.io.vread('t1.avi')[:100] writer = skvideo.io.FFmpegWriter("outputvideo.avi") for image in loaded_video: # Load and resize input image if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run(softmax, feed_dict=feed) print(len(output), type(output), output.shape) # Reshape output from flat vector to 2D Image output = np.transpose(output) shape = image.shape output = output.reshape(num_classes, shape[0], shape[1]) output_image = output[0].reshape(shape[0], shape[1]) # Plot confidences as red-blue overlay rb_image = seg.make_overlay(image, output_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.5 street_prediction = output_image > threshold street_predictions = output > threshold # Plot the hard prediction as green overlay green_image = tv_utils.fast_overlay(image, street_prediction) green_images = [] rb_images = [] output_images = [] for c in range(0,5): green_images.append(tv_utils.fast_overlay(image, street_predictions[c])) rb_images.append(seg.make_overlay(image, output[c])) output_images.append(output[c].reshape(shape[0], shape[1])) # Save output images to disk. if FLAGS.output_image is None: output_base_name = input_image else: output_base_name = FLAGS.output_image #Name and save the the red blue segmentation for first 5 frames as png to debug. green_image_names = [] rb_image_names = [] raw_image_names = [] for c in range(1,6): green_image_names.append(output_base_name.split('.')[0] + str(c) + '_green.png') rb_image_names.append(output_base_name.split('.')[0] + str(c) + '_rb.png') raw_image_names.append(output_base_name.split('.')[0] + str(c) + '_raw.png') for c in range(0,5): print(green_image_names[c], green_images[c].shape) scp.misc.imsave(raw_image_names[c], output_images[c]) scp.misc.imsave(rb_image_names[c], rb_images[c]) scp.misc.imsave(green_image_names[c], green_images[c]) #Output the green masked video as a file and show it to screen writer.writeFrame(green_images[4]) cv2.imshow('frame', green_images[4]) #user can press p to quit during processing. if cv2.waitKey(1) & 0xFF == ord('q'): break writer.close()
def main(): args = parser.parse_args() tv_utils.set_gpus_to_use() logdir = args.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32, shape=(hypes["image_height"], hypes["image_width"], 3)) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") if args.save: save_file = args.save else: save_file = 'video.mp4' if os.path.isfile(save_file): os.remove(save_file) # Get all images to make video image_names = sorted(os.listdir(args.image_dir))[:args.limit] if len(image_names) == 0: logging.error("No image found in given image_dir.") exit(1) start = time.time() if os.path.isfile(save_file): os.remove(save_file) logging.info("Making video") with imageio.get_writer(save_file, mode='I', fps=args.framerate) as writer: for i, image_name in enumerate(image_names): input_image = os.path.join(args.image_dir, image_name) # logging.info("Starting inference using %s as input %d/%d" % (input_image, i, len(image_names))) # Load and resize input image oimage = scp.misc.imread(input_image) oshape = oimage.shape[:2] rh = oshape[0] / float(hypes["image_height"]) rw = oshape[1] / float(hypes["image_width"]) image = scp.misc.imresize(oimage, (hypes["image_height"], hypes["image_width"]), interp='cubic') feed = {image_pl: image} # Run KittiBox model on image pred_boxes = prediction['pred_boxes_new'] pred_confidences = prediction['pred_confidences'] (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, pred_confidences], feed_dict=feed) # np_pred_boxes[:, :, 0] *= rw # np_pred_boxes[:, :, 2] *= rw # np_pred_boxes[:, :, 1] *= rh # np_pred_boxes[:, :, 3] *= rh # Apply non-maximal suppression # and draw predictions on the image threshold = 0.5 output_image, rectangles = kittibox_utils.add_rectangles( hypes, [image], np_pred_confidences, np_pred_boxes, show_removed=False, use_stitching=True, rnn_len=1, min_conf=threshold, tau=hypes['tau'], color_acc=(0, 255, 0)) output_image = scp.misc.imresize(output_image, oshape, interp='cubic') writer.append_data(output_image) time_taken = time.time() - start logging.info('Video saved as %s' % save_file) logging.info('Number of images: %d' % len(image_names)) logging.info('Time takes: %.2f s' % (time_taken)) logging.info('Frequency: %.2f fps' % (len(image_names) / time_taken))
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") #create tf graph and build net with tf.Graph().as_default(): image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") sess = tf.Session() saver = tf.train.Saver() core.load_weights(logdir, sess, saver) input_image = FLAGS.input_image #logging.info("start inference using {} as input".format(input_image)) cap = cv2.VideoCapture( 0) #live camera and change as video path if you have save video while True: #image_bgr = cv2.imread(input_image) #b, g, r = cv2.split(image_bgr) # get b,g,r #image = cv2.merge([r, g, b]) # switch it to rgb ret, image = cap.read() if hypes['jitter']['reseize_image']: image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), inerp='cubic') feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run([softmax], feed_dict=feed) #reshape shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) rb_image = seg.make_overlay(image, output_image) threshold = 0.5 street_prediction = output_image > threshold green_image = tv_utils.fast_overlay(image, street_prediction) cv2.imshow('ori', green_image) #live camera: imshow or video:save if cv2.waitKey(25) & 0xFF == ord( 'q'): #live camera imshow, delete if occur error in video show cv2.destroyAllWindows() break
def load_united_model(logdir): subhypes = {} subgraph = {} submodules = {} subqueues = {} subgraph['debug_ops'] = {} first_iter = True meta_hypes = utils.load_hypes_from_logdir(logdir, subdir="", base_path='hypes') for model in meta_hypes['model_list']: subhypes[model] = utils.load_hypes_from_logdir(logdir, subdir=model) hypes = subhypes[model] hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir'] hypes['dirs']['image_dir'] = meta_hypes['dirs']['image_dir'] hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir'] submodules[model] = utils.load_modules_from_logdir(logdir, dirname=model, postfix=model) modules = submodules[model] logging.info("Build %s computation Graph.", model) with tf.name_scope("Queues_%s" % model): subqueues[model] = modules['input'].create_queues(hypes, 'train') logging.info('Building Model: %s' % model) subgraph[model] = build_training_graph(hypes, subqueues[model], modules, first_iter) first_iter = False if len(meta_hypes['model_list']) == 2: _recombine_2_losses(meta_hypes, subgraph, subhypes, submodules) else: _recombine_3_losses(meta_hypes, subgraph, subhypes, submodules) hypes = subhypes[meta_hypes['model_list'][0]] tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] saver = tv_sess['saver'] cur_step = core.load_weights(logdir, sess, saver) for model in meta_hypes['model_list']: hypes = subhypes[model] modules = submodules[model] optimizer = modules['solver'] with tf.name_scope('Validation_%s' % model): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) inf_out = core.build_inference_graph(hypes, modules, image=image) subgraph[model]['image_pl'] = image_pl subgraph[model]['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, subqueues[model], 'train', sess) target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json') with open(target_file, 'w') as outfile: json.dump(meta_hypes, outfile, indent=2, sort_keys=True) return meta_hypes, subhypes, submodules, subgraph, tv_sess, cur_step
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiBox') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) # Load and resize input image image = scp.misc.imread(input_image) # image = scp.misc.imresize(image, (hypes["image_height"], hypes["image_width"]),interp='cubic') feed = {image_pl: image} # Run KittiBox model on image pred_boxes = prediction['pred_boxes_new'] pred_confidences = prediction['pred_confidences'] print('asdffffffffffffffffffffffffffffffffffff') print(pred_boxes,pred_confidences,hypes["image_height"],hypes["image_width"]) start = timer() for i in range(11): if i==1: start = timer() (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes,pred_confidences],feed_dict=feed) end = timer() print((end - start)/10 ) # Apply non-maximal suppression # and draw predictions on the image output_image, rectangles = kittibox_utils.add_rectangles( hypes, [image], np_pred_confidences, np_pred_boxes, show_removed=False, use_stitching=True, rnn_len=1, min_conf=0.50, tau=hypes['tau'], color_acc=(0, 255, 0)) threshold = 0.5 accepted_predictions = [] # removing predictions <= threshold for rect in rectangles: if rect.score >= threshold: accepted_predictions.append(rect) print('') logging.info("{} Cars detected".format(len(accepted_predictions))) # Printing coordinates of predicted rects. for i, rect in enumerate(accepted_predictions): logging.info("") logging.info("Coordinates of Box {}".format(i)) logging.info(" x1: {}".format(rect.x1)) logging.info(" x2: {}".format(rect.x2)) logging.info(" y1: {}".format(rect.y1)) logging.info(" y2: {}".format(rect.y2)) logging.info(" Confidence: {}".format(rect.score)) # save Image if FLAGS.output_image is None: output_name = input_image.split('.')[0] + '_rects.png' else: output_name = FLAGS.output_image scp.misc.imsave(output_name, output_image) logging.info("") logging.info("Output image saved to {}".format(output_name)) logging.info("") logging.warning("Do NOT use this Code to evaluate multiple images.") logging.warning("Demo.py is **very slow** and designed " "to be a tutorial to show how the KittiBox works.") logging.warning("") logging.warning("Please see this comment, if you like to apply demo.py to" "multiple images see:") logging.warning("https://github.com/MarvinTeichmann/KittiBox/" "issues/15#issuecomment-301800058") np.load = np_load_old print((end - start)/10 )
def build_united_model(meta_hypes): logging.info("Initialize training folder") subhypes = {} subgraph = {} submodules = {} subqueues = {} subgraph['debug_ops'] = {} base_path = meta_hypes['dirs']['base_path'] first_iter = True for model in meta_hypes['model_list']: subhypes_file = os.path.join(base_path, meta_hypes['models'][model]) with open(subhypes_file, 'r') as f: logging.info("f: %s", f) subhypes[model] = json.load(f) hypes = subhypes[model] utils.set_dirs(hypes, subhypes_file) hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir'] hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir'] train.initialize_training_folder(hypes, files_dir=model, logging=first_iter) meta_hypes['dirs']['image_dir'] = hypes['dirs']['image_dir'] submodules[model] = utils.load_modules_from_hypes(hypes, postfix="_%s" % model) modules = submodules[model] logging.info("Build %s computation Graph.", model) with tf.name_scope("Queues_%s" % model): subqueues[model] = modules['input'].create_queues(hypes, 'train') logging.info('Building Model: %s' % model) subgraph[model] = build_training_graph(hypes, subqueues[model], modules, first_iter) first_iter = False if len(meta_hypes['models']) == 2: _recombine_2_losses(meta_hypes, subgraph, subhypes, submodules) else: _recombine_3_losses(meta_hypes, subgraph, subhypes, submodules) hypes = subhypes[meta_hypes['model_list'][0]] tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] for model in meta_hypes['model_list']: hypes = subhypes[model] modules = submodules[model] optimizer = modules['solver'] with tf.name_scope('Validation_%s' % model): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) inf_out = core.build_inference_graph(hypes, modules, image=image) subgraph[model]['image_pl'] = image_pl subgraph[model]['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, subqueues[model], 'train', sess) target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json') with open(target_file, 'w') as outfile: json.dump(meta_hypes, outfile, indent=2, sort_keys=True) return subhypes, submodules, subgraph, tv_sess
def main(_): tv_utils.set_gpus_to_use() # if FLAGS.input_image is None: # logging.error("No input_image was given.") # logging.info( # "Usage: python demo.py --input_image data/test.png " # "[--output_image output_image] [--logdir /path/to/weights] " # "[--gpus GPUs_to_use] ") # exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") dataset = kitti_object( os.path.join(ROOT_DIR, 'free-space/dataset/KITTI/object')) point_pub = rospy.Publisher('cloud', PointCloud2, queue_size=50) # picture_pub = rospy.Publisher("kitti_image",newImage,queue_size=50) rospy.init_node('point-cloud', anonymous=True) # h = std_msgs.msg.Header() # h.frame_id="base_link" # h.stamp=rospy.Time.now() #rate = rospy.Rate(10) #point_msg=PointCloud2() video_dir = '/home/user/Data/lrx_work/free-space/kitti.avi' fps = 10 num = 4541 img_size = (1241, 376) fourcc = 'mp4v' videoWriter = cv2.VideoWriter(video_dir, cv2.VideoWriter_fourcc(*fourcc), fps, img_size) calib = dataset.get_calibration(0) for data_idx in range(len(dataset)): #objects = dataset.get_label_objects(data_idx) # Load and resize input image image = dataset.get_image(data_idx) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) #scp.misc.imsave('new.png', image) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') img_height, img_width, img_channel = image.shape print("picture-shape") print(len(image)) print(len(image[0])) pc_velo = dataset.get_lidar(data_idx)[:, 0:3] print(len(pc_velo)) velo_len = len(pc_velo) #calib = dataset.get_calibration(data_idx) # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run([softmax], feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) # Plot confidences as red-blue overlay rb_image = seg.make_overlay(image, output_image) # scp.misc.imsave('new0.png', rb_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.5 street_prediction = output_image > threshold index = np.where(street_prediction == True) chang = len(index[0]) print(chang) # test = np.zeros((velo_len,2),dtype=np.int) # for tmp0 in range(chang): # test[tmp0][0]=index[0][tmp0] # test[tmp0][1]=index[1][tmp0] print("suoyindayin") # if (chang>0): # print(test[0][0]) # print(test[0][1]) pts_2d = calib.project_velo_to_image(pc_velo) print(pts_2d.shape) # print(pts_2d[1][0]) # print(pts_2d[1][1]) fig = mlab.figure(figure=None, bgcolor=(0, 0, 0), fgcolor=None, engine=None, size=(1000, 500)) fov_inds = (pts_2d[:,0]<1242) & (pts_2d[:,0]>=0) & \ (pts_2d[:,1]<370) & (pts_2d[:,1]>=0) print(fov_inds.shape) # print(fov_inds[1000]) # print(pc_velo.shape) print("okok") fov_inds = fov_inds & (pc_velo[:, 0] > 0) print(fov_inds.shape) imgfov_pts_2d = pts_2d[fov_inds, :] imgfov_pc_velo = pc_velo[fov_inds, :] pts_2d0 = calib.project_velo_to_image(imgfov_pc_velo) fov_inds0 = (pts_2d0[:,0]<len(image[0])) & (pts_2d0[:,0]>=0) & \ (pts_2d0[:,1]<len(image)) & (pts_2d0[:,1]>=0) fov_inds0 = fov_inds0 & (imgfov_pc_velo[:, 0] > 2.0) print(fov_inds0.shape) print(street_prediction.shape) print(pts_2d0.shape) # if(chang>0): # print(int(imgfov_pts_2d[5,0])) # print(int(imgfov_pts_2d[5,1])) # print(street_prediction[int(imgfov_pts_2d[5,1]),int(imgfov_pts_2d[5,0])]) if (chang > 0): for i in range(len(fov_inds0)): if ((pts_2d0[i, 1] < len(street_prediction)) & (pts_2d0[i, 0] < len(street_prediction[0]))): fov_inds0[i] = fov_inds0[i] & ( street_prediction[int(pts_2d0[i, 1]), int(pts_2d0[i, 0])] == True) imgfov_pc_velo0 = imgfov_pc_velo[fov_inds0, :] print("number") green_image = tv_utils.fast_overlay(image, street_prediction) # pub point-cloud topic print(imgfov_pc_velo0.shape) number = len(imgfov_pc_velo0) header = std_msgs.msg.Header() header.stamp = rospy.Time.now() header.frame_id = "velodyne" points = pc2.create_cloud_xyz32(header, imgfov_pc_velo0) # point=Point() # for t in range(0,number): # point_x=imgfov_pc_velo0[t][0] # point_y=imgfov_pc_velo0[t][1] # point_z=imgfov_pc_velo0[t][2] # point_msg.points[t].point.x=point_x # point_msg.points[t].point.y=point_y # point_msg.points[t].point.z=point_z # point_pub.publish(points) # videoWriter.write(green_image) # bridge=CvBridge() # picture_pub.publish(bridge.cv2_to_imgmsg(green_image,"rgb8")) # minx=imgfov_pc_velo0[0][0] # miny=imgfov_pc_velo0[0][1] # minz=imgfov_pc_velo0[0][2] # maxx=imgfov_pc_velo0[0][0] # maxy=imgfov_pc_velo0[0][1] # maxz=imgfov_pc_velo0[0][2] # for t in range(len(imgfov_pc_velo0)): # minx=min(minx,imgfov_pc_velo0[t][0]) # miny=min(miny,imgfov_pc_velo0[t][1]) # minz=min(minz,imgfov_pc_velo0[t][2]) # maxx=max(maxx,imgfov_pc_velo0[t][0]) # maxy=max(maxy,imgfov_pc_velo0[t][1]) # maxz=max(maxz,imgfov_pc_velo0[t][2]) # print(minx,miny,minz,maxx,maxy,maxz) # width=1024 # height=1024 # img_res=np.zeros([width,height,3],dtype=np.uint8) # for p in range(len(imgfov_pc_velo0)): # velo_x=5*(int(imgfov_pc_velo0[p][0])+50) # velo_y=5*(int(imgfov_pc_velo0[p][1])+50) # img_res[velo_x][velo_y]=255 # scale=25 # if((velo_x>scale)&(velo_x+scale<1024)&(velo_y>scale)&(velo_y+scale<1024)): # for q in range(scale): # for m in range(scale): # img_res[velo_x-q][velo_y-m]=255 # img_res[velo_x-q][velo_y+m]=255 # img_res[velo_x+q][velo_y-m]=255 # img_res[velo_x+q][velo_y+m]=255 # scp.misc.imsave('res.png',img_res) draw_lidar(imgfov_pc_velo0, fig=fig) # for obj in objects: # if obj.type == 'DontCare': continue # # Draw 3d bounding box # box3d_pts_2d, box3d_pts_3d = utils.compute_box_3d(obj, calib.P) # box3d_pts_3d_velo = calib.project_rect_to_velo(box3d_pts_3d) # # Draw heading arrow # ori3d_pts_2d, ori3d_pts_3d = utils.compute_orientation_3d(obj, calib.P) # ori3d_pts_3d_velo = calib.project_rect_to_velo(ori3d_pts_3d) # x1, y1, z1 = ori3d_pts_3d_velo[0, :] # x2, y2, z2 = ori3d_pts_3d_velo[1, :] # draw_gt_boxes3d([box3d_pts_3d_velo], fig=fig) # mlab.plot3d([x1, x2], [y1, y2], [z1, z2], color=(0.5, 0.5, 0.5), # tube_radius=None, line_width=1, figure=fig) #mlab.show(1) # Plot the hard prediction as green overlay # Save output images to disk. if FLAGS.output_image is None: output_base_name = image else: output_base_name = FLAGS.output_image # raw_image_name = output_base_name.split('.')[0] + '_raw.png' # rb_image_name = output_base_name.split('.')[0] + '_rb.png' # green_image_name = output_base_name.split('.')[0] + '_green.png' # scp.misc.imsave('1.png', output_image) # scp.misc.imsave('2.png', rb_image) # scp.misc.imsave('3.png', green_image) raw_input()
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiBox') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for inputclassLabel image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # yang. image.set_shape([1, None, None, 3]) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) # yang. for img in os.listdir(input_image): # print(input_image) # filelist = ['100k/train/a9c4268f-b97f4325.jpg','100k/train/1f416cf1-10a9e2ed.jpg','100k/train/4d22354a-6ce691da.jpg','100k/train/2786c637-b689c08a.jpg','100k/train/1c56c68b-b520ae6d.jpg','100k/train/7170cba6-dd72fe05.jpg','100k/train/71cf580c-d819c189.jpg','100k/train/70b5dbac-87ba4de7.jpg','100k/train/219da1a7-030143c1.jpg','100k/train/2f45c868-4399445f.jpg'] # for img in filelist: #if img.split('.')[-1] == 'png': #if img.split('.')[-1] == 'png': # Load and resize input image # image = scp.misc.imread('/home/yang/home/yang/MultiNet/submodules/KittiBox/DATA/KittiBox/' + img) image = scp.misc.imread(input_image + img) # if hypes["resize_image"]: # image = scp.misc.imresize(image, (hypes["image_height"],hypes["image_width"]),interp='cubic') image = image[8:,:,:] # print(image.shape) # break # image = scp.misc.imread(input_image) feed = {image_pl: image} # Run KittiBox model on image pred_boxes = prediction['pred_boxes'] pred_confidences = prediction['pred_confidences'] # yang. pred_logits = prediction['pred_logits'] # print(pred_boxes,pred_confidences,pred_logits) # yang. #print('----------------------demo.time------------------------') t0 = time.time() # (np_pred_boxes, np_pred_confidences) = sess.run([pred_boxes, # pred_confidences], # feed_dict=feed) (np_pred_boxes, np_pred_confidences, np_pred_logits) = sess.run([pred_boxes, pred_confidences, pred_logits], feed_dict=feed) # print(np_pred_boxes,np_pred_confidences,np_pred_logits) print(time.time() - t0) # # print('--------------------------logits-----------------------') # print('-------------------------------------------------------') # print(np_pred_confidences) # # print(np_pred_logits) # print('-------------------------------------------------------') # break #print('-------------------------------------------------------') # Apply non-maximal suppression # and draw predictions on the image # yang. # print(pred_logits.shape) # break # output_image, rectangles = kittibox_utils.add_rectangles( # hypes, [image], np_pred_confidences, # np_pred_boxes, np_pred_logits, show_removed=False, # use_stitching=True, rnn_len=1, # min_conf=0.50, tau=hypes['tau'], color_acc=(0, 255, 0)) output_image, rectangles = kittibox_utils.add_rectangles( hypes, [image], np_pred_confidences, np_pred_boxes, show_removed=False, use_stitching=True, rnn_len=1, min_conf=0.75, tau=hypes['tau'], color_acc=(0, 255, 0)) threshold = 0.75 accepted_predictions = [] # removing predictions <= threshold for rect in rectangles: if rect.score >= threshold: accepted_predictions.append(rect) print('') logging.info("{} Cars detected".format(len(accepted_predictions))) # Printing coordinates of predicted rects. for i, rect in enumerate(accepted_predictions): logging.info("") logging.info("Coordinates of Box {}".format(i)) logging.info(" class: {}".format(rect.classID)) logging.info(" x1: {}".format(rect.x1)) logging.info(" x2: {}".format(rect.x2)) logging.info(" y1: {}".format(rect.y1)) logging.info(" y2: {}".format(rect.y2)) logging.info(" Confidence: {}".format(rect.score)) savedata(rect,img,FLAGS.output_image) # save Image output_name = os.path.join(FLAGS.output_image, img.split('.')[0] + '_rects.jpg') #output_image = scp.misc.imresize(output_image, (512,640),interp='cubic') # scp.misc.imsave(output_name, output_image) logging.info("") logging.info("Output image saved to {}".format(output_name))
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) # Load and resize input image image = scp.misc.imread(input_image) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') # classes classes_colors = [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run(softmax, feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output.reshape(shape[0], shape[1], -1) x = np.argmax(output_image, axis=2) im = np.zeros((shape[0], shape[1], 3), dtype=np.uint8) for i, _ in enumerate(x): for j, _ in enumerate(x[i]): value = x[i][j] color_code = classes_colors[value] im[i][j] = color_code # Save output images to disk. if FLAGS.output_image is None: output_base_name = input_image else: output_base_name = FLAGS.output_image raw_image_name = output_base_name.split('.')[0] + '_raw.png' scp.misc.imsave(raw_image_name, im)
def main(_): tv_utils.set_gpus_to_use() if FLAGS.input_image is None: logging.error("No input_image was given.") logging.info( "Usage: python demo.py --input_image data/test.png " "[--output_image output_image] [--logdir /path/to/weights] " "[--gpus GPUs_to_use] ") exit(1) if FLAGS.logdir is None: # Download and use weights from the MultiNet Paper if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' maybe_download_and_extract(runs_dir) logdir = os.path.join(runs_dir, default_run) else: logging.info("Using weights found in {}".format(FLAGS.logdir)) logdir = FLAGS.logdir # Loading hyperparameters from logdir hypes = tv_utils.load_hypes_from_logdir(logdir, base_path='hypes') logging.info("Hypes loaded successfully.") # Loading tv modules (encoder.py, decoder.py, eval.py) from logdir modules = tv_utils.load_modules_from_logdir(logdir) logging.info("Modules loaded successfully. Starting to build tf graph.") # Create tf graph and build module. with tf.Graph().as_default(): # Create placeholder for input image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) # build Tensorflow graph using the model from logdir prediction = core.build_inference_graph(hypes, modules, image=image) logging.info("Graph build successfully.") # Create a session for running Ops on the Graph. sess = tf.Session() saver = tf.train.Saver() # Load weights from logdir core.load_weights(logdir, sess, saver) logging.info("Weights loaded successfully.") input_image = FLAGS.input_image logging.info("Starting inference using {} as input".format(input_image)) # Load and resize input image image = scp.misc.imread(input_image) if hypes['jitter']['reseize_image']: # Resize input only, if specified in hypes image_height = hypes['jitter']['image_height'] image_width = hypes['jitter']['image_width'] image = scp.misc.imresize(image, size=(image_height, image_width), interp='cubic') # Run KittiSeg model on image feed = {image_pl: image} softmax = prediction['softmax'] output = sess.run([softmax], feed_dict=feed) # Reshape output from flat vector to 2D Image shape = image.shape output_image = output[0][:, 1].reshape(shape[0], shape[1]) # Plot confidences as red-blue overlay rb_image = seg.make_overlay(image, output_image) # Accept all pixel with conf >= 0.5 as positive prediction # This creates a `hard` prediction result for class street threshold = 0.5 street_prediction = output_image > threshold # Plot the hard prediction as green overlay green_image = tv_utils.fast_overlay(image, street_prediction) # Save output images to disk. if FLAGS.output_image is None: output_base_name = input_image else: output_base_name = FLAGS.output_image raw_image_name = output_base_name.split('.')[0] + '_raw.png' rb_image_name = output_base_name.split('.')[0] + '_rb.png' green_image_name = output_base_name.split('.')[0] + '_green.png' scp.misc.imsave(raw_image_name, output_image) scp.misc.imsave(rb_image_name, rb_image) scp.misc.imsave(green_image_name, green_image) logging.info("") logging.info("Raw output image has been saved to: {}".format( os.path.realpath(raw_image_name))) logging.info("Red-Blue overlay of confs have been saved to: {}".format( os.path.realpath(rb_image_name))) logging.info("Green plot of predictions have been saved to: {}".format( os.path.realpath(green_image_name))) logging.info("") logging.warning("Do NOT use this Code to evaluate multiple images.") logging.warning("Demo.py is **very slow** and designed " "to be a tutorial to show how the KittiSeg works.") logging.warning("") logging.warning("Please see this comment, if you like to apply demo.py to" "multiple images see:") logging.warning("https://github.com/MarvinTeichmann/KittiBox/" "issues/15#issuecomment-301800058")
def load_united_model(logdir): subhypes = {} subgraph = {} submodules = {} subqueues = {} subgraph['debug_ops'] = {} first_iter = True #load the hypes meta_hypes = utils.load_hypes_from_logdir(logdir, subdir="", base_path='hypes') #get the models from model_list in hypes for model in meta_hypes['model_list']: subhypes[model] = utils.load_hypes_from_logdir(logdir, subdir=model) hypes = subhypes[model] #get the output directory hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir'] #image input directory hypes['dirs']['image_dir'] = meta_hypes['dirs']['image_dir'] #training data directory hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir'] submodules[model] = utils.load_modules_from_logdir(logdir, dirname=model, postfix=model) modules = submodules[model] logging.info("Build %s computation Graph.", model) #build the computational graph with tf.name_scope("Queues_%s" % model): subqueues[model] = modules['input'].create_queues(hypes, 'train') logging.info('Building Model: %s' % model) #build the model subgraph[model] = build_training_graph(hypes, subqueues[model], modules, first_iter) first_iter = False #if model list is having detection and segmentation if len(meta_hypes['model_list']) == 2: #call the function to calculate the losses _recombine_2_losses(meta_hypes, subgraph, subhypes, submodules) else: _recombine_3_losses(meta_hypes, subgraph, subhypes, submodules) hypes = subhypes[meta_hypes['model_list'][0]] #using the context manager launch the graph in session tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] saver = tv_sess['saver'] #load weights cur_step = core.load_weights(logdir, sess, saver) #for each of the models in model list expand the image dimension for model in meta_hypes['model_list']: hypes = subhypes[model] modules = submodules[model] optimizer = modules['solver'] with tf.name_scope('Validation_%s' % model): tf.get_variable_scope().reuse_variables() #returns the tensor that may be used as handle for feeding a value image_pl = tf.placeholder(tf.float32) #expand the shape of the array by inserting new axes in 0th positon image = tf.expand_dims(image_pl, 0) inf_out = core.build_inference_graph(hypes, modules, image=image) subgraph[model]['image_pl'] = image_pl subgraph[model]['inf_out'] = inf_out # Start the data load modules['input'].start_enqueuing_threads(hypes, subqueues[model], 'train', sess) target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json') with open(target_file, 'w') as outfile: json.dump(meta_hypes, outfile, indent=2, sort_keys=True) return meta_hypes, subhypes, submodules, subgraph, tv_sess, cur_step