def main(_): utils.set_gpus_to_use() if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) logging.info("Start finetuning") do_finetuning(hypes)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() runs_dir = 'RUNS' utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) logging.info("Evaluating on Validation data.") logdir = os.path.join(runs_dir, FLAGS.RUN) # logging.info("Output images will be saved to {}".format) ana.do_analyze(logdir) logging.info("Creating output on test data.") kitti_test.do_inference(logdir) logging.info("Analysis for pretrained model complete.") logging.info("For evaluating your own models I recommend using:" "`tv-analyze --logdir /path/to/run`.") logging.info("tv-analysis has a much cleaner interface.")
def main(_): utils.set_gpus_to_use() try: import tensorvision.train except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) #utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiBox') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) #train.maybe_download_and_extract(hypes) logging.info("Start training") train.do_training(hypes)
def main(): with open('../config/fcn8_seg.json', 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.set_dirs(hypes, '../config/fcn8_seg.json') utils._add_paths_to_sys(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) logging.info("Start training") print('start') train.do_training(hypes) print('end')
def main(_): utils.set_gpus_to_use() load_weights = tf.app.flags.FLAGS.logdir is not None if not load_weights: with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'MultiNet') # with tf.Session() as sess: # king with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess: if not load_weights: utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) # Build united Model subhypes, submodules, subgraph, tv_sess = build_united_model(hypes) start_step = 0 else: logdir = tf.app.flags.FLAGS.logdir logging_file = os.path.join(logdir, "output.log") utils.create_filewrite_handler(logging_file, mode='a') hypes, subhypes, submodules, subgraph, tv_sess, start_step = \ load_united_model(logdir) if start_step is None: start_step = 0 # Run united training run_united_training(hypes, subhypes, submodules, subgraph, tv_sess, start_step=start_step) # stopping input Threads tv_sess['coord'].request_stop() tv_sess['coord'].join(tv_sess['threads'])
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) hypes['dist'] = FLAGS.dist if FLAGS.layers: hypes['arch']['layers'] = FLAGS.layers if FLAGS.lr: hypes['solver']['learning_rate'] = FLAGS.lr if FLAGS.optimizer: hypes['solver']['opt'] = FLAGS.optimizer utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) train.do_training(hypes)
def main(_): utils.set_gpus_to_use() sys.path.append("submodules/tensorflow-fcn") sys.path.append("submodules/TensorVision") import tensorvision.train import tensorflow_fcn.utils # try: # import tensorvision.train # import tensorflow_fcn.utils # except ImportError: # logging.error("Could not import the submodules.") # logging.error("Please execute:" # "'git submodule update --init --recursive'") # exit(1) if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) logging.info("Start training") train.do_training(hypes)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) logging.info("Start training") encoder_path = hypes['model']['architecture_file'] hypes['model']['architecture_file'] = '../encoder/stub.py' hypes['ga_data'] = 'ga_data.json' run_genetic_algorithm(hypes, encoder_path)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) maybe_download_and_extract(runs_dir) logging.info("Evaluating on Validation data.") logdir = os.path.join(runs_dir, FLAGS.RUN) # logging.info("Output images will be saved to {}".format) ana.do_analyze(logdir) logging.info("Creating output on test data.") kitti_test.do_inference(logdir) logging.info("Analysis for pretrained model complete.") logging.info("For evaluating your own models I recommend using:" "`tv-analyze --logdir /path/to/run`.") logging.info("tv-analysis has a much cleaner interface.")
def main(_): utils.set_gpus_to_use() with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'MediSeg') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) train.maybe_download_and_extract(hypes) logging.info("Start training") train.do_training(hypes)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) os.environ["TV_DIR_DATA"] = "../../SemSeg_DATA/DATA" os.environ["TV_DIR_RUNS"] = "../../SemSeg_DATA/RUNS" # print(os.environ["TV_DIR_DATA"]) utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) logging.info("Start training") train.do_training(hypes)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiBox') else: runs_dir = 'RUNS' utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) maybe_download_and_extract(runs_dir) logging.info("Evaluating on Validation data.") logdir = os.path.join(runs_dir, FLAGS.RUN) # logging.info("Output images will be saved to {}".format) ana.do_analyze(logdir, base_path='hypes') logging.info("Analysis for pretrained model complete.") logging.info("For evaluating your own models I recommend using:" "`tv-analyze --logdir /path/to/run`.") logging.info("") logging.info( "Output images can be found in {}/analyse/images.".format(logdir))
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) if 'TV_DIR_RUNS' in os.environ: os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) logging.info("Start training") train.do_training(hypes)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) if tf.app.flags.FLAGS.hypes is None: logging.error("No hype file is given.") logging.info("Usage: python train.py --hypes hypes/KittiClass.json") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = commentjson.load(f) utils.load_plugins() if tf.app.flags.FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod) dict_merge(hypes, mod_dict) if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' logdir = os.path.join(runs_dir, FLAGS.RUN) utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.continue_training(logdir)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) hypes_path = FLAGS.logdir hypes_path = os.path.join(hypes_path, "model_files/hypes.json") with open(hypes_path, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'FacadeSeg') else: runs_dir = 'RUNS' utils.set_dirs(hypes, FLAGS.hypes) utils._add_paths_to_sys(hypes) logging.info("Evaluating on Validation data.") ana.do_analyze(FLAGS.logdir) logging.info("Segmenting and test data. Creating output.") ana.do_inference(FLAGS.logdir) logging.info("Analysis for pretrained model complete.")
def main(_): logging.info( "Initializing GPUs, plugins and creating the essential folders") utils.set_gpus_to_use() if FLAGS.hypes is None: logging.error("No hypes are given.") logging.error("Usage: python train.py --hypes hypes.json") logging.error(" tf: tv-train --hypes hypes.json") exit(1) with open(FLAGS.hypes) as f: logging.info("f: %s", f) hypes = commentjson.load(f) if FLAGS.mod is not None: import ast mod_dict = ast.literal_eval(FLAGS.mod) dict_merge(hypes, mod_dict) logging.info("Loading plugins") utils.load_plugins() logging.info("Set dirs") utils.set_dirs(hypes, FLAGS.hypes) logging.info("Add paths to sys") utils._add_paths_to_sys(hypes) logging.info("Initialize training folder") train.initialize_training_folder(hypes) tf.reset_default_graph() logging.info("Start training") train.do_training(hypes)
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'VOCSeg') else: runs_dir = 'RUNS' utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) maybe_download_and_extract(runs_dir, "VOCSeg_2017_04_14_00.49") logging.info("Evaluating on Validation data.") logdir = os.path.join(runs_dir, "VOCSeg_2017_04_14_00.49") # logging.info("Output images will be saved to {}".format) # ana.do_analyze(logdir) logging.info("Creating output on test data.") voc_test.do_inference(logdir) logging.info("Analysis for pretrained model complete.")
def main(_): utils.set_gpus_to_use() try: import tensorvision.train import tensorflow_fcn.utils except ImportError: logging.error("Could not import the submodules.") logging.error("Please execute:" "'git submodule update --init --recursive'") exit(1) with open(tf.app.flags.FLAGS.hypes, 'r') as f: logging.info("f: %s", f) hypes = json.load(f) utils.load_plugins() if 'TV_DIR_RUNS' in os.environ: runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg') else: runs_dir = 'RUNS' utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes) utils._add_paths_to_sys(hypes) train.maybe_download_and_extract(hypes) maybe_download_and_extract(runs_dir) logging.info("Trimming weights.") logdir = os.path.join(runs_dir, FLAGS.RUN) modules = utils.load_modules_from_hypes(hypes) with tf.Graph().as_default(): # build the graph based on the loaded modules with tf.name_scope("Queues"): queue = modules['input'].create_queues(hypes, 'train') tv_graph = core.build_training_graph(hypes, queue, modules) # prepare the tv session with tf.Session().as_default(): tv_sess = core.start_tv_session(hypes) sess = tv_sess['sess'] saver = tv_sess['saver'] cur_step = core.load_weights(logdir, sess, saver) if cur_step is None: logging.warning("Loaded global_step is None.") logging.warning("This could mean," " that no weights have been loaded.") logging.warning("Starting Training with step 0.") cur_step = 0 with tf.name_scope('Validation'): tf.get_variable_scope().reuse_variables() image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) tv_graph['image_pl'] = image_pl tv_graph['inf_out'] = inf_out # prepaire the tv session image_pl = tf.placeholder(tf.float32) image = tf.expand_dims(image_pl, 0) image.set_shape([1, None, None, 3]) inf_out = core.build_inference_graph(hypes, modules, image=image) # Create a session for running Ops on the Graph. trim_dir = 'RUNS/trimmed' shutil.copytree(logdir, trim_dir) shutil.copy(tf.app.flags.FLAGS.hypes, os.path.join(trim_dir, 'model_files', 'hypes.json')) sess = tf.Session() saver = tf.train.Saver() core.load_weights(trim_dir, sess, saver) for weight in tf.contrib.model_pruning.get_masks(): if any([ layer in weight.name for layer in hypes['layer_pruning']['layers'] ]): weight_value = tv_sess['sess'].run(weight) kernel_count = int(weight_value.shape[3] * hypes['layer_pruning']['layer_sparsity']) l1_values = np.sum(np.abs(weight_value), axis=(0, 1, 2)) toss_kernels = l1_values.argsort()[:kernel_count] weight_value[:, :, :, toss_kernels] = 0 assign_op = tf.assign(weight, tf.constant(weight_value)) tv_sess['sess'].run(assign_op) checkpoint_path = os.path.join(trim_dir, 'model.ckpt') tv_sess['saver'].save(sess, checkpoint_path, global_step=cur_step) train.continue_training(trim_dir)