def main(_):
"""Run main function."""
if FLAGS.hypes is None:
logging.error("No hypes are given.")
logging.error("Usage: tv-train --hypes hypes.json")
exit(1)
if FLAGS.gpus is None:
if 'TV_USE_GPUS' in os.environ:
if os.environ['TV_USE_GPUS'] == 'force':
logging.error('Please specify a GPU.')
logging.error('Usage tv-train --gpus <ids>')
exit(1)
else:
gpus = os.environ['TV_USE_GPUS']
logging.info("GPUs are set to: %s", gpus)
os.environ['CUDA_VISIBLE_DEVICES'] = gpus
else:
logging.info("GPUs are set to: %s", FLAGS.gpus)
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpus
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
logging.info("Initialize training folder")
initialize_training_folder(hypes)
maybe_download_and_extract(hypes)
logging.info("Start training")
do_training(hypes)
def main(_):
utils.set_gpus_to_use()
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
logging.info("Start finetuning")
do_finetuning(hypes)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
runs_dir = 'RUNS'
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
logging.info("Evaluating on Validation data.")
logdir = os.path.join(runs_dir, FLAGS.RUN)
# logging.info("Output images will be saved to {}".format)
ana.do_analyze(logdir)
logging.info("Creating output on test data.")
kitti_test.do_inference(logdir)
logging.info("Analysis for pretrained model complete.")
logging.info("For evaluating your own models I recommend using:"
"`tv-analyze --logdir /path/to/run`.")
logging.info("tv-analysis has a much cleaner interface.")
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
#utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiBox')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
#train.maybe_download_and_extract(hypes)
logging.info("Start training")
train.do_training(hypes)
def test_set_dirs():
"""Test if setting plugins works."""
hype_file = "examples/cifar10_minimal.json"
with open(hype_file, 'r') as f:
import json
hypes = json.load(f)
from tensorvision.utils import set_dirs
set_dirs(hypes, hype_file)
def main():
with open('../config/fcn8_seg.json', 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.set_dirs(hypes, '../config/fcn8_seg.json')
utils._add_paths_to_sys(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
logging.info("Start training")
print('start')
train.do_training(hypes)
print('end')
def test_set_dirs():
"""Test if setting plugins works."""
hype_file = "examples/cifar10_minimal.json"
with open(hype_file, 'r') as f:
import json
hypes = json.load(f)
from tensorvision.utils import set_dirs
import tensorflow as tf
flags = tf.app.flags
flags.DEFINE_string('name', 'debug', 'Append a name Tag to run.')
tf.app.flags.DEFINE_boolean('save', True, (''))
set_dirs(hypes, hype_file)
def test_set_dirs():
"""Test if setting plugins works."""
hype_file = "examples/cifar10_minimal.json"
with open(hype_file, 'r') as f:
import json
hypes = json.load(f)
from tensorvision.utils import set_dirs
import tensorflow as tf
flags = tf.app.flags
flags.DEFINE_string('name', 'debug',
'Append a name Tag to run.')
tf.app.flags.DEFINE_boolean(
'save', True, (''))
set_dirs(hypes, hype_file)
def main(_):
utils.set_gpus_to_use()
load_weights = tf.app.flags.FLAGS.logdir is not None
if not load_weights:
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'MultiNet')
# with tf.Session() as sess:
# king
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)) as sess:
if not load_weights:
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
# Build united Model
subhypes, submodules, subgraph, tv_sess = build_united_model(hypes)
start_step = 0
else:
logdir = tf.app.flags.FLAGS.logdir
logging_file = os.path.join(logdir, "output.log")
utils.create_filewrite_handler(logging_file, mode='a')
hypes, subhypes, submodules, subgraph, tv_sess, start_step = \
load_united_model(logdir)
if start_step is None:
start_step = 0
# Run united training
run_united_training(hypes,
subhypes,
submodules,
subgraph,
tv_sess,
start_step=start_step)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
hypes['dist'] = FLAGS.dist
if FLAGS.layers:
hypes['arch']['layers'] = FLAGS.layers
if FLAGS.lr:
hypes['solver']['learning_rate'] = FLAGS.lr
if FLAGS.optimizer:
hypes['solver']['opt'] = FLAGS.optimizer
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
train.do_training(hypes)
def main(_):
utils.set_gpus_to_use()
sys.path.append("submodules/tensorflow-fcn")
sys.path.append("submodules/TensorVision")
import tensorvision.train
import tensorflow_fcn.utils
# try:
# import tensorvision.train
# import tensorflow_fcn.utils
# except ImportError:
# logging.error("Could not import the submodules.")
# logging.error("Please execute:"
# "'git submodule update --init --recursive'")
# exit(1)
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
logging.info("Start training")
train.do_training(hypes)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
logging.info("Start training")
encoder_path = hypes['model']['architecture_file']
hypes['model']['architecture_file'] = '../encoder/stub.py'
hypes['ga_data'] = 'ga_data.json'
run_genetic_algorithm(hypes, encoder_path)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
else:
runs_dir = 'RUNS'
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
maybe_download_and_extract(runs_dir)
logging.info("Evaluating on Validation data.")
logdir = os.path.join(runs_dir, FLAGS.RUN)
# logging.info("Output images will be saved to {}".format)
ana.do_analyze(logdir)
logging.info("Creating output on test data.")
kitti_test.do_inference(logdir)
logging.info("Analysis for pretrained model complete.")
logging.info("For evaluating your own models I recommend using:"
"`tv-analyze --logdir /path/to/run`.")
logging.info("tv-analysis has a much cleaner interface.")
def main(_):
"""Run main function."""
if FLAGS.hypes is None:
logging.error("No hypes are given.")
logging.error("Usage: tv-train --hypes hypes.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.set_gpus_to_use()
utils.load_plugins()
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
logging.info("Initialize training folder")
initialize_training_folder(hypes)
maybe_download_and_extract(hypes)
logging.info("Start training")
do_training(hypes)
def main(_):
utils.set_gpus_to_use()
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'MediSeg')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Start training")
train.do_training(hypes)
def main(_):
"""Run main function."""
if FLAGS.hypes is None:
logging.error("No hypes are given.")
logging.error("Usage: tv-train --hypes hypes.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.set_gpus_to_use()
utils.load_plugins()
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
logging.info("Initialize training folder")
initialize_training_folder(hypes)
maybe_download_and_extract(hypes)
logging.info("Start training")
do_training(hypes)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
os.environ["TV_DIR_DATA"] = "../../SemSeg_DATA/DATA"
os.environ["TV_DIR_RUNS"] = "../../SemSeg_DATA/RUNS"
# print(os.environ["TV_DIR_DATA"])
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
logging.info("Start training")
train.do_training(hypes)
def main(_):
"""Run main function."""
if FLAGS.hypes is None:
logging.error("No hypes are given.")
logging.error("Usage: python train.py --hypes hypes.json")
logging.error(" tf: tv-train --hypes hypes.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
logging.info(
"Initializing GPUs, plugins and creating the essential folders")
utils.set_gpus_to_use()
utils.load_plugins()
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
initialize_training_folder(hypes)
logging.info("Training settings")
do_training(hypes)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiBox')
else:
runs_dir = 'RUNS'
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
maybe_download_and_extract(runs_dir)
logging.info("Evaluating on Validation data.")
logdir = os.path.join(runs_dir, FLAGS.RUN)
# logging.info("Output images will be saved to {}".format)
ana.do_analyze(logdir, base_path='hypes')
logging.info("Analysis for pretrained model complete.")
logging.info("For evaluating your own models I recommend using:"
"`tv-analyze --logdir /path/to/run`.")
logging.info("")
logging.info(
"Output images can be found in {}/analyse/images.".format(logdir))
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'KittiSeg')
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
logging.info("Start training")
train.do_training(hypes)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
if tf.app.flags.FLAGS.hypes is None:
logging.error("No hype file is given.")
logging.info("Usage: python train.py --hypes hypes/KittiClass.json")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
utils.load_plugins()
if tf.app.flags.FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(tf.app.flags.FLAGS.mod)
dict_merge(hypes, mod_dict)
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
logdir = os.path.join(runs_dir, FLAGS.RUN)
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.continue_training(logdir)
def main(_):
logging.info(
"Initializing GPUs, plugins and creating the essential folders")
utils.set_gpus_to_use()
if FLAGS.hypes is None:
logging.error("No hypes are given.")
logging.error("Usage: python train.py --hypes hypes.json")
logging.error(" tf: tv-train --hypes hypes.json")
exit(1)
with open(FLAGS.hypes) as f:
logging.info("f: %s", f)
hypes = commentjson.load(f)
if FLAGS.mod is not None:
import ast
mod_dict = ast.literal_eval(FLAGS.mod)
dict_merge(hypes, mod_dict)
logging.info("Loading plugins")
utils.load_plugins()
logging.info("Set dirs")
utils.set_dirs(hypes, FLAGS.hypes)
logging.info("Add paths to sys")
utils._add_paths_to_sys(hypes)
logging.info("Initialize training folder")
train.initialize_training_folder(hypes)
tf.reset_default_graph()
logging.info("Start training")
train.do_training(hypes)
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'VOCSeg')
else:
runs_dir = 'RUNS'
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
maybe_download_and_extract(runs_dir, "VOCSeg_2017_04_14_00.49")
logging.info("Evaluating on Validation data.")
logdir = os.path.join(runs_dir, "VOCSeg_2017_04_14_00.49")
# logging.info("Output images will be saved to {}".format)
# ana.do_analyze(logdir)
logging.info("Creating output on test data.")
voc_test.do_inference(logdir)
logging.info("Analysis for pretrained model complete.")
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
hypes_path = FLAGS.logdir
hypes_path = os.path.join(hypes_path, "model_files/hypes.json")
with open(hypes_path, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'FacadeSeg')
else:
runs_dir = 'RUNS'
utils.set_dirs(hypes, FLAGS.hypes)
utils._add_paths_to_sys(hypes)
logging.info("Evaluating on Validation data.")
ana.do_analyze(FLAGS.logdir)
logging.info("Segmenting and test data. Creating output.")
ana.do_inference(FLAGS.logdir)
logging.info("Analysis for pretrained model complete.")
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
maybe_download_and_extract(runs_dir)
logging.info("Trimming weights.")
logdir = os.path.join(runs_dir, FLAGS.RUN)
modules = utils.load_modules_from_hypes(hypes)
with tf.Graph().as_default():
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepare the tv session
with tf.Session().as_default():
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
saver = tv_sess['saver']
cur_step = core.load_weights(logdir, sess, saver)
if cur_step is None:
logging.warning("Loaded global_step is None.")
logging.warning("This could mean,"
" that no weights have been loaded.")
logging.warning("Starting Training with step 0.")
cur_step = 0
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# prepaire the tv session
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
# Create a session for running Ops on the Graph.
trim_dir = 'RUNS/trimmed'
shutil.copytree(logdir, trim_dir)
shutil.copy(tf.app.flags.FLAGS.hypes,
os.path.join(trim_dir, 'model_files', 'hypes.json'))
sess = tf.Session()
saver = tf.train.Saver()
core.load_weights(trim_dir, sess, saver)
for weight in tf.contrib.model_pruning.get_masks():
if any([
layer in weight.name
for layer in hypes['layer_pruning']['layers']
]):
weight_value = tv_sess['sess'].run(weight)
kernel_count = int(weight_value.shape[3] *
hypes['layer_pruning']['layer_sparsity'])
l1_values = np.sum(np.abs(weight_value), axis=(0, 1, 2))
toss_kernels = l1_values.argsort()[:kernel_count]
weight_value[:, :, :, toss_kernels] = 0
assign_op = tf.assign(weight, tf.constant(weight_value))
tv_sess['sess'].run(assign_op)
checkpoint_path = os.path.join(trim_dir, 'model.ckpt')
tv_sess['saver'].save(sess, checkpoint_path, global_step=cur_step)
train.continue_training(trim_dir)
def build_united_model(meta_hypes):
logging.info("Initialize training folder")
subhypes = {}
subgraph = {}
submodules = {}
subqueues = {}
subgraph['debug_ops'] = {}
base_path = meta_hypes['dirs']['base_path']
first_iter = True
for model in meta_hypes['model_list']:
subhypes_file = os.path.join(base_path, meta_hypes['models'][model])
with open(subhypes_file, 'r') as f:
logging.info("f: %s", f)
subhypes[model] = json.load(f)
hypes = subhypes[model]
utils.set_dirs(hypes, subhypes_file)
#output directories
hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir']
#data directories
hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir']
#initialize the training folders
train.initialize_training_folder(hypes, files_dir=model,
logging=first_iter)
#get the image directory from hypes
meta_hypes['dirs']['image_dir'] = hypes['dirs']['image_dir']
#load the modules from hypes
submodules[model] = utils.load_modules_from_hypes(
hypes, postfix="_%s" % model)
modules = submodules[model]
logging.info("Build %s computation Graph.", model)
with tf.name_scope("Queues_%s" % model):
subqueues[model] = modules['input'].create_queues(hypes, 'train')
logging.info('Building Model: %s' % model)
subgraph[model] = build_training_graph(hypes,
subqueues[model],
modules,
first_iter)
first_iter = False
#if model list is having detection and segmentation
if len(meta_hypes['models']) == 2:
#calculate the losses
_recombine_2_losses(meta_hypes, subgraph, subhypes, submodules)
#if model list is having detection, segmentation and classification
else:
#calculate combined losses
_recombine_3_losses(meta_hypes, subgraph, subhypes, submodules)
hypes = subhypes[meta_hypes['model_list'][0]]
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
#for each of the models in model list expand the image dimension
for model in meta_hypes['model_list']:
hypes = subhypes[model]
modules = submodules[model]
optimizer = modules['solver']
with tf.name_scope('Validation_%s' % model):
tf.get_variable_scope().reuse_variables()
#returns the tensor that may be used as handle for feeding a value
image_pl = tf.placeholder(tf.float32)
#expand the shape of the array by inserting new axes in 0th positon
image = tf.expand_dims(image_pl, 0)
inf_out = core.build_inference_graph(hypes, modules,
image=image)
subgraph[model]['image_pl'] = image_pl
subgraph[model]['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, subqueues[model],
'train', sess)
#join paths for the output result
target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json')
with open(target_file, 'w') as outfile:
json.dump(meta_hypes, outfile, indent=2, sort_keys=True)
return subhypes, submodules, subgraph, tv_sess
def build_united_model(meta_hypes):
logging.info("Initialize training folder")
subhypes = {}
subgraph = {}
submodules = {}
subqueues = {}
subgraph['debug_ops'] = {}
base_path = meta_hypes['dirs']['base_path']
first_iter = True
for model in meta_hypes['model_list']:
subhypes_file = os.path.join(base_path, meta_hypes['models'][model])
with open(subhypes_file, 'r') as f:
logging.info("f: %s", f)
subhypes[model] = json.load(f)
hypes = subhypes[model]
utils.set_dirs(hypes, subhypes_file)
hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir']
hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir']
train.initialize_training_folder(hypes,
files_dir=model,
logging=first_iter)
meta_hypes['dirs']['image_dir'] = hypes['dirs']['image_dir']
submodules[model] = utils.load_modules_from_hypes(hypes,
postfix="_%s" %
model)
modules = submodules[model]
logging.info("Build %s computation Graph.", model)
with tf.name_scope("Queues_%s" % model):
subqueues[model] = modules['input'].create_queues(hypes, 'train')
logging.info('Building Model: %s' % model)
subgraph[model] = build_training_graph(hypes, subqueues[model],
modules, first_iter)
first_iter = False
if len(meta_hypes['models']) == 2:
_recombine_2_losses(meta_hypes, subgraph, subhypes, submodules)
else:
_recombine_3_losses(meta_hypes, subgraph, subhypes, submodules)
hypes = subhypes[meta_hypes['model_list'][0]]
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
for model in meta_hypes['model_list']:
hypes = subhypes[model]
modules = submodules[model]
optimizer = modules['solver']
with tf.name_scope('Validation_%s' % model):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
inf_out = core.build_inference_graph(hypes, modules, image=image)
subgraph[model]['image_pl'] = image_pl
subgraph[model]['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, subqueues[model],
'train', sess)
target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json')
with open(target_file, 'w') as outfile:
json.dump(meta_hypes, outfile, indent=2, sort_keys=True)
return subhypes, submodules, subgraph, tv_sess
