def do_training(hypes):
"""
Train model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
modules = utils.load_modules_from_hypes(hypes)
# set to allocate memory on GPU as needed
# For more details, look at
# https://stackoverflow.com/questions/36927607/how-can-i-solve-ran-out-of-gpu-memory-in-tensorflow
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session(config=config) as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepaire the tv session
tv_sess = core.start_tv_session(hypes)
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
train.run_training(hypes, modules, tv_graph, tv_sess)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def do_training(hypes):
"""
Train model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
modules = utils.load_modules_from_hypes(hypes)
data_input, arch, objective, solver = modules
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# build the graph based on the loaded modules
graph_ops = build_training_graph(hypes, modules)
q = graph_ops[0]
# prepaire the tv session
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
# Start the data load
_start_enqueuing_threads(hypes, q, sess, data_input)
# And then after everything is built, start the training loop.
start_time = time.time()
for step in xrange(hypes['solver']['max_steps']):
start_time = run_training_step(hypes, step, start_time,
graph_ops, sess_coll, modules,
image_pl, softmax)
if hasattr(solver, 'update_learning_rate'):
solver.update_learning_rate(hypes, step)
# stopping input Threads
coord.request_stop()
coord.join(threads)
def do_training(hypes):
"""
Train model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
modules = utils.load_modules_from_hypes(hypes)
data_input, arch, objective, solver = modules
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# build the graph based on the loaded modules
graph_ops = build_training_graph(hypes, modules)
q = graph_ops[0]
# prepaire the tv session
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
# Start the data load
_start_enqueuing_threads(hypes, q, sess, data_input)
# And then after everything is built, start the training loop.
start_time = time.time()
for step in xrange(hypes['solver']['max_steps']):
start_time = run_training_step(hypes, step, start_time,
graph_ops, sess_coll, objective,
image_pl, softmax)
if hasattr(solver, 'update_learning_rate'):
solver.update_learning_rate(hypes, step)
# stopping input Threads
coord.request_stop()
coord.join(threads)
def do_training(hypes):
"""
Train model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
modules = utils.load_modules_from_hypes(hypes)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session() as sess:
# build the graph based on the loaded modules
logging.info("..Creating queues")
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
logging.info("..Building training graph")
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepare the tv session
tv_sess = core.start_tv_session(hypes)
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# Start the data load
logging.info("..Enqueuing files")
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
logging.info("..Initializing the training")
run_training(hypes, modules, tv_graph, tv_sess)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def do_training(hypes):
"""
Train model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
modules = utils.load_modules_from_hypes(hypes)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session() as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepaire the tv session
tv_sess = core.start_tv_session(hypes)
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules,
image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
run_training(hypes, modules, tv_graph, tv_sess)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def load_trained_model(sess, hypes):
"""
Load an exsiting model trained for hand segmentation
Parameters
----------
hypes : dict
Hyperparameters
"""
# load modules
modules = utils.load_modules_from_hypes(hypes)
# start loading Graph from the pretrained model
model_dir = hypes['transfer']['model_folder']
model_name = hypes['transfer']['model_name']
# meta file
meta_file = os.path.join(model_dir, model_name, ".meta")
# restore
saver = tf.train.import_meta_graph(meta_file)
saver.restore(sess, tf.train.latest_checkpoint(model_dir))
def do_finetuning(hypes):
"""
Finetune model for a number of steps.
This finetunes the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
try:
import tensorvision.core as core
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
modules = utils.load_modules_from_hypes(hypes)
# set to allocate memory on GPU as needed
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session(config=config) as sess:
# with tf.Session() as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = build_training_graph(hypes, queue, modules)
# restoring vars
vars_to_restore = restoring_vars(hypes)
restorer = tf.train.Saver(vars_to_restore)
# load pre-trained model of hand segmentation
logging.info("Loading pretrained model's weights")
model_dir = hypes['transfer']['model_folder']
model_file = hypes['transfer']['model_name']
# DEBUG: check the model file
# check_model(os.path.join(model_dir, model_file))
"""
# Get a list of vars to restore
vars_to_restore = restoring_vars(sess)
print("vars to restore:", vars_to_restore)
# Create another Saver for restoring pre-trained vars
saver = tf.train.Saver(vars_to_restore)
"""
core.load_weights(model_dir, sess, restorer)
# load_trained_model(sess, hypes)
saver = tf.train.Saver(max_to_keep=int(utils.cfg.max_to_keep))
# prepaire the tv session
tv_sess = prepare_tv_session(hypes, sess, saver)
# DEBUG: print weights
# check_weights(tv_sess['sess'])
# check_graph(tv_sess['sess'])
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
num_channels = hypes['arch']['num_channels']
image_pl = tf.placeholder(tf.float32, [None, None, num_channels])
image = tf.expand_dims(image_pl, 0)
if hypes['jitter']['resize_image']:
height = hypes['jitter']['image_height']
width = hypes['jitter']['image_width']
# set the pre-defined image size here
image.set_shape([1, height, width, num_channels])
inf_out = core.build_inference_graph(hypes, modules, image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
run_training(hypes, modules, tv_graph, tv_sess)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def build_united_model(meta_hypes):
logging.info("Initialize training folder")
subhypes = {}
subgraph = {}
submodules = {}
subqueues = {}
subgraph['debug_ops'] = {}
base_path = meta_hypes['dirs']['base_path']
first_iter = True
for model in meta_hypes['model_list']:
subhypes_file = os.path.join(base_path, meta_hypes['models'][model])
with open(subhypes_file, 'r') as f:
logging.info("f: %s", f)
subhypes[model] = json.load(f)
hypes = subhypes[model]
utils.set_dirs(hypes, subhypes_file)
#output directories
hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir']
#data directories
hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir']
#initialize the training folders
train.initialize_training_folder(hypes, files_dir=model,
logging=first_iter)
#get the image directory from hypes
meta_hypes['dirs']['image_dir'] = hypes['dirs']['image_dir']
#load the modules from hypes
submodules[model] = utils.load_modules_from_hypes(
hypes, postfix="_%s" % model)
modules = submodules[model]
logging.info("Build %s computation Graph.", model)
with tf.name_scope("Queues_%s" % model):
subqueues[model] = modules['input'].create_queues(hypes, 'train')
logging.info('Building Model: %s' % model)
subgraph[model] = build_training_graph(hypes,
subqueues[model],
modules,
first_iter)
first_iter = False
#if model list is having detection and segmentation
if len(meta_hypes['models']) == 2:
#calculate the losses
_recombine_2_losses(meta_hypes, subgraph, subhypes, submodules)
#if model list is having detection, segmentation and classification
else:
#calculate combined losses
_recombine_3_losses(meta_hypes, subgraph, subhypes, submodules)
hypes = subhypes[meta_hypes['model_list'][0]]
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
#for each of the models in model list expand the image dimension
for model in meta_hypes['model_list']:
hypes = subhypes[model]
modules = submodules[model]
optimizer = modules['solver']
with tf.name_scope('Validation_%s' % model):
tf.get_variable_scope().reuse_variables()
#returns the tensor that may be used as handle for feeding a value
image_pl = tf.placeholder(tf.float32)
#expand the shape of the array by inserting new axes in 0th positon
image = tf.expand_dims(image_pl, 0)
inf_out = core.build_inference_graph(hypes, modules,
image=image)
subgraph[model]['image_pl'] = image_pl
subgraph[model]['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, subqueues[model],
'train', sess)
#join paths for the output result
target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json')
with open(target_file, 'w') as outfile:
json.dump(meta_hypes, outfile, indent=2, sort_keys=True)
return subhypes, submodules, subgraph, tv_sess
def build_united_model(meta_hypes):
logging.info("Initialize training folder")
subhypes = {}
subgraph = {}
submodules = {}
subqueues = {}
subgraph['debug_ops'] = {}
base_path = meta_hypes['dirs']['base_path']
first_iter = True
for model in meta_hypes['model_list']:
subhypes_file = os.path.join(base_path, meta_hypes['models'][model])
with open(subhypes_file, 'r') as f:
logging.info("f: %s", f)
subhypes[model] = json.load(f)
hypes = subhypes[model]
utils.set_dirs(hypes, subhypes_file)
hypes['dirs']['output_dir'] = meta_hypes['dirs']['output_dir']
hypes['dirs']['data_dir'] = meta_hypes['dirs']['data_dir']
train.initialize_training_folder(hypes,
files_dir=model,
logging=first_iter)
meta_hypes['dirs']['image_dir'] = hypes['dirs']['image_dir']
submodules[model] = utils.load_modules_from_hypes(hypes,
postfix="_%s" %
model)
modules = submodules[model]
logging.info("Build %s computation Graph.", model)
with tf.name_scope("Queues_%s" % model):
subqueues[model] = modules['input'].create_queues(hypes, 'train')
logging.info('Building Model: %s' % model)
subgraph[model] = build_training_graph(hypes, subqueues[model],
modules, first_iter)
first_iter = False
if len(meta_hypes['models']) == 2:
_recombine_2_losses(meta_hypes, subgraph, subhypes, submodules)
else:
_recombine_3_losses(meta_hypes, subgraph, subhypes, submodules)
hypes = subhypes[meta_hypes['model_list'][0]]
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
for model in meta_hypes['model_list']:
hypes = subhypes[model]
modules = submodules[model]
optimizer = modules['solver']
with tf.name_scope('Validation_%s' % model):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
inf_out = core.build_inference_graph(hypes, modules, image=image)
subgraph[model]['image_pl'] = image_pl
subgraph[model]['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, subqueues[model],
'train', sess)
target_file = os.path.join(meta_hypes['dirs']['output_dir'], 'hypes.json')
with open(target_file, 'w') as outfile:
json.dump(meta_hypes, outfile, indent=2, sort_keys=True)
return subhypes, submodules, subgraph, tv_sess
def main(_):
utils.set_gpus_to_use()
try:
import tensorvision.train
import tensorflow_fcn.utils
except ImportError:
logging.error("Could not import the submodules.")
logging.error("Please execute:"
"'git submodule update --init --recursive'")
exit(1)
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
runs_dir = os.path.join(os.environ['TV_DIR_RUNS'], 'KittiSeg')
else:
runs_dir = 'RUNS'
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
train.maybe_download_and_extract(hypes)
maybe_download_and_extract(runs_dir)
logging.info("Trimming weights.")
logdir = os.path.join(runs_dir, FLAGS.RUN)
modules = utils.load_modules_from_hypes(hypes)
with tf.Graph().as_default():
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepare the tv session
with tf.Session().as_default():
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
saver = tv_sess['saver']
cur_step = core.load_weights(logdir, sess, saver)
if cur_step is None:
logging.warning("Loaded global_step is None.")
logging.warning("This could mean,"
" that no weights have been loaded.")
logging.warning("Starting Training with step 0.")
cur_step = 0
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# prepaire the tv session
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
# Create a session for running Ops on the Graph.
trim_dir = 'RUNS/trimmed'
shutil.copytree(logdir, trim_dir)
shutil.copy(tf.app.flags.FLAGS.hypes,
os.path.join(trim_dir, 'model_files', 'hypes.json'))
sess = tf.Session()
saver = tf.train.Saver()
core.load_weights(trim_dir, sess, saver)
for weight in tf.contrib.model_pruning.get_masks():
if any([
layer in weight.name
for layer in hypes['layer_pruning']['layers']
]):
weight_value = tv_sess['sess'].run(weight)
kernel_count = int(weight_value.shape[3] *
hypes['layer_pruning']['layer_sparsity'])
l1_values = np.sum(np.abs(weight_value), axis=(0, 1, 2))
toss_kernels = l1_values.argsort()[:kernel_count]
weight_value[:, :, :, toss_kernels] = 0
assign_op = tf.assign(weight, tf.constant(weight_value))
tv_sess['sess'].run(assign_op)
checkpoint_path = os.path.join(trim_dir, 'model.ckpt')
tv_sess['saver'].save(sess, checkpoint_path, global_step=cur_step)
train.continue_training(trim_dir)
def do_training(hypes):
"""
Train model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
modules = utils.load_modules_from_hypes(hypes)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session() as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
regression_weights = tf.placeholder(dtype=tf.float32, shape=(3,))
hypes['solver']['regression_weights'] = regression_weights
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepaire the tv session
tv_sess = core.start_tv_session(hypes)
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
calib = tf.placeholder(tf.float32, shape=[1, hypes['grid_height'], hypes['grid_width'], 3, 4])
xy_scale = tf.placeholder(tf.float32, shape=[1, hypes['grid_height'], hypes['grid_width'], 2])
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, 384, 1248, 3])
inf_out, encoder_out = core.build_inference_graph(hypes, modules,
image, calib, xy_scale)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
tv_graph['calib_pl'] = calib
tv_graph['xy_scale_pl'] = xy_scale
tv_graph['encoder_out'] = encoder_out
all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES)
sess.run(tf.variables_initializer(all_variables))
var_list = [var for var in all_variables if "beta" not in var.name and 'Adam' not in var.name]
saver = tf.train.Saver(var_list=var_list)
saver.restore(sess, hypes['pretrained'])
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
run_training(hypes, modules, tv_graph, tv_sess)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def do_evaling(hypes):
"""
eval model for a number of steps.
This trains the model for at most hypes['solver']['max_steps'].
It shows an update every utils.cfg.step_show steps and writes
the model to hypes['dirs']['output_dir'] every utils.cfg.step_eval
steps.
Paramters
---------
hypes : dict
Hyperparameters
"""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
modules = utils.load_modules_from_hypes(hypes)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session() as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepaire the tv session
tv_sess = core.start_tv_session(hypes)
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
calib = tf.placeholder(
tf.float32,
shape=[1, hypes['grid_height'], hypes['grid_width'], 3, 4])
xy_scale = tf.placeholder(
tf.float32,
shape=[1, hypes['grid_height'], hypes['grid_width'], 2])
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, 384, 1248, 3])
inf_out = core.build_inference_graph(hypes, modules, image, calib,
xy_scale)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
tv_graph['calib_pl'] = calib
tv_graph['xy_scale_pl'] = xy_scale
all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES)
sess.run(tf.variables_initializer(all_variables))
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
saver = tf.train.Saver()
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(hypes["dirs"]["ckpt_dir"])
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
# And then after everything is built, start the training loop.
run_evaling(hypes, modules, tv_graph, tv_sess)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
