def continue_training(logdir):
"""
Continues training of a model.
This will load model files and weights found in logdir and continues
an aborted training.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session() as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepaire the tv session
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
saver = tv_sess['saver']
logging_file = os.path.join(logdir, 'output.log')
utils.create_filewrite_handler(logging_file, mode='a')
logging.info("Continue training.")
cur_step = core.load_weights(logdir, sess, saver)
if cur_step is None:
logging.warning("Loaded global_step is None.")
logging.warning("This could mean,"
" that no weights have been loaded.")
logging.warning("Starting Training with step 0.")
cur_step = 0
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules, image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
run_training(hypes, modules, tv_graph, tv_sess, cur_step)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def initialize_training_folder(hypes):
"""
Creating the training folder and copy all model files into it.
The model will be executed from the training folder and all
outputs will be saved there.
Parameters
----------
hypes : dict
Hyperparameters
"""
target_dir = os.path.join(hypes['dirs']['output_dir'], "model_files")
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# Creating an additional logging saving the console outputs
# into the training folder
logging_file = os.path.join(hypes['dirs']['output_dir'], "output.log")
utils.create_filewrite_handler(logging_file)
# TODO: read more about loggers and make file logging neater.
target_file = os.path.join(target_dir, 'hypes.json')
with open(target_file, 'w') as outfile:
json.dump(hypes, outfile, indent=2)
_copy_parameters_to_traindir(
hypes, hypes['model']['input_file'], "data_input.py", target_dir)
_copy_parameters_to_traindir(
hypes, hypes['model']['architecture_file'], "architecture.py",
target_dir)
_copy_parameters_to_traindir(
hypes, hypes['model']['objective_file'], "objective.py", target_dir)
_copy_parameters_to_traindir(
hypes, hypes['model']['optimizer_file'], "solver.py", target_dir)
def initialize_training_folder(hypes):
"""
Creating the training folder and copy all model files into it.
The model will be executed from the training folder and all
outputs will be saved there.
Parameters
----------
hypes : dict
Hyperparameters
"""
target_dir = os.path.join(hypes['dirs']['output_dir'], "model_files")
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# Creating an additional logging saving the console outputs
# into the training folder
logging_file = os.path.join(hypes['dirs']['output_dir'], "output.log")
utils.create_filewrite_handler(logging_file)
# TODO: read more about loggers and make file logging neater.
target_file = os.path.join(target_dir, 'hypes.json')
with open(target_file, 'w') as outfile:
json.dump(hypes, outfile, indent=2)
_copy_parameters_to_traindir(
hypes, hypes['model']['input_file'], "data_input.py", target_dir)
_copy_parameters_to_traindir(
hypes, hypes['model']['architecture_file'], "architecture.py",
target_dir)
_copy_parameters_to_traindir(
hypes, hypes['model']['objective_file'], "objective.py", target_dir)
_copy_parameters_to_traindir(
hypes, hypes['model']['optimizer_file'], "solver.py", target_dir)
def continue_training(logdir):
"""
Continues training of a model.
This will load model files and weights found in logdir and continues
an aborted training.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
# append output to output.log
logging_file = os.path.join(logdir, 'output.log')
utils.create_filewrite_handler(logging_file, mode='a')
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default() as graph:
# build the graph based on the loaded modules
graph_ops = build_training_graph(hypes, modules)
q = graph_ops[0]
# prepaire the tv session
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
if hasattr(objective, 'evaluate'):
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
# Load weights from logdir
cur_step = core.load_weights(logdir, sess, saver)
# Start the data load
_start_enqueuing_threads(hypes, q, sess, data_input)
# And then after everything is built, start the training loop.
start_time = time.time()
for step in xrange(cur_step+1, hypes['solver']['max_steps']):
start_time = run_training_step(hypes, step, start_time,
graph_ops, sess_coll, modules,
image_pl, softmax)
# stopping input Threads
coord.request_stop()
coord.join(threads)
def continue_training(logdir):
"""
Continues training of a model.
This will load model files and weights found in logdir and continues
an aborted training.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
# append output to output.log
logging_file = os.path.join(logdir, 'output.log')
utils.create_filewrite_handler(logging_file, mode='a')
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default() as graph:
# build the graph based on the loaded modules
graph_ops = build_training_graph(hypes, modules)
q = graph_ops[0]
# prepaire the tv session
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
if hasattr(objective, 'evaluate'):
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
# Load weights from logdir
cur_step = core.load_weights(logdir, sess, saver)
# Start the data load
_start_enqueuing_threads(hypes, q, sess, data_input)
# And then after everything is built, start the training loop.
start_time = time.time()
for step in xrange(cur_step+1, hypes['solver']['max_steps']):
start_time = run_training_step(hypes, step, start_time,
graph_ops, sess_coll, objective,
image_pl, softmax)
# stopping input Threads
coord.request_stop()
coord.join(threads)
def do_analyze(logdir, base_path=None):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
if base_path is not None:
hypes['dirs']['base_path'] = base_path
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# prepaire the tv session
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules,
image=image)
# Create a session for running Ops on the Graph.
sess = tf.Session()
saver = tf.train.Saver()
core.load_weights(logdir, sess, saver)
logging.info("Graph loaded succesfully. Starting evaluation.")
output_dir = os.path.join(logdir, 'analyse')
logging.info("Output Images will be written to: {}".format(
os.path.join(output_dir, "images/")))
logging_file = os.path.join(logdir, "analyse/output.log")
utils.create_filewrite_handler(logging_file)
eval_dict, images = modules['eval'].evaluate(
hypes, sess, image_pl, inf_out)
logging.info("Evaluation Succesfull. Results:")
utils.print_eval_dict(eval_dict)
_write_images_to_logdir(images, output_dir)
def main(_):
utils.set_gpus_to_use()
load_weights = tf.app.flags.FLAGS.logdir is not None
if not load_weights:
with open(tf.app.flags.FLAGS.hypes, 'r') as f:
logging.info("f: %s", f)
hypes = json.load(f)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'MultiNet')
# with tf.Session() as sess:
# king
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)) as sess:
if not load_weights:
utils.set_dirs(hypes, tf.app.flags.FLAGS.hypes)
utils._add_paths_to_sys(hypes)
# Build united Model
subhypes, submodules, subgraph, tv_sess = build_united_model(hypes)
start_step = 0
else:
logdir = tf.app.flags.FLAGS.logdir
logging_file = os.path.join(logdir, "output.log")
utils.create_filewrite_handler(logging_file, mode='a')
hypes, subhypes, submodules, subgraph, tv_sess, start_step = \
load_united_model(logdir)
if start_step is None:
start_step = 0
# Run united training
run_united_training(hypes,
subhypes,
submodules,
subgraph,
tv_sess,
start_step=start_step)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def do_analyze(logdir):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# prepaire the tv session
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
core.load_weights(logdir, sess, saver)
eval_dict, images = objective.tensor_eval(hypes, sess, image_pl,
softmax)
logging_file = os.path.join(logdir, "eval/analysis.log")
utils.create_filewrite_handler(logging_file)
utils.print_eval_dict(eval_dict)
_write_images_to_logdir(images, logdir)
return
def do_analyze(logdir):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# prepaire the tv session
with tf.name_scope('Validation'):
image_pl, label_pl = _create_input_placeholder()
image = tf.expand_dims(image_pl, 0)
softmax = core.build_inference_graph(hypes, modules,
image=image,
label=label_pl)
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
core.load_weights(logdir, sess, saver)
eval_dict, images = objective.tensor_eval(hypes, sess, image_pl,
softmax)
logging_file = os.path.join(logdir, "eval/analysis.log")
utils.create_filewrite_handler(logging_file)
utils.print_eval_dict(eval_dict)
_write_images_to_logdir(images, logdir)
return
def main(_):
utils.set_gpus_to_use()
load_weights = tf.app.flags.FLAGS.logdir is not None
if not load_weights:
print("You must specify --logdir path/to/trained/model")
exit(1)
utils.load_plugins()
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'MultiNet')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1)
config = tf.ConfigProto(gpu_options=gpu_options)
with tf.Session(config=config) as sess:
logdir = tf.app.flags.FLAGS.logdir
logging_file = os.path.join(logdir, "output.log")
utils.create_filewrite_handler(logging_file, mode='a')
hypes, subhypes, submodules, subgraph, tv_sess, start_step = load_united_model(
logdir)
if start_step is None:
start_step = 0
# Run united training
run_united_evaluation(hypes,
subhypes,
submodules,
subgraph,
tv_sess,
step=start_step)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])
def main(_):
utils.set_gpus_to_use()
logdir = FLAGS.logdir
data_file = FLAGS.data
#if input is not given pass the error message
if logdir is None:
logging.error('Usage python predict_joint --logdir /path/to/logdir'
'--data /path/to/data/txt')
exit(1)
output_folder = os.path.join(logdir, res_folder)
if not os.path.exists(output_folder):
os.mkdir(output_folder)
logdir = logdir
utils.load_plugins()
#if data directory exist, join the path
if 'TV_DIR_DATA' in os.environ:
data_file = os.path.join(os.environ['TV_DIR_DATA'], data_file)
else:
#else create a directory DATA
data_file = os.path.join('DATA', data_file)
if not os.path.exists(data_file):
logging.error('Please provide a valid data_file.')
logging.error('Use --data_file')
exit(1)
if 'TV_DIR_RUNS' in os.environ:
os.environ['TV_DIR_RUNS'] = os.path.join(os.environ['TV_DIR_RUNS'],
'UnitedVision2')
logging_file = os.path.join(output_folder, "analysis.log") #log file
utils.create_filewrite_handler(logging_file, mode='a')
load_out = load_united_model(logdir)
run_eval(load_out, output_folder, data_file)
def do_analyze(logdir):
"""
Analyze a trained model.
This will load model files and weights found in logdir and run a basic
analysis.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
data_input, arch, objective, solver = modules
logging_file = os.path.join(logdir, "eval/analysis.log")
utils.create_filewrite_handler(logging_file)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# build the graph based on the loaded modules
graph_ops = core.build_graph(hypes, modules, train=False)
q, train_op, loss, eval_lists = graph_ops
q = graph_ops[0]
# prepaire the tv session
sess_coll = core.start_tv_session(hypes)
sess, saver, summary_op, summary_writer, coord, threads = sess_coll
core.load_weights(logdir, sess, saver)
# Start the data load
data_input.start_enqueuing_threads(hypes, q['val'], 'val', sess,
hypes['dirs']['data_dir'])
return core.do_eval(hypes, eval_lists, 'val', sess)
def continue_training(logdir):
"""
Continues training of a model.
This will load model files and weights found in logdir and continues
an aborted training.
Parameters
----------
logdir : string
Directory with logs.
"""
hypes = utils.load_hypes_from_logdir(logdir)
modules = utils.load_modules_from_logdir(logdir)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Session() as sess:
# build the graph based on the loaded modules
with tf.name_scope("Queues"):
queue = modules['input'].create_queues(hypes, 'train')
tv_graph = core.build_training_graph(hypes, queue, modules)
# prepaire the tv session
tv_sess = core.start_tv_session(hypes)
sess = tv_sess['sess']
saver = tv_sess['saver']
logging_file = os.path.join(logdir, 'output.log')
utils.create_filewrite_handler(logging_file, mode='a')
logging.info("Continue training.")
cur_step = core.load_weights(logdir, sess, saver)
if cur_step is None:
logging.warning("Loaded global_step is None.")
logging.warning("This could mean,"
" that no weights have been loaded.")
logging.warning("Starting Training with step 0.")
cur_step = 0
with tf.name_scope('Validation'):
tf.get_variable_scope().reuse_variables()
image_pl = tf.placeholder(tf.float32)
image = tf.expand_dims(image_pl, 0)
image.set_shape([1, None, None, 3])
inf_out = core.build_inference_graph(hypes, modules,
image=image)
tv_graph['image_pl'] = image_pl
tv_graph['inf_out'] = inf_out
# Start the data load
modules['input'].start_enqueuing_threads(hypes, queue, 'train', sess)
# And then after everything is built, start the training loop.
run_training(hypes, modules, tv_graph, tv_sess, cur_step)
# stopping input Threads
tv_sess['coord'].request_stop()
tv_sess['coord'].join(tv_sess['threads'])