def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
with tf.device(deploy_config.inputs_device()):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
[image, shape, glabels, gbboxes] = provider.get(
['image', 'shape', 'object/label', 'object/bbox'])
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list(
[b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=2 * deploy_config.num_clones)
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple
clones of network_fn."""
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.moving_average_decay:
update_ops.append(
variable_averages.apply(moving_average_variables))
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
summaries.add(tf.summary.scalar('total_loss', total_loss))
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
summary_op = tf.summary.merge(list(summaries), name='summary_op')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
global_step = slim.create_global_step()
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
tf_class = nets_factory.get_network(FLAGS.model_name)
tf_model = tf_class()
tf_anchors = tf_model.anchors()
tf_input_shape = tf_model.input_shape
[image, shape, glabels, gbboxes
] = provider.get(['image', 'shape', 'object/label', 'object/bbox'])
# Add Data Augmentation
# image : -> random crop & resize & whitening
# bboxes : -> random flip & change to relative style
image, glabels, gbboxes = image_preprocessing_fn(
image,
glabels,
gbboxes,
out_shape=tf_input_shape,
data_format=DATA_FORMAT)
# Match anchors to gt_bboxes
gclasses, glocalisations, gscores = \
tf_model.bboxes_encode(glabels, gbboxes, tf_anchors)
b_image, b_gclasses, b_glocalisations, b_gscores = tf.train.batch(
[image, gclasses, glocalisations, gscores],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
predictions, localisations, logits, end_points = \
tf_model.TF_net(b_image, is_training=True)
total_losses = tf_model.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
summaries.add(tf.summary.scalar('total_loss', total_losses))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES'):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
# summaries.add(tf.summary.scalar('learning_rate', learning_rate))
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
grad_vars = optimizer.compute_gradients(total_losses,
variables_to_train)
grad_updates = optimizer.apply_gradients(grad_vars,
global_step=global_step)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_losses,
name='train_op')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# missing_vars = tf_utils.missing_vars(FLAGS)
# if FLAGS.ignore_missing_vars is not None:
# FLAGS.ignore_missing_vars += missing_vars
slim.learning.train(train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
tf_recorders = glob.glob('./tf_records/*.tfrecord')
sess = tf.InteractiveSession()
dataset = tf.data.TFRecordDataset(tf_recorders)
global_step = tf.train.get_or_create_global_step()
dataset = dataset.map(_parse_function)
dataset = dataset.shuffle(buffer_size=200)
dataset = dataset.repeat(600)
dataset = dataset.batch(BATCH_SIZE)
print dataset.output_shapes
iterator = dataset.make_initializable_iterator()
next_element = iterator.get_next()
img, gt_maps, img_shape = gt_encoder(next_element)
img = tf.image.random_brightness(img, 0.3)
img = tf.image.random_contrast(img, 0.8, 1.2)
print img.get_shape().as_list()
print gt_maps.get_shape().as_list()
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
summaries.add(tf.summary.image("input_image", tf.cast(img, tf.float32)))
summaries.add(tf.summary.image("gt_map", tf.cast(gt_maps[:,:,:,1:], tf.float32)))
net_class = nets_factory.get_network('mobilenet_lane_net')
net_params = net_class.default_params._replace(num_classes=2)
lane_net = net_class(net_params)
net_shape = lane_net.params.img_shape
arg_scope = lane_net.arg_scope(weight_decay=0.00004,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
lane_prediction, lane_logits, end_points = \
lane_net.net(img, is_training = True)
summaries.add(tf.summary.image("prediction",
tf.cast(lane_prediction[:,:,:,1:],
tf.float32)))
lane_net.losses(lane_logits, gt_maps, 0)
total_loss = tf.losses.get_total_loss()
summaries.add(tf.summary.scalar('loss', total_loss))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for variable in tf.trainable_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
with tf.name_scope('Optimizer'):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, 700, global_step
)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
summarize_gradients=False
)
summary_op = tf.summary.merge(list(summaries), name='summary_op')
train_writer = tf.summary.FileWriter('./logs/', sess.graph)
variables_to_train = tf.trainable_variables()
variables_to_restore = \
tf.contrib.framework.filter_variables(variables_to_train,
exclude_patterns=['Lane_Prediction',
'_fpn'])
restorer = tf.train.Saver(variables_to_restore)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=tf.train.SaverDef.V2,
pad_step_number=False)
sess.run(tf.global_variables_initializer())
restorer.restore(sess, FLAGS.checkpoint_path)
i = 0
with slim.queues.QueueRunners(sess):
sess.run(iterator.initializer)
while (i < FLAGS.max_number_of_steps):
_, summary_str = sess.run([train_op, summary_op])
if i % 50 == 0:
global_step_str = global_step.eval()
print('%d iteration' % (global_step_str))
train_writer.add_summary(summary_str, global_step_str)
if i % 100 == 0:
global_step_str = global_step.eval()
saver.save(sess, "./logs/", global_step=global_step_str)
i += 1
def main(_):
if not FLAGS.data_dir:
raise ValueError('You must supply the dataset directory with --data_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
#with tf.Graph().as_default():
# with tf.Graph().as_default(), tf.device('/cpu:0'):
with tf.device('/cpu:0'):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.data_dir)
# Get the RON network and its anchors.
ron_class = nets_factory.get_network(FLAGS.model_name)
ron_params = ron_class.default_params._replace(num_classes=FLAGS.num_classes)
ron_net = ron_class(ron_params)
ron_shape = ron_net.params.img_shape
ron_anchors = ron_net.anchors(ron_shape)
# for i in range(len(ron_anchors)):
# for j in range(len(ron_anchors[i])):
# print(ron_anchors[i][j].shape)
tf_utils.print_configuration(FLAGS.__flags, ron_params,
dataset.data_sources, FLAGS.model_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=120 * FLAGS.batch_size,
common_queue_min=80 * FLAGS.batch_size,
shuffle=True)
# Get for RON network: image, labels, bboxes.
# (ymin, xmin, ymax, xmax) fro gbboxes
[image, shape, glabels, gbboxes, isdifficult] = provider.get(['image', 'shape',
'object/label',
'object/bbox',
'object/difficult'])
#glabels = tf.cast(isdifficult < tf.ones_like(isdifficult), glabels.dtype) * glabels
isdifficult_mask =tf.cond(tf.reduce_sum(tf.cast(tf.logical_not(tf.equal(tf.ones_like(isdifficult), isdifficult)), tf.float32)) < 1., lambda : tf.one_hot(0, tf.shape(isdifficult)[0], on_value=True, off_value=False, dtype=tf.bool), lambda : isdifficult < tf.ones_like(isdifficult))
glabels = tf.boolean_mask(glabels, isdifficult_mask)
gbboxes = tf.boolean_mask(gbboxes, isdifficult_mask)
#glabels = tf.Print(glabels, [glabels,isdifficult], message='glabels: ', summarize=200)
#### DEBUG ####
#image = tf.Print(image, [shape, glabels, gbboxes], message='before preprocess: ', summarize=20)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ron_shape,
data_format=DATA_FORMAT)
#### DEBUG ####
#image = tf.Print(image, [shape, glabels, gbboxes], message='after preprocess: ', summarize=20)
#glabels = tf.Print(glabels, [glabels,isdifficult], message='glabels: ', summarize=200)
# save_image_op = tf.py_func(save_image_with_bbox,
# [image,
# tf.reshape(tf.clip_by_value(glabels, 0, 22), [-1]),
# #tf.convert_to_tensor(list(rscores.keys()), dtype=tf.int64),
# tf.reshape(tf.ones_like(gbboxes), [-1]),
# tf.reshape(gbboxes, [-1, 4])],
# tf.int64, stateful=True)
# Encode groundtruth labels and bboxes.
# glocalisations is our regression object
# gclasses is the ground_trutuh label
# gscores is the the jaccard score with ground_truth
gclasses, glocalisations, gscores, gbboxes = \
ron_net.bboxes_encode(glabels, gbboxes, ron_anchors, positive_threshold=FLAGS.match_threshold, ignore_threshold=FLAGS.neg_threshold)
#gclasses[1] = tf.Print(gclasses[1], [gclasses[1]], message='gclasses[1]: ', summarize=200)
# save_image_op = tf.py_func(save_image_with_bbox,
# [image,
# tf.reshape(tf.clip_by_value(gclasses[3], 0, 22), [-1]),
# #tf.convert_to_tensor(list(rscores.keys()), dtype=tf.int64),
# tf.reshape(gscores[3], [-1]),
# tf.reshape(gbboxes[3], [-1, 4])],
# tf.int64, stateful=True)
# save_image_op = tf.py_func(save_image_with_bbox,
# [image,
# tf.clip_by_value(tf.concat([tf.reshape(_, [-1]) for _ in gclasses], axis=0), 0, 22),
# tf.concat([tf.reshape(_, [-1]) for _ in gscores], axis=0),
# tf.concat([tf.reshape(_, [-1, 4]) for _ in gbboxes], axis=0)],
# tf.int64, stateful=True)
# each size of the batch elements
# include one image, three others(gclasses, glocalisations, gscores)
batch_shape = [1] + [len(ron_anchors)] * 3
#with tf.control_dependencies([save_image_op]):
# Training batches and queue.
r = tf.train.batch(
tf_utils.reshape_list([image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=120 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
with tf.device('/gpu:0'):
# Construct RON network.
arg_scope = ron_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, logits, objness_pred, objness_logits, localisations, end_points = \
ron_net.net(b_image, is_training=True)
# Add loss function.
ron_net.losses(logits, localisations, objness_logits, objness_pred,
b_gclasses, b_glocalisations, b_gscores,
match_threshold = FLAGS.match_threshold,
neg_threshold = FLAGS.neg_threshold,
objness_threshold = FLAGS.objectness_thres,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
beta=FLAGS.loss_beta,
label_smoothing=FLAGS.label_smoothing)
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# Add summaries for losses and extra losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES'):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
# learning_rate = tf_utils.configure_learning_rate(FLAGS,
# dataset.num_samples,
# global_step)
lr_values = [FLAGS.learning_rate * decay for decay in [1., 0.1, 0.001]]
learning_rate_ = tf.train.piecewise_constant(tf.cast(global_step, tf.int32), [90000, 115000], lr_values)
learning_rate = tf.maximum(learning_rate_, tf.constant(FLAGS.end_learning_rate, dtype=learning_rate_.dtype))
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss = tf.losses.get_total_loss()
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grads = optimizer.compute_gradients(
total_loss,
variables_to_train)
grad_updates = optimizer.apply_gradients(grads,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
name='train_op')
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction = FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement = False, intra_op_parallelism_threads = FLAGS.num_cpu_threads, inter_op_parallelism_threads = FLAGS.num_cpu_threads, gpu_options = gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours = FLAGS.save_interval_secs/3600.,
write_version=2,
pad_step_number=False)
def wrapper_debug(sess):
sess = tf_debug.LocalCLIDebugWrapperSession(sess, thread_name_filter="MainThread$")
sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
return sess
slim.learning.train(
train_tensor,
logdir=FLAGS.model_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS, os.path.join(FLAGS.data_dir, 'vgg_model/vgg16_reducedfc.ckpt')),#'vgg_model/vgg16_reducedfc.ckpt'
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
session_wrapper=None,#wrapper_debug,#
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Config model_deploy. Keep TF Slim Models structure.
# Useful if want to need multiple GPUs and/or servers in the future.
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones, # 1
clone_on_cpu=FLAGS.clone_on_cpu, # False
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
# Create global_step.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
# Select the dataset.
# 'pascalvoc_2012', 'train', tfr文件存储位置
# TFR文件命名格式：'voc_2012_%s_*.tfrecord'，%s使用train或者test
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name) # 'ssd_300_vgg'
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes) # 替换类属性为21
ssd_net = ssd_class(ssd_params) # 创建类实例
ssd_shape = ssd_net.params.img_shape # 获取类属性(300,300)
ssd_anchors = ssd_net.anchors(ssd_shape) # 调用类方法，创建搜素框
# Select the preprocessing function.
# 'ssd_300_vgg', 如果 preprocessing_name 是 None 就使用 model_name
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
# '/job:ps/device:CPU:0' 或者 '/device:CPU:0'
with tf.device(deploy_config.inputs_device()):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset, # DatasetDataProvider 需要 slim.dataset.Dataset 做参数
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.c
# DatasetDataProvider可以通过TFR字段获取batch size数据
[image, shape, glabels, gbboxes] = provider.get(
['image', 'shape', 'object/label', 'object/bbox'])
# Pre-processing image, labels and bboxes.
# 'CHW' (n,) (n, 4)
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape, # (300,300)
data_format=DATA_FORMAT) # 'NCHW'
# Encode groundtruth labels and bboxes.
# f层个(m,m,k)，f层个(m,m,k,4xywh)，f层个(m,m,k) f层表示提取ssd特征的层的数目
# 0-20数字,方便loss的坐标记录,IOU值
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3 # (1,f层,f层,f层)
# Training batches and queue.
r = tf.train.batch( # 图片，中心点类别，真实框坐标，得分
tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size, # 32
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
# pp.pprint([image, gclasses, glocalisations, gscores])
# pp.pprint(r)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# pp.pprint([b_image, b_gclasses, b_glocalisations, b_gscores])
# Intermediate queueing: unique batch computation pipeline for all
# GPUs running the training.
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list(
[b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=2 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple
clones of network_fn."""
# Dequeue batch.
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape) # 重整list
# Construct SSD network.
# 这个实例方法会返回之前定义的函数ssd_arg_scope（允许修改两个参数）
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
# predictions: (BS, H, W, 4, 21)
# localisations: (BS, H, W, 4, 4)
# logits: (BS, H, W, 4, 21)
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
# Add loss function.
ssd_net.losses(
logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold, # .5
negative_ratio=FLAGS.negative_ratio, # 3
alpha=FLAGS.loss_alpha, # 1
label_smoothing=FLAGS.label_smoothing) # .0
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# =================================================================== #
# Add summaries from first clone.
# =================================================================== #
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses and extra losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op#
# total_loss 并不参与优化，仅记录用
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS), # 看函数实现就明白了
summary_op=summary_op, # tf.summary.merge节点
number_of_steps=FLAGS.max_number_of_steps, # 训练step
log_every_n_steps=FLAGS.log_every_n_steps, # 每次model保存step间隔
save_summaries_secs=FLAGS.save_summaries_secs, # 每次summary时间间隔
saver=saver, # tf.train.Saver节点
save_interval_secs=FLAGS.save_interval_secs,
session_config=config, # sess参数
sync_optimizer=None)
def main(_):
if FLAGS.train_on_cpu:
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
else:
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu_device
if not FLAGS.dataset_dir:
raise ValueError(
"You must supply the dataset directory with --dataset-dir.")
tf.logging.set_verbosity(tf.logging.DEBUG)
g = tf.Graph()
with g.as_default():
# select the dataset
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# create global step, used for optimizer moving average decay
with tf.device("/cpu:0"):
global_step = tf.train.create_global_step()
# pdb.set_trace()
# get the ssd network and its anchors
ssd_cls = ssd.SSDnet
ssd_params = ssd_cls.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_cls(ssd_params)
image_size = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(img_shape=image_size)
# select the preprocessing function
preprocessing_name = FLAGS.preprocessing_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
# create a dataset provider and batches.
with tf.device("/cpu:0"):
with tf.name_scope(FLAGS.dataset_name + "_data_provider"):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# get for ssd network: image,labels,bboxes
[image, shape, glabels, gbboxes] = provider.get(
["image", "shape", "object/label", "object/bbox"])
# pdb.set_trace()
# preprocessing
image,glabels,gbboxes = \
image_preprocessing_fn(image,
glabels,gbboxes,
out_shape=image_size,
data_format="NHWC")
# encode groundtruth labels and bboxes
gclasses,glocalisations,gscores= \
ssd_net.bboxes_encode(glabels,gbboxes,ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# training batches and queue
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image,b_gclasses,b_glocalisations,b_gscores = \
tf_utils.reshape_list(r,batch_shape)
# prefetch queue
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list(
[b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=8)
# dequeue batch
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# gather initial summaries
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay)
with slim.arg_scope(arg_scope):
predictions,localisations,logits,end_points,mobilenet_var_list = \
ssd_net.net(b_image,is_training=True)
# add loss function
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# add summaries for end_points
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram("activations/" + end_point, x))
summaries.add(
tf.summary.scalar("sparsity/" + end_point,
tf.nn.zero_fraction(x)))
# add summaries for losses and extra losses
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection("EXTRA_LOSSES"):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# add summaries for variables
for var in slim.get_model_variables():
summaries.add(tf.summary.histogram(var.op.name, var))
# configure the moving averages
if FLAGS.moving_average_decay: # use moving average decay on weights variables
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# configure the optimization procedure
with tf.device("/cpu:0"):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar("learning_rate", learning_rate))
if FLAGS.moving_average_decay:
# update ops executed by trainer
update_ops.append(
variable_averages.apply(moving_average_variables))
# get variables to train
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# return a train tensor and summary op
total_losses = tf.get_collection(tf.GraphKeys.LOSSES)
total_loss = tf.add_n(total_losses, name="total_loss")
summaries.add(tf.summary.scalar("total_loss", total_loss))
# create gradient updates
grads = optimizer.compute_gradients(total_loss,
var_list=variables_to_train)
grad_updates = optimizer.apply_gradients(grads,
global_step=global_step)
update_ops.append(grad_updates)
# create train op
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name="train_op")
# merge all summaries together
summary_op = tf.summary.merge(list(summaries), name="summary_op")
# start training
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction,
allow_growth=FLAGS.allow_growth)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=2,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
# create initial assignment op
init_assign_op, init_feed_dict = slim.assign_from_checkpoint(
FLAGS.checkpoint_path,
mobilenet_var_list,
ignore_missing_vars=FLAGS.ignore_missing_vars)
# create an initial assignment function
for k, v in init_feed_dict.items():
if "global_step" in k.name:
g_step = k
init_feed_dict[g_step] = 0 # change the global_step to zero.
init_fn = lambda sess: sess.run(init_assign_op, init_feed_dict)
# run training
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
init_fn=init_fn,
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
saver=saver,
)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Create global_step.
# with tf.device('/gpu:0'):
global_step = slim.create_global_step()
# ckpt = tf.train.get_checkpoint_state(os.path.dirname('./logs/checkpoint'))
#os.path.dirname('./logs/')
ckpt_filename = os.path.dirname(
'./logs/') + '/mobilenet_v1_1.0_224.ckpt'
sess = tf.InteractiveSession()
# Select the dataset.
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
dataset_kitti = dataset_factory.get_dataset('kitti',
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
[image, shape, glabels, gbboxes
] = provider.get(['image', 'shape', 'object/label', 'object/bbox'])
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes, out_shape = ssd_shape, data_format = DATA_FORMAT)
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
summaries.add(tf.summary.image("imgs", tf.cast(b_image, tf.float32)))
f_i = 0
for gt_map in b_gscores:
gt_features = tf.reduce_max(gt_map, axis=3)
gt_features = tf.expand_dims(gt_features, -1)
summaries.add(
tf.summary.image("gt_map_%d" % f_i,
tf.cast(gt_features, tf.float32)))
f_i += 1
# for festures in gt_list:
# summaries.add(tf.summary.image("gt_map_%d" % f_i, tf.cast(festures, tf.float32)))
# f_i += 1
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
f_i = 0
for predict_map in predictions:
predict_map = predict_map[:, :, :, :, 1:]
predict_map = tf.reduce_max(predict_map, axis=4)
predict_map = tf.reduce_max(predict_map, axis=3)
predict_map = tf.expand_dims(predict_map, -1)
summaries.add(
tf.summary.image("predicte_map_%d" % f_i,
tf.cast(predict_map, tf.float32)))
f_i += 1
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
0,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
# with tf.name_scope('kitti' + '_data_provider'):
# provider_k = slim.dataset_data_provider.DatasetDataProvider(
# dataset_kitti,
# num_readers = FLAGS.num_readers,
# common_queue_capacity = 20 * FLAGS.batch_size,
# common_queue_min = 10 * FLAGS.batch_size,
# shuffle = True
# )
# [image_k, shape_k, glabels_k, gbboxes_k] = provider_k.get(['image', 'shape', 'object/label', 'object/bbox'])
#
# image_preprocessing_fn_k = preprocessing_factory.get_preprocessing('kitti', is_training=True)
# image_k, glabels_k, gbboxes_k = \
# image_preprocessing_fn_k(image_k, glabels_k, gbboxes_k, out_shape = ssd_shape, data_format = DATA_FORMAT)
#
# gclasses_k, glocalisations_k, gscores_k = \
# ssd_net.bboxes_encode(glabels_k, gbboxes_k, ssd_anchors)
# #batch_shape = [1] + [len(ssd_anchors)] * 3
#
# r_k = tf.train.batch(
# tf_utils.reshape_list([image_k, gclasses_k, glocalisations_k, gscores_k]),
# batch_size=FLAGS.batch_size,
# num_threads=FLAGS.num_preprocessing_threads,
# capacity= 5 * FLAGS.batch_size
# )
#
# b_image_k, b_gclasses_k, b_glocalisations_k, b_gscores_k = \
# tf_utils.reshape_list(r_k, batch_shape)
#
# summaries.add(tf.summary.image("k_imgs", tf.cast(b_image_k, tf.float32)))
#
# f_i = 0
# for gt_map in b_gscores_k:
# gt_features = tf.reduce_max(gt_map, axis=3)
# gt_features = tf.expand_dims(gt_features, -1)
# summaries.add(tf.summary.image("k_gt_map_%d" % f_i, tf.cast(gt_features, tf.float32)))
# f_i += 1
# # for festures in gt_list:
# # summaries.add(tf.summary.image("gt_map_%d" % f_i, tf.cast(festures, tf.float32)))
# # f_i += 1
#
# arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay, data_format=DATA_FORMAT)
# with slim.arg_scope(arg_scope):
# predictions_k, localisations_k, logits_k, end_points_k = \
# ssd_net.net(b_image_k, is_training=True, reuse=True)
#
# f_i = 0
# for predict_map in predictions_k:
# predict_map = predict_map[:, :, :, :, 1:]
# predict_map = tf.reduce_max(predict_map, axis=4)
# predict_map = tf.reduce_max(predict_map, axis=3)
# predict_map = tf.expand_dims(predict_map, -1)
# summaries.add(tf.summary.image("k_predicte_map_%d" % f_i, tf.cast(predict_map, tf.float32)))
# f_i += 1
#
# ssd_net.losses(logits_k, localisations_k, b_gclasses_k, b_glocalisations_k, b_gscores_k, 2,
# match_threshold=FLAGS.match_threshold,
# negative_ratio=FLAGS.negative_ratio,
# alpha=FLAGS.loss_alpha,
# label_smoothing=FLAGS.label_smoothing)
#total_loss = slim.losses.get_total_loss()
total_loss = tf.losses.get_total_loss()
summaries.add(tf.summary.scalar('loss', total_loss))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# for variable in slim.get_model_variables():
# summaries.add(tf.summary.histogram(variable.op.name, variable))
for variable in tf.trainable_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
# optimizer = tf.train.AdamOptimizer(learning_rate, beta1=FLAGS.adam_beta1,
# beta2=FLAGS.adam_beta2, epsilon=FLAGS.opt_epsilon)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = slim.learning.create_train_op(total_loss,
optimizer,
summarize_gradients=False)
summary_op = tf.summary.merge(list(summaries), name='summary_op')
train_writer = tf.summary.FileWriter('./logs/', sess.graph)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
#variables_to_exclude = slim.get_variables_by_suffix("Adam")
variables_to_restore = slim.get_variables_to_restore(
exclude=["MobilenetV1/Logits", "MobilenetV1/Box", "global_step"])
restorer = tf.train.Saver(variables_to_restore)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
sess.run(tf.global_variables_initializer())
restorer.restore(sess, ckpt_filename)
# if ckpt and ckpt.model_checkpoint_path:
# saver.restore(sess, ckpt.model_checkpoint_path)
i = 0
with slim.queues.QueueRunners(sess):
while (i < FLAGS.max_number_of_steps):
_, summary_str = sess.run([train_op, summary_op])
if i % 50 == 0:
global_step_str = global_step.eval()
print('%diteraton' % (global_step_str))
train_writer.add_summary(summary_str, global_step_str)
if i % 100 == 0:
global_step_str = global_step.eval()
saver.save(sess, "./logs/", global_step=global_step_str)
i += 1
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Create global_step.
global_step = slim.create_global_step()
sess = tf.InteractiveSession()
# Select the dataset.
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes
] = provider.get(['image', 'shape', 'object/label', 'object/bbox'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
summaries.add(
tf.summary.image("input_image", tf.cast(b_image, tf.float32)))
f_i = 0
for gt_map in b_gscores:
gt_features = tf.reduce_max(gt_map, axis=3)
gt_features = tf.expand_dims(gt_features, -1)
summaries.add(
tf.summary.image("gt_map_%d" % f_i,
tf.cast(gt_features, tf.float32)))
f_i += 1
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
f_i = 0
for predict_map in predictions:
predict_map = predict_map[:, :, :, :, 1:]
predict_map = tf.reduce_max(predict_map, axis=4)
predict_map = tf.reduce_max(predict_map, axis=3)
predict_map = tf.expand_dims(predict_map, -1)
summaries.add(
tf.summary.image("predicte_map_%d" % f_i,
tf.cast(predict_map, tf.float32)))
f_i += 1
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
total_loss = tf.losses.get_total_loss()
summaries.add(tf.summary.scalar('loss', total_loss))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for variable in tf.trainable_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
with tf.name_scope('Optimizer'):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = slim.learning.create_train_op(total_loss,
optimizer,
summarize_gradients=False)
summary_op = tf.summary.merge(list(summaries), name='summary_op')
train_writer = tf.summary.FileWriter('./logs/', sess.graph)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
variables_to_train = tf.trainable_variables()
variables_to_restore = \
tf.contrib.framework.filter_variables(variables_to_train,
exclude_patterns=['_box'])
restorer = tf.train.Saver(variables_to_restore)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
sess.run(tf.global_variables_initializer())
restorer.restore(sess, FLAGS.checkpoint_path)
# get_init_fn
i = 0
with slim.queues.QueueRunners(sess):
while (i < FLAGS.max_number_of_steps):
_, summary_str = sess.run([train_op, summary_op])
if i % 50 == 0:
global_step_str = global_step.eval()
print('%diteraton' % (global_step_str))
train_writer.add_summary(summary_str, global_step_str)
if i % 100 == 0:
global_step_str = global_step.eval()
saver.save(sess, "./logs/", global_step=global_step_str)
i += 1
def main():
# 打印tf日志
tf.logging.set_verbosity(tf.logging.DEBUG)
# 建立生成默认计算图上下文
with tf.Graph().as_default():
# 选择相应的数据集
dataset = dataset_factory.get_dataset(config.dataset_name,
config.dataset_split_name,
config.dataset_dir)
data_provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=config.num_readers,
common_queue_capacity=20 * config.batch_size,
common_queue_min=10 * config.batch_size,
shuffle=True)
image_raw, label = data_provider.get(['image', 'label'])
# 选择模型
model = nets_factory.get_network_fn(
config.model_name,
7,
is_training=True,
dropout_keep_prob=config.dropout_keep_prob)
# 选择预处理器
preprocessing_name = config.preprocessing_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
with tf.device('/gpu:0'):
#建立全局步数
global_step = slim.create_global_step()
# 预处理
image = image_preprocessing_fn(image_raw, 299, 299)
# 喂入数据
images, images_raws, labels = tf.train.batch(
[image, image_raw, label],
batch_size=config.batch_size,
num_threads=config.num_preprocessing_threads,
capacity=5 * config.batch_size)
# 加载模型
logits, end_points = model(images,
num_classes=dataset.num_classes,
is_training=True)
# 打印模型信息
for k, v in end_points.items():
print('name = {}, shape = {}'.format(v.name, v.get_shape()))
# 打印模型参数
print("\n")
print("Parameters")
for v in slim.get_model_variables():
print('name = {}, shape = {}'.format(v.name, v.get_shape()))
# 损失函数
slim.losses.softmax_cross_entropy(logits, labels)
total_loss = slim.losses.get_total_loss()
# 收集摘要
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
for end_point in end_points:
x = end_points[end_point]
summaries.add(
tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
summaries.add(tf.summary.scalar('total_loss', total_loss))
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# 优化器
learning_rate = tf_utils.configure_learning_rate(
config, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(config, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
# 摘要合并.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# 训练步骤
train_op = slim.learning.create_train_op(total_loss, optimizer)
# gpu配置
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=config.gpu_memory_fraction)
gpu_config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True,
gpu_options=gpu_options)
# 断点续训，存储器
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
# 训练
final_loss = slim.learning.train(
train_op,
logdir=config.train_dir,
init_fn=tf_utils.get_init_fn(config),
summary_op=summary_op,
number_of_steps=config.max_number_of_steps,
log_every_n_steps=config.log_every_n_steps,
save_summaries_secs=config.save_summaries_secs,
saver=saver,
save_interval_secs=config.save_interval_secs,
session_config=gpu_config,
sync_optimizer=None)
print("Finished training. Last batch loss:", final_loss)
print("Checkpoint saved in %s" % config.train_dir)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Config model_deploy. Keep TF Slim Models structure.
# Useful if want to need multiple GPUs and/or servers in the future.
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
# Create global_step.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device(deploy_config.inputs_device()):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes] = provider.get(['image', 'shape',
'object/label',
'object/bbox'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(
tf_utils.reshape_list([image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# Intermediate queueing: unique batch computation pipeline for all
# GPUs running the training.
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list([b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=2 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple
clones of network_fn."""
# Dequeue batch.
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# Construct SSD network.
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
# Add loss function.
ssd_net.losses(logits, localisations,
b_gclasses, b_glocalisations, b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# =================================================================== #
# Add summaries from first clone.
# =================================================================== #
clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses and extra losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(FLAGS,
dataset.num_samples,
global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones,
optimizer,
var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
global_step = slim.create_global_step()
# Get the SSD network and its anchors.
#ssd_params = ssd_class.default_params._replace(num_classes=FLAGS.num_classes)
ssd_net = ssd_vgg_300.SSDNet()
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
b_image, b_gclasses, b_glocalisations, b_gscores = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
ssd_shape,
ssd_net,
ssd_anchors,
FLAGS.num_preprocessing_threads,
is_training = True)
with tf.device(FLAGS.gpu_train):
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
# Add loss function.
total_loss = ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
for loss in tf.get_collection('EXTRA_LOSSES'):
summaries.add(tf.summary.scalar(loss.op.name, loss))
with tf.device(FLAGS.gpu_train):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
# Configure the optimization procedure
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
## Training
train_op = slim.learning.create_train_op(total_loss, optimizer)
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True)
saver = tf.train.Saver(max_to_keep=1,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_op,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
##logging tools
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Create global_step.
with tf.device('/cpu:0'):
global_step = tf.train.create_global_step()
# Select the dataset.
dataset = FLAGS.dataset_dir + '%s_%s.tfrecord' %(FLAGS.dataset_name,FLAGS.dataset_split_name)
#image = tf.placeholder(tf.float32, shape=[62,62])
#label = tf.placeholder(tf.float32, shape=[62,62])
with tf.device('/cpu:0'):
with tf.name_scope('input'):
filename_queue = tf.train.string_input_producer([dataset], num_epochs=FLAGS.num_epochs)
image,mask_class,_ = tf_utils.read_and_decode_for_lstm(filename_queue, batch_size = FLAGS.batch_size,\
capacity=20 * FLAGS.batch_size,\
num_threads=FLAGS.num_readers,\
min_after_dequeue=10 * FLAGS.batch_size, is_training=True)
#image = tf.image.resize_images(image,[256,256])
#labels = tf.expand_dims(mask_class, axis=-1)
#labels = tf.image.resize_nearest_neighbor(labels,[256,256])
# reshape mask_class as [-1, num_classes]
labels = tf.to_float(tf.contrib.layers.one_hot_encoding(mask_class,FLAGS.num_classes))
mask_class_onehot_for_d = tf.to_float(tf.contrib.layers.one_hot_encoding(mask_class,FLAGS.num_classes, on_value=0.99,off_value=0.01))
labels = tf.reshape(labels, (-1, FLAGS.num_classes))
#input_d = tf.reshape(mask_class_onehot_for_d, (-1, FLAGS.num_classes))
# onedim_class = tf.reshape(mask_class, (-1,))
#image = preprocessing.data_augmentation(image, is_training=True)
#image,mask_class,mask_instance = preprocessing(image,mask_class,mask_instance,is_train=True,data_format= 'NHWC')
logits,_,_= SpinePathNet.g_l_net(image, batch_size=FLAGS.batch_size, class_num=FLAGS.num_classes, reuse=False, is_training=True, scope='g_SpinePathNet')
#Gan simultanously descriminate the input is the segmentation predictions or ground truth.
D_logit_real = SpinePathNet.d_net(mask_class_onehot_for_d, class_num = FLAGS.num_classes, reuse=None, is_training=True, scope='d_gan')
D_logit_fake = SpinePathNet.d_net(logits, class_num = FLAGS.num_classes, reuse=True, is_training=True, scope='d_gan')
with tf.name_scope('cross_entropy_loss'):
cross_entropy_loss = losses.weighted_cross_entropy_with_logits(FLAGS, Fold, logits, labels)
with tf.name_scope('gan_loss'): # GAN.
D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))
#Accor
D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))
G_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))
#lmada = tf.Variable(2., name = 'weight_of_gan')
D_loss = D_loss_real + D_loss_fake
G_loss = cross_entropy_loss + G_loss_fake
# trainable varibles of generative and discrimative models.
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'd_' in var.name]
g_vars = [var for var in t_vars if 'g_' in var.name]
# print (g_vars)
learning_rate = tf_utils.configure_learning_rate(FLAGS,FLAGS.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
optimizer_gan = tf.train.AdamOptimizer(beta1=0.5, learning_rate=0.001)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
#Note: when training, the moving_mean and moving_variance need to be updated.
with tf.control_dependencies(update_ops):
train_op_G = optimizer.minimize(G_loss, global_step=global_step, var_list=g_vars)
#train_op_fake = optimizer.minimize(G_loss_fake, var_list=g_vars)
train_op_D = optimizer_gan.minimize(D_loss, global_step=global_step, var_list=d_vars)
# The op for initializing the variables.
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
# Create a session for running operations in the Graph.
#config = tf.ConfigProto(allow_soft_placement = True)
sess = tf.Session()
# Initialize the variables (the trained variables and the
# epoch counter).
sess.run(init_op)
#Include max_to_keep=5
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
exclude_list = FLAGS.ignore_missing_vars
variables_list = tf.contrib.framework.get_variables_to_restore(exclude=exclude_list)
restore = tf.train.Saver(variables_list)
restore.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
global_step = int(global_step)
else: global_step = 0
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# Save models.
if not tf.gfile.Exists(FLAGS.train_dir):
#tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
try:
step = global_step # TODO: Continue to train the model, if the checkpoints are exist.
while not coord.should_stop():
start_time = time.time()
#for i in xrange(50):
_, dl, dlr, dlf = sess.run([train_op_D, D_loss, D_loss_real, D_loss_fake])
#cl = sess.run(cross_entropy_loss)
#print('Step %d: cross entropy loss = %.2f, real discrimator loss = %.2f, fake discrimator loss = %.2f' % (step, cl, dlr,dlf))
#step += 50
#for i in xrange(50):
#
_,gl, cel, fake_loss, lr = sess.run([train_op_G, G_loss, cross_entropy_loss, G_loss_fake, learning_rate])
duration = time.time() - start_time
if step % 10 == 0:
print('Step %d: All generative loss = %.2f (Cross entropy loss = %.2f, Fake loss of generatation = %.2f); All discrimator loss = %.2f (Discrimator loss of real = %.2f, Discrimator loss of fake = %.2f); Learning rate = %.4f (%.3f sec)' % (step, gl, cel, fake_loss, dl, dlr, dlf, lr, duration))
step += 1
if step % 1000 == 0: #or (step + 1) == FLAGS.max_steps
#Increase Gan_loss.
#lmada.assign_add(0.1)
saver.save(sess, checkpoint_path, global_step=step)
##Add the model evaluatation in the future.
except tf.errors.OutOfRangeError:
print('Done training for %d epochs, %d steps.' % (FLAGS.num_epochs, step))
saver.save(sess, checkpoint_path, global_step=step)
##Add the model evaluatation in the future.
print('Model is saved as %s') % checkpoint_path
finally:
# When done, ask the threads to stop.
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
sess.close()
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# initalize the net
net = txtbox_300.TextboxNet()
out_shape = net.params.img_shape
anchors = net.anchors(out_shape)
# Create global_step.
global_step = slim.create_global_step()
# create batch dataset
with tf.device(FLAGS.gpu_data):
b_image, b_glocalisations, b_gscores = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS.num_preprocessing_threads,
is_training = True)
with tf.device(FLAGS.gpu_train):
arg_scope = net.arg_scope(weight_decay=FLAGS.weight_decay)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=True)
# Add loss function.
total_loss = net.losses(logits,
localisations,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
# Gather summaries.
for end_point in end_points:
x = end_points[end_point]
tf.summary.histogram('activations/' + end_point, x)
tf.summary.scalar('sparsity/' + end_point, tf.nn.zero_fraction(x))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
tf.summary.scalar(loss.op.name, loss)
for loss in tf.get_collection('EXTRA_LOSSES'):
tf.summary.scalar(loss.op.name, loss)
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
with tf.device(FLAGS.gpu_train):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
# Configure the optimization procedure
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
tf.summary.scalar('learning_rate', learning_rate)
## Training
train_op = slim.learning.create_train_op(total_loss, optimizer)
merged = tf.summary.merge_all()
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = FLAGS.train_dir
checkpoint_prefix = os.path.join(checkpoint_dir, "model.ckpt")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)
saver = tf.train.Saver(max_to_keep=1,
keep_checkpoint_every_n_hours=1.0,
pad_step_number=False)
path = tf.train.latest_checkpoint(FLAGS.train_dir)
if path:
saver.restore(sess, path)
print sess.run([global_step])
with slim.queues.QueueRunners(sess):
for i in xrange(FLAGS.max_number_of_steps):
loss, _ , summary_, global_step_= \
sess.run([total_loss,train_op,merged,global_step])
current_step = tf.train.global_step(sess, global_step)
if i % 10 == 0:
print loss
if global_step_ % 2 == 0:
train_writer.add_summary(summary_, global_step_)
if global_step_ % 100 == 0:
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
def main(_):
if not FLAGS.data_dir:
raise ValueError('You must supply the dataset directory with --data_dir')
num_gpus = FLAGS.num_gpus
if num_gpus < 1: num_gpus = 1
# ps_spec = FLAGS.ps_hosts.split(",")
# worker_spec = FLAGS.worker_hosts.split(",")
# num_workers = len(worker_spec)
# cluster = tf.train.ClusterSpec({
# "ps": ps_spec,
# "worker": worker_spec})
# server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index)
# if FLAGS.job_name == "ps":
# with tf.device("/cpu:0"):
# server.join()
# return
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.device('/cpu:0'):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.data_dir)
# Get the RON network and its anchors.
ron_class = nets_factory.get_network(FLAGS.model_name)
ron_params = ron_class.default_params._replace(num_classes=FLAGS.num_classes)
ron_net = ron_class(ron_params)
ron_shape = ron_net.params.img_shape
ron_anchors = ron_net.anchors(ron_shape)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=120 * FLAGS.batch_size * num_gpus,
common_queue_min=80 * FLAGS.batch_size * num_gpus,
shuffle=True)
# Get for RON network: image, labels, bboxes.
# (ymin, xmin, ymax, xmax) fro gbboxes
[image, shape, glabels, gbboxes, isdifficult] = provider.get(['image', 'shape',
'object/label',
'object/bbox',
'object/difficult'])
isdifficult_mask =tf.cond(tf.reduce_sum(tf.cast(tf.logical_not(tf.equal(tf.ones_like(isdifficult), isdifficult)), tf.float32)) < 1., lambda : tf.one_hot(0, tf.shape(isdifficult)[0], on_value=True, off_value=False, dtype=tf.bool), lambda : isdifficult < tf.ones_like(isdifficult))
glabels = tf.boolean_mask(glabels, isdifficult_mask)
gbboxes = tf.boolean_mask(gbboxes, isdifficult_mask)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ron_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
# glocalisations is our regression object
# gclasses is the ground_trutuh label
# gscores is the the jaccard score with ground_truth
gclasses, glocalisations, gscores = \
ron_net.bboxes_encode(glabels, gbboxes, ron_anchors, positive_threshold=FLAGS.match_threshold, ignore_threshold=FLAGS.neg_threshold)
# each size of the batch elements
# include one image, three others(gclasses, glocalisations, gscores)
batch_shape = [1] + [len(ron_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(
tf_utils.reshape_list([image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size * num_gpus,
num_threads=FLAGS.num_preprocessing_threads,
capacity=120 * FLAGS.batch_size * num_gpus)
all_batch = tf_utils.reshape_list(r, batch_shape)
b_image = tf.split(all_batch[0], num_or_size_splits=num_gpus, axis=0)
_b_gclasses = [tf.split(b, num_or_size_splits=num_gpus, axis=0) for b in all_batch[1]]
b_gclasses = [_ for _ in zip(*_b_gclasses)]
_b_glocalisations = [tf.split(b, num_or_size_splits=num_gpus, axis=0) for b in all_batch[2]]
b_glocalisations = [_ for _ in zip(*_b_glocalisations)]
_b_gscores = [tf.split(b, num_or_size_splits=num_gpus, axis=0) for b in all_batch[3]]
b_gscores = [_ for _ in zip(*_b_gscores)]
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
learning_rate = tf_utils.configure_learning_rate(FLAGS,
dataset.num_samples,
global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
# Construct RON network.
arg_scope = ron_net.arg_scope(weight_decay=FLAGS.weight_decay, data_format=DATA_FORMAT)
reuse_variables = False
tower_grads = []
loss_list = []
with slim.arg_scope(arg_scope):
for index in range(num_gpus):
with tf.device('/gpu:%d' % index):
predictions, logits, objness_pred, objness_logits, localisations, end_points = ron_net.net(b_image[index], is_training=True, reuse = reuse_variables)
# Add loss function.
ron_net.losses(logits, localisations, objness_logits, objness_pred,
b_gclasses[index], b_glocalisations[index], b_gscores[index],
match_threshold = FLAGS.match_threshold,
neg_threshold = FLAGS.neg_threshold,
objness_threshold = FLAGS.objectness_thres,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
beta=FLAGS.loss_beta,
label_smoothing=FLAGS.label_smoothing)
reuse_variables = True
# and returns a train_tensor and summary_op
loss = tf.losses.get_total_loss()
loss_list.append(loss)
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# Create gradient updates.
grads = optimizer.compute_gradients(loss, variables_to_train)
tower_grads.append(grads)
reduce_grads = average_gradients(tower_grads)
total_loss = tf.reduce_mean(tf.stack(loss_list, axis=0), axis=0)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
grad_updates = optimizer.apply_gradients(reduce_grads, global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss, name='train_op')
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours = FLAGS.save_interval_secs/3600.,
write_version=2,
pad_step_number=False)
slim.learning.train(
train_tensor,
logdir=FLAGS.model_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS, os.path.join(FLAGS.data_dir, 'vgg_16.ckpt')),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
session_wrapper=None,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
######################
# Config model_deploy#
######################
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.worker_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
network_fn = nets_factory.get_network(FLAGS.model_name)
params = network_fn.default_params
params = params._replace(match_threshold=FLAGS.match_threshold)
# initalize the net
net = network_fn(params)
out_shape = net.params.img_shape
anchors = net.anchors(out_shape)
# create batch dataset
with tf.device(deploy_config.inputs_device()):
b_image, b_glocalisations, b_gscores = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
allgscores = []
allglocalization = []
for i in range(len(anchors)):
allgscores.append(tf.reshape(b_gscores[i], [-1]))
allglocalization.append(
tf.reshape(b_glocalisations[i], [-1, 4]))
b_gscores = tf.concat(allgscores, 0)
b_glocalisations = tf.concat(allglocalization, 0)
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list([b_image, b_glocalisations, b_gscores]),
num_threads=8,
capacity=16 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
#Allows data parallelism by creating multiple
#clones of network_fn.
# Dequeue batch.
batch_shape = [1] * 3
b_image, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# Construct SSD network.
arg_scope = net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=FLAGS.data_format)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=True, use_batch=FLAGS.use_batch)
# Add loss function.
net.losses(logits,
localisations,
b_glocalisations,
b_gscores,
negative_ratio=FLAGS.negative_ratio,
use_hard_neg=FLAGS.use_hard_neg,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
#########################################
# Configure the optimization procedure. #
#########################################
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.fine_tune:
gradient_multipliers = pickle.load(
open('nets/multiplier_300.pkl', 'rb'))
else:
gradient_multipliers = None
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
#train_tensor = slim.learning.create_train_op(total_loss, optimizer, gradient_multipliers=gradient_multipliers)
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction,
allocator_type="BFC")
config = tf.ConfigProto(
gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True,
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=1,
)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.worker_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = tf.train.get_or_create_global_step()
net = model_cmc.Model()
with tf.device(deploy_config.inputs_device()):
if (FLAGS.model == 'unet'):
batch_queue = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
None,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
elif (FLAGS.model == 'patch'):
batch_queue = \
load_batch_patch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
None,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
elif (FLAGS.model == 'cmc'):
batch_queue = \
load_batch_cmc.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
None,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
print("Batch_loading_successful")
def clone_fn(batch_queue):
batch_shape = [1] * 3
b_image, label = batch_queue
logits, end_points = net.net(b_image)
# Add loss function.
loss, mean_iou = net.weighted_losses(logits, label)
return end_points, mean_iou
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
end_points, mean_iou = clones[0].outputs
update_ops.append(mean_iou[1])
#for end_point in end_points:
# x = end_points[end_point]
# summaries.add(tf.summary.histogram('activations/' + end_point, x))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
#
#for variable in slim.get_model_variables():
# summaries.add(tf.summary.histogram(variable.op.name, variable))
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
#########################################
# Configure the optimization procedure. #
#########################################
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.fine_tune:
gradient_multipliers = pickle.load(
open('nets/multiplier_300.pkl', 'rb'))
else:
gradient_multipliers = None
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
if gradient_multipliers:
with ops.name_scope('multiply_grads'):
clones_gradients = slim.learning.multiply_gradients(
clones_gradients, gradient_multipliers)
if FLAGS.clip_gradient_norm > 0:
with ops.name_scope('clip_grads'):
clones_gradients = slim.learning.clip_gradient_norms(
clones_gradients, FLAGS.clip_gradient_norm)
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
#train_tensor = slim.learning.create_train_op(total_loss, optimizer, gradient_multipliers=gradient_multipliers)
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
def train_step_fn(session, *args, **kwargs):
# visualizer = Beholder(session=session, logdir=FLAGS.train_dir)
total_loss, should_stop = train_step(session, *args, **kwargs)
if train_step_fn.step % FLAGS.validation_check == 0:
_mean_iou = session.run(train_step_fn.mean_iou)
print('evaluation step %d - loss = %.4f mean_iou = %.2f%%' %\
(train_step_fn.step, total_loss, _mean_iou ))
# evaluated_tensors = session.run([end_points['conv4'], end_points['up1']])
# example_frame = session.run(end_points['up2'])
# visualizer.update(arrays=evaluated_tensors, frame=example_frame)
train_step_fn.step += 1
return [total_loss, should_stop]
train_step_fn.step = 0
train_step_fn.end_points = end_points
train_step_fn.mean_iou = mean_iou[0]
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction,
allocator_type="BFC")
config = tf.ConfigProto(
gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True,
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=1,
)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
train_step_fn=train_step_fn,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Config model_deploy. Keep TF Slim Models structure.
# Useful if want to need multiple GPUs and/or servers in the future.
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
# Create global_step.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device(deploy_config.inputs_device()):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes] = provider.get(
['image', 'shape', 'object/label', 'object/bbox'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# Intermediate queueing: unique batch computation pipeline for all
# GPUs running the training.
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list(
[b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=2 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple
clones of network_fn."""
# Dequeue batch.
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# Construct SSD network.
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
# Add loss function.
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# =================================================================== #
# Add summaries from first clone.
# =================================================================== #
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses and extra losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
network_fn = nets_factory.get_network(FLAGS.model_name)
params = network_fn.default_params
params = params._replace(match_threshold=FLAGS.match_threshold)
# initalize the net
net = network_fn(params)
out_shape = net.params.img_shape
anchors = net.anchors(out_shape)
# Create global_step.
global_step = slim.create_global_step()
# create batch dataset
b_image, b_glocalisations, b_gscores = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
with tf.device(FLAGS.gpu_train):
#with tf.device(FLAGS.gpu_train):
arg_scope = net.arg_scope(weight_decay=FLAGS.weight_decay)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=True, use_batch=FLAGS.use_batch)
# Add loss function.
total_loss = net.losses(logits,
localisations,
b_glocalisations,
b_gscores,
negative_ratio=FLAGS.negative_ratio,
use_hard_neg=FLAGS.use_hard_neg,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
# Gather summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
'''
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
#for loss in tf.get_collection(tf.GraphKeys.LOSSES):
# summaries.add(tf.summary.scalar(loss.op.name, loss))
'''
for loss in tf.get_collection('EXTRA_LOSSES'):
summaries.add(tf.summary.scalar(loss.op.name, loss))
'''
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
'''
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
with tf.device(FLAGS.gpu_train):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
# Configure the optimization procedure
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
#summaries.add(tf.summary.scalar('learning_rate', learning_rate))
## Training
#loss = tf.get_collection(tf.GraphKeys.LOSSES)
#total_loss = tf.add_n(loss)
'''
if FLAGS.fine_tune:
gradient_multipliers = pickle.load(open('nets/multiplier_300.pkl','rb'))
else:
gradient_multipliers = None
'''
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
vars_grad = optimizer.compute_gradients(total_loss, variables_to_train)
grad_updates = optimizer.apply_gradients(vars_grad,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_op = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
#train_op = slim.learning.create_train_op(total_loss, optimizer, gradient_multipliers=gradient_multipliers)
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_op,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
