def main(_):
# Caffe scope...
caffemodel = caffe_scope.CaffeScope()
caffemodel.load(FLAGS.caffemodel_path)
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
global_step = slim.create_global_step()
num_classes = int(FLAGS.num_classes)
# Select the network.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(num_classes=num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
# Image placeholder and model.
shape = (1, ssd_shape[0], ssd_shape[1], 3)
img_input = tf.placeholder(shape=shape, dtype=tf.float32)
# Create model.
with slim.arg_scope(ssd_net.arg_scope_caffe(caffemodel)):
ssd_net.net(img_input, is_training=False)
init_op = tf.global_variables_initializer()
with tf.Session() as session:
# Run the init operation.
session.run(init_op)
# Save model in checkpoint.
saver = tf.train.Saver()
ckpt_path = FLAGS.caffemodel_path.replace('.caffemodel', '.ckpt')
saver.save(session, ckpt_path, write_meta_graph=False)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Config model_deploy. Keep TF Slim Models structure.
# Useful if want to need multiple GPUs and/or servers in the future.
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
# Create global_step.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
# using if-else is ugly, but this is the only way I can think of
# without changing the original structure.
if FLAGS.model_name == 'modular_ssd':
ssd_net = ssd_class(FLAGS.feature_extractor, FLAGS.model)
ssd_params = ssd_net.params._replace(num_classes=FLAGS.num_classes)
ssd_net.params = ssd_params
else:
ssd_params = ssd_class.default_params._replace(num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device(deploy_config.inputs_device()):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes] = provider.get(['image', 'shape',
'object/label',
'object/bbox'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(
tf_utils.reshape_list([image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# Intermediate queueing: unique batch computation pipeline for all
# GPUs running the training.
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list([b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=2 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple
clones of network_fn."""
# Dequeue batch.
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# Construct SSD network.
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
# Add loss function.
ssd_net.losses(logits, localisations,
b_gclasses, b_glocalisations, b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# =================================================================== #
# Add summaries from first clone.
# =================================================================== #
clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses and extra losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(FLAGS,
dataset.num_samples,
global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones,
optimizer,
var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options,
allow_soft_placement=True)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
# =================================================================== #
# Dataset + SSD model + Pre-processing
# =================================================================== #
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
# Evaluation shape and associated anchors: eval_image_size
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.eval_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
#with tf.device('/cpu:0'):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size,
shuffle=False)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes
] = provider.get(['image', 'shape', 'object/label', 'object/bbox'])
if FLAGS.remove_difficult:
[gdifficults] = provider.get(['object/difficult'])
else:
gdifficults = tf.zeros(tf.shape(glabels), dtype=tf.int64)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes, gbbox_img = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT,
resize=FLAGS.eval_resize,
difficults=None)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] * 5 + [len(ssd_anchors)] * 3
# Evaluation batch.
r = tf.train.batch(tf_utils.reshape_list([
image, glabels, gbboxes, gdifficults, gbbox_img, gclasses,
glocalisations, gscores
]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size,
dynamic_pad=True)
(b_image, b_glabels, b_gbboxes, b_gdifficults, b_gbbox_img, b_gclasses,
b_glocalisations, b_gscores) = tf_utils.reshape_list(r, batch_shape)
# =================================================================== #
# SSD Network + Ouputs decoding.
# =================================================================== #
dict_metrics = {}
arg_scope = ssd_net.arg_scope(data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=False)
# Add losses functions.
ssd_net.losses(logits, localisations, b_gclasses, b_glocalisations,
b_gscores)
# Performing post-processing on CPU: loop-intensive, usually more efficient.
with tf.device('/device:CPU:0'):
# Detected objects from SSD output.
localisations = ssd_net.bboxes_decode(localisations, ssd_anchors)
rscores, rbboxes = \
ssd_net.detected_bboxes(predictions, localisations,
select_threshold=FLAGS.select_threshold,
nms_threshold=FLAGS.nms_threshold,
clipping_bbox=None,
top_k=FLAGS.select_top_k,
keep_top_k=FLAGS.keep_top_k)
# Compute TP and FP statistics.
num_gbboxes, tp, fp, rscores = \
tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
b_glabels, b_gbboxes, b_gdifficults,
matching_threshold=FLAGS.matching_threshold)
# Variables to restore: moving avg. or normal weights.
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
# =================================================================== #
# Evaluation metrics.
# =================================================================== #
with tf.device('/device:CPU:0'):
dict_metrics = {}
# First add all losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Extra losses as well.
for loss in tf.get_collection('EXTRA_LOSSES'):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Add metrics to summaries and Print on screen.
for name, metric in dict_metrics.items():
# summary_name = 'eval/%s' % name
summary_name = name
op = tf.summary.scalar(summary_name, metric[0], collections=[])
# op = tf.Print(op, [metric[0]], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# FP and TP metrics.
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp,
rscores)
for c in tp_fp_metric[0].keys():
dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c],
tp_fp_metric[1][c])
# Add to summaries precision/recall values.
aps_voc07 = {}
aps_voc12 = {}
for c in tp_fp_metric[0].keys():
# Precison and recall values.
prec, rec = tfe.precision_recall(*tp_fp_metric[0][c])
# Average precision VOC07.
v = tfe.average_precision_voc07(prec, rec)
summary_name = 'AP_VOC07/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc07[c] = v
# Average precision VOC12.
v = tfe.average_precision_voc12(prec, rec)
summary_name = 'AP_VOC12/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
aps_voc12[c] = v
# Mean average precision VOC07.
summary_name = 'AP_VOC07/mAP'
mAP = tf.add_n(list(aps_voc07.values())) / len(aps_voc07)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Mean average precision VOC12.
summary_name = 'AP_VOC12/mAP'
mAP = tf.add_n(list(aps_voc12.values())) / len(aps_voc12)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# for i, v in enumerate(l_precisions):
# summary_name = 'eval/precision_at_recall_%.2f' % LIST_RECALLS[i]
# op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
# tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Split into values and updates ops.
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(
dict_metrics)
# =================================================================== #
# Evaluation loop.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
# config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
# Number of batches...
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
num_batches = math.ceil(dataset.num_samples /
float(FLAGS.batch_size))
if not FLAGS.wait_for_checkpoints:
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(
FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Standard evaluation loop.
start = time.time()
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
session_config=config)
# Log time spent.
elapsed = time.time()
elapsed = elapsed - start
print('Time spent : %.3f seconds.' % elapsed)
print('Time spent per BATCH: %.3f seconds.' %
(elapsed / num_batches))
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Waiting loop.
slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
eval_interval_secs=10,
max_number_of_evaluations=np.inf,
session_config=config,
timeout=None)
with tf.Session() as sess:
net_shape = (300, 300)
data_format = 'NHWC'
img_input = tf.placeholder(tf.uint8, shape=(None, None, 3))
image_pre, labels_pre, bboxes_pre, bbox_img = ssd_vgg_preprocessing.preprocess_for_eval(
img_input,
None,
None,
net_shape,
data_format,
resize=ssd_vgg_preprocessing.Resize.WARP_RESIZE)
image_4d = tf.expand_dims(image_pre, 0)
reuse = True if 'ssd_net' in locals() else None
ssd_class = nets_factory.get_network('ssd_300_vgg')
ssd_params = ssd_class.default_params._replace(num_classes=3)
# ssd_net = ssd_vgg_300.SSDNet()
ssd_net = ssd_class(ssd_params)
with slim.arg_scope(ssd_net.arg_scope(data_format=data_format)):
predictions, localisations, _, _ = ssd_net.net(image_4d,
is_training=False,
reuse=reuse)
ckpt_filename = './logs_unt_aerial_dataset_20190103_4_0.3/model.ckpt-21274'
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, ckpt_filename)
ssd_anchors = ssd_net.anchors(net_shape)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
with tf.Session() as sess:
# =========================================================================== #
# Main evaluation flags.
# =========================================================================== #
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
# Evaluation shape and associated anchors: eval_image_size
ssd_shape = ssd_net.params.img_shape
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
# =================================================================== #
# Create a dataset provider.
# =================================================================== #
input_tfrecord_paths = [
v for v in FLAGS.input_tfrecord_paths.split(',') if v
]
filename_queue = tf.train.string_input_producer(
input_tfrecord_paths, shuffle=False, num_epochs=1)
tf_record_reader = tf.TFRecordReader()
_, serialized_example_tensor = tf_record_reader.read(
filename_queue)
feature_map = {"image/encoded": tf.FixedLenFeature([], tf.string)}
features = tf.parse_single_example(serialized_example_tensor,
feature_map)
encoded_image = features["image/encoded"]
image_tensor = tf.image.decode_image(encoded_image, channels=3)
image_tensor.set_shape([None, None, 3])
image_tensor, _1, _2, _3 = \
image_preprocessing_fn(image_tensor, labels=None, bboxes=None,
out_shape=ssd_shape,
data_format=DATA_FORMAT,
resize=FLAGS.eval_resize,
difficults=None)
image_tensor = tf.placeholder(tf.float32,
shape=[1, 300, 300, 3],
name="Placeholder")
image_data = np.ones([1, 300, 300, 3])
# =================================================================== #
# Import SSD Network
# =================================================================== #
with tf.gfile.FastGFile(FLAGS.inference_graph_path,
'rb') as graph_def_file:
graph_content = graph_def_file.read()
graph_def = tf.GraphDef()
graph_def.MergeFromString(graph_content)
tf.import_graph_def(
graph_def,
input_map={'eval_batch:0': tf.to_float(image_tensor)},
name="",
)
graph = tf.get_default_graph()
predictions = [
graph.get_tensor_by_name(
'ssd_300_vgg/softmax/Reshape_1:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/softmax_1/Reshape_1:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/softmax_2/Reshape_1:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/softmax_3/Reshape_1:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/softmax_4/Reshape_1:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/softmax_5/Reshape_1:0')
]
localisations = [
graph.get_tensor_by_name(
'ssd_300_vgg/block4_box/Reshape:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/block7_box/Reshape:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/block8_box/Reshape:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/block9_box/Reshape:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/block10_box/Reshape:0'),
graph.get_tensor_by_name(
'ssd_300_vgg/block11_box/Reshape:0')
]
# =================================================================== #
# Inference loop.
# =================================================================== #
sess.run(tf.local_variables_initializer())
tf.train.start_queue_runners()
start = time.time()
try:
for counter in range(0, 100):
tf.logging.log_every_n(tf.logging.INFO,
'Processed %d images...', 10,
counter)
_pre, _loc = sess.run(
[predictions, localisations],
feed_dict={'Placeholder:0': image_data})
except tf.errors.OutOfRangeError:
tf.logging.info('Finished processing records')
# Log time spent.
elapsed = time.time()
elapsed = elapsed - start
print('Time spent : %.3f seconds.' % elapsed)
print('Time spent per BATCH: %.3f seconds.' % (elapsed / 100))
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Create global_step.
# with tf.device('/gpu:0'):
global_step = slim.create_global_step()
# ckpt = tf.train.get_checkpoint_state(os.path.dirname('./logs/checkpoint'))
#os.path.dirname('./logs/')
ckpt_filename = os.path.dirname(
'./logs/') + '/mobilenet_v1_1.0_224.ckpt'
sess = tf.InteractiveSession()
# Select the dataset.
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
dataset_kitti = dataset_factory.get_dataset('kitti',
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
[image, shape, glabels, gbboxes
] = provider.get(['image', 'shape', 'object/label', 'object/bbox'])
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes, out_shape = ssd_shape, data_format = DATA_FORMAT)
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
summaries.add(tf.summary.image("imgs", tf.cast(b_image, tf.float32)))
f_i = 0
for gt_map in b_gscores:
gt_features = tf.reduce_max(gt_map, axis=3)
gt_features = tf.expand_dims(gt_features, -1)
summaries.add(
tf.summary.image("gt_map_%d" % f_i,
tf.cast(gt_features, tf.float32)))
f_i += 1
# for festures in gt_list:
# summaries.add(tf.summary.image("gt_map_%d" % f_i, tf.cast(festures, tf.float32)))
# f_i += 1
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
f_i = 0
for predict_map in predictions:
predict_map = predict_map[:, :, :, :, 1:]
predict_map = tf.reduce_max(predict_map, axis=4)
predict_map = tf.reduce_max(predict_map, axis=3)
predict_map = tf.expand_dims(predict_map, -1)
summaries.add(
tf.summary.image("predicte_map_%d" % f_i,
tf.cast(predict_map, tf.float32)))
f_i += 1
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
0,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
# with tf.name_scope('kitti' + '_data_provider'):
# provider_k = slim.dataset_data_provider.DatasetDataProvider(
# dataset_kitti,
# num_readers = FLAGS.num_readers,
# common_queue_capacity = 20 * FLAGS.batch_size,
# common_queue_min = 10 * FLAGS.batch_size,
# shuffle = True
# )
# [image_k, shape_k, glabels_k, gbboxes_k] = provider_k.get(['image', 'shape', 'object/label', 'object/bbox'])
#
# image_preprocessing_fn_k = preprocessing_factory.get_preprocessing('kitti', is_training=True)
# image_k, glabels_k, gbboxes_k = \
# image_preprocessing_fn_k(image_k, glabels_k, gbboxes_k, out_shape = ssd_shape, data_format = DATA_FORMAT)
#
# gclasses_k, glocalisations_k, gscores_k = \
# ssd_net.bboxes_encode(glabels_k, gbboxes_k, ssd_anchors)
# #batch_shape = [1] + [len(ssd_anchors)] * 3
#
# r_k = tf.train.batch(
# tf_utils.reshape_list([image_k, gclasses_k, glocalisations_k, gscores_k]),
# batch_size=FLAGS.batch_size,
# num_threads=FLAGS.num_preprocessing_threads,
# capacity= 5 * FLAGS.batch_size
# )
#
# b_image_k, b_gclasses_k, b_glocalisations_k, b_gscores_k = \
# tf_utils.reshape_list(r_k, batch_shape)
#
# summaries.add(tf.summary.image("k_imgs", tf.cast(b_image_k, tf.float32)))
#
# f_i = 0
# for gt_map in b_gscores_k:
# gt_features = tf.reduce_max(gt_map, axis=3)
# gt_features = tf.expand_dims(gt_features, -1)
# summaries.add(tf.summary.image("k_gt_map_%d" % f_i, tf.cast(gt_features, tf.float32)))
# f_i += 1
# # for festures in gt_list:
# # summaries.add(tf.summary.image("gt_map_%d" % f_i, tf.cast(festures, tf.float32)))
# # f_i += 1
#
# arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay, data_format=DATA_FORMAT)
# with slim.arg_scope(arg_scope):
# predictions_k, localisations_k, logits_k, end_points_k = \
# ssd_net.net(b_image_k, is_training=True, reuse=True)
#
# f_i = 0
# for predict_map in predictions_k:
# predict_map = predict_map[:, :, :, :, 1:]
# predict_map = tf.reduce_max(predict_map, axis=4)
# predict_map = tf.reduce_max(predict_map, axis=3)
# predict_map = tf.expand_dims(predict_map, -1)
# summaries.add(tf.summary.image("k_predicte_map_%d" % f_i, tf.cast(predict_map, tf.float32)))
# f_i += 1
#
# ssd_net.losses(logits_k, localisations_k, b_gclasses_k, b_glocalisations_k, b_gscores_k, 2,
# match_threshold=FLAGS.match_threshold,
# negative_ratio=FLAGS.negative_ratio,
# alpha=FLAGS.loss_alpha,
# label_smoothing=FLAGS.label_smoothing)
#total_loss = slim.losses.get_total_loss()
total_loss = tf.losses.get_total_loss()
summaries.add(tf.summary.scalar('loss', total_loss))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# for variable in slim.get_model_variables():
# summaries.add(tf.summary.histogram(variable.op.name, variable))
for variable in tf.trainable_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
# optimizer = tf.train.AdamOptimizer(learning_rate, beta1=FLAGS.adam_beta1,
# beta2=FLAGS.adam_beta2, epsilon=FLAGS.opt_epsilon)
#optimizer = tf.train.GradientDescentOptimizer(learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = slim.learning.create_train_op(total_loss,
optimizer,
summarize_gradients=False)
summary_op = tf.summary.merge(list(summaries), name='summary_op')
train_writer = tf.summary.FileWriter('./logs/', sess.graph)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
#variables_to_exclude = slim.get_variables_by_suffix("Adam")
variables_to_restore = slim.get_variables_to_restore(
exclude=["MobilenetV1/Logits", "MobilenetV1/Box", "global_step"])
restorer = tf.train.Saver(variables_to_restore)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
sess.run(tf.global_variables_initializer())
restorer.restore(sess, ckpt_filename)
# if ckpt and ckpt.model_checkpoint_path:
# saver.restore(sess, ckpt.model_checkpoint_path)
i = 0
with slim.queues.QueueRunners(sess):
while (i < FLAGS.max_number_of_steps):
_, summary_str = sess.run([train_op, summary_op])
if i % 50 == 0:
global_step_str = global_step.eval()
print('%diteraton' % (global_step_str))
train_writer.add_summary(summary_str, global_step_str)
if i % 100 == 0:
global_step_str = global_step.eval()
saver.save(sess, "./logs/", global_step=global_step_str)
i += 1
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
######################
# Config model_deploy#
######################
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.worker_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
network_fn = nets_factory.get_network(FLAGS.model_name)
params = network_fn.default_params
params = params._replace(match_threshold=FLAGS.match_threshold)
# initalize the net
net = network_fn(params)
out_shape = net.params.img_shape
anchors = net.anchors(out_shape)
# create batch dataset
with tf.device(deploy_config.inputs_device()):
b_image, b_glocalisations, b_gscores = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
allgscores = []
allglocalization = []
for i in range(len(anchors)):
allgscores.append(tf.reshape(b_gscores[i], [-1]))
allglocalization.append(
tf.reshape(b_glocalisations[i], [-1, 4]))
b_gscores = tf.concat(allgscores, 0)
b_glocalisations = tf.concat(allglocalization, 0)
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list([b_image, b_glocalisations, b_gscores]),
num_threads=8,
capacity=16 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
#Allows data parallelism by creating multiple
#clones of network_fn.
# Dequeue batch.
batch_shape = [1] * 3
b_image, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# Construct SSD network.
arg_scope = net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=FLAGS.data_format)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=True, use_batch=FLAGS.use_batch)
# Add loss function.
net.losses(logits,
localisations,
b_glocalisations,
b_gscores,
negative_ratio=FLAGS.negative_ratio,
use_hard_neg=FLAGS.use_hard_neg,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
#end_points = clones[0].outputs
#for end_point in end_points:
# x = end_points[end_point]
# summaries.add(tf.summary.histogram('activations/' + end_point, x))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
#
#for variable in slim.get_model_variables():
# summaries.add(tf.summary.histogram(variable.op.name, variable))
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
#########################################
# Configure the optimization procedure. #
#########################################
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.fine_tune:
gradient_multipliers = pickle.load(
open('nets/multiplier_300.pkl', 'rb'))
else:
gradient_multipliers = None
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
if gradient_multipliers:
with ops.name_scope('multiply_grads'):
clones_gradients = slim.learning.multiply_gradients(
clones_gradients, gradient_multipliers)
if FLAGS.clip_gradient_norm > 0:
with ops.name_scope('clip_grads'):
clones_gradients = slim.learning.clip_gradient_norms(
clones_gradients, FLAGS.clip_gradient_norm)
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
#train_tensor = slim.learning.create_train_op(total_loss, optimizer, gradient_multipliers=gradient_multipliers)
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction,
allocator_type="BFC")
config = tf.ConfigProto(
gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True,
inter_op_parallelism_threads=0,
intra_op_parallelism_threads=1,
)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Config model_deploy. Keep TF Slim Models structure.
# Useful if want to need multiple GPUs and/or servers in the future.
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
# Create global_step.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
tf_utils.print_configuration(FLAGS.__flags, ssd_params,
dataset.data_sources, FLAGS.train_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device(deploy_config.inputs_device()):
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes] = provider.get(['image', 'shape',
'object/label',
'object/bbox'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(
tf_utils.reshape_list([image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# Intermediate queueing: unique batch computation pipeline for all
# GPUs running the training.
batch_queue = slim.prefetch_queue.prefetch_queue(
tf_utils.reshape_list([b_image, b_gclasses, b_glocalisations, b_gscores]),
capacity=2 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple
clones of network_fn."""
# Dequeue batch.
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(batch_queue.dequeue(), batch_shape)
# Construct SSD network.
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
# Add loss function.
ssd_net.losses(logits, localisations,
b_gclasses, b_glocalisations, b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# =================================================================== #
# Add summaries from first clone.
# =================================================================== #
clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses and extra losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for loss in tf.get_collection('EXTRA_LOSSES', first_clone_scope):
summaries.add(tf.summary.scalar(loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(FLAGS,
dataset.num_samples,
global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones,
optimizer,
var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# Create global_step.
global_step = slim.create_global_step()
sess = tf.InteractiveSession()
# Select the dataset.
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# Get the SSD network and its anchors.
ssd_class = nets_factory.get_network(FLAGS.model_name)
ssd_params = ssd_class.default_params._replace(
num_classes=FLAGS.num_classes)
ssd_net = ssd_class(ssd_params)
ssd_shape = ssd_net.params.img_shape
ssd_anchors = ssd_net.anchors(ssd_shape)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# Get for SSD network: image, labels, bboxes.
[image, shape, glabels, gbboxes
] = provider.get(['image', 'shape', 'object/label', 'object/bbox'])
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = \
image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
gclasses, glocalisations, gscores = \
ssd_net.bboxes_encode(glabels, gbboxes, ssd_anchors)
batch_shape = [1] + [len(ssd_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
b_image, b_gclasses, b_glocalisations, b_gscores = \
tf_utils.reshape_list(r, batch_shape)
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
summaries.add(
tf.summary.image("input_image", tf.cast(b_image, tf.float32)))
f_i = 0
for gt_map in b_gscores:
gt_features = tf.reduce_max(gt_map, axis=3)
gt_features = tf.expand_dims(gt_features, -1)
summaries.add(
tf.summary.image("gt_map_%d" % f_i,
tf.cast(gt_features, tf.float32)))
f_i += 1
arg_scope = ssd_net.arg_scope(weight_decay=FLAGS.weight_decay,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, localisations, logits, end_points = \
ssd_net.net(b_image, is_training=True)
f_i = 0
for predict_map in predictions:
predict_map = predict_map[:, :, :, :, 1:]
predict_map = tf.reduce_max(predict_map, axis=4)
predict_map = tf.reduce_max(predict_map, axis=3)
predict_map = tf.expand_dims(predict_map, -1)
summaries.add(
tf.summary.image("predicte_map_%d" % f_i,
tf.cast(predict_map, tf.float32)))
f_i += 1
ssd_net.losses(logits,
localisations,
b_gclasses,
b_glocalisations,
b_gscores,
match_threshold=FLAGS.match_threshold,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
total_loss = tf.losses.get_total_loss()
summaries.add(tf.summary.scalar('loss', total_loss))
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
summaries.add(tf.summary.scalar(loss.op.name, loss))
for variable in tf.trainable_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
with tf.name_scope('Optimizer'):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = slim.learning.create_train_op(total_loss,
optimizer,
summarize_gradients=False)
summary_op = tf.summary.merge(list(summaries), name='summary_op')
train_writer = tf.summary.FileWriter('./logs/', sess.graph)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
variables_to_train = tf.trainable_variables()
variables_to_restore = \
tf.contrib.framework.filter_variables(variables_to_train,
exclude_patterns=['_box'])
restorer = tf.train.Saver(variables_to_restore)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
sess.run(tf.global_variables_initializer())
restorer.restore(sess, FLAGS.checkpoint_path)
# get_init_fn
i = 0
with slim.queues.QueueRunners(sess):
while (i < FLAGS.max_number_of_steps):
_, summary_str = sess.run([train_op, summary_op])
if i % 50 == 0:
global_step_str = global_step.eval()
print('%diteraton' % (global_step_str))
train_writer.add_summary(summary_str, global_step_str)
if i % 100 == 0:
global_step_str = global_step.eval()
saver.save(sess, "./logs/", global_step=global_step_str)
i += 1
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('必须指定一个TFRecords的数据集目录')
# 设置打印级别
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# 在默认的图当中进行编写训练逻辑
# DeploymentConfig以及全部参数
# 配置计算机相关情况
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones, # GPU设备数量
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1, # 1台计算机
num_ps_tasks=0)
# 定义一个全局步长参数（网络训练都会这么去进行配置）
# 使用指定设备 tf.device
with tf.device(deploy_config.variables_device()):
global_step = tf.train.create_global_step()
# 2、获取图片数据，做一些处理
# 图片有什么？image, shape, bbox, label
# image会做一些数据增强，大小变换
# 直接训练？需要将anchor bbox进行样本标记正负样本，目的使的GT目标样本的数量与default bboxes数量一致
# 将每个模块结果先获取
# (1) 通过数据工厂取出规范信息
dataset = dataset_factory.get_datasets(FLAGS.dataset_name,
FLAGS.train_or_test,
FLAGS.dataset_dir)
# （2）获取网络计算的anchors结果
ssd_class = nets_factory.get_network(FLAGS.model_name)
# 获取默认网络参数
ssd_params = ssd_class.default_params._replace(num_classes=9)
# 初始化网络init函数
ssd_net = ssd_class(ssd_params)
# 获取形状，用于输入到anchors函数参数当中
ssd_shape = ssd_net.params.img_shape
# 获取anchors, SSD网络当中6层的所有计算出来的默认候选框
ssd_anchors = ssd_net.anchors(ssd_shape)
# （3）获取预处理函数
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
FLAGS.model_name, is_training=True)
# 打印网络相关参数
train_tools.print_configuration(ssd_params, dataset.data_sources)
# 2.2
# 1，slim.dataset_data_provider.DatasetDataProvider通过GET方法获取数据
# 2，对数据进行预处理
# 3，对获取出来的groundtruth标签和bbox。进行编码
# 4，获取的单个样本数据，要进行批处理以及返回队列
with tf.device(deploy_config.inputs_device()):
# 给当前操作取一个作用域名称
with tf.name_scope(FLAGS.model_name + '_data_provider'):
# slim.dataset_data_provider.DatasetDataProvider通过GET方法获取数据
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=4,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
# 通过get获取数据
# 真正获取参数
[image, shape, glabels, gbboxes] = provider.get(
['image', 'shape', 'object/label', 'object/bbox'])
# 直接进行数据预处理
# image [?, ?, 3]---->[300， 300， 3]
image, glabels, gbboxes = image_preprocessing_fn(
image,
glabels,
gbboxes,
out_shape=ssd_shape,
data_format=DATA_FORMAT)
# 对原始anchor bboxes进行正负样本标记
# 得到目标值，编码之后，返回？
# 训练？ 预测值类别，物体位置，物体类别概率，目标值
# 8732 anchor, 得到8732个与GT 对应的标记的anchor
# gclasses:目标类别
# glocalisations：目标类别的真是位置
# gscores：是否是正负样本
gclasses, glocalisations, gscores = ssd_net.bboxes_encode(
glabels, gbboxes, ssd_anchors)
# print(gclasses, glocalisations)
# 特征值、目标
# 批处理以及队列处理
# tensor_list:tensor列表 [tensor, tensor, ]
# tf.train.batch(tensor_list, batch_size, num_threads, capacity)
# [Tensor, [6], [6], [6]] 嵌套的列表要转换成单列表形式
r = tf.train.batch(train_tools.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=4,
capacity=5 * FLAGS.batch_size)
# r应该是一个19个Tensor组成的一个列表
# print(r)
# 批处理数据放入队列
# 1个r:批处理的样本， 5个设备，5个r, 5组32张图片
# 队列的目的是为了不同设备需求
batch_queue = slim.prefetch_queue.prefetch_queue(
r, capacity=deploy_config.num_clones)
# 3、赋值模型到不同的GPU设备，以及损失、变量的观察
# train_tools.deploy_loss_summary(deploy_config,batch_queue,ssd_net,summaries,batch_shape,FLAGS)
# summarties:摘要
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
# batch_shape：解析这个batch_queue的大小，指的是获取的一个默认队列大小，指上面r的大小
batch_shape = [1] + 3 * [len(ssd_anchors)]
update_ops, first_clone_scope, clones = train_tools.deploy_loss_summary(
deploy_config, batch_queue, ssd_net, summaries, batch_shape, FLAGS)
# 4、定义学习率以及优化器
# 学习率：0.001
# 终止学习率：0.0001
# 优化器选择:adam优化器
# learning_rate = tf_utils.configure_learning_rate(FLAGS, num_samples, global_step)
# FLAGS：将会用到学习率设置相关参数
# global_step: 全局步数
# optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
# learning_rate: 学习率
with tf.device(deploy_config.optimizer_device()):
# 定义学习率和优化器
learning_rate = train_tools.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
optimizer = train_tools.configure_optimizer(FLAGS, learning_rate)
# 观察学习率的变化情况，添加到summaries
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
# 5、计算所有设备的平均损失以及每个变量的梯度总和
train_op, summaries_op = train_tools.get_trainop(
optimizer, summaries, clones, global_step, first_clone_scope,
update_ops)
# 配置config以及saver
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)
config = tf.ConfigProto(
log_device_placement=False, # 若果打印会有许多变量的设备信息出现
gpu_options=gpu_options)
# saver
saver = tf.train.Saver(
max_to_keep=5, # 默认保留最近几个模型文件
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
# 6、进行训练
print(FLAGS.train_model_dir)
slim.learning.train(
train_op, # 训练优化器tensor
logdir=FLAGS.pre_trained_model, # 模型存储目录
master='',
is_chief=True,
init_fn=train_tools.get_init_fn(FLAGS), # 初始化参数的逻辑，预训练模型的读取和微调模型判断
summary_op=summaries_op, # 摘要
number_of_steps=FLAGS.max_number_of_steps, # 最大步数
log_every_n_steps=10, # 打印频率
save_summaries_secs=60, # 保存摘要频率
saver=saver, # 保存模型参数
save_interval_secs=600, # 保存模型间隔
session_config=config, # 会话参数配置
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
# initalize the net
network_fn = nets_factory.get_network(FLAGS.model_name)
net = network_fn()
out_shape = net.params.img_shape
out_shape = (300,300)
anchors = net.anchors(out_shape)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device('/cpu:0'):
b_image, glabels, b_gbboxes, g_bbox_img, b_glocalisations, b_gscores =\
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS,
file_pattern = '*.tfrecord',
is_training = False,
shuffe = FLAGS.shuffle_data)
b_gdifficults = tf.zeros(tf.shape(glabels), dtype=tf.int64)
dict_metrics = {}
arg_scope = net.arg_scope(data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=False, use_batch=FLAGS.use_batch)
# Add losses functions.
#total_loss = net.losses(logits, localisations,
# b_glocalisations, b_gscores)
predictions = []
for i in range(len(logits)):
predictions.append(slim.softmax(logits[i]))
# Performing post-processing on CPU: loop-intensive, usually more efficient.
with tf.device('/device:CPU:0'):
# Detected objects from SSD output.
localisations = net.bboxes_decode(localisations, anchors)
rscores, rbboxes = \
net.detected_bboxes(predictions, localisations,
select_threshold=FLAGS.select_threshold,
nms_threshold=FLAGS.nms_threshold,
clipping_bbox=None,
top_k=FLAGS.select_top_k,
keep_top_k=FLAGS.keep_top_k)
# Compute TP and FP statistics.
num_gbboxes, tp, fp, rscores = \
tfe.bboxes_matching_batch(rscores.keys(), rscores, rbboxes,
glabels, b_gbboxes, b_gdifficults,
matching_threshold=FLAGS.matching_threshold)
# Variables to restore: moving avg. or normal weights.
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
# =================================================================== #
# Evaluation metrics.
# =================================================================== #
with tf.device(FLAGS.gpu_eval):
dict_metrics = {}
# Extra losses as well.
for loss in tf.get_collection('EXTRA_LOSSES'):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
# Add metrics to summaries and Print on screen.
for name, metric in dict_metrics.items():
# summary_name = 'eval/%s' % name
summary_name = name
op = tf.summary.scalar(summary_name, metric[0], collections=[])
# op = tf.Print(op, [metric[0]], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# FP and TP metrics.
tp_fp_metric = tfe.streaming_tp_fp_arrays(num_gbboxes, tp, fp, rscores)
for c in tp_fp_metric[0].keys():
dict_metrics['tp_fp_%s' % c] = (tp_fp_metric[0][c],
tp_fp_metric[1][c])
# Add to summaries precision/recall values.
icdar2013 = {}
for c in tp_fp_metric[0].keys():
# Precison and recall values.
prec, rec = tfe.precision_recall(*tp_fp_metric[0][c])
op = tf.summary.scalar('precision', tf.reduce_mean(prec), collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
op = tf.summary.scalar('recall', tf.reduce_mean(rec), collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Average precision VOC07.
v = tfe.average_precision_voc12(prec, rec)
#v = (prec + rec)/2.
summary_name = 'ICDAR13/%s' % c
op = tf.summary.scalar(summary_name, v, collections=[])
# op = tf.Print(op, [v], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
icdar2013[c] = v
# Mean average precision VOC07.
summary_name = 'ICDAR13/mAP'
mAP = tf.add_n(list(icdar2013.values())) / len(icdar2013)
op = tf.summary.scalar(summary_name, mAP, collections=[])
op = tf.Print(op, [mAP], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# Split into values and updates ops.
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(dict_metrics)
# =================================================================== #
# Evaluation loop.
# =================================================================== #
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False, gpu_options=gpu_options)
# config.graph_options.optimizer_options.global_jit_level = tf.OptimizerOptions.ON_1
# Number of batches...
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
num_batches = math.ceil(FLAGS.num_samples / float(FLAGS.batch_size))
if not FLAGS.wait_for_checkpoints:
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Standard evaluation loop.
start = time.time()
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
session_config=config)
# Log time spent.
elapsed = time.time()
elapsed = elapsed - start
print('Time spent : %.3f seconds.' % elapsed)
print('Time spent per BATCH: %.3f seconds.' % (elapsed / num_batches))
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
# Waiting loop.
slim.evaluation.evaluation_loop(
master=FLAGS.master,
checkpoint_dir=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
eval_interval_secs=60,
max_number_of_evaluations=np.inf,
session_config=config,
timeout=None)
def main(_):
if not FLAGS.data_dir:
raise ValueError('You must supply the dataset directory with --data_dir')
num_gpus = FLAGS.num_gpus
if num_gpus < 1: num_gpus = 1
# ps_spec = FLAGS.ps_hosts.split(",")
# worker_spec = FLAGS.worker_hosts.split(",")
# num_workers = len(worker_spec)
# cluster = tf.train.ClusterSpec({
# "ps": ps_spec,
# "worker": worker_spec})
# server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index)
# if FLAGS.job_name == "ps":
# with tf.device("/cpu:0"):
# server.join()
# return
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.device('/cpu:0'):
global_step = slim.create_global_step()
# Select the dataset.
dataset = dataset_factory.get_dataset(
FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.data_dir)
# Get the RON network and its anchors.
ron_class = nets_factory.get_network(FLAGS.model_name)
ron_params = ron_class.default_params._replace(num_classes=FLAGS.num_classes)
ron_net = ron_class(ron_params)
ron_shape = ron_net.params.img_shape
ron_anchors = ron_net.anchors(ron_shape)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=120 * FLAGS.batch_size * num_gpus,
common_queue_min=80 * FLAGS.batch_size * num_gpus,
shuffle=True)
# Get for RON network: image, labels, bboxes.
# (ymin, xmin, ymax, xmax) fro gbboxes
[image, shape, glabels, gbboxes, isdifficult] = provider.get(['image', 'shape',
'object/label',
'object/bbox',
'object/difficult'])
isdifficult_mask =tf.cond(tf.reduce_sum(tf.cast(tf.logical_not(tf.equal(tf.ones_like(isdifficult), isdifficult)), tf.float32)) < 1., lambda : tf.one_hot(0, tf.shape(isdifficult)[0], on_value=True, off_value=False, dtype=tf.bool), lambda : isdifficult < tf.ones_like(isdifficult))
glabels = tf.boolean_mask(glabels, isdifficult_mask)
gbboxes = tf.boolean_mask(gbboxes, isdifficult_mask)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = image_preprocessing_fn(image, glabels, gbboxes,
out_shape=ron_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
# glocalisations is our regression object
# gclasses is the ground_trutuh label
# gscores is the the jaccard score with ground_truth
gclasses, glocalisations, gscores = \
ron_net.bboxes_encode(glabels, gbboxes, ron_anchors, positive_threshold=FLAGS.match_threshold, ignore_threshold=FLAGS.neg_threshold)
# each size of the batch elements
# include one image, three others(gclasses, glocalisations, gscores)
batch_shape = [1] + [len(ron_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(
tf_utils.reshape_list([image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size * num_gpus,
num_threads=FLAGS.num_preprocessing_threads,
capacity=120 * FLAGS.batch_size * num_gpus)
all_batch = tf_utils.reshape_list(r, batch_shape)
b_image = tf.split(all_batch[0], num_or_size_splits=num_gpus, axis=0)
_b_gclasses = [tf.split(b, num_or_size_splits=num_gpus, axis=0) for b in all_batch[1]]
b_gclasses = [_ for _ in zip(*_b_gclasses)]
_b_glocalisations = [tf.split(b, num_or_size_splits=num_gpus, axis=0) for b in all_batch[2]]
b_glocalisations = [_ for _ in zip(*_b_glocalisations)]
_b_gscores = [tf.split(b, num_or_size_splits=num_gpus, axis=0) for b in all_batch[3]]
b_gscores = [_ for _ in zip(*_b_gscores)]
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
learning_rate = tf_utils.configure_learning_rate(FLAGS,
dataset.num_samples,
global_step)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
# Construct RON network.
arg_scope = ron_net.arg_scope(weight_decay=FLAGS.weight_decay, data_format=DATA_FORMAT)
reuse_variables = False
tower_grads = []
loss_list = []
with slim.arg_scope(arg_scope):
for index in range(num_gpus):
with tf.device('/gpu:%d' % index):
predictions, logits, objness_pred, objness_logits, localisations, end_points = ron_net.net(b_image[index], is_training=True, reuse = reuse_variables)
# Add loss function.
ron_net.losses(logits, localisations, objness_logits, objness_pred,
b_gclasses[index], b_glocalisations[index], b_gscores[index],
match_threshold = FLAGS.match_threshold,
neg_threshold = FLAGS.neg_threshold,
objness_threshold = FLAGS.objectness_thres,
negative_ratio=FLAGS.negative_ratio,
alpha=FLAGS.loss_alpha,
beta=FLAGS.loss_beta,
label_smoothing=FLAGS.label_smoothing)
reuse_variables = True
# and returns a train_tensor and summary_op
loss = tf.losses.get_total_loss()
loss_list.append(loss)
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# Create gradient updates.
grads = optimizer.compute_gradients(loss, variables_to_train)
tower_grads.append(grads)
reduce_grads = average_gradients(tower_grads)
total_loss = tf.reduce_mean(tf.stack(loss_list, axis=0), axis=0)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
grad_updates = optimizer.apply_gradients(reduce_grads, global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss, name='train_op')
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Kicks off the training.
# =================================================================== #
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours = FLAGS.save_interval_secs/3600.,
write_version=2,
pad_step_number=False)
slim.learning.train(
train_tensor,
logdir=FLAGS.model_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS, os.path.join(FLAGS.data_dir, 'vgg_16.ckpt')),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
session_wrapper=None,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
# =================================================================== #
# Config model_deploy. #
# Keep TF Slim Models structure. #
# Useful if want to need multiple GPUs and/or servers in the future. #
# =================================================================== #
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=0,
num_replicas=1,
num_ps_tasks=0)
# Create global_step.
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
# =================================================================== #
# Select the dataset.
# =================================================================== #
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
# =================================================================== #
# Select the network
# =================================================================== #
if FLAGS.model_name == "crnn":
crnn_net = nets_factory.get_network(FLAGS.model_name)
network_fn = crnn_net(phase='Train',
num_classes=(dataset.num_classes -
FLAGS.labels_offset))
else:
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=True)
# =================================================================== #
# Select the preprocessing function.
# =================================================================== #
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
#tf_utils.print_configuration(FLAGS.__flags,
# dataset.data_sources, save_dir=FLAGS.train_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.device(deploy_config.inputs_device()):
if FLAGS.dataset_name == "ocr":
image, label = tf_utils.read_features(ops.join(
FLAGS.dataset_dir, "ocr_train_000.tfrecord"),
num_epochs=None)
else:
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size,
shuffle=True)
[image, label] = provider.get(['image', 'label'])
# Pre-processing image, labels and bboxes.
train_image_size = FLAGS.train_image_size or network_fn.default_image_size
#image = image_preprocessing_fn(image, 32, 256)
# Resize the image to the specified height and width.
image = tf.expand_dims(image, 0)
image = tf.image.resize_bilinear(image, [IMAGE_H, IMAGE_W],
align_corners=False)
image = tf.squeeze(image, [0])
image = tf.subtract(image, 0.5)
image = tf.multiply(image, 2.0)
#label = tf.reshape(label,[MAX_CHAR_LEN])
images, labels = tf.train.shuffle_batch(
tensors=[image, label],
batch_size=32,
capacity=1000 + 2 * 32,
min_after_dequeue=100,
#enqueue_many=True,
num_threads=1)
images = tf.cast(x=images, dtype=tf.float32)
if FLAGS.model_name != "crnn":
labels = slim.one_hot_encoding(
labels, dataset.num_classes - FLAGS.labels_offset)
batch_queue = slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * deploy_config.num_clones)
# =================================================================== #
# Define the model running on every GPU.
# =================================================================== #
#def clone_fn(batch_queue):
def clone_fn(images, labels):
"""Allows data parallelism by creating multiple
clones of network_fn."""
# Dequeue batch.
#images, labels = batch_queue.dequeue()
with tf.variable_scope('crnn'):
logits, end_points = network_fn.build_CRNNnet(images)
#############################
# Specify the loss function #
#############################
if FLAGS.model_name == "crnn":
if FLAGS.dataset_name == "mnist":
idx = tf.where(tf.not_equal(labels, 0))
labels = tf.SparseTensor(idx, tf.gather_nd(labels, idx),
labels.get_shape())
labels = tf.cast(labels, tf.int32)
ctc_loss = tf.nn.ctc_loss(
labels=labels,
inputs=logits,
sequence_length=SEQ_LENGTH,
ctc_merge_repeated=True,
ignore_longer_outputs_than_inputs=True,
time_major=True)
ctc_loss = tf.reduce_mean(ctc_loss)
ctc_loss = tf.Print(ctc_loss, [ctc_loss], message='* Loss : ')
tf.losses.add_loss(ctc_loss)
decoded, log_prob = tf.nn.ctc_beam_search_decoder(
logits, sequence_length=SEQ_LENGTH, merge_repeated=False)
sequence_dist = tf.reduce_mean(
tf.edit_distance(tf.cast(decoded[0], tf.int32), labels))
else:
if 'AuxLogits' in end_points:
slim.losses.softmax_cross_entropy(
end_points['AuxLogits'],
labels,
label_smoothing=FLAGS.label_smoothing,
weights=0.4,
scope='aux_loss')
slim.losses.softmax_cross_entropy(
logits,
labels,
label_smoothing=FLAGS.label_smoothing,
weights=1.0)
return end_points, ctc_loss, sequence_dist, labels, decoded
if FLAGS.model_name == "crnn":
end_points, ctc_loss, sequence_dist, labels, decoded = clone_fn(
images, labels)
network_fn.train_crnn(FLAGS, global_step, ctc_loss, sequence_dist,
labels, decoded)
else:
# =================================================================== #
# Add summaries from first clone.
# =================================================================== #
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
#clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue])
clones = model_deploy.create_clones(deploy_config, clone_fn,
[images, labels])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
# Add summaries for end_points.
end_points, ctc_loss, sequence_dist, labels, decoded = clones[
0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(
tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES,
first_clone_scope):
summaries.add(
tf.summary.scalar('losses/%s' % loss.op.name, loss))
summaries.add(tf.summary.scalar('losses/ctc_loss',
tensor=ctc_loss))
summaries.add(tf.summary.scalar('Seq_Dist', tensor=sequence_dist))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
# =================================================================== #
# Configure the moving averages.
# =================================================================== #
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
# =================================================================== #
# Configure the optimization procedure.
# =================================================================== #
with tf.device(deploy_config.optimizer_device()):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, dataset.num_samples, global_step)
#optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate).minimize(cost)
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
#optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate).minimize(loss=ctc_loss, global_step=global_step)
summaries.add(
tf.summary.scalar('learning_rate', tensor=learning_rate))
if FLAGS.sync_replicas:
# If sync_replicas is enabled, the averaging will be done in the chief
# queue runner.
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
total_num_replicas=FLAGS.worker_replicas,
variable_averages=variable_averages,
variables_to_average=moving_average_variables)
elif FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones,
optimizer,
#regularization_losses = ctc_loss,
var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_tensor = tf.identity(total_loss, name='train_op')
train_tensor = slim.learning.create_train_op(
total_loss, optimizer)
"""
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
name='train_op')
"""
# Add the summaries from the first clone. These contain the summaries
#summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
# first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
# =================================================================== #
# Configure the saver procedure.
# =================================================================== #
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
model_save_dir = './checkpoints/' + FLAGS.model_name
if not ops.exists(model_save_dir):
os.makedirs(model_save_dir)
train_start_time = time.strftime('%Y-%m-%d-%H-%M-%S',
time.localtime(time.time()))
model_name = 'CRNNnet_{:s}.ckpt'.format(str(train_start_time))
model_save_path = ops.join(model_save_dir, model_name)
# =================================================================== #
# Kicks off the training.
# =================================================================== #
#summary_writer = tf.summary.FileWriter("tensorboard_%d" %(ctx.worker_num),graph=tf.get_default_graph())
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(log_device_placement=False,
gpu_options=gpu_options)
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
#session_wrapper=tfdbg.LocalCLIDebugWrapperSession,
sync_optimizer=None)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
# Get the RON network and its anchors.
ron_class = nets_factory.get_network(FLAGS.model_name)
ron_params = ron_class.default_params._replace(
num_classes=FLAGS.num_classes)
ron_net = ron_class(ron_params)
ron_shape = ron_net.params.img_shape
ron_anchors = ron_net.anchors(ron_shape)
# Get for RON network: image, labels, bboxes.
# (ymin, xmin, ymax, xmax) fro gbboxes
image_input = tf.placeholder(tf.int32, shape=(None, None, 3))
shape_input = tf.placeholder(tf.int32, shape=(2, ))
glabels_input = tf.placeholder(tf.int32, shape=(None, ))
gbboxes_input = tf.placeholder(tf.float32, shape=(None, 4))
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes, bbox_img = image_preprocessing_fn(
image_input,
glabels_input,
gbboxes_input,
out_shape=ron_shape,
data_format=DATA_FORMAT)
#### DEBUG ####
#image = tf.Print(image, [shape, glabels, gbboxes], message='after preprocess: ', summarize=20)
# Construct RON network.
arg_scope = ron_net.arg_scope(is_training=False,
data_format=DATA_FORMAT)
with slim.arg_scope(arg_scope):
predictions, _, objness_pred, _, localisations, _ = ron_net.net(
tf.expand_dims(image, axis=0), is_training=False)
bboxes = ron_net.bboxes_decode(localisations, ron_anchors)
flaten_scores, flaten_labels, flaten_bboxes = flaten_predict(
predictions, objness_pred, bboxes)
#objness_pred = tf.reduce_max(tf.cast(tf.greater(objness_pred[-1], FLAGS.objectness_thres), tf.float32))
flaten_bboxes = tfe.bboxes.bboxes_clip(bbox_img, flaten_bboxes)
flaten_scores, flaten_labels, flaten_bboxes = filter_boxes(
flaten_scores, flaten_labels, flaten_bboxes, 0.03, shape_input,
[320., 320.])
#flaten_scores, flaten_labels, flaten_bboxes = tf_bboxes_nms_by_class(flaten_scores, flaten_labels, flaten_bboxes, nms_threshold=FLAGS.nms_threshold, keep_top_k=FLAGS.nms_topk_percls, mode = 'union')
flaten_scores, flaten_labels, flaten_bboxes = tf_bboxes_nms(
flaten_scores,
flaten_labels,
flaten_bboxes,
nms_threshold=FLAGS.nms_threshold,
keep_top_k=FLAGS.nms_topk,
mode='union')
# Resize bboxes to original image shape.
flaten_bboxes = tfe.bboxes.bboxes_resize(bbox_img, flaten_bboxes)
# configure model restore
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Restoring model from %s. Ignoring missing vars: %s' %
(checkpoint_path, FLAGS.ignore_missing_vars))
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay)
variables_to_restore = variable_averages.variables_to_restore()
else:
variables_to_restore = slim.get_variables_to_restore()
init_fn = slim.assign_from_checkpoint_fn(
checkpoint_path,
variables_to_restore,
ignore_missing_vars=FLAGS.ignore_missing_vars)
def wrapper_debug(sess):
sess = tf_debug.LocalCLIDebugWrapperSession(
sess, thread_name_filter="MainThread$")
sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
return sess
# no need for specify local_variables_initializer and tables_initializer, Supervisor will do this via default local_init_op
init_op = tf.group(tf.global_variables_initializer())
# Pass the init function to the supervisor.
# - The init function is called _after_ the variables have been initialized by running the init_op.
# - manage summary in current process by ourselves for memory saving
# - no need to specify global_step, supervisor will find this automately
# - initialize order: checkpoint -> local_init_op -> init_op -> init_func
sv = tf.train.Supervisor(logdir=FLAGS.test_dir,
init_fn=init_fn,
init_op=init_op,
summary_op=None,
save_model_secs=0)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(
log_device_placement=False,
allow_soft_placement=True,
intra_op_parallelism_threads=FLAGS.num_cpu_threads,
inter_op_parallelism_threads=FLAGS.num_cpu_threads,
gpu_options=gpu_options)
cur_step = 0
tf.logging.info(
datetime.now().strftime('Evaluation Start: %Y-%m-%d %H:%M:%S'))
detector_eval = voc_eval.DetectorEvalPascal('../PASCAL/VOC2007TEST/',
'./eval_logs/',
set_type='test')
num_images = pascalvoc_2007.SPLITS_TO_SIZES['test']
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
all_boxes = [[[] for _ in range(num_images)]
for _ in range(len(pascalvoc_common.VOC_CLASSES) + 1)]
output_dir = detector_eval.output_dir
det_file = os.path.join(output_dir, 'detections.pkl')
with sv.managed_session(config=config) as sess:
while True:
if sv.should_stop():
tf.logging.info('Supervisor emited finish!')
break
if cur_step >= len(detector_eval.image_ids):
break
start_time = time.time()
input_datas = _process_image(
detector_eval.image_ids[cur_step][0],
detector_eval.image_ids[cur_step][1])
with tf.device('/gpu:0'):
image_, shape_, _, _, scores_, labels_, bboxes_ = sess.run(
[
image, shape_input, glabels, gbboxes,
flaten_scores, flaten_labels, flaten_bboxes
],
feed_dict={
image_input: input_datas[0],
shape_input: input_datas[1],
glabels_input: input_datas[2],
gbboxes_input: input_datas[3]
})
# print(image_)
# print(len(a),a[0].shape,a[1].shape,a[2].shape,a[3].shape)
# print(len(b),b[0].shape,b[1].shape,b[2].shape,b[3].shape)
# print(len(c),c[0].shape,c[1].shape,c[2].shape,c[3].shape)
print(scores_)
print(labels_)
print(bboxes_)
# print(a)
# print(FLAGS.objectness_thres)
img_to_draw = np.copy(
preprocessing_factory.ssd_vgg_preprocessing.
np_image_unwhitened(image_))
img_to_draw = draw_toolbox.bboxes_draw_on_img(img_to_draw,
labels_,
scores_,
bboxes_,
thickness=2)
imsave('./Debug/{}.jpg'.format(cur_step), img_to_draw)
unique_labels = []
for l in labels_:
if l not in unique_labels:
unique_labels.append(l)
print('unique_labels:', unique_labels)
# skip j = 0, because it's the background class
for j in unique_labels:
mask = labels_ == j
boxes = bboxes_[mask]
# all detections are collected into:
# all_boxes[cls][image] = N x 5 array of detections in
# (x1, y1, x2, y2, score)
boxes[:, 0] *= shape_[0]
boxes[:, 2] *= shape_[0]
boxes[:, 1] *= shape_[1]
boxes[:, 3] *= shape_[1]
boxes[:, [0, 1]] = boxes[:, [1, 0]]
boxes[:, [2, 3]] = boxes[:, [3, 2]]
scores = scores_[mask]
cls_dets = np.hstack(
(boxes, scores[:, np.newaxis])).astype(np.float32,
copy=False)
print(cls_dets)
all_boxes[j][cur_step] = cls_dets
time_elapsed = time.time() - start_time
if cur_step % FLAGS.log_every_n_steps == 0:
tf.logging.info(
'Eval Speed: {:5.3f}sec/image, {}/{}'.format(
time_elapsed, cur_step,
len(detector_eval.image_ids)))
cur_step += 1
with open(det_file, 'wb') as f:
pickle.dump(all_boxes, f, pickle.HIGHEST_PROTOCOL)
detector_eval.evaluate_detections(all_boxes)
tf.logging.info(
datetime.now().strftime('Evaluation Finished: %Y-%m-%d %H:%M:%S'))
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.DEBUG)
with tf.Graph().as_default():
network_fn = nets_factory.get_network(FLAGS.model_name)
params = network_fn.default_params
params = params._replace(match_threshold=FLAGS.match_threshold)
# initalize the net
net = network_fn(params)
out_shape = net.params.img_shape
anchors = net.anchors(out_shape)
# Create global_step.
global_step = slim.create_global_step()
# create batch dataset
b_image, b_glocalisations, b_gscores = \
load_batch.get_batch(FLAGS.dataset_dir,
FLAGS.num_readers,
FLAGS.batch_size,
out_shape,
net,
anchors,
FLAGS,
file_pattern = FLAGS.file_pattern,
is_training = True,
shuffe = FLAGS.shuffle_data)
with tf.device(FLAGS.gpu_train):
#with tf.device(FLAGS.gpu_train):
arg_scope = net.arg_scope(weight_decay=FLAGS.weight_decay)
with slim.arg_scope(arg_scope):
localisations, logits, end_points = \
net.net(b_image, is_training=True, use_batch=FLAGS.use_batch)
# Add loss function.
total_loss = net.losses(logits,
localisations,
b_glocalisations,
b_gscores,
negative_ratio=FLAGS.negative_ratio,
use_hard_neg=FLAGS.use_hard_neg,
alpha=FLAGS.loss_alpha,
label_smoothing=FLAGS.label_smoothing)
# Gather summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
'''
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
#for loss in tf.get_collection(tf.GraphKeys.LOSSES):
# summaries.add(tf.summary.scalar(loss.op.name, loss))
'''
for loss in tf.get_collection('EXTRA_LOSSES'):
summaries.add(tf.summary.scalar(loss.op.name, loss))
'''
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
'''
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
with tf.device(FLAGS.gpu_train):
learning_rate = tf_utils.configure_learning_rate(
FLAGS, FLAGS.num_samples, global_step)
# Configure the optimization procedure
optimizer = tf_utils.configure_optimizer(FLAGS, learning_rate)
#summaries.add(tf.summary.scalar('learning_rate', learning_rate))
## Training
#loss = tf.get_collection(tf.GraphKeys.LOSSES)
#total_loss = tf.add_n(loss)
'''
if FLAGS.fine_tune:
gradient_multipliers = pickle.load(open('nets/multiplier_300.pkl','rb'))
else:
gradient_multipliers = None
'''
if FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = tf_utils.get_variables_to_train(FLAGS)
vars_grad = optimizer.compute_gradients(total_loss, variables_to_train)
grad_updates = optimizer.apply_gradients(vars_grad,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_op = control_flow_ops.with_dependencies([update_op],
total_loss,
name='train_op')
#train_op = slim.learning.create_train_op(total_loss, optimizer, gradient_multipliers=gradient_multipliers)
# =================================================================== #
# Kicks off the training.
# =================================================================== #
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False,
allow_soft_placement=True)
saver = tf.train.Saver(max_to_keep=5,
keep_checkpoint_every_n_hours=1.0,
write_version=2,
pad_step_number=False)
slim.learning.train(train_op,
logdir=FLAGS.train_dir,
master='',
is_chief=True,
init_fn=tf_utils.get_init_fn(FLAGS),
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
saver=saver,
save_interval_secs=FLAGS.save_interval_secs,
session_config=config,
sync_optimizer=None)
def main(_):
if not FLAGS.data_dir:
raise ValueError(
'You must supply the dataset directory with --data_dir')
tf.logging.set_verbosity(tf.logging.INFO)
validate_batch_size_for_multi_gpu(FLAGS.batch_size)
# Get the RON network and its anchors.
ron_class = nets_factory.get_network(FLAGS.model_name)
ron_params = ron_class.default_params._replace(
num_classes=FLAGS.num_classes)
ron_net = ron_class(ron_params)
ron_shape = ron_net.params.img_shape
ron_anchors = ron_net.anchors(ron_shape)
# There are two steps required if using multi-GPU: (1) wrap the model_fn,
# and (2) wrap the optimizer. The first happens here, and (2) happens
# in the model_fn itself when the optimizer is defined.
model_function = replicate_model_fn(
lambda features, labels, mode, params, config: model_fn(
ron_net, features, labels['b_gclasses'], labels[
'b_glocalisations'], labels['b_gscores'], mode, params),
loss_reduction=tf.losses.Reduction.MEAN)
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
intra_op_parallelism_threads=FLAGS.num_cpu_threads,
inter_op_parallelism_threads=FLAGS.num_cpu_threads,
gpu_options=gpu_options)
ron_detector = tf.estimator.Estimator(
model_fn=model_function,
model_dir=FLAGS.model_dir,
params=None,
config=tf.estimator.RunConfig(
save_summary_steps=FLAGS.save_summaries_steps,
save_checkpoints_secs=FLAGS.save_interval_secs,
session_config=config,
keep_checkpoint_max=5,
keep_checkpoint_every_n_hours=FLAGS.save_interval_secs / 3600.,
log_step_count_steps=FLAGS.log_every_n_steps))
# Train the model
def train_input_fn():
# Select the dataset.
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.data_dir)
tf_utils.print_configuration(FLAGS.__flags, ron_params,
dataset.data_sources, FLAGS.model_dir)
# =================================================================== #
# Create a dataset provider and batches.
# =================================================================== #
with tf.name_scope(FLAGS.dataset_name + '_data_provider'):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=120 * FLAGS.batch_size,
common_queue_min=80 * FLAGS.batch_size,
shuffle=True)
# Get for RON network: image, labels, bboxes.
# (ymin, xmin, ymax, xmax) fro gbboxes
[image, shape, glabels, gbboxes, isdifficult] = provider.get([
'image', 'shape', 'object/label', 'object/bbox', 'object/difficult'
])
isdifficult_mask = tf.cond(
tf.reduce_sum(
tf.cast(
tf.logical_not(
tf.equal(tf.ones_like(isdifficult), isdifficult)),
tf.float32)) < 1.,
lambda: tf.one_hot(0,
tf.shape(isdifficult)[0],
on_value=True,
off_value=False,
dtype=tf.bool),
lambda: isdifficult < tf.ones_like(isdifficult))
glabels = tf.boolean_mask(glabels, isdifficult_mask)
gbboxes = tf.boolean_mask(gbboxes, isdifficult_mask)
# Select the preprocessing function.
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
# Pre-processing image, labels and bboxes.
image, glabels, gbboxes = image_preprocessing_fn(
image,
glabels,
gbboxes,
out_shape=ron_shape,
data_format=DATA_FORMAT)
# Encode groundtruth labels and bboxes.
# glocalisations is our regression object
# gclasses is the ground_trutuh label
# gscores is the the jaccard score with ground_truth
gclasses, glocalisations, gscores = ron_net.bboxes_encode(
glabels,
gbboxes,
ron_anchors,
positive_threshold=FLAGS.match_threshold,
ignore_threshold=FLAGS.neg_threshold)
# each size of the batch elements
# include one image, three others(gclasses, glocalisations, gscores)
batch_shape = [1] + [len(ron_anchors)] * 3
# Training batches and queue.
r = tf.train.batch(tf_utils.reshape_list(
[image, gclasses, glocalisations, gscores]),
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=120 * FLAGS.batch_size,
shared_name=None)
b_image, b_gclasses, b_glocalisations, b_gscores = tf_utils.reshape_list(
r, batch_shape)
return b_image, {
'b_gclasses': b_gclasses,
'b_glocalisations': b_glocalisations,
'b_gscores': b_gscores
}
# Set up training hook that logs the training accuracy every 100 steps.
tensors_to_log = {
'total_loss': 'loss_to_log',
'learning_rate': 'learning_rate_to_log',
'global_step': 'global_step_to_log'
}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=FLAGS.log_every_n_steps)
#with tf.contrib.tfprof.ProfileContext('./train_dir') as pctx:
ron_detector.train(input_fn=train_input_fn,
hooks=[logging_hook],
max_steps=FLAGS.max_number_of_steps)