def init_config(image_shape,
batch_size=None,
weight_decay=0.0005,
num_gpus=1,
train_with_ignored=False,
seg_loc_loss_weight=1.0,
link_cls_loss_weight=1.0,
seg_conf_threshold=0.5,
link_conf_threshold=0.5):
_set_det_th(seg_conf_threshold, link_conf_threshold)
_set_loss_weight(seg_loc_loss_weight, link_cls_loss_weight)
_set_train_with_ignored(train_with_ignored)
h, w = image_shape
from nets import anchor_layer
from nets import seglink_symbol
fake_image = tf.ones((batch_size, h, w, 3))
fake_net = seglink_symbol.SegLinkNet(inputs=fake_image,
weight_decay=weight_decay)
feat_shapes = fake_net.get_shapes()
# the placement of the following lines are extremely important
_set_image_shape(image_shape)
_set_feat_shapes(feat_shapes)
anchors, _ = anchor_layer.generate_anchors()
global default_anchors
default_anchors = anchors
global num_anchors
num_anchors = len(anchors)
_build_anchor_map()
global num_links
num_links = num_anchors * 8 + (num_anchors -
np.prod(feat_shapes[feat_layers[0]])) * 4
#init batch size
global gpus
gpus = util.tf.get_available_gpus(num_gpus)
global num_clones
num_clones = len(gpus)
global clone_scopes
clone_scopes = ['clone_%d' % (idx) for idx in xrange(num_clones)]
_set_batch_size(batch_size)
global batch_size_per_gpu
batch_size_per_gpu = batch_size / num_clones
if batch_size_per_gpu < 1:
raise ValueError(
'Invalid batch_size [=%d], resulting in 0 images per gpu.' %
(batch_size))
def eval():
with tf.name_scope('test'):
with tf.variable_scope(tf.get_variable_scope(),
reuse=True): # the variables has been created in config.init_config
image = tf.placeholder(dtype=tf.int32, shape=[None, None, 3])
image_shape = tf.placeholder(dtype=tf.int32, shape=[3, ])
processed_image, _, _, _, _ = ssd_vgg_preprocessing.preprocess_image(image, None, None, None, None,
out_shape=config.image_shape,
data_format=config.data_format,
is_training=False)
b_image = tf.expand_dims(processed_image, axis=0)
b_shape = tf.expand_dims(image_shape, axis=0)
net = seglink_symbol.SegLinkNet(inputs=b_image, data_format=config.data_format)
bboxes_pred = seglink.tf_seglink_to_bbox(net.seg_scores, net.link_scores,
net.seg_offsets,
image_shape=b_shape,
seg_conf_threshold=config.seg_conf_threshold,
link_conf_threshold=config.link_conf_threshold)
image_names = util.io.ls(FLAGS.dataset_dir)
sess_config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True)
if FLAGS.gpu_memory_fraction < 0:
sess_config.gpu_options.allow_growth = True
elif FLAGS.gpu_memory_fraction > 0:
sess_config.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_memory_fraction
checkpoint_dir = util.io.get_dir(FLAGS.checkpoint_path)
logdir = util.io.join_path(FLAGS.checkpoint_path, 'test', FLAGS.dataset_name + '_' + FLAGS.dataset_split_name)
saver = tf.train.Saver()
if util.io.is_dir(FLAGS.checkpoint_path):
checkpoint = util.tf.get_latest_ckpt(FLAGS.checkpoint_path)
else:
checkpoint = FLAGS.checkpoint_path
tf.logging.info('testing', checkpoint)
with tf.Session(config=sess_config) as sess:
saver.restore(sess, checkpoint)
checkpoint_name = util.io.get_filename(str(checkpoint))
dump_path = util.io.join_path(logdir, checkpoint_name,
'seg_link_conf_th_%f_%f' % (
config.seg_conf_threshold, config.link_conf_threshold))
txt_path = util.io.join_path(dump_path, 'txt')
zip_path = util.io.join_path(dump_path, '%s_seg_link_conf_th_%f_%f.zip' % (
checkpoint_name, config.seg_conf_threshold, config.link_conf_threshold))
# write detection result as txt files
def write_result_as_txt(image_name, bboxes, path):
filename = util.io.join_path(path, 'res_%s.txt' % (image_name))
lines = []
for b_idx, bbox in enumerate(bboxes):
values = [int(v) for v in bbox]
line = "%d, %d, %d, %d, %d, %d, %d, %d\n" % tuple(values)
lines.append(line)
util.io.write_lines(filename, lines)
print('result has been written to:', filename)
for iter, image_name in enumerate(image_names):
image_data = util.img.imread(util.io.join_path(FLAGS.dataset_dir, image_name), rgb=True)
image_name = image_name.split('.')[0]
image_bboxes = sess.run([bboxes_pred], feed_dict={image: image_data, image_shape: image_data.shape})
print('%d/%d: %s' % (iter + 1, len(image_names), image_name))
write_result_as_txt(image_name, image_bboxes[0], txt_path)
self.counter = self.counter + 1
b = np.array(layer.blobs[1].data)
print('Load biases from convolution layer:', layer.name, b.shape)
return tf.cast(b, dtype)
return _initializer
caffe_scope = CaffeScope()
# def get_seglink_model():
fake_image = tf.placeholder(dtype=tf.float32, shape=[1, 512, 1024, 3])
seglink_net = seglink_symbol.SegLinkNet(
inputs=fake_image,
weight_decay=0.01,
weights_initializer=caffe_scope.conv_weights_init(),
biases_initializer=caffe_scope.conv_biases_init())
init_op = tf.global_variables_initializer()
with tf.Session() as session:
# Run the init operation.
session.run(init_op)
# Save model in checkpoint.
saver = tf.train.Saver(write_version=2)
parent_dir = util.io.get_dir(caffemodel_path)
filename = util.io.get_filename(caffemodel_path)
parent_dir = util.io.mkdir(util.io.join_path(parent_dir, 'seglink'))
filename = filename.replace('.caffemodel', '.ckpt')
ckpt_path = util.io.join_path(parent_dir, filename)
saver.save(session, ckpt_path, write_meta_graph=False)
def create_clones(batch_queue):
with tf.device('/cpu:0'):
global_step = slim.create_global_step()
learning_rate = tf.constant(FLAGS.learning_rate, name='learning_rate')
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.MomentumOptimizer(learning_rate,
momentum=FLAGS.momentum,
name='Momentum')
# place clones
seglink_loss = 0
# for summary only
gradients = []
for clone_idx, gpu in enumerate(config.gpus):
do_summary = clone_idx == 0 # only summary on the first clone
with tf.variable_scope(tf.get_variable_scope(
), reuse=True): # the variables has been created in config.init_config
with tf.name_scope(config.clone_scopes[clone_idx]) as clone_scope:
with tf.device(gpu) as clone_device:
b_image, b_seg_label, b_seg_loc, b_link_label = batch_queue.dequeue(
)
net = seglink_symbol.SegLinkNet(
inputs=b_image, data_format=config.data_format)
# build seglink loss
net.build_loss(seg_labels=b_seg_label,
seg_offsets=b_seg_loc,
link_labels=b_link_label,
do_summary=do_summary)
# gather seglink losses
losses = tf.get_collection(tf.GraphKeys.LOSSES,
clone_scope)
assert len(
losses
) == 3 # 3 is the number of seglink losses: seg_cls, seg_loc, link_cls
total_clone_loss = tf.add_n(losses) / config.num_clones
seglink_loss = seglink_loss + total_clone_loss
# gather regularization loss and add to clone_0 only
if clone_idx == 0:
regularization_loss = tf.add_n(
tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES))
total_clone_loss = total_clone_loss + regularization_loss
# compute clone gradients
clone_gradients = optimizer.compute_gradients(
total_clone_loss) # all variables will be updated.
gradients.append(clone_gradients)
tf.summary.scalar('seglink_loss', seglink_loss)
tf.summary.scalar('regularization_loss', regularization_loss)
# add all gradients together
# note that the gradients do not need to be averaged, because the average operation has been done on loss.
averaged_gradients = sum_gradients(gradients)
update_op = optimizer.apply_gradients(averaged_gradients,
global_step=global_step)
train_ops = [update_op]
# moving average
if FLAGS.using_moving_average:
tf.logging.info('using moving average in training, \
with decay = %f' % (FLAGS.moving_average_decay))
ema = tf.train.ExponentialMovingAverage(FLAGS.moving_average_decay)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([update_op]): # ema after updating
train_ops.append(tf.group(ema_op))
train_op = control_flow_ops.with_dependencies(train_ops,
seglink_loss,
name='train_op')
return train_op
def eval(dataset):
dict_metrics = {}
checkpoint_dir = util.io.get_dir(FLAGS.checkpoint_path)
logdir = util.io.join_path(
checkpoint_dir, 'eval',
"%s_%s" % (FLAGS.dataset_name, FLAGS.dataset_split_name))
global_step = slim.get_or_create_global_step()
with tf.name_scope('evaluation_%dx%d' %
(FLAGS.eval_image_height, FLAGS.eval_image_width)):
with tf.variable_scope(tf.get_variable_scope(
), reuse=True): # the variables has been created in config.init_config
# get input tensor
image, seg_label, seg_loc, link_gt, filename, shape, gignored, gxs, gys = read_dataset(
dataset)
# expand dim if needed
b_image = tf.expand_dims(image, axis=0)
b_seg_label = tf.expand_dims(seg_label, axis=0)
b_seg_loc = tf.expand_dims(seg_loc, axis=0)
b_link_gt = tf.expand_dims(link_gt, axis=0)
b_shape = tf.expand_dims(shape, axis=0)
# build seglink loss
net = seglink_symbol.SegLinkNet(inputs=b_image,
data_format=config.data_format)
net.build_loss(
seg_labels=b_seg_label,
seg_offsets=b_seg_loc,
link_labels=b_link_gt,
do_summary=False
) # the summary will be added in the following lines
# gather seglink losses
losses = tf.get_collection(tf.GraphKeys.LOSSES)
assert len(
losses
) == 3 # 3 is the number of seglink losses: seg_cls, seg_loc, link_cls
for loss in tf.get_collection(tf.GraphKeys.LOSSES):
dict_metrics[loss.op.name] = slim.metrics.streaming_mean(loss)
seglink_loss = tf.add_n(losses)
dict_metrics['seglink_loss'] = slim.metrics.streaming_mean(
seglink_loss)
# Add metrics to summaries.
for name, metric in dict_metrics.items():
tf.summary.scalar(name, metric[0])
# shape = (height, width, channels) when format = NHWC TODO
gxs = gxs * tf.cast(shape[1], gxs.dtype)
gys = gys * tf.cast(shape[0], gys.dtype)
if FLAGS.do_grid_search:
# grid search
seg_ths = np.arange(0.5, 0.91, 0.1)
link_ths = seg_ths
else:
seg_ths = [FLAGS.seg_conf_threshold]
link_ths = [FLAGS.link_conf_threshold]
eval_result_path = util.io.join_path(
logdir, 'eval_on_%s_%s.log' %
(FLAGS.dataset_name, FLAGS.dataset_split_name))
for seg_th in seg_ths:
for link_th in link_ths:
config._set_det_th(seg_th, link_th)
eval_result_msg = 'seg_conf_threshold=%f, link_conf_threshold = %f, '\
%(config.seg_conf_threshold, config.link_conf_threshold)
eval_result_msg += 'iter = %r, recall = %r, precision = %f, fmean = %r'
with tf.name_scope('seglink_conf_th_%f_%f'\
%(config.seg_conf_threshold, config.link_conf_threshold)):
# decode seglink to bbox output, with absolute length, instead of being within [0,1]
bboxes_pred = seglink.tf_seglink_to_bbox(
net.seg_scores,
net.link_scores,
net.seg_offsets,
b_shape,
seg_conf_threshold=seg_th,
link_conf_threshold=link_th)
# bboxes_pred = tf.Print(bboxes_pred, [tf.shape(bboxes_pred)], '%f_%f, shape of bboxes = '%(seg_th, link_th))
# calculate true positive and false positive
# the xs and ys from tfrecord is 0~1, resize them to absolute length before matching.
num_gt_bboxes, tp, fp = tfe_bboxes.bboxes_matching(
bboxes_pred, gxs, gys, gignored)
tp_fp_metric = tfe_metrics.streaming_tp_fp_arrays(
num_gt_bboxes, tp, fp)
dict_metrics['tp_fp_%f_%f' %
(config.seg_conf_threshold,
config.link_conf_threshold)] = (
tp_fp_metric[0], tp_fp_metric[1])
# precision and recall
precision, recall = tfe_metrics.precision_recall(
*tp_fp_metric[0])
fmean = tfe_metrics.fmean(precision, recall)
fmean = util.tf.Print(
fmean,
data=[global_step, recall, precision, fmean],
msg=eval_result_msg,
file=eval_result_path,
mode='a')
fmean = tf.Print(
fmean, [recall, precision, fmean],
'%f_%f, Recall, Precision, Fmean = ' %
(seg_th, link_th))
tf.summary.scalar('Precision', precision)
tf.summary.scalar('Recall', recall)
tf.summary.scalar('F-mean', fmean)
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map(
dict_metrics)
sess_config = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)
if FLAGS.gpu_memory_fraction < 0:
sess_config.gpu_options.allow_growth = True
elif FLAGS.gpu_memory_fraction > 0:
sess_config.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_memory_fraction
# Variables to restore: moving avg. or normal weights.
if FLAGS.using_moving_average:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[global_step.op.name] = global_step
else:
variables_to_restore = slim.get_variables_to_restore()
if util.io.is_dir(FLAGS.checkpoint_path):
slim.evaluation.evaluation_loop(
master='',
eval_op=list(names_to_updates.values()),
num_evals=dataset.num_samples,
variables_to_restore=variables_to_restore,
checkpoint_dir=checkpoint_dir,
logdir=logdir,
session_config=sess_config)
else:
slim.evaluation.evaluate_once(
master='',
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore,
num_evals=500, #dataset.num_samples,
checkpoint_path=FLAGS.checkpoint_path,
logdir=logdir,
session_config=sess_config)
