def extract_features(self, preprocessed_inputs):
"""Extract features from preprocessed inputs.
Args:
preprocessed_inputs: a [batch, height, width, channels] float tensor
representing a batch of images.
Returns:
feature_maps: a list of tensors where the ith tensor has shape
[batch, height_i, width_i, depth_i]
"""
preprocessed_inputs = shape_utils.check_min_image_dim(
33, preprocessed_inputs)
feature_map_layout = {
'from_layer':
['layer_15/expansion_output', 'layer_19', '', '', '', ''],
'layer_depth': [-1, -1, 512, 256, 256, 128],
'use_depthwise': self._use_depthwise,
'use_explicit_padding': self._use_explicit_padding,
}
with tf.variable_scope('MobilenetV2',
reuse=self._reuse_weights) as scope:
with slim.arg_scope(
mobilenet_v2.training_scope(is_training=None, bn_decay=0.9997)), \
slim.arg_scope(
[mobilenet.depth_multiplier], min_depth=self._min_depth):
with (slim.arg_scope(self._conv_hyperparams_fn())
if self._override_base_feature_extractor_hyperparams else
context_manager.IdentityContextManager()):
_, image_features = mobilenet_v2.mobilenet_base(
ops.pad_to_multiple(preprocessed_inputs,
self._pad_to_multiple),
final_endpoint='layer_19',
depth_multiplier=self._depth_multiplier,
use_explicit_padding=self._use_explicit_padding,
scope=scope)
with slim.arg_scope(self._conv_hyperparams_fn()):
feature_maps = feature_map_generators.multi_resolution_feature_maps(
feature_map_layout=feature_map_layout,
depth_multiplier=self._depth_multiplier,
min_depth=self._min_depth,
insert_1x1_conv=True,
image_features=image_features)
return feature_maps.values()
def model_fn(features, mode):
train_mode = mode == tf.estimator.ModeKeys.TRAIN
tf.summary.image("images", features['image/encoded'])
#Create the model inference
with slim.arg_scope(
mobilenet_v2.training_scope(is_training=train_mode,
weight_decay=0.0005,
stddev=1.,
bn_decay=0.99)):
#TODO: Check mobilenet_v1 module, var "excluding
logits, _ = mobilenet_v2.mobilenet(features['image/encoded'],
depth_multiplier=1.4,
num_classes=len(labels_to_name))
excluding = ['MobilenetV2/Logits']
variables_to_restore = slim.get_variables_to_restore(exclude=excluding)
if (not ckpt_state) and checkpoint_file and train_mode:
variables_to_restore = variables_to_restore[1:]
tf.train.init_from_checkpoint(
checkpoint_file,
{v.name.split(':')[0]: v
for v in variables_to_restore})
predicted_classes = tf.argmax(logits, axis=1)
labels = tf.cast(features["image/class/label"], tf.int32)
#Defining losses and regulization ops:
with tf.name_scope("loss_op"):
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels,
logits=logits)
total_loss = tf.losses.get_total_loss(
) #obtain the regularization losses as well
#FIXME: Replace classifier function (sigmoid / softmax)
if mode != tf.estimator.ModeKeys.PREDICT:
metrics = {
'Accuracy':
tf.metrics.accuracy(labels, predicted_classes, name="acc_op"),
'Precision':
tf.metrics.precision(labels,
predicted_classes,
name="precision_op"),
'Recall':
tf.metrics.recall(labels, predicted_classes, name="recall_op")
}
for name, value in metrics.items():
tf.summary.scalar(name, value[1])
if mode == tf.estimator.ModeKeys.TRAIN:
#Create the global step for monitoring the learning_rate and training:
global_step = tf.train.get_or_create_global_step()
with tf.name_scope("learning_rate"):
lr = tf.train.exponential_decay(
learning_rate=initial_learning_rate,
global_step=global_step,
decay_steps=decay_steps,
decay_rate=learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', lr)
#Define Optimizer with decay learning rate:
with tf.name_scope("optimizer"):
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
update_ops=tf.get_collection(tf.GraphKeys.UPDATE_OPS))
return tf.estimator.EstimatorSpec(mode,
loss=total_loss,
train_op=train_op)
#For Evaluation Mode
if mode == tf.estimator.ModeKeys.EVAL:
return tf.estimator.EstimatorSpec(mode,
loss=total_loss,
eval_metric_ops=metrics)
#For Predict/Inference Mode:
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'classes': predicted_classes,
'probabilities': tf.nn.softmax(logits, name="Softmax")
}
export_outputs = {
'prediction': tf.estimator.export.PredictOutput(predictions)
}
return tf.estimator.EstimatorSpec(mode,
predictions=predictions,
export_outputs=export_outputs)
def extract_features(self, preprocessed_inputs):
"""Extract features from preprocessed inputs.
Args:
preprocessed_inputs: a [batch, height, width, channels] float tensor
representing a batch of images.
Returns:
feature_maps: a list of tensors where the ith tensor has shape
[batch, height_i, width_i, depth_i]
"""
preprocessed_inputs = shape_utils.check_min_image_dim(
33, preprocessed_inputs)
with tf.variable_scope('MobilenetV2',
reuse=self._reuse_weights) as scope:
with slim.arg_scope(
mobilenet_v2.training_scope(is_training=None, bn_decay=0.9997)), \
slim.arg_scope(
[mobilenet.depth_multiplier], min_depth=self._min_depth):
with (slim.arg_scope(self._conv_hyperparams_fn())
if self._override_base_feature_extractor_hyperparams else
context_manager.IdentityContextManager()):
_, image_features = mobilenet_v2.mobilenet_base(
ops.pad_to_multiple(preprocessed_inputs,
self._pad_to_multiple),
final_endpoint='layer_19',
depth_multiplier=self._depth_multiplier,
conv_defs=_CONV_DEFS if self._use_depthwise else None,
use_explicit_padding=self._use_explicit_padding,
scope=scope)
depth_fn = lambda d: max(int(d * self._depth_multiplier), self.
_min_depth)
with slim.arg_scope(self._conv_hyperparams_fn()):
with tf.variable_scope('fpn', reuse=self._reuse_weights):
feature_blocks = [
'layer_4', 'layer_7', 'layer_14', 'layer_19'
]
base_fpn_max_level = min(self._fpn_max_level, 5)
feature_block_list = []
for level in range(self._fpn_min_level,
base_fpn_max_level + 1):
feature_block_list.append(feature_blocks[level - 2])
fpn_features = feature_map_generators.fpn_top_down_feature_maps(
[(key, image_features[key])
for key in feature_block_list],
depth=depth_fn(self._additional_layer_depth),
use_depthwise=self._use_depthwise)
feature_maps = []
for level in range(self._fpn_min_level,
base_fpn_max_level + 1):
feature_maps.append(fpn_features['top_down_{}'.format(
feature_blocks[level - 2])])
last_feature_map = fpn_features['top_down_{}'.format(
feature_blocks[base_fpn_max_level - 2])]
# Construct coarse features
for i in range(base_fpn_max_level + 1,
self._fpn_max_level + 1):
if self._use_depthwise:
conv_op = functools.partial(slim.separable_conv2d,
depth_multiplier=1)
else:
conv_op = slim.conv2d
last_feature_map = conv_op(
last_feature_map,
num_outputs=depth_fn(self._additional_layer_depth),
kernel_size=[3, 3],
stride=2,
padding='SAME',
scope='bottom_up_Conv2d_{}'.format(
i - base_fpn_max_level + 19))
feature_maps.append(last_feature_map)
return feature_maps
checkpoint_file = os.path.join(checkpoint_dir, "model-115001")
image_size = 224
#Labels
labels_to_name = {0: 'negative', 1: 'positive'}
tf.logging.set_verbosity(tf.logging.INFO)
path_img = ""
file_input = tf.placeholder(tf.string, ())
image = tf.image.decode_image(tf.read_file(file_input), channels=3)
copy = tf.identity(image)
image = tf.image.convert_image_dtype(image, tf.float32)
image.set_shape([None, None, 3])
image_a = dp.preprocess_image(image, 224, 224, is_training=False)
images_bis = tf.expand_dims(image_a, 0)
with slim.arg_scope(mobilenet_v2.training_scope(is_training=False)):
#TODO: Check mobilenet_v1 module, var "excluding
logits, end_points = mobilenet_v2.mobilenet(
images_bis, depth_multiplier=1.4, num_classes=len(labels_to_name))
variables = slim.get_variables_to_restore()
init_fn = slim.assign_from_checkpoint_fn(checkpoint_file, variables)
embedding = end_points["layer_18/output"][0]
weights = tf.reduce_mean(tf.get_default_graph().get_tensor_by_name(
"MobilenetV2/expanded_conv_16/project/weights:0"),
axis=[0, 1, 2])
with tf.Session() as sess:
init_fn(sess)
emb, ng, raw_img = sess.run([embedding, weights, image],
feed_dict={file_input: path_img})
cam = np.zeros(emb.shape[0:2], dtype=np.float32)
def model(images, weight_decay=1e-5, is_training=True):
'''
define the model, we use slim's implemention of resnet
'''
images = mean_image_subtraction(images)
if FLAGS.backbone == 'Resnet':
with slim.arg_scope(
resnet_v1.resnet_arg_scope(weight_decay=weight_decay)):
logits, end_points = resnet_v1.resnet_v1_50(
images, is_training=is_training, scope='resnet_v1_50')
end_points['pool2'] = end_points[
'resnet_v1_50/block1/unit_2/bottleneck_v1/conv3']
end_points['pool3'] = end_points[
'resnet_v1_50/block2/unit_3/bottleneck_v1/conv3']
end_points['pool4'] = end_points[
'resnet_v1_50/block3/unit_5/bottleneck_v1/conv3']
end_points['pool5'] = end_points[
'resnet_v1_50/block4/unit_3/bottleneck_v1/conv3']
elif FLAGS.backbone == 'Mobilenet':
with slim.arg_scope(
mobilenet_v2.training_scope(weight_decay=weight_decay)):
logits, endpoints = mobilenet_v2.mobilenet_base(
images, is_training=is_training)
end_points = dict()
end_points['pool2'] = endpoints['layer_4/output']
end_points['pool3'] = endpoints['layer_6/output']
end_points['pool4'] = endpoints['layer_13/output']
end_points['pool5'] = endpoints['layer_19']
print(end_points['pool2'], end_points['pool3'], end_points['pool4'],
end_points['pool5'])
elif FLAGS.backbone == 'Pvanet' or 'Pvanet2x':
print("Backbone is Pvanet")
expansion = 1 if FLAGS.backbone == 'Pvanet' else 2
with slim.arg_scope(
pvanet_scope(is_training, weight_decay=weight_decay)):
net, end_points = pvanet(images, expansion=expansion)
end_points['pool2'] = end_points['conv2']
end_points['pool3'] = end_points['conv3']
end_points['pool4'] = end_points['conv4']
end_points['pool5'] = end_points['conv5']
if FLAGS.decoder == 'CRAFT':
print("deep decoder")
with tf.variable_scope('feature_fusion', values=[end_points.values]):
batch_norm_params = {
'decay': 0.997,
'epsilon': 1e-5,
'scale': True,
'is_training': is_training
}
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params,
weights_regularizer=slim.l2_regularizer(weight_decay)):
x = unpool(end_points['pool5'])
x = tf.concat([x, end_points['pool4']], axis=-1)
x = slim.conv2d(x, 256, 1)
x = slim.conv2d(x, 128, 3)
print("up_4 shape : ", x.shape)
x = unpool(x)
x = tf.concat([x, end_points['pool3']], axis=-1)
x = slim.conv2d(x, 128, 1)
x = slim.conv2d(x, 64, 3)
print("up_3 shape : ", x.shape)
x = unpool(x)
x = tf.concat([x, end_points['pool2']], axis=-1)
x = slim.conv2d(x, 64, 1)
x = slim.conv2d(x, 32, 3)
print("up_2 shape : ", x.shape)
x = slim.conv2d(x, 32, 3)
x = slim.conv2d(x, 32, 3)
x = slim.conv2d(x, 16, 3)
x = slim.conv2d(x, 16, 3)
F_score = slim.conv2d(x,
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None)
# 4 channel of axis aligned bbox and 1 channel rotation angle
geo_map = slim.conv2d(
x, 4, 1, activation_fn=tf.nn.sigmoid,
normalizer_fn=None) * 512
angle_map = (slim.conv2d(
x, 1, 1, activation_fn=tf.nn.sigmoid, normalizer_fn=None) -
0.5) * np.pi / 2 # angle is between [-45, 45]
F_geometry = tf.concat([geo_map, angle_map], axis=-1)
elif FLAGS.decoder == 'Expand':
expand_kwargs = {
'expansion_size': expand_input(6),
'split_expansion': 1,
'normalizer_fn': slim.batch_norm,
'residual': True
}
batch_norm_params = {
'center': True,
'scale': True,
'is_training': is_training
}
with tf.variable_scope('decoder'):
with slim.arg_scope(
[slim.conv2d, slim.fully_connected, slim.separable_conv2d],
activation_fn=tf.nn.relu6,
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params,
weights_regularizer=slim.l2_regularizer(weight_decay)):
with slim.arg_scope([ops.expanded_conv], **expand_kwargs):
with slim.arg_scope([slim.conv2d, slim.separable_conv2d],
padding='SAME'):
x = unpool(end_points['pool5'])
x = tf.concat([x, end_points['pool4']], axis=-1)
x = ops.expanded_conv(x,
48,
depthwise_location='input',
expansion_size=expand_input(
1, divisible_by=1))
print("up_4 : ", x.shape)
x = unpool(x)
x = tf.concat([x, end_points['pool3']], axis=-1)
print("up_3 concat : ", x.shape)
x = ops.expanded_conv(x, 24)
x = unpool(x)
x = tf.concat([x, end_points['pool2']], axis=-1)
print("up_3 concat : ", x.shape)
x = ops.expanded_conv(x, 16)
x = slim.conv2d(x, 16, 3)
x = slim.conv2d(x, 16, 3)
F_score = slim.conv2d(x,
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None)
# 4 channel of axis aligned bbox and 1 channel rotation angle
geo_map = slim.conv2d(x,
4,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None) * 512
angle_map = (
slim.conv2d(x,
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None) -
0.5) * np.pi / 2 # angle is between [-45, 45]
F_geometry = tf.concat([geo_map, angle_map], axis=-1)
elif FLAGS.decoder == 'CAST':
expand_kwargs = {
'expansion_size': expand_input(6),
'split_expansion': 1,
'normalizer_fn': slim.batch_norm,
'residual': True
}
batch_norm_params = {
'center': True,
'scale': True,
'is_training': is_training
}
with tf.variable_scope('decoder'):
with slim.arg_scope(
[slim.conv2d, slim.fully_connected, slim.separable_conv2d],
activation_fn=tf.nn.relu6,
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params,
weights_regularizer=slim.l2_regularizer(weight_decay)):
with slim.arg_scope([ops.expanded_conv], **expand_kwargs):
with slim.arg_scope([slim.conv2d, slim.separable_conv2d],
padding='SAME'):
x = unpool(end_points['pool5'])
x = tf.concat([x, end_points['pool4']], axis=-1)
x = ops.expanded_conv(x,
128,
expansion_size=expand_input(
1, divisible_by=1))
x = ops.expanded_conv(x, 96)
print("up_4 shape : ", x.shape)
x = unpool(x)
x = tf.concat([x, end_points['pool3']], axis=-1)
print("up_3 concat : ", x.shape)
x = ops.expanded_conv(x, 48)
x = slim.conv2d(x, 64, 3)
print("up_3 shape : ", x.shape)
x = unpool(x)
x = tf.concat([x, end_points['pool2']], axis=-1)
print("up_2 concat : ", x.shape)
x = slim.conv2d(x, 32, 3)
print("up_2 shape : ", x.shape)
x = slim.conv2d(x, 16, 3)
x = slim.conv2d(x, 16, 3)
F_score = slim.conv2d(x,
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None)
# 4 channel of axis aligned bbox and 1 channel rotation angle
geo_map = slim.conv2d(x,
4,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None) * 512
angle_map = (
slim.conv2d(x,
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None) -
0.5) * np.pi / 2 # angle is between [-45, 45]
F_geometry = tf.concat([geo_map, angle_map], axis=-1)
else:
print("original decoder")
with tf.variable_scope('feature_fusion', values=[end_points.values]):
batch_norm_params = {
'decay': 0.997,
'epsilon': 1e-5,
'scale': True,
'is_training': is_training
}
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params,
weights_regularizer=slim.l2_regularizer(weight_decay)):
f = [
end_points['pool5'], end_points['pool4'],
end_points['pool3'], end_points['pool2']
]
for i in range(4):
print('Shape of f_{} {}'.format(i, f[i].shape))
g = [None, None, None, None]
h = [None, None, None, None]
num_outputs = [None, 128, 64, 32]
for i in range(4):
if i == 0:
h[i] = f[i]
else:
c1_1 = slim.conv2d(
tf.concat([g[i - 1], f[i]], axis=-1),
num_outputs[i], 1)
h[i] = slim.conv2d(c1_1, num_outputs[i], 3)
if i <= 2:
g[i] = unpool(h[i])
else:
g[i] = slim.conv2d(h[i], num_outputs[i], 3)
print('Shape of h_{} {}, g_{} {}'.format(
i, h[i].shape, i, g[i].shape))
# here we use a slightly different way for regression part,
# we first use a sigmoid to limit the regression range, and also
# this is do with the angle map
F_score = slim.conv2d(g[3],
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None)
# 4 channel of axis aligned bbox and 1 channel rotation angle
geo_map = slim.conv2d(g[3],
4,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None) * 512
angle_map = (slim.conv2d(g[3],
1,
1,
activation_fn=tf.nn.sigmoid,
normalizer_fn=None) -
0.5) * np.pi / 2 # angle is between [-45, 45]
F_geometry = tf.concat([geo_map, angle_map], axis=-1)
return F_score, F_geometry