def build_unet(self, x, training=False):
dir_bins = self.cfg.dir_bins
drop_rate = self.cfg.drop_rate
batch_norm = self.cfg.batch_norm
layers_per_block = self.cfg.layers_per_block
# first conv layer:
x = conv2d(x, 48)
stack_skip = []
# downsample (encoder):
for i in range(self.n_block_layers):
n = layers_per_block[str(i)]
name = 'down_block_' + str(i)
x_new = dense_block(x, name, n_layers=n, training=training)
x = concat(x, x_new)
stack_skip.append(x)
x = transition_down(x, training)
# bottleneck:
n = layers_per_block[str(self.n_block_layers)]
x_new = dense_block(x, name='bottleneck_block', n_layers=n,
training=training)
rpn_inputs = {self.n_block_layers: concat(x, x_new)}
# upsample (decoder):
for i in reversed(range(self.n_block_layers)):
n = layers_per_block[str(i)]
name = 'up_block_' + str(i)
x_skip = stack_skip[i]
x = transition_up(x_new)
x = concat(x, x_skip)
x_new = dense_block(x, name, n_layers=n, training=training,
batch_norm=batch_norm, drop_rate=drop_rate)
rpn_inputs[i] = concat(x, x_new)
# last conv layer:
mask_logits = conv2d(x_new, 1 + dir_bins, k_size=1)
return mask_logits, rpn_inputs
def denseUnpool(denselist, previous, depth):
# Unpooling to double size and a different feature count.
assert type(denselist) is list
assert type(previous) is list
dense_conv2d = lambda p1, p2, p3: layers.denselist_conv2d(
p1, p2, p3, relu_factor=relu_factor)
feature_count = upscale_feature_counts[depth]
if config.model.unpool_type == 'conv':
# Convolution unpool directly to the correct feature count.
input = layers.concat(
denselist
) # TODO: Use layers.denselist_unpool_conv to save memory.
current = unpool(input, feature_count)
else:
# Convolve to the correct feature count, then unpool.
current = layers.denselist_conv2d(denselist, feature_count, 1)
current = unpool(current, None)
result = []
result.extend(previous)
result.append(current)
return result
def __init__(self,
input,
gt_label,
gt_box,
gt_difficult,
class_num,
background_label=0,
overlap_threshold=0.5,
evaluate_difficult=True,
ap_version='integral'):
super(DetectionMAP, self).__init__("map_eval")
gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype)
gt_difficult = layers.cast(x=gt_difficult, dtype=gt_box.dtype)
label = layers.concat([gt_label, gt_difficult, gt_box], axis=1)
# calculate mean average precision (mAP) of current mini-batch
map = layers.detection_map(input,
label,
class_num,
background_label,
overlap_threshold=overlap_threshold,
evaluate_difficult=evaluate_difficult,
ap_version=ap_version)
self.create_state(dtype='int32', shape=None, suffix='accum_pos_count')
self.create_state(dtype='float32', shape=None, suffix='accum_true_pos')
self.create_state(dtype='float32',
shape=None,
suffix='accum_false_pos')
self.has_state = None
var = self.helper.create_variable(persistable=True,
dtype='int32',
shape=[1])
self.helper.set_variable_initializer(
var, initializer=Constant(value=int(0)))
self.has_state = var
# calculate accumulative mAP
accum_map = layers.detection_map(input,
label,
class_num,
background_label,
overlap_threshold=overlap_threshold,
evaluate_difficult=evaluate_difficult,
has_state=self.has_state,
input_states=self.states,
out_states=self.states,
ap_version=ap_version)
layers.fill_constant(shape=self.has_state.shape,
value=1,
dtype=self.has_state.dtype,
out=self.has_state)
self.cur_map = map
self.accum_map = accum_map
def conv_block(self, inputs):
outputs = batch_normalization()(inputs)
if self.bottleneck:
outputs = relu()(outputs)
outputs = conv2d(filters=4 * self.growth_rate,
kernel_size=(1, 1))(outputs)
outputs = batch_normalization()(outputs)
outputs = relu()(outputs)
outputs = conv2d(filters=self.growth_rate, kernel_size=(3, 3))(outputs)
outputs = concat()([inputs, outputs])
return outputs
def u_net(self, x, layers=4, base_channel=64, train=True):
ds_layers = {}
ds_layer_shape = {}
# down sample layers
for layer in range(0, layers-1):
f_channels = base_channel * (2**layer)
layer_name = 'ds_{}'.format(layer)
if layer == 0:
x = conv2d(x, [3, 3, 3, f_channels], layer_name + '_1')
else:
x = conv2d(x, [3, 3, f_channels/2, f_channels], layer_name + '_1')
x = conv2d(x, [3, 3, f_channels, f_channels], layer_name + '_2')
ds_layers[layer] = x
ds_layer_shape[layer] = tf.shape(x)
x = maxpooling(x)
# bottom layer
f_channels = base_channel * (2**(layers-1))
x = conv2d(x, [3, 3, f_channels/2, f_channels], 'bottom_1')
x = conv2d(x, [3, 3, f_channels, f_channels], 'bottom_2')
# up sample layers
for layer in range(layers-2, -1, -1):
f_channels = base_channel * (2**layer)
layer_name = 'up_{}'.format(layer)
x = deconv2d(x, [3, 3, f_channels, 2*f_channels], ds_layer_shape[layer], layer_name + '_deconv2d')
# add the previous down sumple layer to the up sample layer
x = concat(ds_layers[layer], x)
x = conv2d(x, [3, 3, 2*f_channels, f_channels], layer_name + '_conv_1')
x = conv2d(x, [3, 3, f_channels, f_channels], layer_name + '_conv_2')
#if train:
# x = tf.nn.dropout(x, self.dropout)
# add 1x1 convolution layer to change channel to one
x = conv2d(x, [1, 1, base_channel, 1], 'conv_1x1', activation='no')
logits = tf.squeeze(x, axis=3)
return logits
def inception(self, inputs, filters, layers=None):
"""
Inception Module (default is inception v1)
If you want to use another inception module,
set `layers' appropriate layer function.
Below is inception v2 (Module A) example:
inception_layers = [[conv2d(filters=filters, kernel_size=(1, 1))],
[conv2d(filters=filters, kernel_size=(1, 1)),
conv2d(filters=filters, kernel_size=(3, 3))],
[conv2d(filters=filters, kernel_size=(1, 1)),
conv2d(filters=filters, kernel_size=(3, 3))
conv2d(filters=filters, kernel_size=(3, 3))],
[max_pooling2d(pool_size=(3, 3), strides=1),
conv2d(filters=filters, kernel_size=(1, 1))]]
outputs = self.inception(inputs, filters, layers=inception_layers)
"""
if layers is None:
layers = [[conv2d(filters=filters, kernel_size=(1, 1))],
[
conv2d(filters=filters, kernel_size=(1, 1)),
conv2d(filters=filters, kernel_size=(3, 3))
],
[
conv2d(filters=filters, kernel_size=(1, 1)),
conv2d(filters=filters, kernel_size=(5, 5))
],
[
max_pooling2d(pool_size=(3, 3), strides=1),
conv2d(filters=filters, kernel_size=(1, 1))
]]
outputs = [inputs] * len(layers)
for i in range(len(layers)):
for layer in layers[i]:
outputs[i] = layer(outputs[i])
outputs = concat()(outputs)
return outputs
def build(color_inputs, num_classes, is_training):
"""
Build unet network:
----------
Args:
color_inputs: Tensor, [batch_size, height, width, 3]
num_classes: Integer, number of segmentation (annotation) labels
is_training: Boolean, in training mode or not (for dropout & bn)
Returns:
logits: Tensor, predicted annotated image flattened
[batch_size * height * width, num_classes]
"""
dropout_keep_prob = tf.where(is_training, 0.2, 1.0)
# Encoder Section
# Block 1
color_conv1_1 = layers.conv_btn(color_inputs, [3, 3], 64, 'conv1_1', is_training = is_training)
color_conv1_2 = layers.conv_btn(color_conv1_1, [3, 3], 64, 'conv1_2', is_training = is_training)
color_pool1 = layers.maxpool(color_conv1_2, [2, 2], 'pool1')
# Block 2
color_conv2_1 = layers.conv_btn(color_pool1, [3, 3], 128, 'conv2_1', is_training = is_training)
color_conv2_2 = layers.conv_btn(color_conv2_1, [3, 3], 128, 'conv2_2', is_training = is_training)
color_pool2 = layers.maxpool(color_conv2_2, [2, 2], 'pool2')
# Block 3
color_conv3_1 = layers.conv_btn(color_pool2, [3, 3], 256, 'conv3_1', is_training = is_training)
color_conv3_2 = layers.conv_btn(color_conv3_1, [3, 3], 256, 'conv3_2', is_training = is_training)
color_pool3 = layers.maxpool(color_conv3_2, [2, 2], 'pool3')
color_drop3 = layers.dropout(color_pool3, dropout_keep_prob, 'drop3')
# Block 4
color_conv4_1 = layers.conv_btn(color_drop3, [3, 3], 512, 'conv4_1', is_training = is_training)
color_conv4_2 = layers.conv_btn(color_conv4_1, [3, 3], 512, 'conv4_2', is_training = is_training)
color_pool4 = layers.maxpool(color_conv4_2, [2, 2], 'pool4')
color_drop4 = layers.dropout(color_pool4, dropout_keep_prob, 'drop4')
# Block 5
color_conv5_1 = layers.conv_btn(color_drop4, [3, 3], 1024, 'conv5_1', is_training = is_training)
color_conv5_2 = layers.conv_btn(color_conv5_1, [3, 3], 1024, 'conv5_2', is_training = is_training)
color_drop5 = layers.dropout(color_conv5_2, dropout_keep_prob, 'drop5')
# Decoder Section
# Block 1
upsample6 = layers.deconv_upsample(color_drop5, 2, 'upsample6')
concat6 = layers.concat(upsample6, color_conv4_2, 'contcat6')
color_conv6_1 = layers.conv_btn(concat6, [3, 3], 512, 'conv6_1', is_training = is_training)
color_conv6_2 = layers.conv_btn(color_conv6_1, [3, 3], 512, 'conv6_1', is_training = is_training)
color_drop6 = layers.dropout(color_conv6_2, dropout_keep_prob, 'drop6')
# Block 2
upsample7 = layers.deconv_upsample(color_drop6, 2, 'upsample7')
concat7 = layers.concat(upsample7, color_conv3_2, 'concat7')
color_conv7_1 = layers.conv_btn(concat7, [3, 3], 256, 'conv7_1', is_training = is_training)
color_conv7_2 = layers.conv_btn(color_conv7_1, [3, 3], 256, 'conv7_1', is_training = is_training)
color_drop7 = layers.dropout(color_conv7_2, dropout_keep_prob, 'drop7')
# Block 3
upsample8 = layers.deconv_upsample(color_drop7, 2, 'upsample8')
concat8 = layers.concat(upsample8, color_conv2_2, 'concat8')
color_conv8_1 = layers.conv_btn(concat8, [3, 3], 128, 'conv8_1', is_training = is_training)
color_conv8_2 = layers.conv_btn(color_conv8_1, [3, 3], 128, 'conv8_1', is_training = is_training)
# Block 4
upsample9 = layers.deconv_upsample(color_conv9_2, 2, 'upsample9')
concat9 = layers.concat(upsample8, color_conv1_2, 'concat9')
color_conv9_1 = layers.conv_btn(concat9, [3, 3], 64, 'conv9_1', is_training = is_training)
color_conv9_2 = layers.conv_btn(color_conv9_1, [3, 3], 64, 'conv9_1', is_training = is_training)
# Block 5
score = layers.conv(color_conv9_2, [1, 1], num_classes, 'score', activation_fn = None)
logits = tf.reshape(score, (-1, num_classes))
return logits
def build_30s(color_inputs, num_classes, is_training):
"""
Build unet network:
----------
Args:
color_inputs: Tensor, [batch_size, length, 3]
num_classes: Integer, number of segmentation (annotation) labels
is_training: Boolean, in training mode or not (for dropout & bn)
Returns:
logits: Tensor, predicted annotated image flattened
[batch_size * length, num_classes]
"""
dropout_keep_prob = tf.where(is_training, 0.2, 1.0)
# Encoder Section
# Block 1
# color_conv1_1 = layers.conv_btn(color_inputs, [3, 3], 64, 'conv1_1', is_training = is_training)
color_conv1_1 = layers.conv_btn1(color_inputs,
3,
32,
'conv1_1',
is_training=is_training)
#layers.conv1(current_layer, c, ksize, stride=2, scope='conv{}'.format(i + 1), padding='SAME')
color_conv1_2 = layers.conv_btn1(color_conv1_1,
3,
32,
'conv1_2',
is_training=is_training)
color_pool1 = layers.maxpool(color_conv1_2, 4, 'pool1')
# Block 2
color_conv2_1 = layers.conv_btn1(color_pool1,
3,
32,
'conv2_1',
is_training=is_training)
color_conv2_2 = layers.conv_btn1(color_conv2_1,
3,
32,
'conv2_2',
is_training=is_training)
color_pool2 = layers.maxpool(color_conv2_2, 4, 'pool2')
# Block 3
color_conv3_1 = layers.conv_btn1(color_pool2,
3,
64,
'conv3_1',
is_training=is_training)
color_conv3_2 = layers.conv_btn1(color_conv3_1,
3,
64,
'conv3_2',
is_training=is_training)
color_pool3 = layers.maxpool(color_conv3_2, 4, 'pool3')
color_drop3 = layers.dropout(color_pool3, dropout_keep_prob, 'drop3')
# Block 4
color_conv4_1 = layers.conv_btn1(color_drop3,
3,
64,
'conv4_1',
is_training=is_training)
color_conv4_2 = layers.conv_btn1(color_conv4_1,
3,
64,
'conv4_2',
is_training=is_training)
color_pool4 = layers.maxpool(color_conv4_2, 4, 'pool4')
color_drop4 = layers.dropout(color_pool4, dropout_keep_prob, 'drop4')
# Block 5
color_conv5_1 = layers.conv_btn1(color_drop4,
3,
128,
'conv5_1',
is_training=is_training)
color_conv5_2 = layers.conv_btn1(color_conv5_1,
3,
128,
'conv5_2',
is_training=is_training)
color_drop5 = layers.dropout(color_conv5_2, dropout_keep_prob, 'drop5')
# Decoder Section
# Block 1
upsample61 = layers.deconv_upsample(color_drop5, 4, 'upsample6')
upsample61 = Cropping1D(cropping=((0, 1)))(upsample61)
concat6 = layers.concat(upsample61, color_conv4_2, 'concat6')
color_conv6_1 = layers.conv_btn1(concat6,
3,
128,
'conv6_1',
is_training=is_training)
# color_conv6_2 = layers.conv_btn1(color_conv6_1, 6, 128, 'conv6_2', is_training = is_training)
color_drop6 = layers.dropout(color_conv6_1, dropout_keep_prob, 'drop6')
# Block 2
upsample7 = layers.deconv_upsample(color_drop6, 4, 'upsample7')
# upsample7 = Cropping1D(cropping=((0, 1)))(upsample7)
concat7 = layers.concat(upsample7, color_conv3_2, 'concat7')
color_conv7_1 = layers.conv_btn1(concat7,
3,
64,
'conv7_1',
is_training=is_training)
# color_conv7_2 = layers.conv_btn1(color_conv7_1, 6, 64, 'conv7_1', is_training = is_training)
color_drop7 = layers.dropout(color_conv7_1, dropout_keep_prob, 'drop7')
# Block 3
upsample81 = layers.deconv_upsample(color_drop7, 4, 'upsample8')
upsample81 = Cropping1D(cropping=((0, 1)))(upsample81)
concat8 = layers.concat(upsample81, color_conv2_2, 'concat8')
color_conv8_1 = layers.conv_btn1(concat8,
3,
32,
'conv8_1',
is_training=is_training)
# color_conv8_2 = layers.conv_btn1(color_conv8_1, 3, 32, 'conv8_1', is_training = is_training)
# Block 4
upsample91 = layers.deconv_upsample(color_conv8_1, 4, 'upsample9')
upsample91 = Cropping1D(cropping=((1, 2)))(upsample91)
concat9 = layers.concat(upsample91, color_conv1_2, 'concat9')
color_conv9_1 = layers.conv_btn1(concat9,
3,
32,
'conv9_1',
is_training=is_training)
# color_conv9_2 = layers.conv_btn1(color_conv9_1, 3, 32, 'conv9_1', is_training = is_training)
# Block 5
score = layers.conv(color_conv9_1,
1,
num_classes,
'score',
activation_fn=None)
logits = tf.reshape(score, (-1, num_classes))
return logits
def __init__(self,
input,
gt_label,
gt_box,
gt_difficult=None,
class_num=None,
background_label=0,
overlap_threshold=0.5,
evaluate_difficult=True,
ap_version='integral'):
super(DetectionMAP, self).__init__("map_eval")
gt_label = layers.cast(x=gt_label, dtype=gt_box.dtype)
if gt_difficult:
gt_difficult = layers.cast(x=gt_difficult, dtype=gt_box.dtype)
label = layers.concat([gt_label, gt_difficult, gt_box], axis=1)
else:
label = layers.concat([gt_label, gt_box], axis=1)
# calculate mean average precision (mAP) of current mini-batch
map = layers.detection_map(
input,
label,
class_num,
background_label,
overlap_threshold=overlap_threshold,
evaluate_difficult=evaluate_difficult,
ap_version=ap_version)
self.create_state(dtype='int32', shape=None, suffix='accum_pos_count')
self.create_state(dtype='float32', shape=None, suffix='accum_true_pos')
self.create_state(dtype='float32', shape=None, suffix='accum_false_pos')
self.has_state = None
var = self.helper.create_variable(
persistable=True, dtype='int32', shape=[1])
self.helper.set_variable_initializer(
var, initializer=Constant(value=int(0)))
self.has_state = var
# calculate accumulative mAP
accum_map = layers.detection_map(
input,
label,
class_num,
background_label,
overlap_threshold=overlap_threshold,
evaluate_difficult=evaluate_difficult,
has_state=self.has_state,
input_states=self.states,
out_states=self.states,
ap_version=ap_version)
layers.fill_constant(
shape=self.has_state.shape,
value=1,
dtype=self.has_state.dtype,
out=self.has_state)
self.cur_map = map
self.accum_map = accum_map