def attention_block(g, x, nr_of_convolutions):
"""
Taken from https://github.com/LeeJunHyun/Image_Segmentation
"""
g1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(g)
g1 = BatchNormalization()(g1)
x1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(x)
x1 = BatchNormalization()(x1)
psi = Concatenate()([g1, x1])
psi = Activation(activation='relu')(psi)
psi = Convolution3D(1,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(psi)
psi = BatchNormalization()(psi)
psi = Activation(activation='sigmoid')(psi)
return multiply([x, psi])
def attention_block_oktay(g, x, nr_of_convolutions):
"""
Following the original paper and implementation at https://github.com/ozan-oktay/Attention-Gated-Networks
"""
g1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(g)
g1 = BatchNormalization()(g1)
x1 = MaxPooling3D([2, 2, 2])(x)
x1 = Convolution3D(nr_of_convolutions,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(x1)
x1 = BatchNormalization()(x1)
psi = Concatenate()([g1, x1])
psi = Activation(activation='relu')(psi)
psi = Convolution3D(1,
kernel_size=1,
strides=1,
padding='same',
use_bias=True)(psi)
psi = BatchNormalization()(psi)
psi = Activation(activation='sigmoid')(psi)
return multiply([x, psi])
def cnn_3d():
#3DCNN base model
img_in3D = Input(shape=(3, 120, 160, 3), name='img_in')
x = img_in3D
x = Cropping3D(cropping=((0, 0), (60, 0), (0, 0)))(x)
x = Convolution3D(8, (3, 3, 3), strides=(1, 2, 2), activation='relu')(x)
x = MaxPooling3D(pool_size=(1, 2, 2))(x)
x = BatchNormalization()(x)
x = Dropout(0.1)(x)
x = Flatten(name='flattened')(x)
x = Dense(50, activation='relu')(x)
x = Dropout(0.2)(x)
angle_out = Dense(15, activation='softmax', name='angle_out')(x)
throttle_out = Dense(1, activation='relu', name='throttle_out')(x)
model = Model(inputs=[img_in3D], outputs=[angle_out, throttle_out])
model.compile(optimizer='adam',
loss={
'angle_out': 'categorical_crossentropy',
'throttle_out': 'mean_absolute_error'
},
loss_weights={
'angle_out': 0.9,
'throttle_out': 0.01
})
model.summary()
return model
def call(self, x):
input_shape = x.get_shape().as_list()
_, h, w, d, filters = input_shape
b_layer = Convolution3D(filters // 8, 1, use_bias=False)(x)
c_layer = Convolution3D(filters // 8, 1, use_bias=False)(x)
d_layer = Convolution3D(filters, 1, use_bias=False)(x)
b_layer = tf.transpose(Reshape(target_shape=(h * w * d,
filters // 8))(b_layer),
perm=[0, 2, 1])
c_layer = Reshape(target_shape=(h * w * d, filters // 8))(c_layer)
d_layer = Reshape(target_shape=(h * w * d, filters))(d_layer)
# The bc_mul matrix should be of size (H*W*D) * (H*W*D)
bc_mul = tf.linalg.matmul(c_layer, b_layer)
activation_bc_mul = Activation(activation='softmax')(bc_mul)
bcd_mul = tf.linalg.matmul(activation_bc_mul, d_layer)
bcd_mul = Reshape(target_shape=(h, w, d, filters))(bcd_mul)
out = (self.gamma * bcd_mul) + x
return out
def convolution_block(x,
nr_of_convolutions,
use_bn=False,
spatial_dropout=None):
for i in range(2):
x = Convolution3D(nr_of_convolutions, 3, padding='same')(x)
if use_bn:
x = BatchNormalization()(x)
x = Activation('relu')(x)
if spatial_dropout:
x = SpatialDropout3D(spatial_dropout)(x)
return x
def encoder_block_pyramid(x,
input_ds,
nr_of_convolutions,
use_bn=False,
spatial_dropout=None):
pyramid_conv = Convolution3D(filters=nr_of_convolutions,
kernel_size=(3, 3, 3),
padding='same',
activation='relu')(input_ds)
x = Concatenate(axis=-1)([pyramid_conv, x])
x_before_downsampling = convolution_block(x, nr_of_convolutions, use_bn,
spatial_dropout)
downsample = [2, 2, 2]
for i in range(1, 4):
if x.shape[i] <= 4:
downsample[i - 1] = 1
x = MaxPooling3D(downsample)(x_before_downsampling)
return x, x_before_downsampling
def create(self):
"""
Create model and return it
:return: keras model
"""
input_layer = Input(shape=self.input_shape)
x = input_layer
init_size = max(self.input_shape[:-1])
size = init_size
convolutions = self.convolutions
connection = []
i = 0
if self.input_pyramid:
scaled_input = []
scaled_input.append(x)
for i, nbc in enumerate(self.convolutions[:-1]):
ds_input = AveragePooling3D(pool_size=(2, 2,
2))(scaled_input[i])
scaled_input.append(ds_input)
for i, nbc in enumerate(self.convolutions[:-1]):
if not self.input_pyramid or (i == 0):
x, x_before_ds = encoder_block(
x,
nbc,
use_bn=self.encoder_use_bn,
spatial_dropout=self.encoder_spatial_dropout)
else:
x, x_before_ds = encoder_block_pyramid(
x,
scaled_input[i],
nbc,
use_bn=self.encoder_use_bn,
spatial_dropout=self.encoder_spatial_dropout)
connection.insert(
0, x_before_ds
) # Append in reverse order for easier use in the next block
x = convolution_block(x, self.convolutions[-1], self.encoder_use_bn,
self.encoder_spatial_dropout)
connection.insert(0, x)
inverse_conv = self.convolutions[::-1]
inverse_conv = inverse_conv[1:]
decoded_layers = []
for i, nbc in enumerate(inverse_conv):
x = decoder_block(x,
connection[i + 1],
nbc,
use_bn=self.decoder_use_bn,
spatial_dropout=self.decoder_spatial_dropout)
decoded_layers.append(x)
if not self.deep_supervision:
# Final activation layer
x = Convolution3D(self.nb_classes, 1, activation='softmax')(x)
else:
recons_list = []
for i, lay in enumerate(decoded_layers):
x = Convolution3D(self.nb_classes, 1,
activation='softmax')(lay)
recons_list.append(x)
x = recons_list[::-1]
return Model(inputs=input_layer, outputs=x)
def create(self):
"""
Create model and return it
:return: keras model
"""
input_layer = Input(shape=self.input_shape)
x = input_layer
if self.dim == 3:
init_size = min(self.input_shape[0], self.input_shape[1],
self.input_shape[2])
if self.dim == 2:
init_size = min(self.input_shape[0], self.input_shape[1])
size = init_size
convolutions = self.convolutions
if convolutions is None:
# Create convolutions
convolutions = []
nr_of_convolutions = 8
for i in range(self.get_depth()):
convolutions.append(nr_of_convolutions)
nr_of_convolutions *= 2
convolutions.append(nr_of_convolutions)
for i in range(self.get_depth()):
convolutions.append(nr_of_convolutions)
nr_of_convolutions /= 2
if self.dim == 3:
connection = {}
i = 0
while size % 2 == 0 and size > 4:
x, connection[size] = encoder_block_3(
x, convolutions[i], self.encoder_use_bn,
self.encoder_spatial_dropout)
size /= 2
i += 1
x = convolution_block_3(x, convolutions[i], self.encoder_use_bn,
self.encoder_spatial_dropout)
i += 1
while size < init_size:
size *= 2
x = decoder_block_3(x, convolutions[i], connection[size],
self.decoder_use_bn,
self.decoder_spatial_dropout)
i += 1
x = Convolution3D(self.nb_classes, 1, activation='softmax')(x)
if self.dim == 2:
connection = {}
i = 0
while size % 2 == 0 and size > 4:
x, connection[size] = encoder_block_2(
x, convolutions[i], self.encoder_use_bn,
self.encoder_spatial_dropout)
size /= 2
i += 1
x = convolution_block_2(x, convolutions[i], self.encoder_use_bn,
self.encoder_spatial_dropout)
i += 1
while size < init_size:
size *= 2
x = decoder_block_2(x, convolutions[i], connection[size],
self.decoder_use_bn,
self.decoder_spatial_dropout)
i += 1
x = Convolution2D(self.nb_classes, 1, activation='softmax')(x)
return Model(inputs=input_layer, outputs=x)
def create_model(num_frames=30, num_classes=27, batch_size=10):
inputs = Input(shape=(num_frames, 112, 112, 3),
batch_size=batch_size) # Not quite sure this line
# conv layer1
conv1 = Convolution3D(64,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding="same")(inputs)
norm1 = BatchNormalization()(conv1)
act1 = Activation('relu')(norm1)
pol1 = MaxPool3D(pool_size=(1, 2, 2), strides=(1, 2, 2))(act1)
# conv layer2
conv2 = Convolution3D(128,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same')(pol1)
norm2 = BatchNormalization()(conv2)
act2 = Activation('relu')(norm2)
pol2 = MaxPool3D(pool_size=(2, 2, 2), strides=(2, 2, 2))(act2)
# conv layer3
conv3 = Convolution3D(256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same')(pol2)
# conv layer4
conv4 = Convolution3D(256,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same')(conv3)
norm3 = BatchNormalization()(conv4)
act3 = Activation('relu')(norm3)
pre_output_temp = ConvLSTM2D(256,
kernel_size=3,
strides=(1, 1),
padding='same',
batch_input_shape=(batch_size, num_frames / 2,
1),
return_sequences=True,
stateful=True)(act3)
pre_output = ConvLSTM2D(256,
kernel_size=3,
strides=(1, 1),
padding='same',
batch_input_shape=(batch_size, num_frames / 2, 1),
stateful=True)(pre_output_temp)
# SPP Layer
spp1 = MaxPool2D(pool_size=(28, 28), strides=(28, 28))(pre_output)
spp1 = Flatten()(spp1)
spp2 = MaxPool2D(pool_size=(14, 14), strides=(14, 14))(pre_output)
spp2 = Flatten()(spp2)
spp4 = MaxPool2D(pool_size=(7, 7), strides=(7, 4))(pre_output)
spp4 = Flatten()(spp4)
spp7 = MaxPool2D(pool_size=(4, 4), strides=(4, 4))(pre_output)
spp7 = Flatten()(spp7)
merge = concatenate([spp1, spp2, spp4, spp7], name="Concat")
# final_model = Sequential()
# final_model.add(merge)
# FC Layer
# merge = Dense(1024, activation='relu')(merge)
classes = Dense(num_classes, activation='softmax')(merge)
final_model = Model(inputs=[
inputs,
], outputs=classes)
# final_model.add(Dense(classes, activation='softmax'))
return final_model