def init_model(self):
"""
Build the UNet model with the specified input image shape.
"""
inputs = Input(shape=self.img_shape)
# Apply regularization if not None or 0
kr = regularizers.l2(self.l2_reg) if self.l2_reg else None
"""
Encoding path
"""
filters = 64
in_ = inputs
residual_connections = []
for i in range(self.depth):
conv = Conv3D(int(filters*self.cf), self.kernel_size,
activation=self.activation, padding=self.padding,
kernel_regularizer=kr)(in_)
conv = Conv3D(int(filters * self.cf), self.kernel_size,
activation=self.activation, padding=self.padding,
kernel_regularizer=kr)(conv)
bn = BatchNormalization()(conv)
in_ = MaxPooling3D(pool_size=(2, 2, 2))(bn)
# Update filter count and add bn layer to list for residual conn.
filters *= 2
residual_connections.append(bn)
"""
Bottom (no max-pool)
"""
conv = Conv3D(int(filters * self.cf), self.kernel_size,
activation=self.activation, padding=self.padding,
kernel_regularizer=kr)(in_)
conv = Conv3D(int(filters * self.cf), self.kernel_size,
activation=self.activation, padding=self.padding,
kernel_regularizer=kr)(conv)
bn = BatchNormalization()(conv)
"""
Up-sampling
"""
residual_connections = residual_connections[::-1]
for i in range(self.depth):
# Reduce filter count
filters /= 2
# Up-sampling block
# Note: 2x2 filters used for backward comp, but you probably
# want to use 3x3 here instead.
up = UpSampling3D(size=(2, 2, 2))(bn)
conv = Conv3D(int(filters * self.cf), 2, activation=self.activation,
padding=self.padding, kernel_regularizer=kr)(up)
bn = BatchNormalization()(conv)
# Crop and concatenate
cropped_res = self.crop_nodes_to_match(residual_connections[i], bn)
merge = Concatenate(axis=-1)([cropped_res, bn])
conv = Conv3D(int(filters * self.cf), self.kernel_size,
activation=self.activation, padding=self.padding,
kernel_regularizer=kr)(merge)
conv = Conv3D(int(filters * self.cf), self.kernel_size,
activation=self.activation, padding=self.padding,
kernel_regularizer=kr)(conv)
bn = BatchNormalization()(conv)
"""
Output modeling layer
"""
out = Conv3D(self.n_classes, 1, activation=self.out_activation)(bn)
if self.flatten_output:
out = Reshape([np.prod(self.img_shape[:3]),
self.n_classes], name='flatten_output')(out)
return [inputs], [out]
pooling='avg')
"""
with open('modelsummary.txt', 'w') as f:
with redirect_stdout(f):
model_InceptionV3.summary()
"""
"""
for layer in final_model.layers:
print(layer.output_shape)
"""
for layer in model_InceptionV3.layers[:-12]:
layer.trainable = True
x = model_InceptionV3.output
x = Dense(512, activation='elu', kernel_regularizer=l2(0.001))(x)
y = Dense(46, activation='softmax', name='img')(x)
final_model = Model(inputs=model_InceptionV3.input, outputs=y)
#optimizer로 Stochastic Gradient Descent 사용
opt = SGD(lr=0.0001, momentum=0.9, nesterov=True)
#Model.compile에 관하여 : https://keras.io/models/model/
#optimizer : 옵티마이저 선택
#loss : 손실함수 선택
#img 신경망 출력에서는 categorical_crossentropy를, bbox 출력에는 mse를 사용하겠다는 의미
final_model.compile(
optimizer=opt,
loss={'img': 'categorical_crossentropy'},
def VGG16():
# Build the network of vgg
model = Sequential()
weight_decay = 0.0005
#layer_1
model.add(
Conv2D(64, (3, 3),
padding='same',
input_shape=(230, 124, 2),
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
#layer_2
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
#layer_3
model.add(
Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(256, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
#layer_4
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
#layer_5
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(
Conv2D(512, (3, 3),
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(
Dense(1024,
kernel_regularizer=regularizers.l2(weight_decay),
activation='relu'))
model.add(
Dense(512,
kernel_regularizer=regularizers.l2(weight_decay),
activation='relu'))
model.add(
Dense(256,
kernel_regularizer=regularizers.l2(weight_decay),
activation='relu'))
model.add(Dense(3))
return model
#x = preprocess_input(input_img)
#x = tf.keras.applications.mobilenet_v2.preprocess_input(input_img)
x = preprocess_input(input_img)
encoded_features = base_model(x)
x3 = GlobalAveragePooling2D()(encoded_features)
x4 = Dense(1024, activation='relu')(x3)
x5 = Dropout(0.4)(x4)
predictions = Dense(
n_classes,
activation='softmax',
name='softmax',
kernel_regularizer=l2(0.01),
bias_regularizer=l2(0.01),
kernel_initializer=tf.keras.initializers.glorot_normal())(x5)
# this is the model we will train
model = Model(inputs=input_img, outputs=predictions)
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional resnet50 layers
model.summary()
# In[10]:
callbacks = [
ReduceLROnPlateau(monitor='val_accuracy',
factor=0.2,
def get_model(args):
model_name = args.model_architecture
label_count = 12
model_settings = prepare_model_settings(label_count, args)
if model_name == "fc4":
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(
input_shape=(model_settings['spectrogram_length'],
model_settings['dct_coefficient_count'])),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout(0.2),
tf.keras.layers.BatchNormalization(),
tf.keras.layers.Dense(model_settings['label_count'],
activation="softmax")
])
elif model_name == 'ds_cnn':
print("DS CNN model invoked")
input_shape = [
model_settings['spectrogram_length'],
model_settings['dct_coefficient_count'], 1
]
filters = 64
weight_decay = 1e-4
regularizer = l2(weight_decay)
final_pool_size = (int(input_shape[0] / 2), int(input_shape[1] / 2))
# Model layers
# Input pure conv2d
inputs = Input(shape=input_shape)
x = Conv2D(filters, (10, 4),
strides=(2, 2),
padding='same',
kernel_regularizer=regularizer)(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(rate=0.2)(x)
# First layer of separable depthwise conv2d
# Separable consists of depthwise conv2d followed by conv2d with 1x1 kernels
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Second layer of separable depthwise conv2d
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Third layer of separable depthwise conv2d
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Fourth layer of separable depthwise conv2d
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Reduce size and apply final softmax
x = Dropout(rate=0.4)(x)
x = AveragePooling2D(pool_size=final_pool_size)(x)
x = Flatten()(x)
outputs = Dense(model_settings['label_count'], activation='softmax')(x)
# Instantiate model.
model = Model(inputs=inputs, outputs=outputs)
elif model_name == 'td_cnn':
print("TD CNN model invoked")
input_shape = [
model_settings['spectrogram_length'],
model_settings['dct_coefficient_count'], 1
]
print(f"Input shape = {input_shape}")
filters = 64
weight_decay = 1e-4
regularizer = l2(weight_decay)
# Model layers
# Input time-domain conv
inputs = Input(shape=input_shape)
x = Conv2D(filters, (512, 1),
strides=(384, 1),
padding='valid',
kernel_regularizer=regularizer)(inputs)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(rate=0.2)(x)
x = Reshape((41, 64, 1))(x)
# True conv
x = Conv2D(filters, (10, 4),
strides=(2, 2),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dropout(rate=0.2)(x)
# First layer of separable depthwise conv2d
# Separable consists of depthwise conv2d followed by conv2d with 1x1 kernels
# First layer of separable depthwise conv2d
# Separable consists of depthwise conv2d followed by conv2d with 1x1 kernels
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Second layer of separable depthwise conv2d
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Third layer of separable depthwise conv2d
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Fourth layer of separable depthwise conv2d
x = DepthwiseConv2D(depth_multiplier=1,
kernel_size=(3, 3),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters, (1, 1),
padding='same',
kernel_regularizer=regularizer)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Reduce size and apply final softmax
x = Dropout(rate=0.4)(x)
# x = AveragePooling2D(pool_size=(25,5))(x)
x = GlobalAveragePooling2D()(x)
x = Flatten()(x)
outputs = Dense(model_settings['label_count'], activation='softmax')(x)
# Instantiate model.
model = Model(inputs=inputs, outputs=outputs)
else:
raise ValueError("Model name {:} not supported".format(model_name))
model.compile(
#optimizer=keras.optimizers.RMSprop(learning_rate=args.learning_rate), # Optimizer
optimizer=keras.optimizers.Adam(
learning_rate=args.learning_rate), # Optimizer
# Loss function to minimize
loss=keras.losses.SparseCategoricalCrossentropy(),
# List of metrics to monitor
metrics=[keras.metrics.SparseCategoricalAccuracy()],
)
return model
def resnet(input_tensor, block_settings, use_bias, weight_decay, trainable, bn_trainable):
'''
https://arxiv.org/pdf/1512.03385.pdf
Bottleneck architecture
Arguments
input_tensor:
block_settings:
[[64, 64, 256], [3, [2, 2]], [4, [2, 2]], [6, [2, 2]], [3, [2, 2]]] # Resnet 50, pool 64
[[64, 64, 256], [3, [2, 2]], [4, [2, 2]], [23, [2, 2]], [3, [2, 2]]] # Resnet 101, pool 64
[[64, 64, 256], [3, [2, 2]], [8, [2, 2]], [36, [2, 2]], [3, [2, 2]]] # Resnet 152, pool 64
trainable:
Return
tensor:
'''
filters = np.array(block_settings[0])
n_C2, strides_C2 = block_settings[1]
tensors = []
# C1
tensor = Conv2D(
filters=filters[0],
kernel_size=[7, 7],
strides=[2, 2],
padding='same',
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name='conv1')(input_tensor)
tensor = BatchNormalization(trainable=bn_trainable, name='conv1_bn')(tensor)
tensor = Activation('relu')(tensor)
# C2
tensor = MaxPool2D(pool_size=[3, 3], strides=strides_C2, padding='same')(tensor)
tensor = conv_block(
input_tensor=tensor,
kernel_size=[3, 3],
filters=filters,
strides=[1, 1],
block_name='stg1_blk0_',
use_bias=use_bias,
weight_decay=weight_decay,
trainable=trainable,
bn_trainable=bn_trainable)
for n in range(1, n_C2):
tensor = identity_block(
input_tensor=tensor,
kernel_size=[3, 3],
filters=filters,
block_name='stg1_blk'+str(n)+'_',
use_bias=use_bias,
weight_decay=weight_decay,
trainable=trainable,
bn_trainable=bn_trainable)
tensors.append(tensor)
# C34...
for c in range(2, 2+len(block_settings[2:])):
n_C, strides_C = block_settings[c]
tensor = conv_block(
input_tensor=tensor,
kernel_size=[3, 3],
filters=(2**(c-1))*filters,
strides=strides_C,
block_name='stg'+str(c)+'_blk0_',
use_bias=use_bias,
weight_decay=weight_decay,
trainable=trainable,
bn_trainable=bn_trainable)
for n in range(1, n_C):
tensor = identity_block(
input_tensor=tensor,
kernel_size=[3, 3],
filters=(2**(c-1))*filters,
block_name='stg'+str(c)+'_blk'+str(n)+'_',
use_bias=use_bias,
weight_decay=weight_decay,
trainable=trainable,
bn_trainable=bn_trainable)
tensors.append(tensor)
return tensors
def conv_block(input_tensor, kernel_size, filters, strides, block_name, use_bias, weight_decay, trainable, bn_trainable):
'''
https://arxiv.org/pdf/1512.03385.pdf
Bottleneck architecture
Arguments
input_tensor:
kernel_size:
filters:
strides:
trainable:
Return
tensor:
'''
filters1, filters2, filters3 = filters
tensor = Conv2D(
filters=filters1,
kernel_size=[1, 1],
strides=strides,
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name=block_name+'_conv1')(input_tensor)
tensor = BatchNormalization(trainable=bn_trainable, name=block_name+'_conv1_bn')(tensor)
tensor = Activation('relu')(tensor)
tensor = Conv2D(
filters=filters2,
kernel_size=kernel_size,
padding='same',
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name=block_name+'_conv2')(tensor)
tensor = BatchNormalization(trainable=bn_trainable, name=block_name+'_conv2_bn')(tensor)
tensor = Activation('relu')(tensor)
tensor = Conv2D(
filters=filters3,
kernel_size=[1, 1],
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name=block_name+'_conv3')(tensor)
tensor = BatchNormalization(trainable=bn_trainable, name=block_name+'_conv3_bn')(tensor)
input_tensor = Conv2D(
filters=filters3,
kernel_size=[1, 1],
strides=strides,
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name=block_name+'_conv4')(input_tensor)
input_tensor = BatchNormalization(trainable=bn_trainable, name=block_name+'_conv4_bn')(input_tensor, training=trainable)
tensor = Add()([tensor, input_tensor])
tensor = Activation('relu')(tensor)
return tensor
def DarknetConv2D(*args, **kwargs):
darknet_conv_kwargs = {'kernel_regularizer': l2(5e-4)}
darknet_conv_kwargs['padding'] = 'valid' if kwargs.get('strides') == (
2, 2) else 'same'
darknet_conv_kwargs.update(kwargs)
return Conv2D(*args, **darknet_conv_kwargs)
def centernet(num_classes,
input_size=512,
max_objects=100,
score_threshold=0.1,
nms=True,
flip_test=False,
training=True,
l2_norm=5e-4):
output_size = input_size // 4
image_input = Input(shape=(input_size, input_size, 3))
hm_input = Input(shape=(output_size, output_size, num_classes))
wh_input = Input(shape=(max_objects, 2))
reg_input = Input(shape=(max_objects, 2))
reg_mask_input = Input(shape=(max_objects, ))
index_input = Input(shape=(max_objects, ))
resnet = ResNet50(include_top=False, input_tensor=image_input)
x = resnet.outputs[-1]
num_filters = 256
for i in range(3):
num_filters = num_filters // pow(2, i)
x = Conv2DTranspose(num_filters, (4, 4),
strides=2,
use_bias=False,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
x = BatchNormalization()(x)
x = relu(x)
# hm header
y1 = Conv2D(64,
3,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
y1 = BatchNormalization()(y1)
y1 = relu(y1)
y1 = Conv2D(num_classes,
1,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm),
activation='sigmoid')(y1)
# wh header
y2 = Conv2D(64,
3,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
y2 = BatchNormalization()(y2)
y2 = relu(y2)
y2 = Conv2D(2,
1,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(y2)
# reg header
y3 = Conv2D(64,
3,
padding='same',
use_bias=False,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(x)
y3 = BatchNormalization()(y3)
y3 = relu(y3)
y3 = Conv2D(2,
1,
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_norm))(y3)
loss_ = Lambda(loss, name='centernet_loss')([
y1, y2, y3, hm_input, wh_input, reg_input, reg_mask_input, index_input
])
if training:
model = Model(inputs=[
image_input, hm_input, wh_input, reg_input, reg_mask_input,
index_input
],
outputs=[loss_])
else:
# detections = decode(y1, y2, y3)
detections = Lambda(lambda x: decode(*x,
max_objects=max_objects,
score_threshold=score_threshold,
nms=nms,
flip_test=flip_test,
num_classes=num_classes))(
[y1, y2, y3])
model = Model(inputs=image_input, outputs=detections)
return model
def resnet_3d_cnn_hybrid(self,
voxel_dim=64,
deviation_channels=3,
categoric_outputs=25,
w_val=0):
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow.keras.models import Model
from tensorflow.keras import regularizers
from tensorflow.keras.layers import Conv3D, MaxPooling3D, Add, BatchNormalization, Input, LeakyReLU, Activation, Lambda, Concatenate, Flatten, Dense, UpSampling3D, GlobalAveragePooling3D
from tensorflow.keras.utils import plot_model
if (w_val == 0):
w_val = np.zeros(self.output_dimension)
w_val[:] = 1 / self.output_dimension
def weighted_mse(val):
def loss(yTrue, yPred):
#val = np.array([0.1,0.1,0.1,0.1,0.1])
w_var = K.variable(value=val,
dtype='float32',
name='weight_vec')
#weight_vec = K.ones_like(yTrue[0,:]) #a simple vector with ones shaped as (60,)
#idx = K.cumsum(ones) #similar to a 'range(1,61)'
return K.mean((w_var) * K.square(yTrue - yPred))
return loss
input_size = (voxel_dim, voxel_dim, voxel_dim, deviation_channels)
inputs = Input(input_size)
x = inputs
y = Conv3D(32,
kernel_size=(4, 4, 4),
strides=(2, 2, 2),
name="conv_block_1")(x)
res1 = y
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_2")(y)
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_3")(y)
y = Add()([res1, y])
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(2, 2, 2),
name="conv_block_4")(y)
res2 = y
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_5")(y)
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_6")(y)
y = Add()([res2, y])
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(2, 2, 2),
name="conv_block_7")(y)
res3 = y
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_8")(y)
y = LeakyReLU()(y)
y = Conv3D(32,
kernel_size=(3, 3, 3),
strides=(1, 1, 1),
padding='same',
name="conv_block_9")(y)
y = Add()([res3, y])
y = LeakyReLU()(y)
y = Flatten()(y)
y = Dense(128,
kernel_regularizer=regularizers.l2(0.01),
activation='relu')(y)
y = Dense(64,
kernel_regularizer=regularizers.l2(0.01),
activation='relu')(y)
output_regression = Dense(self.output_dimension - categoric_outputs,
name="regression_output")(y)
output_classification = Dense(categoric_outputs,
activation="sigmoid",
name="classification_output")(y)
output = [output_regression, output_classification]
model = Model(inputs, outputs=output, name='Res_3D_CNN_hybrid')
def mse_scaled(y_true, y_pred):
return K.mean(K.square((y_pred - y_true)))
#loss_regression=tf.keras.losses.MeanSquaredError()
loss_regression = mse_scaled
loss_classification = tf.keras.losses.BinaryCrossentropy()
loss_list = [loss_regression, loss_classification]
model.compile(loss=loss_list,
optimizer=tf.keras.optimizers.Adam(),
experimental_run_tf_function=False,
metrics=[
tf.keras.metrics.MeanAbsoluteError(),
tf.keras.metrics.Accuracy()
])
#plot_model(model,to_file='resnet_3d_cnn_hybrid.png',show_shapes=True, show_layer_names=True)
print(model.summary())
return model
input_shape=(height, width, constants.NUM_CHANNELS)
)
# First time run, no unlocking
conv_base.trainable = False
# Let's see it
print('Summary')
print(conv_base.summary())
# Let's construct that top layer replacement
x = conv_base.output
x = AveragePooling2D(pool_size=(8, 8))(x)
x - Dropout(0.4)(x)
x = Flatten()(x)
x = Dense(256, activation='relu', kernel_initializer=initializers.he_normal(seed=None), kernel_regularizer=regularizers.l2(.0005))(x)
x = Dropout(0.5)(x)
# Essential to have another layer for better accuracy
x = Dense(128,activation='relu', kernel_initializer=initializers.he_normal(seed=None))(x)
x = Dropout(0.25)(x)
predictions = Dense(constants.NUM_CLASSES, kernel_initializer="glorot_uniform", activation='softmax')(x)
print('Stacking New Layers')
model = Model(inputs = conv_base.input, outputs=predictions)
# Load checkpoint if one is found
#if os.path.exists(checkpoint_path):
#print ("loading ", checkpoint_path)
#model.load_weights(checkpoint_path)
#
# Define callback checkpoint for saving model
def mini_XCEPTION(input_shape, num_classes, l2_regularization=0.01):
regularization = l2(l2_regularization)
# base
img_input = Input(input_shape)
x = Conv2D(8, (3, 3),
strides=(1, 1),
kernel_regularizer=regularization,
use_bias=False)(img_input)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(8, (3, 3),
strides=(1, 1),
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# module 1
residual = Conv2D(16, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(16, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(16, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
# module 2
residual = Conv2D(32, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(32, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
# module 3
residual = Conv2D(64, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(64, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
# module 4
residual = Conv2D(128, (1, 1),
strides=(2, 2),
padding='same',
use_bias=False)(x)
residual = BatchNormalization()(residual)
x = SeparableConv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = SeparableConv2D(128, (3, 3),
padding='same',
kernel_regularizer=regularization,
use_bias=False)(x)
x = BatchNormalization()(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
x = layers.add([x, residual])
x = Conv2D(
num_classes,
(3, 3),
#kernel_regularizer=regularization,
padding='same')(x)
x = GlobalAveragePooling2D()(x)
output = Activation('softmax', name='predictions')(x)
model = Model(img_input, output)
return model
def __create_res_next_imagenet(nb_classes, img_input, include_top, depth, cardinality=32, width=4,
weight_decay=5e-4, pooling=None, attention_module=None):
''' Creates a ResNeXt model with specified parameters
Args:
nb_classes: Number of output classes
img_input: Input tensor or layer
include_top: Flag to include the last dense layer
depth: Depth of the network. List of integers.
Increasing cardinality improves classification accuracy,
width: Width of the network.
weight_decay: weight_decay (l2 norm)
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
Returns: a Keras Model
'''
if type(depth) is list or type(depth) is tuple:
# If a list is provided, defer to user how many blocks are present
N = list(depth)
else:
# Otherwise, default to 3 blocks each of default number of group convolution blocks
N = [(depth - 2) // 9 for _ in range(3)]
filters = cardinality * width
filters_list = []
for i in range(len(N)):
filters_list.append(filters)
filters *= 2 # double the size of the filters
x = __initial_conv_block_inception(img_input, weight_decay)
# block 1 (no pooling)
for i in range(N[0]):
x = __bottleneck_block(x, filters_list[0], cardinality, strides=1,
weight_decay=weight_decay, attention_module=attention_module)
N = N[1:] # remove the first block from block definition list
filters_list = filters_list[1:] # remove the first filter from the filter list
# block 2 to N
for block_idx, n_i in enumerate(N):
for i in range(n_i):
if i == 0:
x = __bottleneck_block(x, filters_list[block_idx], cardinality, strides=2,
weight_decay=weight_decay, attention_module=attention_module)
else:
x = __bottleneck_block(x, filters_list[block_idx], cardinality, strides=1,
weight_decay=weight_decay, attention_module=attention_module)
if include_top:
x = GlobalAveragePooling2D()(x)
x = Dense(nb_classes, use_bias=False, kernel_regularizer=l2(weight_decay),
kernel_initializer='he_normal', activation='softmax')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
return x
# model.add(layer)
#for l in model.layers:
# l.trainable=False
# Projection
model.add(
Conv2D(3, (1, 1),
input_shape=(image_height, image_width, 1),
padding="same"))
model.add(RESNET)
#model.layers[1].trainable=True
model.add(
Dense(512, Activation("relu"), kernel_regularizer=regularizers.l2(0.0001)))
model.add(Dense(1))
optimize = keras.optimizers.Adam(learning_rate=learning_rate)
model.compile(optimizer=optimize, loss='MSE', metrics=['mse'])
class NeptuneMonitor(Callback):
def on_epoch_end(self, epoch, logs={}):
neptune.send_metric('val_loss', epoch, logs['val_loss'])
neptune.send_metric('loss', epoch, logs['loss'])
neptune.send_metric(
'learning_rate', epoch,
float(tf.keras.backend.get_value(self.model.optimizer.lr)))
def u_net_pipeline(train_data, train_labels, val_data, val_labels, test_data, test_labels, classes, weights):
"""This function is the main pipeline for the U-Net.
Parameters :
------------
xxx_data : np.array
Data of the train, validation and test datasets
xxx_labels : np.array
Labels of the train, validation and test datasets
classes : np.array
Classes of the dataset
weights : np.array
Weights applied for each class during training
Returns :
-----------
Accuracy : float
Precision : float
Recall : float
F1-Score : float
Metrics evaluated on the test set
"""
# Determining the weights of each pixel of the images
l, x, y = train_labels.shape
new_shape = (l, x, y, 1)
train_labels = np.reshape(train_labels, new_shape)
train_weights = np.ones(shape=train_labels.shape)
for i in range(len(classes)):
train_weights[train_labels == classes[i]] = weights[i]
val_weights = np.ones(shape=val_labels.shape)
for i in range(len(classes)):
val_weights[val_labels == classes[i]] = weights[i]
test_weights = np.ones(shape=test_labels.shape)
for i in range(len(classes)):
test_weights[test_labels == classes[i]] = weights[i]
# Creating an image generator
image_datagen = ImageDataGenerator(horizontal_flip=True, vertical_flip=True)
mask_datagen = ImageDataGenerator(horizontal_flip=True, vertical_flip=True)
seed = 1
image_datagen.fit(train_data, augment=True, seed=seed)
mask_datagen.fit(train_labels, augment=True, seed=seed)
image_generator = image_datagen.flow(
train_data,
batch_size=100,
seed=seed)
mask_generator = mask_datagen.flow(
train_labels,
batch_size=100,
sample_weight = train_weights,
seed=seed)
del train_data, train_labels, train_weights
def image_mask_generator(image_generator, mask_generator):
train_generator = zip(image_generator, mask_generator)
for (img, mask) in train_generator:
mask_squeezed = np.squeeze(mask)
mask_one_hot = tf.one_hot(mask_squeezed, depth=6, dtype=tf.int8)
img = tf.convert_to_tensor(img)
yield img, mask_one_hot
generator = image_mask_generator(image_generator, mask_generator)
# One-hot encoding the validation labels
val_labels = tf.one_hot(val_labels, depth=6, dtype=tf.int8)
## -- Training the Unet Model --
# Image shape
img_rows = 256
img_cols = 256
img_channels = val_data.shape[3]
#Output
nb_classes = 6
# Architecture Parameters
nb_filters_0 = 16
# Saving the model at each step
checkpoint_filepath = "tmp/checkpoint_SPARCS_unet"
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=True,
monitor='val_loss',
mode='min',
save_best_only=True)
# Early Stopping Callback
ES = EarlyStopping(monitor='val_accuracy', patience=20)
# Deep Learning Model
model = Unet4((img_rows, img_cols, img_channels), nb_filters=nb_filters_0, output_channels=nb_classes, initialization="he_normal", kernel_size=5, drop=0.0, regularization=l2(0.000135))
print(model.summary())
model.compile(loss = "categorical_crossentropy", optimizer=Adam(learning_rate=0.00019), metrics=["accuracy"])
history = model.fit(generator, batch_size=100,
steps_per_epoch=10,
epochs=500,
validation_data = (val_data, val_labels, val_weights),
callbacks=[model_checkpoint_callback, ES])
# Making a prediction on the test data
pred = model.predict(test_data)
# Computing metrics
accuracy = tf.keras.metrics.Accuracy()
accuracy.update_state(np.argmax(pred, axis=3), test_labels)
recall = tf.keras.metrics.Recall()
recall.update_state(tf.one_hot(test_labels, depth=6).numpy().flatten(), pred.flatten())
precision = tf.keras.metrics.Precision()
precision.update_state(tf.one_hot(test_labels, depth=6).numpy().flatten(), pred.flatten())
# Computing the confusion matrix
print(confusion_matrix(np.argmax(pred, axis=3).flatten(), test_labels.flatten(), sample_weight=test_weights.flatten()))
# Plotting images of the test dataset
c = 0
cmap = plt.get_cmap('viridis', 6)
for image in pred[:50]:
plt.imshow(np.argmax(image, axis=2)+1e-5, cmap=cmap, vmin=0, vmax=6)
plt.colorbar()
plt.savefig(f"images/SPARCS/u_net/pred{c}")
plt.clf()
plt.imshow(test_labels[c]+1e-5, vmin=0, vmax=6, cmap=cmap)
plt.colorbar()
plt.savefig(f"images/SPARCS/u_net/GT{c}")
plt.clf()
plt.imshow(test_data[c][:,:,5:9]/np.amax(test_data[c][:,:,5:9]))
plt.savefig(f"images/SPARCS/u_net/original{c}")
c+=1
pred = np.argmax(pred, axis=3)
pred = pred.flatten()
test_labels = test_labels.flatten()
# Computing the confusion matrix
conf = confusion_matrix(test_labels, pred, labels=[0, 1, 2, 3, 4, 5])
np.save("logs/metrics/SPARCS/u_net/confusion_matrix", conf)
# Computing metrics
test_accuracy = accuracy.result().numpy()
test_recall = recall.result().numpy()
test_precision = precision.result().numpy()
test_f1 = 2/(1/test_recall + 1/test_precision)
print("Test accuracy = ", test_accuracy)
print("Test recall =", test_recall)
print("Test precision=", test_precision)
print("Test f1 =", test_f1)
def get_model(input_shape,
input_layer,
num_classes,
model_name='resnet_50',
trainable_layers_amount=0,
dropout=0.5,
path_model=None):
if not path_model:
if model_name == 'resnet_50':
base_model = ResNet50(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=input_shape)
else:
if model_name == 'mobile_net':
base_model = MobileNet(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=input_shape)
elif model_name == 'resnet_34':
base_model = ResNet34(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=input_shape)
else:
print(f'Loading model in path {path_model}')
loaded_model = load_model(path_model)
base_model = loaded_model.layers[3]
# Avoid training layers in resnet model.
for layer in base_model.layers:
layer.trainable = False
layers = base_model.layers
# Training the last
if trainable_layers_amount != 0:
if trainable_layers_amount != -1:
trainable_layers = layers[-trainable_layers_amount:]
assert len(trainable_layers) == trainable_layers_amount
else:
trainable_layers = layers
else:
trainable_layers = []
for layer in trainable_layers:
print("Making layer %s trainable " % layer.name)
layer.trainable = True
base_model.summary()
x1 = PreprocessImage(model_name)(input_layer)
x2 = base_model(x1, training=False)
x3 = GlobalAveragePooling2D()(x2)
x4 = Dense(1024, activation='relu')(x3)
x5 = Dropout(dropout)(x4)
output_layer = Dense(
num_classes,
activation='softmax',
name='softmax',
kernel_regularizer=l2(0.01),
bias_regularizer=l2(0.01),
kernel_initializer=tf.keras.initializers.glorot_normal())(x5)
return output_layer, base_model
def interspeech_TIMIT_Model(d):
n = d.num_layers
sf = d.start_filter
activation = d.act
advanced_act = d.aact
drop_prob = d.dropout
inputShape = (778, 3, 40) # (3,41,None)
filsize = (3, 5)
Axis = 1
if advanced_act != "none":
activation = 'linear'
convArgs = {
"activation": activation,
"data_format": "channels_first",
"padding": "same",
"bias_initializer": "zeros",
"kernel_regularizer": l2(d.l2),
"kernel_initializer": "random_uniform",
}
denseArgs = {
"activation": d.act,
"kernel_regularizer": l2(d.l2),
"kernel_initializer": "random_uniform",
"bias_initializer": "zeros",
"use_bias": True
}
#### Check kernel_initializer for quaternion model ####
if d.model == "quaternion":
convArgs.update({"kernel_initializer": d.quat_init})
#
# Input Layer & CTC Parameters for TIMIT
#
if d.model == "quaternion":
I = Input(shape=(778, 4, 40)) # Input(shape=(4,41,None))
else:
I = Input(shape=inputShape)
labels = Input(name='the_labels', shape=[None], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
#
# Input stage:
#
if d.model == "real":
O = Conv2D(sf, filsize, name='conv', use_bias=True, **convArgs)(I)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
else:
O = QuaternionConv2D(sf,
filsize,
name='conv',
use_bias=True,
**convArgs)(I)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
#
# Pooling
#
O = MaxPooling2D(pool_size=(1, 3), padding='same')(O)
#
# Stage 1
#
for i in range(0, n // 2):
if d.model == "real":
O = Conv2D(sf,
filsize,
name='conv' + str(i),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
else:
O = QuaternionConv2D(sf,
filsize,
name='conv' + str(i),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
#
# Stage 2
#
for i in range(0, n // 2):
if d.model == "real":
O = Conv2D(sf * 2,
filsize,
name='conv' + str(i + n / 2),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
else:
O = QuaternionConv2D(sf * 2,
filsize,
name='conv' + str(i + n / 2),
use_bias=True,
**convArgs)(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
#
# Permutation for CTC
#
print("Last Q-Conv2D Layer (output): ", K.int_shape(O))
print("Shape tuple: ",
K.int_shape(O)[0],
K.int_shape(O)[1],
K.int_shape(O)[2],
K.int_shape(O)[3])
#### O = Permute((3,1,2))(O)
#### print("Last Q-Conv2D Layer (Permute): ", O.shape)
# O = Lambda(lambda x: K.reshape(x, (K.shape(x)[0], K.shape(x)[1],
# K.shape(x)[2] * K.shape(x)[3])),
# output_shape=lambda x: (None, None, x[2] * x[3]))(O)
# O = Lambda(lambda x: K.reshape(x, (K.int_shape(x)[0], K.int_shape(x)[1],
# K.int_shape(x)[2] * K.int_shape(x)[3])),
# output_shape=lambda x: (None, None, x[2] * x[3]))(O)
O = tf.keras.layers.Reshape(
target_shape=[-1, K.int_shape(O)[2] * K.int_shape(O)[3]])(O)
#
# Dense
#
print("Q-Dense input: ", K.int_shape(O))
if d.model == "quaternion":
print("first Q-dense layer: ", O.shape)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(QuaternionDense(256, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
else:
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
O = Dropout(drop_prob)(O)
O = TimeDistributed(Dense(1024, **denseArgs))(O)
if advanced_act == "prelu":
O = PReLU(shared_axes=[1, 0])(O)
# pred = TimeDistributed( Dense(61, activation='softmax', kernel_regularizer=l2(d.l2), use_bias=True, bias_initializer="zeros", kernel_initializer='random_uniform' ))(O)
pred = TimeDistributed(
Dense(61,
kernel_regularizer=l2(d.l2),
use_bias=True,
bias_initializer="zeros",
kernel_initializer='random_uniform'))(O)
output = Activation('softmax', name='softmax')(pred)
network = CTCModel([I], [output])
# network.compile(Adam(lr=0.0001))
""" CTC Loss - Implemented differently
if d.ctc:
# CTC For sequence labelling
O = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([pred, labels,input_length,label_length])
# Creating a function for testing and validation purpose
val_function = K.function([I],[pred])
# Return the model
return Model(inputs=[I, input_length, labels, label_length], outputs=O), val_function
"""
# return Model(inputs=I, outputs=pred)
return network
def autoencoder_model(p_drop, p_l2):
input_img = Input(shape=X_train.shape[1:])
encoder = Dropout(rate=p_drop)(input_img, training=True)
encoder = Conv2D(256 + 25, (3, 3),
activation='relu',
padding='same',
name='Econv2d_1',
bias_regularizer=l2(p_l2),
kernel_regularizer=l2(p_l2))(input_img)
encoder = MaxPooling2D((2, 2), padding='same',
name='Emaxpool2d_1')(encoder)
encoder = Dropout(rate=p_drop)(encoder, training=True)
encoder = Conv2D(128 + 12, (3, 3),
activation='relu',
padding='same',
name='Econv2d_2',
bias_regularizer=l2(p_l2),
kernel_regularizer=l2(p_l2))(encoder)
encoder = MaxPooling2D((2, 2), padding='same',
name='Emaxpool2d_2')(encoder)
encoder = Dropout(rate=p_drop)(encoder, training=True)
encoder = Conv2D(64 + 6, (3, 3),
activation='relu',
padding='same',
name='Econv2d_3',
bias_regularizer=l2(p_l2),
kernel_regularizer=l2(p_l2))(encoder)
encoder = MaxPooling2D((2, 2), padding='same',
name='Emaxpool2d_3')(encoder)
decoder = Dropout(rate=p_drop)(encoder, training=True)
decoder = Conv2D(64 + 6, (3, 3),
activation='relu',
padding='same',
name='Dconv2d_1',
bias_regularizer=l2(p_l2),
kernel_regularizer=l2(p_l2))(decoder)
decoder = UpSampling2D((2, 2), name='Dupsamp_1')(decoder)
decoder = Dropout(rate=p_drop)(decoder, training=True)
decoder = Conv2D(128 + 12, (3, 3),
activation='relu',
padding='same',
name='Dconv2d_2',
bias_regularizer=l2(p_l2),
kernel_regularizer=l2(p_l2))(decoder)
decoder = UpSampling2D((2, 2), name='Dupsamp_2')(decoder)
decoder = Dropout(rate=p_drop)(decoder, training=True)
decoder = Conv2D(256 + 25, (3, 3),
activation='relu',
name='Dconv2d_3',
bias_regularizer=l2(p_l2),
kernel_regularizer=l2(p_l2))(decoder)
decoder = UpSampling2D((2, 2), name='Dupsamp_3')(decoder)
decoder = Dropout(rate=p_drop)(decoder, training=True)
decoder = Conv2D(3, (3, 3),
activation='sigmoid',
padding='same',
name='Dconv2d_out')(decoder)
autoencoder = Model(input_img, decoder)
autoencoder.summary()
return autoencoder
def fpn_top_down(input_tensors, top_down_pyramid_size, use_bias, weight_decay, trainable, bn_trainable):
'''
Top down network in Pyramid Feature Net https://arxiv.org/pdf/1612.03144.pdf
Arguments
input_tensors:
top_down_pyramid_size:
trainable:
Return
P2:
P3;
P4;
'''
C2, C3, C4, C5 = input_tensors
L4 = Conv2D(
filters=top_down_pyramid_size,
kernel_size=[3, 3],
padding='same',
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name='lateral_P4')(C4)
L4 = BatchNormalization(trainable=bn_trainable, name='lateral_P4_bn')(L4)
L4 = Activation('relu')(L4)
L3 = Conv2D(
filters=top_down_pyramid_size,
kernel_size=[3, 3],
padding='same',
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name='lateral_P3')(C3)
L3 = BatchNormalization(trainable=bn_trainable, name='lateral_P3_bn')(L3)
L3 = Activation('relu')(L3)
L2 = Conv2D(
filters=top_down_pyramid_size,
kernel_size=[3, 3],
padding='same',
trainable=trainable,
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
name='lateral_P2')(C2)
L2 = BatchNormalization(trainable=bn_trainable, name='lateral_P2_bn')(L2)
L2 = Activation('relu')(L2)
M5 = Conv2D(
filters=top_down_pyramid_size,
kernel_size=[3, 3],
padding='same',
use_bias=use_bias,
kernel_regularizer=regularizers.l2(weight_decay),
trainable=trainable,
name='M5')(C5)
M5 = BatchNormalization(trainable=bn_trainable, name='M5_bn')(M5)
M5 = Activation('relu')(M5)
M4 = Add(name='M4')([UpSampling2D(size=(2, 2), interpolation='bilinear')(M5), L4])
M3 = Add(name='M3')([UpSampling2D(size=(2, 2), interpolation='bilinear')(M4), L3])
M2 = Add(name='M2')([UpSampling2D(size=(2, 2), interpolation='bilinear')(M3), L2])
P5 = RFE(input_tensor=M5, module_name='RFE4', trainable=trainable, bn_trainable=bn_trainable, use_bias=use_bias, weight_decay=weight_decay)
P4 = RFE(input_tensor=M4, module_name='RFE3', trainable=trainable, bn_trainable=bn_trainable, use_bias=use_bias, weight_decay=weight_decay)
P3 = RFE(input_tensor=M3, module_name='RFE2', trainable=trainable, bn_trainable=bn_trainable, use_bias=use_bias, weight_decay=weight_decay)
P2 = RFE(input_tensor=M2, module_name='RFE1', trainable=trainable, bn_trainable=bn_trainable, use_bias=use_bias, weight_decay=weight_decay)
return [P2, P3, P4, P5]
def get_nmf_model(self):
num_factors = self.num_factors
num_layers = self.num_layers
layer1_dim = self.num_factors * (2**(num_layers - 1))
num_users = self.num_users
num_items = self.num_items
num_genres = self.num_genres
# input layer
user_input_layer = layers.Input(shape=(1, ),
dtype='int32',
name='user_input')
item_input_layer = layers.Input(shape=(1, ),
dtype='int32',
name='item_input')
genre_input_layer = layers.Input(shape=(None, ),
dtype='int32',
name='genre_input')
# GMF embedding layer
GMF_user_embedding = layers.Embedding(
input_dim=num_users,
output_dim=num_factors,
embeddings_regularizer=regularizers.l2(0.),
name='GMF_user_embedding')(user_input_layer)
GMF_item_embedding = layers.Embedding(
input_dim=num_items,
output_dim=self.num_movie_factors,
embeddings_regularizer=regularizers.l2(0.),
name='GMF_item_embedding')(item_input_layer)
GMF_genre_embedding = layers.Embedding(
input_dim=num_genres,
output_dim=self.num_movie_factors,
mask_zero=True,
embeddings_regularizer=regularizers.l2(0.),
name='GMF_genre_embedding')(genre_input_layer)
GMF_genre_emb_mean = tf.reduce_mean(GMF_genre_embedding, 1)
# MLP embedding layer
MLP_user_embedding = layers.Embedding(
input_dim=num_users,
output_dim=num_factors,
embeddings_regularizer=regularizers.l2(0.),
name='MLP_user_embedding')(user_input_layer)
MLP_item_embedding = layers.Embedding(
input_dim=num_items,
output_dim=self.num_movie_factors,
embeddings_regularizer=regularizers.l2(0.),
name='MLP_item_embedding')(item_input_layer)
MLP_genre_embedding = layers.Embedding(
input_dim=num_genres,
output_dim=self.num_movie_factors,
mask_zero=True,
embeddings_regularizer=regularizers.l2(0.),
name='MLP_genre_embedding')(genre_input_layer)
MLP_genre_emb_mean = tf.reduce_mean(MLP_genre_embedding, 1)
# GMF
GMF_user_latent = layers.Flatten()(GMF_user_embedding)
GMF_item_latent = layers.Flatten()(GMF_item_embedding)
GMF_genre_latent = layers.Flatten()(GMF_genre_emb_mean)
GMF_movie_latent = layers.concatenate(
[GMF_item_latent, GMF_genre_latent])
# MLP
MLP_user_latent = layers.Flatten()(MLP_user_embedding)
MLP_item_latent = layers.Flatten()(MLP_item_embedding)
MLP_genre_latent = layers.Flatten()(MLP_genre_emb_mean)
MLP_movie_latent = layers.concatenate(
[MLP_item_latent, MLP_genre_latent])
# gmf - element wise product
GMF_vector = layers.multiply([GMF_user_latent, GMF_movie_latent])
GMF_vector = layers.BatchNormalization()(GMF_vector)
# mlp
MLP_vector = layers.concatenate([MLP_user_latent, MLP_movie_latent])
MLP_vector = layers.BatchNormalization()(MLP_vector)
for i in range(num_layers - 1):
MLP_vector = layers.Dense((int)(layer1_dim / (2**i)),
activation='tanh',
name='layer%s' % str(i + 1))(MLP_vector)
MLP_vector = layers.BatchNormalization()(MLP_vector)
#NeuMF layer
NeuMF_vector = layers.concatenate([GMF_vector, MLP_vector])
prediction = layers.Dense(1, activation='tanh',
name='prediction')(NeuMF_vector)
model = Model([user_input_layer, item_input_layer, genre_input_layer],
prediction)
self.nmf_model = model
self.nmf_model = self.set_nmf_weight()
return self.nmf_model
X_train, y_train = mnist_reader.load_mnist('fashion', kind='train')
X_val, y_val = mnist_reader.load_mnist('fashion', kind='t10k')
X_train = X_train.astype('float32') / 255.0
X_val = X_val.astype('float32') / 255.0
y_train = to_categorical(y_train, len(class_names))
y_val = to_categorical(y_val, len(class_names))
model = Sequential()
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(128, activation='relu',
kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(0.3))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
def fit_the_model():
model.fit(X_train,
y_train,
epochs=50,
verbose=1,
batch_size=1024,
validation_split=0.2,
input_shape=(168, 168, 1)),
BatchNormalization(),
MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='same'),
Conv2D(filters=256,
kernel_size=(5, 5),
strides=(1, 1),
activation=tf.nn.relu,
padding="same"),
BatchNormalization(),
MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='same'),
Conv2D(filters=384,
kernel_size=(3, 3),
strides=(1, 1),
activation=tf.nn.relu,
padding="same",
kernel_regularizer=l2(0.02)),
BatchNormalization(),
Conv2D(filters=384,
kernel_size=(3, 3),
strides=(1, 1),
activation=tf.nn.relu,
padding="same"),
BatchNormalization(),
Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
activation=tf.nn.relu,
padding="same"),
BatchNormalization(),
MaxPool2D(pool_size=(3, 3), strides=(2, 2), padding='same'),
Flatten(),
def __init__(self,
num_classes=19,
output_stride=16,
backbonetype='mobilenetv2',
weights='imagenet',
dl_input_shape=(None, 483, 769, 3),
weight_decay=0.00004,
pooling='global',
residual_shortcut=False):
super(CMSNet, self).__init__(name='cmsnet')
"""
:param num_classes: (Default value = 19)
:param output_stride: (Default value = 16)
if strid count is 4 remove stride from block 13 and inser atrous in 14, 15 and 16
if strid count is 3 remove stride from block 6/13 and inser atrous rate 2 in 7-13/ and rate 4 14-16
:param backbonetype: (Default value = 'mobilenetv2')
:param weights: (Default value = 'imagenet')
:param input_shape: (Default value = (None, 483,769,3)
:param weight_decay: use 0.00004 for MobileNet-V2 or Xcpetion model backbonetype. Use 0.0001 for ResNet backbonetype.
"""
self.logger = logging.getLogger('perception.models.CMSNet')
self.logger.info('creating an instance of CMSNet with backbone ' +
backbonetype + ', OS' + str(output_stride) +
', nclass=' + str(num_classes) + ', input=' +
str(dl_input_shape) + ', pooling=' + pooling +
', residual=' + str(residual_shortcut))
self.num_classes = num_classes
self.output_stride = output_stride
self.dl_input_shape = dl_input_shape
self._createBackbone(backbonetype=backbonetype,
output_stride=output_stride)
# All with 256 filters and batch normalization.
# one 1×1 convolution and three 3×3 convolutions with rates = (6, 12, 18) when output stride = 16.
# Rates are doubled when output stride = 8.
#Create Spatial Pyramid Pooling
x = self.backbone.output
pooling_shape = self.backbone.compute_output_shape(self.dl_input_shape)
pooling_shape_float = tf.cast(pooling_shape[1:3], tf.float32)
assert pooling in [
'aspp', 'spp', 'global'
], "Only suported pooling= 'aspp', 'spp' or 'global'."
if pooling == 'aspp':
if output_stride == 16:
rates = (6, 12, 18)
elif output_stride == 8:
rates = (12, 24, 36)
#gride lavel: pooling
x0 = Conv2D(filters=256,
kernel_size=3,
name='aspp_0_expand',
padding="same",
dilation_rate=rates[0],
kernel_regularizer=l2(weight_decay))(x)
x0 = BatchNormalization(name='aspp_0_expand_BN')(x0) #epsilon=1e-5
x0 = ReLU(name='aspp_0_expand_relu')(x0)
x1 = Conv2D(filters=256,
kernel_size=3,
name='aspp_1_expand',
padding="same",
dilation_rate=rates[1],
kernel_regularizer=l2(weight_decay))(x)
x1 = BatchNormalization(name='aspp_1_expand_BN')(x1) #epsilon=1e-5
x1 = ReLU(name='aspp_1_expand_relu')(x1)
x2 = Conv2D(filters=256,
kernel_size=3,
name='aspp_2_expand',
padding="same",
dilation_rate=rates[2],
kernel_regularizer=l2(weight_decay))(x)
x2 = BatchNormalization(name='aspp_2_expand_BN')(x2) #epsilon=1e-5
x2 = ReLU(name='aspp_2_expand_relu')(x2)
#gride lavel: all
xn = Conv2D(filters=256,
kernel_size=1,
name='aspp_n_expand',
kernel_regularizer=l2(weight_decay))(x)
xn = BatchNormalization(name='aspp_n_expand_BN')(xn) #epsilon=1e-5
xn = ReLU(name='aspp_n_expand_relu')(xn)
#Concatenate spatial pyramid pooling
x0.set_shape(pooling_shape[0:3].concatenate(x0.get_shape()[-1]))
x1.set_shape(pooling_shape[0:3].concatenate(x1.get_shape()[-1]))
x2.set_shape(pooling_shape[0:3].concatenate(x2.get_shape()[-1]))
xn.set_shape(pooling_shape[0:3].concatenate(xn.get_shape()[-1]))
x = Concatenate(name='aspp_concatenate')([x0, x1, x2, xn])
elif pooling == 'spp':
rates = (1, 2, 3, 6)
#gride lavel: pooling
x0 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[0],
tf.int32),
padding="valid",
name='spp_0_average_pooling2d')(x)
x0 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_0_expand',
kernel_regularizer=l2(weight_decay))(x0)
x0 = BatchNormalization(name='spp_0_expand_BN')(x0) #epsilon=1e-5
x0 = ReLU(name='spp_0_expand_relu')(x0)
if tf.__version__.split('.')[0] == '1':
x0 = Lambda(lambda x0: tf.image.resize_bilinear(
x0, pooling_shape[1:3], align_corners=True),
name='spp_0_resize_bilinear')(x0)
else:
x0 = Lambda(lambda x0: tf.image.resize(x0,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_0_resize_bilinear')(x0)
x1 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[1],
tf.int32),
padding="valid",
name='spp_1_average_pooling2d')(x)
x1 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_1_expand',
kernel_regularizer=l2(weight_decay))(x1)
x1 = BatchNormalization(name='spp_1_expand_BN')(x1) #epsilon=1e-5
x1 = ReLU(name='spp_1_expand_relu')(x1)
if tf.__version__.split('.')[0] == '1':
x1 = Lambda(lambda x1: tf.image.resize_bilinear(
x1, pooling_shape[1:3], align_corners=True),
name='spp_1_resize_bilinear')(x1)
else:
x1 = Lambda(lambda x1: tf.image.resize(x1,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_1_resize_bilinear')(x1)
x2 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[2],
tf.int32),
padding="valid",
name='spp_2_average_pooling2d')(x)
x2 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_2_expand',
kernel_regularizer=l2(weight_decay))(x2)
x2 = BatchNormalization(name='spp_2_expand_BN')(x2) #epsilon=1e-5
x2 = ReLU(name='spp_2_expand_relu')(x2)
if tf.__version__.split('.')[0] == '1':
x2 = Lambda(lambda x2: tf.image.resize_bilinear(
x2, pooling_shape[1:3], align_corners=True),
name='spp_2_resize_bilinear')(x2)
else:
x2 = Lambda(lambda x2: tf.image.resize(x2,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_2_resize_bilinear')(x2)
x3 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[3],
tf.int32),
padding="valid",
name='spp_3_average_pooling2d')(x)
x3 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_3_expand',
kernel_regularizer=l2(weight_decay))(x3)
x3 = BatchNormalization(name='spp_3_expand_BN')(x3) #epsilon=1e-5
x3 = ReLU(name='spp_3_expand_relu')(x3)
if tf.__version__.split('.')[0] == '1':
x3 = Lambda(lambda x3: tf.image.resize_bilinear(
x3, pooling_shape[1:3], align_corners=True),
name='spp_3_resize_bilinear')(x3)
else:
x3 = Lambda(lambda x3: tf.image.resize(x3,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_3_resize_bilinear')(x3)
#gride lavel: all
xn = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_n_expand',
kernel_regularizer=l2(weight_decay))(x)
xn = BatchNormalization(name='spp_n_expand_BN')(xn) #epsilon=1e-5
xn = ReLU(name='spp_n_expand_relu')(xn)
#Concatenate spatial pyramid pooling
xn.set_shape(pooling_shape[0:3].concatenate(xn.get_shape()[-1]))
x = Concatenate(name='spp_concatenate')([x0, x1, x2, xn])
elif pooling == 'global':
#gride lavel: pooling
x0 = AvgPool2D(pool_size=pooling_shape[1:3],
padding="valid",
name='spp_0_average_pooling2d')(x)
x0 = Conv2D(filters=256,
kernel_size=1,
name='spp_0_expand',
kernel_regularizer=l2(weight_decay))(x0)
x0 = BatchNormalization(name='spp_0_expand_BN')(x0) #epsilon=1e-5
x0 = ReLU(name='spp_0_expand_relu')(x0)
# x0 = tf.image.resize(x0,
# size=pooling_shape[1:3],
# method=tf.image.ResizeMethod.BILINEAR, name='spp_0_resize_bilinear')
if tf.__version__.split('.')[0] == '1':
x0 = Lambda(lambda x0: tf.image.resize_bilinear(
x0, pooling_shape[1:3], align_corners=True),
name='spp_0_resize_bilinear')(x0)
else:
x0 = Lambda(lambda x0: tf.image.resize(x0,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_0_resize_bilinear')(x0)
#gride lavel: all
xn = Conv2D(filters=256,
kernel_size=1,
name='spp_1_expand',
kernel_regularizer=l2(weight_decay))(x)
xn = BatchNormalization(name='spp_1_expand_BN')(xn) #epsilon=1e-5
xn = ReLU(name='spp_1_expand_relu')(xn)
#Concatenate spatial pyramid pooling
xn.set_shape(pooling_shape[0:3].concatenate(xn.get_shape()[-1]))
x = Concatenate(name='spp_concatenate')([x0, xn])
#Concate Projection
x = Conv2D(filters=256,
kernel_size=1,
name='spp_concat_project',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name='spp_concat_project_BN')(x) #epsilon=1e-5
x = ReLU(name='spp_concat_project_relu')(x)
if residual_shortcut:
assert output_stride == 16, "For while residual shotcut is available for atous with os16."
#self.strideOutput8LayerName #block_6_project_BN (BatchNormal (None, 61, 97, 64)
os8_shape = self.backbone.get_layer(
self.strideOutput8LayerName).output_shape
os8_output = self.backbone.get_layer(
self.strideOutput8LayerName).output
x = Conv2D(filters=os8_shape[-1],
kernel_size=1,
name='shotcut_2x_conv',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name='shotcut_2x_BN')(x) #epsilon=1e-5
if tf.__version__.split('.')[0] == '1':
x = Lambda(lambda x: tf.image.resize_bilinear(
x, os8_shape[1:3], align_corners=True),
name='shotcut_2x_bilinear')(x)
else:
x = Lambda(lambda x: tf.image.resize(
x, os8_shape[1:3], method=tf.image.ResizeMethod.BILINEAR),
name='shotcut_2x_bilinear')(x)
x = ReLU(name='shotcut_2x_relu')(x)
x = Add(name='shotcut_2x_add')([x, os8_output])
x = Dropout(rate=0.1, name='dropout')(x)
#Semantic Segmentation
x = Conv2D(filters=num_classes,
kernel_size=1,
name='segmentation',
kernel_regularizer=l2(weight_decay))(x)
#x = BatchNormalization(name='segmentation_BN')(x)
# x = tf.image.resize(x, size=self.dl_input_shape[1:3],
# method=tf.image.ResizeMethod.BILINEAR, name='segmentation_bilinear')
if tf.__version__.split('.')[0] == '1':
x = Lambda(lambda x: tf.image.resize_bilinear(
x, self.dl_input_shape[1:3], align_corners=True),
name='segmentation_bilinear')(x)
else:
x = Lambda(lambda x: tf.image.resize(x,
self.dl_input_shape[1:3],
method=tf.image.ResizeMethod.
BILINEAR),
name='segmentation_bilinear')(x)
x = Softmax(name='logistic_softmax')(x)
#logist to training
#argmax
super(CMSNet, self).__init__(inputs=self.backbone.input,
outputs=x,
name='cmsnet')
train_ratings = ratings['train_data'] val_ratings = ratings['val_data'] test_ratings = ratings['test_data'] num_users = np.max(train_ratings[:, 0]) + 1 y_train_dict = getDataDict(train_ratings) y_test_dict = getDataDict(test_ratings) y_val_dict = getDataDict(val_ratings) x_train_dict = y_train_dict x_test_dict = get_x_test_dict(x_train_dict, list(y_test_dict.keys())) x_val_dict = get_x_test_dict(x_train_dict, list(y_val_dict.keys())) r_i_th = Input(shape=(num_users, )) h1_th = Dense(50, activation='tanh', kernel_regularizer=l2(0))(r_i_th) r_hat_i_th = Dense(num_users, activation='linear')(h1_th) model_th = Model(inputs=r_i_th, outputs=r_hat_i_th) model_th.compile(optimizer='adam', loss=MSE_observed_ratings, metrics=['mae']) r_i_sel = Input(shape=(num_users, )) h1_sel = Dense(100, activation='tanh', kernel_regularizer=l2(0))(r_i_sel) r_hat_i_sel = Dense(num_users, activation='linear')(h1_sel) model_sel = Model(inputs=r_i_sel, outputs=r_hat_i_sel) model_sel.compile(optimizer='adam', loss=MSE_observed_ratings, metrics=['mae']) r_i_sp = Input(shape=(num_users, )) h1_sp = Dense(200, activation='tanh', kernel_regularizer=l2(0))(r_i_sp) r_hat_i_sp = Dense(num_users, activation='linear')(h1_sp) model_sp = Model(inputs=r_i_sp, outputs=r_hat_i_sp) model_sp.compile(optimizer='adam', loss=MSE_observed_ratings, metrics=['mae'])
def build_CNN_base(n_input = 12, n_filters = 512, n_kernel = 3, optimizer = 'Adam', l2_penalty = 0.01, learning_rate=0.001,):
n_input = n_input
n_features= 1
# define model
model = Sequential()
model.add(Conv1D(filters=n_filters, kernel_size=n_kernel, activation='relu', kernel_regularizer = l2(l2_penalty), input_shape=(n_input, 1)))
model.add(Conv1D(filters=n_filters, kernel_size=n_kernel, activation='relu', kernel_regularizer = l2(l2_penalty)))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(1))
model.compile(loss='mae', optimizer=optimizer, metrics=['mae', 'mape'])
return model
#custom_opt = tf.keras.optimizers.Adam(0.01)
model.compile(optimizer='adam', loss='mse')
report = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=300,
verbose=False)
plt.plot(report.history['loss'], label="loss")
plt.plot(report.history['val_loss'], label="validation_loss")
plt.legend()
"""## Great, the model is overfitting, let's try dropout"""
i_layer = Input(shape=(D, ))
h_layer = Dense(128, activation='relu', kernel_regularizer=l2(0.9))(i_layer)
h_layer = Dense(256, activation='relu', kernel_regularizer=l2(0.9))(h_layer)
h_layer = Dense(256, activation='relu', kernel_regularizer=l2(0.9))(h_layer)
o_layer = Dense(1, activation='relu')(h_layer)
model = Model(i_layer, o_layer)
#custom_opt = tf.keras.optimizers.Adam(0.01)
model.compile(optimizer='adam', loss='mse')
report = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=300,
verbose=False)
plt.plot(report.history['loss'], label="loss")
plt.plot(report.history['val_loss'], label="validation_loss")
def get_model(hyper_params):
"""Generating ssd model for hyper params.
inputs:
hyper_params = dictionary
outputs:
ssd_model = tf.keras.model
"""
# Initial scale factor 20 in the paper.
# Even if this scale factor could cause loss value to be NaN in some of the cases,
# it was decided to remain the same after some tests.
scale_factor = 20.0
reg_factor = 5e-4
total_labels = hyper_params["total_labels"]
# +1 for ratio 1
len_aspect_ratios = [len(x) + 1 for x in hyper_params["aspect_ratios"]]
#
input = Input(shape=(None, None, 3), name="input")
# conv1 block
conv1_1 = Conv2D(64, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv1_1")(input)
conv1_2 = Conv2D(64, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv1_2")(conv1_1)
pool1 = MaxPool2D((2, 2), strides=(2, 2), padding="same",
name="pool1")(conv1_2)
# conv2 block
conv2_1 = Conv2D(128, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv2_1")(pool1)
conv2_2 = Conv2D(128, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv2_2")(conv2_1)
pool2 = MaxPool2D((2, 2), strides=(2, 2), padding="same",
name="pool2")(conv2_2)
# conv3 block
conv3_1 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv3_1")(pool2)
conv3_2 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv3_2")(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv3_3")(conv3_2)
pool3 = MaxPool2D((2, 2), strides=(2, 2), padding="same",
name="pool3")(conv3_3)
# conv4 block
conv4_1 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv4_1")(pool3)
conv4_2 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv4_2")(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv4_3")(conv4_2)
pool4 = MaxPool2D((2, 2), strides=(2, 2), padding="same",
name="pool4")(conv4_3)
# conv5 block
conv5_1 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv5_1")(pool4)
conv5_2 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv5_2")(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv5_3")(conv5_2)
pool5 = MaxPool2D((3, 3), strides=(1, 1), padding="same",
name="pool5")(conv5_3)
# conv6 and conv7 converted from fc6 and fc7 and remove dropouts
# These layers coming from modified vgg16 model
# https://gist.github.com/weiliu89/2ed6e13bfd5b57cf81d6
conv6 = Conv2D(1024, (3, 3),
dilation_rate=6,
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv6")(pool5)
conv7 = Conv2D(1024, (1, 1),
strides=(1, 1),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv7")(conv6)
############################ Extra Feature Layers Start ############################
# conv8 block <=> conv6 block in paper caffe implementation
conv8_1 = Conv2D(256, (1, 1),
strides=(1, 1),
padding="valid",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv8_1")(conv7)
conv8_2 = Conv2D(512, (3, 3),
strides=(2, 2),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv8_2")(conv8_1)
# conv9 block <=> conv7 block in paper caffe implementation
conv9_1 = Conv2D(128, (1, 1),
strides=(1, 1),
padding="valid",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv9_1")(conv8_2)
conv9_2 = Conv2D(256, (3, 3),
strides=(2, 2),
padding="same",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv9_2")(conv9_1)
# conv10 block <=> conv8 block in paper caffe implementation
conv10_1 = Conv2D(128, (1, 1),
strides=(1, 1),
padding="valid",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv10_1")(conv9_2)
conv10_2 = Conv2D(256, (3, 3),
strides=(1, 1),
padding="valid",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv10_2")(conv10_1)
# conv11 block <=> conv9 block in paper caffe implementation
conv11_1 = Conv2D(128, (1, 1),
strides=(1, 1),
padding="valid",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv11_1")(conv10_2)
conv11_2 = Conv2D(256, (3, 3),
strides=(1, 1),
padding="valid",
activation="relu",
kernel_initializer="glorot_normal",
kernel_regularizer=l2(reg_factor),
name="conv11_2")(conv11_1)
############################ Extra Feature Layers End ############################
# l2 normalization for each location in the feature map
conv4_3_norm = L2Normalization(scale_factor)(conv4_3)
#
pred_bbox_deltas, pred_labels = get_head_from_outputs(
hyper_params,
[conv4_3_norm, conv7, conv8_2, conv9_2, conv10_2, conv11_2])
#
return Model(inputs=input, outputs=[pred_bbox_deltas, pred_labels])
# Saving the model at each step
checkpoint_filepath = "tmp/checkpoint_SPARCS_unet"
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_filepath,
save_weights_only=True,
monitor='val_loss',
mode='min',
save_best_only=True)
# Early Stopping Callback
ES = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=30)
# Deep Learning Model
model = UnetSeparable(shape=(256, 256, 10), depth=10, nb_filters=4, kernel_size=5, initialization="he_normal", output_channels=6, drop=0.3, regularization=l2(0.00013443))
print(model.summary())
model.compile(loss = "categorical_crossentropy", optimizer=Adam(0.00046131), metrics=["accuracy"])
# Training the model
history = model.fit(generator, batch_size=20,
steps_per_epoch=50,
epochs=150,
validation_data = (val_data, val_labels),
callbacks=[model_checkpoint_callback])
np.save("logs/metrics/SPARCS/u_net_sep/train_accuracy", history.history['accuracy'])
np.save("logs/metrics/SPARCS/u_net_sep/val_accuracy", history.history['val_accuracy'])
# Making a prediction on the test data
def build_model(image_size,
n_classes,
mode='training',
l2_regularization=0.0,
min_scale=0.1,
max_scale=0.9,
scales=None,
aspect_ratios_global=[0.5, 1.0, 2.0],
aspect_ratios_per_layer=None,
two_boxes_for_ar1=True,
steps=None,
offsets=None,
clip_boxes=False,
variances=[1.0, 1.0, 1.0, 1.0],
coords='centroids',
normalize_coords=False,
subtract_mean=None,
divide_by_stddev=None,
swap_channels=False,
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
nms_max_output_size=400,
return_predictor_sizes=False):
'''
Build a Keras model with SSD architecture, see references.
The model consists of convolutional feature layers and a number of convolutional
predictor layers that take their input from different feature layers.
The model is fully convolutional.
The implementation found here is a smaller version of the original architecture
used in the paper (where the base network consists of a modified VGG-16 extended
by a few convolutional feature layers), but of course it could easily be changed to
an arbitrarily large SSD architecture by following the general design pattern used here.
This implementation has 7 convolutional layers and 4 convolutional predictor
layers that take their input from layers 4, 5, 6, and 7, respectively.
Most of the arguments that this function takes are only needed for the anchor
box layers. In case you're training the network, the parameters passed here must
be the same as the ones used to set up `SSDBoxEncoder`. In case you're loading
trained weights, the parameters passed here must be the same as the ones used
to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Training currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
mode (str, optional): One of 'training', 'inference' and 'inference_fast'. In 'training' mode,
the model outputs the raw prediction tensor, while in 'inference' and 'inference_fast' modes,
the raw predictions are decoded into absolute coordinates and filtered via confidence thresholding,
non-maximum suppression, and top-k filtering. The difference between latter two modes is that
'inference' follows the exact procedure of the original Caffe implementation, while
'inference_fast' uses a faster prediction decoding procedure.
l2_regularization (float, optional): The L2-regularization rate. Applies to all convolutional layers.
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used. If a list is passed,
this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all predictor layers. The original implementation uses more aspect ratios
for some predictor layers and fewer for others. If you want to do that, too, then use the next argument instead.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each predictor layer.
This allows you to set the aspect ratios for each predictor layer individually. If a list is passed,
it overrides `aspect_ratios_global`.
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size,
which is also the recommended setting.
clip_boxes (bool, optional): If `True`, clips the anchor box coordinates to stay within image boundaries.
variances (list, optional): A list of 4 floats >0. The anchor box offset for each coordinate will be divided by
its respective variance value.
coords (str, optional): The box coordinate format to be used internally by the model (i.e. this is not the input format
of the ground truth labels). Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width,
and height), 'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
swap_channels (list, optional): Either `False` or a list of integers representing the desired order in which the input
image channels should be swapped.
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes that have a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box's confidence score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage.
nms_max_output_size (int, optional): The maximal number of predictions that will be left over after the NMS stage.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. They are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD model.
predictor_sizes (optional): A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
'''
n_predictor_layers = 4 # The number of predictor conv layers in the network
n_classes += 1 # Account for the background class.
l2_reg = l2_regularization # Make the internal name shorter.
img_height, img_width, img_channels = image_size[0], image_size[
1], image_size[2]
############################################################################
# Get a few exceptions out of the way.
############################################################################
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError(
"`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified."
)
if aspect_ratios_per_layer:
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError(
"It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}."
.format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError(
"Either `min_scale` and `max_scale` or `scales` need to be specified."
)
if scales:
if len(scales) != n_predictor_layers + 1:
raise ValueError(
"It must be either scales is None or len(scales) == {}, but len(scales) == {}."
.format(n_predictor_layers + 1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1)
if len(
variances
) != 4: # We need one variance value for each of the four box coordinates
raise ValueError(
"4 variance values must be pased, but {} values were received.".
format(len(variances)))
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError(
"All variances must be >0, but the variances given are {}".format(
variances))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError(
"You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError(
"You must provide at least one offset value per predictor layer.")
############################################################################
# Compute the anchor box parameters.
############################################################################
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) +
1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
############################################################################
# Define functions for the Lambda layers below.
############################################################################
def identity_layer(tensor):
return tensor
def input_mean_normalization(tensor):
return tensor - np.array(subtract_mean)
def input_stddev_normalization(tensor):
return tensor / np.array(divide_by_stddev)
#def input_channel_swap(tensor):
# if len(swap_channels) == 3:
# return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]]], axis=-1)
# elif len(swap_channels) == 4:
# return K.stack([tensor[...,swap_channels[0]], tensor[...,swap_channels[1]], tensor[...,swap_channels[2]], tensor[...,swap_channels[3]]], axis=-1)
############################################################################
# Build the network.
############################################################################
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that the subsequent lambda layers can be optional.
x1 = Lambda(identity_layer,
output_shape=(img_height, img_width, img_channels),
name='identity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(input_mean_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(input_stddev_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_stddev_normalization')(x1)
#if swap_channels: #REMOVED FOR TFLITE
# x1 = Lambda(input_channel_swap, output_shape=(img_height, img_width, img_channels), name='input_channel_swap')(x1)
conv1 = Conv2D(32, (5, 5),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1')(x1)
conv1 = BatchNormalization(axis=3, momentum=0.99, name='bn1')(
conv1
) # Tensorflow uses filter format [filter_height, filter_width, in_channels, out_channels], hence axis = 3
conv1 = ELU(name='elu1')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1)
conv2 = Conv2D(48, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2')(pool1)
conv2 = BatchNormalization(axis=3, momentum=0.99, name='bn2')(conv2)
conv2 = ELU(name='elu2')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2)
conv3 = Conv2D(64, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3')(pool2)
conv3 = BatchNormalization(axis=3, momentum=0.99, name='bn3')(conv3)
conv3 = ELU(name='elu3')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3)
conv4 = Conv2D(64, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4')(pool3)
conv4 = BatchNormalization(axis=3, momentum=0.99, name='bn4')(conv4)
conv4 = ELU(name='elu4')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4)
conv5 = Conv2D(48, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5')(pool4)
conv5 = BatchNormalization(axis=3, momentum=0.99, name='bn5')(conv5)
conv5 = ELU(name='elu5')(conv5)
pool5 = MaxPooling2D(pool_size=(2, 2), name='pool5')(conv5)
conv6 = Conv2D(48, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6')(pool5)
conv6 = BatchNormalization(axis=3, momentum=0.99, name='bn6')(conv6)
conv6 = ELU(name='elu6')(conv6)
pool6 = MaxPooling2D(pool_size=(2, 2), name='pool6')(conv6)
conv7 = Conv2D(32, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7')(pool6)
conv7 = BatchNormalization(axis=3, momentum=0.99, name='bn7')(conv7)
conv7 = ELU(name='elu7')(conv7)
# The next part is to add the convolutional predictor layers on top of the base network
# that we defined above. Note that I use the term "base network" differently than the paper does.
# To me, the base network is everything that is not convolutional predictor layers or anchor
# box layers. In this case we'll have four predictor layers, but of course you could
# easily rewrite this into an arbitrarily deep base network and add an arbitrary number of
# predictor layers on top of the base network by simply following the pattern shown here.
# Build the convolutional predictor layers on top of conv layers 4, 5, 6, and 7.
# We build two predictor layers on top of each of these layers: One for class prediction (classification), one for box coordinate prediction (localization)
# We precidt `n_classes` confidence values for each box, hence the `classes` predictors have depth `n_boxes * n_classes`
# We predict 4 box coordinates for each box, hence the `boxes` predictors have depth `n_boxes * 4`
# Output shape of `classes`: `(batch, height, width, n_boxes * n_classes)`
classes4 = Conv2D(n_boxes[0] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes4')(conv4)
classes5 = Conv2D(n_boxes[1] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes5')(conv5)
classes6 = Conv2D(n_boxes[2] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes6')(conv6)
classes7 = Conv2D(n_boxes[3] * n_classes, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='classes7')(conv7)
# Output shape of `boxes`: `(batch, height, width, n_boxes * 4)`
boxes4 = Conv2D(n_boxes[0] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes4')(conv4)
boxes5 = Conv2D(n_boxes[1] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes5')(conv5)
boxes6 = Conv2D(n_boxes[2] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes6')(conv6)
boxes7 = Conv2D(n_boxes[3] * 4, (3, 3),
strides=(1, 1),
padding="same",
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='boxes7')(conv7)
# Generate the anchor boxes
# Output shape of `anchors`: `(batch, height, width, n_boxes, 8)`
anchors4 = AnchorBoxes(img_height,
img_width,
this_scale=scales[0],
next_scale=scales[1],
aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[0],
this_offsets=offsets[0],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors4')(boxes4)
anchors5 = AnchorBoxes(img_height,
img_width,
this_scale=scales[1],
next_scale=scales[2],
aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[1],
this_offsets=offsets[1],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors5')(boxes5)
anchors6 = AnchorBoxes(img_height,
img_width,
this_scale=scales[2],
next_scale=scales[3],
aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[2],
this_offsets=offsets[2],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors6')(boxes6)
anchors7 = AnchorBoxes(img_height,
img_width,
this_scale=scales[3],
next_scale=scales[4],
aspect_ratios=aspect_ratios[3],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[3],
this_offsets=offsets[3],
clip_boxes=clip_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='anchors7')(boxes7)
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)`
# We want the classes isolated in the last axis to perform softmax on them
#Shape inference is different for keras.layers and tf.keras.layers (-1), check documentation
#So I had to manually input the intended shape of the reshape
#cba = classes, boxes, anchors SHAPE: shape will be reused later anyways
cba_4 = classes4.shape[1] * classes4.shape[2] * n_boxes[0]
cba_5 = classes5.shape[1] * classes5.shape[2] * n_boxes[1]
cba_6 = classes6.shape[1] * classes6.shape[2] * n_boxes[2]
cba_7 = classes7.shape[1] * classes7.shape[2] * n_boxes[3]
classes4_reshaped = Reshape((cba_4, n_classes),
name='classes4_reshape')(classes4)
classes5_reshaped = Reshape((cba_5, n_classes),
name='classes5_reshape')(classes5)
classes6_reshaped = Reshape((cba_6, n_classes),
name='classes6_reshape')(classes6)
classes7_reshaped = Reshape((cba_7, n_classes),
name='classes7_reshape')(classes7)
# Reshape the box coordinate predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
#shape for classes_reshaped is SAME with boxes EXCEPT for n_classes and anchors so NO NEED TO RECOMPUTE
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
boxes4_reshaped = Reshape((cba_4, 4), name='boxes4_reshape')(boxes4)
boxes5_reshaped = Reshape((cba_5, 4), name='boxes5_reshape')(boxes5)
boxes6_reshaped = Reshape((cba_6, 4), name='boxes6_reshape')(boxes6)
boxes7_reshaped = Reshape((cba_7, 4), name='boxes7_reshape')(boxes7)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
anchors4_reshaped = Reshape((cba_4, 8), name='anchors4_reshape')(anchors4)
anchors5_reshaped = Reshape((cba_5, 8), name='anchors5_reshape')(anchors5)
anchors6_reshaped = Reshape((cba_6, 8), name='anchors6_reshape')(anchors6)
anchors7_reshaped = Reshape((cba_7, 8), name='anchors7_reshape')(anchors7)
# Concatenate the predictions from the different layers and the assosciated anchor box tensors
# Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1
# Output shape of `classes_concat`: (batch, n_boxes_total, n_classes)
classes_concat = Concatenate(axis=1, name='classes_concat')([
classes4_reshaped, classes5_reshaped, classes6_reshaped,
classes7_reshaped
])
# Output shape of `boxes_concat`: (batch, n_boxes_total, 4)
boxes_concat = Concatenate(axis=1, name='boxes_concat')(
[boxes4_reshaped, boxes5_reshaped, boxes6_reshaped, boxes7_reshaped])
# Output shape of `anchors_concat`: (batch, n_boxes_total, 8)
anchors_concat = Concatenate(axis=1, name='anchors_concat')([
anchors4_reshaped, anchors5_reshaped, anchors6_reshaped,
anchors7_reshaped
])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
classes_softmax = Activation('softmax',
name='classes_softmax')(classes_concat)
# Concatenate the class and box coordinate predictions and the anchors to one large predictions tensor
# Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')(
[classes_softmax, boxes_concat, anchors_concat])
if mode == 'training':
model = Model(inputs=x, outputs=predictions)
elif mode == 'inference':
decoded_predictions = DecodeDetections(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=x, outputs=decoded_predictions)
elif mode == 'inference_fast':
decoded_predictions = DecodeDetectionsFast(
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
nms_max_output_size=nms_max_output_size,
coords=coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width,
name='decoded_predictions')(predictions)
model = Model(inputs=x, outputs=decoded_predictions)
else:
raise ValueError(
"`mode` must be one of 'training', 'inference' or 'inference_fast', but received '{}'."
.format(mode))
if return_predictor_sizes:
# The spatial dimensions are the same for the `classes` and `boxes` predictor layers.
predictor_sizes = np.array([
classes4._keras_shape[1:3], classes5._keras_shape[1:3],
classes6._keras_shape[1:3], classes7._keras_shape[1:3]
])
return model, predictor_sizes
else:
return model
def aysennet_model(img_shape=(300, 300, 3), n_classes=12, l2_reg=0.01):
# Initialize model
aysennet = Sequential()
# Layer 1
aysennet.add(
Conv2D(128, (3, 3),
input_shape=img_shape,
padding='same',
kernel_regularizer=l2(l2_reg)))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 2
aysennet.add(Conv2D(256, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 4
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(256, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 5
aysennet.add(ZeroPadding2D((1, 1)))
aysennet.add(Conv2D(128, (3, 3), padding='same'))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(MaxPooling2D(pool_size=(2, 2)))
# Layer 6
aysennet.add(Flatten())
aysennet.add(Dense(1024))
aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(Dropout(0.5))
""" # Layer 7
aysennet.add(Dense(4096))
#aysennet.add(BatchNormalization())
aysennet.add(Activation('relu'))
aysennet.add(Dropout(0.5))"""
# Layer 8
aysennet.add(Dense(n_classes))
aysennet.add(BatchNormalization())
aysennet.add(Activation('softmax'))
return aysennet
