def get_unet_model_yuanqing(self):
# Model inspired by https://github.com/yuanqing811/ISIC2018
unet_input = Input(shape=(self.input_dim_x, self.input_dim_y,
self.num_channels))
conv1 = Conv2D(self.n_filters,
kernel_size=3,
activation='relu',
padding='same')(unet_input)
conv1 = Conv2D(self.n_filters,
kernel_size=3,
activation='relu',
padding='same')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(pool1)
conv2 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(pool2)
conv3 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(conv3)
conv3 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(pool3)
conv4 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(conv4)
conv4 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(pool4)
conv5 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(conv5)
conv5 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(conv5)
up6 = Conv2D(self.n_filters * 4, 2, activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(conv5))
feature4 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(conv4)
concat6 = Concatenate()([feature4, up6])
conv6 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(concat6)
conv6 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(conv6)
up7 = Conv2D(self.n_filters * 2, 2, activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(conv6))
feature3 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(conv3)
concat7 = Concatenate()([feature3, up7])
conv7 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(concat7)
conv7 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(conv7)
up8 = Conv2D(self.n_filters * 1, 2, activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(conv7))
feature2 = Conv2D(self.n_filters * 1,
kernel_size=3,
activation='relu',
padding='same')(conv2)
concat8 = Concatenate()([feature2, up8])
conv8 = Conv2D(self.n_filters * 1,
kernel_size=3,
activation='relu',
padding='same')(concat8)
conv8 = Conv2D(self.n_filters * 1,
kernel_size=3,
activation='relu',
padding='same')(conv8)
up9 = Conv2D(int(self.n_filters / 2),
2,
activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(conv8))
feature1 = Conv2D(int(self.n_filters / 2),
kernel_size=3,
activation='relu',
padding='same')(conv1)
concat9 = Concatenate()([feature1, up9])
conv9 = Conv2D(int(self.n_filters / 2),
kernel_size=3,
activation='relu',
padding='same')(concat9)
conv9 = Conv2D(int(self.n_filters / 2),
kernel_size=3,
activation='relu',
padding='same')(conv9)
conv9 = Conv2D(3, kernel_size=3, activation='relu',
padding='same')(conv9)
conv10 = Conv2D(1, kernel_size=1, activation='sigmoid')(conv9)
return Model(outputs=conv10, inputs=unet_input), 'unet_model_yuanqing'
def Net(n_label,img_input, drop_rate=0.2):
# ---------left branch -----
x = conv_block(img_input, 32, (3, 3), strides=1, name='L_conv1-1')
x = SpatialDropout2D(drop_rate)(x)
L1 = conv_block(x, 32, (3, 3), strides=1, name='L_conv1-2')
x = conv_block(L1, 32, (3, 3), strides=2, name='L_conv1-3')
# 400 -> 200
x = conv_block(x, 64, (3, 3), strides=1, name='L_conv2-1')
x = SpatialDropout2D(drop_rate)(x)
L2 = conv_block(x, 64, (3, 3), strides=1, name='L_conv2-2')
x = conv_block(L2, 32, (3, 3), strides=2, name='L_conv2-3')
# 200 -> 100
x = conv_block(x, 128, (3, 3), strides=1, name='L_conv3-1')
x = SpatialDropout2D(drop_rate)(x)
L3 = conv_block(x, 128, (3, 3), strides=1, name='L_conv3-2')
x = conv_block(L3, 32, (3, 3), strides=2, name='L_conv3-3')
# 100 -> 50
x = conv_block(x, 256, (3, 3), strides=1, name='L_conv4-1')
x = SpatialDropout2D(drop_rate)(x)
L4 = conv_block(x, 256, (3, 3), strides=1, name='L_conv4-2')
x = conv_block(L4, 32, (3, 3), strides=2, name='L_conv4-3')
# 50 -> 25
x = conv_block(x, 512, (3, 3), strides=1, name='L_conv5-1')
x = conv_block(x, 512, (3, 3), strides=1, dila=2, name='L_conv5-2')
x = SpatialDropout2D(drop_rate)(x)
x = conv_block(x, 512, (3, 3), strides=1, dila=2, name='L_conv5-3')
L5 = conv_block(x, 512, (3, 3), strides=1, name='L_conv5-4')
# 25
# ---------Right branch -----
# 25 -> 50
x = Deconv2D(256, kernel_size=2, strides=2, padding='same',name='R_conv1-1')(L5)
x = BatchNormalization(axis=bn_axis, name='R_conv1-1_' + 'bn')(x)
x = conv_block(Concatenate(axis=-1)([x, L4]), 256, (3, 3), strides=1, name='R_conv1-2')
x = SpatialDropout2D(0.1)(x)
x = conv_block(x, 256, (3, 3), strides=1, name='R_conv1-3')
R_out1 = Conv2D(n_label,(1,1),name='R_out1')(x)
# 50 -> 100
x = Deconv2D(128, kernel_size=2, strides=2, padding='same', name='R_conv2-1')(x)
x = BatchNormalization(axis=bn_axis, name='R_conv2-1_' + 'bn')(x)
x = conv_block(Concatenate(axis=-1)([x, L3]), 128, (3, 3), strides=1, name='R_conv2-2')
x = SpatialDropout2D(0.1)(x)
x = conv_block(x, 128, (3, 3), strides=1, name='R_conv2-3')
R_out2 = Conv2D(n_label, (1, 1), name='R_out2')(x)
# 100 -> 200
x = Deconv2D(64, kernel_size=2, strides=2, padding='same', name='R_conv3-1')(x)
x = BatchNormalization(axis=bn_axis, name='R_conv3-1_' + 'bn')(x)
x = conv_block(Concatenate(axis=-1)([x, L2]), 64, (3, 3), strides=1, name='R_conv3-2')
x = SpatialDropout2D(0.1)(x)
x = conv_block(x, 64, (3, 3), strides=1, name='R_conv3-3')
R_out3 = Conv2D(n_label, (1, 1), name='R_out3')(x)
# 200 -> 400
x = Deconv2D(32, kernel_size=2, strides=2, padding='same', name='R_conv4-1')(x)
x = BatchNormalization(axis=bn_axis, name='R_conv4-1_' + 'bn')(x)
x = conv_block(Concatenate(axis=-1)([x, L1]), 32, (3, 3), strides=1, name='R_conv4-2')
x = SpatialDropout2D(0.1)(x)
x = conv_block(x, 32, (3, 3), strides=1, name='R_conv4-3')
R_out4 = Conv2D(n_label, (1, 1), name='R_out4')(x)
# ---------Recoding branch -----
x = conv_block(R_out4, 32, (1, 1), strides=1, name='E_conv1-1')
x = conv_block(x, 32, (3, 3), strides=1, name='E_conv1-2')
x = SpatialDropout2D(drop_rate)(x)
x = conv_block(x, 32, (3, 3), strides=2, name='E_conv1-3')
# 400 -> 200
x = conv_block(Concatenate(axis=-1)([x, conv_block(R_out3,64, (1, 1), strides=1,name='c1')]), 64, (3, 3), strides=1, name='E_conv2-1')
x = conv_block(x, 64, (3, 3), strides=1, name='E_conv2-2')
x = SpatialDropout2D(drop_rate)(x)
x = conv_block(x, 64, (3, 3), strides=2, name='E_conv2-3')
# 200 -> 100
x = conv_block(Concatenate(axis=-1)([x, conv_block(R_out2,128, (1, 1), strides=1,name='c2')]), 128, (3, 3), strides=1, name='E_conv3-1')
x = conv_block(x, 128, (3, 3), strides=1, name='E_conv3-2')
x = SpatialDropout2D(drop_rate)(x)
x = conv_block(x, 128, (3, 3), strides=2, name='E_conv3-3')
# 100 -> 50
x = conv_block(Concatenate(axis=-1)([x, conv_block(R_out1,256, (1, 1), strides=1,name='c3')]), 256, (3, 3), strides=1, name='E_conv4-1')
x = conv_block(x, 256, (3, 3), strides=1, name='E_conv4-2')
x = SpatialDropout2D(drop_rate)(x)
x = conv_block(x, 256, (3, 3), strides=1, dila=2, name='E_conv4-3')
x = conv_block(x, 256, (3, 3), strides=1, dila=2, name='E_conv4-4')
x = conv_block(x, 256, (3, 3), strides=1, name='E_conv4-5')
# 50
x = global_context_block(x, channels=64)
# -----------------------------------------
final_out = Conv2D(n_label,(1,1), name='final_out')(x)
final_out = UpSampling2D(size=(8,8))(final_out)
final_out = Activation('softmax',name='l0')(Reshape((400 * 400, n_label))(final_out))
out1 = Activation('softmax',name='l1')(Reshape((400 * 400, n_label))(R_out4))
out2 = Activation('softmax',name='l2')(Reshape((200 * 200, n_label))(R_out3))
out3 = Activation('softmax',name='l3')(Reshape((100 * 100, n_label))(R_out2))
out4 = Activation('softmax',name='l4')(Reshape((50 * 50, n_label))(R_out1))
return [final_out, out1, out2, out3, out4]
def gamba_unet():
from keras import regularizers
from keras.activations import softmax
from keras.layers import Input, Conv2D, Conv2DTranspose, BatchNormalization, Concatenate, Lambda, Activation, Reshape, Add
from keras.models import Model
inputs = Input(shape=(432, 432, 2))
weight_matrix = Lambda(lambda z: z[:, :, :, 1])(inputs)
weight_matrix = Reshape((432, 432, 1))(weight_matrix)
reshape = Lambda(lambda z: z[:, :, :, 0])(inputs)
reshape = Reshape((432, 432, 1))(reshape)
reg = 0.01
#reshape=Dropout(0.2)(reshape) ## Hyperparameter optimization only on visible layer
Level1_l = Conv2D(filters=32,
kernel_size=(1, 1),
strides=1,
kernel_regularizer=regularizers.l2(reg))(reshape)
Level1_l = BatchNormalization(axis=-1)(Level1_l)
Level1_l_shortcut = Level1_l #Level1_l#
Level1_l = Activation('relu')(Level1_l)
Level1_l = Conv2D(
filters=32,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(
Level1_l
) #(Level1_l)# ## kernel_initializer='glorot_uniform' is the default
Level1_l = BatchNormalization(axis=-1)(Level1_l)
#Level1_l=InstanceNormalization(axis=-1)(Level1_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level1_l = Activation('relu')(Level1_l)
#Level1_l=Dropout(0.5)(Level1_l)
Level1_l = Conv2D(filters=32,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level1_l)
Level1_l = BatchNormalization(axis=-1)(Level1_l)
#Level1_l=InstanceNormalization(axis=-1)(Level1_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level1_l = Add()([Level1_l, Level1_l_shortcut])
Level1_l = Activation('relu')(Level1_l)
Level2_l = Conv2D(filters=64,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level1_l)
Level2_l = BatchNormalization(axis=-1)(Level2_l)
Level2_l_shortcut = Level2_l
Level2_l = Activation('relu')(Level2_l)
Level2_l = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level2_l)
Level2_l = BatchNormalization(axis=-1)(Level2_l)
#Level2_l=InstanceNormalization(axis=-1)(Level2_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level2_l = Activation('relu')(Level2_l)
#Level2_l=Dropout(0.5)(Level2_l)
Level2_l = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level2_l)
Level2_l = BatchNormalization(axis=-1)(Level2_l)
#Level2_l=InstanceNormalization(axis=-1)(Level2_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level2_l = Add()([Level2_l, Level2_l_shortcut])
Level2_l = Activation('relu')(Level2_l)
Level3_l = Conv2D(filters=128,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level2_l)
Level3_l = BatchNormalization(axis=-1)(Level3_l)
Level3_l_shortcut = Level3_l
Level3_l = Activation('relu')(Level3_l)
Level3_l = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level3_l)
Level3_l = BatchNormalization(axis=-1)(Level3_l)
#Level3_l=InstanceNormalization(axis=-1)(Level3_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level3_l = Activation('relu')(Level3_l)
#Level3_l=Dropout(0.5)(Level3_l)
Level3_l = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level3_l)
Level3_l = BatchNormalization(axis=-1)(Level3_l)
#Level3_l=InstanceNormalization(axis=-1)(Level3_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level3_l = Add()([Level3_l, Level3_l_shortcut])
Level3_l = Activation('relu')(Level3_l)
Level4_l = Conv2D(filters=256,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level3_l)
Level4_l = BatchNormalization(axis=-1)(Level4_l)
Level4_l_shortcut = Level4_l
Level4_l = Activation('relu')(Level4_l)
Level4_l = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level4_l)
Level4_l = BatchNormalization(axis=-1)(Level4_l)
#Level4_l=InstanceNormalization(axis=-1)(Level4_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level4_l = Activation('relu')(Level4_l)
#Level4_l=Dropout(0.5)(Level4_l)
Level4_l = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level4_l)
Level4_l = BatchNormalization(axis=-1)(Level4_l)
#Level4_l=InstanceNormalization(axis=-1)(Level4_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level4_l = Add()([Level4_l, Level4_l_shortcut])
Level4_l = Activation('relu')(Level4_l)
Level5_l = Conv2D(filters=512,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level4_l)
Level5_l = BatchNormalization(axis=-1)(Level5_l)
Level5_l_shortcut = Level5_l
Level5_l = Activation('relu')(Level5_l)
Level5_l = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level5_l)
Level5_l = BatchNormalization(axis=-1)(Level5_l)
#Level5_l=InstanceNormalization(axis=-1)(Level5_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level5_l = Activation('relu')(Level5_l)
#Level5_l=Dropout(0.5)(Level5_l)
Level5_l = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level5_l)
Level5_l = BatchNormalization(axis=-1)(Level5_l)
#Level5_l=InstanceNormalization(axis=-1)(Level5_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level5_l = Add()([Level5_l, Level5_l_shortcut])
Level5_l = Activation('relu')(Level5_l)
Level6_l = Conv2D(filters=1024,
kernel_size=(3, 3),
strides=3,
kernel_regularizer=regularizers.l2(reg))(Level5_l)
Level6_l = BatchNormalization(axis=-1)(Level6_l)
Level6_l_shortcut = Level6_l
Level6_l = Activation('relu')(Level6_l)
Level6_l = Conv2D(filters=1024,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level6_l)
Level6_l = BatchNormalization(axis=-1)(Level6_l)
#Level5_l=InstanceNormalization(axis=-1)(Level5_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level6_l = Activation('relu')(Level6_l)
#Level5_l=Dropout(0.5)(Level5_l)
Level6_l = Conv2D(filters=1024,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level6_l)
Level6_l = BatchNormalization(axis=-1)(Level6_l)
#Level5_l=InstanceNormalization(axis=-1)(Level5_l) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level6_l = Add()([Level6_l, Level6_l_shortcut])
Level6_l = Activation('relu')(Level6_l)
Level5_r = Conv2DTranspose(
filters=512,
kernel_size=(3, 3),
strides=3,
kernel_regularizer=regularizers.l2(reg))(Level6_l)
Level5_r = BatchNormalization(axis=-1)(Level5_r)
Level5_r_shortcut = Level5_r
#Level4_r=InstanceNormalization(axis=-1)(Level4_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level5_r = Activation('relu')(Level5_r)
merge5 = Concatenate(axis=-1)([Level5_l, Level5_r])
Level5_r = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(merge5)
Level5_r = BatchNormalization(axis=-1)(Level5_r)
#Level4_r=InstanceNormalization(axis=-1)(Level4_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level5_r = Activation('relu')(Level5_r)
#Level4_r=Dropout(0.5)(Level4_r)
Level5_r = Conv2D(filters=512,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level5_r)
Level5_r = BatchNormalization(axis=-1)(Level5_r)
#Level4_r=InstanceNormalization(axis=-1)(Level4_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level5_r = Add()([Level5_r, Level5_r_shortcut])
Level5_r = Activation('relu')(Level5_r)
Level4_r = Conv2DTranspose(
filters=256,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level5_r)
Level4_r = BatchNormalization(axis=-1)(Level4_r)
Level4_r_shortcut = Level4_r
#Level4_r=InstanceNormalization(axis=-1)(Level4_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level4_r = Activation('relu')(Level4_r)
merge4 = Concatenate(axis=-1)([Level4_l, Level4_r])
Level4_r = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(merge4)
Level4_r = BatchNormalization(axis=-1)(Level4_r)
#Level4_r=InstanceNormalization(axis=-1)(Level4_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level4_r = Activation('relu')(Level4_r)
#Level4_r=Dropout(0.5)(Level4_r)
Level4_r = Conv2D(filters=256,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level4_r)
Level4_r = BatchNormalization(axis=-1)(Level4_r)
#Level4_r=InstanceNormalization(axis=-1)(Level4_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level4_r = Add()([Level4_r, Level4_r_shortcut])
Level4_r = Activation('relu')(Level4_r)
Level3_r = Conv2DTranspose(
filters=128,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level4_r)
Level3_r = BatchNormalization(axis=-1)(Level3_r)
Level3_r_shortcut = Level3_r
#Level3_r=InstanceNormalization(axis=-1)(Level3_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level3_r = Activation('relu')(Level3_r)
merge3 = Concatenate(axis=-1)([Level3_l, Level3_r])
Level3_r = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(merge3)
Level3_r = BatchNormalization(axis=-1)(Level3_r)
#Level3_r=InstanceNormalization(axis=-1)(Level3_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level3_r = Activation('relu')(Level3_r)
#Level3_r=Dropout(0.5)(Level3_r)
Level3_r = Conv2D(filters=128,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level3_r)
Level3_r = BatchNormalization(axis=-1)(Level3_r)
#Level3_r=InstanceNormalization(axis=-1)(Level3_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level3_r = Add()([Level3_r, Level3_r_shortcut])
Level3_r = Activation('relu')(Level3_r)
Level2_r = Conv2DTranspose(
filters=64,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level3_r)
Level2_r = BatchNormalization(axis=-1)(Level2_r)
Level2_r_shortcut = Level2_r
#Level2_r=InstanceNormalization(axis=-1)(Level2_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level2_r = Activation('relu')(Level2_r)
merge2 = Concatenate(axis=-1)([Level2_l, Level2_r])
Level2_r = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(merge2)
Level2_r = BatchNormalization(axis=-1)(Level2_r)
#Level2_r=InstanceNormalization(axis=-1)(Level2_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level2_r = Activation('relu')(Level2_r)
#Level2_r=Dropout(0.5)(Level2_r)
Level2_r = Conv2D(filters=64,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level2_r)
Level2_r = BatchNormalization(axis=-1)(Level2_r)
#Level2_r=InstanceNormalization(axis=-1)(Level2_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level2_r = Add()([Level2_r, Level2_r_shortcut])
Level2_r = Activation('relu')(Level2_r)
Level1_r = Conv2DTranspose(
filters=32,
kernel_size=(2, 2),
strides=2,
kernel_regularizer=regularizers.l2(reg))(Level2_r)
Level1_r = BatchNormalization(axis=-1)(Level1_r)
Level1_r_shortcut = Level1_r
#Level1_r=InstanceNormalization(axis=-1)(Level1_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level1_r = Activation('relu')(Level1_r)
merge1 = Concatenate(axis=-1)([Level1_l, Level1_r])
Level1_r = Conv2D(filters=32,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(merge1)
Level1_r = BatchNormalization(axis=-1)(Level1_r)
#Level1_r=InstanceNormalization(axis=-1)(Level1_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level1_r = Activation('relu')(Level1_r)
#Level1_r=Dropout(0.5)(Level1_r)
Level1_r = Conv2D(filters=32,
kernel_size=(3, 3),
strides=1,
padding='same',
kernel_regularizer=regularizers.l2(reg))(Level1_r)
Level1_r = BatchNormalization()(Level1_r)
#Level1_r=InstanceNormalization(axis=-1)(Level1_r) ## Instance Normalization. Use InstanceNormalization() for Layer Normalization.
Level1_r = Add()([Level1_r, Level1_r_shortcut])
Level1_r = Activation('relu')(Level1_r)
output = Conv2D(filters=7,
kernel_size=(1, 1),
strides=1,
kernel_regularizer=regularizers.l2(reg))(Level1_r)
#output=BatchNormalization(axis=-1)(output)
output = Lambda(lambda x: softmax(x, axis=-1))(output)
output = Concatenate(axis=-1)([output, weight_matrix])
model = Model(inputs=inputs, outputs=output)
return model
def ssc_300(image_size,
n_classes,
l2_regularization=0.0005,
min_scale=None,
max_scale=None,
scales=None,
aspect_ratios_global=None,
aspect_ratios_per_layer=[[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5], [1.0, 2.0, 0.5]],
two_boxes_for_ar1=True,
steps=[8, 16, 32, 64, 100, 300],
offsets=None,
subtract_mean=[123, 117, 104],
divide_by_stddev=None,
swap_channels=[2, 1, 0],
predictors=[
'conv4_3', 'fc7', 'conv6_2', 'conv7_2', 'conv8_2', 'conv9_2'
],
hidden_size=[250, 250, 100],
output_activation=False,
lstm=False,
condense_predictors=False):
"""
Build a Keras model with SSC300 architecture, see references.
The base network is a reduced atrous VGG-16, extended by the SSD architecture,
as described in the paper. Most of the arguments that this function takes are only needed for the anchor
box layers. In case you're training the network.
Note: Requires Keras v2.0 or later. Currently works only with the
TensorFlow backend (v1.0 or later).
References: https://arxiv.org/abs/1512.02325v5
:param tuple image_size: The input image size in the format `(height, width, channels)`.
:param int n_classes: The number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO.
:param float l2_regularization: The L2-regularization rate. Applies to all convolutional layers.
Set to zero to deactivate L2-regularization.
:param float min_scale: The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images.
:param float max_scale: The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`.
:param list scales: A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used. If a list is passed,
this argument overrides `min_scale` and `max_scale`. All scaling factors must be greater than zero.
:param list aspect_ratios_global: The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all prediction layers.
:param list aspect_ratios_per_layer: A list containing one aspect ratio list for each prediction layer.
This allows you to set the aspect ratios for each predictor layer individually, which is the case for the
original SSD300 implementation. If a list is passed, it overrides `aspect_ratios_global`.
:param bool two_boxes_for_ar1: Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor.
:param list steps: `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions.
:param list offsets: `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size.
:param list subtract_mean: `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images.
:param list divide_by_stddev: `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
:param list swap_channels: Either `False` or a list of integers representing the desired order in which the input
image channels should be swapped.
:param list predictors: names of the convolutional layers used as predictors
:param list hidden_size: number of neurons for the 3 hidden fully-connected layers
:param bool output_activation: whether to include or not the softplus activation function after the hidden layers
:param bool lstm: whether to add or not an LSTM cell on top of the hidden layer
:param bool condense_predictors: whether to condense or not the predictors in a single prediction
:return model: The Keras SSC300 model.
"""
n_predictor_layers = len(
predictors
) # The number of predictor conv layers in the network is 6 for the original SSD300.
l2_reg = l2_regularization # Make the internal name shorter.
img_height, img_width, img_channels = image_size[0], image_size[
1], image_size[2]
############################################################################
# Get a few exceptions out of the way.
############################################################################
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError(
"`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified."
)
if aspect_ratios_per_layer:
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError(
"It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}."
.format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError(
"Either `min_scale` and `max_scale` or `scales` need to be specified."
)
if scales:
if len(scales) != n_predictor_layers + 1:
raise ValueError(
"It must be either scales is None or len(scales) == {}, but len(scales) == {}."
.format(n_predictor_layers + 1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1)
if len(hidden_size) != 3:
raise ValueError(
"3 hidden size values must be passed, but {} values were received."
.format(len(hidden_size)))
hidden_size = np.array(hidden_size)
if np.any(hidden_size <= 0):
raise ValueError(
"All hidden sizes must be >0, but the sizes given are {}".format(
hidden_size))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError(
"You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError(
"You must provide at least one offset value per predictor layer.")
############################################################################
# Compute the anchor box parameters.
############################################################################
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) +
1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
############################################################################
# Define functions for the Lambda layers below.
############################################################################
def identity_layer(tensor):
return tensor
def input_mean_normalization(tensor):
return tensor - np.array(subtract_mean)
def input_stddev_normalization(tensor):
return tensor / np.array(divide_by_stddev)
def input_channel_swap(tensor):
if len(swap_channels) == 3:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]]
],
axis=-1)
elif len(swap_channels) == 4:
return K.stack([
tensor[..., swap_channels[0]], tensor[..., swap_channels[1]],
tensor[..., swap_channels[2]], tensor[..., swap_channels[3]]
],
axis=-1)
############################################################################
# Build the network.
############################################################################
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that the subsequent lambda layers can be optional.
x1 = Lambda(identity_layer,
output_shape=(img_height, img_width, img_channels),
name='identity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(input_mean_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(input_stddev_normalization,
output_shape=(img_height, img_width, img_channels),
name='input_stddev_normalization')(x1)
if swap_channels:
x1 = Lambda(input_channel_swap,
output_shape=(img_height, img_width, img_channels),
name='input_channel_swap')(x1)
conv1_1 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1_1')(x1)
conv1_2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv1_2')(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool1')(conv1_2)
conv2_1 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2_1')(pool1)
conv2_2 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv2_2')(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool2')(conv2_2)
conv3_1 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_1')(pool2)
conv3_2 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_2')(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv3_3')(conv3_2)
pool3 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool3')(conv3_3)
conv4_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_1')(pool3)
conv4_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_2')(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv4_3')(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool4')(conv4_3)
conv5_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_1')(pool4)
conv5_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_2')(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv5_3')(conv5_2)
pool5 = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
padding='same',
name='pool5')(conv5_3)
fc6 = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc6')(pool5)
fc7 = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='fc7')(fc6)
conv6_1 = Conv2D(256, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_1')(fc7)
conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv6_padding')(conv6_1)
conv6_2 = Conv2D(512, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv6_2')(conv6_1)
conv7_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_1')(conv6_2)
conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv7_padding')(conv7_1)
conv7_2 = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv7_2')(conv7_1)
conv8_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_1')(conv7_2)
conv8_2 = Conv2D(256, (3, 3),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv8_2')(conv8_1)
conv9_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_1')(conv8_2)
conv9_2 = Conv2D(256, (3, 3),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
kernel_regularizer=l2(l2_reg),
name='conv9_2')(conv9_1)
# Feed conv4_3 into the L2 normalization layer
conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3)
conv_features = {
'conv4_3': conv4_3_norm,
'fc7': fc7,
'conv6_2': conv6_2,
'conv7_2': conv7_2,
'conv8_2': conv8_2,
'conv9_2': conv9_2
}
predictor_layers = []
### Build the predictor layers on top of the base network
for predictor in predictors:
flatten = Flatten(name='{}_flat'.format(predictor))(
conv_features[predictor])
d1 = Dense(hidden_size[0], name='{}_d1'.format(predictor))(flatten)
d1bn = BatchNormalization(name='{}_bn1'.format(predictor))(d1)
r1 = Activation(activation='relu',
name='{}_r1'.format(predictor))(d1bn)
d2 = Dense(hidden_size[1], name='{}_d2'.format(predictor))(r1)
d2bn = BatchNormalization(name='{}_bn2'.format(predictor))(d2)
r2 = Activation(activation='relu',
name='{}_r2'.format(predictor))(d2bn)
d3 = Dense(hidden_size[2], name='{}_d3'.format(predictor))(r2)
d3bn = BatchNormalization(name='{}_bn3'.format(predictor))(d3)
r3 = Activation(activation='relu',
name='{}_r3'.format(predictor))(d3bn)
pred = Dense(n_classes, name='{}_pred'.format(predictor))(r3)
predictor_layers.append(pred)
# Concatenate the output of the different predictors
# Output shape of `predictions`: (batch, n_predictors, n_classes)
predictions = Concatenate(axis=1, name='predictions1')(predictor_layers)
if output_activation:
predictions = Activation(activation='softplus')(predictions)
if lstm:
predictions = Reshape((n_predictor_layers, n_classes),
name='lstm_predictions_res')(predictions)
predictions = Bidirectional(LSTM(20, return_sequences=False),
name='lstm_predictions')(predictions)
if condense_predictors:
predictions = Dense(n_classes,
name='predictions_condensed')(predictions)
return Model(inputs=x, outputs=predictions)
def bidaff(num_words, embeddings_matrix, ce_loader, scope, embedding_dim=64):
# (batch, input_len) => (batch, input_len, embedding_dim)
q_input = Input(shape=(QUESTION_LEN,), name="q_input")
a_input = Input(shape=(ANSWER_LEN,), name="a_input")
c_input = Input(shape=(CONTEXT_LEN,), name="c_input")
q_char_input = Input(shape=(QUESTION_LEN * MAX_WORD_LEN,),
name="q_char_input")
a_char_input = Input(shape=(ANSWER_LEN * MAX_WORD_LEN,),
name="a_char_input")
c_char_input = Input(shape=(CONTEXT_LEN * MAX_WORD_LEN,),
name="c_char_input")
# Word embedders.
q_emb = Embedding(input_dim=num_words + 1, # word 0 used for padding
output_dim=embedding_dim,
weights=[embeddings_matrix],
input_length=QUESTION_LEN,
name="embedding_q2_" + scope,
mask_zero=False,
trainable=False)
a_emb = Embedding(input_dim=num_words + 1, # word 0 used for padding
output_dim=embedding_dim,
weights=[embeddings_matrix],
input_length=ANSWER_LEN,
name="embedding_a2_" + scope,
mask_zero=False,
trainable=False)
c_emb = Embedding(input_dim=num_words + 1, # word 0 used for padding
output_dim=embedding_dim,
weights=[embeddings_matrix],
input_length=CONTEXT_LEN,
name="embedding_c2_" + scope,
mask_zero=False,
trainable=False)
# Char embedders.
q_char_emb = Embedding(input_dim=ce_loader.get_num_words() + 1,
output_dim=ce_loader.get_embedding_len(),
weights=[ce_loader.get_embeddings_matrix()],
input_length=MAX_WORD_LEN * QUESTION_LEN,
name="char_embedding_q2_" + scope,
mask_zero=False,
trainable=False)
a_char_emb = Embedding(input_dim=ce_loader.get_num_words() + 1,
output_dim=ce_loader.get_embedding_len(),
weights=[ce_loader.get_embeddings_matrix()],
input_length=MAX_WORD_LEN * ANSWER_LEN,
name="char_embedding_a2_" + scope,
mask_zero=False,
trainable=False)
c_char_emb = Embedding(input_dim=ce_loader.get_num_words() + 1,
output_dim=ce_loader.get_embedding_len(),
weights=[ce_loader.get_embeddings_matrix()],
input_length=MAX_WORD_LEN * CONTEXT_LEN,
name="char_embedding_c2_" + scope,
mask_zero=False,
trainable=False)
q = q_emb(q_input)
a = a_emb(a_input)
c = c_emb(c_input)
q_char = q_char_emb(q_char_input)
a_char = a_char_emb(a_char_input)
c_char = c_char_emb(c_char_input)
assert(CHAR_EMBEDDINGS_DIM == ce_loader.get_embedding_len())
q_char = Reshape((QUESTION_LEN, MAX_WORD_LEN, CHAR_EMBEDDINGS_DIM))(q_char)
a_char = Reshape((ANSWER_LEN, MAX_WORD_LEN, CHAR_EMBEDDINGS_DIM))(a_char)
c_char = Reshape((CONTEXT_LEN, MAX_WORD_LEN, CHAR_EMBEDDINGS_DIM))(c_char)
# CharCNNs for char level embeddings.
q_char = CharCNN(q_char, name="q_charcnn")
a_char = CharCNN(a_char, name="a_charcnn")
c_char = CharCNN(c_char, name="c_charcnn")
# Concatenate GloVe word embeddings with char-level embeddings.
q = Concatenate(axis=-1)([q, q_char])
a = Concatenate(axis=-1)([a, a_char])
c = Concatenate(axis=-1)([c, c_char])
q = Dropout(0.2)(q)
c = Dropout(0.2)(c)
a = Dropout(0.2)(a)
# Pass them through a 2 layer highway network.
for highway_index in range(1, 2):
q = TimeDistributedHighway(q, 92,
"highway_q_bidaff_{}".format(highway_index))
a = TimeDistributedHighway(a, 92,
"highway_a_bidaff_{}".format(highway_index))
c = TimeDistributedHighway(c, 92,
"highway_c_bidaff_{}".format(highway_index))
# Contextual Embed Layer
q_lstm = Bidirectional(LSTM(30, recurrent_dropout=0.15,
return_sequences=True))(q)
c_lstm = Bidirectional(LSTM(30, recurrent_dropout=0.15,
return_sequences=True))(c)
sim = Similarity()([c_lstm, q_lstm])
# ************* Context-to-query attention. ***********************
# Softmax on each line.
col_softmax = Lambda(lambda x: K.softmax(x, axis=-1))(sim)
# Product between sofmax prob and each query vector.
UT = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=(2, 1)),
output_shape=lambda x: x[0][:2] + x[1][2:])(
[col_softmax, q_lstm])
# ************* Query-to-context attention. ***********************
# Max per line then softmax.
line_softmax = Lambda(lambda x: K.max(x, axis=-1),
output_shape=lambda x: (x[0], x[1]))(sim)
line_softmax = Lambda(lambda x: K.softmax(x, axis=-1))(line_softmax)
# Make @line_softmax a matrix with 1 row.
line_softmax = Lambda(lambda x: K.expand_dims(x, axis=1),
output_shape=lambda x: (x[0], 1, x[1]))(line_softmax)
# Matrix multiplication.
HT = Lambda(lambda x: K.batch_dot(x[0], x[1], axes=(2, 1)),
output_shape=lambda x: x[0][:2] + x[1][2:])(
[line_softmax, c_lstm])
# Remove one extra row.
HT = Lambda(lambda x: K.squeeze(x, axis=1),
output_shape=lambda x: (x[0], x[2]))(HT)
HT = RepeatVector(CONTEXT_LEN)(HT)
# ************ Combine attention vectors. ***********************
G = Concatenate(axis=-1)([
c_lstm,
UT,
Multiply()([c_lstm, UT]),
Multiply()([c_lstm, HT])
])
a_lstm = Bidirectional(LSTM(20, recurrent_dropout=0.15))(a)
a_lstm = RepeatVector(CONTEXT_LEN)(a_lstm)
cqa = Concatenate(axis=-1)([G, a_lstm])
cqa = Dropout(0.2)(cqa)
cqa = Bidirectional(LSTM(30, recurrent_dropout=0.15))(cqa)
cqa = Dropout(0.25)(cqa)
cqa = Dense(100, activation='relu')(cqa)
cqa = Dropout(0.25)(cqa)
output = Dense(2, activation='softmax')(cqa)
model = Model(inputs=[
q_input, a_input, c_input,
q_char_input, a_char_input, c_char_input,
], outputs=[output])
model.compile(loss=categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
plot_model(model, to_file='2way_model.png', show_shapes=True)
return model
def myCrossLayer(nb_flow=2,
map_height=16,
map_width=8,
nb_layers=3,
window_len=12,
nb_filter=64,
external_dim=None,
filter_size=3):
"""
the final model
:param nb_flow: number of measurements, also number of channels of each picture sample
:param map_height: grid map height, here is 16
:param map_width: grid map width, here is 8
:param nb_layers: number of cnn layers
:return:
"""
window_len_pic_fea = []
main_inputs = []
if external_dim == None:
for i in range(window_len):
inputs = Input(shape=(nb_flow, map_height, map_width))
main_inputs.append(inputs)
cnn_fea = dense_conv3D(nb_filter=nb_filter,
nb_col=filter_size,
nb_row=filter_size,
padding='same',
nb_layers=nb_layers,
dense_units=1024,
dropout_rate=0.5)(inputs)
# cnn_fea_flatten = Reshape(([nb_layers * 1024]))(cnn_fea)
cnn_fea_flatten = Reshape(([1024]))(cnn_fea)
# cnn_fea_flatten = Dropout(rate=0.3)(cnn_fea_flatten)
# cnn_fea_flatten = expand_dims(cnn_fea_flatten, axis=1)
cnn_fea_flatten = Lambda(expand_dim_backend)(cnn_fea_flatten)
window_len_pic_fea.append(cnn_fea_flatten)
# add external feature here
if external_dim != None and external_dim > 0:
for i in range(window_len):
# todo : use two tensor to represent the data and meta_data respectively
inputs = Input(shape=((nb_flow, map_height, map_width),
external_dim))
main_inputs.append(inputs)
inputs_0 = inputs
inputs_1 = inputs
cnn_fea = dense_conv3D(nb_filter=nb_filter,
nb_col=filter_size,
nb_row=filter_size,
padding='same',
nb_layers=nb_layers,
dense_units=1024,
dropout_rate=0.5)(inputs_0)
# cnn_fea_flatten = Reshape(([nb_layers * 1024]))(cnn_fea)
cnn_fea_flatten = Reshape(([1024]))(cnn_fea)
# cnn_fea_flatten = Dropout(rate=0.3)(cnn_fea_flatten)
# cnn_fea_flatten = expand_dims(cnn_fea_flatten, axis=1)
cnn_fea_flatten = Lambda(expand_dim_backend)(cnn_fea_flatten)
window_len_pic_fea.append(cnn_fea_flatten)
external_input = inputs_1
# external_input = Input(shape=(external_dim,))
main_inputs.append(external_input)
# todo: change the code here
embedding = Dense(nb_layers * 1024, activation='relu')(external_input)
external_out = Lambda(expand_dim_backend)(embedding)
new_concatenate_fea = []
for pic_fea in window_len_pic_fea:
tmp_con = Concatenate(axis=-1)([pic_fea, external_out])
new_concatenate_fea.append(tmp_con)
window_len_pic_fea = new_concatenate_fea
outputs = add_densenet(nb_flow=nb_flow,
map_height=map_height,
map_width=map_width)(window_len_pic_fea)
# outputs = add_lstm(nb_flow=nb_flow, map_height=map_height, map_width=map_width)(window_len_pic_fea)
# outputs = attention_after_LSTM(nb_flow=nb_flow, map_height=map_height,
# map_width=map_width, window_len=window_len)(window_len_pic_fea)
model = Model(inputs=main_inputs, outputs=outputs)
return model
def build(self):
'''
1. Build Code Representation Model
'''
logger.debug('Building Code Representation Model')
methname = Input(shape=(self.data_params['methname_len'], ),
dtype='int32',
name='methname')
apiseq = Input(shape=(self.data_params['apiseq_len'], ),
dtype='int32',
name='apiseq')
tokens = Input(shape=(self.data_params['tokens_len'], ),
dtype='int32',
name='tokens')
## method name representation ##
#1.embedding
init_emb_weights = np.load(
self.config['workdir'] +
self.model_params['init_embed_weights_methname']
) if self.model_params[
'init_embed_weights_methname'] is not None else None
init_emb_weights = init_emb_weights if init_emb_weights is None else [
init_emb_weights
]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=init_emb_weights,
mask_zero=
False, #Whether 0 in the input is a special "padding" value that should be masked out.
#If set True, all subsequent layers in the model must support masking, otherwise an exception will be raised.
name='embedding_methname')
methname_embedding = embedding(methname)
dropout = Dropout(0.25, name='dropout_methname_embed')
methname_dropout = dropout(methname_embedding)
methname_conv1 = Conv1D(100,
2,
padding='valid',
activation='relu',
strides=1,
name='methname_conv1')
methname_conv2 = Conv1D(100,
3,
padding='valid',
activation='relu',
strides=1,
name='methname_conv2')
methname_conv3 = Conv1D(100,
4,
padding='valid',
activation='relu',
strides=1,
name='methname_conv3')
methname_conv1_out = methname_conv1(methname_dropout)
methname_conv2_out = methname_conv2(methname_dropout)
methname_conv3_out = methname_conv3(methname_dropout)
dropout = Dropout(0.25, name='dropout_methname_conv')
methname_conv1_dropout = dropout(methname_conv1_out)
methname_conv2_dropout = dropout(methname_conv2_out)
methname_conv3_dropout = dropout(methname_conv3_out)
merged_methname = Concatenate(name='methname_merge', axis=1)([
methname_conv1_dropout, methname_conv2_dropout,
methname_conv3_dropout
])
## API Sequence Representation ##
#1.embedding
init_emb_weights = np.load(
self.config['workdir'] +
self.model_params['init_embed_weights_api']
) if self.model_params['init_embed_weights_api'] is not None else None
init_emb_weights = init_emb_weights if init_emb_weights is None else [
init_emb_weights
]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
#weights=weights,
mask_zero=
False, #Whether 0 in the input is a special "padding" value that should be masked out.
#If set True, all subsequent layers must support masking, otherwise an exception will be raised.
name='embedding_apiseq')
apiseq_embedding = embedding(apiseq)
dropout = Dropout(0.25, name='dropout_apiseq_embed')
apiseq_dropout = dropout(apiseq_embedding)
api_conv1 = Conv1D(100,
2,
padding='valid',
activation='relu',
strides=1,
name='api_conv1')
api_conv2 = Conv1D(100,
3,
padding='valid',
activation='relu',
strides=1,
name='api_conv2')
api_conv3 = Conv1D(100,
4,
padding='valid',
activation='relu',
strides=1,
name='api_conv3')
api_conv1_out = api_conv1(apiseq_dropout)
api_conv2_out = api_conv2(apiseq_dropout)
api_conv3_out = api_conv3(apiseq_dropout)
dropout = Dropout(0.25, name='dropout_api_conv')
api_conv1_dropout = dropout(api_conv1_out)
api_conv2_dropout = dropout(api_conv2_out)
api_conv3_dropout = dropout(api_conv3_out)
merged_api = Concatenate(name='api_merge', axis=1)(
[api_conv1_dropout, api_conv2_dropout, api_conv3_dropout])
## Tokens Representation ##
#1.embedding
init_emb_weights = np.load(
self.config['workdir'] +
self.model_params['init_embed_weights_tokens']
) if self.model_params[
'init_embed_weights_tokens'] is not None else None
init_emb_weights = init_emb_weights if init_emb_weights is None else [
init_emb_weights
]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=init_emb_weights,
#mask_zero=True,#Whether 0 in the input is a special "padding" value that should be masked out.
#If set True, all subsequent layers must support masking, otherwise an exception will be raised.
name='embedding_tokens')
tokens_embedding = embedding(tokens)
dropout = Dropout(0.25, name='dropout_tokens_embed')
tokens_dropout = dropout(tokens_embedding)
tokens_conv1 = Conv1D(100,
2,
padding='valid',
activation='relu',
strides=1,
name='tokens_conv1')
tokens_conv2 = Conv1D(100,
3,
padding='valid',
activation='relu',
strides=1,
name='tokens_conv2')
tokens_conv3 = Conv1D(100,
4,
padding='valid',
activation='relu',
strides=1,
name='tokens_conv3')
tokens_conv1_out = tokens_conv1(tokens_dropout)
tokens_conv2_out = tokens_conv2(tokens_dropout)
tokens_conv3_out = tokens_conv3(tokens_dropout)
dropout = Dropout(0.25, name='dropout_tokens_conv')
tokens_conv1_dropout = dropout(tokens_conv1_out)
tokens_conv2_dropout = dropout(tokens_conv2_out)
tokens_conv3_dropout = dropout(tokens_conv3_out)
merged_tokens = Concatenate(name='tokens_merge', axis=1)(
[tokens_conv1_dropout, tokens_conv2_dropout, tokens_conv3_dropout])
# merge code#
merged_code = Concatenate(name='code_merge', axis=1)(
[merged_methname, merged_api, merged_tokens]) #(122,200)
'''
2. Build Desc Representation Model
'''
## Desc Representation ##
logger.debug('Building Desc Representation Model')
desc = Input(shape=(self.data_params['desc_len'], ),
dtype='int32',
name='desc')
#1.embedding
init_emb_weights = np.load(
self.config['workdir'] +
self.model_params['init_embed_weights_desc']
) if self.model_params['init_embed_weights_desc'] is not None else None
init_emb_weights = init_emb_weights if init_emb_weights is None else [
init_emb_weights
]
embedding = Embedding(
input_dim=self.data_params['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=init_emb_weights,
mask_zero=
False, #Whether 0 in the input is a special "padding" value that should be masked out.
#If set True, all subsequent layers must support masking, otherwise an exception will be raised.
name='embedding_desc')
desc_embedding = embedding(desc)
dropout = Dropout(0.25, name='dropout_desc_embed')
desc_dropout = dropout(desc_embedding)
#2. rnn
desc_conv1 = Conv1D(100,
2,
padding='valid',
activation='relu',
strides=1,
name='desc_conv1')
desc_conv2 = Conv1D(100,
3,
padding='valid',
activation='relu',
strides=1,
name='desc_conv2')
desc_conv3 = Conv1D(100,
4,
padding='valid',
activation='relu',
strides=1,
name='desc_conv3')
desc_conv1_out = desc_conv1(desc_dropout)
desc_conv2_out = desc_conv2(desc_dropout)
desc_conv3_out = desc_conv3(desc_dropout)
dropout = Dropout(0.25, name='dropout_desc_conv')
desc_conv1_dropout = dropout(desc_conv1_out)
desc_conv2_dropout = dropout(desc_conv2_out)
desc_conv3_dropout = dropout(desc_conv3_out)
merged_desc = Concatenate(name='desc_merge', axis=1)(
[desc_conv1_dropout, desc_conv2_dropout, desc_conv3_dropout])
#AP networks#
attention = AttentionLayer(name='attention_layer') # (122,60)
attention_out = attention([merged_code, merged_desc])
gmp_1 = GlobalMaxPooling1D(name='blobalmaxpool_colum')
att_1 = gmp_1(attention_out)
activ1 = Activation('softmax', name='AP_active_colum')
att_1_next = activ1(att_1)
dot1 = Dot(axes=1, normalize=False, name='column_dot')
desc_out = dot1([att_1_next, merged_desc])
attention_trans_layer = Lambda(
lambda x: K.permute_dimensions(x, (0, 2, 1)),
name='trans_attention')
attention_transposed = attention_trans_layer(attention_out)
gmp_2 = GlobalMaxPooling1D(name='blobalmaxpool_row')
att_2 = gmp_2(attention_transposed)
activ2 = Activation('softmax', name='AP_active_row')
att_2_next = activ2(att_2)
dot2 = Dot(axes=1, normalize=False, name='row_dot')
code_out = dot2([att_2_next, merged_code])
self._code_repr_model = Model(inputs=[methname, apiseq, tokens, desc],
outputs=[code_out],
name='desc_repr_model')
print('\nsummary of code representation model')
self._code_repr_model.summary()
fname = self.config['workdir'] + 'models/' + self.model_params[
'model_name'] + '/_desc_repr_model.png'
self._desc_repr_model = Model(inputs=[methname, apiseq, tokens, desc],
outputs=[desc_out],
name='code_repr_model')
print('\nsummary of description representation model')
self._desc_repr_model.summary()
"""
3: calculate the cosine similarity between code and desc
"""
logger.debug('Building similarity model')
code_repr = self._code_repr_model([methname, apiseq, tokens, desc])
desc_repr = self._desc_repr_model([methname, apiseq, tokens, desc])
cos_sim = Dot(axes=1, normalize=True,
name='cos_sim')([code_repr, desc_repr])
sim_model = Model(inputs=[methname, apiseq, tokens, desc],
outputs=[cos_sim],
name='sim_model')
self._sim_model = sim_model #for model evaluation
print("\nsummary of similarity model")
self._sim_model.summary()
fname = self.config['workdir'] + 'models/' + self.model_params[
'model_name'] + '/_sim_model.png'
'''
4:Build training model
'''
good_sim = sim_model(
[self.methname, self.apiseq, self.tokens,
self.desc_good]) # similarity of good output
bad_sim = sim_model(
[self.methname, self.apiseq, self.tokens,
self.desc_bad]) #similarity of bad output
loss = Lambda(lambda x: K.maximum(
1e-6, self.model_params['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0],
name='loss')([good_sim, bad_sim])
logger.debug('Building training model')
self._training_model = Model(inputs=[
self.methname, self.apiseq, self.tokens, self.desc_good,
self.desc_bad
],
outputs=[loss],
name='training_model')
print('\nsummary of training model')
self._training_model.summary()
fname = self.config['workdir'] + 'models/' + self.model_params[
'model_name'] + '/_training_model.png'
right_input = Input(input_shape, name='right_input')
# inception_resnet的bottle_neck输出
# with tf.variable_scope("Inception", reuse=None):
# left_x = base_model(left_input)
# with tf.variable_scope("Inception", reuse=True):
# right_x = base_model(right_input)
with tf.variable_scope("Inception") as scope:
left_x = base_model(left_input)
scope.reuse_variables()
right_x = base_model(right_input)
# Concatenate组合
branches = [left_x, right_x]
x = Concatenate(axis=-1, name='Siamese_Concatenate')(branches)
# x = Conv2D(1536, 3, use_bias= False, name='Conv2d_8a_3x3')(x)
# x = BatchNormalization(axis=3, scale=False, name='Conv2d_8a_BatchNorm')(x)
x = Dense(512, activation='relu', name='Dense_512')(x)
x = Dropout(1.0 - DROPOUT_KEEP_PROB, name='Dropout_Final')(x)
# sigmoid预测分类
left_output = Dense(1, activation='sigmoid', name='left_output')(x)
right_output = Dense(1, activation='sigmoid', name='right_output')(x)
# 实例化模型
model = Model(inputs=[left_input, right_input],
outputs=[left_output, right_output])
# model.summary()
# for i, layer in enumerate(base_model.layers):
def new_lpcnet_model(frame_size=160,
rnn_units1=384,
rnn_units2=16,
nb_used_features=38,
training=False,
use_gpu=True):
pcm = Input(shape=(None, 3))
feat = Input(shape=(None, nb_used_features))
pitch = Input(shape=(None, 1))
dec_feat = Input(shape=(None, 128))
dec_state1 = Input(shape=(rnn_units1, ))
dec_state2 = Input(shape=(rnn_units2, ))
padding = 'valid' if training else 'same'
fconv1 = Conv1D(128,
3,
padding=padding,
activation='tanh',
name='feature_conv1')
fconv2 = Conv1D(128,
3,
padding=padding,
activation='tanh',
name='feature_conv2')
embed = Embedding(256,
embed_size,
embeddings_initializer=PCMInit(),
name='embed_sig')
cpcm = Reshape((-1, embed_size * 3))(embed(pcm))
pembed = Embedding(256, 64, name='embed_pitch')
cat_feat = Concatenate()([feat, Reshape((-1, 64))(pembed(pitch))])
cfeat = fconv2(fconv1(cat_feat))
fdense1 = Dense(128, activation='tanh', name='feature_dense1')
fdense2 = Dense(128, activation='tanh', name='feature_dense2')
cfeat = fdense2(fdense1(cfeat))
rep = Lambda(lambda x: K.repeat_elements(x, frame_size, 1))
if use_gpu:
rnn = CuDNNGRU(rnn_units1,
return_sequences=True,
return_state=True,
name='gru_a')
rnn2 = CuDNNGRU(rnn_units2,
return_sequences=True,
return_state=True,
name='gru_b')
else:
rnn = GRU(rnn_units1,
return_sequences=True,
return_state=True,
recurrent_activation="sigmoid",
reset_after='true',
name='gru_a')
rnn2 = GRU(rnn_units2,
return_sequences=True,
return_state=True,
recurrent_activation="sigmoid",
reset_after='true',
name='gru_b')
rnn_in = Concatenate()([cpcm, rep(cfeat)])
md = MDense(pcm_levels, activation='softmax', name='dual_fc')
gru_out1, _ = rnn(rnn_in)
gru_out2, _ = rnn2(Concatenate()([gru_out1, rep(cfeat)]))
ulaw_prob = md(gru_out2)
model = Model([pcm, feat, pitch], ulaw_prob)
model.rnn_units1 = rnn_units1
model.rnn_units2 = rnn_units2
model.nb_used_features = nb_used_features
model.frame_size = frame_size
encoder = Model([feat, pitch], cfeat)
dec_rnn_in = Concatenate()([cpcm, dec_feat])
dec_gru_out1, state1 = rnn(dec_rnn_in, initial_state=dec_state1)
dec_gru_out2, state2 = rnn2(Concatenate()([dec_gru_out1, dec_feat]),
initial_state=dec_state2)
dec_ulaw_prob = md(dec_gru_out2)
decoder = Model([pcm, dec_feat, dec_state1, dec_state2],
[dec_ulaw_prob, state1, state2])
return model, encoder, decoder
m_train = X_train.shape[0]
m_test = X_test.shape[0]
#parameters of embedding layer
input_dim_embedding = 14322
out_dim_embedding = 100
max_input_length = 200
#parameters of LSTM layer
out_dim_LSTM = 100
#parameter of output layer
output_dim = 1
#timestep
Tx = 200
# defined pre_LSTM
repeator = RepeatVector(Tx, name='rep')
concatenator = Concatenate(axis=-1, name='con')
densor1 = Dense(10, activation="tanh", name='den1')
densor2 = Dense(1, activation="relu", name='den2')
activator = Activation(
'softmax', name='attention_weights'
) # We are using a custom softmax(axis = 1) loaded in this notebook
dotor = Dot(axes=1, name='dot')
permute_2_1 = Permute((2, 1))
#define post_LSTM
post_activation_LSTM_cell = LSTM(out_dim_LSTM, return_state=True)
reshape_post_lstm = Reshape((1, out_dim_LSTM))
concatenator_post_lstm = Concatenate(axis=1, name='con_post_lstm')
densor3 = Dense(1)
densor4 = Dense(output_dim)
output_layer = Activation('sigmoid')
#define cnn_layer:
def Deeplabv3(weights='pascal_voc',
input_tensor=None,
input_shape=(512, 512, 3),
classes=21,
backbone='mobilenetv2',
OS=16,
alpha=1.,
activation=None):
""" Instantiates the Deeplabv3+ architecture
Optionally loads weights pre-trained
on PASCAL VOC. This model is available for TensorFlow only,
and can only be used with inputs following the TensorFlow
data format `(width, height, channels)`.
# Arguments
weights: one of 'pascal_voc' (pre-trained on pascal voc)
or None (random initialization)
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: shape of input image. format HxWxC
PASCAL VOC model was trained on (512,512,3) images
classes: number of desired classes. If classes != 21,
last layer is initialized randomly
backbone: backbone to use. one of {'xception','mobilenetv2'}
activation: optional activation to add to the top of the network.
One of 'softmax', 'sigmoid' or None
OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.
Used only for xception backbone.
alpha: controls the width of the MobileNetV2 network. This is known as the
width multiplier in the MobileNetV2 paper.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
Used only for mobilenetv2 backbone
# Returns
A Keras model instance.
# Raises
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
ValueError: in case of invalid argument for `weights` or `backbone`
"""
if not (weights in {'pascal_voc', 'cityscapes', None}):
raise ValueError(
'The `weights` argument should be either '
'`None` (random initialization), `pascal_voc`, or `cityscapes` '
'(pre-trained on PASCAL VOC)')
if K.backend() != 'tensorflow':
raise RuntimeError('The Deeplabv3+ model is only available with '
'the TensorFlow backend.')
if not (backbone in {'xception', 'mobilenetv2'}):
raise ValueError('The `backbone` argument should be either '
'`xception` or `mobilenetv2` ')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if backbone == 'xception':
if OS == 8:
entry_block3_stride = 1
middle_block_rate = 2 # ! Not mentioned in paper, but required
exit_block_rates = (2, 4)
atrous_rates = (12, 24, 36)
else:
entry_block3_stride = 2
middle_block_rate = 1
exit_block_rates = (1, 2)
atrous_rates = (6, 12, 18)
x = Conv2D(32, (3, 3),
strides=(2, 2),
name='entry_flow_conv1_1',
use_bias=False,
padding='same')(img_input)
x = BatchNormalization(name='entry_flow_conv1_1_BN')(x)
x = Activation('relu')(x)
x = _conv2d_same(x, 64, 'entry_flow_conv1_2', kernel_size=3, stride=1)
x = BatchNormalization(name='entry_flow_conv1_2_BN')(x)
x = Activation('relu')(x)
x = _xception_block(x, [128, 128, 128],
'entry_flow_block1',
skip_connection_type='conv',
stride=2,
depth_activation=False)
x, skip1 = _xception_block(x, [256, 256, 256],
'entry_flow_block2',
skip_connection_type='conv',
stride=2,
depth_activation=False,
return_skip=True)
x = _xception_block(x, [728, 728, 728],
'entry_flow_block3',
skip_connection_type='conv',
stride=entry_block3_stride,
depth_activation=False)
for i in range(16):
x = _xception_block(x, [728, 728, 728],
'middle_flow_unit_{}'.format(i + 1),
skip_connection_type='sum',
stride=1,
rate=middle_block_rate,
depth_activation=False)
x = _xception_block(x, [728, 1024, 1024],
'exit_flow_block1',
skip_connection_type='conv',
stride=1,
rate=exit_block_rates[0],
depth_activation=False)
x = _xception_block(x, [1536, 1536, 2048],
'exit_flow_block2',
skip_connection_type='none',
stride=1,
rate=exit_block_rates[1],
depth_activation=True)
else:
OS = 8
first_block_filters = _make_divisible(32 * alpha, 8)
x = Conv2D(first_block_filters,
kernel_size=3,
strides=(2, 2),
padding='same',
use_bias=False,
name='Conv')(img_input)
x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x)
x = Activation(relu6, name='Conv_Relu6')(x)
#x = Dropout(0.1)(x)
x = _inverted_res_block(x,
filters=16,
alpha=alpha,
stride=1,
expansion=1,
block_id=0,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=2,
expansion=6,
block_id=1,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=1,
expansion=6,
block_id=2,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=2,
expansion=6,
block_id=3,
skip_connection=False)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=4,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=5,
skip_connection=True)
# stride in block 6 changed from 2 -> 1, so we need to use rate = 2
x = _inverted_res_block(
x,
filters=64,
alpha=alpha,
stride=1,
rate=2, # 1!
expansion=6,
block_id=6,
skip_connection=False)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=7,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=8,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=9,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=10,
skip_connection=False)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=11,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=12,
skip_connection=True)
x = _inverted_res_block(
x,
filters=160,
alpha=alpha,
stride=1,
rate=2, # 1!
expansion=6,
block_id=13,
skip_connection=False)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=14,
skip_connection=True)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=15,
skip_connection=True)
x = _inverted_res_block(x,
filters=320,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=16,
skip_connection=False)
# end of feature extractor
# branching for Atrous Spatial Pyramid Pooling
# Image Feature branch
#out_shape = int(np.ceil(input_shape[0] / OS))
b4 = AveragePooling2D(pool_size=(int(np.ceil(input_shape[0] / OS)),
int(np.ceil(input_shape[1] / OS))))(x)
b4 = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='image_pooling')(b4)
b4 = BatchNormalization(name='image_pooling_BN', epsilon=1e-5)(b4)
b4 = Activation('relu')(b4)
b4 = BilinearUpsampling((int(np.ceil(input_shape[0] / OS)),
int(np.ceil(input_shape[1] / OS))))(b4)
# simple 1x1
b0 = Conv2D(256, (1, 1), padding='same', use_bias=False, name='aspp0')(x)
b0 = BatchNormalization(name='aspp0_BN', epsilon=1e-5)(b0)
b0 = Activation('relu', name='aspp0_activation')(b0)
# there are only 2 branches in mobilenetV2. not sure why
#atrous_rates = (6, 12, 18)
#atrous_rates = (12, 24, 36)
if backbone == 'xception':
# rate = 6 (12)
b1 = SepConv_BN(x,
256,
'aspp1',
rate=atrous_rates[0],
depth_activation=True,
epsilon=1e-5)
# rate = 12 (24)
b2 = SepConv_BN(x,
256,
'aspp2',
rate=atrous_rates[1],
depth_activation=True,
epsilon=1e-5)
# rate = 18 (36)
b3 = SepConv_BN(x,
256,
'aspp3',
rate=atrous_rates[2],
depth_activation=True,
epsilon=1e-5)
# concatenate ASPP branches & project
x = Concatenate()([b4, b0, b1, b2, b3])
else:
x = Concatenate()([b4, b0])
x = Conv2D(256, (1, 1),
padding='same',
use_bias=False,
name='concat_projection')(x)
x = BatchNormalization(name='concat_projection_BN', epsilon=1e-5)(x)
x = Activation('relu')(x)
x = Dropout(0.1)(x)
# DeepLab v.3+ decoder
if backbone == 'xception':
# Feature projection
# x4 (x2) block
x = BilinearUpsampling(output_size=(int(np.ceil(input_shape[0] / 4)),
int(np.ceil(input_shape[1] /
4))))(x)
dec_skip1 = Conv2D(48, (1, 1),
padding='same',
use_bias=False,
name='feature_projection0')(skip1)
dec_skip1 = BatchNormalization(name='feature_projection0_BN',
epsilon=1e-5)(dec_skip1)
dec_skip1 = Activation('relu')(dec_skip1)
x = Concatenate()([x, dec_skip1])
x = SepConv_BN(x,
256,
'decoder_conv0',
depth_activation=True,
epsilon=1e-5)
x = SepConv_BN(x,
256,
'decoder_conv1',
depth_activation=True,
epsilon=1e-5)
# you can use it with arbitary number of classes
#if classes == 21:
# last_layer_name = 'logits_semantic'
#else:
# last_layer_name = 'custom_logits_semantic'
#last_layer_name = 'logits_semantic'
last_layer_name = 'custom_logits_semantic'
x = Conv2D(classes, (1, 1), padding='same', name=last_layer_name)(x)
x = BilinearUpsampling(output_size=(input_shape[0], input_shape[1]))(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
if activation in {'softmax', 'sigmoid'}:
x = Activation(activation)(x)
model = Model(inputs, x, name='deeplabv3plus')
# load weights
if weights == 'pascal_voc':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_X,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH_MOBILE,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
elif weights == 'cityscapes':
if backbone == 'xception':
weights_path = get_file(
'deeplabv3_xception_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_X_CS,
cache_subdir='models')
else:
weights_path = get_file(
'deeplabv3_mobilenetv2_tf_dim_ordering_tf_kernels_cityscapes.h5',
WEIGHTS_PATH_MOBILE_CS,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
return model
def create_model_mtl_mtv_exchange_rate(horizon=1,
nb_train_samples=512,
batch_size=32,
feature_count=6,
lag_time=6,
aux_feature_count=8):
x = Input(shape=(lag_time, feature_count), name="input_layer")
conv = Conv1D(filters=5, kernel_size=1, activation='relu')(x)
# conv = Conv1D(filters=5, kernel_size=3, padding='causal', strides=1, activation='relu', dilation_rate=2)(x)
conv = Conv1D(filters=5,
kernel_size=3,
padding='causal',
strides=1,
activation='relu',
dilation_rate=4)(conv)
mp = MaxPooling1D(pool_size=1)(conv)
# conv2 = Conv1D(filters=5, kernel_size=3, activation='relu')(mp)
# mp = MaxPooling1D(pool_size=2)(conv2)
lstm1 = GRU(8, return_sequences=True)(mp)
lstm2 = GRU(16, return_sequences=True)(lstm1)
# lstm2 = GRU(16, return_sequences=True)(mp)
shared_dense = Dense(32, name="shared_layer")(lstm2)
## sub1 is main task; units = reshape dimension multiplication
sub1 = GRU(units=(lag_time * aux_feature_count),
name="task1")(shared_dense)
sub2 = GRU(units=16, name="task2")(shared_dense)
sub3 = GRU(units=16, name="task3")(shared_dense)
sub4 = GRU(units=16, name="task4")(shared_dense)
sub5 = GRU(units=16, name="task5")(shared_dense)
sub1 = Reshape((lag_time, aux_feature_count))(sub1)
auxiliary_input = Input(shape=(lag_time, aux_feature_count),
name='aux_input')
concate = Concatenate(axis=-1)([sub1, auxiliary_input])
# out1_gp = Dense(1, name="out1_gp")(sub1)
out1 = Dense(8, name="spec_out1")(concate)
out1 = Flatten()(out1)
out1 = Dense(1, name="out1")(out1)
out2 = Dense(8, name="spec_out2")(sub2)
out2 = Dense(1, name="out2")(out2)
out3 = Dense(1, name="spec_out3")(sub3)
out3 = Dense(1, name="out3")(out3)
out4 = Dense(1, name="spec_out4")(sub4)
out4 = Dense(1, name="out4")(out4)
out5 = Dense(1, name="spec_out5")(sub5)
out5 = Dense(1, name="out5")(out5)
outputs = [out1, out2, out3, out4, out5]
# outputs = [out1, out2, out3, out4]
model = KerasModel(inputs=[x, auxiliary_input], outputs=outputs)
model.compile(optimizer='sgd',
loss='mse',
metrics=['mae', 'mape', 'mse'],
loss_weights=[1, 0.00, 0.00, 0.00, 0.00])
# Callbacks
# callbacks = [EarlyStopping(monitor='val_mse', patience=10)]
model.summary()
return model
observation_layer = Input(shape=(None, d_x + d_gamma + d_y),
name="observation") # (x_o,y_o) tuples
target_X_layer = Input(shape=(None, d_x + d_gamma), name="target") # x_q
ObsMLP = MLP(d_x + d_gamma + d_y,
obs_mlp_layers,
name='obs_mlp',
parallel_inputs=True) # Network E
obs_representations = ObsMLP(observation_layer) # r_i
general_representation = GlobalAveragePooling1D()(obs_representations) # r
general_representation = Lambda(lambda x: tf.keras.backend.repeat(x[0], tf.shape(x[1])[1]), name='Repeat')\
([general_representation, target_X_layer]) # r in batch form (same)
merged_layer = Concatenate(axis=2, name='merged')(
[general_representation, target_X_layer]) # (r,x_q) tuple
Decoder = MLP(d_x + d_gamma + obs_mlp_layers[-1],
decoder_layers,
name='decoder_mlp',
parallel_inputs=False) # Network Q
output = Decoder(merged_layer) # (mean_q, std_q)
model = Model([observation_layer, target_X_layer], output)
model.compile(optimizer=Adam(lr=1e-4), loss=custom_loss)
model.summary()
plot_model(model, to_file=f'{output_path}model.png')
def generator():
while True:
if train & (not retrain) & server:
spatial_model.load_weights(
'weights/xception_spatial_{}e.h5'.format(spa_epochs))
temporal_model.load_weights('weights/xception_temporal{}_{}e.h5'.format(
opt_size, tem_epochs))
# Fusion
if fusion == 'avg':
x = Average()(
[spatial_model.layers[-4].output, temporal_model.layers[-4].output])
elif fusion == 'max':
x = Maximum()(
[spatial_model.layers[-4].output, temporal_model.layers[-4].output])
elif fusion == 'concat':
x = Concatenate()(
[spatial_model.layers[-4].output, temporal_model.layers[-4].output])
else:
x = Multiply()(
[spatial_model.layers[-4].output, temporal_model.layers[-4].output])
x = GlobalAveragePooling2D()(x)
x = Dropout(drop_rate)(x)
x = Dense(classes, activation='softmax')(x)
# Final touch
result_model = Model(inputs=[spatial_model.input, temporal_model.input],
outputs=x)
result_model.summary()
# Run
# x6 = Lambda(lambda x : x[:,6,:])(inputs)
# x7 = Lambda(lambda x : x[:,7,:])(inputs)
# x8 = Lambda(lambda x : x[:,8,:])(inputs)
# x9 = Lambda(lambda x : x[:,9,:])(inputs)
embeds0 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=not args.embed_not_trainable)(x0)
embeds1 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=not args.embed_not_trainable)(x1)
embeds2 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=not args.embed_not_trainable)(x2)
embeds3 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=not args.embed_not_trainable)(x3)
# embeds4 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=True)(x4)
# embeds5 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=True)(x5)
# embeds6 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=True)(x6)
# embeds7 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=True)(x7)
# embeds8 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=True)(x8)
# embeds9 = Embedding(len(vocab), len(vocab), embeddings_initializer='identity', trainable=True)(x9)
embed_conc = Concatenate()([embeds0,embeds2])
x1 = Dense(args.dense_size,activation='relu')(embed_conc)
x2 = Dense(args.dense_size,activation='relu')(x1)
conc = Concatenate()([x2,embeds1,embeds3])
# conc = Concatenate()([embeds0,embeds2,embeds1,embeds3])
print("k",x0.shape)
# conc = Concatenate()([embeds0,embeds1,embeds2,embeds3])#,embeds4,embeds5])#,embeds6,embeds7,embeds8,embeds9])
if not args.two_LSTM:
lstm1 = conc
else:
lstm1 = LSTM_use(hidden_size, return_sequences=True)(conc)
if args.attention:
lstm1b = Lambda(attentions_layer)(lstm1)
else:
lstm1b = lstm1
lstm4 = LSTM_use(hidden_size, return_sequences=True)(lstm1b)
kernel_size=(filter_sizes[2], embedding_dim),
padding='valid',
kernel_initializer='normal',
activation='relu')(reshape)
maxpool_0 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] + 1, 1),
strides=(1, 1),
padding='valid')(conv_0)
maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[1] + 1, 1),
strides=(1, 1),
padding='valid')(conv_1)
maxpool_2 = MaxPool2D(pool_size=(sequence_length - filter_sizes[2] + 1, 1),
strides=(1, 1),
padding='valid')(conv_2)
concatenated_tensor = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2])
flatten = Flatten()(concatenated_tensor)
dropout = Dropout(drop)(flatten)
output = Dense(units=output_length, activation='softmax')(dropout)
# this creates a model that includes
model = Model(inputs=inputs, outputs=output)
checkpoint = ModelCheckpoint('weights.{epoch:03d}-{val_acc:.4f}.hdf5',
monitor='val_acc',
verbose=1,
save_best_only=True,
mode='auto')
adam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=adam,
# Up-down
rnn1 = Bidirectional(LSTM(M, return_sequences=True))
x1 = rnn1(input_)
x1 = GlobalMaxPooling1D()(x1)
# Left-right
rnn2 = Bidirectional(LSTM(M, return_sequences=True))
# Custom layer
permutor = Lambda(lambda t: K.permute_dimensions(t, pattern=(0, 2, 1)))
x2 = permutor(input_)
x2 = rnn2(x2) # output is N X D X 2M
x2 = GlobalMaxPooling1D()(x2) # output is N X 2M
# Put them together
concatenator = Concatenate(axis=1)
x = concatenator([x1, x2]) # output is N X 4M
# Final dense layer
output = Dense(10, activation='softmax')(x)
model = Model(inputs=input_, outputs=output)
# Compile
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Train
print('Training model...')
r = model.fit(X, Y, batch_size=32, epochs=10, validation_split=0.3)
conv2 = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(conv2) pool2 = MaxPool3D(pool_size=(2, 2, 2))(conv2) conv3 = Conv3D(128, (3, 3, 3), activation='relu', padding='same')(pool2) conv3 = Conv3D(128, (3, 3, 3), activation='relu', padding='same')(conv3) pool3 = MaxPool3D(pool_size=(2, 2, 2))(conv3) conv4 = Conv3D(256, (3, 3, 3), activation='relu', padding='same')(pool3) conv4 = Conv3D(256, (3, 3, 3), activation='relu', padding='same')(conv4) pool4 = MaxPool3D(pool_size=(2, 2, 2))(conv4) conv5 = Conv3D(512, (3, 3, 3), activation='relu', padding='same')(pool4) conv5 = Conv3D(512, (3, 3, 3), activation='relu', padding='same')(conv5) falt = Flatten()(conv5) fc1 = Dense(1024, activation='relu')(falt) fc2 = Dense(1024, activation='relu')(fc1) fc3 = Dense(2, activation='softmax')(fc2) up6 = Conv3DTranspose(256, (2, 2, 2), strides=(2, 2, 2), padding='same')(conv5) up6 = Concatenate(axis=-1)([up6, conv4]) conv6 = Conv3D(256, (3, 3, 3), activation='relu', padding='same')(up6) conv6 = Conv3D(256, (3, 3, 3), activation='relu', padding='same')(conv6) up7 = Conv3DTranspose(128, (2, 2, 2), strides=(2, 2, 2), padding='same')(conv6) up7 = Concatenate(axis=-1)([up7, conv3]) conv7 = Conv3D(128, (3, 3, 3), activation='relu', padding='same')(up7) conv7 = Conv3D(128, (3, 3, 3), activation='relu', padding='same')(conv7) up8 = Conv3DTranspose(64, (2, 2, 2), strides=(2, 2, 2), padding='same')(conv7) up8 = Concatenate(axis=-1)([up8, conv2]) conv8 = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(up8) conv8 = Conv3D(64, (3, 3, 3), activation='relu', padding='same')(conv8) up9 = Conv3DTranspose(32, (2, 2, 2), strides=(2, 2, 2), padding='same')(conv8) conv9 = Conv3D(32, (3, 3, 3), activation='relu', padding='same')(up9) conv9 = Conv3D(32, (3, 3, 3), activation='relu', padding='same')(conv9) conv10 = Conv3D(1, (1, 1, 1), activation='sigmoid')(conv9) model = Model(inputs=[I1], outputs=[conv10, fc3])
def AFNN(filters=8, image_channels=1, use_bnorm=True):
layer_count = 0
inpt = Input(shape=(128, 128, image_channels),
name='input'+str(layer_count))
# 1st layer, Conv+relu
layer_count += 1
x_0 = Conv2D(filters=filters, kernel_size=(3, 3), strides=(2, 2),
kernel_initializer='Orthogonal', padding='same',
name='conv'+str(layer_count))(inpt)
layer_count += 1
x_0 = Activation('relu', name='relu'+str(layer_count))(x_0)
# Path 1
layer_count += 1
x1 = MaxPooling2D(pool_size=(2, 2), strides=None, padding='same', data_format=None,
name='maxpool_p1'+str(layer_count))(x_0)
# 2 layers, Conv+BN+relu+MaxPooling
for i in range(2):
layer_count += 1
x1 = Conv2D(filters=filters, kernel_size=(3, 3), strides=(2, 2),
kernel_initializer='Orthogonal', padding='same', use_bias=False,
name='conv_p1'+str(layer_count))(x1)
if use_bnorm:
layer_count += 1
x1 = BatchNormalization(
axis=3, momentum=0.0, epsilon=0.0001, name='bn_p1'+str(layer_count))(x1)
layer_count += 1
x1 = Activation('relu', name='relu_p1'+str(layer_count))(x1)
x1 = MaxPooling2D(pool_size=(2, 2), strides=None, padding='same',
data_format=None, name='maxpool_p1'+str(layer_count))(x1)
i += 1
# Path 2
# 2 layers, Conv+BN+ReLU+MaxPooling
layer_count += 1
x_1 = Conv2D(filters=filters, kernel_size=(7, 7), strides=(4, 4),
kernel_initializer='Orthogonal', padding='same',
name='conv_p2'+str(layer_count))(x_0)
layer_count += 1
x_1 = Activation('relu', name='relu_p2'+str(layer_count))(x_1)
# Path 2_1
# 1 layer, Conv+BN+ReLU+MaxPooling
layer_count += 1
x2 = MaxPooling2D(pool_size=(2, 2), strides=None, padding='same',
data_format=None, name='maxpool_p2'+str(layer_count))(x_1)
layer_count += 1
x2 = Conv2D(filters=filters, kernel_size=(5, 5), strides=(2, 2),
kernel_initializer='Orthogonal', padding='same', use_bias=False,
name='conv_p2'+str(layer_count))(x2)
if use_bnorm:
layer_count += 1
x2 = BatchNormalization(
axis=3, momentum=0.0, epsilon=0.0001, name='bn_p2'+str(layer_count))(x2)
layer_count += 1
x2 = Activation('relu', name='relu_p2'+str(layer_count))(x2)
x2 = MaxPooling2D(pool_size=(2, 2), strides=None, padding='same', data_format=None,
name='maxpool_p2'+str(layer_count))(x2)
# Path 2_2
# 1 layer, Conv+BN+ReLU+MaxPooling
layer_count += 1
x_2 = Conv2D(filters=filters, kernel_size=(7, 7), strides=(4, 4),
kernel_initializer='Orthogonal', padding='same',
name='conv_p3'+str(layer_count))(x_1)
if use_bnorm:
layer_count += 1
x3 = BatchNormalization(
axis=3, momentum=0.0, epsilon=0.0001, name='bn_p3'+str(layer_count))(x_2)
layer_count += 1
x3 = Activation('relu', name='relu_p3'+str(layer_count))(x3)
layer_count += 1
x3 = MaxPooling2D(pool_size=(2, 2), strides=None, padding='same', data_format=None,
name='maxpool_p3'+str(layer_count))(x3)
# Merge layer
layer_count += 1
x = Concatenate(axis=-1, name='concat'+str(layer_count))([x1, x2, x3])
layer_count += 1
x = Flatten(data_format=None, name='Flat'+str(layer_count))(x)
# Dense output layer
layer_count += 1
x = Dense(128, activation='relu', name='dense'+str(layer_count))(x)
layer_count += 1
x = Dropout(0.5, name='dropout'+str(layer_count))(x)
layer_count += 1
y = Dense(1, activation='relu', name='dense'+str(layer_count))(x)
model = Model(inputs=inpt, outputs=y)
return model
name=side_layer1_name)(reshape1)
shisalayer = Dense(
1,
use_bias=False,
trainable=False,
activation='linear',
kernel_initializer=keras.initializers.constant(value=sum_weight),
name=shisalayer_name)(main_layer)
x_list.append(main_layer1)
y_list.append(side_layer1)
z_list.append(shisalayer)
print(x_list)
print(y_list)
print(z_list)
main_layerall = Concatenate(axis=-1, name='main_layerall')(x_list)
side_layerall = Concatenate(axis=-1, name='side_layerall')(y_list)
wave_layerall = Concatenate(axis=-1, name='wave_layerall')(z_list)
print(main_layerall)
print(side_layerall)
print(wave_layerall)
main_reshape = Reshape(input_shape=(q * q * 3, -1),
target_shape=(q, q, 3),
name='main_reshape')(main_layerall)
print(main_reshape)
side_layerallsum = Dense(1,
use_bias=False,
trainable=False,
activation='linear',
kernel_initializer=keras.initializers.ones(),
inputs = Input(shape=(sequence_length,), dtype='int32')
embedding = Embedding(input_dim=vocabulary_size, output_dim=embedding_dim, input_length=sequence_length)(inputs)
reshape = Reshape((sequence_length,embedding_dim,1))(embedding)
conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embedding_dim), padding='valid', kernel_initializer='normal', activation='relu')(reshape)
conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embedding_dim), padding='valid', kernel_initializer='normal', activation='relu')(reshape)
conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embedding_dim), padding='valid', kernel_initializer='normal', activation='relu')(reshape)
conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embedding_dim), padding='valid', kernel_initializer='normal', activation='relu')(reshape)
conv_4 = Conv2D(num_filters, kernel_size=(filter_sizes[4], embedding_dim), padding='valid', kernel_initializer='normal', activation='relu')(reshape)
conv_5 = Conv2D(num_filters, kernel_size=(filter_sizes[5], embedding_dim), padding='valid', kernel_initializer='normal', activation='relu')(reshape)
maxpool_0 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] + 1, 1), strides=(1,1), padding='valid')(conv_0)
maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[1] + 1, 1), strides=(1,1), padding='valid')(conv_1)
maxpool_2 = MaxPool2D(pool_size=(sequence_length - filter_sizes[2] + 1, 1), strides=(1,1), padding='valid')(conv_2)
maxpool_3 = MaxPool2D(pool_size=(sequence_length - filter_sizes[3] + 1, 1), strides=(1,1), padding='valid')(conv_3)
maxpool_4 = MaxPool2D(pool_size=(sequence_length - filter_sizes[4] + 1, 1), strides=(1,1), padding='valid')(conv_4)
maxpool_5 = MaxPool2D(pool_size=(sequence_length - filter_sizes[5] + 1, 1), strides=(1,1), padding='valid')(conv_5)
concatenated_tensor = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3, maxpool_4,maxpool_5])
flatten = Flatten()(concatenated_tensor)
dropout = Dropout(drop)(flatten)
output = Dense(2, activation='softmax')(dropout)
model = Model(inputs=inputs, outputs=output)
checkpoint = ModelCheckpoint('weights.{epoch:03d}-{val_acc:.4f}.hdf5', monitor='val_acc', verbose=1, save_best_only=True, mode='auto')
adam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])
print("Traning Model...")
model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs, verbose=1, callbacks=[checkpoint], validation_data=(X_test, Y_test)) # starts training
main_input_2 = Input(shape=(50, 1), name='main_input_2')
lstm_out = LSTM(32,
activation='tanh',
recurrent_activation='sigmoid',
return_sequences=True)
max_pooling = MaxPooling1D(pool_size=2, strides=50, padding='valid')
dense = Dense(10000, activation='relu')
dense2 = Dense(10000, activation='relu')
adam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
lstm_out_1 = lstm_out(main_input_1)
lstm_out_2 = lstm_out(main_input_2)
max_pooling_1 = max_pooling(lstm_out_1)
max_pooling_2 = max_pooling(lstm_out_2)
concatenate_layer = Concatenate()([max_pooling_1, max_pooling_2])
dense_output1 = dense2(concatenate_layer)
dense_output2 = dense(dense_output1)
logistic_regression_output = Dense(1, activation='sigmoid',
name='main_output')(dense_output2)
model = Model(inputs=[main_input_1, main_input_2],
outputs=[logistic_regression_output])
model.compile(optimizer='RMSprop',
loss={'main_output': 'mean_squared_error'},
metrics=['mse', 'mae', 'mape', 'cosine'])
tensorboard = TensorBoard(log_dir="logs/".format(time()))
def separable_inception_resnet_block(x,
scale,
block_type,
block_idx,
activation='relu'):
"""Adds a Inception-ResNet block.
This function builds 3 types of Inception-ResNet blocks mentioned
in the paper, controlled by the `block_type` argument (which is the
block name used in the official TF-slim implementation):
- Inception-ResNet-A: `block_type='block35'`
- Inception-ResNet-B: `block_type='block17'`
- Inception-ResNet-C: `block_type='block8'`
# Arguments
x: input tensor.
scale: scaling factor to scale the residuals (i.e., the output of
passing `x` through an inception module) before adding them
to the shortcut branch. Let `r` be the output from the residual branch,
the output of this block will be `x + scale * r`.
block_type: `'block35'`, `'block17'` or `'block8'`, determines
the network structure in the residual branch.
block_idx: an `int` used for generating layer names. The Inception-ResNet blocks
are repeated many times in this network. We use `block_idx` to identify
each of the repetitions. For example, the first Inception-ResNet-A block
will have `block_type='block35', block_idx=0`, ane the layer names will have
a common prefix `'block35_0'`.
activation: activation function to use at the end of the block
(see [activations](../activations.md)).
When `activation=None`, no activation is applied
(i.e., "linear" activation: `a(x) = x`).
# Returns
Output tensor for the block.
# Raises
ValueError: if `block_type` is not one of `'block35'`,
`'block17'` or `'block8'`.
"""
if block_type == 'block35':
branch_0 = conv2d_bn(x, 32, 1)
branch_1 = conv2d_bn(x, 32, 1)
branch_1 = SeparableConv2D(filters=32, kernel_size=3,
padding='same')(branch_1)
branch_2 = conv2d_bn(x, 32, 1)
branch_2 = SeparableConv2D(filters=48, kernel_size=3,
padding='same')(branch_2)
branch_2 = SeparableConv2D(filters=64, kernel_size=3,
padding='same')(branch_2)
branches = [branch_0, branch_1, branch_2]
elif block_type == 'block17':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 128, 1)
branch_1 = SeparableConv2D(filters=160,
kernel_size=[1, 7],
padding='same')(branch_1)
branch_1 = SeparableConv2D(filters=192,
kernel_size=[7, 1],
padding='same')(branch_1)
branches = [branch_0, branch_1]
elif block_type == 'block8':
branch_0 = conv2d_bn(x, 192, 1)
branch_1 = conv2d_bn(x, 192, 1)
branch_1 = SeparableConv2D(filters=224,
kernel_size=[1, 3],
padding='same')(branch_1)
branch_1 = SeparableConv2D(filters=256,
kernel_size=[3, 1],
padding='same')(branch_1)
branches = [branch_0, branch_1]
else:
raise ValueError('Unknown Inception-ResNet block type. '
'Expects "block35", "block17" or "block8", '
'but got: ' + str(block_type))
block_name = block_type + '_' + str(block_idx)
channel_axis = 1 if K.image_data_format() == 'channels_first' else 3
mixed = Concatenate(axis=channel_axis,
name=block_name + '_mixed')(branches)
up = conv2d_bn(mixed,
K.int_shape(x)[channel_axis],
1,
activation=None,
use_bias=True,
name=block_name + '_conv')
x = Lambda(lambda inputs, scale: inputs[0] + inputs[1] * scale,
output_shape=K.int_shape(x)[1:],
arguments={'scale': scale},
name=block_name)([x, up])
if activation is not None:
x = Activation(activation, name=block_name + '_ac')(x)
return x
x = Conv1D(256, 5, strides=1)(x) x = MaxPooling1D(pool_size=5, strides=2)(x) # local view input_local = Input(shape=(201, 1)) y = Conv1D(16, 5, strides=1)(input_local) y = Conv1D(16, 5, strides=1)(y) y = MaxPooling1D(pool_size=7, strides=2)(y) y = Conv1D(32, 5, strides=1)(y) y = Conv1D(32, 5, strides=1)(y) y = MaxPooling1D(pool_size=7, strides=2)(y) # merge layers for fully connected xf = Flatten()(x) yf = Flatten()(y) z = Concatenate()([xf, yf]) z = Dense(512, activation='relu')(z) z = Dense(512, activation='relu')(z) z = Dense(512, activation='relu')(z) output = Dense(1, activation='sigmoid', name='main_output')(z) model = Model(inputs=[input_global, input_local], outputs=output) ''' model.summary() __________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== input_1 (InputLayer) (None, 2001, 1) 0 __________________________________________________________________________________________________ conv1d_1 (Conv1D) (None, 1997, 16) 96 input_1[0][0]
X_test = X[train_dataset_last_idx:]
y_test = y[train_dataset_last_idx:]
y_test_orig = y_orig[train_dataset_last_idx:]
def rmse(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))
# Model
inputs = Input((21, 9))
conv = Conv1D(32, 3)(inputs)
conv = Conv1D(64, 3)(conv)
forward = GRU(50, activation='elu')(conv)
backward = GRU(50, go_backwards=True, activation='elu')(conv)
#forward = Lambda(lambda x: x[:, 11, :])(forward)
#backward = Lambda(lambda x: x[:, 11, :])(backward)
net = Concatenate()([forward, backward])
net = Dense(10, activation='relu')(net)
net = Dense(1, activation='linear')(net)
model = Model(inputs=inputs, outputs=net)
model.compile(optimizer='RMSProp', loss='mae')
# Training
history = History()
model.fit(X_train, y_train,
batch_size=BATCH_SIZE,
validation_split=0.2,
#validation_data=[X_val, y_val],
shuffle=True,
def ssd_300(image_size,
n_classes,
min_scale=None,
max_scale=None,
scales=None,
aspect_ratios_global=None,
aspect_ratios_per_layer=[[0.5, 1.0, 2.0],
[1.0 / 3.0, 0.5, 1.0, 2.0, 3.0],
[1.0 / 3.0, 0.5, 1.0, 2.0, 3.0],
[1.0 / 3.0, 0.5, 1.0, 2.0, 3.0],
[0.5, 1.0, 2.0], [0.5, 1.0, 2.0]],
two_boxes_for_ar1=True,
steps=None,
offsets=None,
limit_boxes=False,
variances=[0.1, 0.1, 0.2, 0.2],
coords='centroids',
normalize_coords=False,
subtract_mean=None,
divide_by_stddev=None,
swap_channels=False,
return_predictor_sizes=False):
'''
Build a Keras model with SSD_300 architecture, see references.
The base network is a reduced atrous VGG-16, extended by the SSD architecture,
as described in the paper.
In case you're wondering why this function has so many arguments: All arguments except
the first two (`image_size` and `n_classes`) are only needed so that the anchor box
layers can produce the correct anchor boxes. In case you're training the network, the
parameters passed here must be the same as the ones used to set up `SSDBoxEncoder`.
In case you're loading trained weights, the parameters passed here must be the same
as the ones used to produce the trained weights.
Some of these arguments are explained in more detail in the documentation of the
`SSDBoxEncoder` class.
Note: Requires Keras v2.0 or later. Currently works only with the
TensorFlow backend (v1.0 or later).
Arguments:
image_size (tuple): The input image size in the format `(height, width, channels)`.
n_classes (int): The number of categories for classification including
the background class (i.e. the number of positive classes +1 for
the background calss).
min_scale (float, optional): The smallest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. Defaults to 0.1.
max_scale (float, optional): The largest scaling factor for the size of the anchor boxes as a fraction
of the shorter side of the input images. All scaling factors between the smallest and the
largest will be linearly interpolated. Note that the second to last of the linearly interpolated
scaling factors will actually be the scaling factor for the last predictor layer, while the last
scaling factor is used for the second box for aspect ratio 1 in the last predictor layer
if `two_boxes_for_ar1` is `True`. Defaults to 0.9.
scales (list, optional): A list of floats containing scaling factors per convolutional predictor layer.
This list must be one element longer than the number of predictor layers. The first `k` elements are the
scaling factors for the `k` predictor layers, while the last element is used for the second box
for aspect ratio 1 in the last predictor layer if `two_boxes_for_ar1` is `True`. This additional
last scaling factor must be passed either way, even if it is not being used.
Defaults to `None`. If a list is passed, this argument overrides `min_scale` and
`max_scale`. All scaling factors must be greater than zero.
aspect_ratios_global (list, optional): The list of aspect ratios for which anchor boxes are to be
generated. This list is valid for all prediction layers. Defaults to None.
aspect_ratios_per_layer (list, optional): A list containing one aspect ratio list for each prediction layer.
This allows you to set the aspect ratios for each predictor layer individually, which is the case for the
original SSD300 implementation. If a list is passed, it overrides `aspect_ratios_global`.
Defaults to the aspect ratios used in the original SSD300 architecture, i.e.:
[[0.5, 1.0, 2.0],
[1.0/3.0, 0.5, 1.0, 2.0, 3.0],
[1.0/3.0, 0.5, 1.0, 2.0, 3.0],
[1.0/3.0, 0.5, 1.0, 2.0, 3.0],
[0.5, 1.0, 2.0],
[0.5, 1.0, 2.0]]
two_boxes_for_ar1 (bool, optional): Only relevant for aspect ratio lists that contain 1. Will be ignored otherwise.
If `True`, two anchor boxes will be generated for aspect ratio 1. The first will be generated
using the scaling factor for the respective layer, the second one will be generated using
geometric mean of said scaling factor and next bigger scaling factor. Defaults to `True`, following the original
implementation.
steps (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either ints/floats or tuples of two ints/floats. These numbers represent for each predictor layer how many
pixels apart the anchor box center points should be vertically and horizontally along the spatial grid over
the image. If the list contains ints/floats, then that value will be used for both spatial dimensions.
If the list contains tuples of two ints/floats, then they represent `(step_height, step_width)`.
If no steps are provided, then they will be computed such that the anchor box center points will form an
equidistant grid within the image dimensions. Defaults to `None`.
offsets (list, optional): `None` or a list with as many elements as there are predictor layers. The elements can be
either floats or tuples of two floats. These numbers represent for each predictor layer how many
pixels from the top and left boarders of the image the top-most and left-most anchor box center points should be
as a fraction of `steps`. The last bit is important: The offsets are not absolute pixel values, but fractions
of the step size specified in the `steps` argument. If the list contains floats, then that value will
be used for both spatial dimensions. If the list contains tuples of two floats, then they represent
`(vertical_offset, horizontal_offset)`. If no offsets are provided, then they will default to 0.5 of the step size.
Defaults to `None`.
limit_boxes (bool, optional): If `True`, limits box coordinates to stay within image boundaries.
This would normally be set to `True`, but here it defaults to `False`, following the original
implementation.
variances (list, optional): A list of 4 floats >0 with scaling factors (actually it's not factors but divisors
to be precise) for the encoded predicted box coordinates. A variance value of 1.0 would apply
no scaling at all to the predictions, while values in (0,1) upscale the encoded predictions and values greater
than 1.0 downscale the encoded predictions. Defaults to `[0.1, 0.1, 0.2, 0.2]`, following the original implementation.
The coordinate format must be 'centroids'.
coords (str, optional): The box coordinate format to be used. Can be either 'centroids' for the format
`(cx, cy, w, h)` (box center coordinates, width, and height) or 'minmax' for the format
`(xmin, xmax, ymin, ymax)`. Defaults to 'centroids', following the original implementation.
normalize_coords (bool, optional): Set to `True` if the model is supposed to use relative instead of absolute coordinates,
i.e. if the model predicts box coordinates within [0,1] instead of absolute coordinates. Defaults to `False`.
subtract_mean (array-like, optional): `None` or an array-like object of integers or floating point values
of any shape that is broadcast-compatible with the image shape. The elements of this array will be
subtracted from the image pixel intensity values. For example, pass a list of three integers
to perform per-channel mean normalization for color images. Defaults to `None`.
divide_by_stddev (array-like, optional): `None` or an array-like object of non-zero integers or
floating point values of any shape that is broadcast-compatible with the image shape. The image pixel
intensity values will be divided by the elements of this array. For example, pass a list
of three integers to perform per-channel standard deviation normalization for color images.
Defaults to `None`.
swap_channels (bool, optional): If `True` the color channel order of the input images will be reversed,
i.e. if the input color channel order is RGB, the color channels will be swapped to BGR. Note that the
original Caffe implementation assumes BGR input. Defaults to `True`.
return_predictor_sizes (bool, optional): If `True`, this function not only returns the model, but also
a list containing the spatial dimensions of the predictor layers. This isn't strictly necessary since
you can always get their sizes easily via the Keras API, but it's convenient and less error-prone
to get them this way. THey are only relevant for training anyway (SSDBoxEncoder needs to know the
spatial dimensions of the predictor layers), for inference you don't need them.
Returns:
model: The Keras SSD model.
predictor_sizes: A Numpy array containing the `(height, width)` portion
of the output tensor shape for each convolutional predictor layer. During
training, the generator function needs this in order to transform
the ground truth labels into tensors of identical structure as the
output tensors of the model, which is in turn needed for the cost
function.
References:
https://arxiv.org/abs/1512.02325v5
'''
n_predictor_layers = 6 # The number of predictor conv layers in the network is 6 for the original SSD300
# Get a few exceptions out of the way first
if aspect_ratios_global is None and aspect_ratios_per_layer is None:
raise ValueError(
"`aspect_ratios_global` and `aspect_ratios_per_layer` cannot both be None. At least one needs to be specified."
)
if aspect_ratios_per_layer:
if len(aspect_ratios_per_layer) != n_predictor_layers:
raise ValueError(
"It must be either aspect_ratios_per_layer is None or len(aspect_ratios_per_layer) == {}, but len(aspect_ratios_per_layer) == {}."
.format(n_predictor_layers, len(aspect_ratios_per_layer)))
if (min_scale is None or max_scale is None) and scales is None:
raise ValueError(
"Either `min_scale` and `max_scale` or `scales` need to be specified."
)
if scales:
if len(scales) != n_predictor_layers + 1:
raise ValueError(
"It must be either scales is None or len(scales) == {}, but len(scales) == {}."
.format(n_predictor_layers + 1, len(scales)))
else: # If no explicit list of scaling factors was passed, compute the list of scaling factors from `min_scale` and `max_scale`
scales = np.linspace(min_scale, max_scale, n_predictor_layers + 1)
if len(variances) != 4:
raise ValueError(
"4 variance values must be pased, but {} values were received.".
format(len(variances)))
variances = np.array(variances)
if np.any(variances <= 0):
raise ValueError(
"All variances must be >0, but the variances given are {}".format(
variances))
if (not (steps is None)) and (len(steps) != n_predictor_layers):
raise ValueError(
"You must provide at least one step value per predictor layer.")
if (not (offsets is None)) and (len(offsets) != n_predictor_layers):
raise ValueError(
"You must provide at least one offset value per predictor layer.")
# Set the aspect ratios for each predictor layer. These are only needed for the anchor box layers.
if aspect_ratios_per_layer:
aspect_ratios = aspect_ratios_per_layer
else:
aspect_ratios = [aspect_ratios_global] * n_predictor_layers
# Compute the number of boxes to be predicted per cell for each predictor layer.
# We need this so that we know how many channels the predictor layers need to have.
if aspect_ratios_per_layer:
n_boxes = []
for ar in aspect_ratios_per_layer:
if (1 in ar) & two_boxes_for_ar1:
n_boxes.append(len(ar) +
1) # +1 for the second box for aspect ratio 1
else:
n_boxes.append(len(ar))
else: # If only a global aspect ratio list was passed, then the number of boxes is the same for each predictor layer
if (1 in aspect_ratios_global) & two_boxes_for_ar1:
n_boxes = len(aspect_ratios_global) + 1
else:
n_boxes = len(aspect_ratios_global)
n_boxes = [n_boxes] * n_predictor_layers
if steps is None:
steps = [None] * n_predictor_layers
if offsets is None:
offsets = [None] * n_predictor_layers
# Input image format
img_height, img_width, img_channels = image_size[0], image_size[
1], image_size[2]
### Build the actual network.
x = Input(shape=(img_height, img_width, img_channels))
# The following identity layer is only needed so that subsequent two lambda layers can be optional.
x1 = Lambda(lambda z: z,
output_shape=(img_height, img_width, img_channels),
name='idendity_layer')(x)
if not (subtract_mean is None):
x1 = Lambda(lambda z: z - np.array(subtract_mean),
output_shape=(img_height, img_width, img_channels),
name='input_mean_normalization')(x1)
if not (divide_by_stddev is None):
x1 = Lambda(lambda z: z / np.array(divide_by_stddev),
output_shape=(img_height, img_width, img_channels),
name='input_stddev_normalization')(x1)
if swap_channels and (img_channels == 3):
x1 = Lambda(lambda z: z[..., ::-1],
output_shape=(img_height, img_width, img_channels),
name='input_channel_swap')(x1)
conv1_1 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv1_1')(x1)
conv1_2 = Conv2D(64, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv1_2')(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool1')(conv1_2)
conv2_1 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv2_1')(pool1)
conv2_2 = Conv2D(128, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv2_2')(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool2')(conv2_2)
conv3_1 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv3_1')(pool2)
conv3_2 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv3_2')(conv3_1)
conv3_3 = Conv2D(256, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv3_3')(conv3_2)
pool3 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool3')(conv3_3)
conv4_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv4_1')(pool3)
conv4_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv4_2')(conv4_1)
conv4_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv4_3')(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='same',
name='pool4')(conv4_3)
conv5_1 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv5_1')(pool4)
conv5_2 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv5_2')(conv5_1)
conv5_3 = Conv2D(512, (3, 3),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv5_3')(conv5_2)
pool5 = MaxPooling2D(pool_size=(3, 3),
strides=(1, 1),
padding='same',
name='pool5')(conv5_3)
fc6 = Conv2D(1024, (3, 3),
dilation_rate=(6, 6),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='fc6')(pool5)
fc7 = Conv2D(1024, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='fc7')(fc6)
conv6_1 = Conv2D(256, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv6_1')(fc7)
conv6_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv6_padding')(conv6_1)
conv6_2 = Conv2D(512, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
name='conv6_2')(conv6_1)
conv7_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv7_1')(conv6_2)
conv7_1 = ZeroPadding2D(padding=((1, 1), (1, 1)),
name='conv7_padding')(conv7_1)
conv7_2 = Conv2D(256, (3, 3),
strides=(2, 2),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
name='conv7_2')(conv7_1)
conv8_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv8_1')(conv7_2)
conv8_2 = Conv2D(256, (3, 3),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
name='conv8_2')(conv8_1)
conv9_1 = Conv2D(128, (1, 1),
activation='relu',
padding='same',
kernel_initializer='he_normal',
name='conv9_1')(conv8_2)
conv9_2 = Conv2D(256, (3, 3),
strides=(1, 1),
activation='relu',
padding='valid',
kernel_initializer='he_normal',
name='conv9_2')(conv9_1)
# Feed conv4_3 into the L2 normalization layer
conv4_3_norm = L2Normalization(gamma_init=20, name='conv4_3_norm')(conv4_3)
### Build the convolutional predictor layers on top of the base network
# We precidt `n_classes` confidence values for each box, hence the confidence predictors have depth `n_boxes * n_classes`
# Output shape of the confidence layers: `(batch, height, width, n_boxes * n_classes)`
conv4_3_norm_mbox_conf = Conv2D(
n_boxes[0] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv4_3_norm_mbox_conf')(conv4_3_norm)
fc7_mbox_conf = Conv2D(n_boxes[1] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='fc7_mbox_conf')(fc7)
conv6_2_mbox_conf = Conv2D(n_boxes[2] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv6_2_mbox_conf')(conv6_2)
conv7_2_mbox_conf = Conv2D(n_boxes[3] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv7_2_mbox_conf')(conv7_2)
conv8_2_mbox_conf = Conv2D(n_boxes[4] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv8_2_mbox_conf')(conv8_2)
conv9_2_mbox_conf = Conv2D(n_boxes[5] * n_classes, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv9_2_mbox_conf')(conv9_2)
# We predict 4 box coordinates for each box, hence the localization predictors have depth `n_boxes * 4`
# Output shape of the localization layers: `(batch, height, width, n_boxes * 4)`
conv4_3_norm_mbox_loc = Conv2D(n_boxes[0] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv4_3_norm_mbox_loc')(conv4_3_norm)
fc7_mbox_loc = Conv2D(n_boxes[1] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='fc7_mbox_loc')(fc7)
conv6_2_mbox_loc = Conv2D(n_boxes[2] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv6_2_mbox_loc')(conv6_2)
conv7_2_mbox_loc = Conv2D(n_boxes[3] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv7_2_mbox_loc')(conv7_2)
conv8_2_mbox_loc = Conv2D(n_boxes[4] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv8_2_mbox_loc')(conv8_2)
conv9_2_mbox_loc = Conv2D(n_boxes[5] * 4, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv9_2_mbox_loc')(conv9_2)
### Generate the anchor boxes (called "priors" in the original Caffe/C++ implementation, so I'll keep their layer names)
# Output shape of anchors: `(batch, height, width, n_boxes, 8)`
conv4_3_norm_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[0],
next_scale=scales[1],
aspect_ratios=aspect_ratios[0],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[0],
this_offsets=offsets[0],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv4_3_norm_mbox_priorbox')(conv4_3_norm_mbox_loc)
fc7_mbox_priorbox = AnchorBoxes(img_height,
img_width,
this_scale=scales[1],
next_scale=scales[2],
aspect_ratios=aspect_ratios[1],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[1],
this_offsets=offsets[1],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='fc7_mbox_priorbox')(fc7_mbox_loc)
conv6_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[2],
next_scale=scales[3],
aspect_ratios=aspect_ratios[2],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[2],
this_offsets=offsets[2],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv6_2_mbox_priorbox')(conv6_2_mbox_loc)
conv7_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[3],
next_scale=scales[4],
aspect_ratios=aspect_ratios[3],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[3],
this_offsets=offsets[3],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv7_2_mbox_priorbox')(conv7_2_mbox_loc)
conv8_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[4],
next_scale=scales[5],
aspect_ratios=aspect_ratios[4],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[4],
this_offsets=offsets[4],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv8_2_mbox_priorbox')(conv8_2_mbox_loc)
conv9_2_mbox_priorbox = AnchorBoxes(
img_height,
img_width,
this_scale=scales[5],
next_scale=scales[6],
aspect_ratios=aspect_ratios[5],
two_boxes_for_ar1=two_boxes_for_ar1,
this_steps=steps[5],
this_offsets=offsets[5],
limit_boxes=limit_boxes,
variances=variances,
coords=coords,
normalize_coords=normalize_coords,
name='conv9_2_mbox_priorbox')(conv9_2_mbox_loc)
### Reshape
# Reshape the class predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, n_classes)`
# We want the classes isolated in the last axis to perform softmax on them
conv4_3_norm_mbox_conf_reshape = Reshape(
(-1, n_classes),
name='conv4_3_norm_mbox_conf_reshape')(conv4_3_norm_mbox_conf)
fc7_mbox_conf_reshape = Reshape(
(-1, n_classes), name='fc7_mbox_conf_reshape')(fc7_mbox_conf)
conv6_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv6_2_mbox_conf_reshape')(conv6_2_mbox_conf)
conv7_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv7_2_mbox_conf_reshape')(conv7_2_mbox_conf)
conv8_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv8_2_mbox_conf_reshape')(conv8_2_mbox_conf)
conv9_2_mbox_conf_reshape = Reshape(
(-1, n_classes), name='conv9_2_mbox_conf_reshape')(conv9_2_mbox_conf)
# Reshape the box predictions, yielding 3D tensors of shape `(batch, height * width * n_boxes, 4)`
# We want the four box coordinates isolated in the last axis to compute the smooth L1 loss
conv4_3_norm_mbox_loc_reshape = Reshape(
(-1, 4), name='conv4_3_norm_mbox_loc_reshape')(conv4_3_norm_mbox_loc)
fc7_mbox_loc_reshape = Reshape((-1, 4),
name='fc7_mbox_loc_reshape')(fc7_mbox_loc)
conv6_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv6_2_mbox_loc_reshape')(conv6_2_mbox_loc)
conv7_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv7_2_mbox_loc_reshape')(conv7_2_mbox_loc)
conv8_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv8_2_mbox_loc_reshape')(conv8_2_mbox_loc)
conv9_2_mbox_loc_reshape = Reshape(
(-1, 4), name='conv9_2_mbox_loc_reshape')(conv9_2_mbox_loc)
# Reshape the anchor box tensors, yielding 3D tensors of shape `(batch, height * width * n_boxes, 8)`
conv4_3_norm_mbox_priorbox_reshape = Reshape(
(-1, 8),
name='conv4_3_norm_mbox_priorbox_reshape')(conv4_3_norm_mbox_priorbox)
fc7_mbox_priorbox_reshape = Reshape(
(-1, 8), name='fc7_mbox_priorbox_reshape')(fc7_mbox_priorbox)
conv6_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv6_2_mbox_priorbox_reshape')(conv6_2_mbox_priorbox)
conv7_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv7_2_mbox_priorbox_reshape')(conv7_2_mbox_priorbox)
conv8_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv8_2_mbox_priorbox_reshape')(conv8_2_mbox_priorbox)
conv9_2_mbox_priorbox_reshape = Reshape(
(-1, 8), name='conv9_2_mbox_priorbox_reshape')(conv9_2_mbox_priorbox)
### Concatenate the predictions from the different layers
# Axis 0 (batch) and axis 2 (n_classes or 4, respectively) are identical for all layer predictions,
# so we want to concatenate along axis 1, the number of boxes per layer
# Output shape of `mbox_conf`: (batch, n_boxes_total, n_classes)
mbox_conf = Concatenate(axis=1, name='mbox_conf')([
conv4_3_norm_mbox_conf_reshape, fc7_mbox_conf_reshape,
conv6_2_mbox_conf_reshape, conv7_2_mbox_conf_reshape,
conv8_2_mbox_conf_reshape, conv9_2_mbox_conf_reshape
])
# Output shape of `mbox_loc`: (batch, n_boxes_total, 4)
mbox_loc = Concatenate(axis=1, name='mbox_loc')([
conv4_3_norm_mbox_loc_reshape, fc7_mbox_loc_reshape,
conv6_2_mbox_loc_reshape, conv7_2_mbox_loc_reshape,
conv8_2_mbox_loc_reshape, conv9_2_mbox_loc_reshape
])
# Output shape of `mbox_priorbox`: (batch, n_boxes_total, 8)
mbox_priorbox = Concatenate(axis=1, name='mbox_priorbox')([
conv4_3_norm_mbox_priorbox_reshape, fc7_mbox_priorbox_reshape,
conv6_2_mbox_priorbox_reshape, conv7_2_mbox_priorbox_reshape,
conv8_2_mbox_priorbox_reshape, conv9_2_mbox_priorbox_reshape
])
# The box coordinate predictions will go into the loss function just the way they are,
# but for the class predictions, we'll apply a softmax activation layer first
mbox_conf_softmax = Activation('softmax',
name='mbox_conf_softmax')(mbox_conf)
# Concatenate the class and box predictions and the anchors to one large predictions vector
# Output shape of `predictions`: (batch, n_boxes_total, n_classes + 4 + 8)
predictions = Concatenate(axis=2, name='predictions')(
[mbox_conf_softmax, mbox_loc, mbox_priorbox])
model = Model(inputs=x, outputs=predictions)
if return_predictor_sizes:
# Get the spatial dimensions (height, width) of the predictor conv layers, we need them to
# be able to generate the default boxes for the matching process outside of the model during training.
# Note that the original implementation performs anchor box matching inside the loss function. We don't do that.
# Instead, we'll do it in the batch generator function.
# The spatial dimensions are the same for the confidence and localization predictors, so we just take those of the conf layers.
predictor_sizes = np.array([
conv4_3_norm_mbox_conf._keras_shape[1:3],
fc7_mbox_conf._keras_shape[1:3],
conv6_2_mbox_conf._keras_shape[1:3],
conv7_2_mbox_conf._keras_shape[1:3],
conv8_2_mbox_conf._keras_shape[1:3],
conv9_2_mbox_conf._keras_shape[1:3]
])
return model, predictor_sizes
else:
return model
def build_model5(batch_norm=True, num_channels=12, num_classes=1):
"""
Builds a 7.7 million parameter model, min. input size = 86-by-86
Parameters
----------
batch_norm : bool, optional
True for batch_norm layers to be used. False otherwise.
num_channels : int, optional
Number of spectral bands in inputs.
num_classes : int, optional
Number of output classes.
Returns
-------
model : keras.models.Model
Uncompiled keras model.
"""
inputs = Input(shape=(None, None, num_channels))
# ------LOCAL INFORMATION GATHERING
x0 = Conv2D(12, (3, 3),
strides=1,
padding='SAME',
activation='tanh',
kernel_initializer='glorot_uniform')(inputs)
x1 = Conv2D(16, (7, 7),
strides=1,
padding='SAME',
activation='tanh',
kernel_initializer='glorot_uniform')(inputs)
x_in = Concatenate()([inputs, x0, x1])
x_in = Conv2D(31, (1, 1),
strides=1,
padding='VALID',
activation='tanh',
kernel_initializer='glorot_uniform')(x_in)
if batch_norm:
x_in = BatchNormalization(axis=-1, momentum=0.99)(x_in)
x_in = Conv2D(18, (1, 1),
strides=1,
padding='VALID',
activation='tanh',
kernel_initializer='glorot_uniform')(x_in)
if batch_norm:
x_in = BatchNormalization(axis=-1, momentum=0.99)(x_in)
x_RES_1 = Conv2D(9, (1, 1),
strides=1,
padding='VALID',
activation='tanh',
kernel_initializer='glorot_uniform')(x_in)
if batch_norm:
x_RES_1 = BatchNormalization(axis=-1, momentum=0.99)(x_RES_1)
# =================================
x = Conv2D(64, (5, 5),
strides=2,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x_RES_1)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2D(128, (7, 7),
strides=3,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x_RES_2 = Conv2D(64, (1, 1),
strides=1,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x_RES_2 = LeakyReLU(alpha=0.01)(x_RES_2)
if batch_norm:
x_RES_2 = BatchNormalization(axis=-1, momentum=0.99)(x_RES_2)
x = Conv2D(256, (5, 5),
strides=2,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x_RES_2)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2D(128, (1, 1),
strides=1,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x_RES_3 = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x_RES_3 = BatchNormalization(axis=-1, momentum=0.99)(x_RES_3)
x = Conv2D(512, (5, 5),
strides=2,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x_RES_3)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
# -----------------------CODE LAYER
x = Conv2DTranspose(256, (5, 5),
strides=2,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Concatenate()([x, x_RES_3])
x = Conv2D(192, (1, 1),
strides=1,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2DTranspose(128, (5, 5),
strides=2,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Concatenate()([x, x_RES_2])
x = Conv2DTranspose(128, (7, 7),
strides=3,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2D(64, (1, 1),
strides=1,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2D(48, (1, 1),
strides=1,
padding='VALID',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2DTranspose(48, (5, 5),
strides=2,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Concatenate()([x, x_RES_1, inputs])
# =============FCL-type convolutions at each pixel...
x = Conv2DTranspose(48, (1, 1),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2DTranspose(32, (1, 1),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2DTranspose(12, (1, 1),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
x = Conv2DTranspose(4, (1, 1),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
if batch_norm:
x = BatchNormalization(axis=-1, momentum=0.99)(x)
X0 = Conv2DTranspose(3, (3, 3),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
X0 = LeakyReLU(alpha=0.01)(X0)
x = Concatenate()([x, X0, inputs]) # even more local info
x = Conv2DTranspose(5, (1, 1),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
x = LeakyReLU(alpha=0.01)(x)
x = Conv2DTranspose(num_classes, (1, 1),
strides=1,
padding='SAME',
activation='linear',
kernel_initializer='glorot_uniform')(x)
if num_classes > 1:
outputs = Activation('softmax')(x)
return Model(inputs=inputs, outputs=outputs)
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(
Embedding(embed_sizes[i], 10)(categorical_inputs[i]))
#categorical_logits = Concatenate()([Flatten()(cat_emb) for cat_emb in categorical_embeddings])
categorical_logits = Flatten()(categorical_embeddings[0])
categorical_logits = Dense(32, activation='relu')(categorical_logits)
numerical_inputs = Input(shape=(11, ), name='num')
numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
numerical_logits = Dense(128, activation='relu')(numerical_logits)
numerical_logits = Dense(64, activation='relu')(numerical_logits)
logits = Concatenate()([numerical_logits, categorical_logits])
logits = Dense(64, activation='relu')(logits)
out = Dense(1, activation='sigmoid')(logits)
model = Model(inputs=categorical_inputs + [numerical_inputs], outputs=out)
model.compile(optimizer='adam', loss=binary_crossentropy)
# In[ ]:
# Lets print our model
model.summary()
# In[ ]:
def get_input(market_train, indices):
def __build_keras_model(
self
): # Build Model. Starting with individual Embeddings for every cat. Using Keras
# Functional API
input_store = Input(shape=(1, ))
output_store = Embedding(1115, 10, name='store_embedding')(input_store)
output_store = Reshape(target_shape=(10, ))(output_store)
input_dow = Input(shape=(1, ))
output_dow = Embedding(7, 6, name='dow_embedding')(input_dow)
output_dow = Reshape(target_shape=(6, ))(output_dow)
input_promo = Input(shape=(1, ))
output_promo = Dense(1)(input_promo)
input_year = Input(shape=(1, ))
output_year = Embedding(3, 2, name='year_embedding')(input_year)
output_year = Reshape(target_shape=(2, ))(output_year)
input_month = Input(shape=(1, ))
output_month = Embedding(12, 6, name='month_embedding')(input_month)
output_month = Reshape(target_shape=(6, ))(output_month)
input_day = Input(shape=(1, ))
output_day = Embedding(31, 10, name='day_embedding')(input_day)
output_day = Reshape(target_shape=(10, ))(output_day)
input_germanstate = Input(shape=(1, ))
output_germanstate = Embedding(
12, 6, name='state_embedding')(input_germanstate)
output_germanstate = Reshape(target_shape=(6, ))(output_germanstate)
input_holiday = Input(shape=(1, ))
output_holiday = Dense(1)(input_holiday)
input_model = [
input_store, input_dow, input_promo, input_year, input_month,
input_day, input_germanstate, input_holiday
]
output_embeddings = [
output_store, output_dow, output_promo, output_year, output_month,
output_day, output_germanstate, output_holiday
]
output_model = Concatenate()(
output_embeddings) # Concatenate inputs to model
output_model = Dense(1000, kernel_initializer="uniform")(output_model)
output_model = Activation('relu')(output_model)
output_model = Dense(500, kernel_initializer="uniform")(output_model)
output_model = Activation('relu')(output_model)
output_model = Dense(1)(output_model)
output_model = Activation('sigmoid')(output_model)
self.model = KerasModel(inputs=input_model, outputs=output_model)
self.model.compile(loss='mean_absolute_error', optimizer='adam')
self.model.summary()
plot_model(self.model,
show_shapes=True,
show_layer_names=True,
rankdir='LR',
to_file='tmp/model.png')
self.model.compile(loss='mean_absolute_error', optimizer='adam')
def get_unet_model_4_levels(self):
unet_input = Input(shape=(self.input_dim_x, self.input_dim_y,
self.num_channels))
conv1 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(unet_input)
conv1 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(conv1)
conv1 = BatchNormalization()(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(pool1)
conv2 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(conv2)
conv2 = BatchNormalization()(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(pool2)
conv3 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(conv3)
conv3 = BatchNormalization()(conv3)
drop3 = Dropout(0.5)(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(drop3)
conv4 = Conv2D(self.n_filters * 16,
kernel_size=3,
activation='relu',
padding='same')(pool3)
conv4 = Conv2D(self.n_filters * 16,
kernel_size=3,
activation='relu',
padding='same')(conv4)
conv4 = BatchNormalization()(conv4)
drop4 = Dropout(0.5)(conv4)
up5 = Conv2D(self.n_filters * 16, 2, activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(drop4))
concat5 = Concatenate()([drop3, up5])
conv5 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(concat5)
conv5 = Conv2D(self.n_filters * 8,
kernel_size=3,
activation='relu',
padding='same')(conv5)
conv5 = BatchNormalization()(conv5)
up6 = Conv2D(self.n_filters * 8, 2, activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(conv5))
concat6 = Concatenate()([conv2, up6])
conv6 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(concat6)
conv6 = Conv2D(self.n_filters * 4,
kernel_size=3,
activation='relu',
padding='same')(conv6)
conv6 = BatchNormalization()(conv6)
up7 = Conv2D(self.n_filters * 4, 2, activation='relu',
padding='same')(UpSampling2D(size=(2, 2))(conv6))
concat7 = Concatenate()([conv1, up7])
conv7 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(concat7)
conv7 = Conv2D(self.n_filters * 2,
kernel_size=3,
activation='relu',
padding='same')(conv7)
conv7 = BatchNormalization()(conv7)
conv9 = Conv2D(3, kernel_size=1, activation='sigmoid',
padding='same')(conv7)
return Model(outputs=conv9, inputs=unet_input), 'unet_model_4_levels'