def block_inception_c(input):
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
branch_0 = conv2d_bn(input, 256, 1, 1)
branch_1 = conv2d_bn(input, 384, 1, 1)
branch_10 = conv2d_bn(branch_1, 256, 1, 3)
branch_11 = conv2d_bn(branch_1, 256, 3, 1)
branch_1 = concatenate([branch_10, branch_11], axis=channel_axis)
branch_2 = conv2d_bn(input, 384, 1, 1)
branch_2 = conv2d_bn(branch_2, 448, 3, 1)
branch_2 = conv2d_bn(branch_2, 512, 1, 3)
branch_20 = conv2d_bn(branch_2, 256, 1, 3)
branch_21 = conv2d_bn(branch_2, 256, 3, 1)
branch_2 = concatenate([branch_20, branch_21], axis=channel_axis)
branch_3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(input)
branch_3 = conv2d_bn(branch_3, 256, 1, 1)
x = concatenate([branch_0, branch_1, branch_2, branch_3], axis=channel_axis)
return x
def ship_cnn(input_dim=(512, 512, 3)): inputs = Input(shape=input_dim) conv1 = Conv2D(filters=64, kernel_size=3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs) pool1 = MaxPooling2D(pool_size=(2, 2), padding='same')(conv1) conv2 = Conv2D(filters=128, kernel_size=3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1) drop2 = Dropout(0.5)(conv2) pool2 = MaxPooling2D(pool_size=(2, 2), padding='same')(drop2) conv3 = Conv2D(filters=256, kernel_size=3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2) drop3 = Dropout(0.5)(conv3) # Up-L1 - merge upsampled L2 with output from L1 #upsampled_conv1 = Conv2DTranspose(filters=32, kernel_size=3, strides=2, padding='same')(conv3) up1 = Conv2D(128, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(drop3)) merge1 = concatenate([drop2, up1], axis=3) conv4 = Conv2D(filters = 128, kernel_size=3, padding='same')(merge1) # Up-L1 - merge upsampled L2 with output from L1 #upsampled_conv2 = Conv2DTranspose(filters=64, kernel_size=3, strides=2, padding='same') up2 = Conv2D(64, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv4)) merge2 = concatenate([conv1, up2], axis=3) conv5 = Conv2D(filters = 64, kernel_size=3, padding='same', kernel_initializer='he_normal')(merge2) conv6 = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5) # Final layer - makes 768x768x1 image segmap = Conv2D(filters = 1, kernel_size = 1, activation = 'sigmoid')(conv6) model = Model(inputs=inputs, outputs=segmap) print(model.summary()) model.compile(optimizer = Adam(lr = 0.0001), loss = 'binary_crossentropy', metrics = ['accuracy']) return model
def inception_v4_base(input):
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
# Input Shape is 299 x 299 x 3 (th) or 3 x 299 x 299 (th)
net = conv2d_bn(input, 32, 3, 3, strides=(2, 2), padding='valid')
net = conv2d_bn(net, 32, 3, 3, padding='valid')
net = conv2d_bn(net, 64, 3, 3)
branch_0 = MaxPooling2D((3, 3), strides=(2, 2), padding='valid')(net)
branch_1 = conv2d_bn(net, 96, 3, 3, strides=(2, 2), padding='valid')
net = concatenate([branch_0, branch_1], axis=channel_axis)
branch_0 = conv2d_bn(net, 64, 1, 1)
branch_0 = conv2d_bn(branch_0, 96, 3, 3, padding='valid')
branch_1 = conv2d_bn(net, 64, 1, 1)
branch_1 = conv2d_bn(branch_1, 64, 1, 7)
branch_1 = conv2d_bn(branch_1, 64, 7, 1)
branch_1 = conv2d_bn(branch_1, 96, 3, 3, padding='valid')
net = concatenate([branch_0, branch_1], axis=channel_axis)
branch_0 = conv2d_bn(net, 192, 3, 3, strides=(2, 2), padding='valid')
branch_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='valid')(net)
net = concatenate([branch_0, branch_1], axis=channel_axis)
# 35 x 35 x 384
# 4 x Inception-A blocks
for idx in range(4):
net = block_inception_a(net)
# 35 x 35 x 384
# Reduction-A block
net = block_reduction_a(net)
# 17 x 17 x 1024
# 7 x Inception-B blocks
for idx in range(7):
net = block_inception_b(net)
# 17 x 17 x 1024
# Reduction-B block
net = block_reduction_b(net)
# 8 x 8 x 1536
# 3 x Inception-C blocks
for idx in range(3):
net = block_inception_c(net)
return net
def ZF_UNET_224(dropout_val=0.2, weights=None):
if K.image_dim_ordering() == 'th':
inputs = Input((INPUT_CHANNELS, 224, 224))
axis = 1
else:
inputs = Input((224, 224, INPUT_CHANNELS))
axis = 3
filters = 32
conv_224 = double_conv_layer(inputs, filters)
pool_112 = MaxPooling2D(pool_size=(2, 2))(conv_224)
conv_112 = double_conv_layer(pool_112, 2*filters)
pool_56 = MaxPooling2D(pool_size=(2, 2))(conv_112)
conv_56 = double_conv_layer(pool_56, 4*filters)
pool_28 = MaxPooling2D(pool_size=(2, 2))(conv_56)
conv_28 = double_conv_layer(pool_28, 8*filters)
pool_14 = MaxPooling2D(pool_size=(2, 2))(conv_28)
conv_14 = double_conv_layer(pool_14, 16*filters)
pool_7 = MaxPooling2D(pool_size=(2, 2))(conv_14)
conv_7 = double_conv_layer(pool_7, 32*filters)
up_14 = concatenate([UpSampling2D(size=(2, 2))(conv_7), conv_14], axis=axis)
up_conv_14 = double_conv_layer(up_14, 16*filters)
up_28 = concatenate([UpSampling2D(size=(2, 2))(up_conv_14), conv_28], axis=axis)
up_conv_28 = double_conv_layer(up_28, 8*filters)
up_56 = concatenate([UpSampling2D(size=(2, 2))(up_conv_28), conv_56], axis=axis)
up_conv_56 = double_conv_layer(up_56, 4*filters)
up_112 = concatenate([UpSampling2D(size=(2, 2))(up_conv_56), conv_112], axis=axis)
up_conv_112 = double_conv_layer(up_112, 2*filters)
up_224 = concatenate([UpSampling2D(size=(2, 2))(up_conv_112), conv_224], axis=axis)
up_conv_224 = double_conv_layer(up_224, filters, dropout_val)
conv_final = Conv2D(OUTPUT_MASK_CHANNELS, (1, 1))(up_conv_224)
conv_final = Activation('sigmoid')(conv_final)
model = Model(inputs, conv_final, name="ZF_UNET_224")
if weights == 'generator' and axis == 3 and INPUT_CHANNELS == 3 and OUTPUT_MASK_CHANNELS == 1:
weights_path = get_file(
'zf_unet_224_weights_tf_dim_ordering_tf_generator.h5',
ZF_UNET_224_WEIGHT_PATH,
cache_subdir='models',
file_hash='203146f209baf34ac0d793e1691f1ab7')
model.load_weights(weights_path)
return model
def unet(input_dim = (512,512,3)): inputs = Input(shape=input_dim) # L1 conv1 = Conv2D(filters = 64, kernel_size = 3, padding='same')(inputs) conv2 = Conv2D(filters = 64, kernel_size = 3, padding='same')(conv1) # L2 down1 = MaxPooling2D(pool_size=(2,2), padding='same')(conv2) conv3 = Conv2D(filters = 128, kernel_size = 3, padding='same')(down1) conv4 = Conv2D(filters = 128, kernel_size = 3, padding='same')(conv3) # L3 down2 = MaxPooling2D(pool_size=(2,2), padding='same')(conv4) conv5 = Conv2D(filters = 256, kernel_size = 3, padding='same')(down2) conv6 = Conv2D(filters = 256, kernel_size = 3, padding='same')(conv5) # L4 down3 = MaxPooling2D(pool_size=(2,2), padding='same')(conv6) conv7 = Conv2D(filters = 512, kernel_size = 3, padding='same')(down3) conv8 = Conv2D(filters = 512, kernel_size = 3, padding='same')(conv7) # L5 down4 = MaxPooling2D(pool_size=(2,2), padding='same')(conv8) conv9 = Conv2D(filters=1024, kernel_size = 3, padding='same')(down4) conv10 = Conv2D(filters=1024, kernel_size=3, padding='same')(conv9) # Merge upsampled L5 with output from L4 upsampled_conv10 = Conv2DTranspose(filters=512, kernel_size=3, strides=2, padding='same')(conv10) print(upsampled_conv10.shape) # Up-L4 up1 = concatenate([conv8, upsampled_conv10]) conv11 = Conv2D(filters = 512, kernel_size= 3, padding='same')(up1) conv12 = Conv2D(filters = 512, kernel_size= 3, padding='same')(conv11) # Up-L3 - merge upsampled L4 with output from L3 upsampled_conv12 = Conv2DTranspose(filters=256, kernel_size = 3, strides=2, padding='same')(conv12) up2 = concatenate([conv6, upsampled_conv12]) conv13 = Conv2D(filters = 256, kernel_size= 3, padding='same')(up2) conv14 = Conv2D(filters = 256, kernel_size= 3, padding='same')(conv13) # Up-L2 - merge upsampled L3 with output from L2 upsampled_conv14 = Conv2DTranspose(filters=128, kernel_size = 3, strides=2, padding='same')(conv14) up3 = concatenate([conv4, upsampled_conv14]) conv15 = Conv2D(filters = 128, kernel_size= 3, padding='same')(up3) conv16 = Conv2D(filters = 128, kernel_size= 3, padding='same')(conv15) # Up-L1 - merge upsampled L2 with output from L1 upsampled_conv16 = Conv2DTranspose(filters=64, kernel_size = 3, strides=2, padding='same')(conv16) up4 = concatenate([conv2, upsampled_conv16]) conv17 = Conv2D(filters = 64, kernel_size = 3, padding='same')(up4) conv18 = Conv2D(filters = 64, kernel_size= 3, padding='same')(conv17) conv19 = Conv2D(filters = 2, kernel_size = 3, padding='same')(conv18) # Final layer - makes 768x768x1 image segmap = Conv2D(filters = 1, kernel_size = 1, activation = 'sigmoid')(conv19) model = Model(inputs=inputs, outputs=segmap) print(model.summary()) model.compile(optimizer = Adam(1e-4, decay=1e-6), loss = dice_p_bce, metrics = ['accuracy', dice_coef, true_positive_rate]) return model
def build_UNet2D_4L(inp_shape, k_size=3):
merge_axis = -1 # Feature maps are concatenated along last axis (for tf backend)
data = Input(shape=inp_shape)
conv1 = Convolution2D(filters=32, kernel_size=k_size, padding='same', activation='relu')(data)
conv1 = Convolution2D(filters=32, kernel_size=k_size, padding='same', activation='relu')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(pool1)
conv2 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(pool2)
conv3 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Convolution2D(filters=128, kernel_size=k_size, padding='same', activation='relu')(pool3)
conv4 = Convolution2D(filters=128, kernel_size=k_size, padding='same', activation='relu')(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(conv4)
conv5 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(pool4)
up1 = UpSampling2D(size=(2, 2))(conv5)
conv6 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(up1)
conv6 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(conv6)
merged1 = concatenate([conv4, conv6], axis=merge_axis)
conv6 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(merged1)
up2 = UpSampling2D(size=(2, 2))(conv6)
conv7 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(up2)
conv7 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(conv7)
merged2 = concatenate([conv3, conv7], axis=merge_axis)
conv7 = Convolution2D(filters=256, kernel_size=k_size, padding='same', activation='relu')(merged2)
up3 = UpSampling2D(size=(2, 2))(conv7)
conv8 = Convolution2D(filters=128, kernel_size=k_size, padding='same', activation='relu')(up3)
conv8 = Convolution2D(filters=128, kernel_size=k_size, padding='same', activation='relu')(conv8)
merged3 = concatenate([conv2, conv8], axis=merge_axis)
conv8 = Convolution2D(filters=128, kernel_size=k_size, padding='same', activation='relu')(merged3)
up4 = UpSampling2D(size=(2, 2))(conv8)
conv9 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(up4)
conv9 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(conv9)
merged4 = concatenate([conv1, conv9], axis=merge_axis)
conv9 = Convolution2D(filters=64, kernel_size=k_size, padding='same', activation='relu')(merged4)
conv10 = Convolution2D(filters=1, kernel_size=k_size, padding='same', activation='sigmoid')(conv9)
output = conv10
model = Model(data, output)
return model
def test_unet(input_dim=(768, 768, 3)): inputs = Input(input_dim) conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs) conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1) pool1 = MaxPooling2D(pool_size=(2, 2))(conv1) conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1) conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2) pool2 = MaxPooling2D(pool_size=(2, 2))(conv2) conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2) conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3) pool3 = MaxPooling2D(pool_size=(2, 2))(conv3) conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool3) conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4) drop4 = Dropout(0.5)(conv4) pool4 = MaxPooling2D(pool_size=(2, 2))(drop4) conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool4) conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5) drop5 = Dropout(0.5)(conv5) up6 = Conv2D(512, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(drop5)) merge6 = concatenate([drop4, up6], axis=3) conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6) conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6) up7 = Conv2D(256, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv6)) merge7 = concatenate([conv3, up7], axis=3) conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7) conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7) up8 = Conv2D(128, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv7)) merge8 = concatenate([conv2, up8], axis=3) conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge8) conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8) up9 = Conv2D(64, 2, activation='relu', padding='same', kernel_initializer='he_normal')(UpSampling2D(size=(2, 2))(conv8)) merge9 = concatenate([conv1, up9], axis=3) conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge9) conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9) conv9 = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9) conv10 = Conv2D(1, 1, activation='sigmoid')(conv9) model = Model(input=inputs, output=conv10) model.compile(optimizer=Adam(lr=1e-4), loss='binary_crossentropy', metrics=['accuracy']) return model
def dense_block(x, stage, nb_layers, nb_filter, growth_rate, dropout_rate=None, weight_decay=1e-4,
grow_nb_filters=True):
"""
Build a dense_block where the output of each conv_block is fed to subsequent ones
# Arguments
x: input tensor
stage: index for dense block
nb_layers: the number of layers of conv_block to append to the model.
nb_filter: number of filters
growth_rate: growth rate
dropout_rate: dropout rate
weight_decay: weight decay factor
grow_nb_filters: flag to decide to allow number of filters to grow
"""
concat_feat = x
for i in range(nb_layers):
branch = i + 1
x = conv_block(concat_feat, stage, branch, growth_rate, dropout_rate, weight_decay)
concat_feat = concatenate(
[concat_feat, x],
axis=concat_axis,
name='concat_' + str(stage) + '_' + str(branch),
)
if grow_nb_filters:
nb_filter += growth_rate
return concat_feat, nb_filter
def _shortcut(input, residual):
"""Adds a shortcut between input and residual block and merges them with "sum"
"""
# Expand channels of shortcut to match residual.
# Stride appropriately to match residual (width, height)
# Should be int if network architecture is correctly configured.
input_shape = K.int_shape(input)
residual_shape = K.int_shape(residual)
stride_width = int(round(input_shape[ROW_AXIS] / residual_shape[ROW_AXIS]))
stride_height = int(round(input_shape[COL_AXIS] / residual_shape[COL_AXIS]))
equal_channels = input_shape[CHANNEL_AXIS] == residual_shape[CHANNEL_AXIS]
shortcut = input
# if shape is different.
if stride_width > 1 or stride_height > 1 or not equal_channels:
if SHORTCUT_OPTION == 'B':
# 1x1 convolution to match dimension
shortcut = Conv2D(filters=residual_shape[CHANNEL_AXIS],
kernel_size=(1, 1),
strides=(stride_width, stride_height),
padding="valid",
kernel_initializer="he_normal",
kernel_regularizer=l2(0.0001))(input)
elif SHORTCUT_OPTION == 'A':
# spatial pooling with padded identity mapping
x = AveragePooling2D(pool_size=(1, 1),
strides=(stride_width, stride_height))(input)
# multiply every element of x by 0 to get zero matrix
mul_zero = Lambda(lambda val: val * 0.0,
output_shape=K.int_shape(x)[1:])(x)
shortcut = concatenate([x, mul_zero], axis=CHANNEL_AXIS)
return add([shortcut, residual])
def get_bidirectional(embed_size_1 = 200 , embedding_matrix_1 = None, embed_size_2 = 200 , embedding_matrix_2 = None,
#num_lstm = 50 ,
rate_drop_dense = 0.1,
num_dense = 50):
print(">> get_model_bidirectional_avg [pre-trained word embeddings]<<")
#embedding_layer = Embedding(max_features,embed_size,weights=[embedding_matrix],input_length=maxlen,trainable=True)
embedding_layer_1 = Embedding(max_features,embed_size_1,weights=[embedding_matrix_1],input_length=maxlen)
embedding_layer_2 = Embedding(max_features,embed_size_2,weights=[embedding_matrix_2],input_length=maxlen)
inp = Input(shape=(maxlen, ) , dtype='int32')
x1 = embedding_layer_1(inp)
x2 = embedding_layer_2(inp)
x = concatenate([x1, x2],axis=2)
x = Bidirectional(GRU(num_dense, return_sequences=True, dropout=rate_drop_dense, recurrent_dropout=rate_drop_dense,trainable=True))(x)
x = GlobalMaxPool1D()(x)
x = Dense(num_dense, activation="relu")(x)
x = Dropout(rate_drop_dense)(x)
x = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer='adam',
#optimizer='nadam',
metrics=['accuracy'])
return model
def define_model(length, vocab_size): # channel 1: 三个 channel 只是对应的 kernel_size 不同, 表示使用不同的窗口大小来控制 n-gram; 4-grams, 6-grams, and 8-grams inputs1 = Input(shape=(length,)) embedding1 = Embedding(vocab_size, 100)(inputs1) conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding1) drop1 = Dropout(0.5)(conv1) pool1 = MaxPooling1D(pool_size=2)(drop1) flat1 = Flatten()(pool1) # channel 2 inputs2 = Input(shape=(length,)) embedding2 = Embedding(vocab_size, 100)(inputs2) conv2 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding2) drop2 = Dropout(0.5)(conv2) pool2 = MaxPooling1D(pool_size=2)(drop2) flat2 = Flatten()(pool2) # channel 3 inputs3 = Input(shape=(length,)) embedding3 = Embedding(vocab_size, 100)(inputs3) conv3 = Conv1D(filters=32, kernel_size=8, activation='relu')(embedding3) drop3 = Dropout(0.5)(conv3) pool3 = MaxPooling1D(pool_size=2)(drop3) flat3 = Flatten()(pool3) # merge merged = concatenate([flat1, flat2, flat3]) # interpretation dense1 = Dense(10, activation='relu')(merged) outputs = Dense(1, activation='sigmoid')(dense1) model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs) # compile model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print(model.summary()) plot_model(model, show_shapes=True, to_file='tmp_multichannel.png') return model
def f(x):
if max_pooling:
x_max = GlobalMaxPool1D()(x)
else:
x_max = None
if mean_pooling:
x_mean = GlobalAveragePooling1D()(x)
else:
x_mean = None
if weighted_average_attention:
x_att = AttentionWeightedAverage()(x)
else:
x_att = None
x = [xi for xi in [x_max, x_mean, x_att] if xi is not None]
if len(x) == 1:
x = x[0]
else:
if concat_mode == 'concat':
x = concatenate(x, axis=-1)
else:
NotImplementedError('only mode concat for now')
for _ in range(repeat_dense):
x = dense_block(dense_size=dense_size,
use_batch_norm=use_batch_norm,
use_prelu=use_prelu,
dropout=dropout,
kernel_reg_l2=kernel_reg_l2,
bias_reg_l2=bias_reg_l2,
batch_norm_first=batch_norm_first)(x)
x = Dense(output_size, activation=output_activation)(x)
return x
def model_discriminator(global_shape=(256, 256, 3), local_shape=(128, 128, 3)):
def crop_image(img, crop):
return tf.image.crop_to_bounding_box(img,
crop[1],
crop[0],
crop[3] - crop[1],
crop[2] - crop[0])
in_pts = Input(shape=(4,), dtype='int32')
cropping = Lambda(lambda x: K.map_fn(lambda y: crop_image(y[0], y[1]), elems=x, dtype=tf.float32),
output_shape=local_shape)
g_img = Input(shape=global_shape)
l_img = cropping([g_img, in_pts])
# Local Discriminator
x_l = Conv2D(64, kernel_size=5, strides=2, padding='same')(l_img)
x_l = BatchNormalization()(x_l)
x_l = Activation('relu')(x_l)
x_l = Conv2D(128, kernel_size=5, strides=2, padding='same')(x_l)
x_l = BatchNormalization()(x_l)
x_l = Activation('relu')(x_l)
x_l = Conv2D(256, kernel_size=5, strides=2, padding='same')(x_l)
x_l = BatchNormalization()(x_l)
x_l = Activation('relu')(x_l)
x_l = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_l)
x_l = BatchNormalization()(x_l)
x_l = Activation('relu')(x_l)
x_l = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_l)
x_l = BatchNormalization()(x_l)
x_l = Activation('relu')(x_l)
x_l = Flatten()(x_l)
x_l = Dense(1024, activation='relu')(x_l)
# Global Discriminator
x_g = Conv2D(64, kernel_size=5, strides=2, padding='same')(g_img)
x_g = BatchNormalization()(x_g)
x_g = Activation('relu')(x_g)
x_g = Conv2D(128, kernel_size=5, strides=2, padding='same')(x_g)
x_g = BatchNormalization()(x_g)
x_g = Activation('relu')(x_g)
x_g = Conv2D(256, kernel_size=5, strides=2, padding='same')(x_g)
x_g = BatchNormalization()(x_g)
x_g = Activation('relu')(x_g)
x_g = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_g)
x_g = BatchNormalization()(x_g)
x_g = Activation('relu')(x_g)
x_g = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_g)
x_g = BatchNormalization()(x_g)
x_g = Activation('relu')(x_g)
x_g = Conv2D(512, kernel_size=5, strides=2, padding='same')(x_g)
x_g = BatchNormalization()(x_g)
x_g = Activation('relu')(x_g)
x_g = Flatten()(x_g)
x_g = Dense(1024, activation='relu')(x_g)
x = concatenate([x_l, x_g])
x = Dense(1, activation='sigmoid')(x)
return Model(inputs=[g_img, in_pts], outputs=x)
def _build(self, models, layers=[]):
for layer in layers:
layer.name = '%s/%s' % (self.scope, layer.name)
inputs, outputs = self._get_inputs_outputs(models)
x = concatenate(outputs)
for layer in layers:
x = layer(x)
model = km.Model(inputs, x, name=self.name)
return model
def __init__(self):
# Input tensors holding the query, positive (clicked) document, and negative (unclicked) documents.
# The first dimension is None because the queries and documents can vary in length.
query = Input(shape = (None, WORD_DEPTH))
pos_doc = Input(shape = (None, WORD_DEPTH))
neg_docs = [Input(shape = (None, WORD_DEPTH)) for j in range(J)]
query_conv = Convolution1D(K, FILTER_LENGTH, padding = "same", input_shape = (None, WORD_DEPTH), activation = "tanh")(query) # See equation (2).
query_max = Lambda(lambda x: backend.max(x, axis = 1), output_shape = (K, ))(query_conv) # See section 3.4.
query_sem = Dense(L, activation = "tanh", input_dim = K)(query_max) # See section 3.5.
# The document equivalent of the above query model.
doc_conv = Convolution1D(K, FILTER_LENGTH, padding = "same", input_shape = (None, WORD_DEPTH), activation = "tanh")
doc_max = Lambda(lambda x: backend.max(x, axis = 1), output_shape = (K, ))
doc_sem = Dense(L, activation = "tanh", input_dim = K)
pos_doc_conv = doc_conv(pos_doc)
neg_doc_convs = [doc_conv(neg_doc) for neg_doc in neg_docs]
pos_doc_max = doc_max(pos_doc_conv)
neg_doc_maxes = [doc_max(neg_doc_conv) for neg_doc_conv in neg_doc_convs]
pos_doc_sem = doc_sem(pos_doc_max)
neg_doc_sems = [doc_sem(neg_doc_max) for neg_doc_max in neg_doc_maxes]
# This layer calculates the cosine similarity between the semantic representations of
# a query and a document.
R_Q_D_p = dot([query_sem, pos_doc_sem], axes = 1, normalize = True) # See equation (4).
R_Q_D_ns = [dot([query_sem, neg_doc_sem], axes = 1, normalize = True) for neg_doc_sem in neg_doc_sems] # See equation (4).
concat_Rs = concatenate([R_Q_D_p] + R_Q_D_ns)
concat_Rs = Reshape((J + 1, 1))(concat_Rs)
# In this step, we multiply each R(Q, D) value by gamma. In the paper, gamma is
# described as a smoothing factor for the softmax function, and it's set empirically
# on a held-out data set. We're going to learn gamma's value by pretending it's
# a single 1 x 1 kernel.
weight = np.array([1]).reshape(1, 1, 1)
with_gamma = Convolution1D(1, 1, padding = "same", input_shape = (J + 1, 1), activation = "linear", use_bias = False, weights = [weight])(concat_Rs) # See equation (5).
with_gamma = Reshape((J + 1, ))(with_gamma)
# Finally, we use the softmax function to calculate P(D+|Q).
prob = Activation("softmax")(with_gamma) # See equation (5).
# We now have everything we need to define our model.
self.model = Model(inputs = [query, pos_doc] + neg_docs, outputs = prob)
self.model.compile(optimizer = "adadelta", loss = "categorical_crossentropy")
self.encoder = Model(inputs=query, outputs=query_sem)
def fire_module(x, fire_id, squeeze=16, expand=64):
s_id = 'fire' + str(fire_id) + '/'
x = Conv2D(squeeze, (1, 1), padding='valid', name=s_id + sq1x1)(x)
x = Activation('relu', name=s_id + relu + sq1x1)(x)
left = Conv2D(expand, (1, 1), padding='valid', name=s_id + exp1x1)(x)
left = Activation('relu', name=s_id + relu + exp1x1)(left)
right = Conv2D(expand, (3, 3), padding='same', name=s_id + exp3x3)(x)
right = Activation('relu', name=s_id + relu + exp3x3)(right)
x = concatenate([left, right], axis=3, name=s_id + 'concat')
return x
def block_reduction_a(input):
if K.image_data_format() == 'channels_first':
channel_axis = 1
else:
channel_axis = -1
branch_0 = conv2d_bn(input, 384, 3, 3, strides=(2, 2), padding='valid')
branch_1 = conv2d_bn(input, 192, 1, 1)
branch_1 = conv2d_bn(branch_1, 224, 3, 3)
branch_1 = conv2d_bn(branch_1, 256, 3, 3, strides=(2, 2), padding='valid')
branch_2 = MaxPooling2D((3, 3), strides=(2, 2), padding='valid')(input)
x = concatenate([branch_0, branch_1, branch_2], axis=channel_axis)
return x
def linknet_decoder(conv1, enc1, enc2, enc3, enc4, enc5, filters=[64, 128, 256, 512, 512], feature_scale=4, skipFirst=False, transposed_conv=False):
decoder5 = decoder(enc5, filters[4], filters[3], name='decoder5', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder5 = add([decoder5, enc4])
decoder4 = decoder(decoder5, filters[3], filters[2], name='decoder4', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder4 = add([decoder4, enc3])
decoder3 = decoder(decoder4, filters[2], filters[1], name='decoder3', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder3 = add([decoder3, enc2])
decoder2 = decoder(decoder3, filters[1], filters[0], name='decoder2', feature_scale=feature_scale, transposed_conv=transposed_conv)
decoder2 = add([decoder2, enc1])
decoder1 = decoder(decoder2, filters[0], filters[0], name='decoder1', feature_scale=feature_scale, transposed_conv=transposed_conv)
if skipFirst:
x = concatenate([conv1, decoder1])
x = conv_bn_relu(x, 32, 3, stride=1, padding='same', name='f2_skip_1')
x = conv_bn_relu(x, 32, 3, stride=1, padding='same', name='f2_skip_2')
else:
x = conv_bn_relu(decoder1, 32, 3, stride=1, padding='same', name='f2')
return x
def inception_model(input, filters_1x1, filters_3x3_reduce, filters_3x3, filters_5x5_reduce, filters_5x5, filters_pool_proj):
conv_1x1 = Conv2D(filters=filters_1x1, kernel_size=(1, 1), padding='same', activation='relu', kernel_regularizer=l2(0.01))(input)
conv_3x3_reduce = Conv2D(filters=filters_3x3_reduce, kernel_size=(1, 1), padding='same', activation='relu', kernel_regularizer=l2(0.01))(input)
conv_3x3 = Conv2D(filters=filters_3x3, kernel_size=(3, 3), padding='same', activation='relu', kernel_regularizer=l2(0.01))(conv_3x3_reduce)
conv_5x5_reduce = Conv2D(filters=filters_5x5_reduce, kernel_size=(1, 1), padding='same', activation='relu', kernel_regularizer=l2(0.01))(input)
conv_5x5 = Conv2D(filters=filters_5x5, kernel_size=(5, 5), padding='same', activation='relu', kernel_regularizer=l2(0.01))(conv_5x5_reduce)
maxpool = MaxPooling2D(pool_size=(3, 3), strides=(1, 1), padding='same')(input)
maxpool_proj = Conv2D(filters=filters_pool_proj, kernel_size=(1, 1), strides=(1, 1), padding='same', activation='relu', kernel_regularizer=l2(0.01))(maxpool)
inception_output = concatenate([conv_1x1, conv_3x3, conv_5x5, maxpool_proj], axis=3) # use tf as backend
return inception_output
def build(input_shape, num_outputs, block_fn, repetitions, nb_init_filter=64,
init_filter_size=7, init_conv_stride=2, pool_size=3, pool_stride=2,
weight_decay=.0001, alpha=1., l1_ratio=.5,
inp_dropout=.0, hidden_dropout=.0, shortcut_with_bn=False):
"""
Builds a custom ResNet like architecture.
:param input_shape: Shall be the input shapes for both CC and MLO views.
:param num_outputs: The number of outputs at final softmax layer
:param block_fn: The block function to use. This is either :func:`basic_block` or :func:`bottleneck`.
The original paper used basic_block for layers < 50
:param repetitions: Number of repetitions of various block units.
At each block unit, the number of filters are doubled and the input size is halved
:return: The keras model.
"""
# First, define a shared CNN model for both CC and MLO views.
input_cc, flatten_cc = ResNetBuilder._shared_conv_layers(
input_shape, block_fn, repetitions,
nb_init_filter=nb_init_filter, init_filter_size=init_filter_size,
init_conv_stride=init_conv_stride,
pool_size=pool_size, pool_stride=pool_stride,
weight_decay=weight_decay,
inp_dropout=inp_dropout, hidden_dropout=hidden_dropout,
shortcut_with_bn=shortcut_with_bn)
input_mlo, flatten_mlo = ResNetBuilder._shared_conv_layers(
input_shape, block_fn, repetitions,
nb_init_filter=nb_init_filter, init_filter_size=init_filter_size,
init_conv_stride=init_conv_stride,
pool_size=pool_size, pool_stride=pool_stride,
weight_decay=weight_decay,
inp_dropout=inp_dropout, hidden_dropout=hidden_dropout,
shortcut_with_bn=shortcut_with_bn)
# Then merge the conv representations of the two views.
merged_repr = concatenate([flatten_cc, flatten_mlo])
enet_penalty = ResNetBuilder.l1l2_penalty_reg(alpha, l1_ratio)
activation = "softmax" if num_outputs > 1 else "sigmoid"
dense = Dense(units=num_outputs, kernel_initializer="he_normal",
activation=activation, kernel_regularizer=enet_penalty)(merged_repr)
discr_model = Model(inputs=[input_cc, input_mlo], outputs=dense)
return discr_model
def yolo_body(inputs, num_anchors, num_classes):
"""Create YOLO_V2 model CNN body in Keras."""
darknet = Model(inputs, darknet_body()(inputs))
conv20 = compose(
DarknetConv2D_BN_Leaky(1024, (3, 3)),
DarknetConv2D_BN_Leaky(1024, (3, 3)))(darknet.output)
conv13 = darknet.layers[43].output
conv21 = DarknetConv2D_BN_Leaky(64, (1, 1))(conv13)
# TODO: Allow Keras Lambda to use func arguments for output_shape?
conv21_reshaped = Lambda(
space_to_depth_x2,
output_shape=space_to_depth_x2_output_shape,
name='space_to_depth')(conv21)
x = concatenate([conv21_reshaped, conv20])
x = DarknetConv2D_BN_Leaky(1024, (3, 3))(x)
x = DarknetConv2D(num_anchors * (num_classes + 5), (1, 1))(x)
return Model(inputs, x)
def get_model_bidirectional_avg(embed_size = 200 , embed_size_2 = 50 ,
embedding_matrix = None,
embedding_matrix_2 = None ,
num_lstm = 50 ,
rate_drop_dense = 0.1,
num_dense = 50):
if embedding_matrix is None:
print(">> get_model_bidirectional_avg [no pre-trained word embeddings]<<")
inp = Input(shape=(maxlen, ))
x = Embedding(max_features, embed_size)(inp)
else:
print(">> get_model_bidirectional_avg [pre-trained word embeddings]<<")
embedding_layer = Embedding(max_features,embed_size,weights=[embedding_matrix],input_length=maxlen,trainable=True)
inp = Input(shape=(maxlen, ) , dtype='int32')
x1 = embedding_layer(inp)
x1 = Bidirectional(GRU(num_lstm, return_sequences=True))(x1)
x1 = Dropout(rate_drop_dense)(x1)
#add a GlobalAveragePooling1D, which will average the embeddings of all words in the document
x1 = GlobalAveragePooling1D()(x1)
if embedding_matrix_2 is not None:
embedding_layer_2 = Embedding(max_features,embed_size_2,weights=[embedding_matrix_2],input_length=maxlen,trainable=True)
x2 = embedding_layer_2(inp)
x2 = Bidirectional(GRU(num_lstm, return_sequences=True))(x2)
x2 = Dropout(rate_drop_dense)(x2)
#add a GlobalAveragePooling1D, which will average the embeddings of all words in the document
x2 = GlobalAveragePooling1D()(x2)
x = concatenate([x1, x2])
else:
x = x1
x = Dense(num_dense, activation="relu")(x)
x = Dropout(rate_drop_dense)(x)
#x = BatchNormalization()(x)
x = Dense(6, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def E_C_lstm():
English_input = Input(shape=(maxlen,))
Engem =Embedding(max_features, 128, input_length=maxlen)(English_input)
Chinese_input =Input(shape=(maxlen,))
CHem =Embedding(max_features,64,input_length=maxlen)(Chinese_input)
ENlstm =LSTM(64,dropout=0.5)(Engem)
CHlstm =LSTM(64,dropout=0.5)(CHem)
ENout =Dense(2,activation='sigmoid',name='ENout')(ENlstm)
CHout =Dense(2,activation='sigmoid',name='CHout')(CHlstm)
loss_out =Lambda(lossfunction,output_shape=(2,),name='losss')([ENout,CHout])
out =concatenate(inputs=[ENout,CHout])
out =Dense(2,activation='softmax',name='Finout')(out)
model =Model(inputs=[English_input,Chinese_input], outputs=[out,ENout,CHout,loss_out])
model.summary()
return model
def _dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None, weight_decay=1e-4,
grow_nb_filters=True, return_concat_list=False):
''' Build a dense_block where the output of each conv_block is fed to subsequent ones
Args:
x: keras tensor
nb_layers: the number of layers of conv_block to append to the model.
nb_filter: number of filters
growth_rate: growth rate
bottleneck: bottleneck block
dropout_rate: dropout rate
weight_decay: weight decay factor
grow_nb_filters: flag to decide to allow number of filters to grow
return_concat_list: return the list of feature maps along with the actual output
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
x_list = [x]
channel_list = [nb_filter]
for i in range(nb_layers):
#nb_channels = sum(_exponential_index_fetch(channel_list))
x = _conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay)
x_list.append(x)
fetch_outputs = _exponential_index_fetch(x_list)
x = concatenate(fetch_outputs, axis=concat_axis)
channel_list.append(growth_rate)
if grow_nb_filters:
nb_filter = sum(_exponential_index_fetch(channel_list))
if return_concat_list:
return x, nb_filter, x_list
else:
return x, nb_filter
def make_model(self,model_type='class2'):
##########################Parameters for the model and dataset
#input layers
seq_input = Input(shape=(self.MAXLEN,len(self.dict_aa['A'])))
fix_input = Input(shape=(self.fix_len,))
#set RNN layer
if self.mask0:
seq_input0 = Masking(mask_value=0.0)(seq_input)
else:
seq_input0 = seq_input
rnn0 = recurrent.LSTM(self.node0, activation=self.act_fun, #recurrent_activation=self.act_fun,
use_bias=True, kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal', bias_initializer='zeros',
unit_forget_bias=True, kernel_regularizer=None,
recurrent_regularizer=None, bias_regularizer=None,
activity_regularizer=None, kernel_constraint=None,
recurrent_constraint=None, bias_constraint=None, dropout=self.drop_out_c,
recurrent_dropout=self.drop_r)(seq_input0)
fix0 = Dense(self.fix_node,activation=self.act_fun)(fix_input)
fix0_d = Dropout(self.drop_out_c)(fix0)
merge_layer = concatenate([rnn0,fix0_d])
combine1 = Dense(self.help_nn,activation=self.act_fun)(merge_layer)
combine1_d = Dropout(self.drop_out_c)(combine1)
combine2 = Dense(self.help_nn,activation=self.act_fun)(combine1_d)
combine2_d = Dropout(self.drop_out_c)(combine2)
if model_type == 'regression':
dense0 = Dense(1)(combine2_d)
final_model = Model(inputs = [seq_input,fix_input],outputs = [dense0])
final_model.compile(loss=self.loss_function0,optimizer='adam')
elif 'class' in model_type:
class_n = int(model_type[-1])
dense0 = Dense(class_n,activation='softmax')(combine2_d)
final_model = Model(inputs = [fix_input,seq_input],outputs = [dense0])
final_model.compile(loss=self.loss_function0, optimizer="RMSprop")
json_string = final_model.to_json()
open(self.path_save+self.file_name0+self.out_name+'_model.json', 'w').write(json_string)
return final_model
def nn_architecture_seg_3d(input_shape, pool_size=(2, 2, 2), n_labels=1, initial_learning_rate=0.00001,
depth=3, n_base_filters=16, metrics=dice_coefficient, batch_normalization=True):
inputs = Input(input_shape)
current_layer = inputs
levels = list()
for layer_depth in range(depth):
layer1 = create_convolution_block(input_layer=current_layer, n_filters=n_base_filters * (2**layer_depth),
batch_normalization=batch_normalization)
layer2 = create_convolution_block(input_layer=layer1, n_filters=n_base_filters * (2**layer_depth) * 2,
batch_normalization=batch_normalization)
if layer_depth < depth - 1:
current_layer = MaxPooling3D(pool_size=pool_size)(layer2)
levels.append([layer1, layer2, current_layer])
else:
current_layer = layer2
levels.append([layer1, layer2])
for layer_depth in range(depth - 2, -1, -1):
up_convolution = UpSampling3D(size=pool_size)
concat = concatenate([up_convolution, levels[layer_depth][1]], axis=1)
current_layer = create_convolution_block(n_filters=levels[layer_depth][1]._keras_shape[1],
input_layer=concat, batch_normalization=batch_normalization)
current_layer = create_convolution_block(n_filters=levels[layer_depth][1]._keras_shape[1],
input_layer=current_layer,
batch_normalization=batch_normalization)
final_convolution = Conv3D(n_labels, (1, 1, 1))(current_layer)
act = Activation('sigmoid')(final_convolution)
model = Model(inputs=inputs, outputs=act)
if not isinstance(metrics, list):
metrics = [metrics]
model.compile(optimizer=Adam(lr=initial_learning_rate), loss=dice_coefficient_loss, metrics=metrics)
return model
def __init__(self,
title_word_length,
content_word_length,
title_char_length,
content_char_length,
fs_btm_tw_cw_length,
fs_btm_tc_length,
class_num,
word_embedding_matrix,
char_embedding_matrix,
optimizer_name,
lr,
metrics):
# set attributes
self.title_word_length = title_word_length
self.content_word_length = content_word_length
self.title_char_length = title_char_length
self.content_char_length = content_char_length
self.fs_btm_tw_cw_length = fs_btm_tw_cw_length
self.fs_btm_tc_length = fs_btm_tc_length
self.class_num = class_num
self.word_embedding_matrix = word_embedding_matrix
self.char_embedding_matrix = char_embedding_matrix
self.optimizer_name = optimizer_name
self.lr = lr
self.metrics = metrics
# Placeholder for input (title and content)
title_word_input = Input(shape=(title_word_length,), dtype='int32', name="title_word_input")
cont_word_input = Input(shape=(content_word_length,), dtype='int32', name="content_word_input")
title_char_input = Input(shape=(title_char_length,), dtype='int32', name="title_char_input")
cont_char_input = Input(shape=(content_char_length,), dtype='int32', name="content_char_input")
# Embedding layer
with K.tf.device("/cpu:0"):
word_embedding_layer = Embedding(len(word_embedding_matrix),
256,
weights=[word_embedding_matrix],
trainable=True, name='word_embedding')
title_word_emb = word_embedding_layer(title_word_input)
cont_word_emb = word_embedding_layer(cont_word_input)
char_embedding_layer = Embedding(len(char_embedding_matrix),
256,
weights=[char_embedding_matrix],
trainable=True, name='char_embedding')
title_char_emb = char_embedding_layer(title_char_input)
cont_char_emb = char_embedding_layer(cont_char_input)
# Create a convolution + max pooling layer
title_content_conv = list()
title_content_pool = list()
for win_size in range(1, 8):
# batch_size x doc_len x embed_size
title_content_conv.append(Conv1D(100, win_size, activation='relu', padding='same')(title_word_emb))
title_content_conv.append(Conv1D(100, win_size, activation='relu', padding='same')(cont_word_emb))
title_content_conv.append(Conv1D(100, win_size, activation='relu', padding='same')(title_char_emb))
title_content_conv.append(Conv1D(100, win_size, activation='relu', padding='same')(cont_char_emb))
for conv_out in title_content_conv:
title_content_pool.append(GlobalMaxPooling1D()(conv_out))
title_content_att = list()
for conv_out, pool_out in zip(title_content_conv, title_content_pool):
title_content_att.append(Attention()([ conv_out, pool_out ]))
# add btm_tw_cw features + btm_tc features
fs_btm_tw_cw_input = Input(shape=(fs_btm_tw_cw_length,), dtype='float32', name="fs_btm_tw_cw_input")
fs_btm_tc_input = Input(shape=(fs_btm_tc_length,), dtype='float32', name="fs_btm_tc_input")
fs_btm_raw_features = concatenate([fs_btm_tw_cw_input, fs_btm_tc_input])
fs_btm_emb_features = Dense(1024, activation='relu', name='fs_btm_embedding')(fs_btm_raw_features)
fs_btm_emb_features = Dropout(0.5, name='fs_btm_embedding_dropout')(fs_btm_emb_features)
title_content_pool_features = concatenate(title_content_pool)
title_content_pool_features = Dense(1600, activation='relu', name='title_content_pool_embedding')(title_content_pool_features)
title_content_pool_features = Dropout(0.1, name='title_content_pool_dropout')(title_content_pool_features)
title_content_att_features = concatenate(title_content_att)
title_content_att_features = Dense(1600, activation='relu', name='title_content_att_embedding')(title_content_att_features)
title_content_att_features = Dropout(0.1, name='title_content_att_dropout')(title_content_att_features)
title_content_features = concatenate([title_content_pool_features, title_content_att_features, fs_btm_emb_features])
# Full connection
title_content_features = Dense(3600, activation='relu', name='fs_embedding')(title_content_features)
title_content_features = Dropout(0.5, name='fs_embedding_dropout')(title_content_features)
# Prediction
preds = Dense(class_num, activation='sigmoid', name='prediction')(title_content_features)
self._model = Model([title_word_input,
cont_word_input,
title_char_input,
cont_char_input,
fs_btm_tw_cw_input,
fs_btm_tc_input], preds)
if 'rmsprop' == optimizer_name:
optimizer = optimizers.RMSprop(lr=lr)
elif 'adam' == optimizer_name:
optimizer = optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
else:
optimizer = None
self._model.compile(loss=binary_crossentropy_sum, optimizer=optimizer, metrics=metrics)
self._model.summary()
def train(run_name, start_epoch, stop_epoch, img_w):
# Input Parameters
img_h = 64
words_per_epoch = 16000
val_split = 0.2
val_words = int(words_per_epoch * (val_split))
# Network parameters
conv_filters = 16
kernel_size = (3, 3)
pool_size = 2
time_dense_size = 32
rnn_size = 512
minibatch_size = 32
if K.image_data_format() == 'channels_first':
input_shape = (1, img_w, img_h)
else:
input_shape = (img_w, img_h, 1)
fdir = os.path.dirname(
get_file('wordlists.tgz',
origin='http://www.mythic-ai.com/datasets/wordlists.tgz',
untar=True))
img_gen = TextImageGenerator(
monogram_file=os.path.join(fdir, 'wordlist_mono_clean.txt'),
bigram_file=os.path.join(fdir, 'wordlist_bi_clean.txt'),
minibatch_size=minibatch_size,
img_w=img_w,
img_h=img_h,
downsample_factor=(pool_size ** 2),
val_split=words_per_epoch - val_words)
act = 'relu'
input_data = Input(name='the_input', shape=input_shape, dtype='float32')
inner = Conv2D(conv_filters, kernel_size, padding='same',
activation=act, kernel_initializer='he_normal',
name='conv1')(input_data)
inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
inner = Conv2D(conv_filters, kernel_size, padding='same',
activation=act, kernel_initializer='he_normal',
name='conv2')(inner)
inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
conv_to_rnn_dims = (img_w // (pool_size ** 2),
(img_h // (pool_size ** 2)) * conv_filters)
inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
# cuts down input size going into RNN:
inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
# Two layers of bidirectional GRUs
# GRU seems to work as well, if not better than LSTM:
gru_1 = GRU(rnn_size, return_sequences=True,
kernel_initializer='he_normal', name='gru1')(inner)
gru_1b = GRU(rnn_size, return_sequences=True,
go_backwards=True, kernel_initializer='he_normal',
name='gru1_b')(inner)
gru1_merged = add([gru_1, gru_1b])
gru_2 = GRU(rnn_size, return_sequences=True,
kernel_initializer='he_normal', name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True,
kernel_initializer='he_normal', name='gru2_b')(gru1_merged)
# transforms RNN output to character activations:
inner = Dense(img_gen.get_output_size(), kernel_initializer='he_normal',
name='dense2')(concatenate([gru_2, gru_2b]))
y_pred = Activation('softmax', name='softmax')(inner)
Model(inputs=input_data, outputs=y_pred).summary()
labels = Input(name='the_labels',
shape=[img_gen.absolute_max_string_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(
ctc_lambda_func, output_shape=(1,),
name='ctc')([y_pred, labels, input_length, label_length])
# clipnorm seems to speeds up convergence
sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
if start_epoch > 0:
weight_file = os.path.join(
OUTPUT_DIR,
os.path.join(run_name, 'weights%02d.h5' % (start_epoch - 1)))
model.load_weights(weight_file)
# captures output of softmax so we can decode the output during visualization
test_func = K.function([input_data], [y_pred])
viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
model.fit_generator(
generator=img_gen.next_train(),
steps_per_epoch=(words_per_epoch - val_words) // minibatch_size,
epochs=stop_epoch,
validation_data=img_gen.next_val(),
validation_steps=val_words // minibatch_size,
callbacks=[viz_cb, img_gen],
initial_epoch=start_epoch)
flat1 = Flatten()(pool3)
conv4 = Conv2D(32, kernel_size=4, activation='relu')(pool1)
pool4 = MaxPooling2D(pool_size=(5, 5))(conv4)
drop2 = Dropout(0.1)(pool4)
flat3 = Flatten()(drop2)
conv5 = Conv2D(32, kernel_size=4, activation='relu')(pool1)
pool5 = MaxPooling2D(pool_size=(3, 3))(conv5)
drop3 = Dropout(0.1)(pool5)
conv6 = Conv2D(16, kernel_size=4, activation='relu')(drop2)
pool6 = MaxPooling2D(pool_size=(3, 3))(conv6)
flat2 = Flatten()(pool6)
merge3 = concatenate([flat1, flat2, flat3])
# interpretation layer
hidden1 = Dense(64, activation='relu')(merge3)
drop4 = Dropout(0.2)(hidden1)
# prediction output
output = Dense(1, activation='sigmoid')(drop4)
model = Model(inputs=visible, outputs=output)
# summarize layers
print(model.summary())
# plot graph
plot_model(model, to_file='shared_input_layer.png')
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# dimension of input (and label)
n_x = X_train.shape[1]
n_y = y_train.shape[1]
# nubmer of epochs, batch size
m = 5
n_epoch = 70
## ENCODER ##
# encoder inputs
X = Input(shape=(w * h, ))
cond = Input(shape=(n_y, ))
# merge pixel representation and label
inputs = concatenate([X, cond])
# dense ReLU layer to mu and sigma
l1 = Dense(1500, activation='relu')(inputs)
l2 = Dense(750, activation='relu')(l1)
l3 = Dense(300, activation='relu')(l2)
mu = Dense(n_z, activation='linear')(l3)
log_sigma = Dense(n_z, activation='linear')(l3)
def sample_z(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], n_z),
mean=0.,
stddev=1.)
return z_mean + K.exp(z_log_var / 2) * epsilon
input1 = Input(shape=(3, )) dense1_1 = Dense(10, activation='relu', name='ait1')(input1) dense1_2 = Dense(20, activation='relu', name='ait2')(dense1_1) dense1_3 = Dense(30, activation='relu', name='ait3')(dense1_2) input2 = Input(shape=(3, )) dense2_1 = Dense(10, activation='relu', name='bit1')(input2) dense2_2 = Dense(20, activation='relu', name='bit2')(dense2_1) dense2_3 = Dense(30, activation='relu', name='bit3')(dense2_2) from keras.layers.merge import concatenate #단순병합 merge1 = concatenate([dense1_3, dense2_3]) middle1 = Dense(30, name='mid1')(merge1) middle1_2 = Dense(40, name='mid2')(middle1) middle1_3 = Dense(30, name='mid3')(middle1_2) ###### output 모델 구성 ###### #첫번째 아웃풋 output1 = Dense(30, name='ot1_1')(middle1_3) #middle 위에 합쳤던거 마지막 이름을 가져다 인풋으로 쓴다. output1_2 = Dense(10, name='ot1_2')(output1) output1_3 = Dense(8, name='ot1_3')(output1_2) output1_4 = Dense(3, name='ot1_4')(output1_3) # #두번째 아웃풋 # output2 = Dense(30, name='ot2_1')(middle1_4)
dense1 = Dense(30,activation='relu',name='m1_d3')(dense1)
dense1 = Dense(31,activation='relu',name='m1_d4')(dense1)
# output1 = Dense(3)(dense1)
# model = Model(inputs=input1, outputs=output1)
# 함수형 모델2
input2 = Input(shape=(3,)) #행 무시, 열 우선
dense2 = Dense(5,activation='relu',name='m2_d1')(input2)
dense2 = Dense(100,activation='relu',name='m2_d2')(dense2)
dense2 = Dense(40,activation='relu',name='m2_d3')(dense2)
dense2 = Dense(41,activation='relu',name='m2_d4')(dense2)
# output2 = Dense(3)(dense1)
# 모델1 + 모델2
from keras.layers.merge import concatenate # concatenate 사전적 의미 : 사슬같이 있다 ( 단순병합 가장 기본적인 앙상블방법 )
merge1 = concatenate([dense1,dense2])
midel1 = Dense(70,name='e1_d5')(merge1)
midel1 = Dense(70,name='e1_d6')(midel1)
midel1 = Dense(70,name='e1_d7')(midel1)
#---------------output모델 구성 ----------------#
output1 = Dense(60,name='e1_d8')(midel1)
output1_2 = Dense(60,name='e1_d9')(output1)
output1_3 = Dense(3,name='e1_outpput')(output1_2)
# 함수형 모델의 선언
model = Model(inputs=[input1,input2],
outputs=output1_3)
# model.summary()
def create180180Model(patchSize):
seed = 5
np.random.seed(seed)
input = Input(shape=(1, patchSize[0, 0], patchSize[0, 1]))
out1 = Conv2D(filters=64,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(input)
out2 = Conv2D(filters=64,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='valid',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out1)
out2 = pool2(pool_size=(2, 2), data_format='channels_first')(out2)
out3 = Conv2D(filters=64,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out2)
out4 = Conv2D(filters=64,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out3)
out4 = add([out2, out4])
out4 = pool2(pool_size=(2, 2), data_format='channels_first')(out4)
out_3 = Conv2D(filters=128,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out4)
out_4 = Conv2D(filters=128,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out_3)
out5_1 = Conv2D(filters=32,
kernel_size=(1, 1),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out_4)
out5_2 = Conv2D(filters=32,
kernel_size=(1, 1),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out_4)
out5_2 = Conv2D(filters=128,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out5_2)
out5_3 = Conv2D(filters=32,
kernel_size=(1, 1),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out_4)
out5_3 = Conv2D(filters=128,
kernel_size=(5, 5),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out5_3)
out5_4 = pool2(pool_size=(3, 3),
strides=(1, 1),
padding='same',
data_format='channels_first')(out_4)
out5_4 = Conv2D(filters=128,
kernel_size=(1, 1),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out5_4)
out5 = concatenate(inputs=[out5_1, out5_2, out5_3], axis=1)
out7 = Conv2D(filters=288,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out5)
out7 = add([out5, out7])
out7 = pool2(pool_size=(2, 2), data_format='channels_first')(out7)
sout7 = Conv2D(filters=256,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out7)
out8 = Conv2D(filters=256,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out7)
out9 = Conv2D(filters=256,
kernel_size=(3, 3),
kernel_initializer='he_normal',
weights=None,
padding='same',
strides=(1, 1),
kernel_regularizer=l2(1e-6),
activation='relu')(out8)
out9 = add([sout7, out9])
out9 = pool2(pool_size=(2, 2), data_format='channels_first')(out9)
out10 = Flatten()(out9)
out11 = Dense(units=11,
kernel_initializer='normal',
kernel_regularizer='l2',
activation='softmax')(out10)
cnn = Model(inputs=input, outputs=out11)
return cnn
def build_model(input_layer, start_neurons, DropoutRatio=0.5):
# 101 -> 50
conv1 = Conv2D(start_neurons * 1, (3, 3), activation=None,
padding="same")(input_layer)
conv1 = residual_block(conv1, start_neurons * 1)
conv1 = residual_block(conv1, start_neurons * 1, True)
pool1 = MaxPooling2D((2, 2))(conv1)
pool1 = Dropout(DropoutRatio / 2)(pool1)
# 50 -> 25
conv2 = Conv2D(start_neurons * 2, (3, 3), activation=None,
padding="same")(pool1)
conv2 = residual_block(conv2, start_neurons * 2)
conv2 = residual_block(conv2, start_neurons * 2, True)
pool2 = MaxPooling2D((2, 2))(conv2)
pool2 = Dropout(DropoutRatio)(pool2)
# 25 -> 12
conv3 = Conv2D(start_neurons * 4, (3, 3), activation=None,
padding="same")(pool2)
conv3 = residual_block(conv3, start_neurons * 4)
conv3 = residual_block(conv3, start_neurons * 4, True)
pool3 = MaxPooling2D((2, 2))(conv3)
pool3 = Dropout(DropoutRatio)(pool3)
# 12 -> 6
conv4 = Conv2D(start_neurons * 8, (3, 3), activation=None,
padding="same")(pool3)
conv4 = residual_block(conv4, start_neurons * 8)
conv4 = residual_block(conv4, start_neurons * 8, True)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(DropoutRatio)(pool4)
# Middle
convm = Conv2D(start_neurons * 16, (3, 3), activation=None,
padding="same")(pool4)
convm = residual_block(convm, start_neurons * 16)
convm = residual_block(convm, start_neurons * 16, True)
# 6 -> 12
deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3),
strides=(2, 2),
padding="same")(convm)
uconv4 = concatenate([deconv4, conv4])
uconv4 = Dropout(DropoutRatio)(uconv4)
uconv4 = Conv2D(start_neurons * 8, (3, 3), activation=None,
padding="same")(uconv4)
uconv4 = residual_block(uconv4, start_neurons * 8)
uconv4 = residual_block(uconv4, start_neurons * 8, True)
# 12 -> 25
#deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv4)
deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3),
strides=(2, 2),
padding="valid")(uconv4)
uconv3 = concatenate([deconv3, conv3])
uconv3 = Dropout(DropoutRatio)(uconv3)
uconv3 = Conv2D(start_neurons * 4, (3, 3), activation=None,
padding="same")(uconv3)
uconv3 = residual_block(uconv3, start_neurons * 4)
uconv3 = residual_block(uconv3, start_neurons * 4, True)
# 25 -> 50
deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3),
strides=(2, 2),
padding="same")(uconv3)
uconv2 = concatenate([deconv2, conv2])
uconv2 = Dropout(DropoutRatio)(uconv2)
uconv2 = Conv2D(start_neurons * 2, (3, 3), activation=None,
padding="same")(uconv2)
uconv2 = residual_block(uconv2, start_neurons * 2)
uconv2 = residual_block(uconv2, start_neurons * 2, True)
# 50 -> 101
#deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv2)
deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3),
strides=(2, 2),
padding="valid")(uconv2)
uconv1 = concatenate([deconv1, conv1])
uconv1 = Dropout(DropoutRatio)(uconv1)
uconv1 = Conv2D(start_neurons * 1, (3, 3), activation=None,
padding="same")(uconv1)
uconv1 = residual_block(uconv1, start_neurons * 1)
uconv1 = residual_block(uconv1, start_neurons * 1, True)
#uconv1 = Dropout(DropoutRatio/2)(uconv1)
#output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1)
output_layer_noActi = Conv2D(1, (1, 1), padding="same",
activation=None)(uconv1)
output_layer = Activation('sigmoid')(output_layer_noActi)
return output_layer
# Next, we build a very simple model.
actor = Sequential()
actor.add(Flatten(input_shape=(1,) + env.observation_space.shape))
actor.add(Dense(16))
actor.add(Activation('relu'))
actor.add(Dense(nb_actions))
actor.add(Activation('linear'))
print(actor.summary())
plot_model(actor, to_file='actor.png', show_shapes=True)
action_input = Input(shape=(nb_actions,), name='action_input')
observation_input = Input(shape=(1,) + env.observation_space.shape, name='observation_input')
flattened_observation = Flatten()(observation_input)
x = concatenate([action_input, flattened_observation])
x = Dense(32)(x)
x = Activation('relu')(x)
x = Dense(1)(x)
x = Activation('linear')(x)
critic = Model(inputs=[action_input, observation_input], outputs=x)
print(critic.summary())
plot_model(critic, to_file='critic.png', show_shapes=True)
# # Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
# # even the metrics!
memory = SequentialMemory(limit=10000, window_length=1)
random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=0.15, mu=0., sigma=.3)
agent = DDPGAgent(nb_actions=nb_actions, actor=actor, critic=critic, critic_action_input=action_input,
memory=memory, nb_steps_warmup_critic=100, nb_steps_warmup_actor=100,
c2 = Conv2D(16, (3, 3), activation='relu', padding='same') (c2) p2 = MaxPooling2D((2, 2)) (c2) c3 = Conv2D(32, (3, 3), activation='relu', padding='same') (p2) c3 = Conv2D(32, (3, 3), activation='relu', padding='same') (c3) p3 = MaxPooling2D((2, 2)) (c3) c4 = Conv2D(64, (3, 3), activation='relu', padding='same') (p3) c4 = Conv2D(64, (3, 3), activation='relu', padding='same') (c4) p4 = MaxPooling2D(pool_size=(2, 2)) (c4) c5 = Conv2D(128, (3, 3), activation='relu', padding='same') (p4) c5 = Conv2D(128, (3, 3), activation='relu', padding='same') (c5) u6 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same') (c5) u6 = concatenate([u6, c4]) c6 = Conv2D(64, (3, 3), activation='relu', padding='same') (u6) c6 = Conv2D(64, (3, 3), activation='relu', padding='same') (c6) u7 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same') (c6) u7 = concatenate([u7, c3]) c7 = Conv2D(32, (3, 3), activation='relu', padding='same') (u7) c7 = Conv2D(32, (3, 3), activation='relu', padding='same') (c7) u8 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same') (c7) u8 = concatenate([u8, c2]) c8 = Conv2D(16, (3, 3), activation='relu', padding='same') (u8) c8 = Conv2D(16, (3, 3), activation='relu', padding='same') (c8) u9 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same') (c8) u9 = concatenate([u9, c1], axis=3)
def Model_Dense_Softmax(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
sourcepossize,
pos_W,
pos_emd_dim,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(Conv1D(50, 3, activation='relu',
padding='same'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
# char_macpool = Dropout(0.5)(char_macpool)
# !!!!!!!!!!!!!!
char_macpool = Dropout(0.25)(char_macpool)
word_embedding = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=True,
weights=[source_W])(word_input)
word_embedding_dropout = Dropout(0.5)(word_embedding)
embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1)
Dense1 = TimeDistributed(Dense(400, activation='tanh'))(embedding)
Dense1 = Dropout(0.5)(Dense1)
Dense2 = TimeDistributed(Dense(200, activation='tanh'))(Dense1)
Dense2 = Dropout(0.3)(Dense2)
Dense3 = TimeDistributed(Dense(100, activation='tanh'))(Dense2)
Dense3 = Dropout(0.2)(Dense3)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(Dense3)
# TimeD = Dropout(0.5)(TimeD)
# !!!!!!!!!!!!!!!delete dropout
model = Activation('softmax')(TimeD)
# crflayer = CRF(targetvocabsize+1, sparse_target=False)
# model = crflayer(TimeD)
Models = Model([word_input, char_input], [model])
Models.compile(loss=loss,
optimizer=optimizers.RMSprop(lr=0.001),
metrics=['acc'])
# Models.compile(loss=crflayer.loss_function, optimizer=optimizers.RMSprop(lr=0.001), metrics=[crflayer.accuracy])
return Models
def Model_BiLSTM_X2_CRF(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(Conv1D(50, 3, activation='relu',
padding='same'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
# char_macpool = Dropout(0.5)(char_macpool)
# !!!!!!!!!!!!!!
char_macpool = Dropout(0.25)(char_macpool)
word_embedding = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=True,
trainable=True,
weights=[source_W])(word_input)
word_embedding_dropout = Dropout(0.5)(word_embedding)
embedding = concatenate([word_embedding_dropout, char_macpool], axis=-1)
BiLSTM = Bidirectional(LSTM(hidden_dim, return_sequences=True),
merge_mode='concat')(embedding)
BiLSTM = BatchNormalization(axis=1)(BiLSTM)
BiLSTM_dropout = Dropout(0.5)(BiLSTM)
BiLSTM2 = Bidirectional(LSTM(hidden_dim // 2, return_sequences=True),
merge_mode='concat')(BiLSTM_dropout)
BiLSTM_dropout2 = Dropout(0.5)(BiLSTM2)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(BiLSTM_dropout2)
# TimeD = TimeDistributed(Dense(int(hidden_dim / 2)))(BiLSTM_dropout)
# TimeD = Dropout(0.5)(TimeD)
# !!!!!!!!!!!!!!!delete dropout
# model = Activation('softmax')(TimeD)
crflayer = CRF(targetvocabsize + 1, sparse_target=False)
model = crflayer(TimeD) #0.8746633147782367
# # model = crf(BiLSTM_dropout)#0.870420501714492
Models = Model([word_input, char_input], [model])
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
# Models.compile(loss=crflayer.loss_function, optimizer='adam', metrics=[crflayer.accuracy])
Models.compile(loss=crflayer.loss_function,
optimizer=optimizers.RMSprop(lr=0.001),
metrics=[crflayer.accuracy])
return Models
from keras.layers import Dense, Input, LSTM
input1 = Input(shape=(25, ))
dense1 = Dense(64)(input1)
dense1 = Dense(64)(dense1)
dense1 = Dense(64)(dense1)
output1 = Dense(32)(dense1)
input2 = Input(shape=(25, ))
dense2 = Dense(64)(input2)
dense2 = Dense(64)(dense2)
dense2 = Dense(64)(dense2)
output2 = Dense(32)(dense2)
from keras.layers.merge import concatenate
merge = concatenate([output1, output2])
output3 = Dense(1)(merge)
model = Model(inputs=[input1, input2], outputs=output3)
model.compile(loss='mse', optimizer='adam', metrics=['mse'])
from keras.callbacks import EarlyStopping
early_stopping = EarlyStopping(patience=20)
model.fit([x1_train_scaled, x2_train_scaled],
y1_train,
validation_split=0.2,
epochs=100,
batch_size=1,
verbose=1,
callbacks=[early_stopping])
def unet_model(im_height, im_width, im_chan):
input_img = Input((im_height, im_width, im_chan), name='img')
res1 = resnet_block(input_img, filters=4, strides=1)
res1 = resnet_block(res1, filters=4, strides=1)
res2 = resnet_block(res1, filters=8, strides=2)
res2 = resnet_block(res2, filters=8, strides=1)
res3 = resnet_block(res2, filters=16, strides=2)
res3 = resnet_block(res3, filters=16, strides=1)
res4 = resnet_block(res3, filters=32, strides=2)
res4 = resnet_block(res4, filters=32, strides=1)
res5 = resnet_block(res4, filters=64, strides=2)
res5 = resnet_block(res5, filters=64, strides=1)
mid = convlayer(res5,
filters=128,
kernel_size=3,
strides=2,
activation='relu')
mid = convlayer(mid,
filters=128,
kernel_size=3,
strides=1,
activation='relu')
up1 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(mid)
up1 = concatenate([up1, res5])
c1 = convlayer(up1,
filters=64,
kernel_size=3,
strides=1,
activation='relu')
c1 = convlayer(c1, filters=64, kernel_size=3, strides=1, activation='relu')
up2 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c1)
up2 = concatenate([up2, res4])
c2 = convlayer(up2,
filters=32,
kernel_size=3,
strides=1,
activation='relu')
c2 = convlayer(c2, filters=32, kernel_size=3, strides=1, activation='relu')
up3 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c2)
up3 = concatenate([up3, res3])
c3 = convlayer(up3,
filters=16,
kernel_size=3,
strides=1,
activation='relu')
c3 = convlayer(c3, filters=16, kernel_size=3, strides=1, activation='relu')
up4 = Conv2DTranspose(8, (2, 2), strides=(2, 2), padding='same')(c3)
up4 = concatenate([up4, res2])
c4 = convlayer(up4, filters=8, kernel_size=3, strides=1, activation='relu')
c4 = convlayer(c4, filters=8, kernel_size=3, strides=1, activation='relu')
up5 = Conv2DTranspose(4, (2, 2), strides=(2, 2), padding='same')(c4)
up5 = concatenate([up5, res1], axis=3)
c5 = convlayer(up5, filters=4, kernel_size=3, strides=1, activation='relu')
c5 = convlayer(c5, filters=4, kernel_size=3, strides=1, activation='relu')
outputs = convlayer(c5,
filters=1,
kernel_size=1,
strides=1,
activation='sigmoid')
model = Model(inputs=[input_img], outputs=[outputs])
return model
wide = Conv1D(64, 5, activation=act, padding=pad)(wide)
wide = MaxPooling1D(5, strides=strmp)(wide)
wide = Conv1D(128, 5, activation=act, padding=pad)(wide)
wide = Conv1D(128, 5, activation=act, padding=pad)(wide)
wide = MaxPooling1D(5, strides=strmp)(wide)
wide = Conv1D(256, 5, activation=act, padding=pad)(wide)
wide = Conv1D(256, 5, activation=act, padding=pad)(wide)
wide = MaxPooling1D(5, strides=strmp)(wide)
wide = Flatten()(wide)
local = Conv1D(16, 5, activation=act, padding=pad)(inlocal)
local = Conv1D(16, 5, activation=act, padding=pad)(local)
local = MaxPooling1D(7, strides=strmp)(local)
local = Conv1D(32, 5, activation=act, padding=pad)(local)
local = Conv1D(32, 5, activation=act, padding=pad)(local)
local = MaxPooling1D(7, strides=strmp)(local)
local = Flatten()(local)
concat = concatenate([wide, local], axis=1)
concat = Dense(units=512, activation=act)(concat)
concat = Dense(units=512, activation=act)(concat)
concat = Dense(units=512, activation=act)(concat)
concat = Dense(units=512, activation=act)(concat)
concat = Dense(units=1, activation="sigmoid")(concat)
model = Model(inputs=[inwide, inlocal], outputs=concat)
model.compile(optimizer="adam", loss="binary_crossentropy")
print(model.summary())
model.fit([Xw, X], lab, epochs=args.e[0], validation_split=0.2)
model.save(output)
# topic_sum = o4(topic_sum)
# # fifth match layer
# match = dot([topic_sum, wt_emb], axes=(2, 2))
# joint_match = add([represent_mu, match])
# joint_match = f5(joint_match)
# topic_sum = add([joint_match, x])
# topic_sum = o5(topic_sum)
x = Reshape((MAX_SEQ_LEN, TOPIC_EMB_DIM, 1))(topic_sum)
x0 = conv_0(x)
x1 = conv_1(x)
x2 = conv_2(x)
mp0 = MaxPool2D(pool_size=(MAX_SEQ_LEN - filter_sizes[0] + 1, 1), strides=(1, 1), padding='valid')(x0)
mp1 = MaxPool2D(pool_size=(MAX_SEQ_LEN - filter_sizes[1] + 1, 1), strides=(1, 1), padding='valid')(x1)
mp2 = MaxPool2D(pool_size=(MAX_SEQ_LEN - filter_sizes[2] + 1, 1), strides=(1, 1), padding='valid')(x2)
out = concatenate([mp0, mp1, mp2], axis=1)
out = Dropout(0.05)(Flatten()(out))
cls_out = s2(out)
def kl_loss(x_true, x_decoded):
kl_term = - 0.5 * K.sum(
1 - K.square(z_mean) + z_log_var - K.exp(z_log_var),
axis=-1)
return kl_term
def nnl_loss(x_true, x_decoder):
nnl_term = - K.sum(x_true * K.log(x_decoder + 1e-32), axis=-1)
return nnl_term
kl_strength = K.variable(1.0)
def _main(args):
config_path = os.path.expanduser(args.config_path)
weights_path = os.path.expanduser(args.weights_path)
assert config_path.endswith('.cfg'), '{} is not a .cfg file'.format(
config_path)
assert weights_path.endswith(
'.weights'), '{} is not a .weights file'.format(weights_path)
output_path = os.path.expanduser(args.output_path)
assert output_path.endswith(
'.h5'), 'output path {} is not a .h5 file'.format(output_path)
output_root = os.path.splitext(output_path)[0]
# Load weights and config.
print('Loading weights.')
weights_file = open(weights_path, 'rb')
weights_header = np.ndarray(
shape=(4, ), dtype='int32', buffer=weights_file.read(16))
print('Weights Header: ', weights_header)
# TODO: Check transpose flag when implementing fully connected layers.
# transpose = (weight_header[0] > 1000) or (weight_header[1] > 1000)
print('Parsing Darknet config.')
unique_config_file = unique_config_sections(config_path)
cfg_parser = configparser.ConfigParser()
cfg_parser.read_file(unique_config_file)
print('Creating Keras model.')
if args.fully_convolutional:
image_height, image_width = None, None
else:
image_height = int(cfg_parser['net_0']['height'])
image_width = int(cfg_parser['net_0']['width'])
prev_layer = Input(shape=(image_height, image_width, 3))
all_layers = [prev_layer]
weight_decay = float(cfg_parser['net_0']['decay']
) if 'net_0' in cfg_parser.sections() else 5e-4
count = 0
for section in cfg_parser.sections():
print('Parsing section {}'.format(section))
if section.startswith('convolutional'):
filters = int(cfg_parser[section]['filters'])
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
pad = int(cfg_parser[section]['pad'])
activation = cfg_parser[section]['activation']
batch_normalize = 'batch_normalize' in cfg_parser[section]
# padding='same' is equivalent to Darknet pad=1
padding = 'same' if pad == 1 else 'valid'
# Setting weights.
# Darknet serializes convolutional weights as:
# [bias/beta, [gamma, mean, variance], conv_weights]
prev_layer_shape = K.int_shape(prev_layer)
# TODO: This assumes channel last dim_ordering.
weights_shape = (size, size, prev_layer_shape[-1], filters)
darknet_w_shape = (filters, weights_shape[2], size, size)
weights_size = np.product(weights_shape)
print('conv2d', 'bn'
if batch_normalize else ' ', activation, weights_shape)
conv_bias = np.ndarray(
shape=(filters, ),
dtype='float32',
buffer=weights_file.read(filters * 4))
count += filters
if batch_normalize:
bn_weights = np.ndarray(
shape=(3, filters),
dtype='float32',
buffer=weights_file.read(filters * 12))
count += 3 * filters
# TODO: Keras BatchNormalization mistakenly refers to var
# as std.
bn_weight_list = [
bn_weights[0], # scale gamma
conv_bias, # shift beta
bn_weights[1], # running mean
bn_weights[2] # running var
]
conv_weights = np.ndarray(
shape=darknet_w_shape,
dtype='float32',
buffer=weights_file.read(weights_size * 4))
count += weights_size
# DarkNet conv_weights are serialized Caffe-style:
# (out_dim, in_dim, height, width)
# We would like to set these to Tensorflow order:
# (height, width, in_dim, out_dim)
# TODO: Add check for Theano dim ordering.
conv_weights = np.transpose(conv_weights, [2, 3, 1, 0])
conv_weights = [conv_weights] if batch_normalize else [
conv_weights, conv_bias
]
# Handle activation.
act_fn = None
if activation == 'leaky':
pass # Add advanced activation later.
elif activation != 'linear':
raise ValueError(
'Unknown activation function `{}` in section {}'.format(
activation, section))
# Create Conv2D layer
conv_layer = (Conv2D(
filters, (size, size),
strides=(stride, stride),
kernel_regularizer=l2(weight_decay),
use_bias=not batch_normalize,
weights=conv_weights,
activation=act_fn,
padding=padding))(prev_layer)
if batch_normalize:
conv_layer = (BatchNormalization(
weights=bn_weight_list))(conv_layer)
prev_layer = conv_layer
if activation == 'linear':
all_layers.append(prev_layer)
elif activation == 'leaky':
act_layer = LeakyReLU(alpha=0.1)(prev_layer)
prev_layer = act_layer
all_layers.append(act_layer)
elif section.startswith('maxpool'):
size = int(cfg_parser[section]['size'])
stride = int(cfg_parser[section]['stride'])
all_layers.append(
MaxPooling2D(
padding='same',
pool_size=(size, size),
strides=(stride, stride))(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('avgpool'):
if cfg_parser.items(section) != []:
raise ValueError('{} with params unsupported.'.format(section))
all_layers.append(GlobalAveragePooling2D()(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('route'):
ids = [int(i) for i in cfg_parser[section]['layers'].split(',')]
layers = [all_layers[i] for i in ids]
if len(layers) > 1:
print('Concatenating route layers:', layers)
concatenate_layer = concatenate(layers)
all_layers.append(concatenate_layer)
prev_layer = concatenate_layer
else:
skip_layer = layers[0] # only one layer to route
all_layers.append(skip_layer)
prev_layer = skip_layer
elif section.startswith('reorg'):
block_size = int(cfg_parser[section]['stride'])
assert block_size == 2, 'Only reorg with stride 2 supported.'
all_layers.append(
Lambda(
space_to_depth_x2,
output_shape=space_to_depth_x2_output_shape,
name='space_to_depth_x2')(prev_layer))
prev_layer = all_layers[-1]
elif section.startswith('region'):
with open('{}_anchors.txt'.format(output_root), 'w') as f:
#print(cfg_parser[section]['anchors'], file=f)
f.write(cfg_parser[section]['anchors'])
elif (section.startswith('net') or section.startswith('cost') or
section.startswith('softmax')):
pass # Configs not currently handled during model definition.
else:
raise ValueError(
'Unsupported section header type: {}'.format(section))
# Create and save model.
model = Model(inputs=all_layers[0], outputs=all_layers[-1])
print(model.summary())
model.save('{}'.format(output_path))
print('Saved Keras model to {}'.format(output_path))
# Check to see if all weights have been read.
remaining_weights = len(weights_file.read()) / 4
weights_file.close()
print('Read {} of {} from Darknet weights.'.format(count, count +
remaining_weights))
if remaining_weights > 0:
print('Warning: {} unused weights'.format(remaining_weights))
if args.plot_model:
plot(model, to_file='{}.png'.format(output_root), show_shapes=True)
print('Saved model plot to {}.png'.format(output_root))
def resnet_v2_stem(input):
'''The stem of the pure Inception-v4 and Inception-ResNet-v2 networks. This is input part of those networks.'''
# Input shape is 299 * 299 * 3 (Tensorflow dimension ordering)
x = Conv2D(32, (3, 3),
kernel_regularizer=l2(0.0002),
activation="relu",
strides=(2, 2))(input) # 149 * 149 * 32
x = Conv2D(32, (3, 3), kernel_regularizer=l2(0.0002),
activation="relu")(x) # 147 * 147 * 32
x = Conv2D(64, (3, 3),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(x) # 147 * 147 * 64
x1 = MaxPooling2D((3, 3), strides=(2, 2))(x)
x2 = Conv2D(96, (3, 3),
kernel_regularizer=l2(0.0002),
activation="relu",
strides=(2, 2))(x)
x = concatenate([x1, x2], axis=3) # 73 * 73 * 160
x1 = Conv2D(64, (1, 1),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(x)
x1 = Conv2D(96, (3, 3), kernel_regularizer=l2(0.0002),
activation="relu")(x1)
x2 = Conv2D(64, (1, 1),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(x)
x2 = Conv2D(64, (7, 1),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(x2)
x2 = Conv2D(64, (1, 7),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(x2)
x2 = Conv2D(96, (3, 3),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="valid")(x2)
x = concatenate([x1, x2], axis=3) # 71 * 71 * 192
x1 = Conv2D(192, (3, 3),
kernel_regularizer=l2(0.0002),
activation="relu",
strides=(2, 2))(x)
x2 = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = concatenate([x1, x2], axis=3) # 35 * 35 * 384
x = BatchNormalization(axis=3)(x)
x = Activation("relu")(x)
return x
def Unet(input_img, n_filters=32, dropout=0.4, batch_norm=True):
c1 = conv_block(input_img, n_filters, 3, batch_norm)
p1 = Conv2D(n_filters,
kernel_size=(3, 3),
strides=2,
padding='same',
kernel_initializer='he_normal')(c1)
#p1 = Dropout(dropout)(p1)
c2 = conv_block(p1, n_filters * 2, 3, batch_norm)
p2 = Conv2D(n_filters,
kernel_size=(3, 3),
strides=2,
padding='same',
kernel_initializer='he_normal')(c2)
#p2 = Dropout(dropout)(p2)
c3 = conv_block(p2, n_filters * 4, 3, batch_norm)
p3 = Conv2D(n_filters,
kernel_size=(3, 3),
strides=2,
padding='same',
kernel_initializer='he_normal')(c3)
#p3 = Dropout(dropout)(p3)
c4 = conv_block(p3, n_filters * 8, 3, batch_norm)
p4 = Conv2D(n_filters,
kernel_size=(3, 3),
strides=2,
padding='same',
kernel_initializer='he_normal')(c4)
#p4 = Dropout(dropout)(p4)
c5 = conv_block(p4, n_filters * 16, 3, batch_norm)
u6 = Conv2DTranspose(n_filters * 8, (3, 3), strides=(2, 2),
padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = conv_block(u6, n_filters * 8, 3, batch_norm)
#c6 = Dropout(dropout)(c6)
u7 = Conv2DTranspose(n_filters * 4, (3, 3), strides=(2, 2),
padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = conv_block(u7, n_filters * 4, 3, batch_norm)
#c7 = Dropout(dropout)(c7)
u8 = Conv2DTranspose(n_filters * 2, (3, 3), strides=(2, 2),
padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = conv_block(u8, n_filters * 2, 3, batch_norm)
#c8 = Dropout(dropout)(c8)
u9 = Conv2DTranspose(n_filters, (3, 3), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1])
c9 = conv_block(u9, n_filters, 3, batch_norm)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=input_img, outputs=outputs)
return model
def inceptionresnetv2(input,
dropout_keep_prob=0.8,
num_classes=1000,
is_training=True,
scope='InceptionResnetV2',
supermd=False):
'''Creates the Inception_ResNet_v2 network.'''
with tf.variable_scope(scope, 'InceptionResnetV2', [input]):
# Input shape is 299 * 299 * 3
x = resnet_v2_stem(input) # Output: 35 * 35 * 256
# 5 x Inception A
for i in range(5):
x = inception_resnet_v2_A(x)
# Output: 35 * 35 * 256
# Reduction A
x = reduction_resnet_A(x, k=256, l=256, m=384,
n=384) # Output: 17 * 17 * 896
# 10 x Inception B
for i in range(10):
x = inception_resnet_v2_B(x)
# Output: 17 * 17 * 896
# auxiliary
loss2_ave_pool = AveragePooling2D(pool_size=(5, 5),
strides=(3, 3),
name='loss2/ave_pool')(x)
loss2_conv_a = Conv2D(128, (1, 1),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(loss2_ave_pool)
loss2_conv_b = Conv2D(768, (5, 5),
kernel_regularizer=l2(0.0002),
activation="relu",
padding="same")(loss2_conv_a)
loss2_conv_b = BatchNormalization(axis=3)(loss2_conv_b)
loss2_conv_b = Activation('relu')(loss2_conv_b)
loss2_flat = Flatten()(loss2_conv_b)
loss2_fc = Dense(1024,
activation='relu',
name='loss2/fc',
kernel_regularizer=l2(0.0002))(loss2_flat)
loss2_drop_fc = Dropout(dropout_keep_prob)(loss2_fc,
training=is_training)
if supermd:
loss2_classifier_a = Dense(
4, name='loss2/classifiera',
kernel_regularizer=l2(0.0002))(loss2_drop_fc)
loss2_classifier_a, loss2_classifier_a2 = tf.split(
loss2_classifier_a, [1, 3], 1)
loss2_classifier_a2 = Activation('relu')(loss2_classifier_a2)
loss2_classifier_b = Dense(
3, name='loss2/classifierb',
kernel_regularizer=l2(0.0002))(loss2_classifier_a2)
loss2_classifier_b, loss2_classifier_b2 = tf.split(
loss2_classifier_b, [1, 2], 1)
loss2_classifier_b2 = Activation('relu')(loss2_classifier_b2)
loss2_classifier_c = Dense(
2, name='loss2/classifierc',
kernel_regularizer=l2(0.0002))(loss2_classifier_b2)
loss2_classifier = concatenate(
[loss2_classifier_a, loss2_classifier_b, loss2_classifier_c],
axis=-1)
else:
loss2_classifier = Dense(
num_classes,
name='loss2/classifier',
kernel_regularizer=l2(0.0002))(loss2_drop_fc)
# Reduction B
x = reduction_resnet_v2_B(x) # Output: 8 * 8 * 1792
# 5 x Inception C
for i in range(5):
x = inception_resnet_v2_C(x)
# Output: 8 * 8 * 1792
net = x
# Average Pooling
x = GlobalAveragePooling2D(name='avg_pool')(x) # Output: 1792
pool5_drop_10x10_s1 = Dropout(dropout_keep_prob)(x,
training=is_training)
if supermd:
loss3_classifier_aw = Dense(4,
name='loss3/classifiera',
kernel_regularizer=l2(0.0002))
loss3_classifier_a = loss3_classifier_aw(pool5_drop_10x10_s1)
loss3_classifier_a, loss3_classifier_a2 = tf.split(
loss3_classifier_a, [1, 3], 1)
loss3_classifier_a2 = Activation('relu')(loss3_classifier_a2)
loss3_classifier_bw = Dense(3,
name='loss3/classifierb',
kernel_regularizer=l2(0.0002))
loss3_classifier_b = loss3_classifier_bw(loss3_classifier_a2)
loss3_classifier_b, loss3_classifier_b2 = tf.split(
loss3_classifier_b, [1, 2], 1)
loss3_classifier_b2 = Activation('relu')(loss3_classifier_b2)
loss3_classifier_cw = Dense(2,
name='loss3/classifierc',
kernel_regularizer=l2(0.0002))
loss3_classifier_c = loss3_classifier_cw(loss3_classifier_b2)
loss3_classifier = concatenate(
[loss3_classifier_a, loss3_classifier_b, loss3_classifier_c],
axis=-1)
w_variables = loss3_classifier_aw.get_weights()
else:
loss3_classifier_w = Dense(num_classes,
name='loss3/classifier',
kernel_regularizer=l2(0.0002))
loss3_classifier = loss3_classifier_w(pool5_drop_10x10_s1)
w_variables = loss3_classifier_w.get_weights()
logits = tf.cond(
tf.equal(is_training, tf.constant(True)),
lambda: tf.add(loss3_classifier,
tf.scalar_mul(tf.constant(0.3), loss2_classifier)),
lambda: loss3_classifier)
return logits, net, tf.convert_to_tensor(w_variables[0])
def make_yolov3_model():
input_image = Input(shape=(None, None, 3))
# Layer 0 => 4
x = _conv_block(input_image, [{
'filter': 32,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 0
}, {
'filter': 64,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 1
}, {
'filter': 32,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 2
}, {
'filter': 64,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 3
}])
# Layer 5 => 8
x = _conv_block(x, [{
'filter': 128,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 5
}, {
'filter': 64,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 6
}, {
'filter': 128,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 7
}])
# Layer 9 => 11
x = _conv_block(x, [{
'filter': 64,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 9
}, {
'filter': 128,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 10
}])
# Layer 12 => 15
x = _conv_block(x, [{
'filter': 256,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 12
}, {
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 13
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 14
}])
# Layer 16 => 36
for i in range(7):
x = _conv_block(x, [{
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 16 + i * 3
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 17 + i * 3
}])
skip_36 = x
# Layer 37 => 40
x = _conv_block(x, [{
'filter': 512,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 37
}, {
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 38
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 39
}])
# Layer 41 => 61
for i in range(7):
x = _conv_block(x, [{
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 41 + i * 3
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 42 + i * 3
}])
skip_61 = x
# Layer 62 => 65
x = _conv_block(x, [{
'filter': 1024,
'kernel': 3,
'stride': 2,
'bnorm': True,
'leaky': True,
'layer_idx': 62
}, {
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 63
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 64
}])
# Layer 66 => 74
for i in range(3):
x = _conv_block(x, [{
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 66 + i * 3
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 67 + i * 3
}])
# Layer 75 => 79
x = _conv_block(x, [{
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 75
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 76
}, {
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 77
}, {
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 78
}, {
'filter': 512,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 79
}],
skip=False)
# Layer 80 => 82
yolo_82 = _conv_block(x, [{
'filter': 1024,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 80
}, {
'filter': 255,
'kernel': 1,
'stride': 1,
'bnorm': False,
'leaky': False,
'layer_idx': 81
}],
skip=False)
# Layer 83 => 86
x = _conv_block(x, [{
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 84
}],
skip=False)
x = UpSampling2D(2)(x)
x = concatenate([x, skip_61])
# Layer 87 => 91
x = _conv_block(x, [{
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 87
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 88
}, {
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 89
}, {
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 90
}, {
'filter': 256,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 91
}],
skip=False)
# Layer 92 => 94
yolo_94 = _conv_block(x, [{
'filter': 512,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 92
}, {
'filter': 255,
'kernel': 1,
'stride': 1,
'bnorm': False,
'leaky': False,
'layer_idx': 93
}],
skip=False)
# Layer 95 => 98
x = _conv_block(x, [{
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 96
}],
skip=False)
x = UpSampling2D(2)(x)
x = concatenate([x, skip_36])
# Layer 99 => 106
yolo_106 = _conv_block(x, [{
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 99
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 100
}, {
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 101
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 102
}, {
'filter': 128,
'kernel': 1,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 103
}, {
'filter': 256,
'kernel': 3,
'stride': 1,
'bnorm': True,
'leaky': True,
'layer_idx': 104
}, {
'filter': 255,
'kernel': 1,
'stride': 1,
'bnorm': False,
'leaky': False,
'layer_idx': 105
}],
skip=False)
model = Model(input_image, [yolo_82, yolo_94, yolo_106])
return model
def get_model(K, K0, lw=1e-4, lw1=1e-4, lr=1e-3, act='relu', batchnorm=False):
embedding_inputs = []
embedding_outputs = []
for i in range(len(embedding_features) - 3):
val_bound = 0.0 if i == 0 else 0.005
tmp_input = Input(shape=(1,), dtype='int32', name=embedding_features[i]+'_input')
tmp_embeddings = Embedding(int(train_embeddings[i].max()+1),
K if i == 0 else K0,
embeddings_initializer=RandomUniform(minval=-val_bound, maxval=val_bound),
embeddings_regularizer=l2(lw),
input_length=1,
trainable=True,
name=embedding_features[i]+'_embeddings')(tmp_input)
tmp_embeddings = Flatten(name=embedding_features[i]+'_flatten')(tmp_embeddings)
embedding_inputs.append(tmp_input)
embedding_outputs.append(tmp_embeddings)
song_id_input = Input(shape=(1,), dtype='int32', name='song_id_input')
before_song_id_input = Input(shape=(1,), dtype='int32', name='before_song_id_input')
after_song_id_input = Input(shape=(1,), dtype='int32', name='after_song_id_input')
embedding_inputs += [song_id_input, before_song_id_input, after_song_id_input]
genre_inputs = []
genre_outputs = []
genre_embeddings = Embedding(int(np.max(train_genre)+1),
K0,
embeddings_initializer=RandomUniform(minval=-0.05, maxval=0.05),
embeddings_regularizer=l2(lw),
input_length=1,
trainable=True,
name='genre_embeddings')
for i in range(len(genre_features)):
tmp_input = Input(shape=(1,), dtype='int32', name=genre_features[i]+'_input')
tmp_embeddings = genre_embeddings(tmp_input)
tmp_embeddings = Flatten(name=genre_features[i]+'_flatten')(tmp_embeddings)
genre_inputs.append(tmp_input)
genre_outputs.append(tmp_embeddings)
usr_input = Embedding(usr_feat.shape[0],
usr_feat.shape[1],
weights=[usr_feat],
input_length=1,
trainable=False,
name='usr_feat')(embedding_inputs[0])
usr_input = Flatten(name='usr_feat_flatten')(usr_input)
song_input = Embedding(song_feat.shape[0],
song_feat.shape[1],
weights=[song_feat],
input_length=1,
trainable=False,
name='song_feat')(song_id_input)
song_input = Flatten(name='song_feat_flatten')(song_input)
usr_component_input = Embedding(usr_component.shape[0],
usr_component.shape[1],
weights=[usr_component],
input_length=1,
trainable=False,
name='usr_component')(embedding_inputs[0])
usr_component_input = Flatten(name='usr_component_flatten')(usr_component_input)
song_component_embeddings = Embedding(song_component.shape[0],
song_component.shape[1],
weights=[song_component],
input_length=1,
trainable=False,
name='song_component')
song_component_input = song_component_embeddings(song_id_input)
song_component_input = Flatten(name='song_component_flatten')(song_component_input)
before_song_component_input = song_component_embeddings(before_song_id_input)
before_song_component_input = Flatten(name='before_song_component_flatten')(before_song_component_input)
after_song_component_input = song_component_embeddings(after_song_id_input)
after_song_component_input = Flatten(name='after_song_component_flatten')(after_song_component_input)
usr_artist_component_input = Embedding(usr_artist_component.shape[0],
usr_artist_component.shape[1],
weights=[usr_artist_component],
input_length=1,
trainable=False,
name='usr_artist_component')(embedding_inputs[0])
usr_artist_component_input = Flatten(name='usr_artist_component_flatten')(usr_artist_component_input)
song_artist_component_embeddings = Embedding(song_artist_component.shape[0],
song_artist_component.shape[1],
weights=[song_artist_component],
input_length=1,
trainable=False,
name='song_artist_component')
song_artist_component_input = song_artist_component_embeddings(song_id_input)
song_artist_component_input = Flatten(name='song_artist_component_flatten')(song_artist_component_input)
before_song_artist_component_input = song_artist_component_embeddings(before_song_id_input)
before_song_artist_component_input = Flatten(name='before_song_artist_component_flatten')(before_song_artist_component_input)
after_song_artist_component_input = song_artist_component_embeddings(after_song_id_input)
after_song_artist_component_input = Flatten(name='after_song_artist_component_flatten')(after_song_artist_component_input)
context_input = Input(shape=(len(context_features),), name='context_feat')
# basic profiles
usr_profile = concatenate(embedding_outputs[1:4]+[usr_input, \
usr_component_input, usr_artist_component_input], name='usr_profile')
song_profile = concatenate(embedding_outputs[4:7]+genre_outputs+[song_input, \
song_component_input, song_artist_component_input], name='song_profile')
multiply_component = dot([usr_component_input, song_component_input], axes=1, name='component_dot')
multiply_artist_component = dot([usr_artist_component_input, \
song_artist_component_input], axes=1, name='artist_component_dot')
multiply_before_song = dot([before_song_component_input, song_component_input], \
normalize=True, axes=1, name='before_component_dot')
multiply_after_song = dot([after_song_component_input, song_component_input], \
normalize=True, axes=1, name='after_component_dot')
multiply_before_artist = dot([before_song_artist_component_input, song_artist_component_input], \
normalize=True, axes=1, name='before_artist_component_dot')
multiply_after_artist = dot([after_song_artist_component_input, song_artist_component_input], \
normalize=True, axes=1, name='after_artist_component_dot')
context_profile = concatenate(embedding_outputs[7:]+[context_input, \
multiply_component, multiply_artist_component, before_song_component_input, \
after_song_component_input, before_song_artist_component_input, \
after_song_artist_component_input, multiply_before_song, multiply_after_song, \
multiply_before_artist, multiply_after_artist], name='context_profile')
# user field
usr_embeddings = FunctionalDense(K*2, usr_profile, lw1=lw1, batchnorm=batchnorm, act=act, name='usr_profile')
usr_embeddings = Dense(K, name='usr_profile_output')(usr_embeddings)
usr_embeddings = add([usr_embeddings, embedding_outputs[0]], name='usr_embeddings')
# song field
song_embeddings = FunctionalDense(K*2, song_profile, lw1=lw1, batchnorm=batchnorm, act=act, name='song_profile')
song_embeddings = Dense(K, name='song_profile_output')(song_embeddings)
#song_embeddings = add([song_embeddings, embedding_outputs[4]], name='song_embeddings')
# context field
context_embeddings = FunctionalDense(K, context_profile, lw1=lw1, batchnorm=batchnorm, act=act, name='context_profile')
# joint embeddings
joint = dot([usr_embeddings, song_embeddings], axes=1, normalize=False, name='pred_cross')
joint_embeddings = concatenate([usr_embeddings, song_embeddings, context_embeddings, joint], name='joint_embeddings')
# top model
preds0 = FunctionalDense(K*2, joint_embeddings, batchnorm=batchnorm, act=act, name='preds_0')
preds1 = FunctionalDense(K*2, preds0, batchnorm=batchnorm, act=act, name='preds_1')
preds2 = FunctionalDense(K*2, preds1, batchnorm=batchnorm, act=act, name='preds_2')
preds = concatenate([preds0, preds1, preds2], name='prediction_aggr')
preds = Dropout(0.5, name='prediction_dropout')(preds)
preds = Dense(1, activation='sigmoid', name='prediction')(preds)
model = Model(inputs=embedding_inputs+genre_inputs+[context_input], outputs=preds)
opt = RMSprop(lr=lr)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['acc'])
return model
def __init__(self, input_size, weights=None):
input_image = Input(shape=(input_size[0], input_size[1], 3))
# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)
def space_to_depth_x2(x):
return tf.space_to_depth(x, block_size=2)
# Layer 1
x = Conv2D(32, (3,3), strides=(1,1), padding='same', name='conv_1', use_bias=False)(input_image)
x = BatchNormalization(name='norm_1')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 2
x = Conv2D(64, (3,3), strides=(1,1), padding='same', name='conv_2', use_bias=False)(x)
x = BatchNormalization(name='norm_2')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 3
x = Conv2D(128, (3,3), strides=(1,1), padding='same', name='conv_3', use_bias=False)(x)
x = BatchNormalization(name='norm_3')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 4
x = Conv2D(64, (1,1), strides=(1,1), padding='same', name='conv_4', use_bias=False)(x)
x = BatchNormalization(name='norm_4')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 5
x = Conv2D(128, (3,3), strides=(1,1), padding='same', name='conv_5', use_bias=False)(x)
x = BatchNormalization(name='norm_5')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 6
x = Conv2D(256, (3,3), strides=(1,1), padding='same', name='conv_6', use_bias=False)(x)
x = BatchNormalization(name='norm_6')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 7
x = Conv2D(128, (1,1), strides=(1,1), padding='same', name='conv_7', use_bias=False)(x)
x = BatchNormalization(name='norm_7')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 8
x = Conv2D(256, (3,3), strides=(1,1), padding='same', name='conv_8', use_bias=False)(x)
x = BatchNormalization(name='norm_8')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 9
x = Conv2D(512, (3,3), strides=(1,1), padding='same', name='conv_9', use_bias=False)(x)
x = BatchNormalization(name='norm_9')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 10
x = Conv2D(256, (1,1), strides=(1,1), padding='same', name='conv_10', use_bias=False)(x)
x = BatchNormalization(name='norm_10')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 11
x = Conv2D(512, (3,3), strides=(1,1), padding='same', name='conv_11', use_bias=False)(x)
x = BatchNormalization(name='norm_11')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 12
x = Conv2D(256, (1,1), strides=(1,1), padding='same', name='conv_12', use_bias=False)(x)
x = BatchNormalization(name='norm_12')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 13
x = Conv2D(512, (3,3), strides=(1,1), padding='same', name='conv_13', use_bias=False)(x)
x = BatchNormalization(name='norm_13')(x)
x = LeakyReLU(alpha=0.1)(x)
skip_connection = x
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 14
x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_14', use_bias=False)(x)
x = BatchNormalization(name='norm_14')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 15
x = Conv2D(512, (1,1), strides=(1,1), padding='same', name='conv_15', use_bias=False)(x)
x = BatchNormalization(name='norm_15')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 16
x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_16', use_bias=False)(x)
x = BatchNormalization(name='norm_16')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 17
x = Conv2D(512, (1,1), strides=(1,1), padding='same', name='conv_17', use_bias=False)(x)
x = BatchNormalization(name='norm_17')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 18
x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_18', use_bias=False)(x)
x = BatchNormalization(name='norm_18')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 19
x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_19', use_bias=False)(x)
x = BatchNormalization(name='norm_19')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 20
x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_20', use_bias=False)(x)
x = BatchNormalization(name='norm_20')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 21
skip_connection = Conv2D(64, (1,1), strides=(1,1), padding='same', name='conv_21', use_bias=False)(skip_connection)
skip_connection = BatchNormalization(name='norm_21')(skip_connection)
skip_connection = LeakyReLU(alpha=0.1)(skip_connection)
skip_connection = Lambda(space_to_depth_x2)(skip_connection)
x = concatenate([skip_connection, x])
# Layer 22
x = Conv2D(1024, (3,3), strides=(1,1), padding='same', name='conv_22', use_bias=False)(x)
x = BatchNormalization(name='norm_22')(x)
x = LeakyReLU(alpha=0.1)(x)
self.feature_extractor = Model(input_image, x)
if weights == 'imagenet':
print('Imagenet for YOLO backend are not available yet, defaulting to random weights')
elif weights == None:
pass
else:
print('Loaded backend weigths: '+weights)
self.feature_extractor.load_weights(weights)
def Model_BiLSTM_CnnDecoder(sourcevocabsize,
targetvocabsize,
source_W,
input_seq_lenth,
output_seq_lenth,
hidden_dim,
emd_dim,
sourcecharsize,
character_W,
input_word_length,
char_emd_dim,
sourcepossize,
pos_W,
pos_emd_dim,
batch_size=32,
loss='categorical_crossentropy',
optimizer='rmsprop'):
# 0.8349149507609669--attention,lstm*2decoder
# pos_input = Input(shape=(input_seq_lenth,), dtype='int32')
# pos_embeding = Embedding(input_dim=sourcepossize + 1,
# output_dim=pos_emd_dim,
# input_length=input_seq_lenth,
# mask_zero=False,
# trainable=True,
# weights=[pos_W])(pos_input)
word_input = Input(shape=(input_seq_lenth, ), dtype='int32')
char_input = Input(shape=(
input_seq_lenth,
input_word_length,
),
dtype='int32')
char_embedding = Embedding(input_dim=sourcecharsize,
output_dim=char_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
input_word_length),
mask_zero=False,
trainable=True,
weights=[character_W])
char_embedding2 = TimeDistributed(char_embedding)(char_input)
char_cnn = TimeDistributed(
Conv1D(50, 3, activation='relu', border_mode='valid'))(char_embedding2)
char_macpool = TimeDistributed(GlobalMaxPooling1D())(char_cnn)
# char_macpool = Dropout(0.5)(char_macpool)
pos_input = Input(shape=(
input_seq_lenth,
3,
), dtype='int32')
pos_embedding = Embedding(input_dim=sourcepossize + 1,
output_dim=pos_emd_dim,
batch_input_shape=(batch_size, input_seq_lenth,
3),
mask_zero=False,
trainable=True,
weights=[pos_W])
pos_embedding2 = TimeDistributed(pos_embedding)(pos_input)
pos_cnn = TimeDistributed(
Conv1D(20, 2, activation='relu', border_mode='valid'))(pos_embedding2)
pos_macpool = TimeDistributed(GlobalMaxPooling1D())(pos_cnn)
word_embedding_RNN = Embedding(input_dim=sourcevocabsize + 1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=False,
trainable=False,
weights=[source_W])(word_input)
# word_embedding_RNN = Dropout(0.5)(word_embedding_RNN)
embedding = concatenate([word_embedding_RNN, char_macpool, pos_macpool],
axis=-1)
embedding = Dropout(0.5)(embedding)
BiLSTM = Bidirectional(LSTM(int(hidden_dim / 2), return_sequences=True),
merge_mode='concat')(embedding)
BiLSTM = BatchNormalization()(BiLSTM)
# BiLSTM = Dropout(0.3)(BiLSTM)
# decodelayer1 = LSTM(50, return_sequences=False, go_backwards=True)(concat_LC_d)#!!!!!
# repeat_decodelayer1 = RepeatVector(input_seq_lenth)(decodelayer1)
# concat_decoder = concatenate([concat_LC_d, repeat_decodelayer1], axis=-1)#!!!!
# decodelayer2 = LSTM(hidden_dim, return_sequences=True)(concat_decoder)
# decodelayer = Dropout(0.5)(decodelayer2)
# decoderlayer1 = LSTM(50, return_sequences=True, go_backwards=False)(BiLSTM)
decoderlayer5 = Conv1D(50, 5, activation='relu', strides=1,
padding='same')(BiLSTM)
decoderlayer2 = Conv1D(50, 2, activation='relu', strides=1,
padding='same')(BiLSTM)
decoderlayer3 = Conv1D(50, 3, activation='relu', strides=1,
padding='same')(BiLSTM)
decoderlayer4 = Conv1D(50, 4, activation='relu', strides=1,
padding='same')(BiLSTM)
# 0.8868111121100423
decodelayer = concatenate(
[decoderlayer2, decoderlayer3, decoderlayer4, decoderlayer5], axis=-1)
decodelayer = BatchNormalization()(decodelayer)
decodelayer = Dropout(0.5)(decodelayer)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(decodelayer)
# TimeD = Dropout(0.5)(TimeD)
model = Activation('softmax')(TimeD) # 0.8769744561783556
# crf = CRF(targetvocabsize + 1, sparse_target=False)
# model = crf(TimeD)
Models = Model([word_input, char_input, pos_input], model)
# Models.compile(loss=my_cross_entropy_Weight, optimizer='adam', metrics=['acc'])
Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
# Models.compile(loss=loss, optimizer='adam', metrics=['acc'], sample_weight_mode="temporal")
# Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.01), metrics=['acc'])
# Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.005), metrics=[crf.accuracy])
return Models
def __init__(self, input_size):
input_image = Input(shape=(input_size, input_size, 3))
# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)
def space_to_depth_x2(x):
import tensorflow as tf
return tf.space_to_depth(x, block_size=2)
# Layer 1
x = Conv2D(32, (3, 3),
strides=(1, 1),
padding='same',
name='conv_1',
use_bias=False)(input_image)
x = BatchNormalization(name='norm_1')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 2
x = Conv2D(64, (3, 3),
strides=(1, 1),
padding='same',
name='conv_2',
use_bias=False)(x)
x = BatchNormalization(name='norm_2')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 3
x = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
name='conv_3',
use_bias=False)(x)
x = BatchNormalization(name='norm_3')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 4
x = Conv2D(64, (1, 1),
strides=(1, 1),
padding='same',
name='conv_4',
use_bias=False)(x)
x = BatchNormalization(name='norm_4')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 5
x = Conv2D(128, (3, 3),
strides=(1, 1),
padding='same',
name='conv_5',
use_bias=False)(x)
x = BatchNormalization(name='norm_5')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 6
x = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
name='conv_6',
use_bias=False)(x)
x = BatchNormalization(name='norm_6')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 7
x = Conv2D(128, (1, 1),
strides=(1, 1),
padding='same',
name='conv_7',
use_bias=False)(x)
x = BatchNormalization(name='norm_7')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 8
x = Conv2D(256, (3, 3),
strides=(1, 1),
padding='same',
name='conv_8',
use_bias=False)(x)
x = BatchNormalization(name='norm_8')(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 9
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_9',
use_bias=False)(x)
x = BatchNormalization(name='norm_9')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 10
x = Conv2D(256, (1, 1),
strides=(1, 1),
padding='same',
name='conv_10',
use_bias=False)(x)
x = BatchNormalization(name='norm_10')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 11
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_11',
use_bias=False)(x)
x = BatchNormalization(name='norm_11')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 12
x = Conv2D(256, (1, 1),
strides=(1, 1),
padding='same',
name='conv_12',
use_bias=False)(x)
x = BatchNormalization(name='norm_12')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 13
x = Conv2D(512, (3, 3),
strides=(1, 1),
padding='same',
name='conv_13',
use_bias=False)(x)
x = BatchNormalization(name='norm_13')(x)
x = LeakyReLU(alpha=0.1)(x)
skip_connection = x
x = MaxPooling2D(pool_size=(2, 2))(x)
# Layer 14
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_14',
use_bias=False)(x)
x = BatchNormalization(name='norm_14')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 15
x = Conv2D(512, (1, 1),
strides=(1, 1),
padding='same',
name='conv_15',
use_bias=False)(x)
x = BatchNormalization(name='norm_15')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 16
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_16',
use_bias=False)(x)
x = BatchNormalization(name='norm_16')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 17
x = Conv2D(512, (1, 1),
strides=(1, 1),
padding='same',
name='conv_17',
use_bias=False)(x)
x = BatchNormalization(name='norm_17')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 18
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_18',
use_bias=False)(x)
x = BatchNormalization(name='norm_18')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 19
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_19',
use_bias=False)(x)
x = BatchNormalization(name='norm_19')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 20
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_20',
use_bias=False)(x)
x = BatchNormalization(name='norm_20')(x)
x = LeakyReLU(alpha=0.1)(x)
# Layer 21
skip_connection = Conv2D(64, (1, 1),
strides=(1, 1),
padding='same',
name='conv_21',
use_bias=False)(skip_connection)
skip_connection = BatchNormalization(name='norm_21')(skip_connection)
skip_connection = LeakyReLU(alpha=0.1)(skip_connection)
skip_connection = Lambda(space_to_depth_x2)(skip_connection)
x = concatenate([skip_connection, x])
# Layer 22
x = Conv2D(1024, (3, 3),
strides=(1, 1),
padding='same',
name='conv_22',
use_bias=False)(x)
x = BatchNormalization(name='norm_22')(x)
x = LeakyReLU(alpha=0.1)(x)
self.feature_extractor = Model(input_image, x)
# self.feature_extractor.load_weights(FULL_YOLO_BACKEND_PATH)
# self.feature_extractor.summary()
print('\n')
#net = MaxPooling2D((2, 2))(net)
#net = Conv2D(16, (4,4), activation='relu', padding='same')(net)
#net = MaxPooling2D((2, 2))(net)
tower2 = Flatten()(net)
# Tower 3
net = Conv2D(16, (2, 2), activation='relu')(visible)
#net = BatchNormalization()(net)
#net = Conv2D(32, (2,2), activation='relu')(net)
#net = Conv2D(64, (4,4), activation='relu')(net)
#net = Conv2D(128, (4,4), activation='relu')(net)
#net = Conv2D(32, (4,4), activation='relu')(net)
#net = BatchNormalization()(net)
tower3 = Flatten()(net)
net = concatenate([tower1, tower2, tower3])
#net = Dense(100, activation='elu')(merge)
net = Dropout(0.8)(net)
output = Dense(10, activation='softmax')(net)
model = Model(inputs=visible, outputs=output)
from keras.utils import plot_model
plot_model(model, to_file='model.png', show_shapes=True)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
def get_unet_model(IMG_HEIGHT=300, IMG_WIDTH=300, IMG_CHANNELS=3):
inputs = layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
s = Lambda(lambda x: x / 1)(inputs)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(s)
c1 = Dropout(0.1)(c1)
c1 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c1)
p1 = MaxPooling2D((2, 2))(c1)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p1)
c2 = Dropout(0.1)(c2)
c2 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c2)
p2 = MaxPooling2D((2, 2))(c2)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p2)
c3 = Dropout(0.2)(c3)
c3 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c3)
p3 = MaxPooling2D((2, 2))(c3)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p3)
c4 = Dropout(0.2)(c4)
c4 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c4)
p4 = MaxPooling2D(pool_size=(2, 2))(c4)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(p4)
c5 = Dropout(0.3)(c5)
c5 = Conv2D(256, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c5)
u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5)
u6 = concatenate([u6, c4])
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u6)
c6 = Dropout(0.2)(c6)
c6 = Conv2D(128, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c6)
u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6)
u7 = concatenate([u7, c3])
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u7)
c7 = Dropout(0.2)(c7)
c7 = Conv2D(64, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c7)
u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7)
u8 = concatenate([u8, c2])
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u8)
c8 = Dropout(0.1)(c8)
c8 = Conv2D(32, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c8)
u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8)
u9 = concatenate([u9, c1], axis=3)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(u9)
c9 = Dropout(0.1)(c9)
c9 = Conv2D(16, (3, 3),
activation='elu',
kernel_initializer='he_normal',
padding='same')(c9)
outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9)
model = Model(inputs=[inputs], outputs=[outputs])
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=[mean_iou])
model.summary()
return model
pos_doc_max = doc_max(pos_doc_conv)
neg_doc_maxes = [doc_max(neg_doc_conv) for neg_doc_conv in neg_doc_convs]
pos_doc_sem = doc_sem(pos_doc_max)
neg_doc_sems = [doc_sem(neg_doc_max) for neg_doc_max in neg_doc_maxes]
# This layer calculates the cosine similarity between the semantic representations of
# a query and a document.
R_Q_D_p = dot([query_sem, pos_doc_sem], axes=1,
normalize=True) # See equation (4).
R_Q_D_ns = [
dot([query_sem, neg_doc_sem], axes=1, normalize=True)
for neg_doc_sem in neg_doc_sems
] # See equation (4).
concat_Rs = concatenate([R_Q_D_p] + R_Q_D_ns)
concat_Rs = Reshape((J + 1, 1))(concat_Rs)
# In this step, we multiply each R(Q, D) value by gamma. In the paper, gamma is
# described as a smoothing factor for the softmax function, and it's set empirically
# on a held-out data set. We're going to learn gamma's value by pretending it's
# a single 1 x 1 kernel.
weight = np.array([1]).reshape(1, 1, 1)
with_gamma = Convolution1D(1,
1,
padding="same",
input_shape=(J + 1, 1),
activation="linear",
use_bias=False,
weights=[weight])(concat_Rs) # See equation (5).
with_gamma = Reshape((J + 1, ))(with_gamma)
def init_model(self):
if K.image_data_format() == 'channels_first':
input_shape = (1, self.IMAGE_WIDTH, self.IMAGE_HEIGHT)
else:
input_shape = (self.IMAGE_WIDTH, self.IMAGE_HEIGHT, 1)
act = 'relu'
input_data = Input(name='the_input',
shape=input_shape,
dtype='float32')
inner = Conv2D(self.CONV_FILTERS,
self.KERNEL_SIZE,
padding='same',
activation=act,
kernel_initializer='he_normal',
name='conv1')(input_data)
inner = MaxPooling2D(pool_size=(self.POOL_SIZE, self.POOL_SIZE),
name='max1')(inner)
inner = Conv2D(self.CONV_FILTERS,
self.KERNEL_SIZE,
padding='same',
activation=act,
kernel_initializer='he_normal',
name='conv2')(inner)
inner = MaxPooling2D(pool_size=(self.POOL_SIZE, self.POOL_SIZE),
name='max2')(inner)
conv_to_rnn_dims = (self.IMAGE_WIDTH // (self.POOL_SIZE**2),
(self.IMAGE_HEIGHT //
(self.POOL_SIZE**2)) * self.CONV_FILTERS)
inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
# cuts down input size going into RNN:
inner = Dense(self.TIME_DENSE_SIZE, activation=act,
name='dense1')(inner)
# Two layers of bidirectional GRUs
# GRU seems to work as well, if not better than LSTM:
gru_1 = GRU(self.RNN_SIZE,
return_sequences=True,
kernel_initializer='he_normal',
name='gru1')(inner)
gru_1b = GRU(self.RNN_SIZE,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru1_b')(inner)
gru1_merged = add([gru_1, gru_1b])
gru_2 = GRU(self.RNN_SIZE,
return_sequences=True,
kernel_initializer='he_normal',
name='gru2')(gru1_merged)
gru_2b = GRU(self.RNN_SIZE,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
# transforms RNN output to character activations:
inner = Dense(len(self.ALPHABET) + 1,
kernel_initializer='he_normal',
name='dense2')(concatenate([gru_2, gru_2b]))
y_pred = Activation('softmax', name='softmax')(inner)
# Model(inputs=input_data, outputs=y_pred).summary()
labels = Input(name='the_labels',
shape=[self.ABSOLUTE_MAX_STRING],
dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
# LOSS FUNCTION NEED TO UNDERSTAND
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ), name='ctc')(
[y_pred, labels, input_length, label_length])
adam = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=adam)
model.summary()
# get keras model structure
self.MODEL = model
self.input_data = input_data
self.y_pred = y_pred
def unet(input_size, base_num_filters=64):
inputs = Input(input_size)
conv1 = Conv2D(base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(inputs)
conv1 = Conv2D(base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(2 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool1)
conv2 = Conv2D(2 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(4 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool2)
conv3 = Conv2D(4 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
conv4 = Conv2D(8 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool3)
conv4 = Conv2D(8 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)
conv5 = Conv2D(16 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(pool4)
conv5 = Conv2D(16 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv5)
drop5 = Dropout(0.5)(conv5)
up6 = Conv2D(8 * base_num_filters,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(drop5))
merge6 = concatenate([drop4, up6])
conv6 = Conv2D(8 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(8 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv6)
up7 = Conv2D(4 * base_num_filters,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv6))
merge7 = concatenate([conv3, up7])
conv7 = Conv2D(4 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge7)
conv7 = Conv2D(4 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv7)
up8 = Conv2D(2 * base_num_filters,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv7))
merge8 = concatenate([conv2, up8])
conv8 = Conv2D(2 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge8)
conv8 = Conv2D(2 * base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv8)
up9 = Conv2D(base_num_filters,
2,
activation='relu',
padding='same',
kernel_initializer='he_normal')(UpSampling2D(size=(2,
2))(conv8))
merge9 = concatenate([conv1, up9])
conv9 = Conv2D(base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(merge9)
conv9 = Conv2D(base_num_filters,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv9 = Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(conv9)
conv10 = Conv2D(1, 1)(conv9)
model = Model(input=inputs, output=conv10)
return model
def build_model(input_layer, start_neurons, DropoutRatio = 0.5):
# 101 -> 50
conv1 = Conv2D(start_neurons * 1, (3, 3), activation=None, padding="same")(input_layer)
conv1 = residual_block(conv1,start_neurons * 1)
conv1 = residual_block(conv1,start_neurons * 1)
conv1 = Activation('relu')(conv1)
pool1 = MaxPooling2D((2, 2))(conv1)
pool1 = Dropout(DropoutRatio/2)(pool1)
# 50 -> 25
conv2 = Conv2D(start_neurons * 2, (3, 3), activation=None, padding="same")(pool1)
conv2 = residual_block(conv2,start_neurons * 2)
conv2 = residual_block(conv2,start_neurons * 2)
conv2 = Activation('relu')(conv2)
pool2 = MaxPooling2D((2, 2))(conv2)
pool2 = Dropout(DropoutRatio)(pool2)
# 25 -> 12
conv3 = Conv2D(start_neurons * 4, (3, 3), activation=None, padding="same")(pool2)
conv3 = residual_block(conv3,start_neurons * 4)
conv3 = residual_block(conv3,start_neurons * 4)
conv3 = Activation('relu')(conv3)
pool3 = MaxPooling2D((2, 2))(conv3)
pool3 = Dropout(DropoutRatio)(pool3)
# 12 -> 6
conv4 = Conv2D(start_neurons * 8, (3, 3), activation=None, padding="same")(pool3)
conv4 = residual_block(conv4,start_neurons * 8)
conv4 = residual_block(conv4,start_neurons * 8)
conv4 = Activation('relu')(conv4)
pool4 = MaxPooling2D((2, 2))(conv4)
pool4 = Dropout(DropoutRatio)(pool4)
# Middle
convm = Conv2D(start_neurons * 16, (3, 3), activation=None, padding="same")(pool4)
convm = residual_block(convm,start_neurons * 16)
convm = residual_block(convm,start_neurons * 16)
convm = Activation('relu')(convm)
# 6 -> 12
deconv4 = Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding="same")(convm)
uconv4 = concatenate([deconv4, conv4])
uconv4 = Dropout(DropoutRatio)(uconv4)
uconv4 = Conv2D(start_neurons * 8, (3, 3), activation=None, padding="same")(uconv4)
uconv4 = residual_block(uconv4,start_neurons * 8)
uconv4 = residual_block(uconv4,start_neurons * 8)
uconv4 = Activation('relu')(uconv4)
# 12 -> 25
#deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="same")(uconv4)
deconv3 = Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding="valid")(uconv4)
uconv3 = concatenate([deconv3, conv3])
uconv3 = Dropout(DropoutRatio)(uconv3)
uconv3 = Conv2D(start_neurons * 4, (3, 3), activation=None, padding="same")(uconv3)
uconv3 = residual_block(uconv3,start_neurons * 4)
uconv3 = residual_block(uconv3,start_neurons * 4)
uconv3 = Activation('relu')(uconv3)
#5 -> 50
deconv2 = Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding="same")(uconv3)
uconv2 = concatenate([deconv2, conv2])
uconv2 = Dropout(DropoutRatio)(uconv2)
uconv2 = Conv2D(start_neurons * 2, (3, 3), activation=None, padding="same")(uconv2)
uconv2 = residual_block(uconv2,start_neurons * 2)
uconv2 = residual_block(uconv2,start_neurons * 2)
uconv2 = Activation('relu')(uconv2) # 50 -> 101
#deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="same")(uconv2)
deconv1 = Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding="valid")(uconv2)
uconv1 = concatenate([deconv1, conv1])
uconv1 = Dropout(DropoutRatio)(uconv1)
uconv1 = Conv2D(start_neurons * 1, (3, 3), activation=None, padding="same")(uconv1)
uconv1 = residual_block(uconv1,start_neurons * 1)
uconv1 = residual_block(uconv1,start_neurons * 1)
uconv1 = Activation('relu')(uconv1)
uconv1 = Dropout(DropoutRatio/2)(uconv1)
output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1)
return output_layer
visible1 = Input(shape=(4,)) embedding_layer = Embedding(4,128,input_length=4)(visible1) v1 = Flatten()(embedding_layer) # In[16]: #input 2 - numerical features as input #visible2 = Input(shape=(1,)) #visible2 = Input(shape=(2,)) visible2 = Input(shape=(3,)) # In[17]: merge = concatenate([v1, visible2]) # In[18]: x= Dense(7,activation='relu')(merge) x=BatchNormalization()(x) x= Dense(25,activation='relu')(x) x= Dense(20,activation='relu')(x) x=Dropout(0.05)(x) x= Dense(15,activation='relu')(x) x = Dense(10, activation='relu')(x) x = Dense(5, activation='relu')(x) predictions = Dense(num_class, activation='softmax')(x)
def __create_fcn_dense_net(nb_classes,
img_input,
include_top,
nb_dense_block=5,
growth_rate=12,
reduction=0.0,
dropout_rate=None,
weight_decay=1e-4,
nb_layers_per_block=4,
nb_upsampling_conv=128,
upsampling_type='upsampling',
init_conv_filters=48,
input_shape=None,
activation='deconv'):
''' Build the DenseNet model
Args:
nb_classes: number of classes
img_input: tuple of shape (channels, rows, columns) or (rows, columns, channels)
include_top: flag to include the final Dense layer
nb_dense_block: number of dense blocks to add to end (generally = 3)
growth_rate: number of filters to add per dense block
reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
dropout_rate: dropout rate
weight_decay: weight decay
nb_layers_per_block: number of layers in each dense block.
Can be a positive integer or a list.
If positive integer, a set number of layers per dense block.
If list, nb_layer is used as provided. Note that list size must
be (nb_dense_block + 1)
nb_upsampling_conv: number of convolutional layers in upsampling via subpixel convolution
upsampling_type: Can be one of 'upsampling', 'deconv' and 'subpixel'. Defines
type of upsampling algorithm used.
input_shape: Only used for shape inference in fully convolutional networks.
activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'.
Note that if sigmoid is used, classes must be 1.
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
if concat_axis == 1: # channels_first dim ordering
_, rows, cols = input_shape
else:
rows, cols, _ = input_shape
if reduction != 0.0:
assert reduction <= 1.0 and reduction > 0.0, 'reduction value must lie between 0.0 and 1.0'
# check if upsampling_conv has minimum number of filters
# minimum is set to 12, as at least 3 color channels are needed for correct upsampling
assert nb_upsampling_conv > 12 and nb_upsampling_conv % 4 == 0, 'Parameter `upsampling_conv` number of channels must ' \
'be a positive number divisible by 4 and greater ' \
'than 12'
# layers in each dense block
if type(nb_layers_per_block) is list or type(nb_layers_per_block) is tuple:
nb_layers = list(nb_layers_per_block) # Convert tuple to list
assert len(nb_layers) == (nb_dense_block + 1), 'If list, nb_layer is used as provided. ' \
'Note that list size must be (nb_dense_block + 1)'
bottleneck_nb_layers = nb_layers[-1]
rev_layers = nb_layers[::-1]
nb_layers.extend(rev_layers[1:])
else:
bottleneck_nb_layers = nb_layers_per_block
nb_layers = [nb_layers_per_block] * (2 * nb_dense_block + 1)
# compute compression factor
compression = 1.0 - reduction
# Initial convolution
x = Conv2D(init_conv_filters, (7, 7),
kernel_initializer='he_normal',
padding='same',
name='initial_conv2D',
use_bias=False,
kernel_regularizer=l2(weight_decay))(img_input)
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
nb_filter = init_conv_filters
skip_list = []
# Add dense blocks and transition down block
for block_idx in range(nb_dense_block):
x, nb_filter = __dense_block(x,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Skip connection
skip_list.append(x)
# add transition_block
x = __transition_block(x,
nb_filter,
compression=compression,
weight_decay=weight_decay)
nb_filter = int(
nb_filter *
compression) # this is calculated inside transition_down_block
# The last dense_block does not have a transition_down_block
# return the concatenated feature maps without the concatenation of the input
_, nb_filter, concat_list = __dense_block(x,
bottleneck_nb_layers,
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay,
return_concat_list=True)
skip_list = skip_list[::-1] # reverse the skip list
# Add dense blocks and transition up block
for block_idx in range(nb_dense_block):
n_filters_keep = growth_rate * nb_layers[nb_dense_block + block_idx]
# upsampling block must upsample only the feature maps (concat_list[1:]),
# not the concatenation of the input with the feature maps (concat_list[0].
l = concatenate(concat_list[1:], axis=concat_axis)
t = __transition_up_block(l,
nb_filters=n_filters_keep,
type=upsampling_type,
weight_decay=weight_decay)
# concatenate the skip connection with the transition block
x = concatenate([t, skip_list[block_idx]], axis=concat_axis)
# Dont allow the feature map size to grow in upsampling dense blocks
x_up, nb_filter, concat_list = __dense_block(x,
nb_layers[nb_dense_block +
block_idx + 1],
nb_filter=growth_rate,
growth_rate=growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay,
return_concat_list=True,
grow_nb_filters=False)
if include_top:
x = Conv2D(nb_classes, (1, 1),
activation='linear',
padding='same',
use_bias=False)(x_up)
if K.image_data_format() == 'channels_first':
channel, row, col = input_shape
else:
row, col, channel = input_shape
x = Reshape((row * col, nb_classes))(x)
x = Activation(activation)(x)
x = Reshape((row, col, nb_classes))(x)
else:
x = x_up
return x
def __init__(self, max_len, emb_train):
# Define hyperparameters
modname = FIXED_PARAMETERS["model_name"]
learning_rate = FIXED_PARAMETERS["learning_rate"]
dropout_rate = FIXED_PARAMETERS["dropout_rate"]
batch_size = FIXED_PARAMETERS["batch_size"]
max_words = FIXED_PARAMETERS["max_words"]
print("Loading data...")
genres_train, sent1_train, sent2_train, labels_train_, scores_train = load_sts_data(
FIXED_PARAMETERS["train_path"])
genres_dev, sent1_dev, sent2_dev, labels_dev_, scores_dev = load_sts_data(
FIXED_PARAMETERS["dev_path"])
print("Building dictionary...")
text = sent1_train + sent2_train + sent1_dev + sent2_dev
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(text)
word_index = tokenizer.word_index
print("Padding and indexing sentences...")
sent1_train_seq, sent2_train_seq, labels_train = tokenizing_and_padding(
FIXED_PARAMETERS["train_path"], tokenizer, max_len)
sent1_dev_seq, sent2_dev_seq, labels_dev = tokenizing_and_padding(
FIXED_PARAMETERS["dev_path"], tokenizer, max_len)
print("Encoding labels...")
train_labels_to_probs = encode_labels(labels_train)
dev_labels_to_probs = encode_labels(labels_dev)
print("Loading embeddings...")
vocab_size = min(max_words, len(word_index)) + 1
embedding_matrix = build_emb_matrix(FIXED_PARAMETERS["embedding_path"],
vocab_size, word_index)
embedding_layer = Embedding(vocab_size,
300,
weights=[embedding_matrix],
input_length=max_len,
trainable=emb_train,
mask_zero=True,
name='VectorLookup')
lstm = Bidirectional(
LSTM(
150, #dropout=dropout_rate, recurrent_dropout=0.1,
return_sequences=False,
kernel_regularizer=regularizers.l2(1e-4),
name='RNN'))
sent1_seq_in = Input(shape=(max_len, ),
dtype='int32',
name='Sentence1')
embedded_sent1 = embedding_layer(sent1_seq_in)
encoded_sent1 = lstm(embedded_sent1)
#mean_pooling_1 = TemporalMeanPooling()(encoded_sent1)
#sent1_lstm = Dropout(0.1)(mean_pooling_1)
sent2_seq_in = Input(shape=(max_len, ),
dtype='int32',
name='Sentence2')
embedded_sent2 = embedding_layer(sent2_seq_in)
encoded_sent2 = lstm(embedded_sent2)
#mean_pooling_2 = TemporalMeanPooling()(encoded_sent2)
#sent2_lstm = Dropout(0.1)(mean_pooling_2)
mul = layers.Multiply(name='S1.S2')(
[encoded_sent1,
encoded_sent2]) #([mean_pooling_1, mean_pooling_2])
sub = layers.Subtract(name='S1-S2')(
[encoded_sent1,
encoded_sent2]) #([mean_pooling_1, mean_pooling_2])
dif = Lambda(lambda x: K.abs(x), name='Abs')(sub)
concatenated = concatenate([mul, dif], name='Concat')
x = Dense(50,
activation='sigmoid',
name='Sigmoid',
kernel_regularizer=regularizers.l2(1e-4))(concatenated)
preds = Dense(6,
activation='softmax',
kernel_regularizer=regularizers.l2(1e-4),
name='Softmax')(x)
model = Model([sent1_seq_in, sent2_seq_in], preds)
model.summary()
def pearson(y_true, y_pred):
"""
Pearson product-moment correlation metric.
"""
gate_mapping = K.variable(
value=np.array([[0.], [1.], [2.], [3.], [4.], [5.]]))
y_true = K.clip(y_true, K.epsilon(), 1)
y_pred = K.clip(y_pred, K.epsilon(), 1)
y_true = K.reshape(K.dot(y_true, gate_mapping), (-1, ))
y_pred = K.reshape(K.dot(y_pred, gate_mapping), (-1, ))
return pearsonr(y_true, y_pred)
early_stopping = EarlyStopping(monitor='pearson',
patience=10,
mode='max')
# checkpointer = ModelCheckpoint(filepath=os.path.join(FIXED_PARAMETERS["ckpt_path"], modname) + '.hdf5',
# verbose=1,
# monitor='val_pearson',
# save_best_only=True,
# mode='max')
Adagrad = optimizers.Adagrad(lr=learning_rate)
model.compile(optimizer=Adagrad, loss='kld', metrics=[pearson])
history = model.fit([sent1_train_seq, sent2_train_seq],
train_labels_to_probs,
epochs=30,
batch_size=batch_size,
shuffle=True,
callbacks=[early_stopping],
validation_data=([sent1_dev_seq, sent2_dev_seq],
dev_labels_to_probs))
