def two_head_ensemble_model(dense1=None,
dense2=None,
dropout=0.25,
k_reg=0.00000001):
"""Not used due to no performance gains"""
k_regularizer = keras.regularizers.l2(k_reg)
input_tensor = keras.layers.Input(shape=(TWO_HEAD_SHAPE, ))
x = layers.Dense(dense1,
activation=None,
kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(input_tensor)
x = layers.PReLU()(x)
out = layers.Dropout(dropout)(x)
half_model = keras.models.Model(inputs=input_tensor, outputs=out)
inp_a = keras.layers.Input(shape=(TWO_HEAD_SHAPE, ))
inp_b = keras.layers.Input(shape=(TWO_HEAD_SHAPE, ))
out_a = half_model(inp_a)
out_b = half_model(inp_b)
concatenated = keras.layers.concatenate([out_a, out_b])
x = layers.Dense(dense2,
activation=None,
kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(concatenated)
x = layers.PReLU()(x)
predictions = layers.Dense(N_CLASSES,
activation='softmax',
kernel_regularizer=k_regularizer)(x)
model = keras.models.Model(inputs=[inp_a, inp_b], outputs=predictions)
return model
def enet_upsample(tensor, nfilters, scale = 8, name=''):
y = tensor
skip = tensor
skip = layers.Conv2D(filters=nfilters, kernel_size=(1, 1), kernel_initializer='he_normal', strides=(1, 1),
padding='same', use_bias=False, name=f'1x1_conv_skip_{name}')(skip)
skip = layers.UpSampling2D(size=(scale, scale), interpolation='bilinear', name=f'upsample_skip_{name}')(skip)
# 1*1 dimensionality reduction
y = layers.Conv2D(filters=nfilters // 4, kernel_size=(1, 1), kernel_initializer='he_normal', strides=(1, 1),
padding='same', use_bias=False, name=f'1x1_conv_{name}')(y)
y = layers.BatchNormalization(momentum=0.1, name=f'bn_1x1_{name}')(y)
y = layers.PReLU(shared_axes=[1, 2], name=f'prelu_1x1_{name}')(y)
# main conv
y = layers.Conv2DTranspose(filters=nfilters // 4, kernel_size=(3, 3),
kernel_initializer='he_normal', strides=(scale, scale),
padding='same', name=f'3x3_deconv_{name}')(y)
y = layers.BatchNormalization(momentum=0.1, name=f'bn_main_{name}')(y)
y = layers.PReLU(shared_axes=[1, 2], name=f'prelu_{name}')(y)
y = layers.Conv2D(filters=nfilters, kernel_size=(1, 1), kernel_initializer='he_normal', use_bias=False,
name=f'final_1x1_{name}')(y)
y = layers.BatchNormalization(momentum=0.1, name=f'bn_final_{name}')(y)
print(f'add_{name}')
y = layers.Add(name=f'add_{name}')([y, skip])
y = layers.ReLU(name=f'relu_out_{name}')(y)
return y
def p_net(training=False):
x = Input(shape=(12, 12, 3)) if training else Input(shape=(None, None, 3))
y = layers.Conv2D(10, 3, padding='valid', strides=(1, 1),
name='p_conv1')(x)
y = layers.PReLU(shared_axes=(1, 2), name='p_prelu1')(y)
y = layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name='p_max_pooling1')(y)
y = layers.Conv2D(16, 3, padding='valid', strides=(1, 1),
name='p_conv2')(y)
y = layers.PReLU(shared_axes=(1, 2), name='p_prelu2')(y)
y = layers.Conv2D(32, 3, padding='valid', strides=(1, 1),
name='p_conv3')(y)
y = layers.PReLU(shared_axes=(1, 2), name='p_prelu3')(y)
classifier = layers.Conv2D(2, 1, activation='softmax',
name='p_classifier')(y)
bbox = layers.Conv2D(4, 1, name='p_bbox')(y)
landmark = layers.Conv2D(10, 1, padding='valid', name='p_landmark')(y)
if training:
classifier = layers.Reshape((2, ), name='p_classifier1')(classifier)
bbox = layers.Reshape((4, ), name='p_bbox1')(bbox)
landmark = layers.Reshape((10, ), name='p_landmark1')(landmark)
outputs = layers.concatenate([classifier, bbox, landmark])
model = Model(inputs=[x], outputs=[outputs], name='P_Net')
else:
model = Model(inputs=[x],
outputs=[classifier, bbox, landmark],
name='P_Net')
return model
def _create_Kao_Onet(self, weight_path='./models/mtcnn/48net.h5'):
'''
'''
input = KL.Input(shape=[48, 48, 3])
x = KL.Conv2D(32, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input)
x = KL.PReLU(shared_axes=[1, 2], name='prelu1')(x)
x = KL.MaxPool2D(pool_size=3, strides=2, padding='same', data_format="channels_last")(x)
x = KL.Conv2D(64, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='prelu2')(x)
x = KL.MaxPool2D(pool_size=3, strides=2, data_format="channels_last")(x)
x = KL.Conv2D(64, (3, 3), strides=1, padding='valid', name='conv3', data_format="channels_last")(x)
x = PReLU(shared_axes=[1, 2], name='prelu3')(x)
x = KL.MaxPool2D(pool_size=2, data_format="channels_last")(x)
x = KL.Conv2D(128, (2, 2), strides=1, padding='valid', name='conv4', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='prelu4')(x)
x = KL.Permute((3, 2, 1))(x)
x = KL.Flatten()(x)
x = KL.Dense(256, name='conv5') (x)
x = KL.PReLU(name='prelu5')(x)
classifier = KL.Dense(2, activation='softmax', name='conv6-1')(x)
bbox_regress = KL.Dense(4, name='conv6-2')(x)
landmark_regress = KL.Dense(10, name='conv6-3')(x)
model = Model([input], [classifier, bbox_regress, landmark_regress])
model.load_weights(weight_path, by_name=True)
return model
def create_generator():
img = L.Input(shape=(32, 32, 3))
x = L.Conv2D(filters=64, kernel_size=(9, 9),
strides=(1, 1), padding='same')(img)
x = L.PReLU(shared_axes=[1, 2])(x)
res = x
for i in range(16):
x = ResBlock(x, 64)
x = L.Conv2D(filters=64, kernel_size=(3, 3),
strides=(1, 1), padding='same')(x)
x = L.BatchNormalization()(x)
x = L.Add()([res, x])
x = L.Conv2D(filters=256, kernel_size=(3, 3),
strides=(1, 1), padding='same')(x)
x = L.UpSampling2D()(x)
x = L.PReLU(shared_axes=[1, 2])(x)
x = L.Conv2D(filters=256, kernel_size=(3, 3),
strides=(1, 1), padding='same')(x)
x = L.PReLU(shared_axes=[1, 2])(x)
x = L.Conv2D(filters=3, kernel_size=(9, 9),
strides=(1, 1), padding='same', activation='relu')(x)
gen = Model(inputs=img, outputs=x)
gen.compile(loss=mean_squared_error,
optimizer=Adam(), metrics=['accuracy'])
return gen
def r_net(training=False):
x = Input(shape=(24, 24, 3))
y = layers.Conv2D(28, 3, padding='same', strides=(1, 1), name='r_conv1')(x)
y = layers.PReLU(shared_axes=(1, 2), name='r_prelu1')(y)
y = layers.MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='p_max_pooling1')(y)
y = layers.Conv2D(48, 3, padding='valid', strides=(1, 1),
name='r_conv2')(y)
y = layers.PReLU(shared_axes=(1, 2), name='r_prelu2')(y)
y = layers.MaxPooling2D(pool_size=(3, 3),
strides=(2, 2),
name='p_max_pooling2')(y)
y = layers.Conv2D(64, 2, padding='valid', name='r_conv3')(y)
y = layers.PReLU(shared_axes=(1, 2), name='r_prelu3')(y)
y = layers.Dense(128, name='r_dense')(y)
y = layers.PReLU(name='r_prelu4')(y)
y = layers.Flatten()(y)
classifier = layers.Dense(2, activation='softmax', name='r_classifier')(y)
bbox = layers.Dense(4, name='r_bbox')(y)
landmark = layers.Dense(10, name='r_landmark')(y)
if training:
outputs = layers.concatenate([classifier, bbox, landmark])
model = Model(inputs=[x], outputs=[outputs], name='R_Net')
else:
model = Model(inputs=[x],
outputs=[classifier, bbox, landmark],
name='R_Net')
return model
def bottleneck_identity_block(inputs,
filters=(64, 64, 256),
activation='relu',
kernel_size=(3, 3)):
"""
Follows the bottleneck architecture implementation of an identity block.
:param inputs:
:param filters:
:param activation:
:param kernel_size:
:return:
"""
# Set residual and shortcut as inputs for clarity
shortcut = inputs
residual = inputs
f1, f2, f3 = filters # Filters take specific numbers only
# First convolution layer -> BN -> activation
residual = Conv2D(f1, kernel_size=(1, 1), strides=(1, 1),
padding='valid')(residual)
residual = BatchNormalization()(residual)
if activation == 'relu':
residual = layers.ReLU()(residual)
elif activation == 'prelu':
residual = layers.PReLU()(residual)
else:
raise NotImplementedError
# Second convolution layer -> BN -> activation
# Padding is 'same' to ensure same dims
residual = Conv2D(f2,
kernel_size=kernel_size,
strides=(1, 1),
padding='same')(residual)
residual = BatchNormalization()(residual)
if activation == 'relu':
output = layers.ReLU()(residual)
elif activation == 'prelu':
output = layers.PReLU()(residual)
else:
raise NotImplementedError
residual = Conv2D(f3, kernel_size=(1, 1), strides=(1, 1),
padding='valid')(residual)
residual = BatchNormalization()(residual)
output = layers.Add()([shortcut, residual])
if activation == 'relu':
output = layers.ReLU()(output)
elif activation == 'prelu':
output = layers.PReLU()(output)
else:
raise NotImplementedError
return output
def bottleneck_projection_block(inputs,
filters,
activation='relu',
kernel_size=(3, 3),
strides=(2, 2)):
# Set residual and shortcut as inputs for clarity
shortcut = inputs
residual = inputs
f1, f2, f3 = filters # Filters take specific numbers only
# First convolution layer -> BN -> activation
residual = Conv2D(f1, kernel_size=(1, 1), strides=strides,
padding='valid')(residual)
residual = BatchNormalization()(residual)
if activation == 'relu':
residual = layers.ReLU()(residual)
elif activation == 'prelu':
residual = layers.PReLU()(residual)
else:
raise NotImplementedError
# Second convolution layer -> BN -> activation
# Padding is 'same' to ensure same dims
residual = Conv2D(f2,
kernel_size=kernel_size,
strides=(1, 1),
padding='same')(residual)
residual = BatchNormalization()(residual)
if activation == 'relu':
output = layers.ReLU()(residual)
elif activation == 'prelu':
output = layers.PReLU()(residual)
else:
raise NotImplementedError
residual = Conv2D(f3, kernel_size=(1, 1), strides=(1, 1),
padding='valid')(residual)
residual = BatchNormalization()(residual)
# Projection of shortcut through convolutional block with same strides as first conv layer
shortcut = Conv2D(f3, kernel_size=(1, 1), strides=strides,
padding='valid')(shortcut)
shortcut = BatchNormalization()(shortcut)
# Add shortcut to residual, pass through activation
output = layers.Add()([shortcut, residual])
if activation == 'relu':
output = layers.ReLU()(output)
elif activation == 'prelu':
output = layers.PReLU()(output)
else:
raise NotImplementedError
return output
def identity_block(inputs, activation='relu', kernel_size=3, strides=(1, 1)):
"""
Regular residual block, not bottlenecked.
:param inputs:
:param activation:
:param kernel_size:
:param strides:
:return:
"""
# Set residual and shortcut as inputs for clarity
shortcut = inputs
residual = inputs
f = K.int_shape(inputs)[
3] # Filters must be fixed to input shape for dimension adding later
# First convolution layer -> BN -> activation
# Padding is 'same' to ensure shortcut/residual have same dims
residual = Conv2D(f,
kernel_size=kernel_size,
strides=strides,
padding='same')(residual)
residual = BatchNormalization()(residual)
if activation == 'relu':
residual = layers.ReLU()(residual)
elif activation == 'prelu':
residual = layers.PReLU()(residual)
else:
raise NotImplementedError
# Second convolution layer -> BN. Activation follows after adding shortcut
residual = Conv2D(f,
kernel_size=kernel_size,
strides=strides,
padding='same')(residual)
residual = BatchNormalization()(residual)
add = layers.Add()([
shortcut, residual
]) # Add shortcut and residual, then send through activation out
if activation == 'relu':
output = layers.ReLU()(add)
elif activation == 'prelu':
output = layers.PReLU()(add)
else:
raise NotImplementedError
return output
def convBN_pool(input_layer, conv_channels):
convlayer = Conv1D(conv_channels, 1, padding = 'valid', strides = 2)(input_layer)
BN = BatchNormalization(axis=-1, momentum = 0.99, epsilon=0.001, center=True, scale = True,
beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros')(convlayer)
activation = layers.PReLU()(BN)
return activation
def convBN(input_layer, conv_channels):
convlayer = Conv1D(conv_channels, 3, padding = 'same')(input_layer)
BN = BatchNormalization(axis=-1, momentum = 0.99, epsilon=0.001, center=True, scale = True,
beta_initializer='zeros', gamma_initializer='ones', moving_mean_initializer='zeros', moving_variance_initializer='ones',
beta_regularizer=None, gamma_regularizer=None, beta_constraint=None, gamma_constraint=None)(convlayer)
activation = layers.PReLU()(BN)
return activation
def preactivated_identity_residual_block(inputs,
activation='relu',
kernel_size=3,
strides=(1, 1)):
# Set residual and shortcut as inputs for clarity
shortcut = inputs
residual = inputs
f = K.int_shape(inputs)[
3] # Filters must be fixed to input shape for dimension adding later
# Pre-activation using BN and P/ReLU
residual = layers.BatchNormalization()(inputs)
if activation == 'relu':
residual = layers.ReLU()(residual)
elif activation == 'prelu':
residual = layers.PReLU()(residual)
else:
raise NotImplementedError
# Padding is 'same' to ensure shortcut/residual have same dims
residual = Conv2D(f,
kernel_size=kernel_size,
strides=strides,
padding='same')(residual)
residual = BatchNormalization()(residual)
if activation == 'relu':
residual = layers.ReLU()(residual)
elif activation == 'prelu':
residual = layers.PReLU()(residual)
else:
raise NotImplementedError
# Second convolution layer -> addition. Only pre-activation, no post.
residual = Conv2D(f,
kernel_size=kernel_size,
strides=strides,
padding='same')(residual)
output = layers.Add()([shortcut, residual]) # Add shortcut and residual
return output
def _create_Kao_Pnet(self, weight_path='./models/mtcnn/12net.h5'):
'''
'''
input = KL.Input(shape=[None, None, 3])
x = KL.Conv2D(10, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input)
x = KL.PReLU(shared_axes=[1, 2], name='PReLU1')(x)
x = KL.MaxPool2D(pool_size=2, data_format="channels_last")(x)
x = KL.Conv2D(16, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='PReLU2')(x)
x = KL.Conv2D(32, (3, 3), strides=1, padding='valid', name='conv3', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='PReLU3')(x)
classifier = KL.Conv2D(2, (1, 1), activation='softmax', name='conv4-1', data_format="channels_last")(x)
bbox_regress = KL.Conv2D(4, (1, 1), name='conv4-2', data_format="channels_last")(x)
model = Model([input], [classifier, bbox_regress])
model.load_weights(weight_path, by_name=True)
return model
def activate(self, layer):
""" activate layer with given activation function
:param layer: the input layer
:return: the layer after activation
"""
if self.activ == 'lrelu':
return layers.LeakyReLU()(layer)
elif self.activ == 'prelu':
return layers.PReLU()(layer)
else:
return Activation(self.activ)(layer)
def ResBlock(x, filters):
res = x
x = L.Conv2D(filters=filters, kernel_size=(
3, 3), strides=(1, 1), padding='same')(x)
x = L.BatchNormalization()(x)
x = L.PReLU(shared_axes=[1, 2])(x)
x = L.Conv2D(filters=filters, kernel_size=(
3, 3), strides=(1, 1), padding='same')(x)
x = L.BatchNormalization()(x)
x = L.Add()([res, x])
return x
def combined_features_model(dense1=1024,
dense2=None,
dropout=0.25,
k_reg=0.0001):
k_regularizer = keras.regularizers.l2(k_reg)
input_tensor = keras.layers.Input(shape=MODEL_INPUT_SHAPE)
x = layers.Dense(dense1,
activation=None,
kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(input_tensor)
x = layers.PReLU()(x)
x = layers.Dropout(dropout)(x)
if dense2:
x = layers.Dense(dense2,
activation=None,
kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(x)
x = layers.PReLU()(x)
x = layers.Dropout(dropout)(x)
predictions = layers.Dense(N_CLASSES, activation='softmax')(x)
model = keras.models.Model(inputs=input_tensor, outputs=predictions)
return model
def inception_resnet_v2(num_classes, input_size=150):
base_model = InceptionResNetV2(include_top=False,
weights='imagenet',
input_shape=(input_size, input_size, 3),
pooling='max')
x = base_model.output
x = layers.BatchNormalization()(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(1024)(x)
x = layers.BatchNormalization()(x)
x = layers.PReLU(shared_axes=[1])(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(512)(x)
x = layers.BatchNormalization()(x)
features = layers.PReLU(shared_axes=[1])(x)
prediction = layers.Dense(num_classes, activation='softmax', name='fc_prediction')(features)
model = Model(inputs=base_model.input, outputs=prediction)
model.summary()
return model, base_model
def ensemble_model(dense1=None,
dense2=None,
n_layers=4,
dropout=0.25,
k_reg=0):
"""Creates NN ensemble model"""
k_regularizer = keras.regularizers.l2(k_reg)
input_tensor = keras.layers.Input(shape=MODEL_INPUT_SHAPE)
if dense1:
x = layers.Dense(dense1,
activation=None,
kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(input_tensor)
x = layers.PReLU()(x)
x = layers.Dropout(dropout)(x)
x = layers.BatchNormalization()(x)
if dense2:
for n in range(n_layers - 1):
x = layers.Dense(dense2,
activation=None,
kernel_initializer='he_uniform',
kernel_regularizer=k_regularizer)(x)
x = layers.PReLU()(x)
if not n == n_layers - 2: # Don't want dropout and BN on last layer
x = layers.Dropout(dropout)(x)
x = layers.BatchNormalization()(x)
if dense1:
predictions = layers.Dense(N_CLASSES,
activation='softmax',
kernel_regularizer=k_regularizer)(x)
else:
predictions = layers.Dense(
N_CLASSES, activation='softmax',
kernel_regularizer=k_regularizer)(input_tensor)
model = keras.models.Model(inputs=input_tensor, outputs=predictions)
return model
def _last_layer(output, last_layer):
if last_layer == 'softmax':
output = layers.Softmax(axis=-1)(output)
elif last_layer == 'sigmoid':
output = layers.Activation(activation='sigmoid')(output)
elif last_layer == 'relu':
output = layers.Activation(activation='relu')(output)
elif last_layer == 'leaky_relu':
output = layers.LeakyReLU()(output)
elif last_layer == 'prelu':
output = layers.PReLU()(output)
return output
def resblock(input, filters, stride=1):
input_channels = input.shape[-1]
residual = layers.Conv2D(filters / 4, kernel_size=3, strides=1, padding='same')(input)
residual = layers.BatchNormalization()(residual)
residual = layers.PReLU()(residual)
residual = layers.Conv2D(filters / 2, kernel_size=3, strides=stride, padding='same')(residual)
residual = layers.BatchNormalization()(residual)
residual = layers.PReLU()(residual)
residual = layers.Conv2D(filters, kernel_size=3, strides=1, padding='same')(residual)
residual = layers.BatchNormalization()(residual)
if stride > 1 or input_channels != filters:
# Perform identity shortcut convolution for dims not matching
shortcut = layers.Conv2D(filters, strides=stride, kernel_size=1)(input)
shortcut = layers.BatchNormalization()(shortcut)
else:
shortcut = input
output = layers.Add()([shortcut, residual])
output = layers.PReLU()(output)
return output
def _create_Kao_Rnet(self, weight_path='./models/mtcnn/24net.h5'):
'''
'''
input = KL.Input(shape=[24, 24, 3]) # change this shape to [None,None,3] to enable arbitraty shape input
x = KL.Conv2D(28, (3, 3), strides=1, padding='valid', name='conv1', data_format="channels_last")(input)
x = KL.PReLU(shared_axes=[1, 2], name='prelu1')(x)
x = KL.MaxPool2D(pool_size=3, strides=2, padding='same', data_format="channels_last")(x)
x = KL.Conv2D(48, (3, 3), strides=1, padding='valid', name='conv2', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='prelu2')(x)
x = KL.MaxPool2D(pool_size=3, strides=2, data_format="channels_last")(x)
x = KL.Conv2D(64, (2, 2), strides=1, padding='valid', name='conv3', data_format="channels_last")(x)
x = KL.PReLU(shared_axes=[1, 2], name='prelu3')(x)
x = KL.Permute((3, 2, 1))(x)
x = KL.Flatten()(x)
x = KL.Dense(128, name='conv4')(x)
x = KL.PReLU(name='prelu4')(x)
classifier = KL.Dense(2, activation='softmax', name='conv5-1')(x)
bbox_regress = KL.Dense(4, name='conv5-2')(x)
model = Model([input], [classifier, bbox_regress])
model.load_weights(weight_path, by_name=True)
return model
def conv_bn_activation_block(inputs,
filters,
activation='relu',
kernel_size=5,
padding='valid',
strides=(1, 1)):
conv_layer = Conv2D(filters,
kernel_size=kernel_size,
strides=strides,
padding=padding)(inputs)
bn_layer = BatchNormalization()(conv_layer)
if activation == 'relu':
output = layers.ReLU()(bn_layer)
elif activation == 'prelu':
output = layers.PReLU()(bn_layer)
else:
return print("Unspecified activation.")
return output
def create_smNet(input_dims=(32, 32, 1), output_dims=1, min_z=-500, max_z=500):
input = layers.Input(shape=input_dims)
x = layers.Conv2D(filters=64, kernel_size=7)(input)
x = layers.BatchNormalization()(x)
x = layers.PReLU()(x)
x = layers.Conv2D(filters=128, kernel_size=5)(x)
x = layers.BatchNormalization()(x)
x = layers.PReLU()(x)
x = resblock(x, 128, 1)
x = resblock(x, 128, 1)
x = resblock(x, 128, 1)
x = resblock(x, 256, 1)
x = resblock(x, 256, 1)
x = resblock(x, 256, 1)
x = resblock(x, 256, 1)
x = layers.Conv2D(128, kernel_size=1, strides=1)(x)
x = layers.BatchNormalization()(x)
x = layers.PReLU()(x)
x = layers.Conv2D(64, kernel_size=1, strides=1)(x)
x = layers.BatchNormalization()(x)
x = layers.PReLU()(x)
x = layers.Conv2D(1, kernel_size=1, strides=1)(x)
x = layers.BatchNormalization()(x)
x = layers.PReLU()(x)
# FCs
x = layers.Flatten()(x)
x = layers.Dense(10)(x)
x = layers.PReLU()(x)
output = layers.Dense(output_dims)(x)
output = HardTanh(min_z=min_z, max_z=max_z)(output)
model = keras.Model(input, output)
return model
################################################################################
################################################################################
################################################################################
# Try replacing GRU, or SimpleRNN.
RNN = layers.LSTM
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 5
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
# Note: In a situation where your input sequences have a variable length,
# use input_shape=(None, num_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(INPUT_LEN, len(chars)), activation=layers.PReLU(), recurrent_activation='sigmoid',\
dropout=0.25, recurrent_dropout=0.125))
model.add(BatchNormalization(center=True, scale=True))
# As the decoder RNN's input, repeatedly provide with the last output of
# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
model.add(layers.RepeatVector(OUTPUT_LEN))
model.add(BatchNormalization(center=True, scale=True))
# The decoder RNN could be multiple layers stacked or a single layer.
for _ in range(LAYERS):
# By setting return_sequences to True, return not only the last output but
# all the outputs so far in the form of (num_samples, timesteps,
# output_dim). This is necessary as TimeDistributed in the below expects
# the first dimension to be the timesteps.
model.add(RNN(HIDDEN_SIZE, return_sequences=True, activation=layers.PReLU(), recurrent_activation='sigmoid',\
dropout=0.25, recurrent_dropout = 0.125))
################################################################################
################################################################################
# Try replacing GRU, or SimpleRNN.
RNN = layers.LSTM
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 1
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
# Note: In a situation where your input sequences have a variable length,
# use input_shape=(None, num_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(INPUT_LEN, len(chars)),\
activation=layers.PReLU(), recurrent_activation='sigmoid',\
dropout=0.25, recurrent_dropout=0.125))
model.add(BatchNormalization(center=True, scale=True))
# As the decoder RNN's input, repeatedly provide with the last output of
# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
model.add(layers.RepeatVector(OUTPUT_LEN))
model.add(BatchNormalization(center=True, scale=True))
# The decoder RNN could be multiple layers stacked or a single layer.
for _ in range(LAYERS):
# By setting return_sequences to True, return not only the last output but
# all the outputs so far in the form of (num_samples, timesteps,
# output_dim). This is necessary as TimeDistributed in the below expects
# the first dimension to be the timesteps.
model.add(RNN(HIDDEN_SIZE, return_sequences=True,\
activation=layers.PReLU(), recurrent_activation='sigmoid',\
def UNet_like2(input_tensor=None):
img_input = input_tensor
### Conv1
conv1 = layers.Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv1_1')(img_input)
conv1 = layers.BatchNormalization(axis=3, name='conv1_1bn')(conv1)
conv1 = layers.PReLU(shared_axes=[1, 2], name='prelu1_1')(conv1)
conv1 = layers.Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv1_2')(conv1)
conv1 = layers.BatchNormalization(axis=3, name='conv1_2bn')(conv1)
conv1 = layers.PReLU(shared_axes=[1, 2], name='prelu1_2')(conv1)
pool1 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(conv1)
### Conv 2
conv2 = layers.Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv2_1')(pool1)
conv2 = layers.BatchNormalization(axis=3, name='conv2_1bn')(conv2)
conv2 = layers.PReLU(shared_axes=[1, 2], name='prelu2_1')(conv2)
conv2 = layers.Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv2_2')(conv2)
conv2 = layers.BatchNormalization(axis=3, name='conv2_2bn')(conv2)
conv2 = layers.PReLU(shared_axes=[1, 2], name='prelu2_2')(conv2)
pool2 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(conv2)
### Conv 3
conv3 = layers.Conv2D(256, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv3_1')(pool2)
conv3 = layers.BatchNormalization(axis=3, name='conv3_1bn')(conv3)
conv3 = layers.PReLU(shared_axes=[1, 2], name='prelu3_1')(conv3)
conv3 = layers.Conv2D(256, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv3_2')(conv3)
conv3 = layers.BatchNormalization(axis=3, name='conv3_2bn')(conv3)
conv3 = layers.PReLU(shared_axes=[1, 2], name='prelu3_2')(conv3)
pool3 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(conv3)
### Conv 4
conv4 = layers.Conv2D(512, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv4_1')(pool3)
conv4 = layers.BatchNormalization(axis=3, name='conv4_1bn')(conv4)
conv4 = layers.PReLU(shared_axes=[1, 2], name='prelu4_1')(conv4)
conv4 = layers.Conv2D(512, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv4_2')(conv4)
conv4 = layers.BatchNormalization(axis=3, name='conv4_2bn')(conv4)
conv4 = layers.PReLU(shared_axes=[1, 2], name='prelu4_2')(conv4)
drop4 = layers.Dropout(0.5)(conv4) ###
pool4 = layers.MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(drop4)
### Conv 5
conv5 = layers.Conv2D(1024, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv5_1')(pool4)
conv5 = layers.BatchNormalization(axis=3, name='conv5_1bn')(conv5)
conv5 = layers.PReLU(shared_axes=[1, 2], name='prelu5_1')(conv5)
conv5 = layers.Conv2D(1024, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='conv5_2')(conv5)
conv5 = layers.BatchNormalization(axis=3, name='conv5_2bn')(conv5)
conv5 = layers.PReLU(shared_axes=[1, 2], name='prelu5_2')(conv5)
drop5 = layers.Dropout(0.5)(conv5) ###
### upconv + conv 6
upconv6 = layers.Conv2D(512, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv6_1')(
layers.UpSampling2D(size=(2, 2))(drop5))
upconv6 = layers.PReLU(shared_axes=[1, 2], name='prelu6_1')(upconv6)
merge6 = layers.concatenate([drop4, upconv6], axis=3)
conv6 = layers.Conv2D(512, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv6_2')(merge6)
conv6 = layers.PReLU(shared_axes=[1, 2], name='prelu6_2')(conv6)
conv6 = layers.Conv2D(512, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv6_3')(conv6)
conv6 = layers.PReLU(shared_axes=[1, 2], name='prelu6_3')(conv6)
### Upconv + Conv 7
upconv7 = layers.Conv2D(256, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv7_1')(
layers.UpSampling2D(size=(2, 2))(conv6))
upconv7 = layers.PReLU(shared_axes=[1, 2], name='prelu7_1')(upconv7)
merge7 = layers.concatenate([conv3, upconv7], axis=3)
conv7 = layers.Conv2D(256, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv7_2')(merge7)
conv7 = layers.PReLU(shared_axes=[1, 2], name='prelu7_2')(conv7)
conv7 = layers.Conv2D(256, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv7_3')(conv7)
conv7 = layers.PReLU(shared_axes=[1, 2], name='prelu7_3')(conv7)
### Upconv + Conv 8
upconv8 = layers.Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv8_1')(
layers.UpSampling2D(size=(2, 2))(conv7))
upconv8 = layers.PReLU(shared_axes=[1, 2], name='prelu8_1')(upconv8)
merge8 = layers.concatenate([conv2, upconv8], axis=3)
conv8 = layers.Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv8_2')(merge8)
conv8 = layers.PReLU(shared_axes=[1, 2], name='prelu8_2')(conv8)
conv8 = layers.Conv2D(128, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv8_3')(conv8)
conv8 = layers.PReLU(shared_axes=[1, 2], name='prelu8_3')(conv8)
### Upconv + Conv 9
upconv9 = layers.Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv9_1')(
layers.UpSampling2D(size=(2, 2))(conv8))
upconv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_1')(upconv9)
merge9 = layers.concatenate([conv1, upconv9], axis=3)
conv9 = layers.Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv9_2')(merge9)
conv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_2')(conv9)
conv9 = layers.Conv2D(64, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv9_3')(conv9)
conv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_3')(conv9)
conv9 = layers.Conv2D(2, (3, 3),
padding='same',
kernel_initializer='he_normal',
name='upconv9_4')(conv9)
conv9 = layers.PReLU(shared_axes=[1, 2], name='prelu9_4')(conv9)
### Conv 10
conv10 = layers.Conv2D(1, (1, 1), activation='sigmoid',
name='conv10')(conv9)
# Create model
model = models.Model(img_input, conv10, name='U-Net')
return model
def WaveNetMinimal():
"""
Repurposed for my use from https://arxiv.org/abs/1609.03499
"""
dilation_depth = 8 #
nb_stacks = 1
nb_output_bins = 4
nb_filters = 64
use_bias = False
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_tanh_s%d' %
(2**i, s),
activation='tanh')(x)
x = layers.Dropout(0.2)(x)
sigm_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_sigm_s%d' %
(2**i, s),
activation='sigmoid')(x)
x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' %
(i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias)(x)
res_x = layers.Merge(mode='sum')([original_x, res_x])
return res_x
_log.info('Building model...')
input = Input(shape=(1000, nb_output_bins), name='input_part')
out = input
out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=1,
border_mode='valid',
causal=True,
name='initial_causal_conv')(out)
for s in range(nb_stacks):
for i in range(0, dilation_depth + 1):
out = residual_block(out)
out = layers.PReLU()(out)
out = layers.Convolution1D(nb_filter=64,
filter_length=3,
border_mode='same',
init='he_normal')(out)
out = layers.Dropout(0.2)(out)
out = layers.Activation('relu')(out)
out = layers.Convolution1D(nb_filter=64,
filter_length=3,
border_mode='same',
init='he_normal')(out)
out = layers.Dropout(0.2)(out)
out = layers.Activation('relu')(out)
out = layers.Flatten()(out)
predictions = layers.Dense(919, name='fc1')(out)
predictions = layers.Activation('sigmoid',
name="output_sigmoid")(predictions)
model = Model(input, predictions)
_log.info('Compiling model...')
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
def run_script():
# Tunable parameters
TRAINING_SIZE = 5000
TEST_SIZE = 1000
INPUT_LEN = 10 # The maximum number of digits in the input integers
DECIMALS = 2 # the number of decimals in the scientific notation
# This number is fixed
OUTPUT_LEN = 6 + DECIMALS
chars = '0123456789e+. '
ctable = CharacterTable(chars)
print('Generating data...')
questions, expected = generate_data_set(TRAINING_SIZE+TEST_SIZE, INPUT_LEN, DECIMALS)
################################################################################
################################################################################
################################################################################
print('Vectorization...')
x = np.zeros((len(questions), INPUT_LEN, len(chars)), dtype=np.bool)
y = np.zeros((len(questions), OUTPUT_LEN, len(chars)), dtype=np.bool)
# Encode all inputs and outputs (i.e. turn strings into matrices)
for i, sentence in enumerate(questions):
x[i] = ctable.encode(sentence, INPUT_LEN)
for i, sentence in enumerate(expected):
y[i] = ctable.encode(sentence, OUTPUT_LEN)
# Shuffle (x, y) in unison as the later parts of x will almost all be larger
# digits.
"""
indices = np.arange(len(y))
np.random.shuffle(indices)
x = x[indices]
y = y[indices]
"""
# Split the data over the training set (90% training and 10% validation) and the test set
split1 = int(0.9*TRAINING_SIZE)
split2 = TRAINING_SIZE
(x_train, x_val, x_test) = x[:split1], x[split1:split2], x[split2:]
(y_train, y_val, y_test) = y[:split1], y[split1:split2], y[split2:]
print('Training Data:')
print(x_train.shape)
print(y_train.shape)
print('Validation Data:')
print(x_val.shape)
print(y_val.shape)
print('Test Data:')
print(x_val.shape)
print(y_val.shape)
################################################################################
################################################################################
################################################################################
# Try replacing GRU, or SimpleRNN.
RNN = layers.LSTM
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 1
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
# Note: In a situation where your input sequences have a variable length,
# use input_shape=(None, num_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(INPUT_LEN, len(chars)),\
activation=layers.PReLU(), recurrent_activation='sigmoid',\
dropout=0.25, recurrent_dropout=0.125))
model.add(BatchNormalization(center=True, scale=True))
# As the decoder RNN's input, repeatedly provide with the last output of
# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
model.add(layers.RepeatVector(OUTPUT_LEN))
model.add(BatchNormalization(center=True, scale=True))
# The decoder RNN could be multiple layers stacked or a single layer.
for _ in range(LAYERS):
# By setting return_sequences to True, return not only the last output but
# all the outputs so far in the form of (num_samples, timesteps,
# output_dim). This is necessary as TimeDistributed in the below expects
# the first dimension to be the timesteps.
model.add(RNN(HIDDEN_SIZE, return_sequences=True,\
activation=layers.PReLU(), recurrent_activation='sigmoid',\
dropout=0.25, recurrent_dropout=0.125))
model.add(BatchNormalization(center=True, scale=True))
# Apply a dense layer to the every temporal slice of an input. For each of step
# of the output sequence, decide which character should be chosen.
model.add(layers.TimeDistributed(layers.Dense(len(chars), activation='softmax')))
model.compile(loss='categorical_crossentropy',
optimizer='Nadam',
metrics=['accuracy'])
model.summary()
#training_accuracies = []
#training_losses = []
val_all = []
val_all_but_one = []
# Train the model each generation and show predictions against the validation
# dataset.
for iteration in range(1, 150):
print()
print('-' * 50)
print('Iteration', iteration)
hist = model.fit(x_train, y_train,
batch_size=BATCH_SIZE,
epochs=1,
validation_data=(x_val, y_val))
# Select 10 samples from the validation set at random so we can visualize
# errors.
#print(hist.history)
#training_accuracies.append([hist.history['acc'][0], hist.history['val_acc'][0]])
#training_losses.append([hist.history['loss'][0], hist.history['val_loss'][0]])
for i in range(10):
ind = np.random.randint(0, len(x_val))
rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]
preds = model.predict_classes(rowx, verbose=0)
q = ctable.decode(rowx[0])
correct = ctable.decode(rowy[0])
guess = ctable.decode(preds[0], calc_argmax=False)
print('Q', q, end=' ')
print('T', correct, end=' ')
if correct == guess:
print('OK', end=' ')
else:
print('..', end=' ')
print(guess)
full, one_off = 0, 0
predict = model.predict_classes(x_val, verbose=0)
for i in range(len(x_val)):
correct = ctable.decode(y_val[i])
guess = ctable.decode(predict[i], calc_argmax=False)
if correct == guess:
full += 1
elif match(correct, guess):
one_off += 1
if (iteration > 50):
q = ctable.decode(x_val[i])
print('Q', q, end=' ')
print('T', correct, end=' ')
print('..', end=' ')
print(guess)
print('{}% of validation examples are completely correct'.format(100.
*float(full)/len(x_val)))
print('{}% of validation examples are one off'.format(100.*float(one_off)/len(x_val)))
val_all.append(100.*float(full)/len(x_val))
val_all_but_one.append(100.*float(one_off)/len(x_val))
################################################################################
################################################################################
################################################################################
scores = model.evaluate(x_test, y_test, verbose=1)
print('-----------------------------------------')
print('Test accuracy: ', scores[1])
#np.save('training_accuracies', np.array(training_accuracies))
#np.save('training_losses', np.array(training_losses))
np.save('results/learn_rule/val_all_'+str(INPUT_LEN), np.array(val_all))
np.save('results/learn_rule/val_all_but_one_'+str(INPUT_LEN), np.array(val_all_but_one))
full, one_off = 0, 0
predict = model.predict_classes(x_test, verbose=0)
for i in range(len(x_test)):
correct = ctable.decode(y_test[i])
guess = ctable.decode(predict[i], calc_argmax=False)
if correct == guess:
full += 1
elif match(correct, guess):
one_off += 1
print('{}% of test examples are completely correct'.format(100.
*float(full)/len(x_test)))
print('{}% of test examples are one off'.format(100.*float(one_off)/len(x_test)))
print(100.*float(full)/len(x_test), 100.*float(one_off)/len(x_test))
def create_model(desired_sample_rate, dilation_depth, nb_stacks):
# desired_sample_rate = 4410
nb_output_bins = 4
# nb_filters = 256
nb_filters = 64
# dilation_depth = 9 #
# nb_stacks = 1
use_bias = False
res_l2 = 0
final_l2 = 0
fragment_length = 488 + compute_receptive_field_(
desired_sample_rate, dilation_depth, nb_stacks)[0]
fragment_stride = 488
use_skip_connections = True
learn_all_outputs = True
def residual_block(x):
original_x = x
# TODO: initalization, regularization?
# Note: The AtrousConvolution1D with the 'causal' flag is implemented in github.com/basveeling/keras#@wavenet.
tanh_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_tanh_s%d' %
(2**i, s),
activation='tanh',
W_regularizer=l2(res_l2))(x)
x = layers.Dropout(0.2)(x)
sigm_out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=2**i,
border_mode='valid',
causal=True,
bias=use_bias,
name='dilated_conv_%d_sigm_s%d' %
(2**i, s),
activation='sigmoid',
W_regularizer=l2(res_l2))(x)
x = layers.Merge(mode='mul', name='gated_activation_%d_s%d' %
(i, s))([tanh_out, sigm_out])
res_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias,
W_regularizer=l2(res_l2))(x)
skip_x = layers.Convolution1D(nb_filters,
1,
border_mode='same',
bias=use_bias,
W_regularizer=l2(res_l2))(x)
res_x = layers.Merge(mode='sum')([original_x, res_x])
return res_x, skip_x
input = Input(shape=(fragment_length, nb_output_bins), name='input_part')
out = input
skip_connections = []
out = CausalAtrousConvolution1D(nb_filters,
2,
atrous_rate=1,
border_mode='valid',
causal=True,
name='initial_causal_conv')(out)
for s in range(nb_stacks):
for i in range(0, dilation_depth + 1):
out, skip_out = residual_block(out)
skip_connections.append(skip_out)
if use_skip_connections:
out = layers.Merge(mode='sum')(skip_connections)
out = layers.PReLU()(out)
# out = layers.Convolution1D(nb_filter=256, filter_length=1, border_mode='same',
# W_regularizer=l2(final_l2))(out)
out = layers.Convolution1D(nb_filter=nb_output_bins,
filter_length=3,
border_mode='same')(out)
out = layers.Dropout(0.5)(out)
out = layers.PReLU()(out)
out = layers.Convolution1D(nb_filter=nb_output_bins,
filter_length=3,
border_mode='same')(out)
if not learn_all_outputs:
raise DeprecationWarning(
'Learning on just all outputs is wasteful, now learning only inside receptive field.'
)
out = layers.Lambda(
lambda x: x[:, -1, :], output_shape=(out._keras_shape[-1], ))(
out) # Based on gif in deepmind blog: take last output?
# out = layers.Activation('softmax', name="output_softmax")(out)
out = layers.PReLU()(out)
# out = layers.Activation('sigmoid', name="output_sigmoid")(out)
out = layers.Flatten()(out)
predictions = layers.Dense(919, activation='sigmoid', name='fc1')(out)
model = Model(input, predictions)
# x = model.output
# x = layers.Flatten()(x)
# # x = layers.Dense(output_dim=1024)(x)
# # x = layers.PReLU()(x)
# # x = layers.Dropout(0.5)(x)
# # x = layers.Dense(output_dim=919)(x)
# # x = layers.Activation('sigmoid')(x)
# model = Model(input=model.input, output=predictions)
receptive_field, receptive_field_ms = compute_receptive_field_(
desired_sample_rate, dilation_depth, nb_stacks)
_log.info('Receptive Field: %d (%dms)' %
(receptive_field, int(receptive_field_ms)))
return model
def coa_res(model_weights = None):
EMBED_HIDDEN_SIZE = 300
#shared weight layers
shared_LSTM = LSTM(EMBED_HIDDEN_SIZE, return_sequences=True)
###########################
sentence = layers.Input(shape=(max_para,dimension), dtype='float32')
encoded_sentence =shared_LSTM(sentence)
question = layers.Input(shape=(max_q,dimension), dtype='float32')
encoded_question = shared_LSTM(question)
#Encoder
merge_1 = layers.dot([encoded_sentence, encoded_question], axes = 2 )
A_Q = layers.Activation("softmax")(merge_1)
merge_2 = layers.dot([encoded_question, encoded_sentence], axes = 2 )
A_D = layers.Activation("softmax")(merge_2)
C_Q = layers.dot([A_Q, encoded_sentence], axes = 1 )
C_Q = layers.concatenate([encoded_question, C_Q], axis=2)
C_D = layers.dot([A_D, C_Q], axes=1)
C_ = layers.concatenate([encoded_sentence, C_D], axis=2)
U = Bidirectional(LSTM(EMBED_HIDDEN_SIZE, return_sequences=True))(C_)
U = Dropout(0.5)(U)
#Decoder
start = convBN(U, 100)
start = RU(start, 100, 1)
start = Dropout(0.5)(start)
start = convBN_pool(start, 64)
start = RU(start, 64, 1)
start = convBN_pool(start, 64)
start = RU(start, 64, 2)
start = convBN_pool(start, 128)
start = RU(start, 128, 1)
start =layers.PReLU()(start)
start = RU(start, 128, 2)
start = Dropout(0.5)(start)
start = convBN_pool(start, 256)
start = RU(start, 256, 2)
start = Dropout(0.5)(start)
start = convBN_pool(start, 128)
start = RU(start, 128, 1)
start = layers.PReLU()(start)
start = Dropout(0.5)(start)
start = convBN(start, 64)
start = RU(start, 64, 1)
start = layers.PReLU()(start)
start = Flatten()(start)
start = Dropout(0.5)(start)
start = Dense(max_para, activation='softmax', name='output_1')(start)
end = GRU(100, return_sequences=True)(U)
end = convBN(end, 100)
end = RU(end, 100, 1)
end = Dropout(0.5)(end)
end = convBN_pool(end, 64)
end = RU(end, 64, 1)
end = convBN_pool(end, 64)
end = RU(end, 64, 2)
end = convBN_pool(end, 128)
end = RU(end, 128, 1)
end = layers.PReLU()(end)
end = RU(end, 128, 2)
end = Dropout(0.5)(end)
end = convBN_pool(end, 256)
end = RU(end, 256, 2)
end = Dropout(0.5)(end)
end = convBN_pool(end, 128)
end = RU(end, 128, 1)
end = layers.PReLU()(end)
end = Dropout(0.5)(end)
end = convBN(end, 64)
end = RU(end, 64, 1)
end = layers.PReLU()(end)
end = Flatten()(end)
end = Dropout(0.5)(end)
end = Dense(max_para, activation='softmax', name='output_2')(end)
model = Model([sentence, question],[start, end])
if model_weights != None:
model.load_weights(model_weights)
model.compile(optimizer='adam',
loss={'output_1': 'categorical_crossentropy', 'output_2': 'categorical_crossentropy'},
metrics=['accuracy'])
return model
