def lstm(embedding_matrix=np.zeros(0),
max_len=21,
input_dim=111857,
max_features=50000,
units=150,
num_filter=128):
inp1 = Input(shape=(max_len, ), dtype='int32')
inp2 = Input(shape=(max_len, ), dtype='int32')
x1 = Embedding(input_dim=input_dim,
output_dim=units * 2,
input_length=max_len,
trainable=True)(inp1)
x1 = SpatialDropout1D(rate=0.24)(x1)
x2 = Embedding(input_dim=input_dim,
output_dim=units * 2,
input_length=max_len,
trainable=True)(inp2)
x2 = SpatialDropout1D(rate=0.24)(x2)
x1 = Bidirectional(layer=LSTM(units,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0,
seed=1029)),
name='bidirectional_lstm1')(x1)
x2 = Bidirectional(layer=LSTM(units,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0,
seed=1029)),
name='bidirectional_lstm2')(x2)
x1 = Reshape((max_len, units * 2))(x1)
x2 = Reshape((max_len, units * 2))(x2)
x1 = Flatten()(x1)
x2 = Flatten()(x2)
diff = abs_layer(Subtract()([x1, x2]))
mul = Multiply()([x1, x2])
outputp = Concatenate()([x1, x2, diff, mul])
outputp = Dense(int(units * 2))(outputp)
outputp = BatchNormalization()(outputp)
outputp = ReLU()(outputp)
finalout = Dense(1, activation='sigmoid')(outputp)
model = Model(inputs=[inp1, inp2], outputs=finalout)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def capsule(maxlen, max_features, embed_size, embedding_matrix, num_classes):
K.clear_session()
inp = Input(shape=(maxlen, ))
x = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=False)(inp)
x = SpatialDropout1D(rate=0.2)(x)
x = Bidirectional(
GRU(100,
return_sequences=True,
kernel_initializer=glorot_normal(seed=12300),
recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x)
x = Capsule(num_capsule=10, dim_capsule=10, routings=4,
share_weights=True)(x)
x = Flatten()(x)
x = Dense(100,
activation="relu",
kernel_initializer=glorot_normal(seed=12300))(x)
x = Dropout(0.12)(x)
x = BatchNormalization()(x)
x = Dense(num_classes, activation="sigmoid")(x)
model = Model(inputs=inp, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer=Adam(),
metrics=['accuracy'])
return model
def create_model(self):
model = Sequential()
print('#3')
model.add(
Embedding(self.input_dim,
self.vec_dim,
input_length=self.maxlen,
trainable=True,
embeddings_initializer=uniform(seed=20170719)))
model.add(BatchNormalization(axis=-1))
print('#4')
model.add(
Masking(mask_value=0, input_shape=(self.maxlen, self.vec_dim)))
model.add(
LSTM(self.n_hidden,
batch_input_shape=(None, self.maxlen, self.vec_dim),
kernel_initializer=glorot_uniform(seed=20170719),
recurrent_initializer=orthogonal(gain=1.0, seed=20170719)))
print('#5')
model.add(BatchNormalization(axis=-1))
print('#6')
model.add(
Dense(self.output_dim,
activation='sigmoid',
use_bias=True,
kernel_initializer=glorot_uniform(seed=20170719)))
model.compile(loss="binary_crossentropy",
optimizer="RMSprop",
metrics=['binary_accuracy'])
return model
def build_fasttext(self):
"""
可用的initialization方法：random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform()
:return:
"""
initial_dict = {
'orthogonal': orthogonal(),
'he_n': he_normal(),
'he_u': he_uniform()
}
# model.add(Embedding(output_dim=self.hidden_layer, input_length=self.max_sequence_length,
# input_dim=self.max_nb_words, embeddings_initializer=initial_dict['he_n'], trainable=True))
# model.add(GlobalAveragePooling1D())
# model.add(Dense(self.n_classes, activation='softmax', kernel_initializer=initial_dict['he_n']))
self.model.add(
Embedding(output_dim=self.hidden_layer,
input_length=self.max_sequence_length,
input_dim=self.max_nb_words,
trainable=True))
self.model.add(GlobalAveragePooling1D())
self.model.add(Dense(self.n_classes, activation='softmax'))
print(self.model.summary())
self.compile()
def attention_capsule(maxlen, max_features, embed_size, embedding_matrix,
num_classes):
inp = Input(shape=(maxlen, ))
x = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=False)(inp)
x = SpatialDropout1D(rate=0.24)(x)
x = Bidirectional(
LSTM(80,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x)
x_1 = Attention(maxlen)(x)
x_1 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_1)
x_2 = Capsule(num_capsule=10,
dim_capsule=10,
routings=4,
share_weights=True)(x)
x_2 = Flatten()(x_2)
x_2 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_2)
conc = concatenate([x_1, x_2])
# conc = add([x_1, x_2])
outp = Dense(num_classes, activation="sigmoid")(conc)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy',
optimizer=Adam(),
metrics=['accuracy'])
return model
def build_my_model(m, n):
inp = Input(shape=(m, ))
x = SpatialDropout1D(rate=0.24)(inp)
x = Bidirectional(
CuDNNLSTM(80,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x)
x_1 = Attention(m)(x)
x_1 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_1)
x_2 = Capsule(num_capsule=10,
dim_capsule=10,
routings=4,
share_weights=True)(x)
x_2 = Flatten()(x_2)
x_2 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_2)
conc = concatenate([x_1, x_2])
# conc = add([x_1, x_2])
outp = Dense(1, activation="sigmoid")(conc)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=[f1])
return model
def build(self, input_shape):
assert len(input_shape) >= 2
m = input_shape[0][-1]
n = input_shape[0][-1]
self.W1 = self.add_weight(name='mixed_agg_weight1',
shape=(m, n),
initializer=orthogonal(seed=9),
trainable=True)
self.W2 = self.add_weight(name='mixed_agg_weight2',
shape=(n, n),
initializer=orthogonal(seed=9),
trainable=True)
self.vm = self.add_weight(name='vmix',
shape=(1, n),
initializer=random_normal(seed=9),
trainable=True)
super(MixedAggregation, self).build(input_shape)
def Transpose_Conv2D(x, filters, kernel, strides, padding, block_id, kernel_init=orthogonal()):
prefix = f'block_{block_id}_'
x = layers.Conv2DTranspose(filters, kernel_size=kernel, strides=strides, padding=padding,
kernel_initializer=kernel_init, name=prefix+'de-conv')(x)
x = layers.LeakyReLU(name=prefix+'lrelu')(x)
x = layers.Dropout(0.2, name=prefix+'drop')((x))
x = layers.BatchNormalization(name=prefix+'conv_bn')(x)
return x
def build(self, input_shape):
assert len(input_shape) >= 2
self.W = self.add_weight(name='inside_agg_weight',
shape=(input_shape[-1], input_shape[-1]),
initializer=orthogonal(seed=9),
trainable=True)
super(InsideAggregation, self).build(input_shape)
def custom_model(data_shape,
layers=1,
units1=128,
units2=128,
units3=128,
units4=128,
units5=128,
optim='rmsprop',
metrics=['accuracy']):
units = {1: units1, 2: units2, 3: units3, 4: units4, 5: units5}
model = Sequential()
model.add(Masking(mask_value=0.0, input_shape=data_shape))
return_sequences = True
for i in range(layers):
if i + 1 >= layers:
return_sequences = False
if i == 0:
model.add(
LSTM(units[i + 1],
input_shape=data_shape,
return_sequences=return_sequences,
kernel_regularizer=l2(0.01),
recurrent_regularizer=l2(0.01),
kernel_initializer=glorot_uniform(seed=42),
recurrent_initializer=orthogonal(seed=42)))
#model.add(Dropout(rate=.1))
else:
model.add(
LSTM(units[i + 1],
return_sequences=return_sequences,
kernel_regularizer=l2(0.01),
recurrent_regularizer=l2(0.01),
kernel_initializer=glorot_uniform(seed=42),
recurrent_initializer=orthogonal(seed=42)))
#model.add(Dropout(rate=.1))
model.add(
Dense(2,
activation='softmax',
kernel_initializer=glorot_uniform(seed=42),
kernel_regularizer=l2(0.01)))
model.compile(loss='binary_crossentropy', optimizer=optim, metrics=metrics)
return model
def _build_fn_orthogonal_i_5(input_shape): # `orthogonal(gain=Real(0.6, 1.6))`
model = Sequential(
[
Dense(Integer(50, 100), input_shape=input_shape),
Dense(1, kernel_initializer=orthogonal(gain=Real(0.6, 1.6))),
]
)
model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
return model
def _build_fn_categorical_3(input_shape): # `Categorical(["glorot_normal", orthogonal(gain=1)])`
model = Sequential(
[
Dense(Integer(50, 100), input_shape=input_shape),
Dense(1, kernel_initializer=Categorical(["glorot_normal", orthogonal(gain=1)])),
]
)
model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
return model
def build(self, input_shape):
assert len(input_shape) == 4
m1 = input_shape[-2]
m2 = input_shape[-1]
self.W = self.add_weight(name='merge_weight',
shape=(m1, m2),
initializer=orthogonal(seed=9),
trainable=True)
super(MergeChannel, self).build(input_shape)
def test_orthogonal_init_does_not_affect_global_rng():
np.random.seed(1337)
before = np.random.randint(0, 100, size=10)
np.random.seed(1337)
init = initializers.orthogonal(seed=9876)
init(shape=(10, 5))
after = np.random.randint(0, 100, size=10)
assert np.array_equal(before, after)
def test_orthogonal_init_does_not_affect_global_rng():
np.random.seed(1337)
before = np.random.randint(0, 100, size=10)
np.random.seed(1337)
init = initializers.orthogonal(seed=9876)
init(shape=(10, 5))
after = np.random.randint(0, 100, size=10)
assert np.array_equal(before, after)
def build(self, input_shape):
assert len(input_shape) == 2
assert len(input_shape[0]) == 3
assert len(input_shape[1]) == 3
m1 = input_shape[0][-1]
m2 = input_shape[1][-1]
self.W = self.add_weight(name='interaction_weight',
shape=(m1, m2),
initializer=orthogonal(seed=9),
trainable=True)
super(MatrixInteraction, self).build(input_shape)
def Model2(embedding_matrix=None):
# wt = [embedding_matrix] if embedding_matrix != None else None
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=embedding_matrix,
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
net = SpatialDropout1D(0.2)(embedded_sequences)
net = Conv1D(100, 3, padding='same', activation='relu')(net)
net = BatchNormalization()(net)
# net=GRU(EMBEDDING_DIM, return_sequences=True,
# kernel_initializer=glorot_normal(seed=1029),
# recurrent_initializer=orthogonal(gain=1.0, seed=1029))(net)
net = Bidirectional(layer=GRU(100,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0,
seed=1029)),
name='bidirectional_gru')(net)
# net = Bidirectional(
# layer=LSTM(EMBEDDING_DIM, return_sequences=True,
# kernel_initializer=glorot_normal(seed=1029),
# recurrent_initializer=orthogonal(gain=1.0, seed=1029)),
# name='bidirectional_lstm')(net)
#net = BatchNormalization()(net)
capsul = Capsule(num_capsule=10,
dim_capsule=10,
routings=4,
share_weights=True)(net) # noqa
capsul = Flatten()(capsul)
capsul = DropConnect(Dense(32, activation="relu"), prob=0.2)(capsul)
atten = Attention(step_dim=MAX_SEQUENCE_LENGTH, name='attention')(net)
atten = DropConnect(Dense(16, activation="relu"), prob=0.2)(atten)
net = Concatenate(axis=-1)([capsul, atten])
# net = GlobalAveragePooling1D()(net)
# output = Dense(units=1, activation='sigmoid', name='output')(net)
# net = GlobalAveragePooling1D()(net)
net = Dense(100, activation='relu')(net)
# net = Dropout(0.3)(net)
output = Dense(2, activation='softmax')(net)
model4 = Model(inputs=sequence_input, outputs=output)
model4.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['acc'])
model4.summary()
chkpt4 = ModelCheckpoint('expertiza_nn_model3.h5',
monitor='val_acc',
verbose=1,
save_best_only=True)
return (model4, chkpt4)
def build(self, input_shape):
assert len(input_shape) == 2
assert len(input_shape[0]) == 3
assert len(input_shape[1]) == 3
dim = max(input_shape[0][-1], input_shape[1][-1])
self.W0 = self.add_weight(name='weight0',
shape=(input_shape[0][-1], dim),
initializer=orthogonal(seed=9),
trainable=True)
self.W1 = self.add_weight(name='weight1',
shape=(input_shape[1][-1], dim),
initializer=orthogonal(seed=9),
trainable=True)
self.vc = self.add_weight(name='vc',
shape=(dim, 1),
initializer=random_normal(seed=9),
trainable=True)
super(AttentionConcat, self).build(input_shape)
def build(self, input_shape):
assert len(input_shape) == 2
assert len(input_shape[0]) == 3
assert len(input_shape[1]) == 3
self.W = self.add_weight(name='bilinear_weight',
shape=(input_shape[1][-1],
input_shape[0][-1]),
initializer=orthogonal(seed=9),
trainable=True)
super(AttentionBilinear, self).build(input_shape)
def create_sequence_model(embedding_matrix, word_index, max_words):
"""This function creates a deep learning model for the Quora competition.
Parameters
----------
embedding_matrix: np.array
The matrix of embeddings used in the embedding layer
word_index: dict
The word_index obtained by the tokenizer
max_words: int
The maximum number of words in a sequence
Returns
-------
A deep learning model
"""
input_layer = Input(shape=(75, ), name='input_layer')
x = Embedding(len(word_index) + 1,
embedding_matrix.shape[1],
weights=[embedding_matrix],
trainable=False,
input_shape=(max_words, ))(input_layer)
x = SpatialDropout1D(rate=0.24)(x)
x = Bidirectional(layer=LSTM(80,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0,
seed=1029)),
name='bidirectional_lstm')(x)
# Capsule layer
capsule = Capsule(num_capsule=10,
dim_capsule=10,
routings=4,
share_weights=True)(x) # noqa
capsule = Flatten()(capsule)
capsule = DropConnect(Dense(32, activation="relu"), prob=0.01)(capsule)
# Attention layer
atten = Attention(step_dim=75, name='attention')(x)
atten = DropConnect(Dense(16, activation="relu"), prob=0.05)(atten)
# Concatenate Capsule and Attention layer
x = Concatenate(axis=-1)([capsule, atten])
output_layer = Dense(units=1, activation='sigmoid', name='output')(x)
model = Model(inputs=input_layer, outputs=output_layer)
# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam')
return model
def create_model(self):
model = Sequential()
model.add(LSTM(self.n_hidden,
batch_input_shape=(None, self.maxlen, self.n_in),
kernel_initializer=glorot_uniform(seed=20170719),
recurrent_initializer=orthogonal(gain=1.0, seed=20170719),
dropout=0.3,
recurrent_dropout=0.3))
model.add(Dropout(0.3))
model.add(Dense(1,kernel_initializer=glorot_uniform(seed=20170719)))
model.add(Activation("linear"))
model.compile(loss="mean_squared_error", optimizer="RMSprop", metrics=['accuracy'])
return model
def create_model(self):
model = Sequential()
model.add(LSTM(self.n_hidden, batch_input_shape=(None, self.maxlen, self.n_in),
kernel_initializer=glorot_uniform(seed=20170719),
recurrent_initializer=orthogonal(gain=1.0, seed=20170719),
dropout=0.3,
recurrent_dropout=0.3))
model.add(Dropout(0.3))
model.add(Dense(self.n_out,
kernel_initializer=glorot_uniform(seed=20170719)))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer="RMSprop", metrics=['categorical_accuracy'])
return model
def build_model(input_eular, input_crp, parameters, seed=None):
dropout = parameters['dropout']
optimizer = parameters['optimizer']
learning_rate = parameters['learning_rate']
rnn_type = parameters['rnn_type']
rnn_size = parameters['rnn_size']
rnn_activation = parameters['rnn_activation']
rnn_dropout = parameters['rnn_dropout']
dense_initializer = RandomNormal(seed=seed)
if rnn_type == 'none': x = concatenate([input_eular, input_crp], axis=1)
else:
rnn_kernel_initializer = glorot_uniform(seed=seed)
rnn_recurrent_initializer = orthogonal(seed=seed)
x = Masking(mask_value=-1)(input_eular)
if rnn_type == 'lstm':
x = LSTM(rnn_size,
activation=rnn_activation,
kernel_initializer=rnn_kernel_initializer,
recurrent_initializer=rnn_recurrent_initializer,
dropout=dropout,
recurrent_dropout=rnn_dropout,
name='LSTM_{}'.format(rnn_size))(x)
elif rnn_type == 'simplernn':
x = SimpleRNN(rnn_size,
activation=rnn_activation,
kernel_initializer=rnn_kernel_initializer,
recurrent_initializer=rnn_recurrent_initializer,
dropout=dropout,
recurrent_dropout=rnn_dropout,
name='SRNN_{}'.format(rnn_size))(x)
x = concatenate([x, input_crp])
for layer_size in [val for val in parameters["dense_layers"] if val != 0]:
x = Dense(layer_size,
activation='linear',
kernel_initializer=dense_initializer)(x)
x = get_activation(x, parameters['activation'])(x)
if dropout != 0:
x = Dropout(rate=dropout)(x)
output = Dense(1, kernel_initializer=dense_initializer)(x)
output = get_activation(output, parameters['last_activation'])(output)
model = Model(input=[input_eular, input_crp], output=output)
return model
def create_sharedmodel(self):
inputs = Input(shape=(self.maxlen, self.height, self.width, 3))
x = TimeDistributed(self.resnet,name="resnet")(inputs)
x = TimeDistributed(GlobalAveragePooling2D(), name="GAP")(x)
x = TimeDistributed(Dense(512,activation='relu'), name="dense")(x)
predictions = Bidirectional(LSTM(128, batch_input_shape = (None, self.maxlen, 512),
kernel_initializer = glorot_normal(seed=20181020),
recurrent_initializer = orthogonal(gain=1.0, seed=20181020),
dropout = 0.01,
recurrent_dropout = 0.01))(x)
shared_layers = Model(inputs, predictions, name="shared_layers")
return shared_layers
def build(self, input_shape):
assert len(input_shape) == 2
assert len(input_shape[0]) == 3
assert len(input_shape[1]) == 3
assert input_shape[0][-1] == input_shape[1][-1]
m = input_shape[0][-1]
n = input_shape[0][-1]
self.W = self.add_weight(name='minus_weight',
shape=(m, n),
initializer=orthogonal(seed=9),
trainable=True)
self.vm = self.add_weight(name='vd',
shape=(n, 1),
initializer=random_normal(seed=9),
trainable=True)
super(AttentionMinus, self).build(input_shape)
def create_model(self):
model = Sequential()
model.add(
LSTM(self.n_hidden,
batch_input_shape=(None, self.maxlen, self.n_in),
kernel_initializer=glorot_uniform(seed=20170719),
recurrent_initializer=orthogonal(gain=1.0, seed=20170719),
dropout=0.5,
recurrent_dropout=0.5))
model.add(Dropout(0.5))
model.add(
Dense(self.n_out,
kernel_initializer=glorot_uniform(seed=20170719)))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy",
optimizer="RMSprop",
metrics=['categorical_accuracy'])
return model
def build(self, input_shape):
self.kernel = self.add_weight(name='kernel',
shape=(input_shape[1][2], self.units),
initializer=initializers.orthogonal(),
regularizer=regularizers.l2(5e-4))
self.bias = self.add_weight(name='bias',
shape=(self.units, ),
initializer=initializers.zeros())
if self.learn_pqr:
self.p = self.add_weight(name='p',
shape=(1, ),
initializer=initializers.constant(0))
self.q = self.add_weight(name='q',
shape=(1, ),
initializer=initializers.constant(0))
# self.trainable_weights = [self.p, self.q]
super(MyGCN, self).build(input_shape)
def get_model(embed_weights):
input_layer = Input(shape=(MAX_LEN, ), name='input')
# 1. embedding layer
# get embedding weights
print('load pre-trained embedding weights ......')
input_dim = embed_weights.shape[0]
output_dim = embed_weights.shape[1]
x = Embedding(input_dim=input_dim,
output_dim=output_dim,
weights=[embed_weights],
trainable=False,
name='embedding')(input_layer)
# clean up
del embed_weights, input_dim, output_dim
gc.collect()
# 2. dropout
x = SpatialDropout1D(rate=SPATIAL_DROPOUT)(x)
# 3. bidirectional lstm
x = Bidirectional(layer=CuDNNLSTM(
RNN_UNITS,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)),
name='bidirectional_lstm')(x)
# 4. capsule layer
capsul = Capsule(num_capsule=10,
dim_capsule=10,
routings=4,
share_weights=True)(x) # noqa
capsul = Flatten()(capsul)
capsul = DropConnect(Dense(32, activation="relu"), prob=0.01)(capsul)
# 5. attention later
atten = Attention(step_dim=MAX_LEN, name='attention')(x)
atten = DropConnect(Dense(16, activation="relu"), prob=0.05)(atten)
x = Concatenate(axis=-1)([capsul, atten])
# 6. output (sigmoid)
output_layer = Dense(units=1, activation='sigmoid', name='output')(x)
model = Model(inputs=input_layer, outputs=output_layer)
# compile model
model.compile(loss='binary_crossentropy', optimizer='adam')
return model
def create_sharedAutoRegLSTMmodel(self):
inputs = Input(shape=(2 * self.maxlen, 1))
mid_lstm = Bidirectional(
LSTM(128,
batch_input_shape=(None, 2 * self.maxlen, 1),
kernel_initializer=glorot_normal(seed=20181020),
recurrent_initializer=orthogonal(gain=1.0, seed=20181020),
dropout=0.01,
recurrent_dropout=0.01,
return_sequences=True))(inputs)
mid_dense = Dense(256, activation='relu')(mid_lstm)
predictions = Dense(self.maxlen, activation='sigmoid')(mid_dense)
shared_layers = Model(inputs,
predictions,
name="shared_AutoRegLSTMlayers")
return shared_layers
def gru_crf(embedding_matrix):
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)
x = SpatialDropout1D(rate=0.24)(x)
x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=seed),
recurrent_initializer=orthogonal(gain=1.0, seed=seed)))(x)
x = CRF(10, learn_mode='marginal', unroll=True)(x)
x = Flatten()(x)
x = Dense(100, activation='relu', kernel_initializer=glorot_normal(seed=seed))(x)
x = Dropout(0.2)(x)
x = BatchNormalization()(x)
outp = Dense(1, activation='sigmoid')(x)
model = Model(inputs=inp, outputs=outp)
model.compile(loss='binary_crossentropy', optimizer='adam')
return model
def AutoEncoder(input_shape):
inputs = layers.Input(shape=input_shape)
# 256 x 256
conv1 = Conv2DLayer(inputs, 64, 3, strides=1, padding='same', block_id=1)
conv2 = Conv2DLayer(conv1, 64, 3, strides=2, padding='same', block_id=2)
# 128 x 128
conv3 = Conv2DLayer(conv2, 128, 5, strides=2, padding='same', block_id=3)
# 64 x 64
conv4 = Conv2DLayer(conv3, 128, 3, strides=1, padding='same', block_id=4)
conv5 = Conv2DLayer(conv4, 256, 5, strides=2, padding='same', block_id=5)
# 32 x 32
conv6 = Conv2DLayer(conv5, 512, 3, strides=2, padding='same', block_id=6)
# 16 x 16
deconv1 = Transpose_Conv2D(conv6, 512, 3, strides=2, padding='same', block_id=7)
# 32 x 32
skip1 = layers.concatenate([deconv1, conv5], name='skip1')
conv7 = Conv2DLayer(skip1, 256, 3, strides=1, padding='same', block_id=8)
deconv2 = Transpose_Conv2D(conv7, 128, 3, strides=2, padding='same', block_id=9)
# 64 x 64
skip2 = layers.concatenate([deconv2, conv3], name='skip2')
conv8 = Conv2DLayer(skip2, 128, 5, strides=1, padding='same', block_id=10)
deconv3 = Transpose_Conv2D(conv8, 64, 3, strides=2, padding='same', block_id=11)
# 128 x 128
skip3 = layers.concatenate([deconv3, conv2], name='skip3')
conv9 = Conv2DLayer(skip3, 64, 5, strides=1, padding='same', block_id=12)
deconv4 = Transpose_Conv2D(conv9, 64, 3, strides=2, padding='same', block_id=13)
# 256 x 256
skip3 = layers.concatenate([deconv4, conv1])
conv10 = layers.Conv2D(3, 3, strides=1, padding='same', activation='sigmoid',
kernel_initializer=orthogonal(), name='final_conv')(skip3)
return models.Model(inputs=inputs, outputs=conv10)
def get_model(embed_weights):
input_layer = Input(shape=(MAX_LEN, ), name='input')
# 1. embedding layer
# get embedding weights
print('load pre-trained embedding weights ......')
input_dim = embed_weights.shape[0]
output_dim = embed_weights.shape[1]
x = Embedding(input_dim=input_dim,
output_dim=output_dim,
weights=[embed_weights],
trainable=False,
name='embedding')(input_layer)
# clean up
del embed_weights, input_dim, output_dim
gc.collect()
# 2. dropout
x = SpatialDropout1D(rate=SPATIAL_DROPOUT)(x)
# 3. bidirectional lstm
x = Bidirectional(layer=CuDNNLSTM(
RNN_UNITS,
return_sequences=True,
kernel_initializer=glorot_normal(seed=1029),
recurrent_initializer=orthogonal(gain=1.0, seed=1029)),
name='bidirectional_lstm')(x)
# 4. capsule layer
x = Capsule(num_capsule=10,
dim_capsule=10,
routings=4,
share_weights=True,
name='capsule')(x)
x = Flatten(name='flatten')(x)
# # 5. dense with dropConnect
# x = DropConnect(
# Dense(DENSE_UNITS, activation="relu"),
# prob=0.05,
# name='dropConnect_dense')(x)
# 6. output (sigmoid)
output_layer = Dense(units=1, activation='sigmoid', name='output')(x)
model = Model(inputs=input_layer, outputs=output_layer)
# compile model
model.compile(loss='binary_crossentropy', optimizer='adam')
return model
def create_LSTM(self,
lstm_units,
lstm_return_state=False,
lstm_return_sequences=False,
lstm_go_backwards=False,
lstm_name='LSTM'):
layer = LSTM(
lstm_units,
name=lstm_name,
return_state=lstm_return_state,
return_sequences=lstm_return_sequences,
go_backwards=lstm_go_backwards,
#recurrent_regularizer=regularizers.l2(self.reg_lambda) ,
#kernel_regularizer=regularizers.l2(self.reg_lambda) ,
kernel_initializer=glorot_uniform(seed=self.seed),
recurrent_initializer=orthogonal(gain=1.0, seed=self.seed),
bias_initializer=Ones(),
dropout=0.5,
recurrent_dropout=0.5)
return layer
def test_orthogonal(tensor_shape):
_runner(initializers.orthogonal(), tensor_shape,
target_mean=0.)
