def lstm(embedding_matrix=np.zeros(0), max_len=21, input_dim=111857, max_features=50000, units=150, num_filter=128): inp1 = Input(shape=(max_len, ), dtype='int32') inp2 = Input(shape=(max_len, ), dtype='int32') x1 = Embedding(input_dim=input_dim, output_dim=units * 2, input_length=max_len, trainable=True)(inp1) x1 = SpatialDropout1D(rate=0.24)(x1) x2 = Embedding(input_dim=input_dim, output_dim=units * 2, input_length=max_len, trainable=True)(inp2) x2 = SpatialDropout1D(rate=0.24)(x2) x1 = Bidirectional(layer=LSTM(units, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm1')(x1) x2 = Bidirectional(layer=LSTM(units, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm2')(x2) x1 = Reshape((max_len, units * 2))(x1) x2 = Reshape((max_len, units * 2))(x2) x1 = Flatten()(x1) x2 = Flatten()(x2) diff = abs_layer(Subtract()([x1, x2])) mul = Multiply()([x1, x2]) outputp = Concatenate()([x1, x2, diff, mul]) outputp = Dense(int(units * 2))(outputp) outputp = BatchNormalization()(outputp) outputp = ReLU()(outputp) finalout = Dense(1, activation='sigmoid')(outputp) model = Model(inputs=[inp1, inp2], outputs=finalout) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def capsule(maxlen, max_features, embed_size, embedding_matrix, num_classes): K.clear_session() inp = Input(shape=(maxlen, )) x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp) x = SpatialDropout1D(rate=0.2)(x) x = Bidirectional( GRU(100, return_sequences=True, kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x) x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) x = Flatten()(x) x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x) x = Dropout(0.12)(x) x = BatchNormalization()(x) x = Dense(num_classes, activation="sigmoid")(x) model = Model(inputs=inp, outputs=x) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy']) return model
def create_model(self): model = Sequential() print('#3') model.add( Embedding(self.input_dim, self.vec_dim, input_length=self.maxlen, trainable=True, embeddings_initializer=uniform(seed=20170719))) model.add(BatchNormalization(axis=-1)) print('#4') model.add( Masking(mask_value=0, input_shape=(self.maxlen, self.vec_dim))) model.add( LSTM(self.n_hidden, batch_input_shape=(None, self.maxlen, self.vec_dim), kernel_initializer=glorot_uniform(seed=20170719), recurrent_initializer=orthogonal(gain=1.0, seed=20170719))) print('#5') model.add(BatchNormalization(axis=-1)) print('#6') model.add( Dense(self.output_dim, activation='sigmoid', use_bias=True, kernel_initializer=glorot_uniform(seed=20170719))) model.compile(loss="binary_crossentropy", optimizer="RMSprop", metrics=['binary_accuracy']) return model
def build_fasttext(self): """ 可用的initialization方法:random_normal(stddev=0.0001), Orthogonal(), glorot_uniform(), lecun_uniform() :return: """ initial_dict = { 'orthogonal': orthogonal(), 'he_n': he_normal(), 'he_u': he_uniform() } # model.add(Embedding(output_dim=self.hidden_layer, input_length=self.max_sequence_length, # input_dim=self.max_nb_words, embeddings_initializer=initial_dict['he_n'], trainable=True)) # model.add(GlobalAveragePooling1D()) # model.add(Dense(self.n_classes, activation='softmax', kernel_initializer=initial_dict['he_n'])) self.model.add( Embedding(output_dim=self.hidden_layer, input_length=self.max_sequence_length, input_dim=self.max_nb_words, trainable=True)) self.model.add(GlobalAveragePooling1D()) self.model.add(Dense(self.n_classes, activation='softmax')) print(self.model.summary()) self.compile()
def attention_capsule(maxlen, max_features, embed_size, embedding_matrix, num_classes): inp = Input(shape=(maxlen, )) x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp) x = SpatialDropout1D(rate=0.24)(x) x = Bidirectional( LSTM(80, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x) x_1 = Attention(maxlen)(x) x_1 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_1) x_2 = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) x_2 = Flatten()(x_2) x_2 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_2) conc = concatenate([x_1, x_2]) # conc = add([x_1, x_2]) outp = Dense(num_classes, activation="sigmoid")(conc) model = Model(inputs=inp, outputs=outp) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy']) return model
def build_my_model(m, n): inp = Input(shape=(m, )) x = SpatialDropout1D(rate=0.24)(inp) x = Bidirectional( CuDNNLSTM(80, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)))(x) x_1 = Attention(m)(x) x_1 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_1) x_2 = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) x_2 = Flatten()(x_2) x_2 = DropConnect(Dense(32, activation="relu"), prob=0.2)(x_2) conc = concatenate([x_1, x_2]) # conc = add([x_1, x_2]) outp = Dense(1, activation="sigmoid")(conc) model = Model(inputs=inp, outputs=outp) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=[f1]) return model
def build(self, input_shape): assert len(input_shape) >= 2 m = input_shape[0][-1] n = input_shape[0][-1] self.W1 = self.add_weight(name='mixed_agg_weight1', shape=(m, n), initializer=orthogonal(seed=9), trainable=True) self.W2 = self.add_weight(name='mixed_agg_weight2', shape=(n, n), initializer=orthogonal(seed=9), trainable=True) self.vm = self.add_weight(name='vmix', shape=(1, n), initializer=random_normal(seed=9), trainable=True) super(MixedAggregation, self).build(input_shape)
def Transpose_Conv2D(x, filters, kernel, strides, padding, block_id, kernel_init=orthogonal()): prefix = f'block_{block_id}_' x = layers.Conv2DTranspose(filters, kernel_size=kernel, strides=strides, padding=padding, kernel_initializer=kernel_init, name=prefix+'de-conv')(x) x = layers.LeakyReLU(name=prefix+'lrelu')(x) x = layers.Dropout(0.2, name=prefix+'drop')((x)) x = layers.BatchNormalization(name=prefix+'conv_bn')(x) return x
def build(self, input_shape): assert len(input_shape) >= 2 self.W = self.add_weight(name='inside_agg_weight', shape=(input_shape[-1], input_shape[-1]), initializer=orthogonal(seed=9), trainable=True) super(InsideAggregation, self).build(input_shape)
def custom_model(data_shape, layers=1, units1=128, units2=128, units3=128, units4=128, units5=128, optim='rmsprop', metrics=['accuracy']): units = {1: units1, 2: units2, 3: units3, 4: units4, 5: units5} model = Sequential() model.add(Masking(mask_value=0.0, input_shape=data_shape)) return_sequences = True for i in range(layers): if i + 1 >= layers: return_sequences = False if i == 0: model.add( LSTM(units[i + 1], input_shape=data_shape, return_sequences=return_sequences, kernel_regularizer=l2(0.01), recurrent_regularizer=l2(0.01), kernel_initializer=glorot_uniform(seed=42), recurrent_initializer=orthogonal(seed=42))) #model.add(Dropout(rate=.1)) else: model.add( LSTM(units[i + 1], return_sequences=return_sequences, kernel_regularizer=l2(0.01), recurrent_regularizer=l2(0.01), kernel_initializer=glorot_uniform(seed=42), recurrent_initializer=orthogonal(seed=42))) #model.add(Dropout(rate=.1)) model.add( Dense(2, activation='softmax', kernel_initializer=glorot_uniform(seed=42), kernel_regularizer=l2(0.01))) model.compile(loss='binary_crossentropy', optimizer=optim, metrics=metrics) return model
def _build_fn_orthogonal_i_5(input_shape): # `orthogonal(gain=Real(0.6, 1.6))` model = Sequential( [ Dense(Integer(50, 100), input_shape=input_shape), Dense(1, kernel_initializer=orthogonal(gain=Real(0.6, 1.6))), ] ) model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"]) return model
def _build_fn_categorical_3(input_shape): # `Categorical(["glorot_normal", orthogonal(gain=1)])` model = Sequential( [ Dense(Integer(50, 100), input_shape=input_shape), Dense(1, kernel_initializer=Categorical(["glorot_normal", orthogonal(gain=1)])), ] ) model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"]) return model
def build(self, input_shape): assert len(input_shape) == 4 m1 = input_shape[-2] m2 = input_shape[-1] self.W = self.add_weight(name='merge_weight', shape=(m1, m2), initializer=orthogonal(seed=9), trainable=True) super(MergeChannel, self).build(input_shape)
def test_orthogonal_init_does_not_affect_global_rng(): np.random.seed(1337) before = np.random.randint(0, 100, size=10) np.random.seed(1337) init = initializers.orthogonal(seed=9876) init(shape=(10, 5)) after = np.random.randint(0, 100, size=10) assert np.array_equal(before, after)
def test_orthogonal_init_does_not_affect_global_rng(): np.random.seed(1337) before = np.random.randint(0, 100, size=10) np.random.seed(1337) init = initializers.orthogonal(seed=9876) init(shape=(10, 5)) after = np.random.randint(0, 100, size=10) assert np.array_equal(before, after)
def build(self, input_shape): assert len(input_shape) == 2 assert len(input_shape[0]) == 3 assert len(input_shape[1]) == 3 m1 = input_shape[0][-1] m2 = input_shape[1][-1] self.W = self.add_weight(name='interaction_weight', shape=(m1, m2), initializer=orthogonal(seed=9), trainable=True) super(MatrixInteraction, self).build(input_shape)
def Model2(embedding_matrix=None): # wt = [embedding_matrix] if embedding_matrix != None else None embedding_layer = Embedding(len(word_index) + 1, EMBEDDING_DIM, weights=embedding_matrix, input_length=MAX_SEQUENCE_LENGTH, trainable=True) sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32') embedded_sequences = embedding_layer(sequence_input) net = SpatialDropout1D(0.2)(embedded_sequences) net = Conv1D(100, 3, padding='same', activation='relu')(net) net = BatchNormalization()(net) # net=GRU(EMBEDDING_DIM, return_sequences=True, # kernel_initializer=glorot_normal(seed=1029), # recurrent_initializer=orthogonal(gain=1.0, seed=1029))(net) net = Bidirectional(layer=GRU(100, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_gru')(net) # net = Bidirectional( # layer=LSTM(EMBEDDING_DIM, return_sequences=True, # kernel_initializer=glorot_normal(seed=1029), # recurrent_initializer=orthogonal(gain=1.0, seed=1029)), # name='bidirectional_lstm')(net) #net = BatchNormalization()(net) capsul = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(net) # noqa capsul = Flatten()(capsul) capsul = DropConnect(Dense(32, activation="relu"), prob=0.2)(capsul) atten = Attention(step_dim=MAX_SEQUENCE_LENGTH, name='attention')(net) atten = DropConnect(Dense(16, activation="relu"), prob=0.2)(atten) net = Concatenate(axis=-1)([capsul, atten]) # net = GlobalAveragePooling1D()(net) # output = Dense(units=1, activation='sigmoid', name='output')(net) # net = GlobalAveragePooling1D()(net) net = Dense(100, activation='relu')(net) # net = Dropout(0.3)(net) output = Dense(2, activation='softmax')(net) model4 = Model(inputs=sequence_input, outputs=output) model4.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc']) model4.summary() chkpt4 = ModelCheckpoint('expertiza_nn_model3.h5', monitor='val_acc', verbose=1, save_best_only=True) return (model4, chkpt4)
def build(self, input_shape): assert len(input_shape) == 2 assert len(input_shape[0]) == 3 assert len(input_shape[1]) == 3 dim = max(input_shape[0][-1], input_shape[1][-1]) self.W0 = self.add_weight(name='weight0', shape=(input_shape[0][-1], dim), initializer=orthogonal(seed=9), trainable=True) self.W1 = self.add_weight(name='weight1', shape=(input_shape[1][-1], dim), initializer=orthogonal(seed=9), trainable=True) self.vc = self.add_weight(name='vc', shape=(dim, 1), initializer=random_normal(seed=9), trainable=True) super(AttentionConcat, self).build(input_shape)
def build(self, input_shape): assert len(input_shape) == 2 assert len(input_shape[0]) == 3 assert len(input_shape[1]) == 3 self.W = self.add_weight(name='bilinear_weight', shape=(input_shape[1][-1], input_shape[0][-1]), initializer=orthogonal(seed=9), trainable=True) super(AttentionBilinear, self).build(input_shape)
def create_sequence_model(embedding_matrix, word_index, max_words): """This function creates a deep learning model for the Quora competition. Parameters ---------- embedding_matrix: np.array The matrix of embeddings used in the embedding layer word_index: dict The word_index obtained by the tokenizer max_words: int The maximum number of words in a sequence Returns ------- A deep learning model """ input_layer = Input(shape=(75, ), name='input_layer') x = Embedding(len(word_index) + 1, embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False, input_shape=(max_words, ))(input_layer) x = SpatialDropout1D(rate=0.24)(x) x = Bidirectional(layer=LSTM(80, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm')(x) # Capsule layer capsule = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) # noqa capsule = Flatten()(capsule) capsule = DropConnect(Dense(32, activation="relu"), prob=0.01)(capsule) # Attention layer atten = Attention(step_dim=75, name='attention')(x) atten = DropConnect(Dense(16, activation="relu"), prob=0.05)(atten) # Concatenate Capsule and Attention layer x = Concatenate(axis=-1)([capsule, atten]) output_layer = Dense(units=1, activation='sigmoid', name='output')(x) model = Model(inputs=input_layer, outputs=output_layer) # Compile model model.compile(loss='binary_crossentropy', optimizer='adam') return model
def create_model(self): model = Sequential() model.add(LSTM(self.n_hidden, batch_input_shape=(None, self.maxlen, self.n_in), kernel_initializer=glorot_uniform(seed=20170719), recurrent_initializer=orthogonal(gain=1.0, seed=20170719), dropout=0.3, recurrent_dropout=0.3)) model.add(Dropout(0.3)) model.add(Dense(1,kernel_initializer=glorot_uniform(seed=20170719))) model.add(Activation("linear")) model.compile(loss="mean_squared_error", optimizer="RMSprop", metrics=['accuracy']) return model
def create_model(self): model = Sequential() model.add(LSTM(self.n_hidden, batch_input_shape=(None, self.maxlen, self.n_in), kernel_initializer=glorot_uniform(seed=20170719), recurrent_initializer=orthogonal(gain=1.0, seed=20170719), dropout=0.3, recurrent_dropout=0.3)) model.add(Dropout(0.3)) model.add(Dense(self.n_out, kernel_initializer=glorot_uniform(seed=20170719))) model.add(Activation("softmax")) model.compile(loss="categorical_crossentropy", optimizer="RMSprop", metrics=['categorical_accuracy']) return model
def build_model(input_eular, input_crp, parameters, seed=None): dropout = parameters['dropout'] optimizer = parameters['optimizer'] learning_rate = parameters['learning_rate'] rnn_type = parameters['rnn_type'] rnn_size = parameters['rnn_size'] rnn_activation = parameters['rnn_activation'] rnn_dropout = parameters['rnn_dropout'] dense_initializer = RandomNormal(seed=seed) if rnn_type == 'none': x = concatenate([input_eular, input_crp], axis=1) else: rnn_kernel_initializer = glorot_uniform(seed=seed) rnn_recurrent_initializer = orthogonal(seed=seed) x = Masking(mask_value=-1)(input_eular) if rnn_type == 'lstm': x = LSTM(rnn_size, activation=rnn_activation, kernel_initializer=rnn_kernel_initializer, recurrent_initializer=rnn_recurrent_initializer, dropout=dropout, recurrent_dropout=rnn_dropout, name='LSTM_{}'.format(rnn_size))(x) elif rnn_type == 'simplernn': x = SimpleRNN(rnn_size, activation=rnn_activation, kernel_initializer=rnn_kernel_initializer, recurrent_initializer=rnn_recurrent_initializer, dropout=dropout, recurrent_dropout=rnn_dropout, name='SRNN_{}'.format(rnn_size))(x) x = concatenate([x, input_crp]) for layer_size in [val for val in parameters["dense_layers"] if val != 0]: x = Dense(layer_size, activation='linear', kernel_initializer=dense_initializer)(x) x = get_activation(x, parameters['activation'])(x) if dropout != 0: x = Dropout(rate=dropout)(x) output = Dense(1, kernel_initializer=dense_initializer)(x) output = get_activation(output, parameters['last_activation'])(output) model = Model(input=[input_eular, input_crp], output=output) return model
def create_sharedmodel(self): inputs = Input(shape=(self.maxlen, self.height, self.width, 3)) x = TimeDistributed(self.resnet,name="resnet")(inputs) x = TimeDistributed(GlobalAveragePooling2D(), name="GAP")(x) x = TimeDistributed(Dense(512,activation='relu'), name="dense")(x) predictions = Bidirectional(LSTM(128, batch_input_shape = (None, self.maxlen, 512), kernel_initializer = glorot_normal(seed=20181020), recurrent_initializer = orthogonal(gain=1.0, seed=20181020), dropout = 0.01, recurrent_dropout = 0.01))(x) shared_layers = Model(inputs, predictions, name="shared_layers") return shared_layers
def build(self, input_shape): assert len(input_shape) == 2 assert len(input_shape[0]) == 3 assert len(input_shape[1]) == 3 assert input_shape[0][-1] == input_shape[1][-1] m = input_shape[0][-1] n = input_shape[0][-1] self.W = self.add_weight(name='minus_weight', shape=(m, n), initializer=orthogonal(seed=9), trainable=True) self.vm = self.add_weight(name='vd', shape=(n, 1), initializer=random_normal(seed=9), trainable=True) super(AttentionMinus, self).build(input_shape)
def create_model(self): model = Sequential() model.add( LSTM(self.n_hidden, batch_input_shape=(None, self.maxlen, self.n_in), kernel_initializer=glorot_uniform(seed=20170719), recurrent_initializer=orthogonal(gain=1.0, seed=20170719), dropout=0.5, recurrent_dropout=0.5)) model.add(Dropout(0.5)) model.add( Dense(self.n_out, kernel_initializer=glorot_uniform(seed=20170719))) model.add(Activation("softmax")) model.compile(loss="categorical_crossentropy", optimizer="RMSprop", metrics=['categorical_accuracy']) return model
def build(self, input_shape): self.kernel = self.add_weight(name='kernel', shape=(input_shape[1][2], self.units), initializer=initializers.orthogonal(), regularizer=regularizers.l2(5e-4)) self.bias = self.add_weight(name='bias', shape=(self.units, ), initializer=initializers.zeros()) if self.learn_pqr: self.p = self.add_weight(name='p', shape=(1, ), initializer=initializers.constant(0)) self.q = self.add_weight(name='q', shape=(1, ), initializer=initializers.constant(0)) # self.trainable_weights = [self.p, self.q] super(MyGCN, self).build(input_shape)
def get_model(embed_weights): input_layer = Input(shape=(MAX_LEN, ), name='input') # 1. embedding layer # get embedding weights print('load pre-trained embedding weights ......') input_dim = embed_weights.shape[0] output_dim = embed_weights.shape[1] x = Embedding(input_dim=input_dim, output_dim=output_dim, weights=[embed_weights], trainable=False, name='embedding')(input_layer) # clean up del embed_weights, input_dim, output_dim gc.collect() # 2. dropout x = SpatialDropout1D(rate=SPATIAL_DROPOUT)(x) # 3. bidirectional lstm x = Bidirectional(layer=CuDNNLSTM( RNN_UNITS, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm')(x) # 4. capsule layer capsul = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x) # noqa capsul = Flatten()(capsul) capsul = DropConnect(Dense(32, activation="relu"), prob=0.01)(capsul) # 5. attention later atten = Attention(step_dim=MAX_LEN, name='attention')(x) atten = DropConnect(Dense(16, activation="relu"), prob=0.05)(atten) x = Concatenate(axis=-1)([capsul, atten]) # 6. output (sigmoid) output_layer = Dense(units=1, activation='sigmoid', name='output')(x) model = Model(inputs=input_layer, outputs=output_layer) # compile model model.compile(loss='binary_crossentropy', optimizer='adam') return model
def create_sharedAutoRegLSTMmodel(self): inputs = Input(shape=(2 * self.maxlen, 1)) mid_lstm = Bidirectional( LSTM(128, batch_input_shape=(None, 2 * self.maxlen, 1), kernel_initializer=glorot_normal(seed=20181020), recurrent_initializer=orthogonal(gain=1.0, seed=20181020), dropout=0.01, recurrent_dropout=0.01, return_sequences=True))(inputs) mid_dense = Dense(256, activation='relu')(mid_lstm) predictions = Dense(self.maxlen, activation='sigmoid')(mid_dense) shared_layers = Model(inputs, predictions, name="shared_AutoRegLSTMlayers") return shared_layers
def gru_crf(embedding_matrix): inp = Input(shape=(maxlen,)) x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp) x = SpatialDropout1D(rate=0.24)(x) x = Bidirectional(CuDNNGRU(100, return_sequences=True, kernel_initializer=glorot_normal(seed=seed), recurrent_initializer=orthogonal(gain=1.0, seed=seed)))(x) x = CRF(10, learn_mode='marginal', unroll=True)(x) x = Flatten()(x) x = Dense(100, activation='relu', kernel_initializer=glorot_normal(seed=seed))(x) x = Dropout(0.2)(x) x = BatchNormalization()(x) outp = Dense(1, activation='sigmoid')(x) model = Model(inputs=inp, outputs=outp) model.compile(loss='binary_crossentropy', optimizer='adam') return model
def AutoEncoder(input_shape): inputs = layers.Input(shape=input_shape) # 256 x 256 conv1 = Conv2DLayer(inputs, 64, 3, strides=1, padding='same', block_id=1) conv2 = Conv2DLayer(conv1, 64, 3, strides=2, padding='same', block_id=2) # 128 x 128 conv3 = Conv2DLayer(conv2, 128, 5, strides=2, padding='same', block_id=3) # 64 x 64 conv4 = Conv2DLayer(conv3, 128, 3, strides=1, padding='same', block_id=4) conv5 = Conv2DLayer(conv4, 256, 5, strides=2, padding='same', block_id=5) # 32 x 32 conv6 = Conv2DLayer(conv5, 512, 3, strides=2, padding='same', block_id=6) # 16 x 16 deconv1 = Transpose_Conv2D(conv6, 512, 3, strides=2, padding='same', block_id=7) # 32 x 32 skip1 = layers.concatenate([deconv1, conv5], name='skip1') conv7 = Conv2DLayer(skip1, 256, 3, strides=1, padding='same', block_id=8) deconv2 = Transpose_Conv2D(conv7, 128, 3, strides=2, padding='same', block_id=9) # 64 x 64 skip2 = layers.concatenate([deconv2, conv3], name='skip2') conv8 = Conv2DLayer(skip2, 128, 5, strides=1, padding='same', block_id=10) deconv3 = Transpose_Conv2D(conv8, 64, 3, strides=2, padding='same', block_id=11) # 128 x 128 skip3 = layers.concatenate([deconv3, conv2], name='skip3') conv9 = Conv2DLayer(skip3, 64, 5, strides=1, padding='same', block_id=12) deconv4 = Transpose_Conv2D(conv9, 64, 3, strides=2, padding='same', block_id=13) # 256 x 256 skip3 = layers.concatenate([deconv4, conv1]) conv10 = layers.Conv2D(3, 3, strides=1, padding='same', activation='sigmoid', kernel_initializer=orthogonal(), name='final_conv')(skip3) return models.Model(inputs=inputs, outputs=conv10)
def get_model(embed_weights): input_layer = Input(shape=(MAX_LEN, ), name='input') # 1. embedding layer # get embedding weights print('load pre-trained embedding weights ......') input_dim = embed_weights.shape[0] output_dim = embed_weights.shape[1] x = Embedding(input_dim=input_dim, output_dim=output_dim, weights=[embed_weights], trainable=False, name='embedding')(input_layer) # clean up del embed_weights, input_dim, output_dim gc.collect() # 2. dropout x = SpatialDropout1D(rate=SPATIAL_DROPOUT)(x) # 3. bidirectional lstm x = Bidirectional(layer=CuDNNLSTM( RNN_UNITS, return_sequences=True, kernel_initializer=glorot_normal(seed=1029), recurrent_initializer=orthogonal(gain=1.0, seed=1029)), name='bidirectional_lstm')(x) # 4. capsule layer x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True, name='capsule')(x) x = Flatten(name='flatten')(x) # # 5. dense with dropConnect # x = DropConnect( # Dense(DENSE_UNITS, activation="relu"), # prob=0.05, # name='dropConnect_dense')(x) # 6. output (sigmoid) output_layer = Dense(units=1, activation='sigmoid', name='output')(x) model = Model(inputs=input_layer, outputs=output_layer) # compile model model.compile(loss='binary_crossentropy', optimizer='adam') return model
def create_LSTM(self, lstm_units, lstm_return_state=False, lstm_return_sequences=False, lstm_go_backwards=False, lstm_name='LSTM'): layer = LSTM( lstm_units, name=lstm_name, return_state=lstm_return_state, return_sequences=lstm_return_sequences, go_backwards=lstm_go_backwards, #recurrent_regularizer=regularizers.l2(self.reg_lambda) , #kernel_regularizer=regularizers.l2(self.reg_lambda) , kernel_initializer=glorot_uniform(seed=self.seed), recurrent_initializer=orthogonal(gain=1.0, seed=self.seed), bias_initializer=Ones(), dropout=0.5, recurrent_dropout=0.5) return layer
def test_orthogonal(tensor_shape): _runner(initializers.orthogonal(), tensor_shape, target_mean=0.)