def build(self): # build word embedding word_ids = Input(batch_shape=(None, None), dtype='int32') lengths = Input(batch_shape=(None, None), dtype='int32') inputs = [word_ids] if self._embeddings is None: word_embeddings = Embedding(input_dim=self._word_vocab_size, output_dim=self._word_embedding_dim, mask_zero=True)(word_ids) else: word_embeddings = Embedding(input_dim=self._embeddings.shape[0], output_dim=self._embeddings.shape[1], mask_zero=True, weights=[self._embeddings])(word_ids) # build character based word embedding if self._use_char: char_ids = Input(batch_shape=(None, None, None), dtype='int32') inputs.append(char_ids) char_embeddings = Embedding(input_dim=self._char_vocab_size, output_dim=self._char_embedding_dim, mask_zero=True)(char_ids) s = K.shape(char_embeddings) char_embeddings = Lambda(lambda x: K.reshape( x, shape=(-1, s[-2], self._char_embedding_dim)))( char_embeddings) fwd_state = LSTM(self._char_lstm_size, return_state=True)(char_embeddings)[-2] bwd_state = LSTM(self._char_lstm_size, return_state=True, go_backwards=True)(char_embeddings)[-2] char_embeddings = Concatenate(axis=-1)([fwd_state, bwd_state]) # shape = (batch size, max sentence length, char hidden size) char_embeddings = Lambda(lambda x: K.reshape( x, shape=[-1, s[1], 2 * self._char_lstm_size]))( char_embeddings) # combine characters and word word_embeddings = Concatenate(axis=-1)( [word_embeddings, char_embeddings]) inputs.append(lengths) word_embeddings = Dropout(self._dropout)(word_embeddings) z = Bidirectional( LSTM(units=self._word_lstm_size, return_sequences=True))(word_embeddings) z = Dropout(self._dropout)(z) z = Dense(self._fc_dim, activation='tanh')(z) z = Dense(self._fc_dim, activation='tanh')(z) if self._use_crf: crf = CRF(self._num_labels, sparse_target=False) self._loss = crf.loss_function pred = crf(z) else: self._loss = 'categorical_crossentropy' pred = Dense(self._num_labels, activation='softmax')(z) self.model = Model(inputs=inputs, outputs=pred)
def build(self): # build word embedding word_ids = Input(batch_shape=(None, None), dtype='int32', name='word_input') inputs = [word_ids] if self._embeddings is None: word_embeddings = Embedding(input_dim=self._word_vocab_size, output_dim=self._word_embedding_dim, mask_zero=True, name='word_embedding')(word_ids) else: word_embeddings = Embedding(input_dim=self._embeddings.shape[0], output_dim=self._embeddings.shape[1], mask_zero=True, weights=[self._embeddings], name='word_embedding')(word_ids) # build character based word embedding if self._use_char: char_ids = Input(batch_shape=(None, None, None), dtype='int32', name='char_input') inputs.append(char_ids) char_embeddings = Embedding(input_dim=self._char_vocab_size, output_dim=self._char_embedding_dim, mask_zero=True, name='char_embedding')(char_ids) char_embeddings = TimeDistributed( Bidirectional(LSTM(self._char_lstm_size)))(char_embeddings) word_embeddings = Concatenate()([word_embeddings, char_embeddings]) word_embeddings = Dropout(self._dropout)(word_embeddings) z = Bidirectional( LSTM(units=self._word_lstm_size, return_sequences=True, dropout=self._layerdropout, recurrent_dropout=self._layerdropout))(word_embeddings) if (self._layer2Flag): z = Bidirectional( LSTM(units=self._word_lstm_size, return_sequences=True, dropout=self._layerdropout, recurrent_dropout=self._layerdropout))(z) z = Dense(self._fc_dim, activation='tanh')(z) if self._use_crf: crf = CRF(self._num_labels, sparse_target=False) loss = crf.loss_function pred = crf(z) else: loss = 'categorical_crossentropy' pred = Dense(self._num_labels, activation='softmax')(z) model = Model(inputs=inputs, outputs=pred) return model, loss
def build(self): # build word embedding #in_id = Input(shape=(self._bretMaxLen,), name="input_ids") #in_mask = Input(shape=(self._bretMaxLen,), name="input_masks") #in_segment = Input(shape=(self._bretMaxLen,), name="segment_ids") #inputs = [in_id, in_mask, in_segment] #word_embeddings = BertLayer(n_fine_tune_layers=3,bert_path=self._bert_path)(inputs) word_ids = Input(batch_shape=(None, None), dtype='int32', name='word_input') word_embeddings = BertEmbedding( model='bert_24_1024_16', dataset_name='book_corpus_wiki_en_cased')(word_ids) # build character based word embedding # if self._use_char: # print("char Embedding layer On") # char_ids = Input(batch_shape=(None, None, None), dtype='int32', name='char_input') # inputs.append(char_ids) # char_embeddings = Embedding(input_dim=self._char_vocab_size, # output_dim=self._char_embedding_dim, # mask_zero=True, # name='char_embedding')(char_ids) # char_embeddings = TimeDistributed(Bidirectional(LSTM(self._char_lstm_size)))(char_embeddings) # word_embeddings = Concatenate()([word_embeddings, char_embeddings]) # # word_embeddings = Dropout(self._dropout)(word_embeddings) z = Bidirectional( LSTM(units=self._word_lstm_size, return_sequences=True, dropout=self._layerdropout, recurrent_dropout=self._layerdropout))(word_embeddings) if (self._layer2Flag): z = Bidirectional( LSTM(units=self._word_lstm_size, return_sequences=True, dropout=self._layerdropout, recurrent_dropout=self._layerdropout))(z) z = Dense(self._fc_dim, activation='tanh')(z) if self._use_crf: crf = CRF(self._num_labels, sparse_target=False) loss = crf.loss_function pred = crf(z) else: loss = 'categorical_crossentropy' pred = Dense(self._num_labels, activation='softmax')(z) model = Model(inputs=inputs, outputs=pred) return model, loss
def build(self): # build word embedding word_ids = Input(batch_shape=(None, None), dtype='int32', name='word_input') if self._embeddings is None: word_embeddings = Embedding(input_dim=self._word_vocab_size, output_dim=self._word_embedding_dim, mask_zero=True, name='word_embedding')(word_ids) else: word_embeddings = Embedding(input_dim=self._embeddings.shape[0], output_dim=self._embeddings.shape[1], mask_zero=False, trainable=False, weights=[self._embeddings], name='word_embedding')(word_ids) # build character based word embedding char_ids = Input(batch_shape=(None, None, None), dtype='int32', name='char_input') char_embeddings = Embedding(input_dim=self._char_vocab_size, output_dim=self._char_embedding_dim, mask_zero=False, trainable=False, name='char_embedding')(char_ids) char_embeddings = TimeDistributed( Bidirectional(CuDNNLSTM(self._char_lstm_size)))(char_embeddings) elmo_embeddings = Input(shape=(None, 1024), dtype='float32') word_embeddings = Concatenate(trainable=False)( [word_embeddings, char_embeddings, elmo_embeddings]) word_embeddings = Dropout(self._dropout)(word_embeddings) z = Bidirectional( CuDNNLSTM(units=self._word_lstm_size, return_sequences=True))(word_embeddings) z = Dense(self._fc_dim, activation='tanh')(z) z = Dropout(self._dropout)(z) crf = CRF(self._num_labels, sparse_target=False) loss = crf.loss_function pred = crf(z) model = Model(inputs=[word_ids, char_ids, elmo_embeddings], outputs=pred) return model, loss
def get_model(max_len, max_features, embed_size, embedding_matrix, num_tags): sequence_input = Input(shape=(max_len, )) embedding = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=config.train_embeddings, mask_zero=True)(sequence_input) x = Dropout(0.5)(embedding) x1 = Bidirectional(LSTM(100, return_sequences=True))(x) td = Dense(100, activation='tanh')(x1) crf = CRF(num_tags, sparse_target=False) # CRF layer out = crf(td) # output model = Model(sequence_input, out) return model, crf
def build_model(self): """ Builds Bidirectional LSTM-CRF model. """ # Word embedding - TODO #word_embeddings = Dropout(self.dropout) # Model model = Sequential() # Input #input = Input(shape=(maxlen,), dtype='int32') # Embeddings embeddings = self.create_embeddings_layer()#(input) model.add(embeddings) #model.add(Embedding(max_features, self.word_lstm_size * 2, input_length=maxlen)) # Bidirectional LSTM #bi_lstm = Bidirectional(LSTM(units=self.word_lstm_size, return_sequences=True)) model.add(Bidirectional(LSTM(units=self.word_lstm_size, return_sequences=True))) #bi_lstm = Dense(self.fc_dim, activation='tanh')(bi_lstm) model.add(Dense(self.fc_dim, activation='tanh')) #model.add(bi_lstm) model.add(Dropout(self.dropout)) # TODO # CRF if self.use_crf: crf = CRF(self.labels_cnt, sparse_target=False) loss = crf.loss_function model.add(crf) else: loss = 'categorical_crossentropy' model.add(Dense(self.labels_cnt, activation='softmax')) return model, loss
def build(self): # build word embedding word_ids = Input(batch_shape=(None, None), dtype='int32', name='word_input') inputs = [word_ids] if self._embeddings is None: word_embeddings = Embedding(input_dim=self._word_vocab_size, output_dim=self._word_embedding_dim, mask_zero=True, name='word_embedding')(word_ids) else: word_embeddings = Embedding(input_dim=self._embeddings.shape[0], output_dim=self._embeddings.shape[1], mask_zero=False, trainable=False, weights=[self._embeddings], name='word_embedding')(word_ids) # build character based word embedding if self._use_char: char_ids = Input(batch_shape=(None, None, None), dtype='int32', name='char_input') inputs.append(char_ids) char_embeddings = Embedding(input_dim=self._char_vocab_size, output_dim=self._char_embedding_dim, mask_zero=False, trainable=True, name='char_embedding')(char_ids) # dropout = Dropout(self._dropout)(char_embeddings) # conv1d_out = TimeDistributed( # Conv1D(kernel_size=3, filters=50, padding='same', activation='tanh', strides=1))(dropout) # maxpool_out = TimeDistributed(MaxPooling1D(self._char_embedding_dim))(conv1d_out) # char = TimeDistributed(Flatten())(maxpool_out) # char_embeddings = Dropout(self._dropout)(char) char_embeddings = TimeDistributed( Bidirectional(CuDNNLSTM( self._char_lstm_size)))(char_embeddings) # chars = TimeDistributed(Conv1D(self._char_lstm_size, 3, padding='same'), name="char_cnn")(char_embeddings) # chars = TimeDistributed(GlobalMaxPooling1D(), name="char_pooling")(chars) # char_embeddings = Dropout(self._dropout)(char_embeddings) word_embeddings = Concatenate()([word_embeddings, char_embeddings]) word_embeddings = Dropout(self._dropout)(word_embeddings) z = Bidirectional( CuDNNLSTM(units=self._word_lstm_size, return_sequences=True))(word_embeddings) z = Bidirectional(CuDNNLSTM(units=100, return_sequences=True))(z) # z = Dropout(self._dropout)(z) z = Dense(self._fc_dim, activation='tanh')(z) z = Dropout(self._dropout)(z) if self._use_crf: crf = CRF(self._num_labels, sparse_target=False) loss = crf.loss_function pred = crf(z) else: loss = 'categorical_crossentropy' pred = Dense(self._num_labels, activation='softmax')(z) model = Model(inputs=inputs, outputs=pred) return model, loss