def test_simple(self): max_seq_len = 10 model = keras.Sequential([ keras.layers.Embedding(input_dim=5, output_dim=15, mask_zero=True), # [B, 10, 12] TransformerEncoderLayer( hidden_size=15, num_heads=5, num_layers=2, intermediate_size=8, hidden_dropout=0.1), # [B, 10, 6] keras.layers.TimeDistributed( keras.layers.Dense(units=2, activation="softmax")), # [B, 10, 2] keras.layers.Lambda(lambda x: x[:, -0, ...]) # [B, 2] ]) model.build(input_shape=(None, max_seq_len)) model.compile(optimizer=keras.optimizers.Adam(lr=0.003), loss=keras.losses.sparse_categorical_crossentropy, metrics=[keras.metrics.sparse_categorical_accuracy] ) model.summary(line_length=120) model.fit_generator(generator=parity_ds_generator(64, max_seq_len), steps_per_epoch=100, epochs=20, validation_data=parity_ds_generator(12, -4+max_seq_len), validation_steps=10, callbacks=[ keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy', patience=5), ], )
def build(self, input_shape): if isinstance(input_shape, list): assert len(input_shape) == 2 input_ids_shape, token_type_ids_shape = input_shape self.input_spec = [tf.keras.layers.InputSpec(shape=input_ids_shape), tf.keras.layers.InputSpec(shape=token_type_ids_shape)] else: input_ids_shape = input_shape self.input_spec = tf.keras.layers.InputSpec(shape=input_ids_shape) self.embeddings_layer = AdvBertEmbeddingsLayer.from_params( self.params, name="embeddings" ) # create all transformer encoder sub-layers self.encoders_layer = TransformerEncoderLayer.from_params( self.params, name="encoder" ) self.dropout_layer = tf.keras.layers.Dropout(rate=self.params.hidden_dropout) self.pooler_layer = PoolerLayer(self.params.hidden_size, name="pooler") super(BertModelLayer, self).build(input_shape)
def _construct(self, **kwargs): super()._construct(**kwargs) self.embeddings_layer = BertEmbeddingsLayer.from_params( self.params, name="embeddings") # create all transformer encoder sub-layers self.encoders_layer = TransformerEncoderLayer.from_params( self.params, name="encoder") self.support_masking = True
def __init__(self, vocab_size, hidden_size): super(Bert, self).__init__() self.embedding = BertEmbedding(vocab_size, config.hidden_size) # 输入表示向量层 d_ff = hidden_size * 4 self.transformer_block = nn.ModuleList( # 堆叠若干层的transformer encoder层 [ TransformerEncoderLayer(d_ff, hidden_size) for _ in range(config.layers) ]) self.mask_obj = Mask()
def build(self, input_shape): if isinstance(input_shape, list): assert len(input_shape) == 2 input_ids_shape, token_type_ids_shape = input_shape self.input_spec = [ keras.layers.InputSpec(shape=input_ids_shape), keras.layers.InputSpec(shape=token_type_ids_shape) ] else: input_ids_shape = input_shape self.input_spec = keras.layers.InputSpec(shape=input_ids_shape) self.embeddings_layer = BertEmbeddingsLayer.from_params(self.params, name="embeddings") # create all transformer encoder sub-layers self.encoders_layer = TransformerEncoderLayer.from_params(self.params, name="encoder") super(BertModelLayer, self).build(input_shape)