def test_simple(self):
max_seq_len = 10
model = keras.Sequential([
keras.layers.Embedding(input_dim=5, output_dim=15, mask_zero=True), # [B, 10, 12]
TransformerEncoderLayer(
hidden_size=15,
num_heads=5,
num_layers=2,
intermediate_size=8,
hidden_dropout=0.1), # [B, 10, 6]
keras.layers.TimeDistributed(
keras.layers.Dense(units=2, activation="softmax")), # [B, 10, 2]
keras.layers.Lambda(lambda x: x[:, -0, ...]) # [B, 2]
])
model.build(input_shape=(None, max_seq_len))
model.compile(optimizer=keras.optimizers.Adam(lr=0.003),
loss=keras.losses.sparse_categorical_crossentropy,
metrics=[keras.metrics.sparse_categorical_accuracy]
)
model.summary(line_length=120)
model.fit_generator(generator=parity_ds_generator(64, max_seq_len),
steps_per_epoch=100,
epochs=20,
validation_data=parity_ds_generator(12, -4+max_seq_len),
validation_steps=10,
callbacks=[
keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_accuracy', patience=5),
],
)
def build(self, input_shape):
if isinstance(input_shape, list):
assert len(input_shape) == 2
input_ids_shape, token_type_ids_shape = input_shape
self.input_spec = [tf.keras.layers.InputSpec(shape=input_ids_shape),
tf.keras.layers.InputSpec(shape=token_type_ids_shape)]
else:
input_ids_shape = input_shape
self.input_spec = tf.keras.layers.InputSpec(shape=input_ids_shape)
self.embeddings_layer = AdvBertEmbeddingsLayer.from_params(
self.params,
name="embeddings"
)
# create all transformer encoder sub-layers
self.encoders_layer = TransformerEncoderLayer.from_params(
self.params,
name="encoder"
)
self.dropout_layer = tf.keras.layers.Dropout(rate=self.params.hidden_dropout)
self.pooler_layer = PoolerLayer(self.params.hidden_size, name="pooler")
super(BertModelLayer, self).build(input_shape)
def _construct(self, **kwargs):
super()._construct(**kwargs)
self.embeddings_layer = BertEmbeddingsLayer.from_params(
self.params, name="embeddings")
# create all transformer encoder sub-layers
self.encoders_layer = TransformerEncoderLayer.from_params(
self.params, name="encoder")
self.support_masking = True
def __init__(self, vocab_size, hidden_size):
super(Bert, self).__init__()
self.embedding = BertEmbedding(vocab_size,
config.hidden_size) # 输入表示向量层
d_ff = hidden_size * 4
self.transformer_block = nn.ModuleList( # 堆叠若干层的transformer encoder层
[
TransformerEncoderLayer(d_ff, hidden_size)
for _ in range(config.layers)
])
self.mask_obj = Mask()
def build(self, input_shape):
if isinstance(input_shape, list):
assert len(input_shape) == 2
input_ids_shape, token_type_ids_shape = input_shape
self.input_spec = [
keras.layers.InputSpec(shape=input_ids_shape),
keras.layers.InputSpec(shape=token_type_ids_shape)
]
else:
input_ids_shape = input_shape
self.input_spec = keras.layers.InputSpec(shape=input_ids_shape)
self.embeddings_layer = BertEmbeddingsLayer.from_params(self.params, name="embeddings")
# create all transformer encoder sub-layers
self.encoders_layer = TransformerEncoderLayer.from_params(self.params, name="encoder")
super(BertModelLayer, self).build(input_shape)