},
))
dataset = dataset.cache()
dataset = dataset.shuffle(BUFFER_SIZE)
dataset = dataset.batch(BATCH_SIZE)
dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
tf.keras.backend.clear_session()
model = tfr.transformer(vocab_size=input_size,
num_layers=num_layer,
dff=dff,
d_model=dmodel,
num_heads=num_head,
dropout=dropout)
learning_rate = tfr.CustomSchedule(dmodel)
optimizer = tf.keras.optimizers.Adam(learning_rate,
beta_1=0.9,
beta_2=0.98,
epsilon=1e-9)
def loss_function(y_true, y_pred):
y_true = tf.reshape(y_true, shape=(-1, seq_mx_len - 1))
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True,
reduction='none')(
y_true, y_pred)
mask = tf.cast(tf.not_equal(y_true, 0), tf.float32)
