def evaluate(inp_sentence, params):
start_token = [tokenizer_source.vocab_size]
end_token = [tokenizer_source.vocab_size + 1]
inp = [start_token + tokenizer_source.encode(inp_sentence) + end_token]
inp = tf.keras.preprocessing.sequence.pad_sequences(
inp, maxlen=params["MAX_LENGTH"], padding="post")
# inp = tf.expand_dims(inp, 0)
enc_padding_mask = create_masks(inp, None)
# predictions.shape == (batch_size, seq_len, vocab_size)
predictions, _, attention_weights = transformer(inp, None, False,
enc_padding_mask, None,
None)
predictions = tf.squeeze(predictions, axis=0)
predictions_index = tf.cast(
tf.argsort(predictions, axis=-1, direction="DESCENDING"), tf.int32)
predictions = predictions.numpy()[predictions_index.numpy()]
_pred = [{
"score": i,
"label": tokenizer_target.int2str(j.numpy())
} for i, j in zip(predictions, predictions_index)
][:params["max_predictions"]]
return _pred, attention_weights
def test_acc(batch=32,
test_dataset=[],
transformer=[],
test_accuracy=[],
test_loss=[]):
real = []
pred = []
for (batch, (inp, tar)) in enumerate(test_dataset):
logger.debug("input: {}".format(inp.shape))
logger.debug("target: {}".format(tar.shape))
enc_padding_mask = create_masks(inp, tar)
predictions, _, _ = transformer(inp, tar, False, enc_padding_mask,
None, None)
logger.debug("predictions: {}".format(predictions.shape))
logger.debug("tar_real: {}".format(tar.shape))
test_accuracy(tar, predictions)
test_loss(loss_function(tar, predictions))
tar = tf.Variable(tar)
predictions = tf.Variable(predictions)
real += tar.numpy().tolist()
pred += [i for i in np.argmax(predictions.numpy(), axis=1)]
test_accuracy(tar, predictions)
test_loss(loss_function(tar, predictions))
# print(classification_report(real, pred))
F1 = f1_score(real, pred, average='macro')
return F1
def train_step(inp, tar):
logger.debug("input: {}".format(inp.shape))
logger.debug("target: {}".format(tar.shape))
enc_padding_mask = create_masks(inp, tar)
logger.debug("enc_padding_mask: {}".format(enc_padding_mask.shape))
with tf.GradientTape() as tape:
predictions, enc_output, _ = transformer(inp, tar, True,
enc_padding_mask, None,
None)
logger.debug("predictions: {}".format(predictions.shape))
logger.debug("tar_real: {}".format(tar.shape))
logger.debug("enc_output: {}".format(enc_output.shape))
# logger.debug("enc_output: {}".format(enc_output.shape))
loss = loss_function(tar, predictions)
gradients = tape.gradient(loss, transformer.trainable_variables)
optimizer.apply_gradients(
zip(gradients, transformer.trainable_variables))
train_loss(loss)
train_accuracy(tar, predictions)
def train_step(inp, tar):
tar_inp = tar[:, :-1]
tar_real = tar[:, 1:]
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
inp, tar_inp)
with tf.GradientTape() as tape:
predictions, _ = transformer(inp, tar_inp, True, enc_padding_mask,
combined_mask, dec_padding_mask)
loss = loss_function(tar_real, predictions)
gradients = tape.gradient(loss, transformer.trainable_variables)
optimizer.apply_gradients(
zip(gradients, transformer.trainable_variables))
train_loss(loss)
train_accuracy(tar_real, predictions)
def evaluate(self, inp_sentence):
start_token = [self.tokenizer_source.vocab_size]
end_token = [self.tokenizer_source.vocab_size + 1]
# inp sentence is portuguese, hence adding the start and end token
inp_sentence = (
start_token + self.tokenizer_source.encode(inp_sentence) + end_token
)
encoder_input = tf.expand_dims(inp_sentence, 0)
# as the target is english, the first word to the transformer should be the
# english start token.
decoder_input = [self.tokenizer_target.vocab_size]
output = tf.expand_dims(decoder_input, 0)
for i in range(self.MAX_LENGTH):
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
encoder_input, output
)
# predictions.shape == (batch_size, seq_len, vocab_size)
predictions, attention_weights = self.transformer(
encoder_input,
output,
False,
enc_padding_mask,
combined_mask,
dec_padding_mask,
)
# select the last word from the seq_len dimension
predictions = predictions[:, -1:, :] # (batch_size, 1, vocab_size)
predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)
# return the result if the predicted_id is equal to the end token
if tf.equal(predicted_id, self.tokenizer_target.vocab_size + 1):
return tf.squeeze(output, axis=0), attention_weights
# concatentate the predicted_id to the output which is given to the decoder
# as its input.
output = tf.concat([output, predicted_id], axis=-1)
return tf.squeeze(output, axis=0), attention_weights
def test_acc(batch=32,
test_dataset=[],
transformer=[],
test_accuracy=[],
test_loss=[]):
for (batch, (inp, tar)) in enumerate(test_dataset):
tar_inp = tar[:, :-1]
tar_real = tar[:, 1:]
enc_padding_mask, combined_mask, dec_padding_mask = create_masks(
inp, tar_inp)
predictions, _ = transformer(inp, tar_inp, True, enc_padding_mask,
combined_mask, dec_padding_mask)
test_accuracy(tar_real, predictions)
test_loss(loss_function(tar_real, predictions))
return test_accuracy, test_loss