def _create_tokenizers(
self, train_data: Iterable[Tuple[str, str]]
) -> Tuple[KerasTokenizer, KerasTokenizer]:
self.logger.info('fitting tokenizers...')
counter_encoder = Counter()
counter_decoder = Counter()
train_preprocessed = (self.preprocessor(d) for d in train_data)
for text_encoder, text_decoder in train_preprocessed:
counter_encoder.update(text_encoder.split())
counter_decoder.update(text_decoder.split())
tokens_encoder = {
token_count[0]
for token_count in counter_encoder.most_common(
self.max_vocab_size_encoder)
}
tokens_decoder = {
token_count[0]
for token_count in counter_decoder.most_common(
self.max_vocab_size_decoder)
}
tokens_encoder.update(
{self.preprocessor.start_token, self.preprocessor.end_token})
tokens_decoder.update(
{self.preprocessor.start_token, self.preprocessor.end_token})
tokenizer_encoder = KerasTokenizer(oov_token=OOV_TOKEN,
lower=False,
filters='')
tokenizer_decoder = KerasTokenizer(oov_token=OOV_TOKEN,
lower=False,
filters='')
tokenizer_encoder.fit(sorted(list(tokens_encoder)))
tokenizer_decoder.fit(sorted(list(tokens_decoder)))
return tokenizer_encoder, tokenizer_decoder
def test_serde_happy_path(self) -> None:
preprocessor = Preprocessor()
tokenizer = KerasTokenizer(oov_token='<unk>')
tokenizer.fit(['a b c {} {}'.format(
preprocessor.start_token, preprocessor.end_token)])
vectorizer = Vectorizer(tokenizer, tokenizer)
summarizer = AttentionSummarizer(lstm_size=10,
max_prediction_len=10,
embedding_size=10,
embedding_encoder_trainable=False)
summarizer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer)
# we need at least a train step to init the weights
train_step = summarizer.new_train_step(masked_crossentropy, batch_size=1, apply_gradients=True)
train_seq = tf.convert_to_tensor(np.array([[1, 1, 1]]), dtype=tf.int32)
train_step(train_seq, train_seq)
save_dir = os.path.join(self.temp_dir, 'summarizer_serde_happy_path')
summarizer.save(save_dir)
summarizer_loaded = AttentionSummarizer.load(save_dir)
self.assertEqual(10, summarizer_loaded.lstm_size)
self.assertEqual(10, summarizer_loaded.max_prediction_len)
self.assertIsNotNone(summarizer_loaded.preprocessor)
self.assertIsNotNone(summarizer_loaded.vectorizer)
self.assertIsNotNone(summarizer_loaded.encoder)
self.assertIsNotNone(summarizer_loaded.decoder)
self.assertFalse(summarizer_loaded.encoder.embedding.trainable)
self.assertTrue(summarizer_loaded.decoder.embedding.trainable)
self.assertIsNotNone(summarizer_loaded.optimizer)
pred = summarizer.predict_vectors('a c', '')
pred_loaded = summarizer_loaded.predict_vectors('a c', '')
np.testing.assert_almost_equal(pred['logits'], pred_loaded['logits'], decimal=6)
def test_keras_tokenizer(self):
tokenizer = KerasTokenizer(filters='', lower=False, oov_token='<unk>')
tokenizer.fit(['a b c d'])
encoded = tokenizer.encode('a b e')
self.assertEqual([2, 3, 1], encoded)
decoded = tokenizer.decode(encoded)
self.assertEqual('a b <unk>', decoded)
self.assertEqual(5, tokenizer.vocab_size)
self.assertEqual({'a', 'b', 'c', 'd', '<unk>'},
tokenizer.token_index.keys())
self.assertEqual({1, 2, 3, 4, 5}, set(tokenizer.token_index.values()))
def test_serde_happy_path(self) -> None:
preprocessor = Preprocessor(start_token='[CLS]', end_token='[SEP]')
tokenizer_encoder = BertTokenizer.from_pretrained('bert-base-uncased')
tokenizer_decoder = KerasTokenizer(oov_token='<unk>')
tokenizer_decoder.fit([
'a b c {} {}'.format(preprocessor.start_token,
preprocessor.end_token)
])
vectorizer = Vectorizer(tokenizer_encoder, tokenizer_decoder)
summarizer = SummarizerBert(num_layers_encoder=1,
num_layers_decoder=1,
bert_embedding_encoder='bert-base-uncased',
num_heads=2,
max_prediction_len=3,
embedding_size_encoder=768,
embedding_size_decoder=10,
embedding_encoder_trainable=False)
summarizer.init_model(preprocessor=preprocessor, vectorizer=vectorizer)
# we need at least a train step to init the weights
train_step = summarizer.new_train_step(masked_crossentropy,
batch_size=1,
apply_gradients=True)
train_seq = tf.convert_to_tensor(np.array([[1, 1, 1]]), dtype=tf.int32)
train_step(train_seq, train_seq)
save_dir = os.path.join(self.temp_dir, 'summarizer_serde_happy_path')
summarizer.save(save_dir)
summarizer_loaded = SummarizerBert.load(save_dir)
self.assertEqual(1, summarizer_loaded.num_layers_encoder)
self.assertEqual(1, summarizer_loaded.num_layers_decoder)
self.assertEqual(2, summarizer_loaded.num_heads)
self.assertEqual(3, summarizer_loaded.max_prediction_len)
self.assertEqual(768, summarizer_loaded.embedding_size_encoder)
self.assertEqual(10, summarizer_loaded.embedding_size_decoder)
self.assertIsNotNone(summarizer_loaded.preprocessor)
self.assertIsNotNone(summarizer_loaded.vectorizer)
self.assertIsNotNone(summarizer_loaded.transformer)
self.assertFalse(
summarizer_loaded.transformer.encoder.embedding.trainable)
self.assertTrue(
summarizer_loaded.transformer.decoder.embedding.trainable)
self.assertIsNotNone(summarizer_loaded.optimizer_encoder)
self.assertIsNotNone(summarizer_loaded.optimizer_decoder)
pred = summarizer.predict_vectors('a c', '')
pred_loaded = summarizer_loaded.predict_vectors('a c', '')
np.testing.assert_almost_equal(pred['logits'],
pred_loaded['logits'],
decimal=6)
def test_vectorize(self):
data = [('a b c', 'd')]
tokenizer_encoder = KerasTokenizer()
tokenizer_decoder = KerasTokenizer()
tokenizer_encoder.fit([data[0][0]])
tokenizer_decoder.fit([data[0][1]])
vectorizer = Vectorizer(tokenizer_encoder,
tokenizer_decoder,
max_output_len=3)
data_vectorized = [vectorizer(d) for d in data]
self.assertEqual([([1, 2, 3], [1, 0, 0])], data_vectorized)
data = [('a b c', 'd d d d')]
data_vectorized = [vectorizer(d) for d in data]
self.assertEqual([([1, 2, 3], [1, 1, 1])], data_vectorized)
def setUp(self) -> None:
tf.random.set_seed(42)
np.random.seed(42)
data = [('I love dogs.', 'Dogs.'), ('I love cats.', 'Cats.')]
tokenizer_encoder = BertTokenizer.from_pretrained('bert-base-uncased')
self.preprocessor = BertPreprocessor(nlp=English())
data_prep = [self.preprocessor(d) for d in data]
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder.fit([d[1] for d in data_prep])
self.vectorizer = BertVectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=10)
batch_generator = DatasetGenerator(2, rank=3)
data_vecs = [self.vectorizer(d) for d in data_prep]
self.dataset = batch_generator(lambda: data_vecs)
self.loss_func = masked_crossentropy
def setUp(self) -> None:
tf.random.set_seed(42)
np.random.seed(42)
self.data = [('a b', 'c'), ('a b c', 'd')]
tokenizer_encoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_encoder.fit(['a b c <start> <end>'])
tokenizer_decoder.fit(['c d <start> <end>'])
self.vectorizer = Vectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=3)
self.preprocessor = Preprocessor()
batch_generator = DatasetGenerator(2)
data_prep = [self.preprocessor(d) for d in self.data]
data_vecs = [self.vectorizer(d) for d in data_prep]
self.dataset = batch_generator(lambda: data_vecs)
self.loss_func = masked_crossentropy
def test_vectorize(self):
data = ('[CLS] I love my dog. [SEP] [CLS] He is the best. [SEP]',
'[CLS] Dog. [SEP]')
tokenizer_encoder = BertTokenizer.from_pretrained('bert-base-uncased')
tokenizer_decoder = KerasTokenizer()
tokenizer_decoder.fit([data[1]])
vectorizer = BertVectorizer(tokenizer_encoder,
tokenizer_decoder,
max_input_len=50,
max_output_len=3)
data_vectorized = vectorizer(data)
expected = ([
101, 1045, 2293, 2026, 3899, 1012, 102, 101, 2002, 2003, 1996,
2190, 1012, 102
], [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1], [1, 2, 3])
self.assertEqual(expected, data_vectorized)
input_decoded = vectorizer.decode_input(expected[0])
expected = '[CLS] i love my dog. [SEP] [CLS] he is the best. [SEP]'
self.assertEqual(expected, input_decoded)
def test_training(self) -> None:
data = [('a b', 'c'), ('a b c', 'd')]
tokenizer_encoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_encoder.fit(['a b c <start> <end>'])
tokenizer_decoder.fit(['c d <start> <end>'])
vectorizer = Vectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=3)
preprocessor = Preprocessor()
batch_generator = DatasetGenerator(2)
data_prep = [preprocessor(d) for d in data]
data_vecs = [vectorizer(d) for d in data_prep]
dataset = batch_generator(lambda: data_vecs)
summarizer_transformer = SummarizerTransformer(num_heads=1,
num_layers=1,
feed_forward_dim=20,
embedding_size=10,
dropout_rate=0,
max_prediction_len=3)
summarizer_transformer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer_attention = SummarizerAttention(lstm_size=10,
embedding_size=10)
summarizer_attention.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer = SummarizerBasic(lstm_size=10, embedding_size=10)
summarizer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
loss_func = masked_crossentropy
loss_attention = 0
train_step = summarizer_attention.new_train_step(
loss_function=loss_func, batch_size=2)
for _ in range(10):
for source_seq, target_seq in dataset.take(-1):
loss_attention = train_step(source_seq, target_seq)
print(str(loss_attention))
self.assertAlmostEqual(1.5810251235961914, float(loss_attention), 10)
output_attention = summarizer_attention.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.069454, 0.00272, 0.007199, -0.039547, 0.014357])
np.testing.assert_allclose(expected_first_logits,
output_attention['logits'][0],
atol=1e-6)
self.assertEqual('a c', output_attention['preprocessed_text'][0])
self.assertEqual('<end>', output_attention['predicted_text'])
loss = 0
train_step = summarizer.new_train_step(loss_function=loss_func,
batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss = train_step(source_seq, target_seq)
self.assertAlmostEqual(1.5771859884262085, float(loss), 10)
output = summarizer.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.03838864, 0.01226684, 0.01055636, -0.05209339, 0.02549592])
np.testing.assert_allclose(expected_first_logits,
output['logits'][0],
atol=1e-6)
self.assertEqual('a c', output['preprocessed_text'][0])
self.assertEqual('<end>', output['predicted_text'])
loss_transformer = 0
train_step = summarizer_transformer.new_train_step(
loss_function=loss_func, batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss_transformer = train_step(source_seq, target_seq)
print(str(loss_transformer))
self.assertAlmostEqual(1.2841172218322754, float(loss_transformer), 10)
output_transformer = summarizer_transformer.predict_vectors('a c', '')
expected_first_logits = np.array(
[0.094787, 0.516092, 1.165521, 0.271338, 0.670318])
np.testing.assert_allclose(expected_first_logits,
output_transformer['logits'][0],
atol=1e-6)
self.assertEqual('a c', output_transformer['preprocessed_text'][0])
self.assertEqual('d <end>', output_transformer['predicted_text'])
def test_training(self) -> None:
data = [('a b', 'c'), ('a b c', 'd')]
tokenizer_encoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_encoder.fit(['a b c <start> <end>'])
tokenizer_decoder.fit(['c d <start> <end>'])
vectorizer = Vectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=3)
preprocessor = Preprocessor()
batch_generator = DatasetGenerator(2)
data_prep = [preprocessor(d) for d in data]
data_vecs = [vectorizer(d) for d in data_prep]
dataset = batch_generator(lambda: data_vecs)
summarizer_transformer = SummarizerTransformer(num_heads=1,
num_layers=1,
feed_forward_dim=20,
embedding_size=10,
dropout_rate=0,
max_prediction_len=3)
summarizer_transformer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer_attention = SummarizerAttention(lstm_size=10,
embedding_size=10)
summarizer_attention.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer = SummarizerBasic(lstm_size=10, embedding_size=10)
summarizer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
loss_func = masked_crossentropy
loss_attention = 0
train_step = summarizer_attention.new_train_step(
loss_function=loss_func, batch_size=2)
for _ in range(10):
for source_seq, target_seq in dataset.take(-1):
loss_attention = train_step(source_seq, target_seq)
print(str(loss_attention))
self.assertAlmostEqual(1.577033519744873, float(loss_attention), 5)
output_attention = summarizer_attention.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.077805, 0.012667, 0.021359, -0.04872, 0.014989])
np.testing.assert_allclose(expected_first_logits,
output_attention['logits'][0],
atol=1e-6)
self.assertEqual('<start> a c <end>',
output_attention['preprocessed_text'][0])
self.assertEqual('d <end>', output_attention['predicted_text'])
loss = 0
train_step = summarizer.new_train_step(loss_function=loss_func,
batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss = train_step(source_seq, target_seq)
self.assertAlmostEqual(1.5713274478912354, float(loss), 5)
output = summarizer.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.051753, 0.013869, 0.010337, -0.073727, 0.033059])
np.testing.assert_allclose(expected_first_logits,
output['logits'][0],
atol=1e-6)
self.assertEqual('<start> a c <end>', output['preprocessed_text'][0])
self.assertEqual('<end>', output['predicted_text'])
loss_transformer = 0
train_step = summarizer_transformer.new_train_step(
loss_function=loss_func, batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss_transformer = train_step(source_seq, target_seq)
print(str(loss_transformer))
self.assertAlmostEqual(1.175953984260559, float(loss_transformer), 5)
output_transformer = summarizer_transformer.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.197903, 0.884185, 1.147212, 0.318798, 0.97936])
np.testing.assert_allclose(expected_first_logits,
output_transformer['logits'][0],
atol=1e-6)
self.assertEqual('<start> a c <end>',
output_transformer['preprocessed_text'][0])
self.assertEqual('d <end>', output_transformer['predicted_text'])