def test_serde_happy_path(self) -> None:
preprocessor = Preprocessor()
tokenizer = KerasTokenizer(oov_token='<unk>')
tokenizer.fit(['a b c {} {}'.format(
preprocessor.start_token, preprocessor.end_token)])
vectorizer = Vectorizer(tokenizer, tokenizer)
summarizer = AttentionSummarizer(lstm_size=10,
max_prediction_len=10,
embedding_size=10,
embedding_encoder_trainable=False)
summarizer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer)
# we need at least a train step to init the weights
train_step = summarizer.new_train_step(masked_crossentropy, batch_size=1, apply_gradients=True)
train_seq = tf.convert_to_tensor(np.array([[1, 1, 1]]), dtype=tf.int32)
train_step(train_seq, train_seq)
save_dir = os.path.join(self.temp_dir, 'summarizer_serde_happy_path')
summarizer.save(save_dir)
summarizer_loaded = AttentionSummarizer.load(save_dir)
self.assertEqual(10, summarizer_loaded.lstm_size)
self.assertEqual(10, summarizer_loaded.max_prediction_len)
self.assertIsNotNone(summarizer_loaded.preprocessor)
self.assertIsNotNone(summarizer_loaded.vectorizer)
self.assertIsNotNone(summarizer_loaded.encoder)
self.assertIsNotNone(summarizer_loaded.decoder)
self.assertFalse(summarizer_loaded.encoder.embedding.trainable)
self.assertTrue(summarizer_loaded.decoder.embedding.trainable)
self.assertIsNotNone(summarizer_loaded.optimizer)
pred = summarizer.predict_vectors('a c', '')
pred_loaded = summarizer_loaded.predict_vectors('a c', '')
np.testing.assert_almost_equal(pred['logits'], pred_loaded['logits'], decimal=6)
def test_serde_happy_path(self) -> None:
preprocessor = Preprocessor(start_token='[CLS]', end_token='[SEP]')
tokenizer_encoder = BertTokenizer.from_pretrained('bert-base-uncased')
tokenizer_decoder = KerasTokenizer(oov_token='<unk>')
tokenizer_decoder.fit([
'a b c {} {}'.format(preprocessor.start_token,
preprocessor.end_token)
])
vectorizer = Vectorizer(tokenizer_encoder, tokenizer_decoder)
summarizer = SummarizerBert(num_layers_encoder=1,
num_layers_decoder=1,
bert_embedding_encoder='bert-base-uncased',
num_heads=2,
max_prediction_len=3,
embedding_size_encoder=768,
embedding_size_decoder=10,
embedding_encoder_trainable=False)
summarizer.init_model(preprocessor=preprocessor, vectorizer=vectorizer)
# we need at least a train step to init the weights
train_step = summarizer.new_train_step(masked_crossentropy,
batch_size=1,
apply_gradients=True)
train_seq = tf.convert_to_tensor(np.array([[1, 1, 1]]), dtype=tf.int32)
train_step(train_seq, train_seq)
save_dir = os.path.join(self.temp_dir, 'summarizer_serde_happy_path')
summarizer.save(save_dir)
summarizer_loaded = SummarizerBert.load(save_dir)
self.assertEqual(1, summarizer_loaded.num_layers_encoder)
self.assertEqual(1, summarizer_loaded.num_layers_decoder)
self.assertEqual(2, summarizer_loaded.num_heads)
self.assertEqual(3, summarizer_loaded.max_prediction_len)
self.assertEqual(768, summarizer_loaded.embedding_size_encoder)
self.assertEqual(10, summarizer_loaded.embedding_size_decoder)
self.assertIsNotNone(summarizer_loaded.preprocessor)
self.assertIsNotNone(summarizer_loaded.vectorizer)
self.assertIsNotNone(summarizer_loaded.transformer)
self.assertFalse(
summarizer_loaded.transformer.encoder.embedding.trainable)
self.assertTrue(
summarizer_loaded.transformer.decoder.embedding.trainable)
self.assertIsNotNone(summarizer_loaded.optimizer_encoder)
self.assertIsNotNone(summarizer_loaded.optimizer_decoder)
pred = summarizer.predict_vectors('a c', '')
pred_loaded = summarizer_loaded.predict_vectors('a c', '')
np.testing.assert_almost_equal(pred['logits'],
pred_loaded['logits'],
decimal=6)
def test_vectorize(self):
data = [('a b c', 'd')]
tokenizer_encoder = KerasTokenizer()
tokenizer_decoder = KerasTokenizer()
tokenizer_encoder.fit([data[0][0]])
tokenizer_decoder.fit([data[0][1]])
vectorizer = Vectorizer(tokenizer_encoder,
tokenizer_decoder,
max_output_len=3)
data_vectorized = [vectorizer(d) for d in data]
self.assertEqual([([1, 2, 3], [1, 0, 0])], data_vectorized)
data = [('a b c', 'd d d d')]
data_vectorized = [vectorizer(d) for d in data]
self.assertEqual([([1, 2, 3], [1, 1, 1])], data_vectorized)
def setUp(self) -> None:
tf.random.set_seed(42)
np.random.seed(42)
self.data = [('a b', 'c'), ('a b c', 'd')]
tokenizer_encoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_encoder.fit(['a b c <start> <end>'])
tokenizer_decoder.fit(['c d <start> <end>'])
self.vectorizer = Vectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=3)
self.preprocessor = Preprocessor()
batch_generator = DatasetGenerator(2)
data_prep = [self.preprocessor(d) for d in self.data]
data_vecs = [self.vectorizer(d) for d in data_prep]
self.dataset = batch_generator(lambda: data_vecs)
self.loss_func = masked_crossentropy
def _init_model(self, summarizer: Summarizer,
train_data: Iterable[Tuple[str, str]]) -> None:
tokenizer_encoder, tokenizer_decoder = self._create_tokenizers(
train_data)
self.logger.info(
'vocab encoder: {vocab_enc}, vocab decoder: {vocab_dec}'.format(
vocab_enc=tokenizer_encoder.vocab_size,
vocab_dec=tokenizer_decoder.vocab_size))
vectorizer = Vectorizer(tokenizer_encoder,
tokenizer_decoder,
max_input_len=self.max_input_len,
max_output_len=self.max_output_len)
embedding_weights_encoder, embedding_weights_decoder = None, None
if self.embedding_path_encoder is not None:
self.logger.info('loading encoder embedding from {}'.format(
self.embedding_path_encoder))
embedding = read_embedding(self.embedding_path_encoder,
summarizer.embedding_size)
embedding_weights_encoder = embedding_to_matrix(
embedding=embedding,
token_index=tokenizer_encoder.token_index,
embedding_dim=summarizer.embedding_size)
if self.embedding_path_decoder is not None:
self.logger.info('loading decoder embedding from {}'.format(
self.embedding_path_decoder))
embedding = read_embedding(self.embedding_path_decoder,
summarizer.embedding_size)
embedding_weights_decoder = embedding_to_matrix(
embedding=embedding,
token_index=tokenizer_decoder.token_index,
embedding_dim=summarizer.embedding_size)
summarizer.init_model(
preprocessor=self.preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=embedding_weights_encoder,
embedding_weights_decoder=embedding_weights_decoder)
def test_training(self) -> None:
data = [('a b', 'c'), ('a b c', 'd')]
tokenizer_encoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_encoder.fit(['a b c <start> <end>'])
tokenizer_decoder.fit(['c d <start> <end>'])
vectorizer = Vectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=3)
preprocessor = Preprocessor()
batch_generator = DatasetGenerator(2)
data_prep = [preprocessor(d) for d in data]
data_vecs = [vectorizer(d) for d in data_prep]
dataset = batch_generator(lambda: data_vecs)
summarizer_transformer = SummarizerTransformer(num_heads=1,
num_layers=1,
feed_forward_dim=20,
embedding_size=10,
dropout_rate=0,
max_prediction_len=3)
summarizer_transformer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer_attention = SummarizerAttention(lstm_size=10,
embedding_size=10)
summarizer_attention.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer = SummarizerBasic(lstm_size=10, embedding_size=10)
summarizer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
loss_func = masked_crossentropy
loss_attention = 0
train_step = summarizer_attention.new_train_step(
loss_function=loss_func, batch_size=2)
for _ in range(10):
for source_seq, target_seq in dataset.take(-1):
loss_attention = train_step(source_seq, target_seq)
print(str(loss_attention))
self.assertAlmostEqual(1.5810251235961914, float(loss_attention), 10)
output_attention = summarizer_attention.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.069454, 0.00272, 0.007199, -0.039547, 0.014357])
np.testing.assert_allclose(expected_first_logits,
output_attention['logits'][0],
atol=1e-6)
self.assertEqual('a c', output_attention['preprocessed_text'][0])
self.assertEqual('<end>', output_attention['predicted_text'])
loss = 0
train_step = summarizer.new_train_step(loss_function=loss_func,
batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss = train_step(source_seq, target_seq)
self.assertAlmostEqual(1.5771859884262085, float(loss), 10)
output = summarizer.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.03838864, 0.01226684, 0.01055636, -0.05209339, 0.02549592])
np.testing.assert_allclose(expected_first_logits,
output['logits'][0],
atol=1e-6)
self.assertEqual('a c', output['preprocessed_text'][0])
self.assertEqual('<end>', output['predicted_text'])
loss_transformer = 0
train_step = summarizer_transformer.new_train_step(
loss_function=loss_func, batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss_transformer = train_step(source_seq, target_seq)
print(str(loss_transformer))
self.assertAlmostEqual(1.2841172218322754, float(loss_transformer), 10)
output_transformer = summarizer_transformer.predict_vectors('a c', '')
expected_first_logits = np.array(
[0.094787, 0.516092, 1.165521, 0.271338, 0.670318])
np.testing.assert_allclose(expected_first_logits,
output_transformer['logits'][0],
atol=1e-6)
self.assertEqual('a c', output_transformer['preprocessed_text'][0])
self.assertEqual('d <end>', output_transformer['predicted_text'])
def test_training(self) -> None:
data = [('a b', 'c'), ('a b c', 'd')]
tokenizer_encoder = KerasTokenizer(lower=False, filters='')
tokenizer_decoder = KerasTokenizer(lower=False, filters='')
tokenizer_encoder.fit(['a b c <start> <end>'])
tokenizer_decoder.fit(['c d <start> <end>'])
vectorizer = Vectorizer(tokenizer_encoder=tokenizer_encoder,
tokenizer_decoder=tokenizer_decoder,
max_output_len=3)
preprocessor = Preprocessor()
batch_generator = DatasetGenerator(2)
data_prep = [preprocessor(d) for d in data]
data_vecs = [vectorizer(d) for d in data_prep]
dataset = batch_generator(lambda: data_vecs)
summarizer_transformer = SummarizerTransformer(num_heads=1,
num_layers=1,
feed_forward_dim=20,
embedding_size=10,
dropout_rate=0,
max_prediction_len=3)
summarizer_transformer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer_attention = SummarizerAttention(lstm_size=10,
embedding_size=10)
summarizer_attention.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
summarizer = SummarizerBasic(lstm_size=10, embedding_size=10)
summarizer.init_model(preprocessor=preprocessor,
vectorizer=vectorizer,
embedding_weights_encoder=None,
embedding_weights_decoder=None)
loss_func = masked_crossentropy
loss_attention = 0
train_step = summarizer_attention.new_train_step(
loss_function=loss_func, batch_size=2)
for _ in range(10):
for source_seq, target_seq in dataset.take(-1):
loss_attention = train_step(source_seq, target_seq)
print(str(loss_attention))
self.assertAlmostEqual(1.577033519744873, float(loss_attention), 5)
output_attention = summarizer_attention.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.077805, 0.012667, 0.021359, -0.04872, 0.014989])
np.testing.assert_allclose(expected_first_logits,
output_attention['logits'][0],
atol=1e-6)
self.assertEqual('<start> a c <end>',
output_attention['preprocessed_text'][0])
self.assertEqual('d <end>', output_attention['predicted_text'])
loss = 0
train_step = summarizer.new_train_step(loss_function=loss_func,
batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss = train_step(source_seq, target_seq)
self.assertAlmostEqual(1.5713274478912354, float(loss), 5)
output = summarizer.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.051753, 0.013869, 0.010337, -0.073727, 0.033059])
np.testing.assert_allclose(expected_first_logits,
output['logits'][0],
atol=1e-6)
self.assertEqual('<start> a c <end>', output['preprocessed_text'][0])
self.assertEqual('<end>', output['predicted_text'])
loss_transformer = 0
train_step = summarizer_transformer.new_train_step(
loss_function=loss_func, batch_size=2)
for e in range(0, 10):
for source_seq, target_seq in dataset.take(-1):
loss_transformer = train_step(source_seq, target_seq)
print(str(loss_transformer))
self.assertAlmostEqual(1.175953984260559, float(loss_transformer), 5)
output_transformer = summarizer_transformer.predict_vectors('a c', '')
expected_first_logits = np.array(
[-0.197903, 0.884185, 1.147212, 0.318798, 0.97936])
np.testing.assert_allclose(expected_first_logits,
output_transformer['logits'][0],
atol=1e-6)
self.assertEqual('<start> a c <end>',
output_transformer['preprocessed_text'][0])
self.assertEqual('d <end>', output_transformer['predicted_text'])