def test_initialize_word_tensorizer(self):
tensorizer = TokenTensorizer(text_column="text")
init = tensorizer.initialize()
init.send(None) # kick
for row in self.data.train:
init.send(row)
init.close()
self.assertEqual(49, len(tensorizer.vocab))
def test_create_word_tensors(self):
tensorizer = TokenTensorizer(text_column="text")
init = tensorizer.initialize()
init.send(None) # kick
for row in self.data.train:
init.send(row)
init.close()
rows = [{"text": "I want some coffee"}, {"text": "Turn it up"}]
tensors = (tensorizer.numberize(row) for row in rows)
tokens, seq_len = next(tensors)
self.assertEqual([24, 0, 0, 0], tokens)
self.assertEqual(4, seq_len)
tokens, seq_len = next(tensors)
self.assertEqual([13, 47, 9], tokens)
self.assertEqual(3, seq_len)
def test_initialize_token_tensorizer(self):
# default (build from data)
tensorizer = TokenTensorizer(text_column="text")
self._initialize_tensorizer(tensorizer)
self.assertEqual(49, len(tensorizer.vocab))
# size limit on tokens from data
tensorizer = TokenTensorizer(
text_column="text", vocab_config=VocabConfig(size_from_data=3)
)
self._initialize_tensorizer(tensorizer)
self.assertEqual(5, len(tensorizer.vocab)) # 3 + unk token + pad token
embed_file = tests_module.test_file("pretrained_embed_raw")
# vocab from data + vocab_file
tensorizer = TokenTensorizer(
text_column="text",
vocab_config=VocabConfig(
size_from_data=3,
vocab_files=[
VocabFileConfig(filepath=embed_file, skip_header_line=True)
],
),
)
self._initialize_tensorizer(tensorizer)
self.assertEqual(15, len(tensorizer.vocab))
# vocab just from vocab_file
tensorizer = TokenTensorizer(
text_column="text",
vocab_config=VocabConfig(
build_from_data=False,
vocab_files=[
VocabFileConfig(
filepath=embed_file, skip_header_line=True, size_limit=5
)
],
),
)
init = tensorizer.initialize()
# Should skip initialization
with self.assertRaises(StopIteration):
init.send(None)
self.assertEqual(7, len(tensorizer.vocab)) # 5 + unk token + pad token