def test_encode_decode_from_loaded_tokenizer_with_1_maxlen(self, temp_dir): tokenizer = Tokenizer(self.captions, 1) tokenizer.save_dictionaries(temp_dir.path) loading_tokenizer = Tokenizer() loading_tokenizer.load_dictionaries(temp_dir.path) phrase_encoded = loading_tokenizer.encode_caption(self.phrase) phrase_decoded = tokenizer.decode_caption(phrase_encoded) self.assertEqual(phrase_decoded, [Tokenizer.END])
def test_encode_decode(self): for value in [None, 5, 10]: with self.subTest(captions=self.captions, user_maxlen=value): tokenizer = Tokenizer(self.captions, value) phrase_encoded = tokenizer.encode_caption(self.phrase) phrase_decoded = tokenizer.decode_caption(phrase_encoded) expected_string = ["THERE", Tokenizer.UNK, "ONE", "HAND"] pad_length = self.max_phraselen if (value is None) or ( value >= self.max_phraselen) else value self.assertEqual( phrase_decoded[:pad_length], expected_string + [Tokenizer.END] * (pad_length - len(expected_string)))
def test_encode_decode_from_loaded_tokenizer(self, temp_dir): for value in [None, 5, 10]: with self.subTest(captions=self.captions, user_maxlen=value): tokenizer = Tokenizer(self.captions, value) tokenizer.save_dictionaries(temp_dir.path) loading_tokenizer = Tokenizer() loading_tokenizer.load_dictionaries(temp_dir.path) phrase_encoded = loading_tokenizer.encode_caption(self.phrase) phrase_decoded = tokenizer.decode_caption(phrase_encoded) expected_string = ["THERE", Tokenizer.UNK, "ONE", "HAND"] pad_length = self.max_phraselen if (value is None) or ( value >= self.max_phraselen) else value self.assertEqual( phrase_decoded[:pad_length], expected_string + [Tokenizer.END] * (pad_length - len(expected_string)))
def test_encode_decode_with_1_maxlen(self): tokenizer = Tokenizer(self.captions, 1) phrase_encoded = tokenizer.encode_caption(self.phrase) phrase_decoded = tokenizer.decode_caption(phrase_encoded) self.assertEqual(phrase_decoded, [Tokenizer.END])