def __call__(self, sentence: List[str]) -> List[str]:
"""Adapted code from BPE.segment """
output = []
for word in sentence:
# Hack. TODO: inspect why there are empty sentences
if len(word) == 0:
output.append(word)
continue
new_word = encode(word, self.bpe.bpe_codes)
for item in new_word[:-1]:
output.append(item + self.bpe.separator)
output.append(new_word[-1])
return output
def __call__(self, sentence: List[str]) -> List[str]:
"""Adapted code from BPE.segment."""
output = []
for word in sentence:
# Hack. TODO: inspect why there are empty sentences
if not word:
output.append(word)
continue
new_word = encode(word, self.bpe.bpe_codes)
for item in new_word[:-1]:
output.append(item + self.bpe.separator)
output.append(new_word[-1])
return output