def test_pad_decr(self): self.assertEqual([2, 1, 0], text_encoder.pad_decr([3, 2, 1])) self.assertEqual([2, 1, 0], text_encoder.pad_decr([3, 2, 1, 0, 0, 0])) self.assertEqual([-1, 2, 1, 0], text_encoder.pad_decr([0, 3, 2, 1, 0, 0])) self.assertEqual([], text_encoder.pad_decr([])) self.assertEqual([], text_encoder.pad_decr(np.array([])))
def decode(self, ids): """Decodes a list of integers into text.""" ids = text_encoder.pad_decr(ids) subword_ids = ids del ids subwords = [] # Some ids correspond to bytes. Because unicode characters are composed of # possibly multiple bytes, we attempt to decode contiguous lists of bytes # all together. Invalid byte sequences are replaced with the unicode # replacement (i.e. unknown) character U+FFFD. prev_bytes = [] def consume_prev_bytes(): if prev_bytes: bytestr = b"".join(prev_bytes) bytes_text = bytestr.decode("utf-8", "replace") subwords.append(bytes_text) return [] for subword_id in subword_ids: subword = self._id_to_subword(subword_id) if isinstance(subword, six.binary_type): # Byte-encoded prev_bytes.append(subword) else: # If there were bytes previously, convert to unicode. prev_bytes = consume_prev_bytes() trimmed, add_space = _trim_underscore_and_tell(subword) subwords.append(trimmed) if add_space: subwords.append(" ") # If there were trailing bytes, convert to unicode. prev_bytes = consume_prev_bytes() return tf.compat.as_text("".join(subwords))