def testIdToToken(self):
with self.session(use_gpu=False):
token_ids = [[12, 9, 16, 16, 19, 2, 2, 2, 2, 2],
[23, 13, 22, 73, 2, 2, 2, 2, 2, 2],
[27, 12, 5, 24, 3, 5, 3, 0, 3, 2],
[5, 17, 9, 22, 13, 7, 5, 32, 23, 2],
[4, 3, 9, 5, 22, 16, 29, 2, 2, 2],
[40, 34, 39, 39, 3, 5, 17, 2, 2, 2],
[52, 2, 2, 2, 2, 2, 2, 2, 2, 2]]
seq_lens = [5, 4, 9, 9, 7, 7, 1]
tokens = self.evaluate(ops.id_to_ascii(token_ids, seq_lens))
self.assertEqual(tokens.tolist(), [
b'hello', b'sir<epsilon>', b'what a <unk> ', b"america's",
b'<noise> early', b'1:00 am', b'%'
])
def IdsToStrings(self, ids, lens):
return ops.id_to_ascii(ids, lens)