def testIdToToken(self):
with self.session(use_gpu=False) as sess:
token_ids = [[12, 9, 16, 16, 19, 2, 2, 2, 2,
2], [23, 13, 22, 73, 2, 2, 2, 2, 2,
2], [27, 12, 5, 24, 3, 5, 3, 0, 3,
2], [5, 17, 9, 22, 13, 7, 5, 32, 23, 2],
[4, 3, 9, 5, 22, 16, 29, 2, 2,
2], [40, 34, 39, 39, 3, 5, 17, 2, 2,
2], [52, 2, 2, 2, 2, 2, 2, 2, 2, 2]]
seq_lens = [5, 4, 9, 9, 7, 7, 1]
tokens = sess.run(py_x_ops.id_to_ascii(token_ids, seq_lens))
self.assertEqual(tokens.tolist(), [
'hello', 'sir<epsilon>', 'what a <unk> ', "america's", '<noise> early',
'1:00 am', '%'
])
def IdsToStrings(self, ids, lens):
return py_x_ops.id_to_ascii(ids, lens)
