def map_tokens_to_ids_py(self, tokens: List[str]) -> np.ndarray:
r"""Maps text tokens into ids.
The input :attr:`tokens` and returned ids are both python
arrays or list.
Args:
tokens: A numpy array or (possibly nested) list of text tokens.
Returns:
A numpy array of token ids of the same shape as :attr:`tokens`.
"""
return dict_lookup(self.token_to_id_map_py, tokens, self.unk_token_id)
def map_ids_to_tokens_py(self, ids: np.ndarray) -> np.ndarray:
r"""Maps ids into text tokens.
The input :attr:`ids` and returned tokens are both python
arrays or list.
Args:
ids: An `int` numpy array or (possibly nested) list of token ids.
Returns:
A numpy array of text tokens of the same shape as :attr:`ids`.
"""
return dict_lookup(self.id_to_token_map_py, ids, self.unk_token)