def _get_subtoken_encoder():
with lock:
global _subtoken_encoder
if not _subtoken_encoder:
_subtoken_encoder = text_encoder.SubwordTextEncoder(
resources.get_vocab_file())
return _subtoken_encoder
def _get_token_vocab():
with lock:
global _token_vocab
if not _token_vocab:
_token_vocab = {}
tokens, _, _ = create_token_vocab.read_vocab(resources.get_vocab_file())
_token_vocab = dict(zip(tokens, range(len(tokens))))
return _token_vocab