def __init__(self, token_vocab, *args, **kwargs):
""""""
super(BaseVocab, self).__init__(*args, **kwargs)
self._cased = super(BaseVocab, self).cased
SubtokenVocab.__setattr__(self, '_token_vocab', token_vocab)
self._multibucket = Multibucket.from_configurable(self, embed_model=self.embed_model, name=self.name)
self._vocabs = [NgramVocab.from_vocab(self.token_vocab, i+1, cased=self.cased) for i in xrange(self.max_n)]
self._special_tokens = super(BaseVocab, self).special_tokens
self._special_tokens_set = set(self._special_tokens)
SubtokenVocab._set_special_tokens(self)
self._tok2idx = {}
for vocab in self:
assert vocab.token_vocab is self.token_vocab
return
def generate_placeholder(self): return SubtokenVocab.generate_placeholder(self)
def index(self, token): return SubtokenVocab.index(self, token)
def __call__(self, placeholder, keep_prob=None, moving_params=None):
print('===ngram_multivocab.py: Call ', ' name=', self.name)
return SubtokenVocab.__call__(self, placeholder, keep_prob=keep_prob, moving_params=moving_params)
def __call__(self, placeholder, keep_prob=None, moving_params=None):
return SubtokenVocab.__call__(self,
placeholder,
keep_prob=keep_prob,
moving_params=moving_params)