def reinit(self, vocabs, parse_files): """ """ self.preopen_parse_file = parse_files self._vocabs = vocabs self._multibuckets = [ Multibucket.from_configurable(vocab, name='%s-%s' % (self.name, vocab.name)) for vocab in self.vocabs ] self._metadata = [] with Bucketer.from_configurable(self, self.n_buckets, name='bucketer-%s' % self.name) as bucketer: splits = bucketer.compute_splits( len(sent) for sent, metadata in self.iterfiles()) for i in range(len(splits)): splits[i] += 1 for multibucket, vocab in self.iteritems(): multibucket.open(splits, depth=vocab.depth) for sent, metadata in self.iterfiles(): self._metadata.append(metadata) for multibucket, vocab in self.iteritems(): tokens = [line[vocab.conll_idx] for line in sent] idxs = [vocab.ROOT] + [vocab.index(token) for token in tokens] multibucket.add(idxs, tokens) for multibucket in self: multibucket.close() self._multibucket = Multibucket.from_dataset(self) return
def __init__(self, vocabs, *args, **kwargs): """ """ nlp_model = kwargs.pop('nlp_model', None) if "parse_files" in kwargs and isinstance( kwargs["parse_files"], io.StringIO): ### SPECIAL CASE - PARSING StringIO self.preopen_parse_file = kwargs.pop( "parse_files" ) #This doesn't really play well with the configparser thing else: self.preopen_parse_file = None super(Dataset, self).__init__(*args, **kwargs) self._vocabs = vocabs self._multibuckets = [ Multibucket.from_configurable(vocab, name='%s-%s' % (self.name, vocab.name)) for vocab in self.vocabs ] self._metadata = [] if nlp_model is not None: self._nlp_model = nlp_model.from_configurable(self, name=self.name) else: self._nlp_model = None with Bucketer.from_configurable(self, self.n_buckets, name='bucketer-%s' % self.name) as bucketer: splits = bucketer.compute_splits( len(sent) for sent, metadata in self.iterfiles()) for i in range(len(splits)): splits[i] += 1 for multibucket, vocab in self.iteritems(): multibucket.open(splits, depth=vocab.depth) for sent, metadata in self.iterfiles(): self._metadata.append(metadata) for multibucket, vocab in self.iteritems(): tokens = [line[vocab.conll_idx] for line in sent] idxs = [vocab.ROOT] + [vocab.index(token) for token in tokens] multibucket.add(idxs, tokens) for multibucket in self: multibucket.close() self._multibucket = Multibucket.from_dataset(self) return
def __init__(self, token_vocab, *args, **kwargs): """ """ recount = kwargs.pop('recount', False) initialize_zero = kwargs.pop('initialize_zero', False) super(TokenVocab, self).__init__(*args, **kwargs) self._token_vocab = token_vocab self._token_counts = Counter() self._multibucket = Multibucket.from_configurable(self, embed_model=self.embed_model, name=self.name) self._tok2idx = {} if recount: self.count() else: if os.path.isfile(self.filename): self.load() else: self.count() self.dump() self.index_vocab() embed_dims = [len(self), self.embed_size] if initialize_zero: self._embeddings_array = np.zeros(embed_dims) else: self._embeddings_array = np.random.randn(*embed_dims) return
def __init__(self, token_vocab, *args, **kwargs): """ """ super(BaseVocab, self).__init__(*args, **kwargs) self._cased = super(BaseVocab, self).cased SubtokenVocab.__setattr__(self, '_token_vocab', token_vocab) self._multibucket = Multibucket.from_configurable(self, embed_model=self.embed_model, name=self.name) self._vocabs = [NgramVocab.from_vocab(self.token_vocab, i+1, cased=self.cased) for i in range(self.max_n)] self._special_tokens = super(BaseVocab, self).special_tokens self._special_tokens_set = set(self._special_tokens) SubtokenVocab._set_special_tokens(self) self._tok2idx = {} for vocab in self: assert vocab.token_vocab is self.token_vocab return