class CharField(Field): def __init__(self, *args, **kwargs): self.fix_len = kwargs.pop('fix_len') if 'fix_len' in kwargs else -1 super(CharField, self).__init__(*args, **kwargs) def build(self, corpus, min_freq=1, embed=None): sequences = getattr(corpus, self.name) counter = Counter(char for sequence in sequences for token in sequence for char in self.transform(token)) self.vocab = Vocab(counter, min_freq, self.specials) if not embed: self.embed = None else: tokens = self.transform(embed.tokens) # if the `unk` token has existed in the pretrained, # then replace it with a self-defined one if embed.unk: tokens[embed.unk_index] = self.unk self.vocab.extend(tokens) self.embed = torch.zeros(len(self.vocab), embed.dim) self.embed[self.vocab.token2id(tokens)] = embed.vectors def numericalize(self, sequences): sequences = [[self.transform(token) for token in sequence] for sequence in sequences] if self.fix_len <= 0: self.fix_len = max(len(token) for sequence in sequences for token in sequence) if self.use_vocab: sequences = [[self.vocab.token2id(token) for token in sequence] for sequence in sequences] if self.bos: sequences = [[self.vocab.token2id(self.bos)] + sequence for sequence in sequences] if self.eos: sequences = [sequence + [self.vocab.token2id(self.eos)] for sequence in sequences] sequences = [ torch.tensor([ids[:self.fix_len] + [0] * (self.fix_len - len(ids)) for ids in sequence]) for sequence in sequences ] return sequences
class Field(object): def __init__(self, name, pad=None, unk=None, bos=None, eos=None, lower=False, use_vocab=True, tokenize=None, fn=None): self.name = name self.pad = pad self.unk = unk self.bos = bos self.eos = eos self.lower = lower self.use_vocab = use_vocab self.tokenize = tokenize self.fn = fn self.specials = [token for token in [pad, unk, bos, eos] if token is not None] def __repr__(self): s, params = f"({self.name}): {self.__class__.__name__}(", [] if self.pad is not None: params.append(f"pad={self.pad}") if self.unk is not None: params.append(f"unk={self.unk}") if self.bos is not None: params.append(f"bos={self.bos}") if self.eos is not None: params.append(f"eos={self.eos}") if self.lower: params.append(f"lower={self.lower}") if not self.use_vocab: params.append(f"use_vocab={self.use_vocab}") s += f", ".join(params) s += f")" return s @property def pad_index(self): return self.specials.index(self.pad) if self.pad is not None else 0 @property def unk_index(self): return self.specials.index(self.unk) if self.unk is not None else 0 @property def bos_index(self): return self.specials.index(self.bos) @property def eos_index(self): return self.specials.index(self.eos) def transform(self, sequence): if self.tokenize is not None: sequence = self.tokenize(sequence) if self.lower: sequence = [str.lower(token) for token in sequence] if self.fn is not None: sequence = [self.fn(token) for token in sequence] return sequence def build(self, corpus, min_freq=1, embed=None): sequences = getattr(corpus, self.name) counter = Counter(token for sequence in sequences for token in self.transform(sequence)) self.vocab = Vocab(counter, min_freq, self.specials) if not embed: self.embed = None else: tokens = self.transform(embed.tokens) # if the `unk` token has existed in the pretrained, # then replace it with a self-defined one if embed.unk: tokens[embed.unk_index] = self.unk self.vocab.extend(tokens) self.embed = torch.zeros(len(self.vocab), embed.dim) self.embed[self.vocab.token2id(tokens)] = embed.vectors self.embed /= torch.std(self.embed) def numericalize(self, sequences): sequences = [self.transform(sequence) for sequence in sequences] if self.use_vocab: sequences = [self.vocab.token2id(sequence) for sequence in sequences] if self.bos: sequences = [[self.bos_index] + sequence for sequence in sequences] if self.eos: sequences = [sequence + [self.eos_index] for sequence in sequences] sequences = [torch.tensor(sequence) for sequence in sequences] return sequences