class CharField(Field): def __init__(self, *args, **kwargs): self.fix_len = kwargs.pop('fix_len') if 'fix_len' in kwargs else -1 super(CharField, self).__init__(*args, **kwargs) def build(self, corpus, min_freq=1, embed=None): sequences = getattr(corpus, self.name) counter = Counter(char for sequence in sequences for token in sequence for char in self.transform(token)) self.vocab = Vocab(counter, min_freq, self.specials) if not embed: self.embed = None else: tokens = self.transform(embed.tokens) # if the `unk` token has existed in the pretrained, # then replace it with a self-defined one if embed.unk: tokens[embed.unk_index] = self.unk self.vocab.extend(tokens) self.embed = torch.zeros(len(self.vocab), embed.dim) self.embed[self.vocab.token2id(tokens)] = embed.vectors def numericalize(self, sequences): sequences = [[self.transform(token) for token in sequence] for sequence in sequences] if self.fix_len <= 0: self.fix_len = max( len(token) for sequence in sequences for token in sequence) if self.use_vocab: sequences = [[self.vocab.token2id(token) for token in sequence] for sequence in sequences] if self.bos: sequences = [[self.vocab.token2id(self.bos)] + sequence for sequence in sequences] if self.eos: sequences = [ sequence + [self.vocab.token2id(self.eos)] for sequence in sequences ] sequences = [ torch.tensor([ ids[:self.fix_len] + [0] * (self.fix_len - len(ids)) for ids in sequence ]) for sequence in sequences ] return sequences
class Field(object): def __init__(self, name, pad=None, unk=None, bos=None, eos=None, lower=False, use_vocab=True, tokenize=None, fn=None): self.name = name self.pad = pad self.unk = unk self.bos = bos self.eos = eos self.lower = lower self.use_vocab = use_vocab self.tokenize = tokenize self.fn = fn self.specials = [ token for token in [pad, unk, bos, eos] if token is not None ] def __repr__(self): s, params = f"({self.name}): {self.__class__.__name__}(", [] if self.pad is not None: params.append(f"pad={self.pad}") if self.unk is not None: params.append(f"unk={self.unk}") if self.bos is not None: params.append(f"bos={self.bos}") if self.eos is not None: params.append(f"eos={self.eos}") if self.lower: params.append(f"lower={self.lower}") if not self.use_vocab: params.append(f"use_vocab={self.use_vocab}") s += f", ".join(params) s += f")" return s @property def pad_index(self): return self.specials.index(self.pad) if self.pad is not None else 0 @property def unk_index(self): return self.specials.index(self.unk) if self.unk is not None else 0 @property def bos_index(self): return self.specials.index(self.bos) @property def eos_index(self): return self.specials.index(self.eos) def transform(self, sequence): if self.tokenize is not None: sequence = self.tokenize(sequence) if self.lower: sequence = [str.lower(token) for token in sequence] if self.fn is not None: sequence = [self.fn(token) for token in sequence] return sequence def build(self, corpus, min_freq=1, embed=None): sequences = getattr(corpus, self.name) counter = Counter(token for sequence in sequences for token in self.transform(sequence)) self.vocab = Vocab(counter, min_freq, self.specials, self.name) if not embed: self.embed = None else: tokens = self.transform(embed.tokens) # if the `unk` token has existed in the pretrained, # then replace it with a self-defined one if embed.unk: tokens[embed.unk_index] = self.unk self.vocab.extend(tokens) self.embed = torch.zeros(len(self.vocab), embed.dim) self.embed[self.vocab.token2id(tokens)] = embed.vectors self.embed /= torch.std(self.embed) # transfer word/pos to it's id def numericalize(self, sequences): sequences = [self.transform(sequence) for sequence in sequences] if self.use_vocab: sequences = [ self.vocab.token2id(sequence) for sequence in sequences ] if self.bos: # if self.name == 'tags': # sequences = [[1] + sequence for sequence in sequences] # else: sequences = [[self.bos_index] + sequence for sequence in sequences] if self.eos: sequences = [sequence + [self.eos_index] for sequence in sequences] sequences = [torch.tensor(sequence) for sequence in sequences] return sequences
class NGramField(Field): def __init__(self, *args, **kwargs): self.n = kwargs.pop('n') if 'n' in kwargs else 1 super(NGramField, self).__init__(*args, **kwargs) def build(self, corpus, min_freq=1, dict_file=None, embed=None): sequences = getattr(corpus, self.name) counter = Counter() sequences = [self.preprocess(sequence) for sequence in sequences] n_pad = self.n - 1 for sequence in sequences: chars = list(sequence) + [eos] * n_pad bichars = [ "".join(chars[i + s] for s in range(self.n)) for i in range(len(chars) - n_pad) ] counter.update(bichars) if dict_file is not None: counter &= self.read_dict(dict_file) self.vocab = Vocab(counter, min_freq, self.specials, self.unk_index) if not embed: self.embed = None else: tokens = self.preprocess(embed.tokens) # if the `unk` token has existed in the pretrained, # then replace it with a self-defined one if embed.unk: tokens[embed.unk_index] = self.unk self.vocab.extend(tokens) self.embed = torch.zeros(len(self.vocab), embed.dim) self.embed[self.vocab.token2id(tokens)] = embed.vectors self.embed /= torch.std(self.embed) def read_dict(self, dict_file): word_list = dict() with open(dict_file, encoding='utf-8') as dict_in: for line in dict_in: line = line.split() if len(line) == 3: word_list[line[0]] = 100 return Counter(word_list) def __repr__(self): s, params = f"({self.name}): {self.__class__.__name__}(", [] params.append(f"n={self.n}") if self.pad is not None: params.append(f"pad={self.pad}") if self.unk is not None: params.append(f"unk={self.unk}") if self.bos is not None: params.append(f"bos={self.bos}") if self.eos is not None: params.append(f"eos={self.eos}") if self.lower: params.append(f"lower={self.lower}") if not self.use_vocab: params.append(f"use_vocab={self.use_vocab}") if self.tohalfwidth: params.append(f"tohalfwidth={self.tohalfwidth}") s += f", ".join(params) s += f")" return s def transform(self, sequences): sequences = [self.preprocess(sequence) for sequence in sequences] n_pad = (self.n - 1) for sent_idx, sequence in enumerate(sequences): chars = list(sequence) + [eos] * n_pad sequences[sent_idx] = [ "".join(chars[i + s] for s in range(self.n)) for i in range(len(chars) - n_pad) ] if self.use_vocab: sequences = [ self.vocab.token2id(sequence) for sequence in sequences ] if self.bos: sequences = [[self.bos_index] + sequence for sequence in sequences] if self.eos: sequences = [sequence + [self.eos_index] for sequence in sequences] sequences = [torch.tensor(sequence) for sequence in sequences] return sequences