def test_character_token_embedder(self): vocab = Dictionary() vocab.add_symbol('hello') vocab.add_symbol('there') embedder = CharacterTokenEmbedder(vocab, [(2, 16), (4, 32), (8, 64), (16, 2)], 64, 5, 2) test_sents = [['hello', 'unk', 'there'], ['there'], ['hello', 'there']] max_len = max(len(s) for s in test_sents) input = torch.LongTensor(len(test_sents), max_len + 2).fill_(vocab.pad()) for i in range(len(test_sents)): input[i][0] = vocab.eos() for j in range(len(test_sents[i])): input[i][j + 1] = vocab.index(test_sents[i][j]) input[i][j + 2] = vocab.eos() embs = embedder(input) assert embs.size() == (len(test_sents), max_len + 2, 5) self.assertAlmostEqual(embs[0][0], embs[1][0]) self.assertAlmostEqual(embs[0][0], embs[0][-1]) self.assertAlmostEqual(embs[0][1], embs[2][1]) self.assertAlmostEqual(embs[0][3], embs[1][1]) embs.sum().backward() assert embedder.char_embeddings.weight.grad is not None
def create_lang_dictionary(cls, langs): unk = "<unk>" # hack to remove symbols other than unk as they are not needed by lang dict lang_dict = Dictionary(pad=unk, eos=unk, unk=unk, bos=unk) for lang in langs: lang_dict.add_symbol(lang) return lang_dict
def setup_task(cls, args, **kwargs): """Setup the task. """ dictionary = Dictionary() for i in range(args.dict_size): dictionary.add_symbol("word{}".format(i)) logger.info("dictionary: {} types".format(len(dictionary))) return cls(args, dictionary)
def build_vocab(data: tp.List[tp.List[str]]) -> Dictionary: d = Dictionary() for s in data: for token in s: d.add_symbol(token) d.finalize() return d
def dummy_dictionary(vocab_size, prefix='token_'): d = Dictionary() for i in range(vocab_size): token = prefix + str(i) d.add_symbol(token) d.finalize(padding_factor=1) # don't add extra padding symbols return d
def dummy_dictionary(vocab_size, prefix='token_'): d = Dictionary() for i in range(vocab_size): token = prefix + str(i) d.add_symbol(token) d.finalize(padding_factor=1) # don't add extra padding symbols return d
def test_huffman_compresses(self): data = make_data() builder = make_code_builder(data) coder = builder.build_code() with TemporaryDirectory() as dirname: prefix = os.path.join(dirname, "huffman") build_dataset(prefix, data, coder) prefix_mmap = os.path.join(dirname, "mmap") mmap_builder = indexed_dataset.make_builder( indexed_dataset.data_file_path(prefix_mmap), "mmap", vocab_size=len(POPULATION), ) dictionary = Dictionary() for c in POPULATION: dictionary.add_symbol(c) dictionary.finalize() for sentence in data: mmap_builder.add_item(dictionary.encode_line(" ".join(sentence))) mmap_builder.finalize(indexed_dataset.index_file_path(prefix_mmap)) huff_size = os.stat(indexed_dataset.data_file_path(prefix)).st_size mmap_size = os.stat(indexed_dataset.data_file_path(prefix_mmap)).st_size self.assertLess(huff_size, mmap_size)
def setup_task(cls, args, **kwargs): """Setup the task. """ dictionary = Dictionary() for i in range(args.dict_size): dictionary.add_symbol('word{}'.format(i)) print('| dictionary: {} types'.format(len(dictionary))) return cls(args, dictionary)
def build_word_dict(word_embed_path): word_dict = Dictionary() with open(word_embed_path, 'r') as f: for line in f: word = line.split(' ', 1)[0] word_dict.add_symbol(word) word_dict.finalize(padding_factor=1) return word_dict
def pad_dict(d: Dictionary, num_extra_symbols: int, padding_factor: int = 8) -> None: i = 0 while (len(d) + num_extra_symbols) % padding_factor != 0: symbol = f"madeupword{i:04d}" d.add_symbol(symbol, n=0) i += 1
def _t2c_to_tsr(self, t2c: Dict[str, List[str]], dict: Dictionary) -> Dict[int, torch.LongTensor]: res_dict = {} for k in t2c.keys(): res_dict[dict.add_symbol(k)] = torch.LongTensor( [dict.add_symbol(v) for v in t2c[k]]) return res_dict
class DummyMaskedLMTask(FairseqTask): def __init__(self, cfg: DummyMaskedLMConfig): super().__init__(cfg) self.dictionary = Dictionary() for i in range(cfg.dict_size): self.dictionary.add_symbol("word{}".format(i)) logger.info("dictionary: {} types".format(len(self.dictionary))) # add mask token self.mask_idx = self.dictionary.add_symbol("<mask>") self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8 mask_idx = 0 pad_idx = 1 seq = torch.arange(cfg.tokens_per_sample) + pad_idx + 1 mask = torch.arange(2, cfg.tokens_per_sample, 7) # ~15% src = seq.clone() src[mask] = mask_idx tgt = torch.full_like(seq, pad_idx) tgt[mask] = seq[mask] self.dummy_src = src self.dummy_tgt = tgt def load_dataset(self, split, epoch=1, combine=False, **kwargs): """Load a given dataset split. Args: split (str): name of the split (e.g., train, valid, test) """ if self.cfg.batch_size is not None: bsz = self.cfg.batch_size else: bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample) self.datasets[split] = DummyDataset( { "id": 1, "net_input": { "src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]), "src_lengths": torch.full( (bsz, ), self.cfg.tokens_per_sample, dtype=torch.long), }, "target": torch.stack([self.dummy_tgt for _ in range(bsz)]), "nsentences": bsz, "ntokens": bsz * self.cfg.tokens_per_sample, }, num_items=self.cfg.dataset_size, item_size=self.cfg.tokens_per_sample, ) @property def source_dictionary(self): return self.dictionary @property def target_dictionary(self): return self.dictionary
def build_sememe_dict(datapath): sememe_dict = Dictionary() with open(os.path.join(datapath, 'HowNet.edge'), 'r') as f: for line in f: sememes = line.strip().split('\t')[1] for s in sememes.split(): sememe_dict.add_symbol(s) sememe_dict.finalize(threshold=5, padding_factor=1) return sememe_dict
def to_dictionary(self) -> Dictionary: dictionary = Dictionary(bos=self.bos, unk=self.unk, pad=self.pad, eos=self.eos) for n in self: dictionary.add_symbol(n.symbol, n=n.count) dictionary.finalize() return dictionary
def label_schema_as_dictionary(label_schema): label_dict = Dictionary() labels = list(label_schema.labels) assert len(labels) == len(set(labels)) for label in labels: label_dict.add_symbol(label) return label_dict
def setup_task(cls, args, **kwargs): """Setup the task. """ dictionary = Dictionary() for i in range(args.dict_size): dictionary.add_symbol('word{}'.format(i)) logger.info('dictionary: {} types'.format(len(dictionary))) args.max_source_positions = args.src_len + dictionary.pad() + 2 args.max_target_positions = args.tgt_len + dictionary.pad() + 2 return cls(args, dictionary)
def get_bnids_dictionary(cls) -> Dictionary: if cls._bnids_dictionary is None: src_dictionary = cls.get_offsets_dictionary() tgt_dictionary = Dictionary() string_map = cls.get_offset_to_bnids_map() for idx, wn in enumerate(src_dictionary.symbols): if wn.startswith('wn:'): tgt_dictionary.add_symbol(string_map[wn]) tgt_dictionary.finalize() cls._bnids_dictionary = tgt_dictionary return cls._bnids_dictionary
def write_dictionary(model_dir, lst): '''Write out dictionary in fair seq format.''' joined_dict = Dictionary() for toks in lst: for t in toks: joined_dict.add_symbol(t) print('| dictionary: {} types'.format(len(joined_dict))) with open(model_dir + '/dict.txt', 'w') as fd: joined_dict.save(fd)
def _get_test_data_with_word_vocab(self, append_eos=True): """ Args: append_eos: if True, each input sentence in the source tokens tensor will have an EOS appended to the end. Returns: vocabs: word vocab x: input tensor containing numberized source tokens, with EOS at the end if append_eos is true src_lengths: and source lengths. """ vocab = Dictionary() vocab.add_symbol("hello") vocab.add_symbol("how") vocab.add_symbol("are") vocab.add_symbol("you") vocab.add_symbol("new") vocab.add_symbol("york") src_tokens = [ ["hello", "new", "york", "you"], ["how", "are", "you", "new", "york"], ] x, src_lengths = self._convert_src_tokens_to_tensor( vocab=vocab, src_tokens=src_tokens, append_eos=append_eos ) return vocab, x, src_lengths
def load_target_dictionary(self): if self.cfg.labels: dict = Dictionary(bos=self.bos, pad=self.pad, eos=self.eos, unk=self.unk, from_tokenizer=True) dict.bos_index = self.tokenizer.encoder[self.bos] dict.pad_index = self.tokenizer.encoder[self.pad] dict.eos_index = self.tokenizer.encoder[self.eos] dict.unk_index = self.tokenizer.encoder[self.unk] for symbol in self.tokenizer.encoder.keys(): dict.add_symbol(symbol) return dict return None
def augment_dictionary( dictionary: Dictionary, language_list: List[str], lang_tok_style: str, langtoks_specs: Sequence[str] = (LangTokSpec.main.value,), extra_data: Optional[Dict[str, str]] = None, ) -> None: for spec in langtoks_specs: for language in language_list: dictionary.add_symbol( get_lang_tok(lang=language, lang_tok_style=lang_tok_style, spec=spec) ) if lang_tok_style == LangTokStyle.mbart.value or ( extra_data is not None and LangTokSpec.mono_dae.value in extra_data ): dictionary.add_symbol("<mask>")
def ensure_symbols_are_present(dictionary: Dictionary, symbols: List[str], ok_to_increase_dict_size: bool) -> None: """ Ensure that the symbols in the source and target dictionary are present. Makes changes to the dictionaries in-place. """ original_size = len(dictionary) _ = remove_madeupwords_from_dictionary(dictionary) for symbol in symbols: dictionary.add_symbol(symbol) dictionary.pad_to_multiple_(8) if not ok_to_increase_dict_size: # Let's not crash - but rather point out that we are not allowed to increase the dictionary size. if len(dictionary) != original_size: logger.warning( "The dictionary size changed. The model loading will probably fail." )
def load_label_dictionary(cls, args, filename, **kwargs): """Load the dictionary from the filename Args: filename (str): the filename """ label_schema = parse_label_schema(filename) label_dict = Dictionary() labels = list(label_schema.labels) assert labels[0] == "NULL", "Expected label at index 0 to be 'NULL'" assert len(labels) == len(set(labels)) for label in labels: label_dict.add_symbol(label) assert label_dict.symbols[ label_dict. nspecial] == "NULL", "Expected first nonspecial token to be 'NULL'" return label_dict, label_schema
def setup_task(cls, args, **kwargs): data_cfg = data_cfg = S2SDataConfig(Path(args.data) / args.config_yaml) tgt_dict = None infer_tgt_lang_id = None if args.target_is_code: if data_cfg.prepend_tgt_lang_tag_as_bos: # dictionary with language tags dict_path = Path(args.data) / data_cfg.vocab_filename if not dict_path.is_file(): raise FileNotFoundError( f"Dict has to be provided when setting prepend_tgt_lang_tag_as_bos: true, but dict not found: {dict_path}" ) tgt_dict = Dictionary.load(dict_path.as_posix()) # target langauge for inference if args.infer_target_lang != "": tgt_lang_tag = SpeechToTextDataset.LANG_TAG_TEMPLATE.format( args.infer_target_lang) infer_tgt_lang_id = tgt_dict.index(tgt_lang_tag) assert infer_tgt_lang_id != tgt_dict.unk() else: assert args.target_code_size is not None tgt_dict = Dictionary() for i in range(args.target_code_size): tgt_dict.add_symbol(str(i)) logger.info(f"dictionary size: " f"{len(tgt_dict):,}") if getattr(args, "train_subset", None) is not None: if not all( s.startswith("train") for s in args.train_subset.split(",")): raise ValueError('Train splits should be named like "train*".') assert args.n_frames_per_step >= 1 assert (not args.eval_inference or (args.target_is_code and args.vocoder == "code_hifigan") or (not args.target_is_code and args.vocoder != "code_hifigan")) return cls(args, tgt_dict, infer_tgt_lang_id=infer_tgt_lang_id)
def setup_task(cls, args, **kwargs): tgt_dict = None if args.target_is_code: assert args.target_code_size is not None tgt_dict = Dictionary() for i in range(args.target_code_size): tgt_dict.add_symbol(str(i)) logger.info(f"dictionary size: " f"{len(tgt_dict):,}") if getattr(args, "train_subset", None) is not None: if not all( s.startswith("train") for s in args.train_subset.split(",")): raise ValueError('Train splits should be named like "train*".') assert args.n_frames_per_step >= 1 assert (not args.eval_inference or (args.target_is_code and args.vocoder == "code_hifigan") or (not args.target_is_code and args.vocoder != "code_hifigan")) return cls(args, tgt_dict)
def _get_test_data_with_bpe_end_marker(self, append_eos=True): """ Args: append_eos: if True, each input sentence in the source tokens tensor will have an EOS appended to the end. Returns: vocabs: BPE vocab with end-of-word markers as suffixes to denote tokens at the end of a word. This is an alternative to fairseq's standard preprocessing framework and is not generally supported within fairseq. x: input tensor containing numberized source tokens, with EOS at the end if append_eos is true src_lengths: and source lengths. """ vocab = Dictionary() vocab.add_symbol("he") vocab.add_symbol("llo_EOW") vocab.add_symbol("how_EOW") vocab.add_symbol("are_EOW") vocab.add_symbol("y") vocab.add_symbol("ou_EOW") vocab.add_symbol("n") vocab.add_symbol("ew_EOW") vocab.add_symbol("or") vocab.add_symbol("k_EOW") src_tokens = [ ["he", "llo_EOW", "n", "ew_EOW", "y", "or", "k_EOW"], ["how_EOW", "are_EOW", "y", "ou_EOW"], ] x, src_lengths = x, src_lengths = self._convert_src_tokens_to_tensor( vocab=vocab, src_tokens=src_tokens, append_eos=append_eos ) return vocab, x, src_lengths
def _get_test_data_with_bpe_cont_marker(self, append_eos=True): """ Args: append_eos: if True, each input sentence in the source tokens tensor will have an EOS appended to the end. Returns: vocabs: BPE vocab with continuation markers as suffixes to denote non-end of word tokens. This is the standard BPE format used in fairseq's preprocessing. x: input tensor containing numberized source tokens, with EOS at the end if append_eos is true src_lengths: and source lengths. """ vocab = Dictionary() vocab.add_symbol("he@@") vocab.add_symbol("llo") vocab.add_symbol("how") vocab.add_symbol("are") vocab.add_symbol("y@@") vocab.add_symbol("ou") vocab.add_symbol("n@@") vocab.add_symbol("ew") vocab.add_symbol("or@@") vocab.add_symbol("k") src_tokens = [ ["he@@", "llo", "n@@", "ew", "y@@", "or@@", "k"], ["how", "are", "y@@", "ou"], ] x, src_lengths = x, src_lengths = self._convert_src_tokens_to_tensor( vocab=vocab, src_tokens=src_tokens, append_eos=append_eos ) return vocab, x, src_lengths
def __init__(self, root, split='train', samples_filter=None, vocabs_from=None, parse_programs=True, limit=None, programs_mapping=None): questions_path = f'{root}/questions/CLEVR_{split}_questions.json' scenes_path = f'{root}/scenes/CLEVR_{split}_scenes.json' scenes_data = None with open(scenes_path) as data: scenes_data = json.load(data)['scenes'] questions_file = os.path.join(root, questions_path) samples = None with open(questions_file) as data: samples = json.load(data)['questions'] if vocabs_from is not None: programs_vocab = vocabs_from.programs_vocab else: programs_vocab = Dictionary() final_samples = [] for sample in samples: sample['prompt'] = sample['question'] sample['target'] = sample['answer'] img_idx = sample['image_index'] scene = scenes_data[img_idx] sample['image_path'] = os.path.join('images', scene['image_filename']) sample['viz_rep'] = self.scene_to_canonical_rep(scene) sample['scene'] = scene if parse_programs: if programs_mapping is None: prog_str = CLEVR.build_prog_str(sample['program']) sample['program_str'] = prog_str program_tokens = [] for token in CLEVR.tokenize_program(prog_str): program_tokens.append(programs_vocab.add_symbol(token)) sample['program_tokens'] = torch.tensor(program_tokens) else: program_str, program_tokens = programs_mapping.get( sample['question_family_index'], ('NONE', [])) sample['program_str'] = program_str sample['program_tokens'] = torch.tensor(program_tokens) if samples_filter is None or samples_filter(sample): final_samples.append(sample) self.programs_vocab = programs_vocab img_transform = tv.transforms.Compose([ tv.transforms.Pad((0, 150), fill=300, padding_mode='constant'), tv.transforms.Resize(224), tv.transforms.ToTensor(), tv.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) super().__init__(os.path.join(root, split), final_samples, img_transform, vocabs_from=vocabs_from, prompt_mode='natural', target_mode='natural', limit=limit)
def _get_test_data(self): vocab = Dictionary() vocab.add_symbol("he@@") vocab.add_symbol("llo") vocab.add_symbol("how") vocab.add_symbol("are") vocab.add_symbol("y@@") vocab.add_symbol("ou") vocab.add_symbol("n@@") vocab.add_symbol("ew") vocab.add_symbol("or@@") vocab.add_symbol("k") src_tokens = [ ["he@@", "llo", "n@@", "ew", "y@@", "or@@", "k"], ["how", "are", "y@@", "ou"], ] src_len = [len(x) for x in src_tokens] x = torch.LongTensor(len(src_tokens), max(src_len) + 1).fill_(vocab.pad()) for i in range(len(src_tokens)): for j in range(len(src_tokens[i])): x[i][j] = vocab.index(src_tokens[i][j]) x[i][j + 1] = vocab.eos() x = x.transpose(1, 0) return vocab, x, torch.LongTensor([i + 1 for i in src_len])
def _get_test_data(self, append_eos=True, bpe=True): vocab = Dictionary() if bpe: vocab.add_symbol("he@@") vocab.add_symbol("llo") vocab.add_symbol("how") vocab.add_symbol("are") vocab.add_symbol("y@@") vocab.add_symbol("ou") vocab.add_symbol("n@@") vocab.add_symbol("ew") vocab.add_symbol("or@@") vocab.add_symbol("k") src_tokens = [ ["he@@", "llo", "n@@", "ew", "y@@", "or@@", "k"], ["how", "are", "y@@", "ou"], ] else: vocab.add_symbol("hello") vocab.add_symbol("how") vocab.add_symbol("are") vocab.add_symbol("you") vocab.add_symbol("new") vocab.add_symbol("york") src_tokens = [ ["hello", "new", "york", "you"], ["how", "are", "you", "new", "york"], ] src_len = [len(x) for x in src_tokens] # If we have to append EOS, we include EOS in counting src length if append_eos: src_len = [length + 1 for length in src_len] x = torch.LongTensor(len(src_tokens), max(src_len)).fill_(vocab.pad()) for i in range(len(src_tokens)): for j in range(len(src_tokens[i])): x[i][j] = vocab.index(src_tokens[i][j]) if append_eos: x[i][j + 1] = vocab.eos() x = x.transpose(1, 0) return vocab, x, torch.LongTensor(src_len)
def build_dict(symbol_list): d = Dictionary() for symbol in symbol_list: d.add_symbol(symbol) return d