def benchmark_experimental_vocab_construction(vocab_file_path, is_raw_text=True, is_legacy=True, num_iters=1): f = open(vocab_file_path, 'r') t0 = time.monotonic() if is_raw_text: if is_legacy: print("Loading from raw text file with legacy python function") for _ in range(num_iters): legacy_vocab_from_file_object(f) print("Construction time:", time.monotonic() - t0) else: print("Loading from raw text file with basic_english_normalize tokenizer") for _ in range(num_iters): tokenizer = basic_english_normalize() jited_tokenizer = torch.jit.script(tokenizer.to_ivalue()) vocab_from_raw_text_file(f, jited_tokenizer, num_cpus=1) print("Construction time:", time.monotonic() - t0) else: for _ in range(num_iters): vocab_from_file(f) print("Construction time:", time.monotonic() - t0)
def test_vocab_from_raw_text_file(self): asset_name = 'vocab_raw_text_test.txt' asset_path = get_asset_path(asset_name) with open(asset_path, 'r') as f: tokenizer = basic_english_normalize() jit_tokenizer = torch.jit.script(tokenizer.to_ivalue()) v = vocab_from_raw_text_file(f, jit_tokenizer, unk_token='<new_unk>') expected_itos = ['<new_unk>', "'", 'after', 'talks', '.', 'are', 'at', 'disappointed', 'fears', 'federal', 'firm', 'for', 'mogul', 'n', 'newall', 'parent', 'pension', 'representing', 'say', 'stricken', 't', 'they', 'turner', 'unions', 'with', 'workers'] expected_stoi = {x: index for index, x in enumerate(expected_itos)} self.assertEqual(v.get_itos(), expected_itos) self.assertEqual(dict(v.get_stoi()), expected_stoi)