def benchmark_experimental_vocab_construction(vocab_file_path,
is_raw_text=True,
is_legacy=True,
num_iters=1):
f = open(vocab_file_path, 'r')
t0 = time.monotonic()
if is_raw_text:
if is_legacy:
print("Loading from raw text file with legacy python function")
for _ in range(num_iters):
legacy_vocab_from_file_object(f)
print("Construction time:", time.monotonic() - t0)
else:
print(
"Loading from raw text file with basic_english_normalize tokenizer"
)
for _ in range(num_iters):
tokenizer = basic_english_normalize()
jited_tokenizer = torch.jit.script(tokenizer)
build_vocab_from_text_file(f, jited_tokenizer, num_cpus=1)
print("Construction time:", time.monotonic() - t0)
else:
for _ in range(num_iters):
load_vocab_from_file(f)
print("Construction time:", time.monotonic() - t0)
def test_vocab_from_raw_text_file(self):
asset_name = 'vocab_raw_text_test.txt'
asset_path = get_asset_path(asset_name)
tokenizer = basic_english_normalize()
jit_tokenizer = torch.jit.script(tokenizer)
v = build_vocab_from_text_file(asset_path, jit_tokenizer, unk_token='<new_unk>')
expected_itos = ['<new_unk>', "'", 'after', 'talks', '.', 'are', 'at', 'disappointed',
'fears', 'federal', 'firm', 'for', 'mogul', 'n', 'newall', 'parent',
'pension', 'representing', 'say', 'stricken', 't', 'they', 'turner',
'unions', 'with', 'workers']
expected_stoi = {x: index for index, x in enumerate(expected_itos)}
self.assertEqual(v.get_itos(), expected_itos)
self.assertEqual(dict(v.get_stoi()), expected_stoi)