def _load_mscoco(invalid_vocab_times=0):
from transformers import BertTokenizer
toker = PretrainedTokenizer(
BertTokenizer('./tests/dataloader/dummy_bertvocab/vocab.txt'))
return MSCOCO("./tests/dataloader/dummy_mscoco#MSCOCO",
tokenizer=toker,
pretrained='bert',
min_rare_vocab_times=invalid_vocab_times)
def _load_mscoco(invalid_vocab_times=0):
from transformers import GPT2Tokenizer
toker = PretrainedTokenizer(
GPT2Tokenizer('./tests/dataloader/dummy_gpt2vocab/vocab.json',
'./tests/dataloader/dummy_gpt2vocab/merges.txt'))
return MSCOCO("./tests/dataloader/dummy_mscoco#MSCOCO",
tokenizer=toker,
pretrained='gpt2',
min_rare_vocab_times=invalid_vocab_times)
def _load_mscoco(invalid_vocab_times=0):
return MSCOCO("./tests/dataloader/dummy_mscoco", invalid_vocab_times=invalid_vocab_times)