def setUp(self): super().setUp() # We have a SentencePiece fixture for testing tokenizer = PegasusTokenizer(SAMPLE_VOCAB, offset=0, mask_token_sent=None, mask_token="[MASK]") tokenizer.save_pretrained(self.tmpdirname)
def setUp(self): super().setUp() # We have a SentencePiece fixture for testing tokenizer = PegasusTokenizer(SAMPLE_VOCAB) tokenizer.save_pretrained(self.tmpdirname)