Python PretrainedTransformerTokenizer.from_params示例

示例#1

 def test_from_params_kwargs(self):
     PretrainedTransformerTokenizer.from_params(
         Params({
             "model_name": "bert-base-uncased",
             "tokenizer_kwargs": {
                 "max_len": 10
             }
         }))

示例#2

 def test_from_params_kwargs(self):
     PretrainedTransformerTokenizer.from_params(
         Params({
             "model_name": "bert-base-uncased",
             "tokenizer_kwargs": {
                 "do_lower_case": True
             }
         }))

示例#3

文件： pretrained_transformer_tokenizer_test.py 项目： himkt/allennlp

 def test_to_params(self):
     tokenizer = PretrainedTransformerTokenizer.from_params(
         Params({
             "model_name": "bert-base-uncased",
             "tokenizer_kwargs": {
                 "max_len": 10
             }
         }))
     params = tokenizer.to_params()
     assert isinstance(params, Params)
     assert params.params == {
         "type": "pretrained_transformer",
         "model_name": "bert-base-uncased",
         "add_special_tokens": True,
         "max_length": None,
         "tokenizer_kwargs": {
             "max_len": 10,
             "use_fast": True
         },
     }