Python BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys示例

示例#1

文件： test_tokenization_auto.py 项目： sshleifer/transformers_fork

    def test_tokenizer_from_pretrained(self):
        for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
            tokenizer = AutoTokenizer.from_pretrained(model_name)
            self.assertIsNotNone(tokenizer)
            self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast))
            self.assertGreater(len(tokenizer), 0)

        for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
            tokenizer = AutoTokenizer.from_pretrained(model_name)
            self.assertIsNotNone(tokenizer)
            self.assertIsInstance(tokenizer, (GPT2Tokenizer, GPT2TokenizerFast))
            self.assertGreater(len(tokenizer), 0)

示例#2

文件： tokenization_auto_test.py 项目： zxlzr/AwesomeMRC

    def test_tokenizer_from_pretrained(self):
        logging.basicConfig(level=logging.INFO)
        for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]:
            tokenizer = AutoTokenizer.from_pretrained(model_name)
            self.assertIsNotNone(tokenizer)
            self.assertIsInstance(tokenizer, BertTokenizer)
            self.assertGreater(len(tokenizer), 0)

        for model_name in list(GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]:
            tokenizer = AutoTokenizer.from_pretrained(model_name)
            self.assertIsNotNone(tokenizer)
            self.assertIsInstance(tokenizer, GPT2Tokenizer)
            self.assertGreater(len(tokenizer), 0)

示例#3

import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from transformers import (OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
                          XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP)
try:
    from transformers import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP
except:
    pass
from transformers import AutoModel
from utils.layers import *
from utils.data_utils import get_gpt_token_num

MODEL_CLASS_TO_NAME = {
    'gpt': list(OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()),
    'bert': list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()),
    'xlnet': list(XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()),
    'roberta': list(ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()),
    'lstm': ['lstm'],
}
try:
    MODEL_CLASS_TO_NAME['albert'] =  list(ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())
except:
    pass

MODEL_NAME_TO_CLASS = {model_name: model_class for model_class, model_name_list in MODEL_CLASS_TO_NAME.items() for model_name in model_name_list}


class LSTMTextEncoder(nn.Module):
    pool_layer_classes = {'mean': MeanPoolLayer, 'max': MaxPoolLayer}

示例#4

文件： eval_utils.py 项目： mikimn/robust-nli

import glob
import logging
from tqdm import tqdm, trange
from transformers import (WEIGHTS_NAME, BertConfig, BertTokenizer, XLMConfig,
                          XLMForSequenceClassification, XLMTokenizer,
                          XLNetConfig, XLNetForSequenceClassification,
                          XLNetTokenizer, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP)
from utils_bert import BertDebiasForSequenceClassification
MODEL_CLASSES = {
    'bert': (BertConfig, BertDebiasForSequenceClassification, BertTokenizer),
    'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
    'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
}

# ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig)), ())
ALL_MODELS = list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())

task_to_data_dir = {
    "snli": "../../data/datasets/SNLI/original",
    "mnli": "../../data/datasets/MNLI",
    "mnli-mm": "../../data/datasets/MNLI",
    "addonerte": "../../data/datasets/AddOneRTE",
    "dpr": "../../data/datasets/DPR/",
    "sprl": "../../data/datasets/SPRL/",
    "fnplus": "../../data/datasets/FNPLUS/",
    "joci": "../../data/datasets/JOCI/",
    "mpe": "../../data/datasets/MPE/",
    "scitail": "../../data/datasets/SciTail/",
    "sick": "../../data/datasets/SICK/",
    "glue": "../../data/datasets/GLUEDiagnostic/",
    "QQP": "../../data/datasets/QQP/",