def test_tokenizer_from_pretrained(self): for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x): tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, (BertTokenizer, BertTokenizerFast)) self.assertGreater(len(tokenizer), 0) for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys(): tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, (GPT2Tokenizer, GPT2TokenizerFast)) self.assertGreater(len(tokenizer), 0)
def test_tokenizer_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, BertTokenizer) self.assertGreater(len(tokenizer), 0) for model_name in list(GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: tokenizer = AutoTokenizer.from_pretrained(model_name) self.assertIsNotNone(tokenizer) self.assertIsInstance(tokenizer, GPT2Tokenizer) self.assertGreater(len(tokenizer), 0)
import torch.nn as nn import torch.nn.functional as F import numpy as np from transformers import (OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP) try: from transformers import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP except: pass from transformers import AutoModel from utils.layers import * from utils.data_utils import get_gpt_token_num MODEL_CLASS_TO_NAME = { 'gpt': list(OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), 'bert': list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), 'xlnet': list(XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), 'roberta': list(ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), 'lstm': ['lstm'], } try: MODEL_CLASS_TO_NAME['albert'] = list(ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()) except: pass MODEL_NAME_TO_CLASS = {model_name: model_class for model_class, model_name_list in MODEL_CLASS_TO_NAME.items() for model_name in model_name_list} class LSTMTextEncoder(nn.Module): pool_layer_classes = {'mean': MeanPoolLayer, 'max': MaxPoolLayer}
import glob import logging from tqdm import tqdm, trange from transformers import (WEIGHTS_NAME, BertConfig, BertTokenizer, XLMConfig, XLMForSequenceClassification, XLMTokenizer, XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP) from utils_bert import BertDebiasForSequenceClassification MODEL_CLASSES = { 'bert': (BertConfig, BertDebiasForSequenceClassification, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), } # ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig)), ()) ALL_MODELS = list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()) task_to_data_dir = { "snli": "../../data/datasets/SNLI/original", "mnli": "../../data/datasets/MNLI", "mnli-mm": "../../data/datasets/MNLI", "addonerte": "../../data/datasets/AddOneRTE", "dpr": "../../data/datasets/DPR/", "sprl": "../../data/datasets/SPRL/", "fnplus": "../../data/datasets/FNPLUS/", "joci": "../../data/datasets/JOCI/", "mpe": "../../data/datasets/MPE/", "scitail": "../../data/datasets/SciTail/", "sick": "../../data/datasets/SICK/", "glue": "../../data/datasets/GLUEDiagnostic/", "QQP": "../../data/datasets/QQP/",