def get_encoder(self): if self.hparams.model_type == 'bert': encoder = BertModel.from_pretrained('bert-base-uncased') elif self.hparams.model_type == 'bert-cased': encoder = BertModel.from_pretrained('bert-base-cased') elif self.hparams.model_type == 'bert-large': encoder = BertModel.from_pretrained('bert-large-uncased') elif self.hparams.model_type == 'distilbert': encoder = DistilBertModel.from_pretrained( 'distilbert-base-uncased') elif self.hparams.model_type == 'roberta': encoder = RobertaModel.from_pretrained('roberta-base') elif self.hparams.model_type == 'roberta-large': encoder = RobertaModel.from_pretrained('roberta-large') elif self.hparams.model_type == 'albert': encoder = AlbertModel.from_pretrained('albert-base-v2') elif self.hparams.model_type == 'albert-xxlarge': encoder = AlbertModel.from_pretrained('albert-xxlarge-v2') elif self.hparams.model_type == 'electra': encoder = ElectraModel.from_pretrained( 'google/electra-base-discriminator') elif self.hparams.model_type == 'electra-large': encoder = ElectraModel.from_pretrained( 'google/electra-large-discriminator') else: raise ValueError return encoder
def __init__(self,pretrained_model_name,num_labels=2): super(ElectraWithContextClassifier, self).__init__() self.num_labels = num_labels self.electra_text = ElectraModel.from_pretrained(pretrained_model_name) self.electra_context = ElectraModel.from_pretrained(pretrained_model_name) self.avg_pool = nn.AvgPool1d(2, 2) self.dense = nn.Linear(self.electra_text.config.hidden_size*2, self.electra_text.config.hidden_size*2) self.dropout = nn.Dropout(self.electra_text.config.hidden_dropout_prob) self.out_proj = nn.Linear(self.electra_text.config.hidden_size*2, self.num_labels)
def __init__(self, pretrained_model_name, num_labels=2): super(DualElectra, self).__init__() self.num_labels = num_labels self.electra_text = ElectraModel.from_pretrained(pretrained_model_name) self.electra_context = ElectraModel.from_pretrained( pretrained_model_name) self.dense = nn.Linear(self.electra_text.config.hidden_size * 2, self.electra_text.config.hidden_size * 2) self.dropout = nn.Dropout(self.electra_text.config.hidden_dropout_prob) self.out_proj = nn.Linear(self.electra_text.config.hidden_size * 2, self.num_labels)
def __init__(self, config: ElectraConfig, num_tasks, embeddings, discriminator=None, embed_layer=None): super().__init__() self.embed_layer = nn.Embedding(num_embeddings=config.vocab_size, embedding_dim=config.embedding_size, padding_idx=config.vocab_size - 1) if embed_layer: self.embed_layer.load_state_dict(torch.load(embed_layer)) else: self.embed_layer.weight = nn.Parameter(embeddings) if discriminator: self.discriminator = ElectraModel.from_pretrained(discriminator, config=config) else: self.discriminator = ElectraModel(config) self.classificationhead = nn.Sequential( nn.Linear(config.hidden_size, config.hidden_size), nn.LayerNorm(config.hidden_size), nn.Linear(config.hidden_size, 2)) self.heads = nn.ModuleList( [self.classificationhead for i in range(num_tasks)]) self.softmax = nn.Softmax(dim=1)
def __init__(self, config, x_embed): super().__init__() self.model = ElectraModel.from_pretrained(config.pretrained_weights) self.encoder_out_size = self.model.config.d_model # 1024 for t-large return
def __init__(self, args, device): self.args = args self.tokenizer = ElectraTokenizer.from_pretrained(args.model_evaluator) self.model = ElectraModel.from_pretrained( args.model_evaluator).eval().to(device) self.alpha = args.alpha self.device = device self.cos = nn.CosineSimilarity(dim=-1, eps=1e-6)
def __init__(self, config): super(Electra, self).__init__() self.electra = ElectraModel.from_pretrained(config.model_path) config_json = 'bert_config.json' if os.path.exists(config.model_path + 'bert_config.json') else 'config.json' self.electra_config = CONFIGS[config.model].from_pretrained(config.model_path + config_json) self.electra_config.num_labels = config.num_class self.fc = ElectraClassificationHead(self.electra_config)
def __init__(self, pretrainfile, num_labels=8): super().__init__() self.num_labels = num_labels self.electra = ElectraModel.from_pretrained(pretrainfile) self.config = self.electra.config self.dropout = nn.Dropout(self.config.hidden_dropout_prob) self.qa_outputs = nn.Linear(self.config.hidden_size, self.num_labels)
def __init__(self, pretrainfile): super(ElectraForTokenClassification, self).__init__() self.electra = ElectraModel.from_pretrained(pretrainfile) self.config = self.electra.config self.dropout = nn.Dropout(self.config.hidden_dropout_prob) self.classifier = nn.Linear(self.config.hidden_size, self.config.num_labels)
def __init__(self, pretrained_model_name, num_labels=2): super(ElectraClassifier, self).__init__() self.num_labels = num_labels self.electra = ElectraModel.from_pretrained(pretrained_model_name) self.dense = nn.Linear(self.electra.config.hidden_size, self.electra.config.hidden_size) self.dropout = nn.Dropout(self.electra.config.hidden_dropout_prob) self.out_proj = nn.Linear(self.electra.config.hidden_size, self.num_labels)
def __init__(self): super(ElectraQAMulti, self).__init__() electra_base = "google/electra-base-discriminator" electra_large = "google/electra-large-discriminator" self.electra = ElectraModel.from_pretrained(electra_large) self.classifier = ElectraClassificationHead() self.dropout = torch.nn.Dropout(self.electra.config.hidden_dropout_prob)
def __init__(self, dropout_rate=0.3, n_outputs=2): super(ElectraClassifier, self).__init__() self.pretrained_model = ElectraModel.from_pretrained( 'google/electra-small-discriminator') self.d1 = torch.nn.Dropout(dropout_rate) self.l1 = torch.nn.Linear(256, 64) self.bn1 = torch.nn.LayerNorm(64) self.d2 = torch.nn.Dropout(dropout_rate) self.l2 = torch.nn.Linear(64, n_outputs)
def __init__(self): super(ElectraClassificationHead, self).__init__() electra_base = "google/electra-base-discriminator" electra_large = "google/electra-large-discriminator" self.electra = ElectraModel.from_pretrained(electra_large) self.dense = torch.nn.Linear(self.electra.config.hidden_size, self.electra.config.hidden_size) self.dropout = torch.nn.Dropout(self.electra.config.hidden_dropout_prob) self.out_proj = torch.nn.Linear(self.electra.config.hidden_size, 1) self.gelu = torch.nn.GELU()
def construct_encoder(self): model = ElectraModel.from_pretrained('google/' + self.model_name, output_hidden_states=True, output_attentions=True) model = ElectraExtractor(model, location=None, heads=None) model.cuda() model = torch.nn.DataParallel(model) model.eval() print('tokenizer', self.model_name) tokenizer = ElectraTokenizer.from_pretrained('google/' + self.model_name, do_lower_case=True) print("Model and tokenzier are constructed!") return model, tokenizer
def main(): start = time.time() cuda = torch.device('cuda:0') cpu = torch.device('cpu') device = cuda if torch.cuda.is_available() else cpu electra = 'monologg/koelectra-small-v2-discriminator' # KoELECTRA 모델 로드(v2) model = ElectraModel.from_pretrained(electra).to(device) tokenizer = ElectraTokenizer.from_pretrained(electra) # 문서 데이터 파일 로드 with open(corpus_dir + 'doc_summary4.json', 'r', encoding='utf-8') as f: j_doc = json.load(f) train_sets = j_doc max_available_length = 0 for doc_id in tqdm(train_sets): sents = j_doc[doc_id]['sents'] encoded_sents = [] max_in_sent = 0 for sent in sents: a = tokenizer.encode(sent) if len(a) > max_in_sent: max_in_sent = len(a) encoded_sents.append(a) for i, e in enumerate(encoded_sents): encoded_sents[i] += [0] * (max_in_sent - len(e)) split_unit = 8 results = [] for i in range(0, len(encoded_sents), split_unit): j = min(i + split_unit, len(encoded_sents)) x_inputs = torch.LongTensor(encoded_sents[i:j]).to(device) y = model(x_inputs) y_np = y[0].cpu().detach() results.append(y_np) y_np = torch.cat(results).numpy() np.save(f'summary_embed2/{doc_id}.embed', y_np) # CPU로 실행 시간 테스트 end = time.time() elapsed = end - start print(elapsed) print('Ok.')
def test_transformers_embedding_1(self): from transformers import ElectraModel, ElectraTokenizer weight_path = "google/electra-small-generator" model = ElectraModel.from_pretrained(weight_path) tokenizer = ElectraTokenizer.from_pretrained(weight_path) encoder = TransformersWordPieceEncoder(model, tokenizer) ds = DataSet({'words': ["this is a test . [SEP]".split()]}) encoder.index_datasets(ds, field_name='words') self.assertTrue(ds.has_field('word_pieces')) result = encoder(torch.LongTensor([[1,2,3,4]])) self.assertEqual(result.size(), (1, 4, model.config.hidden_size))
def test_inference_no_head_absolute_embedding(self): model = ElectraModel.from_pretrained("google/electra-small-discriminator") input_ids = torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]) attention_mask = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]) output = model(input_ids, attention_mask=attention_mask)[0] expected_shape = torch.Size((1, 11, 256)) self.assertEqual(output.shape, expected_shape) expected_slice = torch.tensor( [[[0.4471, 0.6821, -0.3265], [0.4627, 0.5255, -0.3668], [0.4532, 0.3313, -0.4344]]] ) self.assertTrue(torch.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4))
def __init__(self, model_dir, num_labels): super().__init__() self.num_labels = num_labels self.electra = ElectraModel.from_pretrained(model_dir).cuda() self.linear1 = nn.Linear(self.electra.config.hidden_size, self.electra.config.hidden_size) self.activation = nn.Tanh() self.dropout = nn.Dropout(0.1) self.linear2 = nn.Linear(self.electra.config.hidden_size, num_labels) self.loss = nn.CrossEntropyLoss()
def test_transformers_embedding_1(self): from transformers import ElectraModel, ElectraTokenizer weight_path = "google/electra-small-generator" vocab = Vocabulary().add_word_lst("this is a test . [SEP] NotInRoberta".split()) model = ElectraModel.from_pretrained(weight_path) tokenizer = ElectraTokenizer.from_pretrained(weight_path) embed = TransformersEmbedding(vocab, model, tokenizer, word_dropout=0.1) words = torch.LongTensor([[2, 3, 4, 1]]) result = embed(words) self.assertEqual(result.size(), (1, 4, model.config.hidden_size))
def __init__(self, extractor, config, *args, **kwargs): super().__init__(*args, **kwargs) self.extractor = extractor self.config = config if config["pretrained"] == "electra-base-msmarco": self.bert = ElectraModel.from_pretrained( "Capreolus/electra-base-msmarco") elif config["pretrained"] == "bert-base-msmarco": self.bert = BertModel.from_pretrained( "Capreolus/bert-base-msmarco") elif config["pretrained"] == "bert-base-uncased": self.bert = BertModel.from_pretrained("bert-base-uncased") else: raise ValueError( f"unsupported model: {config['pretrained']}; need to ensure correct tokenizers will be used before arbitrary hgf models are supported" ) self.transformer_layer_1 = BertLayer(self.bert.config) self.transformer_layer_2 = BertLayer(self.bert.config) self.num_passages = extractor.config["numpassages"] self.maxseqlen = extractor.config["maxseqlen"] self.linear = nn.Linear(self.bert.config.hidden_size, 1) if config["aggregation"] == "max": raise NotImplementedError() elif config["aggregation"] == "avg": raise NotImplementedError() elif config["aggregation"] == "attn": raise NotImplementedError() elif config["aggregation"] == "transformer": self.aggregation = self.aggregate_using_transformer input_embeddings = self.bert.get_input_embeddings() # TODO hardcoded CLS token id cls_token_id = torch.tensor([[101]]) self.initial_cls_embedding = input_embeddings(cls_token_id).view( 1, self.bert.config.hidden_size) self.full_position_embeddings = torch.zeros( (1, self.num_passages + 1, self.bert.config.hidden_size), requires_grad=True, dtype=torch.float) torch.nn.init.normal_(self.full_position_embeddings, mean=0.0, std=0.02) self.initial_cls_embedding = nn.Parameter( self.initial_cls_embedding, requires_grad=True) self.full_position_embeddings = nn.Parameter( self.full_position_embeddings, requires_grad=True) else: raise ValueError( f"unknown aggregation type: {self.config['aggregation']}")
def __init__(self, config, model_name, only_embedding=True, output_hidden_states=True): super(ElectraTokenEmbedder, self).__init__(config) self.config = config self.only_embedding = only_embedding self.model = ElectraModel.from_pretrained( model_name, output_hidden_states=output_hidden_states) if self.only_embedding: self.model = self.model.get_input_embeddings() self.model.weight.requires_grad = False
def __init__(self, model_name, config, num_speakers=2): super().__init__(config) self.num_speakers = num_speakers self.electra = ElectraModel.from_pretrained(model_name) self.embeddings = SpeakerAwareElectraEmbeddings(config, self.num_speakers) self.num_labels = config.num_labels self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.out_proj = nn.Linear(config.hidden_size, self.num_labels) self.gelu = nn.GELU() self.init_weights()
def __init__(self, output_size, dropout_rate=0.1, device='cpu'): super().__init__() self.device = device self.dropout = nn.Dropout(p=dropout_rate) self.electra = ElectraModel.from_pretrained( 'google/electra-small-discriminator').to(device) self.cls_query = nn.Parameter( torch.randn(1, self.electra.config.hidden_size, device=device) ) # a learnable vector acting as query for output att self.cls_att = AttentionModule(d_model=self.electra.config.hidden_size, d_k=self.electra.config.hidden_size, device=device, dropout=self.dropout) self.output = nn.Linear(self.electra.config.hidden_size, output_size).to(device)
def _get_bert(model_type, model_path_dict): if model_type == 'bert': config = BertConfig.from_pretrained(model_path_dict['config']) config.output_hidden_states = True bert = BertModel.from_pretrained(model_path_dict['model'], config=config) elif model_type == 'electra': config = ElectraConfig.from_pretrained(model_path_dict['config']) config.output_hidden_states = True bert = ElectraModel.from_pretrained(model_path_dict['model'], config=config) elif model_type == 'roberta': config = RobertaConfig.from_pretrained(model_path_dict['config']) config.output_hidden_states = True bert = RobertaModel.from_pretrained(model_path_dict['model'], config=config) return bert, config
def __init__(self, config: TrainConfig, logger: logging.Logger): super().__init__() self.config = config self.electra: ElectraModel = ElectraModel.from_pretrained( config.pretrained_model_name) self.dense = nn.Linear(self.electra.config.hidden_size, self.electra.config.hidden_size) self.dropout = nn.Dropout(self.electra.config.hidden_dropout_prob) self.bias_classifier = nn.Linear(self.electra.config.hidden_size, 3) self.hate_classifier = nn.Linear(self.electra.config.hidden_size, 3) self.criterion = nn.CrossEntropyLoss() self.learning_rate = config.learning_rate self.stream_logger = logger
def __init__( self, intent_class_num, entity_class_num, default_model_path='monologg/koelectra-small-v2-discriminator', pad_token_id=0): super(KoELECTRAFineTuner, self).__init__() self.intent_class_num = intent_class_num self.entity_class_num = entity_class_num self.backbone = ElectraModel.from_pretrained(default_model_path) self.feature_dim = self.backbone.config.hidden_size self.intent_embedding = nn.Linear(self.feature_dim, self.intent_class_num) self.entity_embedding = nn.Linear(self.feature_dim, self.entity_class_num) self.entity_featurizer = CRF(self.entity_class_num, batch_first=True) self.pad_token_id = pad_token_id
def __init__(self, hidden_dim, pretrained_model): super(UtterancePretrainedModel, self).__init__() self._pretrained_model = pretrained_model if pretrained_model == "bert": self._encoder = BertModel.from_pretrained("bert-base-uncased") elif pretrained_model == "roberta": self._encoder = RobertaModel.from_pretrained("roberta-base") elif pretrained_model == "xlnet": self._encoder = XLNetModel.from_pretrained("xlnet-base-cased") elif pretrained_model == "albert": self._encoder = AlbertModel.from_pretrained("albert-base-v2") elif pretrained_model == "electra": self._encoder = ElectraModel.from_pretrained("google/electra-base-discriminator") else: assert False, "Something wrong with the parameter --pretrained_model" self._linear = nn.Linear(UtterancePretrainedModel.HIDDEN_DIM, hidden_dim)
def __init__(self, configs): super(ElectraClassification, self).__init__() self.configs = configs self.bert_hiddensize = self.configs["bert_hiddensize"] self.dense = self.configs["dense"] self.label_nums = self.configs["label_nums"] self.dropout = self.configs["dropout"] self.electra_model = ElectraModel.from_pretrained(self.configs["path"]["electra_path"]) # for p in self.bert_model.parameters(): # p.requires_grad = True # output shape of bert: (batch_size, seqlens, lstm_hiddensize) self.classification = torch.nn.Sequential( torch.nn.Linear(self.bert_hiddensize, self.dense), torch.nn.ReLU(), torch.nn.Dropout(self.dropout), torch.nn.Linear(self.dense, self.label_nums) )
def __init__( self, backbone: None, vocab_size: int, seq_len: int, intent_class_num: int, entity_class_num: int, d_model=512, nhead=8, num_encoder_layers=6, dim_feedforward=2048, dropout=0.1, activation="relu", pad_token_id: int = 0, ): super(EmbeddingTransformer, self).__init__() self.backbone = backbone self.seq_len = seq_len self.pad_token_id = pad_token_id if backbone is None: self.encoder = nn.TransformerEncoder( TransformerEncoderLayer(d_model, nhead, dim_feedforward, dropout, activation), num_encoder_layers, LayerNorm(d_model), ) else: # pre-defined model architecture use if backbone == "kobert": self.encoder = get_kobert_model() elif backbone == "distill_kobert": self.encoder = get_distilkobert_model() elif backbone == "koelectra": self.encoder = ElectraModel.from_pretrained( "monologg/koelectra-small-v2-discriminator") d_model = self.encoder.config.hidden_size self.embedding = nn.Embedding(vocab_size, d_model) self.position_embedding = nn.Embedding(self.seq_len, d_model) self.intent_feature = nn.Linear(d_model, intent_class_num) self.entity_feature = nn.Linear(d_model, entity_class_num)
def __get_model_and_tokenizer(self): model, tokenizer = None, None if self.transformer_model == TransformerType.BERT: tokenizer = BertTokenizer.from_pretrained('bert-base-cased') model = BertModel.from_pretrained('bert-base-cased') if self.transformer_model == TransformerType.XLNet: tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') model = XLNetModel.from_pretrained('xlnet-base-cased') if self.transformer_model == TransformerType.RoBERTa: tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaModel.from_pretrained('roberta-base') if self.transformer_model == TransformerType.ELECTRA: tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator') model = ElectraModel.from_pretrained('google/electra-small-discriminator') return model, tokenizer