def get_encoder(self):
if self.hparams.model_type == 'bert':
encoder = BertModel.from_pretrained('bert-base-uncased')
elif self.hparams.model_type == 'bert-cased':
encoder = BertModel.from_pretrained('bert-base-cased')
elif self.hparams.model_type == 'bert-large':
encoder = BertModel.from_pretrained('bert-large-uncased')
elif self.hparams.model_type == 'distilbert':
encoder = DistilBertModel.from_pretrained(
'distilbert-base-uncased')
elif self.hparams.model_type == 'roberta':
encoder = RobertaModel.from_pretrained('roberta-base')
elif self.hparams.model_type == 'roberta-large':
encoder = RobertaModel.from_pretrained('roberta-large')
elif self.hparams.model_type == 'albert':
encoder = AlbertModel.from_pretrained('albert-base-v2')
elif self.hparams.model_type == 'albert-xxlarge':
encoder = AlbertModel.from_pretrained('albert-xxlarge-v2')
elif self.hparams.model_type == 'electra':
encoder = ElectraModel.from_pretrained(
'google/electra-base-discriminator')
elif self.hparams.model_type == 'electra-large':
encoder = ElectraModel.from_pretrained(
'google/electra-large-discriminator')
else:
raise ValueError
return encoder
def __init__(self,pretrained_model_name,num_labels=2):
super(ElectraWithContextClassifier, self).__init__()
self.num_labels = num_labels
self.electra_text = ElectraModel.from_pretrained(pretrained_model_name)
self.electra_context = ElectraModel.from_pretrained(pretrained_model_name)
self.avg_pool = nn.AvgPool1d(2, 2)
self.dense = nn.Linear(self.electra_text.config.hidden_size*2, self.electra_text.config.hidden_size*2)
self.dropout = nn.Dropout(self.electra_text.config.hidden_dropout_prob)
self.out_proj = nn.Linear(self.electra_text.config.hidden_size*2, self.num_labels)
def __init__(self, pretrained_model_name, num_labels=2):
super(DualElectra, self).__init__()
self.num_labels = num_labels
self.electra_text = ElectraModel.from_pretrained(pretrained_model_name)
self.electra_context = ElectraModel.from_pretrained(
pretrained_model_name)
self.dense = nn.Linear(self.electra_text.config.hidden_size * 2,
self.electra_text.config.hidden_size * 2)
self.dropout = nn.Dropout(self.electra_text.config.hidden_dropout_prob)
self.out_proj = nn.Linear(self.electra_text.config.hidden_size * 2,
self.num_labels)
def __init__(self,
config: ElectraConfig,
num_tasks,
embeddings,
discriminator=None,
embed_layer=None):
super().__init__()
self.embed_layer = nn.Embedding(num_embeddings=config.vocab_size,
embedding_dim=config.embedding_size,
padding_idx=config.vocab_size - 1)
if embed_layer:
self.embed_layer.load_state_dict(torch.load(embed_layer))
else:
self.embed_layer.weight = nn.Parameter(embeddings)
if discriminator:
self.discriminator = ElectraModel.from_pretrained(discriminator,
config=config)
else:
self.discriminator = ElectraModel(config)
self.classificationhead = nn.Sequential(
nn.Linear(config.hidden_size, config.hidden_size),
nn.LayerNorm(config.hidden_size), nn.Linear(config.hidden_size, 2))
self.heads = nn.ModuleList(
[self.classificationhead for i in range(num_tasks)])
self.softmax = nn.Softmax(dim=1)
def __init__(self, config, x_embed):
super().__init__()
self.model = ElectraModel.from_pretrained(config.pretrained_weights)
self.encoder_out_size = self.model.config.d_model # 1024 for t-large
return
def __init__(self, args, device):
self.args = args
self.tokenizer = ElectraTokenizer.from_pretrained(args.model_evaluator)
self.model = ElectraModel.from_pretrained(
args.model_evaluator).eval().to(device)
self.alpha = args.alpha
self.device = device
self.cos = nn.CosineSimilarity(dim=-1, eps=1e-6)
def __init__(self, config):
super(Electra, self).__init__()
self.electra = ElectraModel.from_pretrained(config.model_path)
config_json = 'bert_config.json' if os.path.exists(config.model_path + 'bert_config.json') else 'config.json'
self.electra_config = CONFIGS[config.model].from_pretrained(config.model_path + config_json)
self.electra_config.num_labels = config.num_class
self.fc = ElectraClassificationHead(self.electra_config)
def __init__(self, pretrainfile, num_labels=8):
super().__init__()
self.num_labels = num_labels
self.electra = ElectraModel.from_pretrained(pretrainfile)
self.config = self.electra.config
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
self.qa_outputs = nn.Linear(self.config.hidden_size, self.num_labels)
def __init__(self, pretrainfile):
super(ElectraForTokenClassification, self).__init__()
self.electra = ElectraModel.from_pretrained(pretrainfile)
self.config = self.electra.config
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
self.classifier = nn.Linear(self.config.hidden_size,
self.config.num_labels)
def __init__(self, pretrained_model_name, num_labels=2):
super(ElectraClassifier, self).__init__()
self.num_labels = num_labels
self.electra = ElectraModel.from_pretrained(pretrained_model_name)
self.dense = nn.Linear(self.electra.config.hidden_size,
self.electra.config.hidden_size)
self.dropout = nn.Dropout(self.electra.config.hidden_dropout_prob)
self.out_proj = nn.Linear(self.electra.config.hidden_size,
self.num_labels)
def __init__(self):
super(ElectraQAMulti, self).__init__()
electra_base = "google/electra-base-discriminator"
electra_large = "google/electra-large-discriminator"
self.electra = ElectraModel.from_pretrained(electra_large)
self.classifier = ElectraClassificationHead()
self.dropout = torch.nn.Dropout(self.electra.config.hidden_dropout_prob)
def __init__(self, dropout_rate=0.3, n_outputs=2):
super(ElectraClassifier, self).__init__()
self.pretrained_model = ElectraModel.from_pretrained(
'google/electra-small-discriminator')
self.d1 = torch.nn.Dropout(dropout_rate)
self.l1 = torch.nn.Linear(256, 64)
self.bn1 = torch.nn.LayerNorm(64)
self.d2 = torch.nn.Dropout(dropout_rate)
self.l2 = torch.nn.Linear(64, n_outputs)
def __init__(self):
super(ElectraClassificationHead, self).__init__()
electra_base = "google/electra-base-discriminator"
electra_large = "google/electra-large-discriminator"
self.electra = ElectraModel.from_pretrained(electra_large)
self.dense = torch.nn.Linear(self.electra.config.hidden_size, self.electra.config.hidden_size)
self.dropout = torch.nn.Dropout(self.electra.config.hidden_dropout_prob)
self.out_proj = torch.nn.Linear(self.electra.config.hidden_size, 1)
self.gelu = torch.nn.GELU()
def construct_encoder(self):
model = ElectraModel.from_pretrained('google/' + self.model_name, output_hidden_states=True, output_attentions=True)
model = ElectraExtractor(model, location=None, heads=None)
model.cuda()
model = torch.nn.DataParallel(model)
model.eval()
print('tokenizer', self.model_name)
tokenizer = ElectraTokenizer.from_pretrained('google/' + self.model_name, do_lower_case=True)
print("Model and tokenzier are constructed!")
return model, tokenizer
def main():
start = time.time()
cuda = torch.device('cuda:0')
cpu = torch.device('cpu')
device = cuda if torch.cuda.is_available() else cpu
electra = 'monologg/koelectra-small-v2-discriminator'
# KoELECTRA 모델 로드(v2)
model = ElectraModel.from_pretrained(electra).to(device)
tokenizer = ElectraTokenizer.from_pretrained(electra)
# 문서 데이터 파일 로드
with open(corpus_dir + 'doc_summary4.json', 'r', encoding='utf-8') as f:
j_doc = json.load(f)
train_sets = j_doc
max_available_length = 0
for doc_id in tqdm(train_sets):
sents = j_doc[doc_id]['sents']
encoded_sents = []
max_in_sent = 0
for sent in sents:
a = tokenizer.encode(sent)
if len(a) > max_in_sent:
max_in_sent = len(a)
encoded_sents.append(a)
for i, e in enumerate(encoded_sents):
encoded_sents[i] += [0] * (max_in_sent - len(e))
split_unit = 8
results = []
for i in range(0, len(encoded_sents), split_unit):
j = min(i + split_unit, len(encoded_sents))
x_inputs = torch.LongTensor(encoded_sents[i:j]).to(device)
y = model(x_inputs)
y_np = y[0].cpu().detach()
results.append(y_np)
y_np = torch.cat(results).numpy()
np.save(f'summary_embed2/{doc_id}.embed', y_np)
# CPU로 실행 시간 테스트
end = time.time()
elapsed = end - start
print(elapsed)
print('Ok.')
def test_transformers_embedding_1(self):
from transformers import ElectraModel, ElectraTokenizer
weight_path = "google/electra-small-generator"
model = ElectraModel.from_pretrained(weight_path)
tokenizer = ElectraTokenizer.from_pretrained(weight_path)
encoder = TransformersWordPieceEncoder(model, tokenizer)
ds = DataSet({'words': ["this is a test . [SEP]".split()]})
encoder.index_datasets(ds, field_name='words')
self.assertTrue(ds.has_field('word_pieces'))
result = encoder(torch.LongTensor([[1,2,3,4]]))
self.assertEqual(result.size(), (1, 4, model.config.hidden_size))
def test_inference_no_head_absolute_embedding(self):
model = ElectraModel.from_pretrained("google/electra-small-discriminator")
input_ids = torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
attention_mask = torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])
output = model(input_ids, attention_mask=attention_mask)[0]
expected_shape = torch.Size((1, 11, 256))
self.assertEqual(output.shape, expected_shape)
expected_slice = torch.tensor(
[[[0.4471, 0.6821, -0.3265], [0.4627, 0.5255, -0.3668], [0.4532, 0.3313, -0.4344]]]
)
self.assertTrue(torch.allclose(output[:, 1:4, 1:4], expected_slice, atol=1e-4))
def __init__(self, model_dir, num_labels):
super().__init__()
self.num_labels = num_labels
self.electra = ElectraModel.from_pretrained(model_dir).cuda()
self.linear1 = nn.Linear(self.electra.config.hidden_size,
self.electra.config.hidden_size)
self.activation = nn.Tanh()
self.dropout = nn.Dropout(0.1)
self.linear2 = nn.Linear(self.electra.config.hidden_size, num_labels)
self.loss = nn.CrossEntropyLoss()
def test_transformers_embedding_1(self):
from transformers import ElectraModel, ElectraTokenizer
weight_path = "google/electra-small-generator"
vocab = Vocabulary().add_word_lst("this is a test . [SEP] NotInRoberta".split())
model = ElectraModel.from_pretrained(weight_path)
tokenizer = ElectraTokenizer.from_pretrained(weight_path)
embed = TransformersEmbedding(vocab, model, tokenizer, word_dropout=0.1)
words = torch.LongTensor([[2, 3, 4, 1]])
result = embed(words)
self.assertEqual(result.size(), (1, 4, model.config.hidden_size))
def __init__(self, extractor, config, *args, **kwargs):
super().__init__(*args, **kwargs)
self.extractor = extractor
self.config = config
if config["pretrained"] == "electra-base-msmarco":
self.bert = ElectraModel.from_pretrained(
"Capreolus/electra-base-msmarco")
elif config["pretrained"] == "bert-base-msmarco":
self.bert = BertModel.from_pretrained(
"Capreolus/bert-base-msmarco")
elif config["pretrained"] == "bert-base-uncased":
self.bert = BertModel.from_pretrained("bert-base-uncased")
else:
raise ValueError(
f"unsupported model: {config['pretrained']}; need to ensure correct tokenizers will be used before arbitrary hgf models are supported"
)
self.transformer_layer_1 = BertLayer(self.bert.config)
self.transformer_layer_2 = BertLayer(self.bert.config)
self.num_passages = extractor.config["numpassages"]
self.maxseqlen = extractor.config["maxseqlen"]
self.linear = nn.Linear(self.bert.config.hidden_size, 1)
if config["aggregation"] == "max":
raise NotImplementedError()
elif config["aggregation"] == "avg":
raise NotImplementedError()
elif config["aggregation"] == "attn":
raise NotImplementedError()
elif config["aggregation"] == "transformer":
self.aggregation = self.aggregate_using_transformer
input_embeddings = self.bert.get_input_embeddings()
# TODO hardcoded CLS token id
cls_token_id = torch.tensor([[101]])
self.initial_cls_embedding = input_embeddings(cls_token_id).view(
1, self.bert.config.hidden_size)
self.full_position_embeddings = torch.zeros(
(1, self.num_passages + 1, self.bert.config.hidden_size),
requires_grad=True,
dtype=torch.float)
torch.nn.init.normal_(self.full_position_embeddings,
mean=0.0,
std=0.02)
self.initial_cls_embedding = nn.Parameter(
self.initial_cls_embedding, requires_grad=True)
self.full_position_embeddings = nn.Parameter(
self.full_position_embeddings, requires_grad=True)
else:
raise ValueError(
f"unknown aggregation type: {self.config['aggregation']}")
def __init__(self,
config,
model_name,
only_embedding=True,
output_hidden_states=True):
super(ElectraTokenEmbedder, self).__init__(config)
self.config = config
self.only_embedding = only_embedding
self.model = ElectraModel.from_pretrained(
model_name, output_hidden_states=output_hidden_states)
if self.only_embedding:
self.model = self.model.get_input_embeddings()
self.model.weight.requires_grad = False
def __init__(self, model_name, config, num_speakers=2):
super().__init__(config)
self.num_speakers = num_speakers
self.electra = ElectraModel.from_pretrained(model_name)
self.embeddings = SpeakerAwareElectraEmbeddings(config, self.num_speakers)
self.num_labels = config.num_labels
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.out_proj = nn.Linear(config.hidden_size, self.num_labels)
self.gelu = nn.GELU()
self.init_weights()
def __init__(self, output_size, dropout_rate=0.1, device='cpu'):
super().__init__()
self.device = device
self.dropout = nn.Dropout(p=dropout_rate)
self.electra = ElectraModel.from_pretrained(
'google/electra-small-discriminator').to(device)
self.cls_query = nn.Parameter(
torch.randn(1, self.electra.config.hidden_size, device=device)
) # a learnable vector acting as query for output att
self.cls_att = AttentionModule(d_model=self.electra.config.hidden_size,
d_k=self.electra.config.hidden_size,
device=device,
dropout=self.dropout)
self.output = nn.Linear(self.electra.config.hidden_size,
output_size).to(device)
def _get_bert(model_type, model_path_dict):
if model_type == 'bert':
config = BertConfig.from_pretrained(model_path_dict['config'])
config.output_hidden_states = True
bert = BertModel.from_pretrained(model_path_dict['model'],
config=config)
elif model_type == 'electra':
config = ElectraConfig.from_pretrained(model_path_dict['config'])
config.output_hidden_states = True
bert = ElectraModel.from_pretrained(model_path_dict['model'],
config=config)
elif model_type == 'roberta':
config = RobertaConfig.from_pretrained(model_path_dict['config'])
config.output_hidden_states = True
bert = RobertaModel.from_pretrained(model_path_dict['model'],
config=config)
return bert, config
def __init__(self, config: TrainConfig, logger: logging.Logger):
super().__init__()
self.config = config
self.electra: ElectraModel = ElectraModel.from_pretrained(
config.pretrained_model_name)
self.dense = nn.Linear(self.electra.config.hidden_size,
self.electra.config.hidden_size)
self.dropout = nn.Dropout(self.electra.config.hidden_dropout_prob)
self.bias_classifier = nn.Linear(self.electra.config.hidden_size, 3)
self.hate_classifier = nn.Linear(self.electra.config.hidden_size, 3)
self.criterion = nn.CrossEntropyLoss()
self.learning_rate = config.learning_rate
self.stream_logger = logger
def __init__(
self,
intent_class_num,
entity_class_num,
default_model_path='monologg/koelectra-small-v2-discriminator',
pad_token_id=0):
super(KoELECTRAFineTuner, self).__init__()
self.intent_class_num = intent_class_num
self.entity_class_num = entity_class_num
self.backbone = ElectraModel.from_pretrained(default_model_path)
self.feature_dim = self.backbone.config.hidden_size
self.intent_embedding = nn.Linear(self.feature_dim,
self.intent_class_num)
self.entity_embedding = nn.Linear(self.feature_dim,
self.entity_class_num)
self.entity_featurizer = CRF(self.entity_class_num, batch_first=True)
self.pad_token_id = pad_token_id
def __init__(self, hidden_dim, pretrained_model):
super(UtterancePretrainedModel, self).__init__()
self._pretrained_model = pretrained_model
if pretrained_model == "bert":
self._encoder = BertModel.from_pretrained("bert-base-uncased")
elif pretrained_model == "roberta":
self._encoder = RobertaModel.from_pretrained("roberta-base")
elif pretrained_model == "xlnet":
self._encoder = XLNetModel.from_pretrained("xlnet-base-cased")
elif pretrained_model == "albert":
self._encoder = AlbertModel.from_pretrained("albert-base-v2")
elif pretrained_model == "electra":
self._encoder = ElectraModel.from_pretrained("google/electra-base-discriminator")
else:
assert False, "Something wrong with the parameter --pretrained_model"
self._linear = nn.Linear(UtterancePretrainedModel.HIDDEN_DIM, hidden_dim)
def __init__(self, configs):
super(ElectraClassification, self).__init__()
self.configs = configs
self.bert_hiddensize = self.configs["bert_hiddensize"]
self.dense = self.configs["dense"]
self.label_nums = self.configs["label_nums"]
self.dropout = self.configs["dropout"]
self.electra_model = ElectraModel.from_pretrained(self.configs["path"]["electra_path"])
# for p in self.bert_model.parameters():
# p.requires_grad = True
# output shape of bert: (batch_size, seqlens, lstm_hiddensize)
self.classification = torch.nn.Sequential(
torch.nn.Linear(self.bert_hiddensize, self.dense),
torch.nn.ReLU(),
torch.nn.Dropout(self.dropout),
torch.nn.Linear(self.dense, self.label_nums)
)
def __init__(
self,
backbone: None,
vocab_size: int,
seq_len: int,
intent_class_num: int,
entity_class_num: int,
d_model=512,
nhead=8,
num_encoder_layers=6,
dim_feedforward=2048,
dropout=0.1,
activation="relu",
pad_token_id: int = 0,
):
super(EmbeddingTransformer, self).__init__()
self.backbone = backbone
self.seq_len = seq_len
self.pad_token_id = pad_token_id
if backbone is None:
self.encoder = nn.TransformerEncoder(
TransformerEncoderLayer(d_model, nhead, dim_feedforward,
dropout, activation),
num_encoder_layers,
LayerNorm(d_model),
)
else: # pre-defined model architecture use
if backbone == "kobert":
self.encoder = get_kobert_model()
elif backbone == "distill_kobert":
self.encoder = get_distilkobert_model()
elif backbone == "koelectra":
self.encoder = ElectraModel.from_pretrained(
"monologg/koelectra-small-v2-discriminator")
d_model = self.encoder.config.hidden_size
self.embedding = nn.Embedding(vocab_size, d_model)
self.position_embedding = nn.Embedding(self.seq_len, d_model)
self.intent_feature = nn.Linear(d_model, intent_class_num)
self.entity_feature = nn.Linear(d_model, entity_class_num)
def __get_model_and_tokenizer(self):
model, tokenizer = None, None
if self.transformer_model == TransformerType.BERT:
tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
model = BertModel.from_pretrained('bert-base-cased')
if self.transformer_model == TransformerType.XLNet:
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')
model = XLNetModel.from_pretrained('xlnet-base-cased')
if self.transformer_model == TransformerType.RoBERTa:
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
model = RobertaModel.from_pretrained('roberta-base')
if self.transformer_model == TransformerType.ELECTRA:
tokenizer = ElectraTokenizer.from_pretrained('google/electra-small-discriminator')
model = ElectraModel.from_pretrained('google/electra-small-discriminator')
return model, tokenizer