def __init__(self, config, output_size=6, extra_args=None):
super().__init__(config)
self.extra_args = extra_args
if "vanilla_electra" in extra_args and extra_args["vanilla_electra"] != False:
self.electra = ElectraModel(config)
self.discriminator_predictions = ElectraDiscriminatorPredictions(config)
self.init_weights()
else:
disc_hid = config.hidden_size
seq_len = extra_args["max_seq_length"]
self.electra = ElectraModel(config)
self.discriminator_predictions = ElectraDiscriminatorPredictions(config)
self.rmd_predictions = ElectraRMD(disc_hid, seq_len, output_size)
self.init_weights()
def __init__(self, config, lstm_hidden, open_emb_size, close_emb_size,
open_size, close_size, num_layer, bilstm_flag):
super().__init__(config)
assert open_emb_size == lstm_hidden * 2, "Please set score-embedding-size to twice the lstm-hidden-size"
# 분류할 라벨의 개수
self.num_labels = config.num_labels
# ELECTRA 모델
self.electra = ElectraModel(config)
self.n_hidden = lstm_hidden
self.num_layers = num_layer
self.bidirectional = 2 if bilstm_flag else 1
self.open_label_lstm_first = nn.LSTM(config.hidden_size,
lstm_hidden,
bidirectional=True,
batch_first=True)
self.close_label_lstm_first = nn.LSTM(config.hidden_size,
lstm_hidden,
bidirectional=True,
batch_first=True)
self.lstm_output2open = nn.Linear(lstm_hidden * 2, open_size)
self.lstm_output2close = nn.Linear(lstm_hidden * 2, close_size)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.init_weights()
def __init__(self, config):
super(ElectraCrfForNer, self).__init__(config)
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.crf = CRF(num_tags=config.num_labels, batch_first=True)
self.init_weights()
def __init__(self, config, num_labels):
super(ElectraForSequenceClassification, self).__init__(config)
self.num_labels = num_labels
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, num_labels)
def __init__(self, config):
super(ElectraForQuestionAnswering, self).__init__(config)
self.num_labels = config.num_labels
self.electra = ElectraModel(config)
self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
def __init__(self, config):
super(ElectraSoftmaxForNer, self).__init__(config)
self.num_labels = config.num_labels
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.loss_type = config.loss_type
self.init_weights()
def __init__(self, config, weight=None):
super(ElectraForSequenceClassification, self).__init__(config)
self.num_labels = config.num_labels
self.bert = ElectraModel(config)
self.classifier = ElectraClassificationHead(config)
self.init_weights()
def __init__(self, config, weight=None):
super().__init__(config)
self.num_labels = config.num_labels
self.electra = ElectraModel(config)
self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
def __init__(self, config, pos_weight=None):
super(ElectraForMultiLabelSequenceClassification, self).__init__(config)
self.num_labels = config.num_labels
self.pos_weight = pos_weight
self.electra = ElectraModel(config)
self.pooler = ElectraPooler(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
def __init__(self, config, weight=None):
super().__init__(config)
self.num_labels = config.num_labels
self.electra = ElectraModel(config)
self.pooler = ElectraPooler(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
self.weight = weight
def __init__(self, intent_class_num, entity_class_num):
super(KoElectraModel, self).__init__()
self.bert = ElectraModel.from_pretrained("monologg/koelectra-small-v2-discriminator")
config = self.bert.config
self.pad_idx = config.pad_token_id
##For intent part
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.intent_cls = nn.Linear(config.hidden_size, intent_class_num)
##For entity part
self.entity_cls = nn.Linear(config.hidden_size, entity_class_num)
def __init__(self, config):
super().__init__(config)
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.crf = CRF(num_tags=config.num_labels,
ignore_index=0,
default_label_pad_index=None)
self.init_weights()
def __init__(self):
super(KoELECTRASpacingModel, self).__init__()
self.bert = ElectraModel.from_pretrained(
"monologg/koelectra-small-v2-discriminator")
config = self.bert.config
self.pad_idx = config.pad_token_id
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.token_cls = nn.Linear(config.hidden_size,
config.vocab_size,
bias=False)
def __init__(self, config: Config, output_encoded_layers: bool, *args,
**kwargs) -> None:
super().__init__(config, output_encoded_layers=output_encoded_layers)
# Load config
config_file = os.path.join(config.electra_cpt_dir, "config.json")
electra_config = ElectraConfig.from_json_file(config_file)
print("Electra model config {}".format(electra_config))
# Instantiate model.
model = ElectraModel(electra_config)
weights_path = os.path.join(config.electra_cpt_dir,
"pytorch_model.bin")
# load pre-trained weights if weights_path exists
if config.load_weights and PathManager.isfile(weights_path):
state_dict = torch.load(weights_path)
missing_keys: List[str] = []
unexpected_keys: List[str] = []
error_msgs: List[str] = []
# copy state_dict so _load_from_state_dict can modify it
metadata = getattr(state_dict, "_metadata", None)
state_dict = state_dict.copy()
if metadata is not None:
state_dict._metadata = metadata
def load(module, prefix=""):
local_metadata = ({} if metadata is None else metadata.get(
prefix[:-1], {}))
module._load_from_state_dict(
state_dict,
prefix,
local_metadata,
True,
missing_keys,
unexpected_keys,
error_msgs,
)
for name, child in module._modules.items():
if child is not None:
load(child, prefix + name + ".")
load(model, prefix="" if hasattr(model, "electra") else "electra.")
if len(missing_keys) > 0:
print(
"Weights of {} not initialized from pretrained model: {}".
format(model.__class__.__name__, missing_keys))
if len(unexpected_keys) > 0:
print(
"Weights from pretrained model not used in {}: {}".format(
model.__class__.__name__, unexpected_keys))
self.electra = model
log_class_usage(__class__)
def __init__(self, config, ):
super(ElectraSpanForNer, self).__init__(config)
self.soft_label = config.soft_label
self.num_labels = config.num_labels
self.loss_type = config.loss_type
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.start_fc = PoolerStartLogits(config.hidden_size, self.num_labels)
if self.soft_label:
self.end_fc = PoolerEndLogits(config.hidden_size + self.num_labels, self.num_labels)
else:
self.end_fc = PoolerEndLogits(config.hidden_size + 1, self.num_labels)
self.init_weights()
def __init__(self, config, lstm_hidden, label_emb_size, score_emb_size,
score_size, num_layer, bilstm_flag):
super().__init__(config)
assert score_emb_size == lstm_hidden * 2, "Please set score-embedding-size to twice the lstm-hidden-size"
# 분류할 라벨의 개수
self.num_labels = config.num_labels
# ELECTRA 모델
self.electra = ElectraModel(config)
self.n_hidden = lstm_hidden
self.score_emb = nn.Embedding(score_size,
score_emb_size,
scale_grad_by_freq=True)
self.num_layers = num_layer
self.bidirectional = 2 if bilstm_flag else 1
self.lstm_first = nn.LSTM(config.hidden_size,
lstm_hidden,
bidirectional=True,
batch_first=True)
self.lstm_last = nn.LSTM(lstm_hidden * 4,
lstm_hidden,
num_layers=self.num_layers,
batch_first=True,
bidirectional=bilstm_flag)
self.lstm_score_sequence = nn.LSTM(lstm_hidden * 2 + score_size,
lstm_hidden,
num_layers=self.num_layers,
batch_first=True,
bidirectional=bilstm_flag)
self.label_attn = multihead_attention(
lstm_hidden * 2,
num_heads=1,
dropout_rate=config.hidden_dropout_prob)
self.label_attn_last = multihead_attention(lstm_hidden * 2,
num_heads=1,
dropout_rate=0)
self.lstm_hidden2senti = nn.Linear(lstm_hidden * 2, config.num_labels)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.init_weights()
def __init__(self, config):
super(ElectraForSequenceRegression, self).__init__(config)
self.electra = ElectraModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.regressor = nn.Linear(config.hidden_size, 1)
