def __init__(self, config, output_size=6, extra_args=None): super().__init__(config) self.extra_args = extra_args if "vanilla_electra" in extra_args and extra_args["vanilla_electra"] != False: self.electra = ElectraModel(config) self.discriminator_predictions = ElectraDiscriminatorPredictions(config) self.init_weights() else: disc_hid = config.hidden_size seq_len = extra_args["max_seq_length"] self.electra = ElectraModel(config) self.discriminator_predictions = ElectraDiscriminatorPredictions(config) self.rmd_predictions = ElectraRMD(disc_hid, seq_len, output_size) self.init_weights()
def __init__(self, config, lstm_hidden, open_emb_size, close_emb_size, open_size, close_size, num_layer, bilstm_flag): super().__init__(config) assert open_emb_size == lstm_hidden * 2, "Please set score-embedding-size to twice the lstm-hidden-size" # 분류할 라벨의 개수 self.num_labels = config.num_labels # ELECTRA 모델 self.electra = ElectraModel(config) self.n_hidden = lstm_hidden self.num_layers = num_layer self.bidirectional = 2 if bilstm_flag else 1 self.open_label_lstm_first = nn.LSTM(config.hidden_size, lstm_hidden, bidirectional=True, batch_first=True) self.close_label_lstm_first = nn.LSTM(config.hidden_size, lstm_hidden, bidirectional=True, batch_first=True) self.lstm_output2open = nn.Linear(lstm_hidden * 2, open_size) self.lstm_output2close = nn.Linear(lstm_hidden * 2, close_size) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.init_weights()
def __init__(self, config): super(ElectraCrfForNer, self).__init__(config) self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, config.num_labels) self.crf = CRF(num_tags=config.num_labels, batch_first=True) self.init_weights()
def __init__(self, config, num_labels): super(ElectraForSequenceClassification, self).__init__(config) self.num_labels = num_labels self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, num_labels)
def __init__(self, config): super(ElectraForQuestionAnswering, self).__init__(config) self.num_labels = config.num_labels self.electra = ElectraModel(config) self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) self.init_weights()
def __init__(self, config): super(ElectraSoftmaxForNer, self).__init__(config) self.num_labels = config.num_labels self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, config.num_labels) self.loss_type = config.loss_type self.init_weights()
def __init__(self, config, weight=None): super(ElectraForSequenceClassification, self).__init__(config) self.num_labels = config.num_labels self.bert = ElectraModel(config) self.classifier = ElectraClassificationHead(config) self.init_weights()
def __init__(self, config, weight=None): super().__init__(config) self.num_labels = config.num_labels self.electra = ElectraModel(config) self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) self.init_weights()
def __init__(self, config): super().__init__(config) self.num_labels = config.num_labels self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, config.num_labels) self.init_weights()
def __init__(self, config, pos_weight=None): super(ElectraForMultiLabelSequenceClassification, self).__init__(config) self.num_labels = config.num_labels self.pos_weight = pos_weight self.electra = ElectraModel(config) self.pooler = ElectraPooler(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
def __init__(self, config, weight=None): super().__init__(config) self.num_labels = config.num_labels self.electra = ElectraModel(config) self.pooler = ElectraPooler(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) self.weight = weight
def __init__(self, intent_class_num, entity_class_num): super(KoElectraModel, self).__init__() self.bert = ElectraModel.from_pretrained("monologg/koelectra-small-v2-discriminator") config = self.bert.config self.pad_idx = config.pad_token_id ##For intent part self.dropout = nn.Dropout(config.hidden_dropout_prob) self.intent_cls = nn.Linear(config.hidden_size, intent_class_num) ##For entity part self.entity_cls = nn.Linear(config.hidden_size, entity_class_num)
def __init__(self, config): super().__init__(config) self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.classifier = nn.Linear(config.hidden_size, config.num_labels) self.crf = CRF(num_tags=config.num_labels, ignore_index=0, default_label_pad_index=None) self.init_weights()
def __init__(self): super(KoELECTRASpacingModel, self).__init__() self.bert = ElectraModel.from_pretrained( "monologg/koelectra-small-v2-discriminator") config = self.bert.config self.pad_idx = config.pad_token_id self.dropout = nn.Dropout(config.hidden_dropout_prob) self.token_cls = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
def __init__(self, config: Config, output_encoded_layers: bool, *args, **kwargs) -> None: super().__init__(config, output_encoded_layers=output_encoded_layers) # Load config config_file = os.path.join(config.electra_cpt_dir, "config.json") electra_config = ElectraConfig.from_json_file(config_file) print("Electra model config {}".format(electra_config)) # Instantiate model. model = ElectraModel(electra_config) weights_path = os.path.join(config.electra_cpt_dir, "pytorch_model.bin") # load pre-trained weights if weights_path exists if config.load_weights and PathManager.isfile(weights_path): state_dict = torch.load(weights_path) missing_keys: List[str] = [] unexpected_keys: List[str] = [] error_msgs: List[str] = [] # copy state_dict so _load_from_state_dict can modify it metadata = getattr(state_dict, "_metadata", None) state_dict = state_dict.copy() if metadata is not None: state_dict._metadata = metadata def load(module, prefix=""): local_metadata = ({} if metadata is None else metadata.get( prefix[:-1], {})) module._load_from_state_dict( state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs, ) for name, child in module._modules.items(): if child is not None: load(child, prefix + name + ".") load(model, prefix="" if hasattr(model, "electra") else "electra.") if len(missing_keys) > 0: print( "Weights of {} not initialized from pretrained model: {}". format(model.__class__.__name__, missing_keys)) if len(unexpected_keys) > 0: print( "Weights from pretrained model not used in {}: {}".format( model.__class__.__name__, unexpected_keys)) self.electra = model log_class_usage(__class__)
def __init__(self, config, ): super(ElectraSpanForNer, self).__init__(config) self.soft_label = config.soft_label self.num_labels = config.num_labels self.loss_type = config.loss_type self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.start_fc = PoolerStartLogits(config.hidden_size, self.num_labels) if self.soft_label: self.end_fc = PoolerEndLogits(config.hidden_size + self.num_labels, self.num_labels) else: self.end_fc = PoolerEndLogits(config.hidden_size + 1, self.num_labels) self.init_weights()
def __init__(self, config, lstm_hidden, label_emb_size, score_emb_size, score_size, num_layer, bilstm_flag): super().__init__(config) assert score_emb_size == lstm_hidden * 2, "Please set score-embedding-size to twice the lstm-hidden-size" # 분류할 라벨의 개수 self.num_labels = config.num_labels # ELECTRA 모델 self.electra = ElectraModel(config) self.n_hidden = lstm_hidden self.score_emb = nn.Embedding(score_size, score_emb_size, scale_grad_by_freq=True) self.num_layers = num_layer self.bidirectional = 2 if bilstm_flag else 1 self.lstm_first = nn.LSTM(config.hidden_size, lstm_hidden, bidirectional=True, batch_first=True) self.lstm_last = nn.LSTM(lstm_hidden * 4, lstm_hidden, num_layers=self.num_layers, batch_first=True, bidirectional=bilstm_flag) self.lstm_score_sequence = nn.LSTM(lstm_hidden * 2 + score_size, lstm_hidden, num_layers=self.num_layers, batch_first=True, bidirectional=bilstm_flag) self.label_attn = multihead_attention( lstm_hidden * 2, num_heads=1, dropout_rate=config.hidden_dropout_prob) self.label_attn_last = multihead_attention(lstm_hidden * 2, num_heads=1, dropout_rate=0) self.lstm_hidden2senti = nn.Linear(lstm_hidden * 2, config.num_labels) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.init_weights()
def __init__(self, config): super(ElectraForSequenceRegression, self).__init__(config) self.electra = ElectraModel(config) self.dropout = nn.Dropout(config.hidden_dropout_prob) self.regressor = nn.Linear(config.hidden_size, 1)