def __init__(self, embedding_matrix, opt): super(LCF_GLOVE, self).__init__() self.config = BertConfig.from_json_file("config.json") self.opt = opt self.embed = nn.Embedding.from_pretrained( torch.tensor(embedding_matrix, dtype=torch.float)) self.mha_global = SelfAttention(self.config, opt) self.mha_local = SelfAttention(self.config, opt) self.ffn_global = PositionwiseFeedForward(self.opt.hidden_dim, dropout=self.opt.dropout) self.ffn_local = PositionwiseFeedForward(self.opt.hidden_dim, dropout=self.opt.dropout) self.mha_local_SA = SelfAttention(self.config, opt) self.mha_global_SA = SelfAttention(self.config, opt) self.mha_SA_single = SelfAttention(self.config, opt) self.bert_pooler = BertPooler(self.config) self.bert_pooler1 = BertPooler(self.config) self.bert_pooler2 = BertPooler(self.config) self.dense1 = nn.Linear(opt.hidden_dim, opt.polarities_dim) self.dense2 = nn.Linear(opt.hidden_dim, opt.polarities_dim) self.sentiment_pool = nn.Linear(6, 3) self.dropout = nn.Dropout(opt.dropout) self.mean_pooling_double = nn.Linear(opt.embed_dim * 2, opt.hidden_dim) self.mean_pooling_single = nn.Linear(opt.embed_dim, opt.hidden_dim) self.dense = nn.Linear(opt.hidden_dim, opt.polarities_dim)
def __init__(self, bert, opt): super(BERT_BASE, self).__init__() self.bert = bert self.opt = opt self.dropout = nn.Dropout(opt.dropout) self.pooler = BertPooler(bert.config) self.dense = nn.Linear(opt.embed_dim, opt.polarities_dim)
def __init__(self, config): super(PatientLevelBert, self).__init__() self.config = config self.embeddings = PatientLevelEmbedding(config) self.encoder = BertEncoder(config) self.pooler = BertPooler(config) self.apply(self.init_weights)
def __init__(self, model, opt): super(LCFS_BERT, self).__init__() if 'bert' in opt.pretrained_bert_name: hidden = model.config.hidden_size elif 'xlnet' in opt.pretrained_bert_name: hidden = model.config.d_model self.hidden = hidden sa_config = BertConfig(hidden_size=self.hidden, output_attentions=True) self.bert_spc = model self.bert_g_sa = SelfAttention(sa_config, opt) self.bert_g_pct = PointwiseFeedForward(self.hidden) self.opt = opt self.bert_local = copy.deepcopy(model) self.bert_local_sa = SelfAttention(sa_config, opt) self.bert_local_pct = PointwiseFeedForward(self.hidden) self.dropout = nn.Dropout(opt.dropout) self.bert_sa = SelfAttention(sa_config, opt) # self.mean_pooling_double = nn.Linear(hidden * 2, hidden) self.mean_pooling_double = PointwiseFeedForward( hidden * 2, hidden, hidden) self.bert_pooler = BertPooler(sa_config) self.dense = nn.Linear(hidden, opt.polarities_dim)
def __init__(self, config, args): super(BERT_MAG_model, self).__init__(config) self.newly_added_config = args if args.output_mode == 'regression': self.num_labels = 1 #BertEncoder self.output_attentions = self.config.output_attentions self.output_hidden_states = self.config.output_hidden_states self.layer = nn.ModuleList([ BertLayer(self.config) for _ in range(self.config.num_hidden_layers) ]) self.MAG = MAG(self.config, args) self.MAG_all = nn.ModuleList([ MAG(self.config, args) for _ in range(self.config.num_hidden_layers) ]) # MultimodalBertModel self.embeddings = BertEmbeddings(self.config) self.pooler = BertPooler(self.config) # MultimodalBertForSequenceClassification self.classifier = nn.Linear(self.config.hidden_size, self.num_labels) self.dropout = nn.Dropout(args["hidden_dropout_prob"]) self.apply(self.init_weights)
def __init__(self, bert, opt): super(LCF_BERT_HAT, self).__init__() self.bert_spc = bert self.opt = opt # self.bert_local = copy.deepcopy(bert) # Uncomment the line to use dual Bert self.bert_local = bert # Default to use single Bert and reduce memory requirements self.dropout = nn.Dropout(opt.dropout) self.bert_SA = SelfAttention(bert.config, opt) self.linear_double = nn.Linear(opt.bert_dim * 2, opt.bert_dim) #self.linear_single = nn.Linear(opt.bert_dim, opt.bert_dim) self.bert_pooler = BertPooler(bert.config) #self.dense = self.last = torch.nn.ModuleList() for t in range(self.opt.taskcla): self.last.append(nn.Linear(opt.bert_dim, opt.polarities_dim)) self.gate = torch.nn.Sigmoid() self.ec1 = torch.nn.Embedding(self.opt.taskcla, opt.bert_dim) self.ec2 = torch.nn.Embedding(self.opt.taskcla, bert.config.hidden_size) self.ec3 = torch.nn.Embedding(self.opt.taskcla, bert.config.hidden_size) self.hat = True
def __init__(self, config): super(BertModelDialog, self).__init__(config) self.embeddings = BertEmbeddingsDialog(config) self.encoder = BertEncoder(config) self.pooler = BertPooler(config) self.init_weights()
def __init__(self, bert, opt, is_global_configuration=False): super(LCF_BERT, self).__init__() self.bert_spc = bert self.opt = opt self.is_global_configuration = is_global_configuration # self.bert_local = copy.deepcopy(bert) # Uncomment the line to use dual Bert self.bert_local = ( bert # Default to use single Bert and reduce memory requirements ) self.dropout = nn.Dropout(self.opt.dropout) self.bert_SA = SelfAttention(bert.config, self.opt) self.linear_double = nn.Linear(self.opt.bert_dim * 2, self.opt.bert_dim) self.linear_single = nn.Linear(self.opt.bert_dim, self.opt.bert_dim) self.bert_pooler = BertPooler(bert.config) if self.opt.use_global_context: self.gc_bert = bert self.linear_gc_merger = nn.Linear( self.opt.bert_dim * self.opt.global_context_seqs_per_doc, self.opt.bert_dim, ) self.linear_lcf_and_gc_merger = nn.Linear(self.opt.bert_dim * 2, self.opt.bert_dim) if not self.is_global_configuration: self.dense = nn.Linear(self.opt.bert_dim, self.opt.polarities_dim)
def __init__(self, config): super(BertModel, self).__init__(config) self.embeddings = BertEmbeddings(config) self.encoder = BertEncoder(config) self.pooler = BertPooler(config) self.apply(self.init_weights)
def __init__(self, bert, opt): super(LCF_BERT, self).__init__() self.bert4global = bert self.bert4local = copy.deepcopy(bert) if opt.use_dual_bert else self.bert4global self.opt = opt self.dropout = nn.Dropout(opt.dropout) self.bert_SA = SelfAttention(bert.config, opt) self.linear2 = nn.Linear(opt.embed_dim * 2, opt.embed_dim) self.linear3 = nn.Linear(opt.embed_dim * 3, opt.embed_dim) self.bert_pooler = BertPooler(bert.config) self.dense = nn.Linear(opt.embed_dim, opt.polarities_dim)
def __init__(self, bert, opt): super(LCF_BERT, self).__init__() self.bert_global_focus = bert self.bert_local_focus = copy.deepcopy(bert) if opt.use_single_bert else bert self.opt = opt self.dropout = nn.Dropout(opt.dropout) self.bert_SA = SelfAttention(bert.config, opt) self.linear_double_cdm_or_cdw = nn.Linear(opt.bert_dim * 2, opt.bert_dim) self.linear_triple_lcf_global = nn.Linear(opt.bert_dim * 3, opt.bert_dim) self.bert_pooler = BertPooler(bert.config) self.dense = nn.Linear(opt.bert_dim, opt.polarities_dim)
def __init__(self, bert, opt): super(LCF_BERT, self).__init__() self.bert_spc = bert self.opt = opt # self.bert_local = copy.deepcopy(bert) # Uncomment the line to use dual Bert self.bert_local = bert # Default to use single Bert and reduce memory requirements self.dropout = nn.Dropout(opt.dropout) self.bert_SA = SelfAttention(bert.config, opt) self.linear_double = nn.Linear(opt.bert_dim * 2, opt.bert_dim) self.linear_single = nn.Linear(opt.bert_dim, opt.bert_dim) self.bert_pooler = BertPooler(bert.config) self.dense = nn.Linear(opt.bert_dim, opt.polarities_dim)
def __init__(self, bert_base_model, args): super(LCF_ATEPC_Chinese, self).__init__(config=bert_base_model.config) config = bert_base_model.config self.bert = bert_base_model self.args = args if self.args.local_context_focus is not None: self.local_bert = copy.deepcopy(self.bert) self.pooler = BertPooler(config) self.dense = torch.nn.Linear(768, 2) self.bert_global_focus = self.bert self.dropout = nn.Dropout(self.args.dropout) self.bert_SA = SelfAttention(config, args) self.linear_double = nn.Linear(768 * 2, 768) self.linear_triple = nn.Linear(768 * 3, 768)
def __init__(self, bert_base_model): super(LCF_ATEPC, self).__init__(config=bert_base_model.config) config = bert_base_model.config self.device = torch.device( "cuda:1" if torch.cuda.is_available() else 'cpu') self.bert = bert_base_model # do not init lcf layer if BERT-SPC or BERT-BASE specified self.local_bert = copy.deepcopy(self.bert) self.pooler = BertPooler(config) self.dense = torch.nn.Linear(768, 2) self.bert_global_focus = self.bert self.bert_SA = SelfAttention(config) self.dropout = torch.nn.Dropout(0.0) self.linear_double = nn.Linear(768 * 2, 768) self.linear_triple = nn.Linear(768 * 3, 768)
def __init__(self, embedding_matrix, opt): super(LCF_GLOVE, self).__init__() self.config = BertConfig.from_json_file("utils/bert_config.json") self.opt = opt self.embed = nn.Embedding.from_pretrained(torch.tensor(embedding_matrix, dtype=torch.float)) self.mha_global = SelfAttention(self.config, opt) self.mha_local = SelfAttention(self.config, opt) self.ffn_global = PositionwiseFeedForward(self.opt.embed_dim, dropout=self.opt.dropout) self.ffn_local = PositionwiseFeedForward(self.opt.embed_dim, dropout=self.opt.dropout) self.mha_local_SA = SelfAttention(self.config, opt) self.mha_global_SA = SelfAttention(self.config, opt) self.pool = BertPooler(self.config) self.dropout = nn.Dropout(opt.dropout) self.linear = nn.Linear(opt.embed_dim * 2, opt.embed_dim) self.dense = nn.Linear(opt.embed_dim, opt.polarities_dim)
def __init__(self, bert, opt): super(LCA_BERT, self).__init__() self.bert4global = bert self.bert4local = copy.deepcopy(bert) if opt.use_dual_bert else self.bert4global self.lc_embed = nn.Embedding(opt.max_seq_len, opt.embed_dim) self.opt = opt self.dropout = nn.Dropout(opt.dropout) self.bert_SA_L = SelfAttention(bert.config, opt) self.bert_SA_G = SelfAttention(bert.config, opt) self.linear = nn.Linear(opt.embed_dim * 2, opt.embed_dim) self.pool = BertPooler(bert.config) if self.opt.dataset in {'camera', 'notebook', 'car', 'phone'}: self.dense = nn.Linear(opt.embed_dim, 2) else: self.dense = nn.Linear(opt.embed_dim, 3) self.classifier = nn.Linear(opt.embed_dim, 2)
def __init__(self, embedding_matrix, opt): super(LCA_GLOVE, self).__init__() # Only few of the parameters are necessary in the config.json, such as hidden_size, num_attention_heads self.config = BertConfig.from_json_file("utils/bert_config.json") self.opt = opt self.embed = nn.Embedding.from_pretrained( torch.tensor(embedding_matrix, dtype=torch.float)) self.lc_embed = nn.Embedding(2, opt.embed_dim) self.global_encoder1 = SelfAttention(self.config, opt) self.local_encoder1 = SelfAttention(self.config, opt) self.local_encoder2 = SelfAttention(self.config, opt) self.mha = SelfAttention(self.config, opt) self.pool = BertPooler(self.config) self.dropout = nn.Dropout(opt.dropout) self.linear = nn.Linear(opt.embed_dim * 2, opt.embed_dim) self.dense = nn.Linear(opt.embed_dim, opt.polarities_dim) self.classifier = nn.Linear(opt.embed_dim, 2)
def __init__(self, bert_base_model, args): super(LCF_ATEPC, self).__init__(config=bert_base_model.config) config = bert_base_model.config self.bert = bert_base_model self.args = args # do not init lcf layer if BERT-SPC or BERT-BASE specified if self.args.local_context_focus is not None: self.local_bert = copy.deepcopy(self.bert) self.pooler = BertPooler(config) if args.dataset in {'camera', 'car', 'phone', 'notebook'}: self.dense = torch.nn.Linear(768, 2) else: self.dense = torch.nn.Linear(768, 3) self.bert_global_focus = self.bert self.dropout = nn.Dropout(self.args.dropout) self.bert_SA = SelfAttention(config, args) self.linear_double = nn.Linear(768 * 2, 768) self.linear_triple = nn.Linear(768 * 3, 768)
def __init__(self, bert_or_embedding_matrix, opt): super(BERT_ALBERT_GCN, self).__init__() self.opt = opt bert = bert_or_embedding_matrix self.context_bert = bert self.squeeze_embedding = SqueezeEmbedding() if not opt.no_gnn: self.gcn = GNN_RELU_DIFF(opt.bert_dim, opt.bert_dim, step=opt.gnn_step, drop=0.0) self.encoder_fn = nn.Linear(opt.bert_dim, opt.bert_dim) self.norm = LayerNorm(opt.bert_dim) if not opt.no_sa: self.bert_self_att = SelfAttention(bert.config, opt) if self.opt.pool_tp == 'bert_pool': self.bert_pooler = BertPooler(bert.config) elif self.opt.pool_tp == 'max_pool': self.pool_fc = nn.Linear(opt.bert_dim, opt.bert_dim) self.fc = nn.Linear(opt.bert_dim, 3)
def __init__( self, config='a class with num_attention_heads, hidden_size, attention_probs_dropout_prob, output_attentions', bert_dir='/mnt/sda1/bert/uncased_L-12_H-768_A-12', drop=0.0, L=80, bert_dim=768, num_class=3, SDR=5, tp='cdm'): super(MY_BERT_LCF, self).__init__() self.text_bert = BertModel.from_pretrained(bert_dir) self.aspect_bert = copy.deepcopy(self.text_bert) self.aspect_self_att = SelfAttention(config, L) self.bert_pooler = BertPooler(config) if tp == 'cdm': self.reduce2_bert_dim = nn.Linear(bert_dim * 2, bert_dim) self.reduce2_num_class_linear = nn.Linear(bert_dim, num_class) self.drop = drop self.L = L self.SDR = SDR self.tp = tp
def __init__(self, bert_base_model, args): super(MAAEAC, self).__init__(config=bert_base_model.config) config = bert_base_model.config self.bert_for_global_context = bert_base_model self.args = args # do not init lcf layer if BERT-SPC or BERT-BASE specified # if self.args.local_context_focus in {'cdw', 'cdm', 'fusion'}: if not self.args.use_unique_bert: self.bert_for_local_context = copy.deepcopy( self.bert_for_global_context) else: self.bert_for_local_context = self.bert_for_global_context self.pooler = BertPooler(config) if args.dataset in {'camera', 'car', 'phone', 'notebook'}: self.dense = torch.nn.Linear(768, 2) else: self.dense = torch.nn.Linear(768, 3) self.bert_global_focus = self.bert_for_global_context self.dropout = nn.Dropout(self.args.dropout) self.SA1 = SelfAttention(config, args) self.SA2 = SelfAttention(config, args) self.linear_double = nn.Linear(768 * 2, 768) self.linear_triple = nn.Linear(768 * 3, 768) self.hidden = config.hidden_size
def __init__(self, config, L): super(SelfAttention, self).__init__() self.SA = BertSelfAttention(config) self.L = L self.pooler = BertPooler(config)