def __init__(self, out=3, out_src=0, bert_type='bert'): super(Bert_finetuning, self).__init__() _, self.bert_model = bert.get_bert(bert_type=bert_type) ''' #self.fc=nn.Linear(768,out) #print(self.bert_model) self.fc=nn.Sequential( #nn.BatchNorm1d(768), #nn.ReLU(inplace=True), #nn.LayerNorm(768), #nn.Dropout(p=0.1,inplace=False), nn.Linear(768,out), #nn.GELU() #nn.LayerNorm(out), #nn.Dropout(p=0.1,inplace=False) #nn.LogSigmoid() ) ''' #self.dropout = torch.nn.Dropout(0.1)#0.1 #torch.nn.init.normal_(self.fc[0].weight.data,mean=0.0,std=0.02) #torch.nn.init.normal_(self.fc[0].weight.data,mean=0.0,std=0.02) #self.fc.weight.data.uniform_(0.0,0.02) #self.fc.weight.data.fill_(0.1) #self.fc[0].bias.data.fill_(0) #print("0.5 -1.1,-1 -1") self.trensfer = out_src != 0 self.out_layers = nn.Linear(768, out) self.out_src_layers = nn.Linear(768, out_src) if self.trensfer else None
def __init__(self, out=3, out_src=0, bert_type='bert'): super(Bert_finetuning_CNN_Entity, self).__init__() _, self.bert_model = bert.get_bert(bert_type=bert_type) self.cnn1 = CNN(inc=768, Dr=[0.25, 0.25], nb_filtres=[3], FC_L_size=[100], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=100, out_src=out_src) self.cnn2 = CNN(inc=768, Dr=[0.25, 0.25], nb_filtres=[3], FC_L_size=[100], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=100, out_src=out_src) self.cnn3 = CNN(inc=768, Dr=[0.25, 0.25], nb_filtres=[3], FC_L_size=[100], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=100, out_src=out_src) self.cnn4 = CNN(inc=768, Dr=[0.25, 0.25], nb_filtres=[3], FC_L_size=[100], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=100, out_src=out_src) self.cnn5 = CNN(inc=768, Dr=[0.25, 0.25], nb_filtres=[3], FC_L_size=[100], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=100, out_src=out_src) self.fc = nn.Sequential(nn.LayerNorm(500), nn.Dropout(p=0.1, inplace=False), nn.Linear(500, 100), nn.Hardtanh(), nn.LayerNorm(100), nn.Linear(100, out), nn.Hardtanh())
def __init__(self, out=3, out_src=0, bert_type='bert'): super(Bert_finetuning_CNN_resedual, self).__init__() _, self.bert_model = bert.get_bert(bert_type=bert_type) self.cnn = CNN(inc=768, nb_filtres=[3], FC_L_size=[768], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=out, out_src=0, w_size=768)
def __init__(self, out=3, out_src=0, bert_type='bert'): super(Bert_finetuning_Mixed, self).__init__() _, self.bert_model = bert.get_bert(bert_type=bert_type) self.cnn = CNN(inc=768, Dr=[0.5, 0.5], nb_filtres=[8], FC_L_size=[768], ker_size=[3, 2], padding=False, maxpooling_dir=True, out=out, out_src=out_src) self.out_layers = nn.Linear(768, out)