def __init__(self, num_classes, model_path='./albert_base'):
super(AlbertCrf, self).__init__()
self.albert = AlbertModel.from_pretrained(model_path)
self.dropout = nn.Dropout(0.1)
self.fc1 = nn.Linear(self.albert.config.hidden_size, 256)
self.fc2 = nn.Linear(256, num_classes)
self.crf = CRF(num_classes, batch_first=True)
def __init__(self, config):
super().__init__(config)
self.num_labels = 2
self.albert = AlbertModel(config)
self.qa_outputs = nn.Linear(int(config.hidden_size / 4),
config.num_labels)
self.hypernet = HyberRnnNet(2048, 512, 1, 'lstm')
self.infernet = InferenceRNN(64, 2048, 512, 512, 'lstm')
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.albert = AlbertModel(config)
self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
def __init__(self, ):
super().__init__()
self.tokenizer = BertTokenizer.from_pretrained('./alberttiny')
self.model = AlbertModel.from_pretrained('./alberttiny').to(
Config.device)
self.is_src = None
#self.w2v= gensim.models.KeyedVectors.load_word2vec_format('../news_comment/baike_26g_news_13g_novel_229g.bin', binary=True)
'''
def __init__(self):
super(AlbertClassifier, self).__init__()
D_in, H, D_out = 768, 50, 2
self.albert = nn.DataParallel(
AlbertModel.from_pretrained('albert-base-v1'))
self.classifier = nn.Sequential(nn.Linear(D_in, H), nn.ReLU(),
nn.Dropout(0.4), nn.Linear(H, D_out))
def __init__(self, config, num_rel_labels):
super(AlbertForRelation, self).__init__(config)
self.num_labels = num_rel_labels
self.albert = AlbertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.layer_norm = BertLayerNorm(config.hidden_size * 2)
self.classifier = nn.Linear(config.hidden_size * 2, self.num_labels)
self.init_weights()
def __init__(self, config):
super(AlbertCrfForNer, self).__init__(config)
self.bert = AlbertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.crf = CRF(tagset_size=config.num_labels,
tag_dictionary=config.label2id,
is_bert=True)
self.init_weights()
def __init__(self, config, num_label):
super(AlbertQA, self).__init__()
self.albert = AlbertModel.from_pretrained('albert-xxlarge-v1')
self.fc = nn.Linear(config.hidden_size, num_label)
self.drop = nn.Dropout(config.hidden_dropout_prob)
self.loss = nn.CrossEntropyLoss(reduction='sum')
torch.nn.init.xavier_uniform_(self.fc.weight)
torch.nn.init.constant_(self.fc.bias, 0.)
def __init__(self, config):
super().__init__(config)
self.output_attentions = True
self.output_hidden_states = True
self.num_labels = config.num_labels
self.albert = AlbertModel(config)
self.dropout = nn.Dropout(config.classifier_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
self.init_weights()
def __init__(self, config):
super(Model, self).__init__()
self.config = AlbertConfig.from_pretrained(config.albert_config_path)
self.albert = AlbertModel.from_pretrained(config.albert_model_path,
config=self.config)
for param in self.albert.parameters():
param.requires_grad = True
self.fc = nn.Linear(config.hidden_size, config.num_classes)
def __init__(self,
args,
device,
d_model=256,
nhead=4,
d_ff=1024,
nlayers=2,
dropout=0.5):
super(Autoencoder, self).__init__()
self.model_type = 'Transformer'
self.d_model = d_model
self.src_mask = None
self.pos_encoder = PositionalEncoding(d_model,
dropout) # encoder's position
self.pos_decoder = PositionalEncoding(d_model,
dropout) # decoder's position
decoder_layers = TransformerDecoderLayer(d_model, nhead, d_ff, dropout)
decoder_norm = nn.LayerNorm(d_model)
self.transformer_decoder = TransformerDecoder(decoder_layers, nlayers,
decoder_norm)
# self.bert_encoder = BertModel.from_pretrained(args.PRETRAINED_MODEL_NAME, output_hidden_states=args.distill_2)
if args.use_albert:
self.bert_encoder = AlbertModel.from_pretrained(
"clue/albert_chinese_tiny")
self.bert_embed = self.bert_encoder.embeddings
# self.tgt_embed = self.bert_embed
d_vocab = self.bert_encoder.config.vocab_size + 1
self.tgt_embed = nn.Sequential(
Embeddings(d_model, d_vocab),
PositionalEncoding(d_model, dropout))
elif args.use_tiny_bert:
self.bert_encoder = AutoModel.from_pretrained(
"google/bert_uncased_L-2_H-256_A-4")
self.bert_embed = self.bert_encoder.embeddings
self.tgt_embed = self.bert_embed
elif args.use_distil_bert:
configuration = DistilBertConfig()
self.bert_encoder = DistilBertModel(configuration)
self.bert_embed = self.bert_encoder.embeddings
self.tgt_embed = self.bert_embed
# self.tgt_embed = self.bert.embeddings
else:
self.bert_encoder = BertModel.from_pretrained(
args.PRETRAINED_MODEL_NAME,
output_hidden_states=args.distill_2)
self.bert_embed = self.bert_encoder.embeddings
self.tgt_embed = self.bert_embed
self.distill_2 = args.distill_2
self.gru = nn.GRU(d_model, d_model, 1)
self.lr = nn.Linear(d_model, self.bert_encoder.config.vocab_size + 1)
self.sigmoid = nn.Sigmoid()
self.device = device
self.init_weights()
def __init__(self, drop_prob=0):
super(BertQA, self).__init__()
self.bert = AlbertModel.from_pretrained("albert-large-v2")
#for QA
self.ans_se = nn.Linear(1024, 2)
#for Beer, ten level sentiment
self.sentiment = nn.Linear(1024, 1)
self.sentiment_movie = nn.Linear(1024, 1)
self.sigmoid = nn.Sigmoid()
def __init__(self, bert_model='bert-base-cased'):
super(BERTRepresenter, self).__init__()
if 'albert' in bert_model.lower():
config = AlbertConfig()
self.bert = AlbertModel(config).from_pretrained(bert_model)
else:
config = BertConfig()
# config = BertConfig(vocab_size=24000, hidden_size=264)
self.bert = BertModel(config).from_pretrained(bert_model)
def __init__(self,hidden_dim=768,num_tags=20):
super(TagValueModel,self).__init__()
self.albert = AlbertModel.from_pretrained("ALINEAR/albert-japanese-v2")
self.dropout = nn.Dropout(0.1)
self.tags = nn.Linear(768,num_tags)
#self.tags_insentence = nn.Linear(768,20)
self.starts = nn.Linear(768,num_tags)
self.ends = nn.Linear(768,num_tags)
self.num_tags = num_tags
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.albert = AlbertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.init_weights()
def __init__(self, config):
super().__init__(config)
self.num_labels = config.num_labels
self.model = AlbertModel(config, add_pooling_layer=False)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.loss_func = CrossEntropyLoss()
self.init_weights()
def __init__(self, config):
super().__init__(config)
self.albert = AlbertModel(config)
self.relu = nn.ReLU()
self.layer1 = nn.Linear(config.hidden_size, config.mlp_size)
self.layer2 = nn.Linear(config.mlp_size, config.mlp_size)
self.layer3 = nn.Linear(config.mlp_size, config.hidden_size)
self.init_weights()
def load_pretrained_encoder(mpath,
config="albert_config.json",
model="albert_model.bin"):
b_config = BC.from_pretrained(opt.join(mpath, config))
encoder = AlbertModel.from_pretrained(opt.join(mpath, model),
config=b_config)
return encoder
def __init__(self, config):
super().__init__(config)
self.albert = AlbertModel(config, add_pooling_layer=False)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.output_layer = nn.Linear(config.hidden_size, 1)
self.pooling_type = config.pooling_type
self.init_weights()
def __init__(self, config, label_size):
super().__init__(config)
self.num_labels = config.num_labels
self.model = AlbertModel(config, add_pooling_layer=False)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.crf = CRF(num_tags=label_size, batch_first=True)
self.init_weights()
def __prepare_model_instance(self, config):
self.config_class, self.pretrained_model_archive_map, self.base_model_prefix = self.CONF_REF[
self.model_type]
if self.model_type == "bert":
self.model = BertModel(config)
elif self.model_type == 'roberta':
self.model = RobertaModel(config)
elif self.model_type == 'albert':
self.model = AlbertModel(config)
def __init__(
self,
pretrained_model_name=None,
config_filename=None,
vocab_size=None,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
intermediate_size=3072,
hidden_act="gelu",
max_position_embeddings=512,
):
super().__init__()
# Check that only one of pretrained_model_name, config_filename, and
# vocab_size was passed in
total = 0
if pretrained_model_name is not None:
total += 1
if config_filename is not None:
total += 1
if vocab_size is not None:
total += 1
if total != 1:
raise ValueError(
"Only one of pretrained_model_name, vocab_size, "
+ "or config_filename should be passed into the "
+ "ALBERT constructor."
)
# TK: The following code checks the same once again.
if vocab_size is not None:
config = AlbertConfig(
vocab_size_or_config_json_file=vocab_size,
vocab_size=vocab_size,
hidden_size=hidden_size,
num_hidden_layers=num_hidden_layers,
num_attention_heads=num_attention_heads,
intermediate_size=intermediate_size,
hidden_act=hidden_act,
max_position_embeddings=max_position_embeddings,
)
model = AlbertModel(config)
elif pretrained_model_name is not None:
model = AlbertModel.from_pretrained(pretrained_model_name)
elif config_filename is not None:
config = AlbertConfig.from_json_file(config_filename)
model = AlbertModel(config)
else:
raise ValueError(
"Either pretrained_model_name or vocab_size must" + " be passed into the ALBERT constructor"
)
model.to(self._device)
self.add_module("albert", model)
self.config = model.config
self._hidden_size = model.config.hidden_size
def __init__(self, config, num_rel_labels):
super(AlbertForRelation, self).__init__(config)
self.num_labels = num_rel_labels
self.albert = AlbertModel(self.config)
# self.albert.resize_token_embeddings(vocab_size)
self.dropout = nn.Dropout(self.config.hidden_dropout_prob)
self.layer_norm = torch.nn.LayerNorm(self.config.hidden_size * 2)
self.classifier = nn.Linear(self.config.hidden_size * 2,
self.num_labels)
self.init_weights()
def __init__(self, model_name, char_vectors, hidden_size, drop_prob):
super(Embedding, self).__init__()
size_map = {'albert-base-v2':768, 'albert-large-v2':1024, 'albert-xlarge-v2':2048, 'albert-xxlarge-v2':4096}
self.drop_prob = drop_prob
self.albert = AlbertModel.from_pretrained(model_name)
self.char_embed = nn.Embedding.from_pretrained(char_vectors)
self.proj = nn.Linear(size_map[model_name], hidden_size, bias=False)
self.hwy = HighwayEncoder(2, hidden_size*2+2)
self.avgatt = AverageAttn(hidden_size)
self.cnn = CNN(hidden_size=hidden_size,embed_size=char_vectors.size(1))
def load_pretrained(mpath,
config="albert_config.json",
model="albert_model.bin"):
b_config = BC.from_pretrained(opt.join(mpath, config))
encoder = AlbertModel.from_pretrained(opt.join(mpath, model),
config=b_config)
tokenizer = BertTokenizer.from_pretrained(mpath)
return encoder, tokenizer
def load_albert(path):
"""
加载模型
"""
vocab_file = os.path.join(path, 'vocab.txt')
tokenizer = BertTokenizer.from_pretrained(vocab_file)
# print(tokenizer)
config = AlbertConfig.from_pretrained(path)
model = AlbertModel.from_pretrained(path, config=config)
return model, tokenizer
def __init__(self, config, crf=None):
super().__init__(config)
self.num_labels = config.num_labels
self.albert = AlbertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
self.loss_fct = nn.CrossEntropyLoss()
self.use_crf = False
self.crf_layer = crf
self.init_weights()
def getAlBertEmbeddings(self):
model = AlbertModel.from_pretrained('albert-base-v2')
tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
model.eval()
tokens_tensor, segments_tensors = self.getIndexs(tokenizer)
with torch.no_grad():
last_hidden_states = model(tokens_tensor, attention_mask=segments_tensors)
features = last_hidden_states[0][:,0,:].numpy()
features = np.reshape(features,features.shape[1])
return(features.tolist())
def __init__(self, bert_name, num_class, bert_type='bert', drop_out=0.1):
super(Bert, self).__init__()
if bert_type == 'bert':
self.bert = BertModel.from_pretrained(bert_name)
elif bert_type == 'albert':
self.bert = AlbertModel.from_pretrained(bert_name)
else:
raise Exception('Please enter the correct bert type.')
self.drop_out = nn.Dropout(p=drop_out)
self.classifier = nn.Linear(self.bert.config.hidden_size, num_class)
def __init__(self, albert_model_name,
bio_vocab_size, bio_embed_size,
feat_vocab_size, feat_embed_size,
albert_cache_dir=None):
super(AlbertFeatureRichEmbedding, self).__init__()
self.bio_embed_size = bio_embed_size
self.feat_embed_size = feat_embed_size
self.albert_embeddings = AlbertModel.from_pretrained(albert_model_name, cache_dir=albert_cache_dir)
self.bio_embeddings = nn.Embedding(bio_vocab_size, bio_embed_size)
self.feat_embeddings = nn.Embedding(feat_vocab_size, feat_embed_size)
