def __init__(self, bert, freeze_bert=False):
super(RegressionBert, self).__init__()
self.bert = BertModel.from_pretrained(bert)
if freeze_bert:
for p in self.bert.parameters():
p.requires_grad = False
self.fc0 = torch.nn.Linear(768, 1)
def __init__(self):
super(Bert, self).__init__()
self.tokenizer = BertTokenizer.from_pretrained(os.path.join(config.get('model_config')['language_model_path'], 'bert-base-uncased-vocab.txt'))
modelConfig = BertConfig.from_pretrained(os.path.join(config.get('model_config')['language_model_path'], 'bert_config.json'))
self.textExtractor = BertModel.from_pretrained(
os.path.join(config.get('model_config')['language_model_path'], 'pytorch_model.bin'), config=modelConfig)
def __init__(self, opt):
self.opt = opt
if 'bert' in opt.model_name:
tokenizer = Tokenizer4Bert(opt.max_seq_len,
opt.pretrained_bert_name)
bert = BertModel.from_pretrained(opt.pretrained_bert_name)
self.pretrained_bert_state_dict = bert.state_dict()
self.model = opt.model_class(bert, opt).to(opt.device)
else:
tokenizer = build_tokenizer(
fnames=[opt.dataset_file['train'], opt.dataset_file['test']],
max_seq_len=opt.max_seq_len,
dat_fname='{0}_tokenizer.dat'.format(opt.dataset))
embedding_matrix = build_embedding_matrix(
word2idx=tokenizer.word2idx,
embed_dim=opt.embed_dim,
dat_fname='{0}_{1}_embedding_matrix.dat'.format(
str(opt.embed_dim), opt.dataset))
self.model = opt.model_class(embedding_matrix, opt).to(opt.device)
self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer)
self.testset = ABSADataset(opt.dataset_file['test'], tokenizer)
if opt.device.type == 'cuda':
logger.info('cuda memory allocated: {}'.format(
torch.cuda.memory_allocated(device=opt.device.index)))
self._print_args()
def createCsvData():
config = BertConfig.from_pretrained('bert-base-uncased')
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel(config)
with Cd("lemmadata"):
with open("id_to_sent.json") as sent_id_dict_file:
sent_id_dict = json.load(sent_id_dict_file)
for dir_item in os.listdir():
if os.path.isfile(dir_item):
if dir_item.endswith(".json") and dir_item != "id_to_sent.json":
print(dir_item)
with open(dir_item, "r") as f:
lemma_data = json.load(f)
with Cd("vectors"):
with open(dir_item[:-5]+".csv", "w") as vector_file:
writer = csv.writer(vector_file, delimiter=",")
for instance in lemma_data:
inst_sent_id = instance["sent_id"]
inst_sense = instance["sense"]
inst_sent = sent_id_dict[str(inst_sent_id)]
if(len(inst_sent) > 511):
continue
vector = vectorizeWordInContext(inst_sent, instance["pos"], tokenizer, model)
vec_list = vector.detach().tolist()
row_data = [inst_sent_id, instance["pos"], inst_sense] + vec_list
writer.writerow(row_data)
def get_bert(bert_type='bert'):
tokenizer, model = None, None
if (bert_type == 'bert'):
######## bert ###########
tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)
model = BertModel.from_pretrained('bert-base-cased')
########################
if (bert_type == 'biobert'):
#### Bio BERT #########
model = bm.from_pretrained('biobert_v1.1_pubmed')
tokenizer = BertTokenizer(vocab_file="biobert_v1.1_pubmed/vocab.txt", do_lower_case=False)
#### Bio BERT #########
if (bert_type == 'scibert'):
#### sci bert #########
config = AutoConfig.from_pretrained('allenai/scibert_scivocab_cased', output_hidden_states=False)
tokenizer = AutoTokenizer.from_pretrained('allenai/scibert_scivocab_cased', do_lower_case=False)
model = AutoModel.from_pretrained('allenai/scibert_scivocab_cased', config=config)
#######################
return tokenizer, model
def __init__(self, opt):
self.opt = opt
if 'bert' in opt.model_name:
tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name)
bert = BertModel.from_pretrained(opt.pretrained_bert_name)
self.model = opt.model_class(bert, opt).to(opt.device)
else:
tokenizer = build_tokenizer(
fnames=[opt.dataset_file['train'], opt.dataset_file['test']],
max_seq_len=opt.max_seq_len,
dat_fname='{0}_tokenizer.dat'.format(opt.dataset))
embedding_matrix = build_embedding_matrix(
word2idx=tokenizer.word2idx,
embed_dim=opt.embed_dim,
dat_fname='{0}_{1}_embedding_matrix.dat'.format(str(opt.embed_dim), opt.dataset))
self.model = opt.model_class(embedding_matrix, opt).to(opt.device)
# self.trainset = ABSADataset(opt.dataset_file['train'], tokenizer)
self.testset = ABSADataset(opt.dataset_file['test'], tokenizer)
# assert 0 <= opt.valset_ratio < 1
# if opt.valset_ratio > 0:
# valset_len = int(len(self.trainset) * opt.valset_ratio)
# self.trainset, self.valset = random_split(self.trainset, (len(self.trainset) - valset_len, valset_len))
# else:
# self.valset = self.testset
#
# if opt.device.type == 'cuda':
# logger.info('cuda memory allocated: {}'.format(torch.cuda.memory_allocated(device=opt.device.index)))
model_path = 'saved/'+opt.model_name+'.hdf5'
self.model.load_state_dict(torch.load(model_path))
def __init__(self, args, device, checkpoint=None, bert_from_extractive=None):
super(AbsSummarizer, self).__init__()
self.args = args
self.device = device
self.bert = Bert(args.large, args.temp_dir, args.finetune_bert)
if bert_from_extractive is not None:
self.bert.model.load_state_dict(
dict([(n[11:], p) for n, p in bert_from_extractive.items() if n.startswith('bert.model')]), strict=True)
if (args.encoder == 'baseline'):
bert_config = BertConfig(self.bert.model.config.vocab_size, hidden_size=args.enc_hidden_size,
num_hidden_layers=args.enc_layers, num_attention_heads=8,
intermediate_size=args.enc_ff_size,
hidden_dropout_prob=args.enc_dropout,
attention_probs_dropout_prob=args.enc_dropout)
self.bert.model = BertModel(bert_config)
if(args.max_pos>512):
my_pos_embeddings = nn.Embedding(args.max_pos, self.bert.model.config.hidden_size)
my_pos_embeddings.weight.data[:512] = self.bert.model.embeddings.position_embeddings.weight.data
my_pos_embeddings.weight.data[512:] = self.bert.model.embeddings.position_embeddings.weight.data[-1][None,:].repeat(args.max_pos-512,1)
self.bert.model.embeddings.position_embeddings = my_pos_embeddings
self.vocab_size = self.bert.model.config.vocab_size
tgt_embeddings = nn.Embedding(self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0)
if (self.args.share_emb):
tgt_embeddings.weight = copy.deepcopy(self.bert.model.embeddings.word_embeddings.weight)
self.decoder = TransformerDecoder(
self.args.dec_layers,
self.args.dec_hidden_size, heads=self.args.dec_heads,
d_ff=self.args.dec_ff_size, dropout=self.args.dec_dropout, embeddings=tgt_embeddings)
self.generator = get_generator(self.vocab_size, self.args.dec_hidden_size, device, self.args.task)
self.generator[0].weight = self.decoder.embeddings.weight
if checkpoint is not None:
print("Abstractor is loading.")
self.load_state_dict(checkpoint['model'], strict=True)
else:
for module in self.decoder.modules():
if isinstance(module, (nn.Linear, nn.Embedding)):
module.weight.data.normal_(mean=0.0, std=0.02)
elif isinstance(module, nn.LayerNorm):
module.bias.data.zero_()
module.weight.data.fill_(1.0)
if isinstance(module, nn.Linear) and module.bias is not None:
module.bias.data.zero_()
for p in self.generator.parameters():
if p.dim() > 1:
xavier_uniform_(p)
else:
p.data.zero_()
if(args.use_bert_emb):
tgt_embeddings = nn.Embedding(self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0)
tgt_embeddings.weight = copy.deepcopy(self.bert.model.embeddings.word_embeddings.weight)
self.decoder.embeddings = tgt_embeddings
self.generator[0].weight = self.decoder.embeddings.weight
self.to(device)
def aspectSentiment_api():
data = request.json
opt = get_parameters()
opt.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name)
bert = BertModel.from_pretrained(opt.pretrained_bert_name)
model = AEN_BERT(bert, opt).to(opt.device)
print('loading model {0} ...'.format(opt.model_name))
model.load_state_dict(
torch.load('aen_bert_restaurant_val_acc0.8098',
map_location=opt.device))
model.eval()
torch.autograd.set_grad_enabled(False)
out = []
for entity, sentences in data.items():
for sentence in sentences:
sentence_d = {'aspect': '', 'sentiment': '', 'sentence': ''}
sentiment_d = {-1: 'Negative', 0: 'Neutral', 1: 'Positive'}
left = sentence['left']
aspect = sentence['aspect']
right = sentence['right']
sentence = left + aspect + right
text_bert_indices, bert_segments_ids, text_raw_bert_indices, aspect_bert_indices = prepare_data(
left, aspect, right, tokenizer)
text_bert_indices = torch.tensor([text_bert_indices],
dtype=torch.int64).to(opt.device)
bert_segments_ids = torch.tensor([bert_segments_ids],
dtype=torch.int64).to(opt.device)
text_raw_bert_indices = torch.tensor([text_raw_bert_indices],
dtype=torch.int64).to(
opt.device)
aspect_bert_indices = torch.tensor([aspect_bert_indices],
dtype=torch.int64).to(
opt.device)
inputs = [text_raw_bert_indices, aspect_bert_indices]
outputs = model(inputs)
t_probs = F.softmax(outputs, dim=-1).cpu().numpy()
aspect_sentiment_n = t_probs.argmax(axis=-1) - 1
print(aspect_sentiment_n)
aspect_sentiment = sentiment_d[aspect_sentiment_n[0]]
sentence_d['aspect'] = aspect
sentence_d['sentiment'] = aspect_sentiment
sentence_d['sentence'] = sentence
out.append(sentence_d)
dic = absa_chapter_combined_s(out)
absaChapterCombinedS = absa_chapter_to_react(dic)
returnJson = {
'sentimentTableData': absaChapterCombinedS,
'absaChapter': dic
}
return returnJson
def __init__(self, config, num_labels):
super(BERTCRF, self).__init__(config)
self.num_labels = num_labels # ent label num
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.fc = nn.Linear(config.hidden_size, num_labels)
self.crf = CRF(num_labels, batch_first=True)
self.apply(self.init_bert_weights)
def __init__(self, bert_config, num_choices):
super(BertCloze, self).__init__(bert_config)
self.num_choices = num_choices
self.bert = BertModel(bert_config)
self.idiom_embedding = nn.Embedding(len(config.idiom2index),
bert_config.hidden_size)
self.my_fc = nn.Sequential(nn.Dropout(config.hidden_dropout_prob),
nn.Linear(bert_config.hidden_size, 1))
def __init__(self, config):
super(BertForLes, self).__init__(config)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
self.apply(self.init_weights)
def __init__(self):
super().__init__()
self.bert = BertModel.from_pretrained(
Config.pretrained_model_name_or_path)
self.soft_max = nn.Softmax(dim=-1)
self.fc = nn.Linear(768 * 3, 2)
self.dropout = nn.Dropout(0.8)
self.loss = nn.CrossEntropyLoss()
def __init__(self):
self.tokenizer = BertTokenizer.from_pretrained('bert-base-cased', do_lower_case=False)
self.model = BertModel.from_pretrained('bert-base-cased', output_hidden_states=True)
self.model.eval()
self.pad_token_id = self.tokenizer.encode(self.tokenizer.pad_token)[0]
self.cls_token_id = self.tokenizer.encode(self.tokenizer.cls_token)[0]
self.sep_token_id = self.tokenizer.encode(self.tokenizer.sep_token)[0]
self.device = 'cpu'
def __init__(self, tokenizing_data, device, embedding_path=None):
# Load pretrained model/tokenizer
self.device = device
self.bert_model = BertModel.from_pretrained(
'bert-base-uncased', output_hidden_states=True).to(self.device)
self.tokenizing_data = tokenizing_data
self.embedding_path = embedding_path
self.songs_features = torch.tensor([])
def __init__(self, opt):
super(BertEmbedding, self).__init__()
self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',
do_lower_case=True)
# Load pre-trained model (weights)
self.bert_model = BertModel.from_pretrained('bert-base-uncased',
output_hidden_states=True)
def __init__(self, vocab_size, config):
super(LM, self).__init__()
self.model_name = 'LM'
self.lm_name = config.lm.lm_file
self.out_dim = config.relation_type
self.lm = BertModel.from_pretrained(self.lm_name)
self.fc = nn.Linear(768, self.out_dim)
def __init__(self, hidden_dim, n_layers, tagset_size):
super(BertLSTM, self).__init__()
config = BertConfig.from_pretrained('bert-base-multilingual-cased')
self.model = BertModel(config)
self.decoder = nn.LSTM(768, hidden_dim, n_layers)
self.hiddentotag = nn.Linear(hidden_dim, tagset_size)
def __init__(self, config):
super(ABSABert, self).__init__(config)
self.bert = BertModel(config)
self.num_labels = config.num_labels
self.classifier = torch.nn.Linear(config.hidden_size, self.num_labels)
self.cls = DepBertPreTrainingHeads(config)
self.domain_cls = DomainPredictionHead(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
def prepare_models():
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased',
output_attentions=True)
model.eval()
mask_model = BertForMaskedLM.from_pretrained('bert-base-uncased')
mask_model.eval()
return tokenizer, model, mask_model
def main():
torch.cuda.empty_cache()
parser = setup_parser()
args = parser.parse_args()
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir
) and args.do_train and not args.overwrite_output_dir:
raise ValueError("Output directory already exists and is not empty.")
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
args.n_gpu = torch.cuda.device_count()
args.device = device
set_seed(args)
args.task_name = args.task_name.lower()
if args.task_name not in processors:
raise ValueError("Task not found: {}".format(args.task_name))
processor = processors[args.task_name]()
args.output_mode = output_modes[args.task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
##Load Models
config = BertConfig.from_pretrained(args.config_name)
tokenizer = BertTokenizer.from_pretrained(args.text_encoder_checkpoint,
do_lower_case=args.do_lower_case)
text_encoder = BertModel.from_pretrained(args.text_encoder_checkpoint,
config=config)
graph_encoder = GraphEncoder(args.n_hidden, args.min_score)
if args.graph_encoder_checkpoint:
graph_encoder.gcnnet.load_state_dict(
torch.load(args.graph_encoder_checkpoint))
medsts_classifier = PairClassifier(config.hidden_size + args.n_hidden, 1)
medsts_c_classifier = PairClassifier(config.hidden_size + args.n_hidden, 5)
medsts_type_classifier = PairClassifier(config.hidden_size + args.n_hidden,
4)
model = MedstsNet(text_encoder, graph_encoder, medsts_classifier,
medsts_c_classifier, medsts_type_classifier)
model.to(args.device)
args.n_gpu = 1
if args.do_train:
train_dataset = load_and_cache_examples(args,
args.task_name,
tokenizer,
evaluate=False)
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info('global step = {}, average loss = {}'.format(
global_step, tr_loss))
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
logger.info("saving model checkpoint to {}".format(args.output_dir))
model_to_save = model.module if hasattr(model, 'module') else model
# model_to_save.save_pretrained(args.output_dir)
torch.save(model_to_save.state_dict(),
os.path.join(args.output_dir, 'saved_model.pth'))
tokenizer.save_pretrained(args.output_dir)
torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
def main():
torch.cuda.empty_cache()
parser = setup_parser()
args = parser.parse_args()
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir
) and args.do_train and not args.overwrite_output_dir:
raise ValueError("Output directory already exists and is not empty.")
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
args.n_gpu = torch.cuda.device_count()
args.device = device
set_seed(args)
args.task_name = args.task_name.lower()
if args.task_name not in processors:
raise ValueError("Task not found: {}".format(args.task_name))
processor = processors[args.task_name]()
args.output_mode = output_modes[args.task_name]
label_list = processor.get_labels()
##Load Models
config = BertConfig.from_pretrained(args.config_name)
tokenizer = BertTokenizer.from_pretrained(args.text_encoder_checkpoint,
do_lower_case=args.do_lower_case)
text_encoder = BertModel.from_pretrained(args.text_encoder_checkpoint,
config=config)
graph_encoder = GraphEncoder(args.n_hidden, args.min_score)
medsts_classifier = PairClassifier(config.hidden_size + args.n_hidden, 1)
medsts_c_classifier = PairClassifier(config.hidden_size + args.n_hidden, 5)
medsts_c2_classifier = PairClassifier(config.hidden_size + args.n_hidden,
2)
medsts_type_classifier = PairClassifier(config.hidden_size + args.n_hidden,
4)
model = MedstsNet(text_encoder, graph_encoder, medsts_classifier,
medsts_c_classifier, medsts_c2_classifier,
medsts_type_classifier)
if args.text_only:
medsts_classifier = PairClassifier(config.hidden_size, 1)
medsts_c_classifier = PairClassifier(config.hidden_size, 5)
medsts_c2_classifier = PairClassifier(config.hidden_size, 2)
medsts_type_classifier = PairClassifier(config.hidden_size, 4)
model = MedstsNet_Textonly(text_encoder, medsts_classifier,
medsts_c_classifier, medsts_c2_classifier,
medsts_type_classifier)
model.to(args.device)
args.n_gpu = 1
if args.do_train:
train_dataset = load_and_cache_examples(args,
args.task_name,
tokenizer,
evaluate=False,
reverse=True)
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info('global step = {}, average loss = {}'.format(
global_step, tr_loss))
def __init__(self,
*,
pretrained_model_name=None,
config_filename=None,
vocab_size=None,
hidden_size=768,
num_hidden_layers=12,
num_attention_heads=12,
intermediate_size=3072,
hidden_act="gelu",
max_position_embeddings=512,
random_init=False,
**kwargs):
TrainableNM.__init__(self, **kwargs)
# Check that only one of pretrained_model_name, config_filename, and
# vocab_size was passed in
total = 0
if pretrained_model_name is not None:
total += 1
if config_filename is not None:
total += 1
if vocab_size is not None:
total += 1
if total != 1:
raise ValueError(
"Only one of pretrained_model_name, vocab_size, " +
"or config_filename should be passed into the " +
"BERT constructor.")
if vocab_size is not None:
config = BertConfig(
vocab_size_or_config_json_file=vocab_size,
hidden_size=hidden_size,
num_hidden_layers=num_hidden_layers,
num_attention_heads=num_attention_heads,
intermediate_size=intermediate_size,
hidden_act=hidden_act,
max_position_embeddings=max_position_embeddings)
model = BertModel(config)
elif pretrained_model_name is not None:
model = BertModel.from_pretrained(pretrained_model_name)
elif config_filename is not None:
config = BertConfig.from_json_file(config_filename)
model = BertModel(config)
else:
raise ValueError(
"Either pretrained_model_name or vocab_size must" +
"be passed into the BERT constructor")
model.to(self._device)
self.add_module("bert", model)
self.config = model.config
if random_init:
self.apply(
lambda module: transformer_weights_init(module, xavier=False))
def __init__(self, bert_model_path, num_classes):
super(Bert, self).__init__()
self.bert = BertModel.from_pretrained(bert_model_path)
# 不对bert进行训练
for param in self.bert.parameters():
param.requires_grad = True
self.fc = nn.Linear(self.bert.config.to_dict()['hidden_size'],
num_classes)
def __init__(self):
super(BERTClassifier, self).__init__()
self.bert = BertModel.from_pretrained('bert-base-uncased')
self.fc = nn.Sequential(
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(768, 2)
)
def __init__(self, code_length): # code_length为fc映射到的维度大小
super(TextNet, self).__init__()
modelConfig = BertConfig.from_pretrained(
'./data/bert-base-uncased-config.json')
self.textExtractor = BertModel.from_pretrained(
'./data/bert-base-uncased-pytorch_model.bin', config=modelConfig)
# self.textExtractor.eval()
embedding_dim = self.textExtractor.config.hidden_size
def __init__(self, dataset, model_path):
super(Model, self).__init__()
self.config = config.IMDBConfig()
model = BertModel.from_pretrained(self.config.BERT_MODEL)
self.model = ModelTrainer(model, 2)
self.model.load_state_dict(torch.load(model_path))
self.model.eval()
self.dataset = dataset
self.tokenizer = BertTokenizer.from_pretrained(self.config.BERT_MODEL)
def __init__(self, config):
super(BertForSequenceClassification, self).__init__(config)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)
self.init_weights()
def __init__(self):
super(BSME_model, self).__init__()
self.tokenizer = BertTokenizer.from_pretrained(
'bert-base-multilingual-cased', do_lower_case=False)
self.model = BertModel.from_pretrained(
'bert-base-multilingual-cased',
do_lower_case=False,
output_hidden_states=True).to('cuda')
self.classifier = DropoutClassifier(768 * 2, 4).to('cuda')
def __init__(self, config):
super(NoHiddenLayerClassification, self).__init__(config)
self.num_labels = config.num_labels
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.fc1 = nn.Linear(in_features=768 * 5, out_features=2)
self.init_weights()
def __init__(self, config, num_choices=1, num_docs_rank=30):
super(BertForMultipleChoice, self).__init__(config)
self.num_choices = num_choices
self.bert = BertModel(config)
self.dropout = nn.Dropout(config.hidden_dropout_prob)
self.classifier = nn.Linear(config.hidden_size, num_choices)
self.init_weights()
