def __init__(self, config, bertmodel):
super(Parser, self).__init__()
self.config = config
# build and load BERT G2G model
bertconfig = BertConfig.from_pretrained(
config.main_path+"/model"+"/model_"+config.modelname+'/config.json')
bertconfig.num_hidden_layers = config.n_attention_layer
bertconfig.label_size = config.n_rels
bertconfig.layernorm_value = config.layernorm_value
bertconfig.layernorm_key = config.layernorm_key
if self.config.input_graph:
self.bert = BertGraphModel(bertconfig)
else:
self.bert = BertBaseModel(bertconfig)
self.bert.load_state_dict(bertmodel.state_dict(),strict=False)
self.mlp = Classifier(3*bertconfig.hidden_size,bertconfig.hidden_size,config.n_trans)
self.mlp_rel = Classifier(2*bertconfig.hidden_size,bertconfig.hidden_size,config.n_rels)
self.pad_index = config.pad_index
self.unk_index = config.unk_index
def predict(predict_model_name_or_path, pre_data, pre_dataloader):
print('进行预测')
pro = processer()
labellist = pro.get_labels()
#*****加载模型*****
print('加载模型')
model = BertForSequenceClassification
config = BertConfig.from_pretrained(predict_model_name_or_path,
num_labels=len(labellist))
model = model.from_pretrained(predict_model_name_or_path, config=config)
print('模型加载到GPU或者CPU')
#如果有GPU，使用GPU进行分布式计算，否则使用CPU
if torch.cuda.is_available():
#单GPU计算
torch.cuda.set_device(0)
device = torch.device('cuda', 0) #设置GPU设备号
else:
device = torch.device('cpu')
model.to(device)
print('******** Running prediction ********')
print(" Num examples = %d", len(pre_data))
preds = None
pbar = ProgressBar(n_total=len(pre_dataloader), desc="Predicting")
#***进行预测***
for step, batch in enumerate(pre_dataloader):
model.eval()
batch = tuple(t.to(device) for t in batch)
with torch.no_grad():
inputs = {
'input_ids': batch[0],
'token_type_ids': batch[2],
'attention_mask': batch[1],
'labels': batch[3]
}
outputs = model(**inputs)
_, logits = outputs[:2]
#***汇总每个batch的预测结果***
if preds is None:
preds = logits.softmax(-1).detach().cpu().numpy()
else:
preds = np.append(preds,
logits.softmax(-1).detach().cpu().numpy(),
axis=0)
pbar(step)
predict_label = np.argmax(preds, axis=1)
print(preds)
print(predict_label)
return preds, predict_label
def __init(self, config=None, *args, **kwargs):
super().__init__(*args, **kwargs)
if config is None:
from transformers.configuration_bert import BertConfig
config = BertConfig.from_pretrained('bert-base-uncased')
assert config.hidden_size == self.in_dim
from transformers.modeling_bert import BertPredictionHeadTransform
self.module = nn.Sequential(
nn.Dropout(config.hidden_dropout_prob),
BertPredictionHeadTransform(config),
nn.Linear(self.in_dim, self.out_dim),
)
def train_model_bert(args):
# need remake config with device option for train with another cuda device
config = BertConfig.from_pretrained(args.folder_model)
config = config.to_dict()
config.update({"device": args.device})
config.update({"use_pooler": args.use_pooler})
config.update({"weight_class": args.weight_class})
config.update({"output_hidden_states": args.output_hidden_states})
config = BertConfig.from_dict(config)
tokenizer = BertTokenizer.from_pretrained(args.folder_model)
model = BERTQa.from_pretrained(args.folder_model, config=config)
model = model.to(args.device)
train_squad(args, tokenizer, model)
def __init__(self, in_dim=768, out_dim=2, config=None, *args, **kwargs):
super().__init__()
from transformers.models.bert.modeling_bert import BertPredictionHeadTransform
if config is None:
from transformers.configuration_bert import BertConfig
config = BertConfig.from_pretrained("bert-base-uncased")
assert config.hidden_size == in_dim
self.module = nn.Sequential(
nn.Dropout(config.hidden_dropout_prob),
BertPredictionHeadTransform(config),
nn.Linear(in_dim, out_dim),
)
def __init__(self, device, serial_model_path, par_model_path):
self.device = device
pretrained_path = 'cl-tohoku/bert-base-japanese-whole-word-masking'
self.tokenizer = BertTokenizer.from_pretrained(pretrained_path,
do_lower_case=False)
config = BertConfig.from_pretrained(pretrained_path)
config.num_labels = 4
self.serial_model = BertForSequenceClassification(config)
config.num_labels = 2
self.par_model = BertForSequenceClassification(config)
self.serial_model.load_state_dict(torch.load(serial_model_path))
self.serial_model.to(self.device)
self.serial_model.eval()
self.par_model.load_state_dict(torch.load(par_model_path))
self.par_model.to(self.device)
self.par_model.eval()
def loss(self, input_ids, attention_mask, token_type_ids, label):
target = label
final_output = self.compute(input_ids, attention_mask, token_type_ids)
if self.use_pooler:
logits = self.qa_outputs(final_output)
else:
logits = self.qa_outputs_cat(final_output)
class_weights = torch.FloatTensor(self.weight_class).to(self.device)
loss = F.cross_entropy(logits, target, weight=class_weights)
predict_value = torch.max(logits, 1)[1]
list_predict = predict_value.cpu().numpy().tolist()
list_target = target.cpu().numpy().tolist()
return loss, list_predict, list_target
if __name__ == '__main__':
from transformers.configuration_bert import BertConfig
config = BertConfig.from_pretrained("bert-base-multilingual-uncased",
cache_dir="../resources/cache_model")
config = config.to_dict()
config.update({"weight_class": [1, 1]})
config = BertConfig.from_dict(config)
# model = BERTQa.from_pretrained("bert-base-multilingual-uncased",
# cache_dir="../resources/cache_model", config=config)
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--data_dir",
default='./data/input/',
type=str,
required=True,
help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
parser.add_argument("--bert_model", default='bert-base-chinese', type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--config_file", default='bert-base-chinese', type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, "
"bert-base-multilingual-cased, bert-base-chinese.")
parser.add_argument("--task_name",
default='xgfy',
type=str,
required=True,
help="The name of the task to train.")
parser.add_argument("--vacab_root",
default='./data/model/',
type=str,
required=True,
help="The directory where the vocab file is saved.")
parser.add_argument("--output_dir",
default='./data/output/',
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.")
parser.add_argument("--weight_name",
default='net_weight_1.bin',
type=str,
)
parser.add_argument("--config_name",
default='config_name_1.bin',
type=str,
)
# Other parameters
parser.add_argument("--cache_dir",
default="./data/model/",
type=str,
help="Where do you want to store the pre-trained models downloaded from s3")
parser.add_argument("--max_seq_length",
default=128,
type=int,
help="The maximum total input sequence length after WordPiece tokenization. \n"
"Sequences longer than this will be truncated, and sequences shorter \n"
"than this will be padded.")
parser.add_argument("--do_train",
action='store_true',
help="Whether to run training.")
parser.add_argument("--do_eval",
action='store_true',
help="Whether to run eval on the dev set.")
parser.add_argument("--do_lower_case",
action='store_true',
help="Set this flag if you are using an uncased model.")
parser.add_argument("--train_batch_size",
default=32,
type=int,
help="Total batch size for training.")
parser.add_argument("--eval_batch_size",
default=8,
type=int,
help="Total batch size for eval.")
parser.add_argument("--log_frq",
default=50,
type=int)
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs",
default=1.0,
type=int,
help="Total number of training epochs to perform.")
parser.add_argument("--n_warmup",
default=1000,
type=int,
help="step of training to perform linear learning rate warmup for.")
parser.add_argument("--no_cuda",
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument('--fp16',
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--parall',
action='store_true')
parser.add_argument('--loss_scale',
type=float, default=0,
help="Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n"
"0 (default value): dynamic loss scaling.\n"
"Positive power of 2: static loss scaling value.\n")
args = parser.parse_args()
# 新冠肺炎
processors = {
"xgfy": SimProcessor
}
num_labels_task = {
"xgfy": 2,
}
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda:0" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
# torch.distributed.init_process_group(backend='nccl')
logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}".format(
device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = args.train_batch_size // args.gradient_accumulation_steps
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_eval:
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
# if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train:
# raise ValueError("Output directory ({}) already exists and is not empty.".format(args.output_dir))
# if not os.path.exists(args.output_dir):
# os.makedirs(args.output_dir)
task_name = args.task_name.lower()
if task_name not in processors:
raise ValueError("Task not found: %s" % (task_name))
processor = processors[task_name]
num_labels = num_labels_task[task_name]
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(args.vacab_root, do_lower_case=args.do_lower_case)
train_examples = None
num_train_optimization_steps = None
if args.do_train:
train_examples = processor.get_train_examples(args.data_dir)
num_train_optimization_steps = int(
len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps) * args.num_train_epochs
if args.local_rank != -1:
num_train_optimization_steps = num_train_optimization_steps // torch.distributed.get_world_size()
# Prepare model
cache_dir = args.cache_dir if args.cache_dir else os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{0}')
# cache_dir = args.cache_dir if args.cache_dir else os.path.join(PYTORCH_PRETRAINED_BERT_CACHE, 'distributed_{}'.format(str(args.local_rank)))
config = BertConfig.from_pretrained(args.config_file, num_labels=num_labels)
model = BertForSequenceClassification.from_pretrained(args.bert_model,
config=config,
cache_dir=cache_dir)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
try:
from apex.parallel import DistributedDataParallel as DDP
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")
model = DDP(model)
elif n_gpu > 1 and args.parall:
model = torch.nn.DataParallel(model)
# Prepare optimizer
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
if args.fp16:
try:
from apex.optimizers import FP16_Optimizer
from apex.optimizers import FusedAdam
except ImportError:
raise ImportError(
"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")
optimizer = FusedAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
bias_correction=False,
max_grad_norm=1.0)
if args.loss_scale == 0:
optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
else:
optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)
else:
optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate)
global_step = 0
nb_tr_steps = 0
tr_loss = 0
if args.do_train:
train_features = convert_examples_to_features(
train_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_examples))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_optimization_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
scheduler = get_linear_schedule_with_warmup(
optimizer, num_warmup_steps=args.n_warmup, num_training_steps=t_total
)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
tr_loss = 0
nb_tr_examples, nb_tr_steps = 0, 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
batch = tuple(t.to(device) for t in batch)
input_ids, input_mask, segment_ids, label_ids = batch
loss, _ = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids)
if n_gpu > 1 and args.parall:
loss = loss.mean() # mean() to average on multi-gpu.
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
if args.fp16:
optimizer.backward(loss)
else:
loss.backward()
tr_loss += loss.item()
nb_tr_examples += input_ids.size(0)
nb_tr_steps += 1
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16:
# modify learning rate with special warm up BERT uses
# if args.fp16 is False, BertAdam is used that handles this automatically
for param_group in optimizer.param_groups:
param_group['lr'] = lr_this_step
optimizer.step()
scheduler.step()
optimizer.zero_grad()
global_step += 1
if (global_step) % args.log_frq == 0:
logger.info("TrLoss: {:.2f} | Loss: {:.2f} | Lr: {:.2f}".format(tr_loss, loss.item(), scheduler.get_lr()[0]))
if args.do_train:
# Save a trained model and the associated configuration
model_to_save = model.module if hasattr(model, 'module') else model # Only save the model it-self
output_model_file = os.path.join(args.output_dir, args.weight_name)
torch.save(model_to_save.state_dict(), output_model_file)
output_config_file = os.path.join(args.output_dir, args.config_name)
with open(output_config_file, 'w') as f:
f.write(model_to_save.config.to_json_string())
# Load a trained model and config that you have fine-tuned
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config)
model.load_state_dict(torch.load(output_model_file))
else:
output_model_file = os.path.join(args.output_dir, args.weight_name)
output_config_file = os.path.join(args.output_dir, args.config_name)
config = BertConfig(output_config_file)
model = BertForSequenceClassification(config)
model.load_state_dict(torch.load(output_model_file))
# model = BertForSequenceClassification.from_pretrained(args.bert_model)
model.to(device)
if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
eval_examples = processor.get_dev_examples(args.data_dir)
eval_features = convert_examples_to_features(
eval_examples, label_list, args.max_seq_length, tokenizer)
logger.info("***** Running evaluation *****")
logger.info(" Num examples = %d", len(eval_examples))
logger.info(" Batch size = %d", args.eval_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
# Run prediction for full data
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(eval_dataloader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss, logits = model(input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
loss = tr_loss / nb_tr_steps if args.do_train else None
result = {'eval_loss': eval_loss,
'eval_accuracy': eval_accuracy,
'global_step': global_step,
'loss': loss}
logger.info(result)
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
def init_encoder(cls, args, dropout: float = 0.1):
cfg = BertConfig.from_pretrained("bert-base-uncased")
if dropout != 0:
cfg.attention_probs_dropout_prob = dropout
cfg.hidden_dropout_prob = dropout
return cls.from_pretrained("bert-base-uncased", config=cfg)
def train(args, model_name_or_path, train_data, train_dataloader, valid_data,
valid_dataloader):
pro = processer()
labellist = pro.get_labels()
trainloss = TrainLoss()
#*****加载模型*****
model = BertForSequenceClassification
config = BertConfig.from_pretrained(model_name_or_path,
num_labels=len(labellist))
model = model.from_pretrained(model_name_or_path, config=config)
# *****模型加载到设备*****
if torch.cuda.is_available():
# 单GPU计算
torch.cuda.set_device(0)
device = torch.device('cuda', 0) # 设置GPU设备号
else:
device = torch.device('cpu')
model.to(device)
#*****优化函数*****
t_total = len(train_dataloader
) // args.gradient_accumulation_steps * args.num_train_epochs
warmup_steps = int(t_total * args.warmup_proportion)
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params': [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
'weight_decay':
args.weight_decay
}, {
'params': [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
'weight_decay':
0.0
}]
optimizer = AdamW(optimizer_grouped_parameters,
lr=args.learning_rate,
eps=args.adam_epsilon)
scheduler = WarmupLinearSchedule(optimizer,
warmup_steps=warmup_steps,
t_total=t_total)
#*****训练过程相关信息*****
logger.info("***** Running training *****")
logger.info(" Num examples = %d", len(train_data))
logger.info(" Num Epochs = %d", args.num_train_epochs)
logger.info(" Instantaneous batch size per GPU = %d",
args.train_batch_size)
logger.info(" Gradient Accumulation steps = %d",
args.gradient_accumulation_steps)
logger.info(" Total optimization steps = %d", t_total)
#*****开始训练*****
tr_loss, logging_loss = 0.0, 0.0
model.zero_grad()
seed_everything(args.seed)
for num in range(args.num_train_epochs):
train_all_steps = 0
train_steps = []
train_losses = []
global_step = 0
logger.info(f'****************Train epoch-{num}****************')
pbar = ProgressBar(n_total=len(train_dataloader), desc='Train')
for step, batch in enumerate(train_dataloader):
#***存储step用于绘制Loss曲线***
train_all_steps += 1
train_steps.append(train_all_steps)
model.train()
#***输入模型进行计算***
batch = tuple(t.to(device) for t in batch)
inputs = {
'input_ids': batch[0],
'attention_mask': batch[1],
'token_type_ids': batch[2],
'labels': batch[3]
}
outputs = model(
**inputs) #模型原文件中已经使用损失函数对输出值和标签值进行了计算，返回的outputs中包含损失函数值
#***损失函数值反向传播***
loss = outputs[0]
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm) #梯度裁剪
#***存储loss用于绘制loss曲线***
train_losses.append(loss.detach().cpu().numpy())
#***优化器进行优化***
pbar(step, {'loss': loss.item()})
tr_loss += loss.item()
if (step + 1) % args.gradient_accumulation_steps == 0:
optimizer.step() #优化器优化
scheduler.step() #学习率机制更新
model.zero_grad()
global_step += 1
#训练一个epoch保存一个模型
output_dir = os.path.join(args.output_dir,
f'model_checkpoint_epoch_{num}')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('') #避免输出信息都在同一行
# logger.info(f'save model checkpoint-{global_step} to {output_dir} ')
model.save_pretrained(output_dir) #保存模型
#***训练一个epoch绘制一个Loss曲线***
trainloss.train_loss(steps=train_steps,
losses=train_losses,
epoch=num,
args=args,
type='train',
max_step=train_all_steps)
#*****一个epoch训练结束以后，进行验证*****
print('')
logger.info(f'****************Valid epoch-{num}****************')
logger.info(" Num examples = %d", len(valid_data))
logger.info(" Batch size = %d", args.valid_batch_size)
valid_steps, valid_losses, valid_all_steps = valid(
args=args,
model=model,
device=device,
valid_data=valid_data,
valid_dataloader=valid_dataloader)
trainloss.train_loss(steps=valid_steps,
losses=valid_losses,
epoch=num,
args=args,
type='valid',
max_steps=valid_all_steps)
#每训练一个epoch清空cuda缓存
if 'cuda' in str(device):
torch.cuda.empty_cache()
path_input_test_data = "../module_dataset/dataset/dataset_preprocess/pair_sequence/test_data/" \
"private_test_pair_without_punc.csv"
no_cuda = False
n_gpu = 1
device = "cuda:0"
seed = 42
max_seq_length = 400
max_query_length = 64
weight_class = [1, 1]
args = Args()
device = torch.device(args.device)
tokenizer = BertTokenizer.from_pretrained(args.folder_model,
do_lower_case=args.do_lower_case)
config = BertConfig.from_pretrained(args.folder_model)
# # custom some parameter for custom bert
config = config.to_dict()
config.update({"device": args.device})
config = BertConfig.from_dict(config)
model = BERTQa.from_pretrained(args.folder_model, config=config)
model = model.to(device)
get_predict_dl(model, tokenizer, args)
#%%
import torch
from torch.optim import Adam
from transformers.configuration_albert import AlbertConfig
from transformers.configuration_bert import BertConfig
from src.dataloader.Dataset import EETaskDataloader
from src.dataloader.utils import load_data
from src.model.AlbertCRF import AlbertCrfForNer
from src.model.BertCRF import BertCrfForNer
from src.model.BertSoftMax import BertSoftmaxForNer
from src.util.EETaskRun import Run
from src.util.extract_arguments import extract_arguments_crf, extract_arguments_softmax
from src.util.utils import lcs
#%%
config = BertConfig.from_pretrained(
r"/home/longred/lic2020_baselines/chinese_L-12_H-768_A-12/bert-base-chinese-config.json"
)
config.pretrained_path = r"/home/longred/lic2020_baselines/chinese_L-12_H-768_A-12/bert-base-chinese-pytorch_model.bin"
config.vocab_path = r"/home/longred/lic2020_baselines/chinese_L-12_H-768_A-12/vocab.txt"
config.train_data_path = r"/home/longred/EETask/data/train.json"
config.batch_size = 32
config.event_schema_path = r"/home/longred/EETask/data/event_schema.json"
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
EE = EETaskDataloader(config)
train_loader = EE.get_train_data_loader()
config.num_labels = EE.num_labels
config.label2id = EE.label2id
data = load_data("/home/longred/EETask/data/dev.json")
model = BertCrfForNer.from_pretrained(
pretrained_model_name_or_path=config.pretrained_path,