def __init__(self,
args,
train_dataset=None,
dev_dataset=None,
test_dataset=None):
self.args = args
self.train_dataset = train_dataset
self.dev_dataset = dev_dataset
self.test_dataset = test_dataset
self.intent_label_lst = get_intent_labels(args)
self.slot_label_lst = get_slot_labels(args)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
self.pad_token_label_id = args.ignore_index
"""model_class选择我们自己的JointBert， Config_Class以config基类配置加入我们自己finetuning-task"""
self.config_class, self.model_class, _ = MODEL_CLASSES[args.model_type]
self.bert_config = self.config_class.from_pretrained(
args.model_name_or_path, finetuning_task=args.task)
self.model = self.model_class(self.bert_config, args,
self.intent_label_lst,
self.slot_label_lst)
"""根据do_pred来选择load的model|^"""
# GPU or CPU
self.device = "cuda" if torch.cuda.is_available(
) and not args.no_cuda else "cpu"
self.model.to(self.device)
def __init__(
self,
config: argparse,
ner_train_dataloader: DataLoader,
ner_val_dataloader: DataLoader,
ner_test_dataloader: DataLoader,
):
super().__init__()
self.config = config
self.ner_train_dataloader = ner_train_dataloader
self.ner_val_dataloader = ner_val_dataloader
self.ner_test_dataloader = ner_test_dataloader
self.ignore_index = torch.nn.CrossEntropyLoss().ignore_index
self.total_intent_labels = get_intent_labels(
os.path.join(self.config.data_path, self.config.task))
self.total_slot_labels = get_slot_labels(
os.path.join(self.config.data_path, self.config.task))
self.bert_config = BertConfig.from_pretrained(self.config.bert_model)
self.model = BertModel.from_pretrained(self.config.bert_model)
self.intent_dropout = nn.Dropout(self.config.dropout_rate)
self.intent_linear = nn.Linear(self.bert_config.hidden_size,
len(self.total_intent_labels))
self.slot_dropout = nn.Dropout(self.config.dropout_rate)
self.slot_linear = nn.Linear(self.bert_config.hidden_size,
len(self.total_slot_labels))
def __init__(self,
args,
train_dataset=None,
dev_dataset=None,
test_dataset=None):
self.args = args
self.train_dataset = train_dataset
self.dev_dataset = dev_dataset
self.test_dataset = test_dataset
self.intent_label_lst = get_intent_labels(args) #获取所有意图标签
# self.slot_label_lst = get_slot_labels(args)#获取所有槽值标签
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
self.pad_token_label_id = args.ignore_index # -100
self.best_f1 = 0
self.best_epoch = 0
self.config_class, self.model_class, _ = MODEL_CLASSES[
args.model_type] #BertConfig, JointBERT， BertTokenizer
self.bert_config = self.config_class.from_pretrained(
args.model_name_or_path, finetuning_task=args.task)
#here
self.model = self.model_class(
self.bert_config, args, self.intent_label_lst
) #JointBERT(BertConfig, args, intent_label_lst, slot_label_lst)
# self.model = self.model_class(self.bert_config, args, self.slot_label_lst) # JointBERT(BertConfig, args, intent_label_lst, slot_label_lst)
# GPU or CPU
self.device = "cuda" if torch.cuda.is_available(
) and not args.no_cuda else "cpu"
self.model.to(self.device)
def __init__(self, args, train_dataset=None, dev_dataset=None, test_dataset=None):
self.args = args
self.train_dataset = train_dataset
self.dev_dataset = dev_dataset
self.test_dataset = test_dataset
self.intent_label_lst = get_intent_labels(args)
self.slot_label_lst = get_slot_labels(args)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
self.pad_token_label_id = args.ignore_index
self.config_class, self.model_class, _ = MODEL_CLASSES[args.model_type]
self.config = self.config_class.from_pretrained(args.model_name_or_path, finetuning_task=args.task)
self.model = self.model_class.from_pretrained(args.model_name_or_path,
config=self.config,
args=args,
intent_label_lst=self.intent_label_lst,
slot_label_lst=self.slot_label_lst)
# GPU or CPU
torch.cuda.set_device(self.args.gpu_id)
print(self.args.gpu_id)
print(torch.cuda.current_device())
self.device = "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu"
self.model.to(self.device)
def __init__(self, args):
self.args = args
self.intent_vocab = get_intent_labels(args)
self.slot_vocab = get_slot_labels(args)
self.input_text_file = 'seq.in'
self.slot_labels_file = 'seq.out'
self.intent_label_file = 'label'
def __init__(self,
args,
train_dataset=None,
val_dataset=None,
test_dataset=None):
self.args = args
self.train_dataset = train_dataset
self.val_dataset = val_dataset
self.test_dataset = test_dataset
self.intent_label_lst = get_intent_labels(args)
self.slot_label_lst = get_slot_labels(args)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
self.pad_token_label_id = args.ignore_index
self.config_class, self.model_class = BertConfig, JointBERT
self.config = self.config_class.from_pretrained(
args.model_name_or_path)
self.model = self.model_class.from_pretrained(
args.model_name_or_path,
config=self.config,
args=args,
intent_label_lst=self.intent_label_lst,
slot_label_lst=self.slot_label_lst)
# GPU or CPU
self.device = "cuda" if torch.cuda.is_available(
) and not args.no_cuda else "cpu"
self.model.to(self.device)
def __init__(self, args):
self.args = args
self.intent_labels = get_intent_labels(args) #获得文档中的意图标签
# self.slot_labels = get_slot_labels(args)#获得文档中的槽标签
self.input_text_file = 'seq.in' #输入句子
self.intent_label_file = 'label.txt' #句子标签
self.slot_labels_file = 'seq.out' #句子槽值
def __init__(self, args):
self.args = args
self.intent_labels = get_intent_labels(
args) # 读取intent_label.txt获取unique的label
self.slot_labels = get_slot_labels(args)
self.input_text_file = 'seq.in'
self.intent_label_file = 'label'
self.slot_labels_file = 'seq.out'
def main(pred_config):
init_logger()
device = set_device(pred_config)
# loading train Model args
args_path = os.path.join(pred_config.model_dir, 'train_args.bin')
train_args = torch.load(args_path)
# load labels
intent_vocab = get_intent_labels(train_args)
slot_vocab = get_slot_labels(train_args)
# load preain Model
model = load_pretrainModel(pred_config, train_args, len(intent_vocab),
len(slot_vocab))
model.to(device)
# read data
examples = read_file(pred_config)
pad_label_id = pred_config.pad_label_id
logger.info(f'Start to predict using {pred_config.model_type}')
if pred_config.model_type.endswith('S2S'):
# convert data to tensor
tokenizer = get_word_vocab(train_args)
dataset = convert_input_file_to_RnnTensor(examples,
tokenizer,
train_args,
pred_config,
pad_token_id=pad_label_id)
# get predict!
intent_preds, slot_preds, slot_masks = get_s2s_predict(
model, dataset, pred_config, train_args, slot_vocab, device)
elif pred_config.model_type.endswith('bert'):
# convert data to tensor
tokenizer = load_tokenizer(train_args)
dataset = convert_input_file_to_BertTensor(examples,
tokenizer,
train_args,
pred_config,
pad_label_id=pad_label_id)
# get predict!
intent_preds, slot_preds, slot_masks = get_pretrain_predict(
model, dataset, pred_config, train_args, device)
logger.info('***Display PredictInfo***')
logger.info(f'Predict number:{len(dataset)}')
logger.info(f'Predict max_seqLen:{train_args.max_seqLen}')
logger.info(f'Whether to use CRF:{train_args.use_crf}')
intent_labels, slot_labels = convert_to_labels(intent_vocab, slot_vocab,
intent_preds, slot_preds,
slot_masks, pad_label_id)
# output to file
write_predsFile(pred_config, examples, intent_labels, slot_labels)
def __init__(self, config, preprocessor, mode):
self.config = config
self.mode = mode
self.sentence_list = []
self.tag_list = []
self.intent_list = []
self.preprocessor = preprocessor
self.data_path = os.path.join(self.config.data_path, self.config.task)
self.intent_labels = get_intent_labels(self.data_path)
self.slot_labels = get_slot_labels(self.data_path)
self.load_data()
def __init__(self, args):
self.args = args
self.intent_labels = get_intent_labels(args)
self.slot_labels = get_slot_labels(args)
self.wn_concept_id2name_dict = get_wn_concept_id2name_dict(args)
self.wn_concept_names = get_wn_concept_vector_names(args)
self.nell_concept_names = get_nell_concept_vector_names(args)
self.input_text_file = 'seq.in'
self.intent_label_file = 'label'
self.slot_labels_file = 'seq.out'
self.wn_synsets_file = 'seq.synsets-full' # if args.full else 'seq.synsets-partial'
self.nell_entities_file = 'seq.entities-full' # if args.full else 'seq.entities-partial'
def __init__(self, args, model, test_data):
self.args = args
self.device = args.device
self.model = model
self.test_text = [
line.strip()
for line in open(os.path.join(self.args.data_dir, 'dataset',
self.args.task, 'test', 'seq.in'),
"r",
encoding="utf-8")
]
self.test_data = test_data
self.intent_label_lst = get_intent_labels(args)
self.slot_label_lst = get_slot_labels(args)
# Use cross entropy ignore index as padding label id so that only real label ids contribute to the loss later
self.pad_token_label_id = args.ignore_index
def load_model(pred_config, args, device):
# Check whether model exists
if not os.path.exists(pred_config.model_dir):
raise Exception("Model doesn't exists! Train first!")
try:
model = MODEL_CLASSES[args.model_type][1].from_pretrained(
args.model_dir,
args=args,
intent_label_lst=get_intent_labels(args),
slot_label_lst=get_slot_labels(args))
model.to(device)
model.eval()
logger.info("***** Model Loaded *****")
except:
raise Exception("Some model files might be missing...")
return model
def __init__(self,
train_set,
val_set,
test_set,
args,
model=None,
pretrained_path=None):
self.train_set = train_set
self.val_set = val_set
self.test_set = test_set
self.args = args
self.word_vocab = utils.get_word_vocab(args)
self.intent_vocab = get_intent_labels(args)
self.slot_vocab = get_slot_labels(args)
self.device = torch.device('cuda:0') if torch.cuda.is_available() \
and not args.no_cuda else torch.device('cpu')
self.model = model
def predict(pred_config):
# load model and args
args = get_args(pred_config)
device = get_device(pred_config)
model = load_model(pred_config, args, device)
logger.info(args)
intent_label_lst = get_intent_labels(args)
slot_label_lst = get_slot_labels(args)
# Convert input file to TensorDataset
pad_token_label_id = args.ignore_index
tokenizer = load_tokenizer(args)
lines = read_input_file(pred_config)
dataset = convert_input_file_to_tensor_dataset(lines, pred_config, args,
tokenizer,
pad_token_label_id)
# Predict
sampler = SequentialSampler(dataset)
data_loader = DataLoader(dataset,
sampler=sampler,
batch_size=pred_config.batch_size)
all_slot_label_mask = None
intent_preds = None
slot_preds = None
for batch in tqdm(data_loader, desc="Predicting"):
batch = tuple(t.to(device) for t in batch)
with torch.no_grad():
inputs = {
"input_ids": batch[0],
"attention_mask": batch[1],
"intent_label_ids": None,
"slot_labels_ids": None
}
if args.model_type != "distilbert":
inputs["token_type_ids"] = batch[2]
outputs = model(**inputs)
_, (intent_logits, slot_logits) = outputs[:2]
# Intent Prediction
if intent_preds is None:
intent_preds = intent_logits.detach().cpu().numpy()
else:
intent_preds = np.append(intent_preds,
intent_logits.detach().cpu().numpy(),
axis=0)
# Slot prediction
if slot_preds is None:
if args.use_crf:
# decode() in `torchcrf` returns list with best index directly
slot_preds = np.array(model.crf.decode(slot_logits))
else:
slot_preds = slot_logits.detach().cpu().numpy()
all_slot_label_mask = batch[3].detach().cpu().numpy()
else:
if args.use_crf:
slot_preds = np.append(slot_preds,
np.array(
model.crf.decode(slot_logits)),
axis=0)
else:
slot_preds = np.append(slot_preds,
slot_logits.detach().cpu().numpy(),
axis=0)
all_slot_label_mask = np.append(
all_slot_label_mask,
batch[3].detach().cpu().numpy(),
axis=0)
intent_preds = np.argmax(intent_preds, axis=1)
if args.use_crf:
slot_preds = np.array(model.crf.decode(slot_logits))
else:
slot_preds = np.argmax(slot_preds, axis=2)
slot_label_map = {i: label for i, label in enumerate(slot_label_lst)}
slot_preds_list = [[] for _ in range(slot_preds.shape[0])]
for i in range(slot_preds.shape[0]):
for j in range(slot_preds.shape[1]):
if all_slot_label_mask[i, j] != pad_token_label_id:
slot_preds_list[i].append(slot_label_map[slot_preds[i][j]])
# Write to output file
with open(pred_config.output_file, "w", encoding="utf-8") as f:
for words, slot_preds, intent_pred in zip(lines, slot_preds_list,
intent_preds):
line = ""
for word, pred in zip(words, slot_preds):
if pred == 'O':
line = line + word + " "
else:
line = line + "[{}:{}] ".format(word, pred)
f.write("<{}> -> {}\n".format(intent_label_lst[intent_pred],
line.strip()))
logger.info("Prediction Done!")
def __init__(self, args):
self.args = args
self.intent_labels = get_intent_labels(args)
self.slot_labels = get_slot_labels(args)
