def evaluate_model(config: Config, model: NNCRF, loader: DataLoader, name: str, insts: List[Instance], print_each_type_metric: bool = False):
## evaluation
p_dict, total_predict_dict, total_entity_dict = Counter(), Counter(), Counter()
batch_id = 0
batch_size = loader.batch_size
dev = config.device
with torch.no_grad():
for iter, batch in tqdm(enumerate(loader, 1), desc="--evaluating batch", total=len(loader)):
one_batch_insts = insts[batch_id * batch_size:(batch_id + 1) * batch_size]
batch_max_scores, batch_max_ids = model.decode(words = batch.words.to(dev), word_seq_lens = batch.word_seq_len.to(dev),
context_emb=batch.context_emb.to(dev) if batch.context_emb is not None else None,
chars = batch.chars.to(dev), char_seq_lens = batch.char_seq_lens.to(dev))
batch_p , batch_predict, batch_total = evaluate_batch_insts(one_batch_insts, batch_max_ids, batch.labels, batch.word_seq_len, config.idx2labels)
p_dict += batch_p
total_predict_dict += batch_predict
total_entity_dict += batch_total
batch_id += 1
if print_each_type_metric:
for key in total_entity_dict:
precision_key, recall_key, fscore_key = get_metric(p_dict[key], total_entity_dict[key], total_predict_dict[key])
print(f"[{key}] Prec.: {precision_key:.2f}, Rec.: {recall_key:.2f}, F1: {fscore_key:.2f}")
total_p = sum(list(p_dict.values()))
total_predict = sum(list(total_predict_dict.values()))
total_entity = sum(list(total_entity_dict.values()))
precision, recall, fscore = get_metric(total_p, total_entity, total_predict)
print(colored(f"[{name} set Total] Prec.: {precision:.2f}, Rec.: {recall:.2f}, F1: {fscore:.2f}", 'blue'), flush=True)
return [precision, recall, fscore]
def train_model(config: Config, epoch: int, train_loader: DataLoader, dev_loader: DataLoader, test_loader: DataLoader):
### Data Processing Info
train_num = len(train_loader.dataset)
print("number of instances: %d" % (train_num))
model = NNCRF(config)
optimizer = get_optimizer(config, model)
dev = config.device
model.to(config.device)
best_dev = [-1, 0]
best_test = [-1, 0]
model_folder = config.model_folder
res_folder = "results"
if os.path.exists("model_files/" + model_folder):
raise FileExistsError(
f"The folder model_files/{model_folder} exists. Please either delete it or create a new one "
f"to avoid override.")
model_path = f"model_files/{model_folder}/lstm_crf.m"
config_path = f"model_files/{model_folder}/config.conf"
res_path = f"{res_folder}/{model_folder}.results"
print("[Info] The model will be saved to: %s.tar.gz" % (model_folder))
os.makedirs(f"model_files/{model_folder}", exist_ok= True) ## create model files. not raise error if exist
os.makedirs(res_folder, exist_ok=True)
no_incre_dev = 0
print(colored(f"[Train Info] Start training, you have set to stop if performace not increase for {config.max_no_incre} epochs",'red'))
for i in tqdm(range(1, epoch + 1), desc="Epoch"):
epoch_loss = 0
start_time = time.time()
model.zero_grad()
model.train()
if config.optimizer.lower() == "sgd":
optimizer = lr_decay(config, optimizer, i)
for iter, batch in tqdm(enumerate(train_loader, 1), desc="--training batch", total=len(train_loader)):
loss = model(words = batch.words.to(dev), word_seq_lens = batch.word_seq_len.to(dev),
context_emb= batch.context_emb.to(dev) if batch.context_emb is not None else None,
chars = batch.chars.to(dev), char_seq_lens = batch.char_seq_lens.to(dev), labels = batch.labels.to(dev))
epoch_loss += loss.item()
loss.backward()
optimizer.step()
optimizer.zero_grad()
model.zero_grad()
end_time = time.time()
print("Epoch %d: %.5f, Time is %.2fs" % (i, epoch_loss, end_time - start_time), flush=True)
model.eval()
dev_metrics = evaluate_model(config, model, dev_loader, "dev", dev_loader.dataset.insts)
test_metrics = evaluate_model(config, model, test_loader, "test", test_loader.dataset.insts)
if dev_metrics[2] > best_dev[0]:
print("saving the best model...")
no_incre_dev = 0
best_dev[0] = dev_metrics[2]
best_dev[1] = i
best_test[0] = test_metrics[2]
best_test[1] = i
torch.save(model.state_dict(), model_path)
# Save the corresponding config as well.
f = open(config_path, 'wb')
pickle.dump(config, f)
f.close()
write_results(res_path, test_loader.dataset.insts)
else:
no_incre_dev += 1
model.zero_grad()
if no_incre_dev >= config.max_no_incre:
print("early stop because there are %d epochs not increasing f1 on dev"%no_incre_dev)
break
print("Archiving the best Model...")
with tarfile.open(f"model_files/{model_folder}/{model_folder}.tar.gz", "w:gz") as tar:
tar.add(f"model_files/{model_folder}", arcname=os.path.basename(model_folder))
print("Finished archiving the models")
print("The best dev: %.2f" % (best_dev[0]))
print("The corresponding test: %.2f" % (best_test[0]))
print("Final testing.")
model.load_state_dict(torch.load(model_path))
model.eval()
evaluate_model(config, model, test_loader, "test", test_loader.dataset.insts)
write_results(res_path, test_loader.dataset.insts)
test_dataloader = DataLoader(test_dataset,
shuffle=False,
num_workers=num_workers,
collate_fn=test_dataset.collate_fn)
parser = argparse.ArgumentParser(description="LSTM CRF implementation")
opt = parse_arguments(parser)
conf = Config(opt)
conf.build_emb_table(word2idx=word2idx)
#carregando modelo
folder_name = opt.model_path
f = open(folder_name + "/config.conf", 'rb')
model = NNCRF(pickle.load(f))
model.load_state_dict(
torch.load(folder_name + "/lstm_crf.m", map_location="cpu"))
model.eval()
batch_size = test_dataloader.batch_size
#avaliando
p_dict, total_predict_dict, total_entity_dict = Counter(), Counter(), Counter()
insts = test_dataloader.dataset.insts
batch_id = 0
dev = "cpu"
# arquivo passado como argumento para eval para escrever os resultados
f = open(opt.results_path, 'a')
f2 = open('data/embeddings_contextuais/cojur_embeddings_100d.txt', 'a')