def test_model(config: Config, test_insts):
dep_model_name = config.dep_model.name
if config.dep_model == DepModelType.dggcn:
dep_model_name += '(' + str(config.num_gcn_layers) + ","+str(config.gcn_dropout)+ ","+str(config.gcn_mlp_layers)+")"
model_name = "model_files/lstm_{}_{}_crf_{}_{}_{}_dep_{}_elmo_{}_{}_gate_{}_base_{}_epoch_{}_lr_{}_doubledep_{}_comb_{}.m".format(config.num_lstm_layer, config.hidden_dim,
config.dataset, config.affix,
config.train_num,
dep_model_name,
config.context_emb.name,
config.optimizer.lower(),
config.edge_gate,
config.num_base,
config.num_epochs,
config.learning_rate, config.double_dep_label, config.interaction_func)
res_name = "results/lstm_{}_{}_crf_{}_{}_{}_dep_{}_elmo_{}_{}_gate_{}_base_{}_epoch_{}_lr_{}_doubledep_{}_comb_{}.results".format(config.num_lstm_layer, config.hidden_dim,
config.dataset, config.affix,
config.train_num,
dep_model_name,
config.context_emb.name,
config.optimizer.lower(),
config.edge_gate,
config.num_base,
config.num_epochs,
config.learning_rate, config.double_dep_label, config.interaction_func)
model = NNCRF(config)
model.load_state_dict(torch.load(model_name))
model.eval()
test_batches = batching_list_instances(config, test_insts)
evaluate(config, model, test_batches, "test", test_insts)
write_results(res_name, test_insts)
def test_model(config: Config, test_insts):
model_name = "model_files/lstm_{}_crf_{}_{}_dep_{}_elmo_{}_lr_{}.m".format(
config.hidden_dim,
config.dataset, config.train_num, config.context_emb.name,
config.optimizer.lower(), config.learning_rate)
res_name = "results/lstm_{}_crf_{}_{}_dep_{}_elmo_{}_lr_{}.results".format(
config.hidden_dim,
config.dataset, config.train_num, config.context_emb.name,
config.optimizer.lower(), config.learning_rate)
model = NNCRF(config)
model.load_state_dict(torch.load(model_name))
model.eval()
test_batches = batching_list_instances(config, test_insts)
evaluate_model(config, model, test_batches, "test", test_insts)
write_results(res_name, test_insts)
def test_model(config: Config, test_insts):
dep_model_name = config.dep_model.name
if config.dep_model == DepModelType.dggcn:
dep_model_name += '(' + str(config.num_gcn_layers) + "," + str(
config.gcn_dropout) + "," + str(config.gcn_mlp_layers) + ")"
model_name = "model_files/gcn_{}_hidden_{}_dataset_{}_{}_context_{}.m".format(
config.num_gcn_layers, config.hidden_dim, config.dataset, config.affix,
config.context_emb.name)
res_name = "results/gcn_{}_hidden_{}_dataset_{}_{}_context_{}.results".format(
config.num_gcn_layers, config.hidden_dim, config.dataset, config.affix,
config.context_emb.name)
model = NNCRF(config)
model.load_state_dict(torch.load(model_name))
model.eval()
test_batches = batching_list_instances(config, test_insts)
evaluate(config, model, test_batches, "test", test_insts)
write_results(res_name, test_insts)
def learn_from_insts(config: Config, epoch: int, train_insts, dev_insts,
test_insts):
# train_insts: List[Instance], dev_insts: List[Instance], test_insts: List[Instance], batch_size: int = 1
model = NNCRF(config)
optimizer = get_optimizer(config, model)
train_num = len(train_insts)
print("number of instances: %d" % (train_num))
print(colored("[Shuffled] Shuffle the training instance ids", "red"))
random.shuffle(train_insts)
batched_data = batching_list_instances(config, train_insts)
dev_batches = batching_list_instances(config, dev_insts)
test_batches = batching_list_instances(config, test_insts)
best_dev = [-1, 0]
best_test = [-1, 0]
model_folder = "model_files"
res_folder = "results"
model_name = model_folder + "/lstm_{}_crf_{}_{}_dep_{}_elmo_{}_lr_{}.m".format(
config.hidden_dim,
config.dataset, config.train_num, config.context_emb.name,
config.optimizer.lower(), config.learning_rate)
res_name = res_folder + "/lstm_{}_crf_{}_{}_dep_{}_elmo_{}_lr_{}.results".format(
config.hidden_dim,
config.dataset, config.train_num, config.context_emb.name,
config.optimizer.lower(), config.learning_rate)
print("[Info] The model will be saved to: %s" % (model_name))
if not os.path.exists(model_folder):
os.makedirs(model_folder)
if not os.path.exists(res_folder):
os.makedirs(res_folder)
for i in range(1, epoch + 1):
epoch_loss = 0
start_time = time.time()
model.zero_grad()
if config.optimizer.lower() == "sgd":
optimizer = lr_decay(config, optimizer, i)
for index in np.random.permutation(len(batched_data)):
# for index in range(len(batched_data)):
model.train()
batch_word, batch_wordlen, batch_context_emb, batch_char, batch_charlen, batch_label = batched_data[
index]
loss = model.neg_log_obj(batch_word, batch_wordlen,
batch_context_emb, batch_char,
batch_charlen, batch_label)
epoch_loss += loss.item()
loss.backward()
# # torch.nn.utils.clip_grad_norm_(model.parameters(), config.clip) ##clipping the gradient
optimizer.step()
model.zero_grad()
end_time = time.time()
print("Epoch %d: %.5f, Time is %.2fs" %
(i, epoch_loss, end_time - start_time),
flush=True)
model.eval()
dev_metrics = evaluate_model(config, model, dev_batches, "dev",
dev_insts)
test_metrics = evaluate_model(config, model, test_batches, "test",
test_insts)
if dev_metrics[2] > best_dev[0]:
print("saving the best model...")
best_dev[0] = dev_metrics[2]
best_dev[1] = i
best_test[0] = test_metrics[2]
best_test[1] = i
torch.save(model.state_dict(), model_name)
write_results(res_name, test_insts)
model.zero_grad()
print("The best dev: %.2f" % (best_dev[0]))
print("The corresponding test: %.2f" % (best_test[0]))
print("Final testing.")
model.load_state_dict(torch.load(model_name))
model.eval()
evaluate_model(config, model, test_batches, "test", test_insts)
write_results(res_name, test_insts)
def learn_from_insts(config: Config, epoch: int, train_insts, dev_insts,
test_insts):
# train_insts: List[Instance], dev_insts: List[Instance], test_insts: List[Instance], batch_size: int = 1
model = NNCRF(config)
optimizer = get_optimizer(config, model)
train_num = len(train_insts)
print("number of instances: %d" % (train_num))
print(colored("[Shuffled] Shuffle the training instance ids", "red"))
random.shuffle(train_insts)
batched_data = batching_list_instances(config, train_insts)
dev_batches = batching_list_instances(config, dev_insts)
test_batches = batching_list_instances(config, test_insts)
best_dev = [-1, 0]
best_test = [-1, 0]
dep_model_name = config.dep_model.name
if config.dep_model == DepModelType.dggcn:
dep_model_name += '(' + str(config.num_gcn_layers) + "," + str(
config.gcn_dropout) + "," + str(config.gcn_mlp_layers) + ")"
model_name = "model_files/gcn_{}_hidden_{}_dataset_{}_{}_context_{}.m".format(
config.num_gcn_layers, config.hidden_dim, config.dataset, config.affix,
config.context_emb.name)
res_name = "results/gcn_{}_hidden_{}_dataset_{}_{}_context_{}.results".format(
config.num_gcn_layers, config.hidden_dim, config.dataset, config.affix,
config.context_emb.name)
print(
"[Info] The model will be saved to: %s, please ensure models folder exist"
% (model_name))
if not os.path.exists("model_files"):
os.makedirs("model_files")
if not os.path.exists("results"):
os.makedirs("results")
for i in range(1, epoch + 1):
epoch_loss = 0
start_time = time.time()
model.zero_grad()
if config.optimizer.lower() == "sgd":
optimizer = lr_decay(config, optimizer, i)
for index in np.random.permutation(len(batched_data)):
# for index in range(len(batched_data)):
model.train()
# optimizer.zero_grad()
batch_word, batch_wordlen, batch_context_emb, batch_char, batch_charlen, adj_matrixs, adjs_in, adjs_out, graphs, dep_label_adj, batch_dep_heads, trees, batch_label, batch_dep_label, batch_poslabel = batched_data[
index]
loss = model.neg_log_obj(batch_word, batch_wordlen,
batch_context_emb, batch_char,
batch_charlen, adj_matrixs, adjs_in,
adjs_out, graphs, dep_label_adj,
batch_dep_heads, batch_label,
batch_dep_label, batch_poslabel, trees)
epoch_loss += loss.item()
loss.backward()
if config.dep_model == DepModelType.dggcn:
torch.nn.utils.clip_grad_norm_(
model.parameters(), config.clip) ##clipping the gradient
optimizer.step()
model.zero_grad()
end_time = time.time()
print("Epoch %d: %.5f, Time is %.2fs" %
(i, epoch_loss, end_time - start_time),
flush=True)
if i + 1 >= config.eval_epoch:
model.eval()
dev_metrics = evaluate(config, model, dev_batches, "dev",
dev_insts)
if dev_metrics[2] > best_dev[0]:
test_metrics = evaluate(config, model, test_batches, "test",
test_insts)
print("saving the best model...")
best_dev[0] = dev_metrics[2]
best_dev[1] = i
best_test[0] = test_metrics[2]
best_test[1] = i
torch.save(model.state_dict(), model_name)
write_results(res_name, test_insts)
model.zero_grad()
print("The best dev: %.2f" % (best_dev[0]))
print("The corresponding test: %.2f" % (best_test[0]))
print("Final testing.")
model.load_state_dict(torch.load(model_name))
model.eval()
evaluate(config, model, test_batches, "test", test_insts)
write_results(res_name, test_insts)
def learn_from_insts(config:Config, epoch: int, train_insts, dev_insts, test_insts):
# train_insts: List[Instance], dev_insts: List[Instance], test_insts: List[Instance], batch_size: int = 1
if config.pretrain_dep:
model_path = f"model_files/{config.pdep_model}/{config.pdep_model}.tar.gz"
predictor = Predictor(model_path)
model = NNCRF(config, pretrained_dep_model=predictor.model)
else:
model = NNCRF(config)
optimizer = get_optimizer(config, model)
train_num = len(train_insts)
print("number of instances: %d" % (train_num))
print(colored("[Shuffled] Shuffle the training instance ids", "red"))
random.shuffle(train_insts)
batched_data = batching_list_instances(config, train_insts)
dev_batches = batching_list_instances(config, dev_insts)
test_batches = batching_list_instances(config, test_insts)
best_dev = [-1, 0]
best_test = [-1, 0]
dep_model_name = config.dep_model.name
if config.dep_model == DepModelType.dggcn:
dep_model_name += '(' + str(config.num_gcn_layers) + "," + str(config.gcn_dropout) + "," + str(
config.gcn_mlp_layers) + ")"
model_folder = config.model_folder
res_folder = "results"
model_path = f"model_files/{model_folder}/gnn.pt"
config_path = f"model_files/{model_folder}/config.conf"
res_path = f"{res_folder}/{model_folder}.res"
os.makedirs(f"model_files/{model_folder}", exist_ok=True) ## create model files. not raise error if exist
os.makedirs(res_folder, exist_ok=True)
print(f"[Info] The model will be saved to the directory: model_files/{model_folder}")
for i in range(1, epoch + 1):
epoch_loss = 0
start_time = time.time()
model.zero_grad()
if config.optimizer.lower() == "sgd":
optimizer = lr_decay(config, optimizer, i)
for index in np.random.permutation(len(batched_data)):
# for index in range(len(batched_data)):
model.train()
batch_word, batch_wordlen, batch_context_emb, batch_char, batch_charlen, adj_matrixs, adjs_in, adjs_out, graphs, dep_label_adj, batch_dep_heads, trees, batch_label, batch_dep_label = batched_data[index]
loss = model.neg_log_obj(batch_word, batch_wordlen, batch_context_emb,batch_char, batch_charlen, adj_matrixs, adjs_in, adjs_out, graphs, dep_label_adj, batch_dep_heads, batch_label, batch_dep_label, trees)
epoch_loss += loss.item()
loss.backward()
if config.dep_model == DepModelType.dggcn:
torch.nn.utils.clip_grad_norm_(model.parameters(), config.clip) ##clipping the gradient
optimizer.step()
model.zero_grad()
end_time = time.time()
print("Epoch %d: %.5f, Time is %.2fs" % (i, epoch_loss, end_time - start_time), flush=True)
if i + 1 >= config.eval_epoch:
model.eval()
dev_metrics = evaluate(config, model, dev_batches, "dev", dev_insts)
test_metrics = evaluate(config, model, test_batches, "test", test_insts)
if dev_metrics[2] > best_dev[0]:
print("saving the best model...")
best_dev[0] = dev_metrics[2]
best_dev[1] = i
best_test[0] = test_metrics[2]
best_test[1] = i
torch.save(model.state_dict(), model_path)
write_results(res_path, test_insts)
model.zero_grad()
print("Archiving the best Model...")
with tarfile.open(f"model_files/{model_folder}/{model_folder}.tar.gz", "w:gz") as tar:
tar.add(f"model_files/{model_folder}", arcname=os.path.basename(model_folder))
print("Finished archiving the models")
print("The best dev: %.2f" % (best_dev[0]))
print("The corresponding test: %.2f" % (best_test[0]))
print("Final testing.")
model.load_state_dict(torch.load(model_path))
model.eval()
evaluate(config, model, test_batches, "test", test_insts)
write_results(res_path, test_insts)