def train(conf):
logger = util.Logger(conf)
if not os.path.exists(conf.checkpoint_dir):
os.makedirs(conf.checkpoint_dir)
model_name = conf.model_name
dataset_name = "ClassificationDataset"
collate_name = "FastTextCollator" if model_name == "FastText" \
else "ClassificationCollator"
train_data_loader, validate_data_loader, test_data_loader = \
get_data_loader(dataset_name, collate_name, conf)
empty_dataset = globals()[dataset_name](conf, [], mode="train")
model = get_classification_model(model_name, empty_dataset, conf)
loss_fn = globals()["ClassificationLoss"](label_size=len(
empty_dataset.label_map),
loss_type=conf.train.loss_type)
optimizer = get_optimizer(conf, model)
evaluator = cEvaluator(conf.eval.dir)
trainer = globals()["ClassificationTrainer"](empty_dataset.label_map,
logger, evaluator, conf,
loss_fn)
best_epoch = -1
best_performance = 0
model_file_prefix = conf.checkpoint_dir + "/" + model_name
for epoch in range(conf.train.start_epoch,
conf.train.start_epoch + conf.train.num_epochs):
start_time = time.time()
trainer.train(train_data_loader, model, optimizer, "Train", epoch)
trainer.eval(train_data_loader, model, optimizer, "Train", epoch)
performance = trainer.eval(validate_data_loader, model, optimizer,
"Validate", epoch)
trainer.eval(test_data_loader, model, optimizer, "test", epoch)
if performance > best_performance: # record the best model
best_epoch = epoch
best_performance = performance
save_checkpoint(
{
'epoch': epoch,
'model_name': model_name,
'state_dict': model.state_dict(),
'best_performance': best_performance,
'optimizer': optimizer.state_dict(),
}, model_file_prefix)
time_used = time.time() - start_time
logger.info("Epoch %d cost time: %d second" % (epoch, time_used))
# best model on validateion set
best_epoch_file_name = model_file_prefix + "_" + str(best_epoch)
best_file_name = model_file_prefix + "_best"
shutil.copyfile(best_epoch_file_name, best_file_name)
load_checkpoint(model_file_prefix + "_" + str(best_epoch), conf, model,
optimizer)
trainer.eval(test_data_loader, model, optimizer, "Best test", best_epoch)
def kfold_eval(conf):
logger = util.Logger(conf)
model_name = conf.model_name
dataset_name = "ClassificationDataset"
collate_name = "FastTextCollator" if model_name == "FastText" \
else "ClassificationCollator"
test_dataset = globals()[dataset_name](conf, conf.data.test_json_files)
collate_fn = globals()[collate_name](conf, len(test_dataset.label_map))
test_data_loader = DataLoader(
test_dataset, batch_size=conf.eval.batch_size, shuffle=False,
num_workers=conf.data.num_worker, collate_fn=collate_fn,
pin_memory=True)
empty_dataset = globals()[dataset_name](conf, [])
model = get_classification_model(model_name, empty_dataset, conf)
optimizer = get_optimizer(conf, model)
load_checkpoint(conf.eval.model_dir, conf, model, optimizer)
model.eval()
predict_probs = []
standard_labels = []
evaluator = cEvaluator(conf.eval.dir)
for batch in test_data_loader:
logits = model(batch)
result = torch.sigmoid(logits).cpu().tolist()
predict_probs.extend(result)
standard_labels.extend(batch[ClassificationDataset.DOC_LABEL_LIST])
# ============================ EVALUATION API ============================================================================================
y_test, predictions = [], []
print (standard_labels)
for i, j in zip(standard_labels, predict_probs):
y_test.append(i)
predictions.append(j)
pred, actual = take_values(predictions, y_test , conf.eval.threshold, conf.eval.top_k )
print(pred)
actual=np.array(actual)
pred=np.array(pred)
evaluation_measures={"Accuracy": accuracy(actual, pred) ,
"Precision": precision(actual, pred) ,
"Recall": recall(actual, pred) ,
"F1 score": f1_scor(actual, pred, ) ,
"Hamming Loss":hammingLoss(actual, pred),
"f-1 Macro":macroF1(actual, pred) ,
"f-1 Micro":microF1(actual, pred),
"averagePrecision":averagePrecision(actual, pred)
}
return evaluation_measures
def eval(conf):
logger = util.Logger(conf)
model_name = conf.model_name
dataset_name = "ClassificationDataset"
collate_name = "FastTextCollator" if model_name == "FastText" \
else "ClassificationCollator"
test_dataset = globals()[dataset_name](conf, conf.data.test_json_files)
collate_fn = globals()[collate_name](conf, len(test_dataset.label_map))
test_data_loader = DataLoader(test_dataset,
batch_size=conf.eval.batch_size,
shuffle=False,
num_workers=conf.data.num_worker,
collate_fn=collate_fn,
pin_memory=True)
empty_dataset = globals()[dataset_name](conf, [])
model = get_classification_model(model_name, empty_dataset, conf)
optimizer = get_optimizer(conf, model.parameters())
load_checkpoint(conf.eval.model_dir, conf, model, optimizer)
model.eval()
is_multi = False
if conf.task_info.label_type == ClassificationType.MULTI_LABEL:
is_multi = True
predict_probs = []
standard_labels = []
total_loss = 0.
evaluator = cEvaluator(conf.eval.dir)
for batch in test_data_loader:
logits = model(batch)
if not is_multi:
result = torch.nn.functional.softmax(logits, dim=1).cpu().tolist()
else:
result = torch.sigmoid(logits).cpu().tolist()
predict_probs.extend(result)
standard_labels.extend(batch[ClassificationDataset.DOC_LABEL_LIST])
total_loss = total_loss / len(predict_probs)
(_, precision_list, recall_list, fscore_list, right_list,
predict_list, standard_list) = \
evaluator.evaluate(
predict_probs, standard_label_ids=standard_labels, label_map=empty_dataset.label_map,
threshold=conf.eval.threshold, top_k=conf.eval.top_k,
is_flat=conf.eval.is_flat, is_multi=is_multi)
logger.warn(
"Performance is precision: %f, "
"recall: %f, fscore: %f, right: %d, predict: %d, standard: %d." %
(precision_list[0][cEvaluator.MICRO_AVERAGE],
recall_list[0][cEvaluator.MICRO_AVERAGE],
fscore_list[0][cEvaluator.MICRO_AVERAGE],
right_list[0][cEvaluator.MICRO_AVERAGE],
predict_list[0][cEvaluator.MICRO_AVERAGE],
standard_list[0][cEvaluator.MICRO_AVERAGE]))
evaluator.save()
def train(conf):
model_name = conf.model_name
logger = util.Logger(conf)
if conf.task_info.weak_pretrain:
logger.info("Batch Size: " + str(conf.train.batch_size) +
" Pretrain Num Epoch: " +
str(conf.train.pretrain_num_epochs))
else:
logger.info("Batch Size: " + str(conf.train.batch_size))
if conf.task_info.weak_pretrain and conf.task_info.weak_data_augmentation:
model_teacher = get_classification_model(model_name, empty_dataset,
conf)
if conf.model_name != "BERT":
optimizer_teacher = get_optimizer(conf, model_teacher)
else:
optimizer_teacher = AdamW(model_teacher.parameters(),
lr=5e-2,
eps=1e-2)
# optimizer_teacher: optimizer for teacher model
model_target = get_classification_model(model_name, empty_dataset, conf)
loss_fn = globals()["ClassificationLoss"](label_size=len(
empty_dataset.label_map),
loss_type=conf.train.loss_type)
if conf.task_info.weak_pretrain:
if conf.model_name != "BERT":
optimizer_weak = get_optimizer(conf, model_target)
else:
optimizer_weak = AdamW(model_target.parameters(),
lr=5e-2,
eps=1e-2)
# optimizer_weak: optimizer for target model pretraining stage
if conf.model_name != "BERT":
optimizer_target = get_optimizer(conf, model_target)
else:
optimizer_target = AdamW(model_target.parameters(), lr=5e-2, eps=1e-2)
# optimizer_target: optimizer for target model fine-tuning stage
evaluator = cEvaluator(conf.eval.dir)
trainer_target = globals()["ClassificationTrainer"](
empty_dataset.label_map, logger, evaluator, conf, loss_fn)
# trainer_target: trainer for target model on fine-tuning stage
if conf.task_info.weak_pretrain:
trainer_weak = globals()["ClassificationTrainer"](
empty_dataset.label_map, logger, evaluator, conf, loss_fn)
# trainer_weak: trainer for target model on pretraining stage
if conf.task_info.weak_data_augmentation:
trainer_teacher = globals()["ClassificationTrainer"](
empty_dataset.label_map, logger, evaluator, conf, loss_fn)
# trainer_teacher: trainer for teacher model
if conf.task_info.weak_data_augmentation:
best_epoch = -1
best_performance = 0
model_file_prefix = conf.checkpoint_dir + "/" + model_name + "_teacher"
logger.info("Training Teacher Model on Labeled Data")
for epoch in range(conf.train.start_epoch,
conf.train.start_epoch + conf.train.num_epochs):
start_time = time.time()
trainer_teacher.train(train_data_loader, model_teacher,
optimizer_teacher, "Train", epoch)
trainer_teacher.eval(train_data_loader, model_teacher,
optimizer_teacher, "Train", epoch)
performance = trainer_teacher.eval(validate_data_loader,
model_teacher,
optimizer_teacher, "Validate",
epoch)
trainer_teacher.eval(test_data_loader, model_teacher,
optimizer_teacher, "Test", epoch)
if performance > best_performance: # record the best model
best_epoch = epoch
best_performance = performance
temp_model = model_teacher
save_checkpoint(
{
'epoch': epoch,
'model_name': model_name,
'state_dict': model_teacher.state_dict(),
'best_performance': best_performance,
'optimizer': optimizer_teacher.state_dict(),
}, model_file_prefix)
time_used = time.time() - start_time
logger.info("Epoch %d cost time: %d second" % (epoch, time_used))
best_epoch = -1
best_performance = 0
if conf.task_info.weak_pretrain:
if conf.task_info.weak_data_augmentation:
unlabeled_data_train_data_loader = select_unlabeled_data(
temp_model, unlabeled_train_data_loader,
len(trainer_weak.label_map), conf)
logger.info("Pretraining on Weak Supervision Data")
for epoch in range(
conf.train.start_epoch,
conf.train.start_epoch + conf.train.pretrain_num_epochs):
start_time = time.time()
trainer_weak.train(unlabeled_train_data_loader, model_target,
optimizer_weak, "Train", epoch)
trainer_weak.eval(unlabeled_train_data_loader, model_target,
optimizer_weak, "Train", epoch)
performance = trainer_weak.eval(validate_data_loader, model_target,
optimizer_weak, "Validate", epoch)
trainer_weak.eval(test_data_loader, model_target, optimizer_weak,
"Test", epoch)
if performance > best_performance: # record the best model
temp_model = model_target
time_used = time.time() - start_time
logger.info("Epoch %d cost time: %d second" % (epoch, time_used))
model_target = temp_model
logger.info("Fine-tuning on Labeled Data")
best_epoch = -1
best_performance = 0
if conf.task_info.weak_pretrain:
if conf.task_info.weak_data_augmentation:
model_file_prefix = conf.checkpoint_dir + "/" + model_name + "-Augmentation-" + conf.task_info.Augmentation_Method + "-Pretrain" + str(
conf.train.pretrain_num_epochs) + "-Batch" + str(
conf.train.batch_size)
else:
model_file_prefix = conf.checkpoint_dir + "/" + model_name + "-WeakSupervision-" + "-Pretrain" + str(
conf.train.pretrain_num_epochs) + "-Batch" + str(
conf.train.batch_size)
else:
model_file_prefix = conf.checkpoint_dir + "/" + model_name + "-Batch" + str(
conf.train.batch_size)
for epoch in range(conf.train.start_epoch,
conf.train.start_epoch + conf.train.num_epochs):
start_time = time.time()
trainer_target.train(train_data_loader, model_target, optimizer_target,
"Train", epoch)
trainer_target.eval(train_data_loader, model_target, optimizer_target,
"Train", epoch)
performance = trainer_target.eval(validate_data_loader, model_target,
optimizer_target, "Validate", epoch)
trainer_target.eval(test_data_loader, model_target, optimizer_target,
"Test", epoch)
if performance > best_performance: # record the best model
best_epoch = epoch
best_performance = performance
temp_model = model_target
save_checkpoint(
{
'epoch': epoch,
'model_name': model_name,
'state_dict': model_target.state_dict(),
'best_performance': best_performance,
'optimizer': optimizer_target.state_dict(),
}, model_file_prefix)
time_used = time.time() - start_time
logger.info("Epoch %d cost time: %d second" % (epoch, time_used))
logger.info("The Best Performance on Validation Data and Test Data")
#best_epoch_file_name = model_file_prefix + "_" + str(best_epoch)
#best_file_name = model_file_prefix + "_best"
#shutil.copyfile(best_epoch_file_name, best_file_name)
#load_checkpoint(model_file_prefix + "_" + str(best_epoch), conf, model,
# optimizer)
model = temp_model
trainer_target.eval(train_data_loader, model, optimizer_target,
"Best Train", best_epoch)
trainer_target.eval(validate_data_loader, model, optimizer_target,
"Best Validate", best_epoch)
trainer_target.eval(test_data_loader, model, optimizer_target, "Best Test",
best_epoch)
def eval(conf):
logger = util.Logger(conf)
model_name = conf.model_name
dataset_name = "ClassificationDataset"
collate_name = "FastTextCollator" if model_name == "FastText" \
else "ClassificationCollator"
test_dataset = globals()[dataset_name](conf, conf.data.test_json_files)
collate_fn = globals()[collate_name](conf, len(test_dataset.label_map))
test_data_loader = DataLoader(
test_dataset, batch_size=conf.eval.batch_size, shuffle=False,
num_workers=conf.data.num_worker, collate_fn=collate_fn,
pin_memory=True)
empty_dataset = globals()[dataset_name](conf, [])
model = get_classification_model(model_name, empty_dataset, conf)
optimizer = get_optimizer(conf, model)
load_checkpoint(conf.eval.model_dir, conf, model, optimizer)
model.eval()
is_multi = False
if conf.task_info.label_type == ClassificationType.MULTI_LABEL:
is_multi = True
predict_probs = []
standard_labels = []
evaluator = cEvaluator(conf.eval.dir)
for batch in test_data_loader:
with torch.no_grad():
logits = model(batch)
if not is_multi:
result = torch.nn.functional.softmax(logits, dim=1)
else:
result = torch.sigmoid(logits)
result = result.detach().cpu().tolist()
predict_probs.extend(result)
standard_labels.extend(batch[ClassificationDataset.DOC_LABEL_LIST])
if conf.eval.is_flat:
(_, precision_list, recall_list, fscore_list, right_list,
predict_list, standard_list, pak_dict, rak_dict, rpak_dict, ndcgak_dict) = \
evaluator.evaluate(
predict_probs, standard_label_ids=standard_labels, label_map=empty_dataset.label_map,
threshold=conf.eval.threshold, top_k=conf.eval.top_k,
is_flat=conf.eval.is_flat, is_multi=is_multi,
debug_file_name=conf.eval.debug_file_name,
is_label_split=conf.data.generate_label_group,
label_split_json_file=os.path.join(conf.data.dict_dir,
"{}.json".format(ClassificationDataset.DOC_LABEL_GROUP)),
instance_remove=conf.eval.instance_remove
)
sup_message = ""
for i in range(1, conf.eval.top_k + 1):
for group in pak_dict[i]:
sup_message += "Precision at {} of {} group: {}, ".format(i, group, pak_dict[i][group])
sup_message += "Recall at {} of {} group: {}, ".format(i, group, rak_dict[i][group])
sup_message += "R-Precision at {} of {} group: {}, ".format(i, group, rpak_dict[i][group])
sup_message += "nDCG at {} of {} group: {}, ".format(i, group, ndcgak_dict[i][group])
message = "Performance is precision: {}, recall: {}, fscore: {}, " + \
"macro-fscore: {}, right: {}, predict: {}, standard: {}, "
logger.warn(message.format(
precision_list[0][cEvaluator.MICRO_AVERAGE],
recall_list[0][cEvaluator.MICRO_AVERAGE],
fscore_list[0][cEvaluator.MICRO_AVERAGE],
fscore_list[0][cEvaluator.MACRO_AVERAGE],
right_list[0][cEvaluator.MICRO_AVERAGE],
predict_list[0][cEvaluator.MICRO_AVERAGE],
standard_list[0][cEvaluator.MICRO_AVERAGE]) +
sup_message)
else:
(_, precision_list, recall_list, fscore_list, right_list,
predict_list, standard_list) = \
evaluator.evaluate(
predict_probs, standard_label_ids=standard_labels, label_map=empty_dataset.label_map,
threshold=conf.eval.threshold, top_k=conf.eval.top_k,
is_flat=conf.eval.is_flat, is_multi=is_multi,
is_label_split=conf.data.generate_label_group,
label_split_json_file=os.path.join(conf.data.dict_dir,
"{}.json".format(ClassificationDataset.DOC_LABEL_GROUP))
)
logger.warn(
"Performance is precision: %f, "
"recall: %f, fscore: %f, right: %d, predict: %d, standard: %d." % (
precision_list[0][cEvaluator.MICRO_AVERAGE],
recall_list[0][cEvaluator.MICRO_AVERAGE],
fscore_list[0][cEvaluator.MICRO_AVERAGE],
right_list[0][cEvaluator.MICRO_AVERAGE],
predict_list[0][cEvaluator.MICRO_AVERAGE],
standard_list[0][cEvaluator.MICRO_AVERAGE]))
evaluator.save()
