def test(self, test_bundle, return_output_vecs=False, weighted_instance_loss=False,
print_perf=True, title=None, report_number_of_intervals=20, return_output_vecs_get_details=True):
if len(test_bundle.task_list) > 1:
print('only one task is allowed for testing')
return None
if len(test_bundle.tws) == 0:
return list(), list(), list(), list()
if title is None:
title = ''
else:
title += ' '
self.bert_classifier.to(self.config.device)
self.bert_classifier.zero_grad()
self.bert_classifier.eval()
self.setup_objective(weighted_instance_loss)
test_dt = EBertDataset(test_bundle, self.tokenizer, self.config.max_seq)
batches = self.generate_batches([test_dt], self.config, False, False, 0, EInputListMode.sequential)
result_vecs = list()
result_vecs_detail = list()
tasks = {test_bundle.task_list[0] : ETaskState(test_bundle.task_list[0])}
print(title + 'labeling ', end=' ', flush=True)
with torch.no_grad():
for ba_ind, cur_batch in enumerate(batches):
outcome = self.bert_classifier(cur_batch, False)
self.__process_loss(outcome, cur_batch, tasks, False, weighted_instance_loss)
if return_output_vecs:
result_vecs.extend(self.bert_classifier.output_vecs)
if self.bert_classifier.output_vecs_detail is not None and return_output_vecs_get_details:
result_vecs_detail.extend(self.bert_classifier.output_vecs_detail)
if ELib.progress_made(ba_ind, cur_batch['batch_count'], report_number_of_intervals):
print(ELib.progress_percent(ba_ind, cur_batch['batch_count']), end=' ', flush=True)
self.delete_batch_from_gpu(cur_batch, EInputListMode.sequential)
del cur_batch, outcome
print()
task_out = tasks[test_bundle.task_list[0]]
task_out.loss /= task_out.size
perf = ELib.calculate_metrics(task_out.lbl_true, task_out.lbl_pred)
if print_perf:
print('Test Results L1> Loss: {:.3f} F1: {:.3f} Pre: {:.3f} Rec: {:.3f}'.format(
task_out.loss, perf[0], perf[1], perf[2]) + '\t\t' + ELib.get_time())
self.bert_classifier.cpu()
return task_out.lbl_pred, task_out.logits, [result_vecs, result_vecs_detail], perf
def __run_twin_net(config, lc, query, this_train_bundle, valid_bundle,
test_bundle, unlabeled_bundle):
pretrain_epochs_virtual = 1000000 # to make the learning-rate flat during pre-training
iterations = math.ceil(
1 + (len(unlabeled_bundle.tws) - EPretrainProj.sample_size) /
EPretrainProj.sample_step)
teacher, student = None, None
distr_info = None
config.cls_type = EBertCLSType.simple
config.epoch_count = EPretrainProj.finetune_epochs
seed = config.seed
print(colored('initial teacher training ...', 'blue'))
student = EBert(config)
student.train([this_train_bundle])
for cur_itr in range(iterations):
del teacher
teacher = student
print(
colored(
'>>> iteration {}/{}, sample-size {}'.format(
cur_itr + 1, iterations, EPretrainProj.sample_size),
'red'))
print(colored('teacher labeling ...', 'blue'))
## preparing unlabeled and labeling by teacher
cur_unlabeled_bundle = copy.deepcopy(unlabeled_bundle)
EPretrainProj.__sample_from_unlabeled(cur_unlabeled_bundle,
EPretrainProj.sample_size,
seed + 34 * (cur_itr + 1))
unl_lbl, unl_lgs, _, _ = teacher.test(cur_unlabeled_bundle)
distr_info = EPretrainProj.__twin_net_transform_labels(
distr_info, unl_lbl, unl_lgs, EPretrainProj.tmperT,
EPretrainProj.distr_moment)
print(
'neg-mean: {:.6f}, neg-std: {:.6f}, pos-mean: {:.6f}, pos-std: {:.6f}'
.format(distr_info[0].mean, math.sqrt(distr_info[0].var),
distr_info[1].mean, math.sqrt(distr_info[1].var)))
EPretrainProj.__update_label_info(cur_unlabeled_bundle, [unl_lbl],
[unl_lgs], False,
EPretrainProj.tmperT)
tr_lbl, tr_lgs, _, _ = teacher.test(this_train_bundle)
EPretrainProj.__update_label_info(this_train_bundle,
[this_train_bundle.input_y[0]],
[tr_lgs], True,
EPretrainProj.tmperT)
## training student
del student
print(colored('student training on unlabeled ...', 'blue'))
config_st = copy.deepcopy(teacher.config)
config_st.seed = seed + 566 * (cur_itr + 1)
config_st.epoch_count = EPretrainProj.pretrain_epochs
config_st.train_by_log_softmax = True
config_st.training_log_softmax_weight = 1
config_st.training_softmax_temperature = EPretrainProj.tmperT
config_st.balance_batch_mode = EBalanceBatchMode.label_based
student = EBert(config_st)
student.train([cur_unlabeled_bundle],
setup_learning_tools=True,
extra_scheduled_trainset_size=len(
this_train_bundle.tws),
extra_scheduled_epochs=pretrain_epochs_virtual)
print(colored('student training on labeled ...', 'blue'))
config_st.epoch_count = EPretrainProj.finetune_epochs
config_st.train_by_log_softmax = True
config_st.training_log_softmax_weight = EPretrainProj.finetune_log_softmax_weight
config_st.training_softmax_temperature = EPretrainProj.tmperT
config_st.balance_batch_mode = EBalanceBatchMode.label_based
student.train([this_train_bundle])
gc.collect()
torch.cuda.empty_cache()
if EPretrainProj.sample_size >= len(unlabeled_bundle.tws):
break
seed = seed + 324 * (cur_itr + 1)
EPretrainProj.sample_size += EPretrainProj.sample_step
## testing
print(colored('testing ...', 'green'))
_, logit_t, _, _ = teacher.test(test_bundle)
_, logit_s, _, _ = student.test(test_bundle)
result_lbl = ELib.majority_logits([logit_t, logit_s])
perf = ELib.calculate_metrics(test_bundle.input_y[0], result_lbl)
print('final test results L1> ' + ELib.get_string_metrics(perf))
del teacher, student
gc.collect()
torch.cuda.empty_cache()