def evaluate(self, model, epoch, print_info=True):
        dev_src_pred = model.predict_score.predict(self.X_dev_src,
                                                   batch_size=32)
        train_tgt_pred = model.predict_score.predict(self.X_train_tgt,
                                                     batch_size=32)
        dev_tgt_pred = model.predict_score.predict(self.X_dev_tgt,
                                                   batch_size=32)

        dev_src_pred_int = dev_src_pred.flatten() * 100
        train_tgt_pred_int = rescale_tointscore(train_tgt_pred,
                                                self.X_train_tgt_prompt_ids)
        dev_tgt_pred_int = rescale_tointscore(dev_tgt_pred,
                                              self.X_dev_tgt_prompt_ids)

        self.calc_correl(dev_src_pred_int, train_tgt_pred_int,
                         dev_tgt_pred_int)
        self.calc_kappa(dev_src_pred_int, train_tgt_pred_int, dev_tgt_pred_int)
        self.calc_rmse(dev_src_pred_int, train_tgt_pred_int, dev_tgt_pred_int)

        if self.dev_src_qwk > self.best_src_dev[0]:
            self.best_src_dev = [
                self.dev_src_qwk, self.dev_src_pr, self.dev_src_spr,
                self.dev_src_rmse
            ]
            self.best_tgt_train = [
                self.train_tgt_qwk, self.train_tgt_pr, self.train_tgt_spr,
                self.train_tgt_rmse
            ]
            self.best_tgt_dev = [
                self.dev_tgt_qwk, self.dev_tgt_pr, self.dev_tgt_spr,
                self.dev_tgt_rmse
            ]
            self.best_dev_epoch = epoch
        if print_info:
            self.print_info()
예제 #2
0
 def __init__(self, test_prompt_id, X_dev_src, X_train_tgt, X_dev_tgt, X_dev_src_prompt_ids, X_train_tgt_prompt_ids,
              X_dev_tgt_prompt_ids, Y_dev_src, Y_train_tgt, Y_dev_tgt):
     self.test_prompt_id = test_prompt_id
     self.X_dev_src, self.X_train_tgt, self.X_dev_tgt = X_dev_src, X_train_tgt, X_dev_tgt
     self.X_dev_src_prompt_ids, self.X_train_tgt_prompt_ids, self.X_dev_tgt_prompt_ids = \
         X_dev_src_prompt_ids, X_train_tgt_prompt_ids, X_dev_tgt_prompt_ids
     self.Y_dev_src, self.Y_train_tgt, self.Y_dev_tgt = Y_dev_src, Y_train_tgt, Y_dev_tgt
     self.Y_dev_src_org = Y_dev_src.flatten() * 100
     self.Y_train_tgt_org = rescale_tointscore(Y_train_tgt, self.X_train_tgt_prompt_ids)
     self.Y_dev_tgt_org = rescale_tointscore(Y_dev_tgt, self.X_dev_tgt_prompt_ids)
     self.best_src_dev = [-1, -1, -1, -1]
     self.best_tgt_train = [-1, -1, -1, -1]
     self.best_tgt_dev = [-1, -1, -1, -1]