def evaluate(self, model, epoch, print_info=True): dev_src_pred = model.predict_score.predict(self.X_dev_src, batch_size=32) train_tgt_pred = model.predict_score.predict(self.X_train_tgt, batch_size=32) dev_tgt_pred = model.predict_score.predict(self.X_dev_tgt, batch_size=32) dev_src_pred_int = dev_src_pred.flatten() * 100 train_tgt_pred_int = rescale_tointscore(train_tgt_pred, self.X_train_tgt_prompt_ids) dev_tgt_pred_int = rescale_tointscore(dev_tgt_pred, self.X_dev_tgt_prompt_ids) self.calc_correl(dev_src_pred_int, train_tgt_pred_int, dev_tgt_pred_int) self.calc_kappa(dev_src_pred_int, train_tgt_pred_int, dev_tgt_pred_int) self.calc_rmse(dev_src_pred_int, train_tgt_pred_int, dev_tgt_pred_int) if self.dev_src_qwk > self.best_src_dev[0]: self.best_src_dev = [ self.dev_src_qwk, self.dev_src_pr, self.dev_src_spr, self.dev_src_rmse ] self.best_tgt_train = [ self.train_tgt_qwk, self.train_tgt_pr, self.train_tgt_spr, self.train_tgt_rmse ] self.best_tgt_dev = [ self.dev_tgt_qwk, self.dev_tgt_pr, self.dev_tgt_spr, self.dev_tgt_rmse ] self.best_dev_epoch = epoch if print_info: self.print_info()
def __init__(self, test_prompt_id, X_dev_src, X_train_tgt, X_dev_tgt, X_dev_src_prompt_ids, X_train_tgt_prompt_ids, X_dev_tgt_prompt_ids, Y_dev_src, Y_train_tgt, Y_dev_tgt): self.test_prompt_id = test_prompt_id self.X_dev_src, self.X_train_tgt, self.X_dev_tgt = X_dev_src, X_train_tgt, X_dev_tgt self.X_dev_src_prompt_ids, self.X_train_tgt_prompt_ids, self.X_dev_tgt_prompt_ids = \ X_dev_src_prompt_ids, X_train_tgt_prompt_ids, X_dev_tgt_prompt_ids self.Y_dev_src, self.Y_train_tgt, self.Y_dev_tgt = Y_dev_src, Y_train_tgt, Y_dev_tgt self.Y_dev_src_org = Y_dev_src.flatten() * 100 self.Y_train_tgt_org = rescale_tointscore(Y_train_tgt, self.X_train_tgt_prompt_ids) self.Y_dev_tgt_org = rescale_tointscore(Y_dev_tgt, self.X_dev_tgt_prompt_ids) self.best_src_dev = [-1, -1, -1, -1] self.best_tgt_train = [-1, -1, -1, -1] self.best_tgt_dev = [-1, -1, -1, -1]