def __init__(self,
prompt_id,
use_char,
out_dir,
modelname,
train_x,
dev_x,
test_x,
train_c,
dev_c,
test_c,
train_y,
dev_y,
test_y,
train_context,
dev_context,
test_context,
char_only=False):
# self.dataset = dataset
self.use_char = use_char
self.char_only = char_only
self.prompt_id = prompt_id
self.train_x, self.dev_x, self.test_x = train_x, dev_x, test_x
self.train_c, self.dev_c, self.test_c = train_c, dev_c, test_c
self.train_y, self.dev_y, self.test_y = train_y, dev_y, test_y
self.train_context, self.dev_context, self.test_context = train_context, dev_context, test_context
self.train_y_org = rescale_tointscore(train_y, self.prompt_id)
self.dev_y_org = rescale_tointscore(dev_y, self.prompt_id)
self.test_y_org = rescale_tointscore(test_y, self.prompt_id)
self.out_dir = out_dir
self.modelname = modelname
self.best_dev = [-1, -1, -1, -1]
self.best_test = [-1, -1, -1, -1]
def evaluate(self, model, epoch, print_info=False):
if self.use_char:
if self.char_only:
train_pred = model.predict(self.train_c, batch_size=32).squeeze()
dev_pred = model.predict(self.dev_c, batch_size=32).squeeze()
test_pred = model.predict(self.test_c, batch_size=32).squeeze()
else:
train_pred = model.predict([self.train_x, self.train_c], batch_size=32).squeeze()
dev_pred = model.predict([self.dev_x, self.dev_c], batch_size=32).squeeze()
test_pred = model.predict([self.test_x, self.test_c], batch_size=32).squeeze()
else:
train_pred = model.predict(self.train_x, batch_size=32).squeeze()
dev_pred = model.predict(self.dev_x, batch_size=32).squeeze()
test_pred = model.predict(self.test_x, batch_size=32).squeeze()
# print test_pred
train_pred_int = rescale_tointscore(train_pred, self.prompt_id)
dev_pred_int = rescale_tointscore(dev_pred, self.prompt_id)
test_pred_int = rescale_tointscore(test_pred, self.prompt_id)
self.calc_correl(train_pred_int, dev_pred_int, test_pred_int)
self.calc_kappa(train_pred_int, dev_pred_int, test_pred_int)
self.calc_rmse(train_pred_int, dev_pred_int, test_pred_int)
if self.dev_qwk > self.best_dev[0]:
self.best_dev = [self.dev_qwk, self.dev_pr, self.dev_spr, self.dev_rmse]
self.best_test = [self.test_qwk, self.test_pr, self.test_spr, self.test_rmse]
self.best_dev_epoch = epoch
model.save_weights(self.out_dir + '/' + self.modelname, overwrite=True)
if print_info:
self.print_info()
def __init__(self, prompt_id, use_mask, out_dir, modelname, train_x, dev_x,
test_x, train_y, dev_y, test_y):
# self.dataset = dataset
self.use_mask = use_mask
self.prompt_id = prompt_id
self.train_x, self.dev_x, self.test_x = train_x, dev_x, test_x
#self.train_mask, self.dev_mask, self.test_mask = train_mask, dev_mask, test_mask
self.train_y, self.dev_y, self.test_y = train_y, dev_y, test_y
self.train_y_org = rescale_tointscore(train_y, self.prompt_id)
self.dev_y_org = rescale_tointscore(dev_y, self.prompt_id)
self.test_y_org = rescale_tointscore(test_y, self.prompt_id)
self.out_dir = out_dir
self.modelname = modelname
self.best_dev = [-1, -1, -1, -1]
self.best_test = [-1, -1, -1, -1]
def evaluate(self, model, epoch, print_info=False):
# train_pred = model.predict({'word_input': self.train_x, 'word_input_d': self.train_d}, batch_size=32).squeeze()
# dev_pred = model.predict({'word_input': self.dev_x, 'word_input_d': self.dev_d}, batch_size=32).squeeze()
# test_pred = model.predict({'word_input': self.test_x, 'word_input_d': self.test_d}, batch_size=32).squeeze()
# train_pred = model.predict({'word_input': self.train_x, 'p_input1': self.p_train[:, 0, :], 'p_input2': self.p_train[:, 1, :],
# 'p_input3': self.p_train[:, 2, :], 'p_input4': self.p_train[:, 3, :]}, batch_size=32).squeeze()
# dev_pred = model.predict({'word_input': self.dev_x, 'p_input1': self.p_dev[:, 0, :], 'p_input2': self.p_dev[:, 1, :],
# 'p_input3': self.p_dev[:, 2, :], 'p_input4': self.p_dev[:, 3, :]}, batch_size=32).squeeze()
# test_pred = model.predict({'word_input': self.test_x, 'p_input1': self.p_test[:, 0, :], 'p_input2': self.p_test[:, 1, :],
# 'p_input3': self.p_test[:, 2, :], 'p_input4': self.p_test[:, 3, :]}, batch_size=32).squeeze()
train_pred = model.predict(
{
'word_input': self.train_x,
'p': self.p_train
}, batch_size=32).squeeze()
dev_pred = model.predict({
'word_input': self.dev_x,
'p': self.p_dev
},
batch_size=32).squeeze()
test_pred = model.predict({
'word_input': self.test_x,
'p': self.p_test
},
batch_size=32).squeeze()
train_pred_int = rescale_tointscore(train_pred, self.prompt_id)
dev_pred_int = rescale_tointscore(dev_pred, self.prompt_id)
test_pred_int = rescale_tointscore(test_pred, self.prompt_id)
# self.calc_correl(train_pred_int, dev_pred_int, test_pred_int)
self.calc_kappa(train_pred_int, dev_pred_int, test_pred_int)
#self.calc_rmse(train_pred_int, dev_pred_int, test_pred_int)
if self.dev_qwk > self.best_dev[0]:
self.best_dev = [self.dev_qwk, 0, 0, 0]
self.best_test = [self.test_qwk, 0, 0, 0]
self.best_dev_epoch = epoch
if print_info:
self.print_info()
def __init__(self, prompt_id, fold, train_x, dev_x, test_x, train_y, dev_y,
test_y, train_d, dev_d, test_d, p_train, p_dev, p_test):
# self.dataset = dataset
self.fold = fold
self.prompt_id = prompt_id
self.train_x, self.dev_x, self.test_x = train_x, dev_x, test_x
self.train_y, self.dev_y, self.test_y = train_y, dev_y, test_y
self.train_d = train_d
self.dev_d = dev_d
self.test_d = test_d
self.train_y_org = rescale_tointscore(train_y, self.prompt_id)
self.dev_y_org = rescale_tointscore(dev_y, self.prompt_id)
self.test_y_org = rescale_tointscore(test_y, self.prompt_id)
self.p_train = p_train
self.p_dev = p_dev
self.p_test = p_test
self.best_dev = [-1, -1, -1, -1]
self.best_test = [-1, -1, -1, -1]