batch_parse = eisner_for_dmv.batch_parse(batch_score,
batch_decision_score,
dmv_model.dvalency,
dmv_model.cvalency)
for i in range(len(eval_batch_pos)):
parse_results[eval_batch_sen[i]] = (batch_parse[0][i],
batch_parse[1][i])
utils.eval(parse_results, eval_sentences, devpath,
options.log + '_dev' + str(options.sample_idx), epoch)
# utils.write_distribution(dmv_model)
print "===================================="
pos, sentences, languages, language_map = utils.read_ml_corpus(
options.language_path,
options.train,
stc_length=15,
isPredict=False,
isadd=False
) # pos: str 2 id, sentences, languages: lang 2 id, language_map: stc_id 2 lang
#if not options.load_model:
# pos, sentences, languages, language_map = utils.read_ml_corpus(options.language_path, options.train, stc_length=15, isPredict=False, isadd=False) # pos: str 2 id, sentences, languages: lang 2 id, language_map: stc_id 2 lang
#else:
# pos, sentences, languages, language_map = utils.read_ml_corpus(options.language_path, options.train, stc_length=15, isPredict=False, isadd=True) # language: lang2i
# sentence_language_map = {}
# print 'Data read'
# with open(os.path.join(options.output, options.params + '_' + str(options.sample_idx)), 'w') as paramsfp:
# pickle.dump((pos, options), paramsfp)
# print 'Parameters saved'
def do_eval(dmv_model, m_model, pos, options, epoch):
print "===================================="
print 'Do evaluation on development set'
# eval_sentences = utils.read_data(options.dev, True)
if not options.load_model:
ml_sentences = utils.read_ml_corpus(options.language_path,
options.dev,
stc_length=15,
isPredict=True,
isadd=False)
else:
ml_sentences = utils.read_ml_corpus(options.language_path,
options.dev,
stc_length=15,
isPredict=True,
isadd=True)
eval_sentences = ml_sentences[0]
dmv_model.eval()
eval_sentence_map = {}
eval_sen_idx = 0
eval_data_list = list()
devpath = os.path.join(
options.output,
'eval_pred' + str(epoch + 1) + '_' + str(options.sample_idx))
lang_id = languages[
options.
dev] if options.dev in languages else 0 # 0 is manually specified (when dev_lang is not trained before)
for s in eval_sentences:
_, s_pos = s.set_data_list(None, pos)
s_data_list = list()
s_data_list.append(s_pos)
s_data_list.append([eval_sen_idx])
eval_data_list.append(s_data_list)
eval_sentence_map[eval_sen_idx] = s_pos
eval_sen_idx += 1
eval_batch_data = utils.construct_batch_data(eval_data_list,
options.batchsize)
parse_results = {}
eval_sentence_trans_param = np.zeros(
(len(eval_data_list), len(pos.keys()), len(pos.keys()), 2,
options.c_valency))
for batch_id, one_batch in enumerate(eval_batch_data):
eval_batch_pos, eval_batch_sen = [s[0] for s in one_batch
], [s[1][0] for s in one_batch]
eval_batch_sen = np.array(eval_batch_sen)
eval_batch_pos = np.array(eval_batch_pos)
if dmv_model.initial_flag:
batch_score, batch_root_score, batch_decision_score = dmv_model.evaluate_batch_score(
eval_batch_pos, eval_batch_sen, None, epoch)
else:
batch_rule_samples = dmv_model.find_predict_samples(
eval_batch_pos, eval_batch_sen)
batch_predict_data = utils.construct_ml_predict_data(
batch_rule_samples)
batch_predict_pos_v = torch.LongTensor(
batch_predict_data['pos'])
batch_predict_pos_index = np.array(batch_predict_data['pos'])
batch_predict_dir_v = torch.LongTensor(
batch_predict_data['dir'])
batch_predict_dir_index = np.array(batch_predict_data['dir'])
batch_predict_cvalency_v = torch.LongTensor(
batch_predict_data['cvalency'])
batch_predict_cvalency_index = np.array(
batch_predict_data['cvalency'])
batch_predict_sen_v = []
for sentence_id in batch_predict_data['sentence']:
batch_predict_sen_v.append(eval_sentence_map[sentence_id])
batch_predict_sen_index = np.array(
batch_predict_data['sentence'])
batch_predict_sen_v = torch.LongTensor(batch_predict_sen_v)
batch_predict_sen_len = torch.LongTensor(
np.array([len(i) for i in batch_predict_sen_v]))
batch_predict_lan_v = torch.LongTensor(
np.array([lang_id for _ in batch_predict_sen_v])) # TODO
batch_predicted = m_model.forward_(batch_predict_pos_v,
batch_predict_dir_v,
batch_predict_cvalency_v,
None,
None,
True,
'child',
options.em_type,
batch_predict_lan_v,
batch_predict_sen_v,
batch_predict_sen_len,
epoch=epoch)
eval_sentence_trans_param[
batch_predict_sen_index, batch_predict_pos_index, :,
batch_predict_dir_index,
batch_predict_cvalency_index] = batch_predicted.detach(
).numpy()
batch_score, batch_root_score, batch_decision_score = dmv_model.evaluate_batch_score(
eval_batch_pos, eval_batch_sen, eval_sentence_trans_param,
epoch)
batch_size, sentence_length, _, v_c_num = batch_score.shape
_, _, _, v_d_num, _ = batch_decision_score.shape
batch_score = np.concatenate(
(np.full((batch_size, 1, sentence_length, v_c_num),
-np.inf), batch_score),
axis=1) # for eisner
batch_score = np.concatenate(
(np.full((batch_size, sentence_length + 1, 1, v_c_num),
-np.inf), batch_score),
axis=2) # for eisner
batch_score[:, 0, 1:, 0] = batch_root_score
batch_decision_score = np.concatenate((np.zeros(
(batch_size, 1, 2, v_d_num, 2)), batch_decision_score),
axis=1)
batch_score = np.expand_dims(batch_score, 3)
batch_score = np.expand_dims(batch_score, 4)
batch_decision_score = np.expand_dims(batch_decision_score, 2)
batch_parse = eisner_for_dmv.batch_parse(batch_score,
batch_decision_score,
dmv_model.dvalency,
dmv_model.cvalency)
for i in range(len(eval_batch_pos)):
parse_results[eval_batch_sen[i]] = (batch_parse[0][i],
batch_parse[1][i])
utils.eval(parse_results, eval_sentences, devpath,
options.log + '_dev' + str(options.sample_idx), epoch)
# utils.write_distribution(dmv_model)
print "===================================="
def do_eval(dmv_model, m_model, pos, words, options, epoch):
print("====================================")
print('Do evaluation on development set')
# eval_sentences = utils.read_data(options.dev, True)
if not options.load_model:
ml_sentences = utils.read_ml_corpus(
options.language_path,
options.dev,
stc_length=options.val_stc_length,
isPredict=True,
isadd=False)
else:
ml_sentences = utils.read_ml_corpus(
options.language_path,
options.dev,
stc_length=options.val_stc_length,
isPredict=True,
isadd=True)
eval_sentences = ml_sentences[0]
dmv_model.eval()
eval_sentence_map = {}
eval_sentence_word_map = {}
eval_sen_idx = 0
eval_data_list = list()
devpath = os.path.join(
options.output,
'eval_pred' + str(epoch + 1) + '_' + str(options.sample_idx))
lang_id = languages[
options.
dev] if options.dev in languages else 0 # 0 is manually specified (when dev_lang is not trained before)
for s in eval_sentences:
s_word, s_pos = s.set_data_list(words, pos)
s_data_list = list()
s_data_list.append([s_pos, s_word])
s_data_list.append([eval_sen_idx])
eval_data_list.append(s_data_list)
eval_sentence_map[eval_sen_idx] = s_pos
eval_sentence_word_map[eval_sen_idx] = s_word
eval_sen_idx += 1
eval_batch_data = utils.construct_batch_data(eval_data_list,
options.batchsize)
eval_sentence_trans_param = np.zeros(
(len(eval_data_list), options.val_stc_length,
options.val_stc_length, options.c_valency))
parse_results = {}
for batch_id, one_batch in enumerate(eval_batch_data):
eval_batch_token, eval_batch_sen = [s[0] for s in one_batch], [
s[1][0] for s in one_batch
]
eval_batch_sen = np.array(eval_batch_sen)
eval_batch_token = np.array(eval_batch_token)
if dmv_model.initial_flag:
batch_score, batch_root_score, batch_decision_score = dmv_model.evaluate_batch_score(
eval_batch_token, eval_batch_sen, None, epoch)
else:
# batch_rule_samples = dmv_model.find_predict_samples(eval_batch_token, eval_batch_sen)
# batch_predict_data = utils.construct_ml_predict_data(batch_rule_samples)
# batch_predict_pos_v = torch.LongTensor(batch_predict_data['pos'])
# batch_predict_pos_index = np.array(batch_predict_data['pos'])
# batch_predict_dir_v = torch.LongTensor(batch_predict_data['dir'])
# batch_predict_dir_index = np.array(batch_predict_data['dir'])
# batch_predict_cvalency_v = torch.LongTensor(batch_predict_data['cvalency'])
# batch_predict_cvalency_index = np.array(batch_predict_data['cvalency'])
# batch_predict_sen_v = []
# for sentence_id in batch_predict_data['sentence']:
# batch_predict_sen_v.append(eval_sentence_map[sentence_id])
# batch_predict_sen_index = np.array(batch_predict_data['sentence'])
# batch_predict_sen_v = torch.LongTensor(batch_predict_sen_v)
# batch_predict_sen_len = torch.LongTensor(np.array([len(i) for i in batch_predict_sen_v]))
# batch_predict_lan_v = torch.LongTensor(np.array([lang_id for _ in batch_predict_sen_v])) # TODO
# batch_predicted, batch_word_predicted = m_model.forward_(batch_predict_pos_v, batch_predict_dir_v, batch_predict_cvalency_v,
# None, None, None, True, 'child', options.em_type, batch_predict_lan_v, batch_predict_sen_v,
# batch_predict_sen_len, epoch=epoch)
# eval_sentence_trans_param[batch_predict_sen_index, batch_predict_word_index, batch_predict_output_word_index, batch_predict_cvalency_index] = batch_predicted.detach().numpy()
# batch_score, batch_root_score, batch_decision_score = dmv_model.evaluate_batch_score(eval_batch_token, eval_batch_sen, eval_sentence_trans_param, epoch)
for i in range(len(one_batch)):
sentence_map = eval_sentence_map
sentence_word_map = eval_sentence_word_map
cvalency = options.c_valency
s = eval_batch_sen[i]
batch_target_lan_v = torch.LongTensor([lang_id]).expand(
len(sentence_map[s])**2) # TODO hanwj
batch_input_len = torch.LongTensor([len(
sentence_map[s])]).expand(len(sentence_map[s])**2)
batch_input_sen_v = torch.LongTensor([
sentence_map[s]
]).expand(len(sentence_map[s])**2, len(sentence_map[s]))
batch_input_sen_v_mlist = torch.LongTensor(
[sentence_word_map[s]]).expand(
len(sentence_word_map[s])**2,
len(sentence_word_map[s]))
one_batch_input_pos = torch.LongTensor([
sentence_map[s][h] for h in range(len(sentence_map[s]))
for _ in range(len(sentence_map[s]))
for v in range(cvalency)
])
one_batch_input_word = torch.LongTensor([
sentence_word_map[s][h]
for h in range(len(sentence_word_map[s]))
for c in range(len(sentence_word_map[s]))
for v in range(cvalency)
])
one_batch_output_word = torch.LongTensor([
sentence_word_map[s][c]
for h in range(len(sentence_word_map[s]))
for c in range(len(sentence_word_map[s]))
for v in range(cvalency)
])
one_batch_output_pos = torch.LongTensor([
sentence_map[s][c] for h in range(len(sentence_map[s]))
for c in range(len(sentence_map[s]))
for v in range(cvalency)
])
one_batch_dir = torch.LongTensor([
1 if h < c else 0 for h in range(len(sentence_map[s]))
for c in range(len(sentence_map[s]))
for v in range(cvalency)
])
one_batch_cvalency = torch.LongTensor([
v for h in range(len(sentence_map[s]))
for c in range(len(sentence_map[s]))
for v in range(cvalency)
])
one_batch_input_tag_index = np.array([
h for h in range(len(sentence_map[s]))
for c in range(len(sentence_map[s]))
for v in range(cvalency)
])
one_batch_input_word_index = np.array([
h for h in range(len(sentence_word_map[s]))
for c in range(len(sentence_word_map[s]))
for v in range(cvalency)
])
one_batch_output_tag_index = np.array([
c for h in range(len(sentence_map[s]))
for c in range(len(sentence_map[s]))
for v in range(cvalency)
])
one_batch_output_word_index = np.array([
c for h in range(len(sentence_word_map[s]))
for c in range(len(sentence_word_map[s]))
for v in range(cvalency)
])
one_batch_cvalency_index = np.array([
v for h in range(len(sentence_map[s]))
for c in range(len(sentence_map[s]))
for v in range(cvalency)
])
predicted_trans_param, predicted_trans_param_word = m_model.forward_(
one_batch_input_pos,
one_batch_input_word,
one_batch_input_word_index,
one_batch_dir,
one_batch_cvalency,
None,
None,
None,
True,
'child',
options.em_type,
batch_target_lan_v,
batch_input_sen_v_mlist,
batch_input_sen_v,
batch_input_len,
epoch=epoch)
# eval_sentence_trans_param[eval_batch_sen[s]][one_batch_input_word_index, one_batch_output_word_index, one_batch_cvalency_index] = predicted_trans_param_word.detach().numpy()[range((len(sentence_map[
# s]) ** 2) * cvalency), one_batch_output_word] # .reshape(one_batch_size, target_pos_num, 1, 1)
if options.full_lex:
eval_sentence_trans_param[s][
one_batch_input_word_index, one_batch_output_word_index, one_batch_cvalency_index] = \
predicted_trans_param_word.detach().numpy()[range((len(sentence_word_map[
s]) ** 2) * cvalency), one_batch_output_word] # .reshape(one_batch_size, target_pos_num, 1, 1)
else:
eval_sentence_trans_param[s][
one_batch_input_tag_index, one_batch_output_tag_index, one_batch_cvalency_index] = \
predicted_trans_param.detach().numpy()[range((len(sentence_map[
s]) ** 2) * cvalency), one_batch_output_pos] # .reshape(one_batch_size, target_pos_num, 1, 1)
batch_score, batch_root_score, batch_decision_score = dmv_model.evaluate_batch_score(
eval_batch_token, eval_batch_sen,
eval_sentence_trans_param, epoch)
batch_size, sentence_length, _, v_c_num = batch_score.shape
_, _, _, v_d_num, _ = batch_decision_score.shape
batch_score = np.concatenate(
(np.full((batch_size, 1, sentence_length, v_c_num),
-np.inf), batch_score),
axis=1) # for eisner
batch_score = np.concatenate(
(np.full((batch_size, sentence_length + 1, 1, v_c_num),
-np.inf), batch_score),
axis=2) # for eisner
batch_score[:, 0, 1:, 0] = batch_root_score
batch_decision_score = np.concatenate((np.zeros(
(batch_size, 1, 2, v_d_num, 2)), batch_decision_score),
axis=1)
batch_score = np.expand_dims(batch_score, 3)
batch_score = np.expand_dims(batch_score, 4)
batch_decision_score = np.expand_dims(batch_decision_score, 2)
batch_parse = eisner_for_dmv.batch_parse(batch_score,
batch_decision_score,
dmv_model.dvalency,
dmv_model.cvalency)
for i in range(len(eval_batch_token)):
parse_results[eval_batch_sen[i]] = (batch_parse[0][i],
batch_parse[1][i])
utils.eval(parse_results, eval_sentences, devpath,
options.log + '_dev' + str(options.sample_idx), epoch)
# utils.write_distribution(dmv_model)
print("====================================")