dev_best_score = None
test_best_score = None
test_ood_best_score = None
for epoch in range(0):
epoch_start = time.time()
for batch_i, train_input_data in enumerate(
inter_utils.get_batch(train_dataset,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=True,
withParrallel=True)):
srl_model.train()
target_argument = train_input_data['argument']
flat_argument = train_input_data['flat_argument']
target_batch_variable = get_torch_variable_from_np(
flat_argument)
out = srl_model(train_input_data,
elmo,
withParallel=False,
log('\nStart training...')
dev_best_score = None
test_best_score = None
test_ood_best_score = None
for epoch in range(max_epoch):
epoch_start = time.time()
for batch_i, train_input_data in enumerate(
inter_utils.get_batch(train_dataset,
batch_size,
word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=True)):
target_argument = train_input_data['argument']
flat_argument = train_input_data['flat_argument']
gold_pos = train_input_data['gold_pos']
gold_PI = train_input_data['predicates_flag']
gold_deprel = train_input_data['sep_dep_rel']
def eval_data(model, elmo, dataset, batch_size ,word2idx, fr_word2idx, lemma2idx, pos2idx, pretrain2idx, fr_pretrain2idx, deprel2idx, argument2idx, idx2argument, idx2word, unify_pred = False, predicate_correct=0, predicate_sum=0, isPretrain=False):
model.eval()
golden = []
predict = []
output_data = []
cur_sentence = None
cur_sentence_data = None
for batch_i, input_data in enumerate(inter_utils.get_batch(dataset, batch_size, word2idx, fr_word2idx,
lemma2idx, pos2idx, pretrain2idx,
fr_pretrain2idx, deprel2idx, argument2idx, idx2word, lang="En")):
target_flags = input_data['sen_flags']
flat_argument = input_data['flat_argument']
target_batch_variable = get_torch_variable_from_np(flat_argument)
sentence_id = input_data['sentence_id']
predicate_id = input_data['predicate_id']
word_id = input_data['word_id']
sentence_len = input_data['sentence_len']
seq_len = input_data['seq_len']
bs = input_data['batch_size']
psl = input_data['pad_seq_len']
out = model(input_data, lang='En')
_, pred = torch.max(out, 1)
pred = get_data(pred)
pred = np.reshape(pred, target_flags.shape)
for idx in range(pred.shape[0]):
predict.append(list(pred[idx]))
golden.append(list(target_flags[idx]))
pre_data = []
for b in range(len(seq_len)):
line_data = ['_' for _ in range(sentence_len[b])]
for s in range(seq_len[b]):
wid = word_id[b][s]
#line_data[wid-1] = idx2argument[pred[b][s]]
line_data[wid - 1] = str(pred[b][s])
pre_data.append(line_data)
for b in range(len(sentence_id)):
if cur_sentence != sentence_id[b]:
if cur_sentence_data is not None:
output_data.append(cur_sentence_data)
cur_sentence_data = [[sentence_id[b]]*len(pre_data[b]),pre_data[b]]
cur_sentence = sentence_id[b]
else:
assert cur_sentence_data is not None
cur_sentence_data.append(pre_data[b])
if cur_sentence_data is not None and len(cur_sentence_data)>0:
output_data.append(cur_sentence_data)
score = sem_f1_score(golden, predict, argument2idx, unify_pred, predicate_correct, predicate_sum)
"""
P = correct_pos / NonullPredict_pos
R = correct_pos / NonullTruth_pos
F = 2 * P * R / (P + R)
log("POS: ", P, R, F)
P = correct_PI / NonullPredict_PI
R = correct_PI / NonullTruth_PI
F = 2 * P * R / (P + R)
log("PI: ", P, R, F)
P = correct_deprel / NonullPredict_deprel
R = correct_deprel / NonullTruth_deprel
F = 2 * P * R / (P + R)
log("deprel: ", P, R, F)
P = correct_link / NonullPredict_link
R = correct_link / NonullTruth_link
F = 2 * P * R / (P + R)
log(correct_link, NonullPredict_link, NonullTruth_link)
log("link: ", P, R, F)
"""
model.train()
return score, output_data
def train_1_epoc(srl_model,
criterion,
optimizer,
train_dataset,
labeled_dataset_fr,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
lang='En',
dev_best_score=None,
test_best_score=None,
test_ood_best_score=None):
for batch_i, train_input_data in enumerate(
inter_utils.get_batch(train_dataset,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=shuffle,
lang=lang)):
flat_argument = train_input_data['flat_argument']
target_batch_variable = get_torch_variable_from_np(flat_argument)
out, out_word = srl_model(train_input_data, lang='En')
loss = criterion(out, target_batch_variable)
loss_word = criterion(out_word, target_batch_variable)
if batch_i % 50 == 0:
log(batch_i, loss, loss_word)
optimizer.zero_grad()
(loss + loss_word).backward()
optimizer.step()
if batch_i > 0 and batch_i % show_steps == 0:
_, pred = torch.max(out, 1)
pred = get_data(pred)
# pred = pred.reshape([bs, sl])
log('\n')
log('*' * 80)
eval_train_batch(epoch, batch_i, loss.data[0], flat_argument, pred,
argument2idx)
log('FR test:')
score, dev_output = eval_data(srl_model,
elmo,
labeled_dataset_fr,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2argument,
idx2word,
False,
dev_predicate_correct,
dev_predicate_sum,
lang='Fr')
if dev_best_score is None or score[5] > dev_best_score[5]:
dev_best_score = score
output_predict(
os.path.join(
result_path, 'dev_argument_{:.2f}.pred'.format(
dev_best_score[2] * 100)), dev_output)
# torch.save(srl_model, os.path.join(os.path.dirname(__file__),'model/best_{:.2f}.pkl'.format(dev_best_score[2]*100)))
log('\tdev best P:{:.2f} R:{:.2f} F1:{:.2f} NP:{:.2f} NR:{:.2f} NF1:{:.2f}'
.format(dev_best_score[0] * 100, dev_best_score[1] * 100,
dev_best_score[2] * 100, dev_best_score[3] * 100,
dev_best_score[4] * 100, dev_best_score[5] * 100))
return dev_best_score
def parallel_train_1_epoc(srl_model,
criterion,
optimizer,
train_dataset,
labeled_dataset_fr,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
lang='En',
dev_best_score=None,
test_best_score=None,
test_ood_best_score=None):
unlabeled_dataset_en, unlabeled_dataset_fr = train_dataset
unlabeled_Generator_En = inter_utils.get_batch(unlabeled_dataset_en,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
lang="En")
for batch_i, unlabeled_data_fr in enumerate(
inter_utils.get_batch(unlabeled_dataset_fr,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
lang='Fr')):
srl_model.train()
unlabeled_data_en = unlabeled_Generator_En.next()
predicates_1D = unlabeled_data_en['predicates_idx']
predicates_1D_fr = unlabeled_data_fr['predicates_idx']
#log(predicates_1D, predicates_1D_fr)
u_loss = srl_model((unlabeled_data_en, unlabeled_data_fr),
lang='En',
unlabeled='True')
optimizer.zero_grad()
u_loss.backward()
optimizer.step()
if batch_i % 50 == 0:
log(batch_i, u_loss)
if batch_i % 500 == 0:
log('\n')
log('*' * 80)
srl_model.eval()
# eval_train_batch(epoch, batch_i, loss.data[0], flat_argument, pred, argument2idx)
log('FR test:')
score, dev_output = eval_data(srl_model,
elmo,
labeled_dataset_fr,
30,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2argument,
idx2word,
False,
dev_predicate_correct,
dev_predicate_sum,
lang='Fr')
if dev_best_score is None or score[5] > dev_best_score[5]:
dev_best_score = score
output_predict(
os.path.join(
result_path, 'dev_argument_{:.2f}.pred'.format(
dev_best_score[2] * 100)), dev_output)
# torch.save(srl_model, os.path.join(os.path.dirname(__file__),'model/best_{:.2f}.pkl'.format(dev_best_score[2]*100)))
log('\tdev best P:{:.2f} R:{:.2f} F1:{:.2f} NP:{:.2f} NR:{:.2f} NF1:{:.2f}'
.format(dev_best_score[0] * 100, dev_best_score[1] * 100,
dev_best_score[2] * 100, dev_best_score[3] * 100,
dev_best_score[4] * 100, dev_best_score[5] * 100))
return dev_best_score
print(srl_model)
print('\t model build finished! consuming {} s'.format(
int(time.time() - start_t)))
print('\nStart training...')
dev_best_score = None
test_best_score = None
test_ood_best_score = None
use_bert = True
unlabeled_Generator_En = inter_utils.get_batch(unlabeled_dataset_en,
batch_size,
argument2idx,
shuffle=False,
lang="En",
use_bert=use_bert,
para=True)
unlabeled_Generator_Fr = inter_utils.get_batch(unlabeled_dataset_fr,
batch_size,
argument2idx,
shuffle=False,
lang="Fr",
use_bert=use_bert,
para=True)
for epoch in range(30):
for batch_i, train_input_data in enumerate(
inter_utils.get_batch(train_dataset,
log('\t model build finished! consuming {} s'.format(
int(time.time() - start_t)))
log('\nStart training...')
dev_best_score = None
test_best_score = None
test_ood_best_score = None
unlabeled_Generator_En = inter_utils.get_batch(unlabeled_dataset_en,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
lang="En")
unlabeled_Generator_Fr = inter_utils.get_batch(unlabeled_dataset_fr,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
dev_best_score = None
test_best_score = None
test_ood_best_score = None
for epoch in range(max_epoch):
epoch_start = time.time()
for batch_i, train_input_data in enumerate(
inter_utils.get_batch(train_dataset,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
withParrallel=True)):
target_argument = train_input_data['argument']
flat_argument = train_input_data['flat_argument']
gold_pos = train_input_data['gold_pos']
gold_PI = train_input_data['predicates_flag']
def pruning_eval_data(model,
elmo,
dataset,
batch_size,
out_of_pruning,
word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
deprel2idx,
argument2idx,
idx2argument,
unify_pred=False,
predicate_correct=0,
predicate_sum=0):
model.eval()
golden = []
predict = []
output_data = []
cur_sentence = None
cur_sentence_data = None
for batch_i, input_data in enumerate(
inter_utils.get_batch(dataset, batch_size, word2idx, lemma2idx,
pos2idx, pretrain2idx, deprel2idx,
argument2idx)):
target_argument = input_data['argument']
flat_argument = input_data['flat_argument']
target_batch_variable = get_torch_variable_from_np(flat_argument)
sentence_id = input_data['sentence_id']
predicate_id = input_data['predicate_id']
word_id = input_data['word_id']
sentence_len = input_data['sentence_len']
seq_len = input_data['seq_len']
bs = input_data['batch_size']
psl = input_data['pad_seq_len']
out = model(input_data, elmo)
# loss = criterion(out, target_batch_variable)
_, pred = torch.max(out, 1)
pred = get_data(pred)
# print(target_argument.shape)
# print(pred.shape)
pred = np.reshape(pred, target_argument.shape)
# golden += flat_argument.tolist()
# predict += pred
for idx in range(pred.shape[0]):
predict.append(list(pred[idx]))
golden.append(list(target_argument[idx]))
pre_data = []
for b in range(len(seq_len)):
line_data = ['_' for _ in range(sentence_len[b])]
for s in range(seq_len[b]):
wid = word_id[b][s]
line_data[wid - 1] = idx2argument[pred[b][s]]
pre_data.append(line_data)
for b in range(len(sentence_id)):
if cur_sentence != sentence_id[b]:
if cur_sentence_data is not None:
output_data.append(cur_sentence_data)
cur_sentence_data = [[sentence_id[b]] * len(pre_data[b]),
pre_data[b]]
cur_sentence = sentence_id[b]
else:
assert cur_sentence_data is not None
cur_sentence_data.append(pre_data[b])
if cur_sentence_data is not None and len(cur_sentence_data) > 0:
output_data.append(cur_sentence_data)
score = pruning_sem_f1_score(golden, predict, out_of_pruning, argument2idx,
unify_pred, predicate_correct, predicate_sum)
model.train()
return score, output_data
dev_best_score = None
test_best_score = None
test_ood_best_score = None
for epoch in range(max_epoch):
epoch_start = time.time()
for batch_i, train_input_data in enumerate(
inter_utils.get_batch(train_dataset,
batch_size,
word2idx,
fr_word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
fr_pretrain2idx,
deprel2idx,
argument2idx,
idx2word,
shuffle=False,
lang='Fr')):
flat_flags = train_input_data['flat_flags']
target_batch_variable = get_torch_variable_from_np(flat_flags)
bs = train_input_data['batch_size']
sl = train_input_data['seq_len']
out = srl_model(train_input_data, lang='En')
loss = criterion(out, target_batch_variable)
if batch_i % 50 == 0:
def my_eval(epoch,
step,
model,
criterion,
dataset,
batch_size,
predicate_correct,
predicate_sum,
word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
argument2idx,
use_attention=False,
aug_data=None):
model.eval()
gold = predict = correct = total = num_correct = 0
for batch_i, input_data in enumerate(
inter_utils.get_batch(dataset, batch_size, word2idx, lemma2idx,
pos2idx, pretrain2idx, argument2idx)):
target_argument = input_data["argument"]
flat_argument = input_data["flat_argument"]
target_batch_variable = get_torch_variable_from_np(flat_argument)
if use_attention:
out = model((input_data, aug_data))
else:
out = model(input_data)
_, pred = torch.max(out, 1)
pred = get_data(pred)
loss = criterion(out, target_batch_variable)
cur_correct, cur_golden, cur_predict, cur_total, cur_nc = my_eval_batch(
flat_argument, pred, argument2idx)
gold += cur_golden
predict += cur_predict
correct += cur_correct
total += cur_total
num_correct += cur_nc
P = (correct + predicate_correct) / (predict + predicate_sum + 1e-13)
R = (correct + predicate_correct) / (gold + predicate_sum + 1e-13)
NP = correct / (predict + 1e-13)
NR = correct / (gold + 1e-13)
F1 = 2 * P * R / (P + R + 1e-13)
NF1 = 2 * NP * NR / (NP + NR + 1e-13)
acc = num_correct / total
print_dict = {
'e': epoch,
's': step,
'acc': acc,
'p': predict,
'g': gold,
'c': correct,
'P': P,
'R': R,
'F1': F1,
'NP': NP,
'NR': NR,
'NF1': NF1
}
print_result(print_dict)
model.train()
return print_dict
def eval_data(model,
elmo,
dataset,
batch_size,
word2idx,
lemma2idx,
pos2idx,
pretrain2idx,
deprel2idx,
argument2idx,
idx2argument,
idx2word,
unify_pred=False,
predicate_correct=0,
predicate_sum=0):
model.eval()
golden = []
predict = []
output_data = []
cur_sentence = None
cur_sentence_data = None
correct_pos, NonullPredict_pos, NonullTruth_pos = 0.1, 0.1, 0.
correct_PI, NonullPredict_PI, NonullTruth_PI = 0., 0., 0.
correct_deprel, NonullPredict_deprel, NonullTruth_deprel = 0., 0., 0.
correct_link, NonullPredict_link, NonullTruth_link = 0., 0., 0.
for batch_i, input_data in enumerate(
inter_utils.get_batch(dataset, batch_size, word2idx, lemma2idx,
pos2idx, pretrain2idx, deprel2idx,
argument2idx, idx2word)):
target_argument = input_data['argument']
flat_argument = input_data['flat_argument']
gold_pos = input_data['gold_pos']
gold_PI = input_data['predicates_flag']
gold_deprel = input_data['sep_dep_rel']
gold_link = input_data['sep_dep_link']
target_batch_variable = get_torch_variable_from_np(flat_argument)
gold_pos_batch_variable = get_torch_variable_from_np(gold_pos)
gold_PI_batch_variable = get_torch_variable_from_np(gold_PI)
gold_deprel_batch_variable = get_torch_variable_from_np(gold_deprel)
gold_link_batch_variable = get_torch_variable_from_np(gold_link)
sentence_id = input_data['sentence_id']
predicate_id = input_data['predicate_id']
word_id = input_data['word_id']
sentence_len = input_data['sentence_len']
seq_len = input_data['seq_len']
bs = input_data['batch_size']
psl = input_data['pad_seq_len']
out, out_pos, out_PI, out_deprel, out_link = model(input_data, elmo)
a, b, c = get_PRF(out_pos, gold_pos_batch_variable.view(-1))
correct_pos += a
NonullPredict_pos += b
NonullTruth_pos += c
a, b, c = get_PRF(out_PI, gold_PI_batch_variable.view(-1))
correct_PI += a
NonullPredict_PI += b
NonullTruth_PI += c
a, b, c = get_PRF(out_deprel, gold_deprel_batch_variable.view(-1))
correct_deprel += a
NonullPredict_deprel += b
NonullTruth_deprel += c
a, b, c = get_PRF(out_link, gold_link_batch_variable.view(-1))
correct_link += a
NonullPredict_link += b
NonullTruth_link += c
_, pred = torch.max(out, 1)
pred = get_data(pred)
pred = np.reshape(pred, target_argument.shape)
for idx in range(pred.shape[0]):
predict.append(list(pred[idx]))
golden.append(list(target_argument[idx]))
pre_data = []
for b in range(len(seq_len)):
line_data = ['_' for _ in range(sentence_len[b])]
for s in range(seq_len[b]):
wid = word_id[b][s]
line_data[wid - 1] = idx2argument[pred[b][s]]
pre_data.append(line_data)
for b in range(len(sentence_id)):
if cur_sentence != sentence_id[b]:
if cur_sentence_data is not None:
output_data.append(cur_sentence_data)
cur_sentence_data = [[sentence_id[b]] * len(pre_data[b]),
pre_data[b]]
cur_sentence = sentence_id[b]
else:
assert cur_sentence_data is not None
cur_sentence_data.append(pre_data[b])
if cur_sentence_data is not None and len(cur_sentence_data) > 0:
output_data.append(cur_sentence_data)
score = sem_f1_score(golden, predict, argument2idx, unify_pred,
predicate_correct, predicate_sum)
P = correct_pos / NonullPredict_pos
R = correct_pos / NonullTruth_pos
F = 2 * P * R / (P + R)
log("POS: ", P, R, F)
P = correct_PI / NonullPredict_PI
R = correct_PI / NonullTruth_PI
F = 2 * P * R / (P + R)
log("PI: ", P, R, F)
P = correct_deprel / NonullPredict_deprel
R = correct_deprel / NonullTruth_deprel
F = 2 * P * R / (P + R)
log("deprel: ", P, R, F)
P = correct_link / NonullPredict_link
R = correct_link / NonullTruth_link
F = 2 * P * R / (P + R)
log(correct_link, NonullPredict_link, NonullTruth_link)
log("link: ", P, R, F)
model.train()
return score, output_data
