reentrancies_pred.append(reentrancies(dict_pred, triples_pred))
reentrancies_gold.append(reentrancies(dict_gold, triples_gold))
srl_pred.append(srl(dict_pred, triples_pred))
srl_gold.append(srl(dict_gold, triples_gold))
for score in preds:
if preds[score] > 0:
pr = inters[score] / float(preds[score])
else:
pr = 0
if golds[score] > 0:
rc = inters[score] / float(golds[score])
else:
rc = 0
if pr + rc > 0:
f = 2 * (pr * rc) / (pr + rc)
print(score, '-> P:', f"{pr:.{significant}f}", ', R:',
f"{rc:.{significant}f}", ', F:', f"{f:.{significant}f}")
else:
print(score, '-> P:', f"{pr:.{significant}f}", ', R:',
f"{rc:.{significant}f}", ', F: 0.00')
pr, rc, f = smatch.main(reentrancies_pred, reentrancies_gold, True, r,
significant)
print('Reentrancies -> P:', f"{float(pr):.{significant}f}", ', R:',
f"{float(rc):.{significant}f}", ', F:', f"{float(f):.{significant}f}")
pr, rc, f = smatch.main(srl_pred, srl_gold, True, r, significant)
print('SRL -> P:', f"{float(pr):.{significant}f}", ', R:',
f"{float(rc):.{significant}f}", ', F:', f"{float(f):.{significant}f}")
golds["Wikification"] += len(set(list_gold))
reentrancies_pred.append(reentrancies(dict_pred, triples_pred))
reentrancies_gold.append(reentrancies(dict_gold, triples_gold))
srl_pred.append(srl(dict_pred, triples_pred))
srl_gold.append(srl(dict_gold, triples_gold))
for score in preds:
print score, "->",
if preds[score] > 0:
pr = inters[score] / float(preds[score])
else:
pr = 0
if golds[score] > 0:
rc = inters[score] / float(golds[score])
else:
rc = 0
if pr + rc > 0:
f = 2 * (pr * rc) / (pr + rc)
print 'P: %.2f, R: %.2f, F: %.2f' % (float(pr), float(rc), float(f))
else:
print 'P: %.2f, R: %.2f, F: %.2f' % (float(pr), float(rc),
float("0.00"))
pr, rc, f = smatch.main(reentrancies_pred, reentrancies_gold, True)
print 'Reentrancies -> P: %.2f, R: %.2f, F: %.4f' % (float(pr), float(rc),
float(f))
pr, rc, f = smatch.main(srl_pred, srl_gold, True)
print 'SRL -> P: %.2f, R: %.2f, F: %.2f' % (float(pr), float(rc), float(f))
def main(args):
data = pickle.load(open(args.data, "rb"))
gold_graphs, old_graphs, new_graphs = [], [], []
gold_graphs_reentrancies = []
old_graphs_reentrancies = []
new_graphs_reentrancies = []
if args.method == 'ORACLE':
actions = [x[2] for x in data]
elif args.method == 'RANDOM':
actions = None
elif args.method == 'PATTERN':
actions = open('patterns_train17_thresh.txt').read().splitlines()
# else: # S2S
# actions = []
# for line in open(args.method):
# # actions.append(from_annot(line))
# if line.strip() == 'EMPTY':
# actions.append({args.action_type: []})
# continue
# else:
# words = line.strip().split()
# i = 0
# actions_line = []
# while i < len(words):
# if i + 2 >= len(words) or \
# 'EMPTY' == words[i] or \
# 'EMPTY' == words[i + 1] or \
# 'EMPTY' == words[i + 2]:
# i += 3
# continue
# actions_line.append(
# (words[i], words[i + 1], words[i + 2])
# )
# i += 3
# actions.append({args.action_type: actions_line})
counts = None
for example in data:
_, src, _, gold = example
gold_graphs.append(gold)
gold_graphs_reentrancies.append(reentrancies(gold[0], gold[1]))
old_graphs.append(src)
old_graphs_reentrancies.append(reentrancies(src[0], src[1]))
if args.method == 'ORACLE':
new_graphs, new_graphs_reentrancies, counts = apply_action_oracle(
old_graphs, gold_graphs, actions, args.action_type)
elif args.method == 'RANDOM':
new_graphs, new_graphs_reentrancies = apply_action_random(
old_graphs, gold_graphs, actions, args.action_type)
elif args.method == 'PATTERN':
new_graphs, new_graphs_reentrancies = apply_action_pattern(
old_graphs, gold_graphs, actions, args.action_type)
# else: # S2S
# new_graphs, new_graphs_reentrancies_s2s = apply_action(
# old_graphs, gold_graphs, actions, args.action_type
# )
if not args.silent:
print 'Action type:', args.action_type
if counts is not None:
print 'Number of actions:', counts
print 'Computing Smatch...'
print 'Old Smatch score:', smatch.main(old_graphs, gold_graphs,
True)[2]
print 'Old Reentrancy score:', \
smatch.main(old_graphs_reentrancies, gold_graphs_reentrancies, True)[2]
print 'New Smatch score:', smatch.main(new_graphs, gold_graphs,
True)[2]
print 'New Reentrancy score:', \
smatch.main(new_graphs_reentrancies, gold_graphs_reentrancies, True)[2]
else:
old_smatch_score = smatch.main(old_graphs, gold_graphs, True)[2]
old_reentr_score = smatch.main(old_graphs_reentrancies,
gold_graphs_reentrancies, True)[2]
new_smatch_score = smatch.main(new_graphs, gold_graphs, True)[2]
new_reentr_score = smatch.main(new_graphs_reentrancies,
gold_graphs_reentrancies, True)[2]
print \
counts, \
"{:0.1f}".format(100 * (float(new_smatch_score) - float(old_smatch_score))), \
"{:0.1f}".format(100 * (float(new_reentr_score) - float(old_reentr_score))),
pickle.dump(new_graphs, open(args.data + '.new_graphs.p', "wb"))