def concept_mismatch(all_amr_list, comparison_amr_list, concept):
"""Find cases where dev amr should have 'possibilty' but doesn't"""
missing = list()
spurious = list()
correct = list()
for comparison_amr in comparison_amr_list:
matches = [
base_tuple for base_tuple in all_amr_list
if base_tuple[0] == comparison_amr[0]
]
if len(matches) > 0:
match = matches[0]
match_amr = AMR.parse_AMR_line(match[2])
match_nodes = match_amr.node_values
comparison_nodes = AMR.parse_AMR_line(
comparison_amr[2]).node_values
id_number = int(comparison_amr[0].split('::')[0].split('.')[1])
if concept in match_nodes and concept not in comparison_nodes:
#missing.append(comparison_amr[0])
missing.append((id_number, comparison_amr[1]))
elif concept in comparison_nodes and concept not in match_nodes:
#spurious.append(comparison_amr[0])
spurious.append((id_number, comparison_amr[1]))
else:
#correct.append(comparison_amr[0])
correct.append((id_number, comparison_amr[1]))
return sorted(correct), sorted(missing), sorted(spurious)
def main(data):
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger(__name__)
test = codecs.open(data.test, 'r', 'utf-8')
gold = codecs.open(data.gold, 'r', 'utf-8')
flag = False
sema = Sema()
while True:
cur_amr1 = AMR.get_amr_line(test)
cur_amr2 = AMR.get_amr_line(gold)
if cur_amr1 == '' and cur_amr2 == '':
break
if cur_amr1 == '':
logger.error('Error: File 1 has less AMRs than file 2')
logger.error('Ignoring remaining AMRs')
flag = True
break
if cur_amr2 == '':
logger.error('Error: File 2 has less AMRs than file 1')
logger.error('Ignoring remaining AMRs')
flag = True
break
try:
amr1 = AMR.parse_AMR_line(cur_amr1)
except Exception as e:
logger.error('Error in parsing amr 1: %s' % cur_amr1)
logger.error(
"Please check if the AMR is ill-formatted. Ignoring remaining AMRs"
)
logger.error("Error message: %s" % str(e))
flag = True
break
try:
amr2 = AMR.parse_AMR_line(cur_amr2)
except Exception as e:
logger.error("Error in parsing amr 2: %s" % cur_amr2)
logger.error(
"Please check if the AMR is ill-formatted. Ignoring remaining AMRs"
)
logger.error("Error message: %s" % str(e))
flag = True
break
prefix_test = 'a'
prefix_gold = 'b'
amr1.rename_node(prefix_test)
amr2.rename_node(prefix_gold)
sema.compute_sema(amr1, amr2)
if not flag:
precision, recall, f1 = sema.get_sema_value()
print(f'SEMA: P {precision:.2f} R {recall:.2f} F1 {f1:.2f}')
def score_amr_pair(ref_amr_line, rec_amr_line, restart_num, justinstance=False,
justattribute=False, justrelation=False):
# parse lines
amr1 = AMR.parse_AMR_line(ref_amr_line)
amr2 = AMR.parse_AMR_line(rec_amr_line)
# Fix prefix
prefix1 = "a"
prefix2 = "b"
# Rename node to "a1", "a2", .etc
amr1.rename_node(prefix1)
# Renaming node to "b1", "b2", .etc
amr2.rename_node(prefix2)
# get triples
(instance1, attributes1, relation1) = amr1.get_triples()
(instance2, attributes2, relation2) = amr2.get_triples()
# optionally turn off some of the node comparison
doinstance = doattribute = dorelation = True
if justinstance:
doattribute = dorelation = False
if justattribute:
doinstance = dorelation = False
if justrelation:
doinstance = doattribute = False
(best_mapping, best_match_num) = smatch.get_best_match(
instance1, attributes1, relation1,
instance2, attributes2, relation2,
prefix1, prefix2,
restart_num,
doinstance=doinstance,
doattribute=doattribute,
dorelation=dorelation
)
if justinstance:
test_triple_num = len(instance1)
gold_triple_num = len(instance2)
elif justattribute:
test_triple_num = len(attributes1)
gold_triple_num = len(attributes2)
elif justrelation:
test_triple_num = len(relation1)
gold_triple_num = len(relation2)
else:
test_triple_num = len(instance1) + len(attributes1) + len(relation1)
gold_triple_num = len(instance2) + len(attributes2) + len(relation2)
return best_match_num, test_triple_num, gold_triple_num
def get_named_entities(all_amr_file=None):
"""Get all the named entities
Inputs:
amr_file: file with all the AMRs
Returns:
list of (id, snt, amr) tuples
"""
if all_amr_file is None:
all_amr_file = GOLD_AMRS
match_amrs = list() #(id,snt)
comments_and_amrs = read_amrz(all_amr_file) #(comment_list, amr_list)
comments = comments_and_amrs[0] #{'snt','id'}
amrs = comments_and_amrs[1]
for i in range(len(amrs)):
amr_graph = AMR.parse_AMR_line(amrs[i])
# amr_evaluation var2concept
v2c = {}
for n, v in zip(amr_graph.nodes, amr_graph.node_values):
v2c[n] = v
# print(v2c)
# I don't know why we need these indices but we do
triples = [t for t in amr_graph.get_triples()[1]]
triples.extend([t for t in amr_graph.get_triples()[2]])
#print(triples)
# named_ent(v2c, triples)
named_entities = [
str(v2c[v1]) for (l, v1, v2) in triples if l == "name"
]
print(named_entities)
def _smatch(cur_amr1, cur_amr2, n_iter):
clear_match_triple_dict()
amr1 = AMR.parse_AMR_line(cur_amr1)
amr2 = AMR.parse_AMR_line(cur_amr2)
prefix1 = "a"
prefix2 = "b"
amr1.rename_node(prefix1)
amr2.rename_node(prefix2)
instance1, attributes1, relation1 = amr1.get_triples()
instance2, attributes2, relation2 = amr2.get_triples()
best_mapping, best_match_num = get_best_match(instance1, attributes1,
relation1, instance2,
attributes2, relation2,
prefix1, prefix2)
test_triple_num = len(instance1) + len(attributes1) + len(relation1)
gold_triple_num = len(instance2) + len(attributes2) + len(relation2)
return best_match_num, test_triple_num, gold_triple_num
def get_amrs_with_concept(concept, all_amr_file=None):
"""Get the IDs of all AMRs with 'possible' concept
Inputs:
amr_file: file with all the AMRs
Returns:
list of (id, snt, amr) tuples
"""
if all_amr_file is None:
all_amr_file = GOLD_AMRS
match_amrs = list() #(id,snt)
comments_and_amrs = read_amrz(all_amr_file) #(comment_list, amr_list)
comments = comments_and_amrs[0] #{'snt','id'}
amrs = comments_and_amrs[1]
for i in range(len(amrs)):
amr_graph = AMR.parse_AMR_line(amrs[i])
node_values = amr_graph.node_values
if concept in node_values:
match_amrs.append((comments[i]['id'], comments[i]['snt'], amrs[i]))
#possible_ids.append((comments[i]['id'].encode('utf8'),comments[i]['snt'].encode('utf8'),amrs[i].encode('utf8')))
print("Total number of AMRs with '{}': {}".format(concept,
len(match_amrs)))
return sorted(match_amrs,
key=lambda x: int(x[0].split(' ')[0].split('.')[1])
) #sort by id number
def compute_subscores(pred, gold):
inters = defaultdict(int)
golds = defaultdict(int)
preds = defaultdict(int)
# Loop through all entries
for amr_pred, amr_gold in zip(pred, gold):
# Create the predicted data
amr_pred = AMR.parse_AMR_line(amr_pred.replace("\n", ""))
if amr_pred is None:
logger.error('Empty amr_pred entry')
continue
dict_pred = var2concept(amr_pred)
triples_pred = [t for t in amr_pred.get_triples()[1]]
triples_pred.extend([t for t in amr_pred.get_triples()[2]])
# Create the gold data
amr_gold = AMR.parse_AMR_line(amr_gold.replace("\n", ""))
if amr_gold is None:
logger.error('Empty amr_gold entry')
continue
dict_gold = var2concept(amr_gold)
triples_gold = [t for t in amr_gold.get_triples()[1]]
triples_gold.extend([t for t in amr_gold.get_triples()[2]])
# Non_sense_frames scores
list_pred = non_sense_frames(dict_pred)
list_gold = non_sense_frames(dict_gold)
inters["Non_sense_frames"] += len(
list(set(list_pred) & set(list_gold)))
preds["Non_sense_frames"] += len(set(list_pred))
golds["Non_sense_frames"] += len(set(list_gold))
# Wikification scores
list_pred = wikification(triples_pred)
list_gold = wikification(triples_gold)
inters["Wikification"] += len(list(set(list_pred) & set(list_gold)))
preds["Wikification"] += len(set(list_pred))
golds["Wikification"] += len(set(list_gold))
# Named entity scores
list_pred = namedent(dict_pred, triples_pred)
list_gold = namedent(dict_gold, triples_gold)
inters["Named Ent."] += len(list(set(list_pred) & set(list_gold)))
preds["Named Ent."] += len(set(list_pred))
golds["Named Ent."] += len(set(list_gold))
# Negation scores
list_pred = negations(dict_pred, triples_pred)
list_gold = negations(dict_gold, triples_gold)
inters["Negations"] += len(list(set(list_pred) & set(list_gold)))
preds["Negations"] += len(set(list_pred))
golds["Negations"] += len(set(list_gold))
# Ignore Vars scores
list_pred = everything(dict_pred, triples_pred)
list_gold = everything(dict_gold, triples_gold)
inters["IgnoreVars"] += len(list(set(list_pred) & set(list_gold)))
preds["IgnoreVars"] += len(set(list_pred))
golds["IgnoreVars"] += len(set(list_gold))
# Concepts scores
list_pred = concepts(dict_pred)
list_gold = concepts(dict_gold)
inters["Concepts"] += len(list(set(list_pred) & set(list_gold)))
preds["Concepts"] += len(set(list_pred))
golds["Concepts"] += len(set(list_gold))
# Frames scores
list_pred = frames(dict_pred)
list_gold = frames(dict_gold)
inters["Frames"] += len(list(set(list_pred) & set(list_gold)))
preds["Frames"] += len(set(list_pred))
golds["Frames"] += len(set(list_gold))
# Create the return dictionary
rdict = OrderedDict()
for score in preds:
pr = 0 if preds[score] <= 0 else inters[score] / float(preds[score])
rc = 0 if golds[score] <= 0 else inters[score] / float(golds[score])
f = 0 if pr + rc <= 0 else 2 * (pr * rc) / (pr + rc)
rdict[score] = (pr, rc, f)
return rdict
