def smatch(gold, system, limit=20, values={}, trace=0, faith=True):
gprefix = "g"
sprefix = "s"
ginstances, gattributes, grelations, gn \
= tuples(gold, gprefix, values, faith)
sinstances, sattributes, srelations, sn \
= tuples(system, sprefix, values, faith)
if trace > 1:
print("gold instances [{}]: {}\ngold attributes [{}]: {}\n"
"gold relations [{}]: {}"
"".format(len(ginstances), ginstances, len(gattributes),
gattributes, len(grelations), grelations),
file=sys.stderr)
print("system instances [{}]: {}\nsystem attributes [{}]: {}\n"
"system relations [{}]: {}"
"".format(len(sinstances), sinstances, len(sattributes),
sattributes, len(srelations), srelations),
file=sys.stderr)
correct, gold, system, mapping \
= get_amr_match(None, None, gold.id, limit = limit,
instance1 = ginstances, attributes1 = gattributes,
relation1 = grelations, prefix1 = gprefix,
instance2 = sinstances, attributes2 = sattributes,
relation2 = srelations, prefix2 = sprefix)
return correct, gold - gn, system - sn, mapping
def evaluate(golds, systems, format="json", limit=5, trace=0, useanchor=False):
if not limit: limit = 5
tg = ts = tc = n = 0
gprefix = "g"
sprefix = "s"
scores = dict() if trace else None
for gold, system in intersect(golds, systems):
id = gold.id
ginstances, gattributes, grelations, gn = tuples(
gold, gprefix, useanchor)
sinstances, sattributes, srelations, sn = tuples(
system, sprefix, useanchor)
if trace > 1:
print("gold instances: {}\ngold attributes: {}\ngold relations: {}"
"".format(ginstances, gattributes, grelations))
print(
"system instances: {}\nsystem attributes: {}\nsystem relations: {}"
"".format(sinstances, sattributes, srelations))
correct, gold, system \
= get_amr_match(None, None, gold.id, limit = limit,
instance1 = ginstances, attributes1 = gattributes,
relation1 = grelations, prefix1 = gprefix,
instance2 = sinstances, attributes2 = sattributes,
relation2 = srelations, prefix2 = sprefix)
gold -= gn
system -= sn
tg += gold
ts += system
tc += correct
n += 1
if trace:
if id in scores:
print("smatch.evaluate(): duplicate graph identifier: {}"
"".format(id),
file=sys.stderr)
scores[id] = {
"g": gold,
"s": system,
"c": correct
}
if trace > 1:
p, r, f = fscore(gold, system, correct)
print("G: {}; S: {}; C: {}; P: {}; R: {}; F: {}"
"".format(gold, system, correct, p, r, f),
file=sys.stderr)
p, r, f = fscore(tg, ts, tc)
result = {
"n": n,
"g": tg,
"s": ts,
"c": tc,
"p": p,
"r": r,
"f": f
}
if trace: result["scores"] = scores
return result
def evaluate(golds, systems, format = "json", limit = 5, trace = 0):
if not limit: limit = 5;
tg = ts = tc = n = 0;
gprefix = "g"; sprefix = "s";
scores = dict() if trace else None;
for gold, system in intersect(golds, systems):
id = gold.id;
ginstances, gattributes, grelations = tuples(gold, gprefix);
sinstances, sattributes, srelations = tuples(system, sprefix);
correct, gold, system \
= get_amr_match(None, None, gold.id, limit = limit,
instance1 = ginstances, attributes1 = gattributes,
relation1 = grelations, prefix1 = gprefix,
instance2 = sinstances, attributes2 = sattributes,
relation2 = srelations, prefix2 = sprefix);
tg += gold; ts += system; tc += correct;
n += 1;
if trace:
if id in scores:
print("smatch.evaluate(): duplicate graph identifier: {}"
"".format(id), file = sys.stderr);
scores[id] = {"g": gold, "s": system, "c": correct};
if trace > 1:
p, r, f = fscore(gold, system, correct);
print("G: {}; S: {}; C: {}; P: {}; R: {}; F: {}"
"".format(gold, system, correct, p, r, f), file = sys.stderr);
if f != 1.0:
print("gold instances: {}\ngold attributes {}\ngold relations: {}"
"".format(ginstances, gattributes, grelations), file=sys.stderr);
print("system instances: {}\nsystem attributes {}\nsystem relations: {}"
"".format(sinstances, sattributes, srelations), file=sys.stderr);
p, r, f = fscore(tg, ts, tc);
result = {"n": n, "g": tg, "s": ts, "c": tc, "p": p, "r": r, "f": f};
if trace: result["scores"] = scores;
return result;