def a3_test() -> ScoreHolder[bool]:
def intern(x: float, y: float) -> bool:
return not isclose(x, y) and x > y
gold = [{1, 2}, {3}]
sys = [{1, 2, 3}]
return ScoreHolder.apply(evaluates(gold, gold), intern,
evaluates(gold, sys))
def d2_test() -> ScoreHolder[bool]:
def intern(x: float, y: float) -> bool:
return isclose(x, y)
gold = [{1, 2, 3, 4, 5}, {6, 7, 8, 9, 10}, {11, 12, 13}, {14, 15, 16}]
sys1 = [{1, 2, 3, 4, 6}, {5, 7, 8, 9, 10}, {11, 12, 13}, {14, 15, 16}]
sys2 = [{1, 2, 3, 4, 5}, {6, 7, 8, 9, 10}, {11, 12, 14}, {13, 15, 16}]
return ScoreHolder.apply(evaluates(gold, sys1), intern,
evaluates(gold, sys2))
def b2_test() -> ScoreHolder[bool]:
def intern(x: float, y: float) -> bool:
return not isclose(x, y) and x < y
gold = [{1, 2, 3, 4, 5}, {6, 7}]
sys1 = [{1, 2, 3, 4}, {5, 6, 7}]
sys2 = [{1, 2}, {3, 4, 5}, {6, 7}]
return ScoreHolder.apply(evaluates(gold, sys1), intern,
evaluates(gold, sys2))
def d1_test() -> ScoreHolder[bool]:
def intern(x: float, y: float) -> bool:
return isclose(x, y)
gold1 = [{1, 2}, {3, 4}, {5, 6}]
sys1 = [{1, 3}, {2, 4}, {5, 6}]
gold2 = [{1, 2}, {3, 4}, {5, 6, 7, 8}]
sys2 = [{1, 3}, {2, 4}, {5, 6, 7, 8}]
return ScoreHolder.apply(evaluates(gold1, sys1), intern,
evaluates(gold2, sys2))
def metric_1_symetry_test(gold: Partition,
sys: Partition) -> ScoreHolder[bool]:
"""
evaluates whether score(gold, sys) = score(sys, gold)
"""
def intern(x: float, y: float) -> bool:
return isclose(x, y)
return ScoreHolder.apply(evaluates(gold, sys), intern,
evaluates(sys, gold))
def metric_4_triangle_test(a: Partition, b: Partition,
c: Partition) -> ScoreHolder[bool]:
"""
evaluates whether the (similarity) triangle inequality is respected
s(a,b) + s(b,c) <= s(b,b) + s(a,b)
"""
def intern(x: float, y: float) -> bool:
return isclose(x, y) or x < y
return ScoreHolder.apply(
evaluates(a, b) + evaluates(b, c), intern,
evaluates(b, b) + evaluates(a, c))
def identity_test(gold: Partition) -> ScoreHolder[bool]:
"""
evaluates whether score(gold, gold) = 1
"""
def intern(x: float) -> bool:
return isclose(x, 1)
return evaluates(gold, gold).apply_to_values(intern)
def metric_5_indiscernable(gold: Partition,
sys: Partition) -> ScoreHolder[bool]:
def intern1(x: float, y: float):
return isclose(x, y)
def intern2(x: bool, y: bool):
return x & y
a = evaluates(gold, gold)
b = evaluates(sys, sys)
c = evaluates(gold, sys)
x = ScoreHolder.apply(a, intern1, b)
y = ScoreHolder.apply(a, intern1, c)
z = ScoreHolder.apply(b, intern1, c)
if gold == sys:
return ScoreHolder.apply(ScoreHolder.apply(x, intern2, y), intern2, z)
else:
return ScoreHolder.apply(ScoreHolder.apply(x, intern2, y), intern2,
z).apply_to_values(lambda k: not k)
def non_identity_test(gold: Partition,
sys: Partition) -> Optional[ScoreHolder[bool]]:
"""
evaluates whether score(gold, sys) != 1 with gold != sys
"""
def intern(x: float) -> bool:
return not isclose(x, 1)
#return EasyFail.has_passed_test(not isclose(x, 1))
if gold == sys:
return None
return evaluates(gold, sys).apply_to_values(intern)
def metric_8(gold: Partition, sys: Partition) -> ScoreHolder[bool]:
def intern(x: float):
return isclose(x, 0) or x > 0
return evaluates(gold, sys).apply_to_values(intern)
def metric_7(gold: Partition) -> ScoreHolder[bool]:
def intern(x: float):
return isclose(x, 1)
return evaluates(gold, gold).apply_to_values(intern)
def metric_3(gold: Partition, sys: Partition) -> ScoreHolder[bool]:
def intern(x: float, y: float) -> bool:
return isclose(x, y) or x > y
return ScoreHolder.apply(evaluates(gold, gold), intern,
evaluates(gold, sys))
def metric_2_non_negativity_test(gold: Partition) -> ScoreHolder[bool]:
def intern(x: float) -> bool:
return isclose(x, 0) or x > 0
return evaluates(gold, gold).apply_to_values(intern)
def entity_test(gold: Partition) -> ScoreHolder[float]:
"""
evaluates the score of a partition against a partition composed of only one entity
"""
return evaluates(gold, entity_partition(get_mentions(gold)))
def singleton_test(gold: Partition) -> ScoreHolder[float]:
"""
evaluates the score of a partition against a partition composed only of singletons
"""
return evaluates(gold, singleton_partition(get_mentions(gold)))
def f_test(gold: Partition, sys: Partition) -> ScoreHolder[bool]:
def intern(x: float) -> bool:
return isclose(x, 0)
return ScoreHolder.apply_to_values(evaluates(gold, sys), intern)
