def test_evaluate_multiple_tokens_no_match_match_correct_statistics():
prediction = ["O", "O", "O", "B-SPACESHIP", "L-SPACESHIP", "O"]
model = MockTokensModel(prediction=prediction)
evaluator = Evaluator(model=model, entities_to_keep=["ANIMAL"])
sample = InputSample("I am the walrus amaericanus magnifico",
masked=None,
spans=None)
sample.tokens = ["I", "am", "the", "walrus", "americanus", "magnifico"]
sample.tags = ["O", "O", "O", "B-ANIMAL", "I-ANIMAL", "L-ANIMAL"]
evaluated = evaluator.evaluate_sample(sample, prediction)
evaluation = evaluator.calculate_score([evaluated])
assert np.isnan(evaluation.pii_precision)
assert evaluation.pii_recall == 0
def test_evaluate_multiple_entities_to_keep_correct_statistics():
prediction = ["O", "U-ANIMAL", "O", "U-ANIMAL"]
entities_to_keep = ["ANIMAL", "PLANT", "SPACESHIP"]
model = MockTokensModel(prediction=prediction)
evaluator = Evaluator(model=model, entities_to_keep=entities_to_keep)
sample = InputSample(full_text="I dog the walrus",
masked="I [ANIMAL] the [ANIMAL]",
spans=None)
sample.tokens = ["I", "am", "the", "walrus"]
sample.tags = ["O", "O", "O", "U-ANIMAL"]
evaluation_result = evaluator.evaluate_sample(sample, prediction)
assert evaluation_result.results[("O", "O")] == 2
assert evaluation_result.results[("ANIMAL", "ANIMAL")] == 1
assert evaluation_result.results[("O", "ANIMAL")] == 1
def test_evaluator_simple():
prediction = ["O", "O", "O", "U-ANIMAL"]
model = MockTokensModel(prediction=prediction, entities_to_keep=["ANIMAL"])
evaluator = Evaluator(model=model)
sample = InputSample(full_text="I am the walrus",
masked="I am the [ANIMAL]",
spans=None)
sample.tokens = ["I", "am", "the", "walrus"]
sample.tags = ["O", "O", "O", "U-ANIMAL"]
evaluated = evaluator.evaluate_sample(sample, prediction)
final_evaluation = evaluator.calculate_score([evaluated])
assert final_evaluation.pii_precision == 1
assert final_evaluation.pii_recall == 1
def test_analyzer_simple_input():
model = PresidioAnalyzerWrapper(entities_to_keep=["PERSON"])
sample = InputSample(
full_text="My name is Mike",
masked="My name is [PERSON]",
spans=[Span("PERSON", "Mike", 10, 14)],
create_tags_from_span=True,
)
prediction = model.predict(sample)
evaluator = Evaluator(model=model)
evaluated = evaluator.evaluate_sample(sample, prediction)
metrics = evaluator.calculate_score([evaluated])
assert metrics.pii_precision == 1
assert metrics.pii_recall == 1