def test_when_remove_duplicates_different_then_entity_not_removed():
# test same result with different score will return only the highest
arr = [
RecognizerResult(
start=0,
end=5,
score=0.1,
entity_type="x",
analysis_explanation=AnalysisExplanation(
recognizer="test",
original_score=0,
pattern_name="test",
pattern="test",
validation_result=None,
),
),
RecognizerResult(
start=0,
end=5,
score=0.5,
entity_type="y",
analysis_explanation=AnalysisExplanation(
recognizer="test",
original_score=0,
pattern_name="test",
pattern="test",
validation_result=None,
),
),
]
results = EntityRecognizer.remove_duplicates(arr)
assert len(results) == 2
def test_remove_duplicates_different_entity_no_removal(self):
# test same result with different score will return only the highest
arr = [
RecognizerResult(start=0,
end=5,
score=0.1,
entity_type="x",
analysis_explanation=AnalysisExplanation(
recognizer='test',
original_score=0,
pattern_name='test',
pattern='test',
validation_result=None)),
RecognizerResult(start=0,
end=5,
score=0.5,
entity_type="y",
analysis_explanation=AnalysisExplanation(
recognizer='test',
original_score=0,
pattern_name='test',
pattern='test',
validation_result=None))
]
results = AnalyzerEngine._AnalyzerEngine__remove_duplicates(arr)
assert len(results) == 2
def test_remove_duplicates():
# test same result with different score will return only the highest
arr = [
RecognizerResult(
start=0,
end=5,
score=0.1,
entity_type="x",
analysis_explanation=AnalysisExplanation(
recognizer="test",
original_score=0,
pattern_name="test",
pattern="test",
validation_result=None,
),
),
RecognizerResult(
start=0,
end=5,
score=0.5,
entity_type="x",
analysis_explanation=AnalysisExplanation(
recognizer="test",
original_score=0,
pattern_name="test",
pattern="test",
validation_result=None,
),
),
]
results = AnalyzerEngine._AnalyzerEngine__remove_duplicates(arr)
assert len(results) == 1
assert results[0].score == 0.5
def build_regex_explanation(
recognizer_name: str,
pattern_name: str,
pattern: str,
original_score: float,
validation_result: bool,
) -> AnalysisExplanation:
"""
Construct an explanation for why this entity was detected.
:param recognizer_name: Name of recognizer detecting the entity
:param pattern_name: Regex pattern name which detected the entity
:param pattern: Regex pattern logic
:param original_score: Score given by the recognizer
:param validation_result: Whether validation was used and its result
:return: Analysis explanation
"""
explanation = AnalysisExplanation(
recognizer=recognizer_name,
original_score=original_score,
pattern_name=pattern_name,
pattern=pattern,
validation_result=validation_result,
)
return explanation
def build_spacy_explanation(recognizer_name, original_score, explanation):
explanation = AnalysisExplanation(
recognizer=recognizer_name,
original_score=original_score,
textual_explanation=explanation,
)
return explanation
def build_regex_explanation(recognizer_name, pattern_name, pattern,
original_score, validation_result):
explanation = AnalysisExplanation(recognizer=recognizer_name,
original_score=original_score,
pattern_name=pattern_name,
pattern=pattern,
validation_result=validation_result)
return explanation
def _build_explanation(original_score: float,
entity_type: str) -> AnalysisExplanation:
explanation = AnalysisExplanation(
recognizer=TextAnalyticsRecognizer.__class__.__name__,
original_score=original_score,
textual_explanation=
f"Identified as {entity_type} by Text Analytics",
)
return explanation
def build_spacy_explanation(
self, original_score: float, explanation: str
) -> AnalysisExplanation:
"""
Create explanation for why this result was detected.
:param original_score: Score given by this recognizer
:param explanation: Explanation string
:return:
"""
explanation = AnalysisExplanation(
recognizer=self.__class__.__name__,
original_score=original_score,
textual_explanation=explanation,
)
return explanation
def build_spacy_explanation(recognizer_name, original_score, entity):
explanation = AnalysisExplanation(
recognizer=recognizer_name,
original_score=original_score,
textual_explanation=SPACY_DEFAULT_EXPLANATION.format(entity))
return explanation