def test_when_remove_duplicates_different_then_entity_not_removed():
# test same result with different score will return only the highest
arr = [
RecognizerResult(
start=0,
end=5,
score=0.1,
entity_type="x",
analysis_explanation=AnalysisExplanation(
recognizer="test",
original_score=0,
pattern_name="test",
pattern="test",
validation_result=None,
),
),
RecognizerResult(
start=0,
end=5,
score=0.5,
entity_type="y",
analysis_explanation=AnalysisExplanation(
recognizer="test",
original_score=0,
pattern_name="test",
pattern="test",
validation_result=None,
),
),
]
results = EntityRecognizer.remove_duplicates(arr)
assert len(results) == 2
def __analyze_patterns(
self, text: str, flags: int = None
) -> List[RecognizerResult]:
"""
Evaluate all patterns in the provided text.
Including words in the provided deny-list
:param text: text to analyze
:param flags: regex flags
:return: A list of RecognizerResult
"""
flags = flags if flags else re.DOTALL | re.MULTILINE
results = []
for pattern in self.patterns:
match_start_time = datetime.datetime.now()
matches = re.finditer(pattern.regex, text, flags=flags)
match_time = datetime.datetime.now() - match_start_time
logger.debug(
"--- match_time[%s]: %s.%s seconds",
pattern.name,
match_time.seconds,
match_time.microseconds,
)
for match in matches:
start, end = match.span()
current_match = text[start:end]
# Skip empty results
if current_match == "":
continue
score = pattern.score
validation_result = self.validate_result(current_match)
description = self.build_regex_explanation(
self.name, pattern.name, pattern.regex, score, validation_result
)
pattern_result = RecognizerResult(
self.supported_entities[0], start, end, score, description
)
if validation_result is not None:
if validation_result:
pattern_result.score = EntityRecognizer.MAX_SCORE
else:
pattern_result.score = EntityRecognizer.MIN_SCORE
invalidation_result = self.invalidate_result(current_match)
if invalidation_result is not None and invalidation_result:
pattern_result.score = EntityRecognizer.MIN_SCORE
if pattern_result.score > EntityRecognizer.MIN_SCORE:
results.append(pattern_result)
results = EntityRecognizer.remove_duplicates(results)
return results
def analyze(
self,
text: str,
language: str,
entities: Optional[List[str]] = None,
correlation_id: Optional[str] = None,
score_threshold: Optional[float] = None,
return_decision_process: Optional[bool] = False,
) -> List[RecognizerResult]:
"""
Find PII entities in text using different PII recognizers for a given language.
:param text: the text to analyze
:param language: the language of the text
:param entities: List of PII entities that should be looked for in the text.
If entities=None then all entities are looked for.
:param correlation_id: cross call ID for this request
:param score_threshold: A minimum value for which
to return an identified entity
:param return_decision_process: Whether the analysis decision process steps
returned in the response.
:return: an array of the found entities in the text
:example:
>>> from presidio_analyzer import AnalyzerEngine
>>> # Set up the engine, loads the NLP module (spaCy model by default)
>>> # and other PII recognizers
>>> analyzer = AnalyzerEngine()
>>> # Call analyzer to get results
>>> results = analyzer.analyze(text='My phone number is 212-555-5555', entities=['PHONE_NUMBER'], language='en') # noqa D501
>>> print(results)
[type: PHONE_NUMBER, start: 19, end: 31, score: 0.85]
"""
all_fields = not entities
recognizers = self.registry.get_recognizers(
language=language, entities=entities, all_fields=all_fields
)
if all_fields:
# Since all_fields=True, list all entities by iterating
# over all recognizers
entities = self.get_supported_entities(language=language)
# run the nlp pipeline over the given text, store the results in
# a NlpArtifacts instance
nlp_artifacts = self.nlp_engine.process_text(text, language)
if self.log_decision_process:
self.app_tracer.trace(
correlation_id, "nlp artifacts:" + nlp_artifacts.to_json()
)
results = []
for recognizer in recognizers:
# Lazy loading of the relevant recognizers
if not recognizer.is_loaded:
recognizer.load()
recognizer.is_loaded = True
# analyze using the current recognizer and append the results
current_results = recognizer.analyze(
text=text, entities=entities, nlp_artifacts=nlp_artifacts
)
if current_results:
results.extend(current_results)
if self.log_decision_process:
self.app_tracer.trace(
correlation_id,
json.dumps([str(result.to_dict()) for result in results]),
)
# Remove duplicates or low score results
results = EntityRecognizer.remove_duplicates(results)
results = self.__remove_low_scores(results, score_threshold)
if not return_decision_process:
results = self.__remove_decision_process(results)
return results