def test_when_analyze_then_apptracer_has_value(loaded_registry, unit_test_guid,
nlp_engine):
text = "My name is Bart Simpson, and Credit card: 4095-2609-9393-4932, my phone is 425 8829090" # noqa E501
language = "en"
entities = ["CREDIT_CARD", "PHONE_NUMBER", "PERSON"]
app_tracer_mock = AppTracerMock(enable_interpretability=True)
analyzer_engine_with_spacy = AnalyzerEngine(
loaded_registry,
app_tracer=app_tracer_mock,
enable_trace_pii=True,
nlp_engine=nlp_engine,
)
results = analyzer_engine_with_spacy.analyze(
correlation_id=unit_test_guid,
text=text,
entities=entities,
language=language,
all_fields=False,
trace=True,
)
assert len(results) == 3
for result in results:
assert result.analysis_explanation is not None
assert app_tracer_mock.get_msg_counter() == 2
assert app_tracer_mock.get_last_trace() is not None
def test_given_text_with_pii_using_package_then_analyze_and_anonymize_complete_successfully():
text_to_test = "John Smith drivers license is AC432223"
expected_response = [RecognizerResult("PERSON", 0, 10, 0.85),
RecognizerResult("US_DRIVER_LICENSE", 30, 38, 0.6499999999999999)
]
# Create configuration containing engine name and models
configuration = {
"nlp_engine_name": "spacy",
"models": [{"lang_code": "en", "model_name": "en_core_web_sm"}],
}
# Create NLP engine based on configuration
provider = NlpEngineProvider(nlp_configuration=configuration)
nlp_engine = provider.create_engine()
# Pass the created NLP engine and supported_languages to the AnalyzerEngine
analyzer = AnalyzerEngine(
nlp_engine=nlp_engine,
supported_languages=["en"]
)
analyzer_results = analyzer.analyze(text_to_test, "en")
for i in range(len(analyzer_results)):
assert analyzer_results[i] == expected_response[i]
expected_response = AnonymizerResult(text="<PERSON> drivers license is <US_DRIVER_LICENSE>")
expected_response.add_item(AnonymizedEntity("replace", "US_DRIVER_LICENSE", 28, 47, "<US_DRIVER_LICENSE>"))
expected_response.add_item(AnonymizedEntity("replace", "PERSON", 0, 8, "<PERSON>"))
anonymizer = AnonymizerEngine()
anonymizer_results = anonymizer.anonymize(text_to_test, analyzer_results)
assert anonymizer_results == expected_response
def test_get_recognizers_returns_predefined(self):
analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(),
nlp_engine=loaded_spacy_nlp_engine)
request = RecognizersAllRequest(language="en")
response = analyze_engine.GetAllRecognizers(request, None)
# there are 15 predefined recognizers that detect the 17 entities
assert len(response) == 15
def __init__(
self,
analyzer_engine: Optional[AnalyzerEngine] = None,
entities_to_keep: List[str] = None,
verbose: bool = False,
labeling_scheme: str = "BIO",
score_threshold: float = 0.4,
language: str = "en",
entity_mapping: Optional[Dict[str, str]] = None,
):
"""
Evaluation wrapper for the Presidio Analyzer
:param analyzer_engine: object of type AnalyzerEngine (from presidio-analyzer)
"""
super().__init__(
entities_to_keep=entities_to_keep,
verbose=verbose,
labeling_scheme=labeling_scheme,
entity_mapping=entity_mapping,
)
self.score_threshold = score_threshold
self.language = language
if not analyzer_engine:
analyzer_engine = AnalyzerEngine()
self._update_recognizers_based_on_entities_to_keep(analyzer_engine)
self.analyzer_engine = analyzer_engine
def test_when_read_test_spacy_nlp_conf_file_then_returns_spacy_nlp_engine(
mock_registry,
):
engine = AnalyzerEngine(registry=mock_registry)
assert isinstance(engine.nlp_engine, SpacyNlpEngine)
assert engine.nlp_engine.nlp is not None
def test_added_pattern_recognizer_works(self):
pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8)
pattern_recognizer = PatternRecognizer("ROCKET",
name="Rocket recognizer",
patterns=[pattern])
# Make sure the analyzer doesn't get this entity
recognizers_store_api_mock = RecognizerStoreApiMock()
analyze_engine = AnalyzerEngine(
registry=MockRecognizerRegistry(recognizers_store_api_mock),
nlp_engine=MockNlpEngine())
text = "rocket is my favorite transportation"
entities = ["CREDIT_CARD", "ROCKET"]
results = analyze_engine.analyze(self.unit_test_guid,
text=text,
entities=entities,
language='en',
all_fields=False)
assert len(results) == 0
# Add a new recognizer for the word "rocket" (case insensitive)
recognizers_store_api_mock.add_custom_pattern_recognizer(
pattern_recognizer)
# Check that the entity is recognized:
results = analyze_engine.analyze(self.unit_test_guid,
text=text,
entities=entities,
language='en',
all_fields=False)
assert len(results) == 1
assert_result(results[0], "ROCKET", 0, 7, 0.8)
def test_demo_text(self):
text = "Here are a few examples sentences we currently support:\n\n" \
"Hello, my name is David Johnson and I live in Maine.\n" \
"My credit card number is 4095-2609-9393-4932 and my " \
"Crypto wallet id is 16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ.\n\n" \
"On September 18 I visited microsoft.com and sent an " \
"email to [email protected], from the IP 192.168.0.1.\n\n" \
"My passport: 991280345 and my phone number: (212) 555-1234.\n\n" \
"Please transfer using this IBAN IL150120690000003111111.\n\n" \
"Can you please check the status on bank account 954567876544 " \
"in PresidiBank?\n\n" \
"" \
"Kate's social security number is 078-05-1120. " \
"Her driver license? it is 9234567B.\n\n" \
"" \
"This project welcomes contributions and suggestions.\n" \
"Most contributions require you to agree to a " \
"Contributor License Agreement (CLA) declaring " \
"that you have the right to, and actually do, " \
"grant us the rights to use your contribution. " \
"For details, visit https://cla.microsoft.com " \
"When you submit a pull request, " \
"a CLA-bot will automatically determine whether " \
"you need to provide a CLA and decorate the PR " \
"appropriately (e.g., label, comment).\n" \
"Simply follow the instructions provided by the bot. " \
"You will only need to do this once across all repos using our CLA.\n" \
"This project has adopted the Microsoft Open Source Code of Conduct.\n" \
"For more information see the Code of Conduct FAQ or " \
"contact [email protected] with any additional questions or comments."
language = "en"
analyzer_engine = AnalyzerEngine(default_score_threshold=0.35, nlp_engine=loaded_spacy_nlp_engine)
results = analyzer_engine.analyze(correlation_id=self.unit_test_guid, text=text, entities=None,
language=language, all_fields=True)
for result in results:
logger.info("Entity = {}, Text = {}, Score={}, Start={}, End={}".format(result.entity_type,
text[result.start:result.end],
result.score,
result.start, result.end))
detected_entities = [result.entity_type for result in results]
assert len([entity for entity in detected_entities if entity == "CREDIT_CARD"]) == 1
assert len([entity for entity in detected_entities if entity == "CRYPTO"]) == 1
assert len([entity for entity in detected_entities if entity == "DATE_TIME"]) == 1
assert len([entity for entity in detected_entities if entity == "DOMAIN_NAME"]) == 4
assert len([entity for entity in detected_entities if entity == "EMAIL_ADDRESS"]) == 2
assert len([entity for entity in detected_entities if entity == "IBAN_CODE"]) == 1
assert len([entity for entity in detected_entities if entity == "IP_ADDRESS"]) == 1
assert len([entity for entity in detected_entities if entity == "LOCATION"]) == 1
assert len([entity for entity in detected_entities if entity == "PERSON"]) == 2
assert len([entity for entity in detected_entities if entity == "PHONE_NUMBER"]) == 1
assert len([entity for entity in detected_entities if entity == "US_BANK_NUMBER"]) == 1
assert len([entity for entity in detected_entities if entity == "US_DRIVER_LICENSE"]) == 1
assert len([entity for entity in detected_entities if entity == "US_PASSPORT"]) == 1
assert len([entity for entity in detected_entities if entity == "US_SSN"]) == 1
assert len(results) == 19
def initialize(self):
SpacyRecognizer.ENTITIES = ["PERSON"]
Replace.NEW_VALUE = 'replace_text'
nlp_engine = SpacyNlpEngine()
nlp_engine.nlp['en'] = spacy.load('en_core_web_lg', disable=["parser", "tagger", "lemmatizer"])
self.analyzer_engine = AnalyzerEngine(nlp_engine=nlp_engine)
self.anonymizer_engine = AnonymizerEngine()
def __init__(self, analyzer_engine: AnalyzerEngine = None, ocr: OCR = None):
if not analyzer_engine:
analyzer_engine = AnalyzerEngine()
self.analyzer_engine = analyzer_engine
if not ocr:
ocr = TesseractOCR()
self.ocr = ocr
def test_when_analyze_two_entities_embedded_then_return_results(nlp_engine):
analyzer = AnalyzerEngine(nlp_engine=nlp_engine)
# Name with driver license in it
text = "My name is John 1234567 Doe"
results = analyzer.analyze(text=text, language="en", score_threshold=0)
# currently we only remove duplicates when the two have the same entity type
assert len(results) == 2
def loaded_analyzer_engine(loaded_registry, app_tracer):
mock_nlp_artifacts = NlpArtifacts([], [], [], [], None, "en")
analyzer_engine = AnalyzerEngine(
loaded_registry,
NlpEngineMock(stopwords=[], punct_words=[], nlp_artifacts=mock_nlp_artifacts),
app_tracer=app_tracer,
log_decision_process=True,
)
return analyzer_engine
def test_when_get_supported_fields_then_return_all_languages(
mock_registry, unit_test_guid, nlp_engine):
analyzer = AnalyzerEngine(registry=mock_registry, nlp_engine=nlp_engine)
entities = analyzer.get_supported_entities()
assert len(entities) == 3
assert "CREDIT_CARD" in entities
assert "DOMAIN_NAME" in entities
assert "PHONE_NUMBER" in entities
def test_simple():
dic = {
"text": "John Smith drivers license is AC432223",
"language": "en",
"score_threshold": 0.7,
"entities": ["CRYPTO", "NRP", "DATE_TIME", "LOCATION", "PERSON"],
}
analyzer = AnalyzerEngine()
analyzer.analyze(**dic)
def test_when_allFields_is_true_full_recognizers_list_return_all_fields(
nlp_engine):
analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(),
nlp_engine=nlp_engine)
request = AnalyzeRequest()
request.analyzeTemplate.allFields = True
request.text = "My name is David and I live in Seattle." "Domain: microsoft.com "
response = analyze_engine.Apply(request, None)
returned_entities = [field.field.name for field in response.analyzeResults]
assert response.analyzeResults is not None
assert "DOMAIN_NAME" in returned_entities
def test_when_allFields_is_true_and_entities_not_empty_exception():
analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(),
nlp_engine=NlpEngineMock())
request = AnalyzeRequest()
request.text = "My name is David and I live in Seattle." "Domain: microsoft.com "
request.analyzeTemplate.allFields = True
new_field = request.analyzeTemplate.fields.add()
new_field.name = "CREDIT_CARD"
new_field.minScore = "0.5"
with pytest.raises(ValueError):
analyze_engine.Apply(request, None)
def __init__(self, *args, **kwargs):
super(TestAnalyzerEngine, self).__init__(*args, **kwargs)
self.loaded_registry = MockRecognizerRegistry(RecognizerStoreApiMock())
mock_nlp_artifacts = NlpArtifacts([], [], [], [], None, "en")
self.app_tracer = AppTracerMock(enable_interpretability=True)
self.loaded_analyzer_engine = AnalyzerEngine(self.loaded_registry,
MockNlpEngine(stopwords=[],
punct_words=[],
nlp_artifacts=mock_nlp_artifacts),
app_tracer=self.app_tracer,
enable_trace_pii=True)
self.unit_test_guid = "00000000-0000-0000-0000-000000000000"
def anonymize_text(text: str) -> str:
analyzer = AnalyzerEngine()
anonymizer = AnonymizerEngine()
analyzer_results = analyzer.analyze(text=text, language="en")
anonymized_results = anonymizer.anonymize(
text=text,
analyzer_results=analyzer_results,
anonymizers_config={
"DEFAULT": AnonymizerConfig("replace",
{"new_value": "<ANONYMIZED>"})
},
)
return anonymized_results
def __init__(self, **data: Any):
super().__init__(**data)
if not self.engine_config:
self.engine_config = PresidioEngineConfig()
if not self.engine_config.models or len(
self.engine_config.models) == 0:
self.engine_config.models = [PresidioModelConfig()]
# If spacy engine then load Spacy models and select languages
languages = []
for model_config in self.engine_config.models:
languages.append(model_config.lang_code)
# Check SpacyNlpEngine.engine_name
if (self.engine_config.nlp_engine_name == "spacy"
and model_config.model_name is not None):
try:
spacy_model = __import__(model_config.model_name)
spacy_model.load()
logger.info(
f"Spacy model {model_config.model_name} is already downloaded"
)
except:
logger.warning(
f"Spacy model {model_config.model_name} is not downloaded"
)
logger.warning(
f"Downloading spacy model {model_config.model_name}, it might take some time"
)
from spacy.cli import download
download(model_config.model_name)
# Create NLP engine based on configuration
provider = NlpEngineProvider(
nlp_configuration=self.engine_config.dict())
nlp_engine = provider.create_engine()
# Pass the created NLP engine and supported_languages to the AnalyzerEngine
self._analyzer = AnalyzerEngine(nlp_engine=nlp_engine,
supported_languages=languages)
# self._analyzer.registry.load_predefined_recognizers()
if self.entity_recognizers:
for entity_recognizer in self.entity_recognizers:
self._analyzer.registry.add_recognizer(entity_recognizer)
# Initialize the anonymizer with logger
self._anonymizer = AnonymizerEngine()
def test_removed_pattern_recognizer_doesnt_work(unit_test_guid):
pattern = Pattern("spaceship pattern", r"\W*(spaceship)\W*", 0.8)
pattern_recognizer = PatternRecognizer("SPACESHIP",
name="Spaceship recognizer",
patterns=[pattern])
# Make sure the analyzer doesn't get this entity
recognizers_store_api_mock = RecognizerStoreApiMock()
analyze_engine = AnalyzerEngine(
registry=MockRecognizerRegistry(recognizers_store_api_mock),
nlp_engine=NlpEngineMock(),
)
text = "spaceship is my favorite transportation"
entities = ["CREDIT_CARD", "SPACESHIP"]
results = analyze_engine.analyze(
correlation_id=unit_test_guid,
text=text,
entities=entities,
language="en",
all_fields=False,
)
assert len(results) == 0
# Add a new recognizer for the word "rocket" (case insensitive)
recognizers_store_api_mock.add_custom_pattern_recognizer(
pattern_recognizer)
# Check that the entity is recognized:
results = analyze_engine.analyze(
correlation_id=unit_test_guid,
text=text,
entities=entities,
language="en",
all_fields=False,
)
assert len(results) == 1
assert_result(results[0], "SPACESHIP", 0, 10, 0.8)
# Remove recognizer
recognizers_store_api_mock.remove_recognizer("Spaceship recognizer")
# Test again to see we didn't get any results
results = analyze_engine.analyze(
correlation_id=unit_test_guid,
text=text,
entities=entities,
language="en",
all_fields=False,
)
assert len(results) == 0
def analyze(self, image: object, **kwargs) -> List[ImageRecognizerResult]:
"""Analyse method to analyse the given image.
:param image: PIL Image/numpy array or file path(str) to be processed
:return: list of the extract entities with image bounding boxes
"""
ocr_result = OCR().perform_ocr(image)
text = OCR().get_text_from_ocr_dict(ocr_result)
analyzer = AnalyzerEngine()
analyzer_result = analyzer.analyze(text=text, language="en", **kwargs)
bboxes = self.map_analyzer_results_to_bounding_boxes(
analyzer_result, ocr_result, text)
return bboxes
def test_when_default_threshold_is_zero_all_results_pass(self):
text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090"
language = "en"
entities = ["CREDIT_CARD", "PHONE_NUMBER"]
# This analyzer engine is different from the global one, as this one
# also loads SpaCy so it can detect the phone number entity
analyzer_engine = AnalyzerEngine(
registry=self.loaded_registry, nlp_engine=MockNlpEngine())
results = analyzer_engine.analyze(self.unit_test_guid, text,
entities, language,
all_fields=False)
assert len(results) == 2
def test_when_entities_is_none_then_return_all_fields(loaded_registry):
analyze_engine = AnalyzerEngine(registry=loaded_registry,
nlp_engine=NlpEngineMock())
threshold = 0
text = (" Credit card: 4095-2609-9393-4932, my phone is 425 8829090 "
"Domain: microsoft.com")
response = analyze_engine.analyze(text=text,
score_threshold=threshold,
language="en")
returned_entities = [response.entity_type for response in response]
assert response is not None
assert "CREDIT_CARD" in returned_entities
assert "PHONE_NUMBER" in returned_entities
assert "DOMAIN_NAME" in returned_entities
def test_when_entities_is_none_all_recognizers_loaded_then_return_all_fields(
nlp_engine, ):
analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(),
nlp_engine=nlp_engine)
threshold = 0
text = "My name is Sharon and I live in Seattle." "Domain: microsoft.com "
response = analyze_engine.analyze(text=text,
score_threshold=threshold,
language="en")
returned_entities = [response.entity_type for response in response]
assert response is not None
assert "PERSON" in returned_entities
assert "LOCATION" in returned_entities
assert "DOMAIN_NAME" in returned_entities
def test_when_allFields_is_true_return_all_fields(self):
analyze_engine = AnalyzerEngine(registry=MockRecognizerRegistry(),
nlp_engine=MockNlpEngine())
request = AnalyzeRequest()
request.analyzeTemplate.allFields = True
request.analyzeTemplate.resultsScoreThreshold = 0
request.text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090 " \
"Domain: microsoft.com"
response = analyze_engine.Apply(request, None)
returned_entities = [
field.field.name for field in response.analyzeResults]
assert response.analyzeResults is not None
assert "CREDIT_CARD" in returned_entities
assert "PHONE_NUMBER" in returned_entities
assert "DOMAIN_NAME" in returned_entities
def anonymize_text(text: str) -> str:
try:
analyzer = AnalyzerEngine()
anonymizer = AnonymizerEngine()
analyzer_results = analyzer.analyze(text=text, language="en")
anonymized_results = anonymizer.anonymize(
text=text,
analyzer_results=analyzer_results,
operators={
"DEFAULT":
AnonymizerConfig("replace", {"new_value": "<ANONYMIZED>"})
},
)
return anonymized_results.text
except Exception as e:
print(f"An exception occurred. {e}")
def test_when_get_supported_fields_specific_language_then_return_single_result(
loaded_registry, unit_test_guid, nlp_engine):
pattern = Pattern("rocket pattern", r"\W*(rocket)\W*", 0.8)
pattern_recognizer = PatternRecognizer(
"ROCKET",
name="Rocket recognizer RU",
patterns=[pattern],
supported_language="ru",
)
analyzer = AnalyzerEngine(registry=loaded_registry, nlp_engine=nlp_engine)
analyzer.registry.add_recognizer(pattern_recognizer)
entities = analyzer.get_supported_entities(language="ru")
assert len(entities) == 1
assert "ROCKET" in entities
def test_get_recognizers_returns_supported_language(self):
pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8)
pattern_recognizer = PatternRecognizer("ROCKET",
name="Rocket recognizer RU",
patterns=[pattern],
supported_language="ru")
recognizers_store_api_mock = RecognizerStoreApiMock()
recognizers_store_api_mock.add_custom_pattern_recognizer(
pattern_recognizer)
analyze_engine = AnalyzerEngine(
registry=MockRecognizerRegistry(recognizers_store_api_mock),
nlp_engine=MockNlpEngine())
request = RecognizersAllRequest(language="ru")
response = analyze_engine.GetAllRecognizers(request, None)
# there is only 1 mocked russian recognizer
assert len(response) == 1
def test_when_get_recognizers_then_returns_supported_language():
pattern = Pattern("rocket pattern", r"\W*(rocket)\W*", 0.8)
pattern_recognizer = PatternRecognizer(
"ROCKET",
name="Rocket recognizer RU",
patterns=[pattern],
supported_language="ru",
)
mock_recognizer_registry = RecognizerRegistryMock()
mock_recognizer_registry.add_recognizer(pattern_recognizer)
analyze_engine = AnalyzerEngine(
registry=mock_recognizer_registry,
nlp_engine=NlpEngineMock(),
)
response = analyze_engine.get_recognizers(language="ru")
# there is only 1 mocked russian recognizer
assert len(response) == 1
def test_when_analyze_then_apptracer_has_value(self):
text = "My name is Bart Simpson, and Credit card: 4095-2609-9393-4932, my phone is 425 8829090"
language = "en"
entities = ["CREDIT_CARD", "PHONE_NUMBER", "PERSON"]
analyzer_engine_with_spacy = AnalyzerEngine(self.loaded_registry,
app_tracer=self.app_tracer,
enable_trace_pii=True,
nlp_engine=TESTS_NLP_ENGINE)
results = analyzer_engine_with_spacy.analyze(correlation_id=self.unit_test_guid,
text=text,
entities=entities,
language=language,
all_fields=False,
trace=True)
assert len(results) == 3
for result in results:
assert result.analysis_explanation is not None
assert self.app_tracer.get_msg_counter() == 2
assert self.app_tracer.get_last_trace() is not None
def test_get_recognizers_returns_added_custom(self):
pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8)
pattern_recognizer = PatternRecognizer("ROCKET",
name="Rocket recognizer",
patterns=[pattern])
recognizers_store_api_mock = RecognizerStoreApiMock()
analyze_engine = AnalyzerEngine(
registry=MockRecognizerRegistry(recognizers_store_api_mock),
nlp_engine=MockNlpEngine())
request = RecognizersAllRequest(language="en")
response = analyze_engine.GetAllRecognizers(request, None)
# there are 15 predefined recognizers
assert len(response) == 15
recognizers_store_api_mock.add_custom_pattern_recognizer(
pattern_recognizer)
response = analyze_engine.GetAllRecognizers(request, None)
# there are 15 predefined recognizers and one custom
assert len(response) == 16