def test_added_pattern_recognizer_works(self): pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8) pattern_recognizer = PatternRecognizer("ROCKET", name="Rocket recognizer", patterns=[pattern]) # Make sure the analyzer doesn't get this entity recognizers_store_api_mock = RecognizerStoreApiMock() analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=MockNlpEngine()) text = "rocket is my favorite transportation" entities = ["CREDIT_CARD", "ROCKET"] results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 0 # Add a new recognizer for the word "rocket" (case insensitive) recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) # Check that the entity is recognized: results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 1 assert_result(results[0], "ROCKET", 0, 7, 0.8)
def test_simple(): dic = { "text": "John Smith drivers license is AC432223", "language": "en", "score_threshold": 0.7, "entities": ["CRYPTO", "NRP", "DATE_TIME", "LOCATION", "PERSON"], } analyzer = AnalyzerEngine() analyzer.analyze(**dic)
def test_removed_pattern_recognizer_doesnt_work(unit_test_guid): pattern = Pattern("spaceship pattern", r"\W*(spaceship)\W*", 0.8) pattern_recognizer = PatternRecognizer("SPACESHIP", name="Spaceship recognizer", patterns=[pattern]) # Make sure the analyzer doesn't get this entity recognizers_store_api_mock = RecognizerStoreApiMock() analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=NlpEngineMock(), ) text = "spaceship is my favorite transportation" entities = ["CREDIT_CARD", "SPACESHIP"] results = analyze_engine.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language="en", all_fields=False, ) assert len(results) == 0 # Add a new recognizer for the word "rocket" (case insensitive) recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) # Check that the entity is recognized: results = analyze_engine.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language="en", all_fields=False, ) assert len(results) == 1 assert_result(results[0], "SPACESHIP", 0, 10, 0.8) # Remove recognizer recognizers_store_api_mock.remove_recognizer("Spaceship recognizer") # Test again to see we didn't get any results results = analyze_engine.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language="en", all_fields=False, ) assert len(results) == 0
def test_given_text_with_pii_using_package_then_analyze_and_anonymize_complete_successfully(): text_to_test = "John Smith drivers license is AC432223" expected_response = [RecognizerResult("PERSON", 0, 10, 0.85), RecognizerResult("US_DRIVER_LICENSE", 30, 38, 0.6499999999999999) ] # Create configuration containing engine name and models configuration = { "nlp_engine_name": "spacy", "models": [{"lang_code": "en", "model_name": "en_core_web_sm"}], } # Create NLP engine based on configuration provider = NlpEngineProvider(nlp_configuration=configuration) nlp_engine = provider.create_engine() # Pass the created NLP engine and supported_languages to the AnalyzerEngine analyzer = AnalyzerEngine( nlp_engine=nlp_engine, supported_languages=["en"] ) analyzer_results = analyzer.analyze(text_to_test, "en") for i in range(len(analyzer_results)): assert analyzer_results[i] == expected_response[i] expected_response = AnonymizerResult(text="<PERSON> drivers license is <US_DRIVER_LICENSE>") expected_response.add_item(AnonymizedEntity("replace", "US_DRIVER_LICENSE", 28, 47, "<US_DRIVER_LICENSE>")) expected_response.add_item(AnonymizedEntity("replace", "PERSON", 0, 8, "<PERSON>")) anonymizer = AnonymizerEngine() anonymizer_results = anonymizer.anonymize(text_to_test, analyzer_results) assert anonymizer_results == expected_response
def test_when_analyze_then_apptracer_has_value(loaded_registry, unit_test_guid, nlp_engine): text = "My name is Bart Simpson, and Credit card: 4095-2609-9393-4932, my phone is 425 8829090" # noqa E501 language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER", "PERSON"] app_tracer_mock = AppTracerMock(enable_interpretability=True) analyzer_engine_with_spacy = AnalyzerEngine( loaded_registry, app_tracer=app_tracer_mock, enable_trace_pii=True, nlp_engine=nlp_engine, ) results = analyzer_engine_with_spacy.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language=language, all_fields=False, trace=True, ) assert len(results) == 3 for result in results: assert result.analysis_explanation is not None assert app_tracer_mock.get_msg_counter() == 2 assert app_tracer_mock.get_last_trace() is not None
class HansardCleaner: def initialize(self): SpacyRecognizer.ENTITIES = ["PERSON"] Replace.NEW_VALUE = 'replace_text' nlp_engine = SpacyNlpEngine() nlp_engine.nlp['en'] = spacy.load('en_core_web_lg', disable=["parser", "tagger", "lemmatizer"]) self.analyzer_engine = AnalyzerEngine(nlp_engine=nlp_engine) self.anonymizer_engine = AnonymizerEngine() def run(self, xml_filename): etree = ET.parse(xml_filename) for elem in etree.iter(): text = elem.text.strip() if text: results = self.analyzer_engine.analyze(correlation_id=0, text = text, entities=[], language='en', score_threshold=0.5) if results: elem.text = run_anonymizer(self.anonymizer_engine, text, results) new_filename = f'out-{xml_filename}' etree.write(new_filename) return new_filename
class HansardTextFormatter: def __init__(self): SpacyRecognizer.ENTITIES = ["PERSON"] Replace.NEW_VALUE = 'replace_text' nlp_engine = SpacyNlpEngine() nlp_engine.nlp['en'] = spacy.load( 'en_core_web_lg', disable=["parser", "tagger", "lemmatizer"]) self.analyzer_engine = AnalyzerEngine(nlp_engine=nlp_engine) self.anonymizer_engine = AnonymizerEngine() def run_anonymizer(self, text): results = self.analyzer_engine.analyze(text=text, entities=[], language='en', score_threshold=0.5) if results: config = { "PERSON": AnonymizerConfig("replace", {"replace_text": "[GDPRREDACT]"}) } return self.anonymizer_engine.anonymize(text, results, config) @staticmethod def clean_text(text): text = text.replace('\n', '') text = text.replace('<BR />', '\n') return text def run_formatter(self, text): anon_text = self.run_anonymizer(text) cleaned_text = self.clean_text(anon_text) return cleaned_text
def test_demo_text(self): text = "Here are a few examples sentences we currently support:\n\n" \ "Hello, my name is David Johnson and I live in Maine.\n" \ "My credit card number is 4095-2609-9393-4932 and my " \ "Crypto wallet id is 16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ.\n\n" \ "On September 18 I visited microsoft.com and sent an " \ "email to [email protected], from the IP 192.168.0.1.\n\n" \ "My passport: 991280345 and my phone number: (212) 555-1234.\n\n" \ "Please transfer using this IBAN IL150120690000003111111.\n\n" \ "Can you please check the status on bank account 954567876544 " \ "in PresidiBank?\n\n" \ "" \ "Kate's social security number is 078-05-1120. " \ "Her driver license? it is 9234567B.\n\n" \ "" \ "This project welcomes contributions and suggestions.\n" \ "Most contributions require you to agree to a " \ "Contributor License Agreement (CLA) declaring " \ "that you have the right to, and actually do, " \ "grant us the rights to use your contribution. " \ "For details, visit https://cla.microsoft.com " \ "When you submit a pull request, " \ "a CLA-bot will automatically determine whether " \ "you need to provide a CLA and decorate the PR " \ "appropriately (e.g., label, comment).\n" \ "Simply follow the instructions provided by the bot. " \ "You will only need to do this once across all repos using our CLA.\n" \ "This project has adopted the Microsoft Open Source Code of Conduct.\n" \ "For more information see the Code of Conduct FAQ or " \ "contact [email protected] with any additional questions or comments." language = "en" analyzer_engine = AnalyzerEngine(default_score_threshold=0.35, nlp_engine=loaded_spacy_nlp_engine) results = analyzer_engine.analyze(correlation_id=self.unit_test_guid, text=text, entities=None, language=language, all_fields=True) for result in results: logger.info("Entity = {}, Text = {}, Score={}, Start={}, End={}".format(result.entity_type, text[result.start:result.end], result.score, result.start, result.end)) detected_entities = [result.entity_type for result in results] assert len([entity for entity in detected_entities if entity == "CREDIT_CARD"]) == 1 assert len([entity for entity in detected_entities if entity == "CRYPTO"]) == 1 assert len([entity for entity in detected_entities if entity == "DATE_TIME"]) == 1 assert len([entity for entity in detected_entities if entity == "DOMAIN_NAME"]) == 4 assert len([entity for entity in detected_entities if entity == "EMAIL_ADDRESS"]) == 2 assert len([entity for entity in detected_entities if entity == "IBAN_CODE"]) == 1 assert len([entity for entity in detected_entities if entity == "IP_ADDRESS"]) == 1 assert len([entity for entity in detected_entities if entity == "LOCATION"]) == 1 assert len([entity for entity in detected_entities if entity == "PERSON"]) == 2 assert len([entity for entity in detected_entities if entity == "PHONE_NUMBER"]) == 1 assert len([entity for entity in detected_entities if entity == "US_BANK_NUMBER"]) == 1 assert len([entity for entity in detected_entities if entity == "US_DRIVER_LICENSE"]) == 1 assert len([entity for entity in detected_entities if entity == "US_PASSPORT"]) == 1 assert len([entity for entity in detected_entities if entity == "US_SSN"]) == 1 assert len(results) == 19
def test_when_analyze_two_entities_embedded_then_return_results(nlp_engine): analyzer = AnalyzerEngine(nlp_engine=nlp_engine) # Name with driver license in it text = "My name is John 1234567 Doe" results = analyzer.analyze(text=text, language="en", score_threshold=0) # currently we only remove duplicates when the two have the same entity type assert len(results) == 2
def test_ad_hoc_with_context_support_higher_confidence(nlp_engine, zip_code_recognizer): text = "Mr. John Smith's zip code is 10023" analyzer_engine = AnalyzerEngine(nlp_engine=nlp_engine) responses1 = analyzer_engine.analyze( text=text, language="en", ad_hoc_recognizers=[zip_code_recognizer]) zip_code_recognizer.context = ["zip", "code"] responses2 = analyzer_engine.analyze( text=text, language="en", ad_hoc_recognizers=[zip_code_recognizer]) zip_result_no_context = [ resp for resp in responses1 if resp.entity_type == "ZIP" ] zip_result_with_context = [ resp for resp in responses2 if resp.entity_type == "ZIP" ] assert zip_result_no_context[0].score < zip_result_with_context[0].score
class Presidio: def __init__(self): self.analyzer = AnalyzerEngine() self.anonymizer = AnonymizerEngine() def analyze_and_anonymize(self, text) -> str: analyzer_results = self.analyzer.analyze(text=text, language='en') operators = {"DEFAULT": OperatorConfig("redact")} anonymizer_results = self.anonymizer.anonymize( text=text, analyzer_results=analyzer_results, operators=operators) return anonymizer_results.text
def anonymize_text(text: str) -> str: analyzer = AnalyzerEngine() anonymizer = AnonymizerEngine() analyzer_results = analyzer.analyze(text=text, language="en") anonymized_results = anonymizer.anonymize( text=text, analyzer_results=analyzer_results, anonymizers_config={ "DEFAULT": AnonymizerConfig("replace", {"new_value": "<ANONYMIZED>"}) }, ) return anonymized_results
def test_when_analyze_added_pattern_recognizer_then_succeed(unit_test_guid): pattern = Pattern("rocket pattern", r"\W*(rocket)\W*", 0.8) pattern_recognizer = PatternRecognizer("ROCKET", name="Rocket recognizer", patterns=[pattern]) mock_recognizer_registry = RecognizerRegistryMock() # Make sure the analyzer doesn't get this entity analyze_engine = AnalyzerEngine( registry=mock_recognizer_registry, nlp_engine=NlpEngineMock(), ) text = "rocket is my favorite transportation" entities = ["CREDIT_CARD", "ROCKET"] results = analyze_engine.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language="en", ) assert len(results) == 0 # Add a new recognizer for the word "rocket" (case insensitive) mock_recognizer_registry.add_recognizer(pattern_recognizer) # Check that the entity is recognized: results = analyze_engine.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language="en", ) assert len(results) == 1 assert_result(results[0], "ROCKET", 0, 7, 0.8)
def test_when_entities_is_none_all_recognizers_loaded_then_return_all_fields( nlp_engine, ): analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(), nlp_engine=nlp_engine) threshold = 0 text = "My name is Sharon and I live in Seattle." "Domain: microsoft.com " response = analyze_engine.analyze(text=text, score_threshold=threshold, language="en") returned_entities = [response.entity_type for response in response] assert response is not None assert "PERSON" in returned_entities assert "LOCATION" in returned_entities assert "DOMAIN_NAME" in returned_entities
def test_when_entities_is_none_then_return_all_fields(loaded_registry): analyze_engine = AnalyzerEngine(registry=loaded_registry, nlp_engine=NlpEngineMock()) threshold = 0 text = (" Credit card: 4095-2609-9393-4932, my phone is 425 8829090 " "Domain: microsoft.com") response = analyze_engine.analyze(text=text, score_threshold=threshold, language="en") returned_entities = [response.entity_type for response in response] assert response is not None assert "CREDIT_CARD" in returned_entities assert "PHONE_NUMBER" in returned_entities assert "DOMAIN_NAME" in returned_entities
def analyze(self, image: object, **kwargs) -> List[ImageRecognizerResult]: """Analyse method to analyse the given image. :param image: PIL Image/numpy array or file path(str) to be processed :return: list of the extract entities with image bounding boxes """ ocr_result = OCR().perform_ocr(image) text = OCR().get_text_from_ocr_dict(ocr_result) analyzer = AnalyzerEngine() analyzer_result = analyzer.analyze(text=text, language="en", **kwargs) bboxes = self.map_analyzer_results_to_bounding_boxes( analyzer_result, ocr_result, text) return bboxes
def test_when_default_threshold_is_zero_all_results_pass(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can detect the phone number entity analyzer_engine = AnalyzerEngine( registry=self.loaded_registry, nlp_engine=MockNlpEngine()) results = analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 2
def anonymize_text(text: str) -> str: try: analyzer = AnalyzerEngine() anonymizer = AnonymizerEngine() analyzer_results = analyzer.analyze(text=text, language="en") anonymized_results = anonymizer.anonymize( text=text, analyzer_results=analyzer_results, operators={ "DEFAULT": AnonymizerConfig("replace", {"new_value": "<ANONYMIZED>"}) }, ) return anonymized_results.text except Exception as e: print(f"An exception occurred. {e}")
class PDM: def __init__(self, language='en'): self.analyzer = AnalyzerEngine() self.anonymizer = AnonymizerEngine() self.language = language def predict(self, text, entities_of_interest=ENTITIES_OF_INTEREST): t0 = time() analyzer_results = self.analyzer.analyze(text, entities=entities_of_interest, language=self.language) t1 = time() anonymized_results = self.anonymizer.anonymize(text=text, analyzer_results=analyzer_results) t2 = time() results = {'time_to_analyze': f'{t1-t0:.4f} seconds', 'time_to_anonymize': f'{t2-t1:.4f} seconds', 'anonymized_text': anonymized_results.text, 'detected_items': [{'start': item.start, 'end': item.end, 'entity_type': item.entity_type} for item in anonymized_results.items]} return results
def test_when_analyze_then_apptracer_has_value(self): text = "My name is Bart Simpson, and Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER", "PERSON"] analyzer_engine_with_spacy = AnalyzerEngine(self.loaded_registry, app_tracer=self.app_tracer, enable_trace_pii=True, nlp_engine=TESTS_NLP_ENGINE) results = analyzer_engine_with_spacy.analyze(correlation_id=self.unit_test_guid, text=text, entities=entities, language=language, all_fields=False, trace=True) assert len(results) == 3 for result in results: assert result.analysis_explanation is not None assert self.app_tracer.get_msg_counter() == 2 assert self.app_tracer.get_last_trace() is not None
def test_when_threshold_is_more_than_half_then_only_credit_card_passes( loaded_registry, unit_test_guid): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can detect the phone number entity analyzer_engine = AnalyzerEngine(registry=loaded_registry, nlp_engine=NlpEngineMock()) results = analyzer_engine.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language=language, score_threshold=0.51, ) assert len(results) == 1
def test_when_add_recognizer_then_also_outputs_others(nlp_engine): pattern = Pattern("rocket pattern", r"\W*(rocket)\W*", 0.8) pattern_recognizer = PatternRecognizer( "ROCKET", name="Rocket recognizer", patterns=[pattern], supported_language="en", ) registry = RecognizerRegistry() registry.add_recognizer(pattern_recognizer) registry.load_predefined_recognizers() assert len(registry.recognizers) > 1 analyzer = AnalyzerEngine(registry=registry, nlp_engine=nlp_engine) text = "Michael Jones has a rocket" results = analyzer.analyze(text=text, language="en") assert len(results) == 2
def test_analyze_with_multiple_predefined_recognizers(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can use the context words analyzer_engine_with_spacy = AnalyzerEngine( registry=self.loaded_registry, nlp_engine=loaded_spacy_nlp_engine) results = analyzer_engine_with_spacy.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 2 assert_result(results[0], "CREDIT_CARD", 14, 33, EntityRecognizer.MAX_SCORE) expected_score = UsPhoneRecognizer.MEDIUM_REGEX_SCORE + \ PatternRecognizer.CONTEXT_SIMILARITY_FACTOR # 0.5 + 0.35 = 0.85 assert_result(results[1], "PHONE_NUMBER", 48, 59, expected_score)
def test_when_analyze_with_multiple_predefined_recognizers_then_succeed( loaded_registry, unit_test_guid, nlp_engine, max_score): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] analyzer_engine_with_spacy = AnalyzerEngine(registry=loaded_registry, nlp_engine=nlp_engine) results = analyzer_engine_with_spacy.analyze( correlation_id=unit_test_guid, text=text, entities=entities, language=language, ) assert len(results) == 2 medium_regex_score = 0.5 # see UsPhoneRecognizer.PATTERNS context_similarity_factor = 0.35 # PatternRecognizer.CONTEXT_SIMILARITY_FACTOR assert_result(results[0], "CREDIT_CARD", 14, 33, max_score) expected_score = medium_regex_score + context_similarity_factor assert_result(results[1], "PHONE_NUMBER", 48, 59, expected_score)
class PresidioPIIAnalyzer(BaseAnalyzer): _analyzer: AnalyzerEngine = PrivateAttr() _anonymizer: AnonymizerEngine = PrivateAttr() TYPE: str = "PresidioPII" engine_config: Optional[PresidioEngineConfig] = None # To see list of supported entities refer https://microsoft.github.io/presidio/supported_entities/ # To add customer recognizers refer https://microsoft.github.io/presidio/analyzer/adding_recognizers/ entity_recognizers: Optional[List[EntityRecognizer]] = None # To find more details refer https://microsoft.github.io/presidio/anonymizer/ anonymizers_config: Optional[Dict[str, OperatorConfig]] = None def __init__(self, **data: Any): super().__init__(**data) if not self.engine_config: self.engine_config = PresidioEngineConfig() # If spacy engine then load Spacy models and select languages languages = [] for model_config in self.engine_config.models: languages.append(model_config.lang_code) # Check SpacyNlpEngine.engine_name if self.engine_config.nlp_engine_name == "spacy": try: spacy_model = __import__(model_config.model_name) spacy_model.load() logger.info( f"Spacy model {model_config.model_name} is already downloaded" ) except: logger.warning( f"Spacy model {model_config.model_name} is not downloaded" ) logger.warning( f"Downloading spacy model {model_config.model_name}, it might take some time" ) from spacy.cli import download download(model_config.model_name) # Create NLP engine based on configuration provider = NlpEngineProvider( nlp_configuration=self.engine_config.dict()) nlp_engine = provider.create_engine() # Pass the created NLP engine and supported_languages to the AnalyzerEngine self._analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=languages) # self._analyzer.registry.load_predefined_recognizers() if self.entity_recognizers: for entity_recognizer in self.entity_recognizers: self._analyzer.registry.add_recognizer(entity_recognizer) # Initialize the anonymizer with logger self._anonymizer = AnonymizerEngine() def analyze_input( self, source_response_list: List[AnalyzerRequest], analyzer_config: PresidioPIIAnalyzerConfig, language: Optional[str] = "en", **kwargs, ) -> List[AnalyzerResponse]: analyzer_output: List[AnalyzerResponse] = [] for source_response in source_response_list: analyzer_result = self._analyzer.analyze( text=source_response.processed_text, entities=analyzer_config.entities, return_decision_process=analyzer_config. return_decision_process, language=language, ) anonymized_result = None if not analyzer_config.analyze_only: anonymizers_config = (analyzer_config.anonymizers_config or self.anonymizers_config) if (source_response.processed_text is not None and len(source_response.processed_text) > 0): anonymized_result = self._anonymizer.anonymize( text=source_response.processed_text, operators=anonymizers_config, analyzer_results=analyzer_result, ) if analyzer_config.replace_original_text and anonymized_result is not None: text = anonymized_result.text else: text = source_response.processed_text analyzer_output.append( AnalyzerResponse( processed_text=text, meta=source_response.meta, segmented_data={ "analyzer_result": [vars(result) for result in analyzer_result], "anonymized_result": None if not anonymized_result else [vars(item) for item in anonymized_result.items], "anonymized_text": None if not anonymized_result else anonymized_result.text, }, source_name=source_response.source_name, )) return analyzer_output
class TestAnalyzerEngine(TestCase): def __init__(self, *args, **kwargs): super(TestAnalyzerEngine, self).__init__(*args, **kwargs) self.loaded_registry = MockRecognizerRegistry(RecognizerStoreApiMock()) mock_nlp_artifacts = NlpArtifacts([], [], [], [], None, "en") self.app_tracer = AppTracerMock(enable_interpretability=True) self.loaded_analyzer_engine = AnalyzerEngine( self.loaded_registry, MockNlpEngine(stopwords=[], punct_words=[], nlp_artifacts=mock_nlp_artifacts), app_tracer=self.app_tracer, enable_trace_pii=True) self.unit_test_guid = "00000000-0000-0000-0000-000000000000" def test_analyze_with_predefined_recognizers_return_results(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD"] results = self.loaded_analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 1 assert_result(results[0], "CREDIT_CARD", 14, 33, EntityRecognizer.MAX_SCORE) def test_analyze_with_multiple_predefined_recognizers(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can use the context words analyzer_engine_with_spacy = AnalyzerEngine( registry=self.loaded_registry, nlp_engine=loaded_spacy_nlp_engine) results = analyzer_engine_with_spacy.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 2 assert_result(results[0], "CREDIT_CARD", 14, 33, EntityRecognizer.MAX_SCORE) expected_score = UsPhoneRecognizer.MEDIUM_REGEX_SCORE + \ PatternRecognizer.CONTEXT_SIMILARITY_FACTOR # 0.5 + 0.35 = 0.85 assert_result(results[1], "PHONE_NUMBER", 48, 59, expected_score) def test_analyze_without_entities(self): with pytest.raises(ValueError): language = "en" text = " Credit card: 4095-2609-9393-4932, my name is John Oliver, DateTime: September 18 Domain: microsoft.com" entities = [] self.loaded_analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) def test_analyze_with_empty_text(self): language = "en" text = "" entities = ["CREDIT_CARD", "PHONE_NUMBER"] results = self.loaded_analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 0 def test_analyze_with_unsupported_language(self): with pytest.raises(ValueError): language = "de" text = "" entities = ["CREDIT_CARD", "PHONE_NUMBER"] self.loaded_analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) def test_remove_duplicates(self): # test same result with different score will return only the highest arr = [ RecognizerResult(start=0, end=5, score=0.1, entity_type="x", analysis_explanation=AnalysisExplanation( recognizer='test', original_score=0, pattern_name='test', pattern='test', validation_result=None)), RecognizerResult(start=0, end=5, score=0.5, entity_type="x", analysis_explanation=AnalysisExplanation( recognizer='test', original_score=0, pattern_name='test', pattern='test', validation_result=None)) ] results = AnalyzerEngine._AnalyzerEngine__remove_duplicates(arr) assert len(results) == 1 assert results[0].score == 0.5 # TODO: add more cases with bug: # bug# 597: Analyzer remove duplicates doesn't handle all cases of one result as a substring of the other def test_remove_duplicates_different_entity_no_removal(self): # test same result with different score will return only the highest arr = [ RecognizerResult(start=0, end=5, score=0.1, entity_type="x", analysis_explanation=AnalysisExplanation( recognizer='test', original_score=0, pattern_name='test', pattern='test', validation_result=None)), RecognizerResult(start=0, end=5, score=0.5, entity_type="y", analysis_explanation=AnalysisExplanation( recognizer='test', original_score=0, pattern_name='test', pattern='test', validation_result=None)) ] results = AnalyzerEngine._AnalyzerEngine__remove_duplicates(arr) assert len(results) == 2 def test_added_pattern_recognizer_works(self): pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8) pattern_recognizer = PatternRecognizer("ROCKET", name="Rocket recognizer", patterns=[pattern]) # Make sure the analyzer doesn't get this entity recognizers_store_api_mock = RecognizerStoreApiMock() analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=MockNlpEngine()) text = "rocket is my favorite transportation" entities = ["CREDIT_CARD", "ROCKET"] results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 0 # Add a new recognizer for the word "rocket" (case insensitive) recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) # Check that the entity is recognized: results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 1 assert_result(results[0], "ROCKET", 0, 7, 0.8) def test_removed_pattern_recognizer_doesnt_work(self): pattern = Pattern("spaceship pattern", r'\W*(spaceship)\W*', 0.8) pattern_recognizer = PatternRecognizer("SPACESHIP", name="Spaceship recognizer", patterns=[pattern]) # Make sure the analyzer doesn't get this entity recognizers_store_api_mock = RecognizerStoreApiMock() analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=MockNlpEngine()) text = "spaceship is my favorite transportation" entities = ["CREDIT_CARD", "SPACESHIP"] results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 0 # Add a new recognizer for the word "rocket" (case insensitive) recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) # Check that the entity is recognized: results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 1 assert_result(results[0], "SPACESHIP", 0, 10, 0.8) # Remove recognizer recognizers_store_api_mock.remove_recognizer("Spaceship recognizer") # Test again to see we didn't get any results results = analyze_engine.analyze(self.unit_test_guid, text=text, entities=entities, language='en', all_fields=False) assert len(results) == 0 def test_apply_with_language_returns_correct_response(self): request = AnalyzeRequest() request.analyzeTemplate.language = 'en' request.analyzeTemplate.resultsScoreThreshold = 0 new_field = request.analyzeTemplate.fields.add() new_field.name = 'CREDIT_CARD' new_field.minScore = '0.5' request.text = "My credit card number is 4916994465041084" response = self.loaded_analyzer_engine.Apply(request, None) assert response.analyzeResults is not None def test_apply_with_no_language_returns_default(self): request = AnalyzeRequest() request.analyzeTemplate.language = '' request.analyzeTemplate.resultsScoreThreshold = 0 new_field = request.analyzeTemplate.fields.add() new_field.name = 'CREDIT_CARD' new_field.minScore = '0.5' request.text = "My credit card number is 4916994465041084" response = self.loaded_analyzer_engine.Apply(request, None) assert response.analyzeResults is not None def test_when_allFields_is_true_return_all_fields(self): analyze_engine = AnalyzerEngine(registry=MockRecognizerRegistry(), nlp_engine=MockNlpEngine()) request = AnalyzeRequest() request.analyzeTemplate.allFields = True request.analyzeTemplate.resultsScoreThreshold = 0 request.text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090 " \ "Domain: microsoft.com" response = analyze_engine.Apply(request, None) returned_entities = [ field.field.name for field in response.analyzeResults ] assert response.analyzeResults is not None assert "CREDIT_CARD" in returned_entities assert "PHONE_NUMBER" in returned_entities assert "DOMAIN_NAME" in returned_entities def test_when_allFields_is_true_full_recognizers_list_return_all_fields( self): analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(), nlp_engine=loaded_spacy_nlp_engine) request = AnalyzeRequest() request.analyzeTemplate.allFields = True request.text = "My name is David and I live in Seattle." \ "Domain: microsoft.com " response = analyze_engine.Apply(request, None) returned_entities = [ field.field.name for field in response.analyzeResults ] assert response.analyzeResults is not None assert "PERSON" in returned_entities assert "LOCATION" in returned_entities assert "DOMAIN_NAME" in returned_entities def test_when_allFields_is_true_and_entities_not_empty_exception(self): analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(), nlp_engine=MockNlpEngine()) request = AnalyzeRequest() request.text = "My name is David and I live in Seattle." \ "Domain: microsoft.com " request.analyzeTemplate.allFields = True new_field = request.analyzeTemplate.fields.add() new_field.name = 'CREDIT_CARD' new_field.minScore = '0.5' with pytest.raises(ValueError): analyze_engine.Apply(request, None) def test_when_analyze_then_apptracer_has_value(self): text = "My name is Bart Simpson, and Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER", "PERSON"] analyzer_engine_with_spacy = AnalyzerEngine( self.loaded_registry, app_tracer=self.app_tracer, enable_trace_pii=True, nlp_engine=TESTS_NLP_ENGINE) results = analyzer_engine_with_spacy.analyze( correlation_id=self.unit_test_guid, text=text, entities=entities, language=language, all_fields=False, trace=True) assert len(results) == 3 for result in results: assert result.analysis_explanation is not None assert self.app_tracer.get_msg_counter() == 2 assert self.app_tracer.get_last_trace() is not None def test_when_threshold_is_zero_all_results_pass(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can detect the phone number entity analyzer_engine = AnalyzerEngine(registry=self.loaded_registry, nlp_engine=MockNlpEngine()) results = analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False, score_threshold=0) assert len(results) == 2 def test_when_threshold_is_more_than_half_only_credit_card_passes(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can detect the phone number entity analyzer_engine = AnalyzerEngine(registry=self.loaded_registry, nlp_engine=MockNlpEngine()) results = analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False, score_threshold=0.51) assert len(results) == 1 def test_when_default_threshold_is_more_than_half_only_one_passes(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can detect the phone number entity analyzer_engine = AnalyzerEngine(registry=self.loaded_registry, nlp_engine=MockNlpEngine(), default_score_threshold=0.7) results = analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 1 def test_when_default_threshold_is_zero_all_results_pass(self): text = " Credit card: 4095-2609-9393-4932, my phone is 425 8829090" language = "en" entities = ["CREDIT_CARD", "PHONE_NUMBER"] # This analyzer engine is different from the global one, as this one # also loads SpaCy so it can detect the phone number entity analyzer_engine = AnalyzerEngine(registry=self.loaded_registry, nlp_engine=MockNlpEngine()) results = analyzer_engine.analyze(self.unit_test_guid, text, entities, language, all_fields=False) assert len(results) == 2 def test_demo_text(self): text = "Here are a few examples sentences we currently support:\n\n" \ "Hello, my name is David Johnson and I live in Maine.\n" \ "My credit card number is 4095-2609-9393-4932 and my " \ "Crypto wallet id is 16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ.\n\n" \ "On September 18 I visited microsoft.com and sent an " \ "email to [email protected], from the IP 192.168.0.1.\n\n" \ "My passport: 991280345 and my phone number: (212) 555-1234.\n\n" \ "Please transfer using this IBAN IL150120690000003111111.\n\n" \ "Can you please check the status on bank account 954567876544 " \ "in PresidiBank?\n\n" \ "" \ "Kate's social security number is 078-05-1120. " \ "Her driver license? it is 9234567B.\n\n" \ "" \ "This project welcomes contributions and suggestions.\n" \ "Most contributions require you to agree to a " \ "Contributor License Agreement (CLA) declaring " \ "that you have the right to, and actually do, " \ "grant us the rights to use your contribution. " \ "For details, visit https://cla.microsoft.com " \ "When you submit a pull request, " \ "a CLA-bot will automatically determine whether " \ "you need to provide a CLA and decorate the PR " \ "appropriately (e.g., label, comment).\n" \ "Simply follow the instructions provided by the bot. " \ "You will only need to do this once across all repos using our CLA.\n" \ "This project has adopted the Microsoft Open Source Code of Conduct.\n" \ "For more information see the Code of Conduct FAQ or " \ "contact [email protected] with any additional questions or comments." language = "en" analyzer_engine = AnalyzerEngine(default_score_threshold=0.35, nlp_engine=loaded_spacy_nlp_engine) results = analyzer_engine.analyze(correlation_id=self.unit_test_guid, text=text, entities=None, language=language, all_fields=True) for result in results: logger.info( "Entity = {}, Text = {}, Score={}, Start={}, End={}".format( result.entity_type, text[result.start:result.end], result.score, result.start, result.end)) detected_entities = [result.entity_type for result in results] assert len([ entity for entity in detected_entities if entity == "CREDIT_CARD" ]) == 1 assert len([ entity for entity in detected_entities if entity == "CRYPTO" ]) == 1 assert len([ entity for entity in detected_entities if entity == "DATE_TIME" ]) == 1 assert len([ entity for entity in detected_entities if entity == "DOMAIN_NAME" ]) == 4 assert len([ entity for entity in detected_entities if entity == "EMAIL_ADDRESS" ]) == 2 assert len([ entity for entity in detected_entities if entity == "IBAN_CODE" ]) == 1 assert len([ entity for entity in detected_entities if entity == "IP_ADDRESS" ]) == 1 assert len([ entity for entity in detected_entities if entity == "LOCATION" ]) == 1 assert len([ entity for entity in detected_entities if entity == "PERSON" ]) == 2 assert len([ entity for entity in detected_entities if entity == "PHONE_NUMBER" ]) == 1 assert len([ entity for entity in detected_entities if entity == "US_BANK_NUMBER" ]) == 1 assert len([ entity for entity in detected_entities if entity == "US_DRIVER_LICENSE" ]) == 1 assert len([ entity for entity in detected_entities if entity == "US_PASSPORT" ]) == 1 assert len([ entity for entity in detected_entities if entity == "US_SSN" ]) == 1 assert len(results) == 19 def test_get_recognizers_returns_predefined(self): analyze_engine = AnalyzerEngine(registry=RecognizerRegistry(), nlp_engine=loaded_spacy_nlp_engine) request = RecognizersAllRequest(language="en") response = analyze_engine.GetAllRecognizers(request, None) # there are 15 predefined recognizers that detect the 17 entities assert len(response) == 15 def test_get_recognizers_returns_custom(self): pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8) pattern_recognizer = PatternRecognizer("ROCKET", name="Rocket recognizer", patterns=[pattern]) recognizers_store_api_mock = RecognizerStoreApiMock() recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=MockNlpEngine()) request = RecognizersAllRequest(language="en") response = analyze_engine.GetAllRecognizers(request, None) # there are 15 predefined recognizers and one custom assert len(response) == 16 rocket_recognizer = [ recognizer for recognizer in response if recognizer.name == "Rocket recognizer" and recognizer.entities == ["ROCKET"] and recognizer.language == "en" ] assert len(rocket_recognizer) == 1 def test_get_recognizers_returns_added_custom(self): pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8) pattern_recognizer = PatternRecognizer("ROCKET", name="Rocket recognizer", patterns=[pattern]) recognizers_store_api_mock = RecognizerStoreApiMock() analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=MockNlpEngine()) request = RecognizersAllRequest(language="en") response = analyze_engine.GetAllRecognizers(request, None) # there are 15 predefined recognizers assert len(response) == 15 recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) response = analyze_engine.GetAllRecognizers(request, None) # there are 15 predefined recognizers and one custom assert len(response) == 16 def test_get_recognizers_returns_supported_language(self): pattern = Pattern("rocket pattern", r'\W*(rocket)\W*', 0.8) pattern_recognizer = PatternRecognizer("ROCKET", name="Rocket recognizer RU", patterns=[pattern], supported_language="ru") recognizers_store_api_mock = RecognizerStoreApiMock() recognizers_store_api_mock.add_custom_pattern_recognizer( pattern_recognizer) analyze_engine = AnalyzerEngine( registry=MockRecognizerRegistry(recognizers_store_api_mock), nlp_engine=MockNlpEngine()) request = RecognizersAllRequest(language="ru") response = analyze_engine.GetAllRecognizers(request, None) # there is only 1 mocked russian recognizer assert len(response) == 1
def anonymize_reverse_lambda(analyzer_results, text_to_anonymize): anonymized_results = anonymizer.anonymize( text=text_to_anonymize, analyzer_results=analyzer_results, operators={"EMAIL_ADDRESS": OperatorConfig("custom", {"lambda": lambda x: x[::-1]})} ) return anonymized_results def anonymize_faker_lambda(analyzer_results, text_to_anonymize): anonymized_results = anonymizer.anonymize( text=text_to_anonymize, analyzer_results=analyzer_results, operators={"EMAIL_ADDRESS": OperatorConfig("custom", {"lambda": lambda x: fake.safe_email()})} ) return anonymized_results fake = Faker('en_US') fake.add_provider(internet) analyzer = AnalyzerEngine() anonymizer = AnonymizerEngine() text = 'The user has the following two emails: [email protected] and [email protected]' analyzer_results = analyzer.analyze(text=text, entities=["EMAIL_ADDRESS"], language='en') print("Origina Text: ", text) print("Analyzer result:", analyzer_results, '\n') print("Reverse lambda result: ",anonymize_reverse_lambda(analyzer_results, text).text, '\n') print("Faker lambda result: ",anonymize_faker_lambda(analyzer_results, text).text, '\n')
from presidio_analyzer import AnalyzerEngine, PatternRecognizer from presidio_anonymizer import AnonymizerEngine from presidio_anonymizer.entities.engine import OperatorConfig text_to_anonymize = "His name is Tom and his phone number is 212-555-5555" analyzer = AnalyzerEngine() anonymizer = AnonymizerEngine() analyzer_results = analyzer.analyze(text=text_to_anonymize, language='en') print("\nPII Detection:") print(analyzer_results) anonymized_results = anonymizer.anonymize( text=text_to_anonymize, analyzer_results=analyzer_results, operators={ "DEFAULT": OperatorConfig("replace", {"new_value": "<ANONYMIZED>"}) }) print("\nPII Anonymization:") print(anonymized_results.to_json())
print("\nDocument properties for", doc) print("Author:", document.core_properties.author) print("Last Modified By:", document.core_properties.last_modified_by) print("Date:", document.core_properties.modified) print("\n===============================================") paras = document.paragraphs doctext = "" for i in paras: doctext += i.text print("\n===============================================") print("Extracted text from", doc) print(doctext) print("\n===============================================") engine = AnalyzerEngine() response = engine.analyze(correlation_id=0, text = doctext, entities=[], language='en', all_fields=True, score_threshold=0.5) for item in response: print("Start = {}, end = {}, entity = {}, confidence = {}".format(item.start, item.end, item.entity_type, item.score))
def analyze(text): analyzer = AnalyzerEngine() analyzer_results = analyzer.analyze(text=text, language='en') return analyzer_results