def test_entity_synonyms_substitute_and_replace(): initial_text = "Looking for a chines restaurant in New York tomorrow for three people" initial_entities = [{ "entity": "type", "value": "chines", "start": 14, "end": 20 }, { "entity": "city", "value": "New York", "start": 35, "end": 43 }, { "entity": "count", "value": "three", "start": 57, "end": 62 }] example = Message(text=initial_text, data={ "entities": initial_entities, }) ent_synonyms = {"chines": "chinese", "new york": "NYC", "three": "3"} EntitySynonymBegin(synonyms=ent_synonyms).process(example) EntitySynonymEnd().process(example)
def test_entity_synonyms_substitute_two_entity(): example = Message( text="Looking for a chines restaurant in New York tomorrow", data={ "entities": [{ "entity": "type", "value": "chinese", "start": 14, "end": 20 }, { "entity": "city", "value": "New York", "start": 35, "end": 43 }] }) ent_synonyms = {"chines": "chinese", "new york": "NYC"} EntitySynonymBegin(synonyms=ent_synonyms).process(example) assert example.text == "Looking for a chinese restaurant in NYC tomorrow" e_type = list( filter(lambda e: e["entity"] == 'type', example.get("entities")))[0] e_city = list( filter(lambda e: e["entity"] == 'city', example.get("entities")))[0] assert e_type["start"] == 14 assert e_type["end"] == 21 assert e_city["start"] == 36 assert e_city["end"] == 39
def _get_example(config=None, gazette=None, primary=None): if primary is None: primary = { "entity": "type", "value": "chines", "start": 14, "end": 20, "extractor": "ner_crf", } return _process_example( Message( text="Looking for a chines restaurant in New York", data={ "entities": [ primary, { "entity": "type", "value": "restaurant", "start": 21, "end": 31, "extractor": "ner_crf", }, { "entity": "city", "value": "New York", "start": 35, "end": 43, "extractor": "ner_crf", }, ] }, ), config=config, gazette=gazette, )
def test_entity_sweeper(): entities = [{ "entity": "cuisine", "value": "chinese", "start": 0, "end": 6 }, { "entity": "time", "value": "whatever", "start": 0, "end": 6 }] sweeper = Sweeper(component_config={'entity_names': ['time']}) message = Message("xxx", {'entities': entities}) sweeper.process(message) assert len(message.get('entities')) == 1 assert message.get('entities')[0]["entity"] == "cuisine"
def test_entity_synonyms_substitute_three_entity(): example = Message( text= "Looking for a chines restaurant in New York tomorrow for three people", data={ "entities": [ { "entity": "type", "value": "chines", "start": 14, "end": 20 }, { "entity": "city", "value": "New York", "start": 35, "end": 43 }, { "entity": "count", "value": "three", "start": 57, "end": 62 }, ] }, ) ent_synonyms = {"chines": "chinese", "new york": "NYC", "three": "3"} EntitySynonymBegin(synonyms=ent_synonyms).process(example) assert (example.text == "Looking for a chinese restaurant in NYC tomorrow for 3 people") e_type = list( filter(lambda e: e["entity"] == "type", example.get("entities")))[0] e_city = list( filter(lambda e: e["entity"] == "city", example.get("entities")))[0] e_count = list( filter(lambda e: e["entity"] == "count", example.get("entities")))[0] assert e_type["start"] == 14 assert e_type["end"] == 21 assert e_city["start"] == 36 assert e_city["end"] == 39 assert e_count["start"] == 53 assert e_count["end"] == 54
def test_entity_synonyms_substitute(): example = Message(text="Looking for a chines restaurant in New York", data={ "entities": [{ "entity": "type", "value": "chinese", "start": 14, "end": 20 }, { "entity": "city", "value": "New York", "start": 35, "end": 43 }] }) ent_synonyms = {"chines": "chinese", "new york": "NYC"} EntitySynonymBegin(synonyms=ent_synonyms).process(example) assert example.text == "Looking for a chinese restaurant in NYC"
def test_entity_synonyms_substitute_one_entity(): example = Message(text="Looking for a chines restaurant", data={ "entities": [{ "entity": "type", "value": "chinese", "start": 14, "end": 20 }] }) ent_synonyms = {"chines": "chinese"} EntitySynonymBegin(synonyms=ent_synonyms).process(example) assert example.text == "Looking for a chinese restaurant" e_type = list( filter(lambda e: e["entity"] == 'type', example.get("entities")))[0] assert e_type["start"] == 14 assert e_type["end"] == 21
def _setup_example(config=None): instance = _get_instance(config=config) message = Message(text='This is a tst message') flagged_tokens = [ { "offset": 10, "token": "tst", "type": "UnknownToken", "suggestions": [ { "suggestion": "test", "score": 0.95155325585711 }, { "suggestion": "text", "score": 0.805342621979041 } ] } ] return instance, message, flagged_tokens
def test_classification(self, trained_classifier, message, intent): text = Message(message) trained_classifier.process(text) assert text.get("intent").get("name", "NOT_CLASSIFIED") == intent
def test_entity_synonyms_substitute_and_replace_w_insertions(): text_initial = "Looking for a chines restaurant in New York tomorrow for three people" initial_entities = [{ "entity": "type", "value": "chines", "start": 14, "end": 20 }, { "entity": "city", "value": "New York", "start": 35, "end": 43 }, { "entity": "count", "value": "three", "start": 57, "end": 62 }] example = Message(text=text_initial, data={ "entities": initial_entities, }) ent_synonyms = {"chines": "chinese", "new york": "NYC", "three": "3"} EntitySynonymBegin(synonyms=ent_synonyms).process(example) # import IPython # IPython.embed() example.data["entities"].extend([ { "entity": "action", "value": "Looking", "start": 0, "end": 7, }, { "entity": "place", "value": "restaurant", "start": 22, "end": 32, }, { "entity": "species", "value": "people", "start": 55, "end": 61, }, ]) EntitySynonymEnd().process(example) def has_changed(entity): return entity["value"] != example.text[entity["start"]:entity["end"]] assert example.text == text_initial changed_entities = filter(has_changed, example.data["entities"]) # Check the unchanged entities match value <-> text[start:end] assert len(list(changed_entities)) == 3 # Check the changed entities are reverted properly for initial, entity in zip(initial_entities, changed_entities): assert raises(KeyError, lambda x: print(x["literal"]), entity) assert entity["start"] == initial["start"] assert entity["end"] == initial["end"]
def test_multiple_errors(): instance = _get_instance() message = Message(text='Ths i a tst mesae') flagged_tokens = [ { "offset": 0, "token": "Ths", "type": "UnknownToken", "suggestions": [ { "suggestion": "This", "score": 0.825389307284585 } ] }, { "offset": 4, "token": "i", "type": "UnknownToken", "suggestions": [ { "suggestion": "is", "score": 0.825389307284585 } ] }, { "offset": 8, "token": "tst", "type": "UnknownToken", "suggestions": [ { "suggestion": "test", "score": 0.825389307284585 }, { "suggestion": "text", "score": 0.646529276890009 } ] }, { "offset": 12, "token": "mesae", "type": "UnknownToken", "suggestions": [ { "suggestion": "message", "score": 0.825389307284585 }, { "suggestion": "mesa", "score": 0.761621385590906 } ] } ] tokens = instance._get_replacements(flagged_tokens) assert len(tokens) == len(flagged_tokens) text = instance._replace(message.text, tokens) assert text == 'This is a test message'