def test_filter_prediction_results1():
store = ModelStore()
default = [{
'start': 0,
'end': 2,
'entity': 'karan singla',
'tag': 'PERSON'
}, {
'start': 1,
'end': 3,
'entity': 'singla is',
'tag': 'MISC'
}, {
'start': 2,
'end': 6,
'entity': 'is famous in Delhi',
'tag': 'LOCATION'
}]
resp = store.filter_prediction_results(default, ['PERSON', 'LOCATION'])
expected = [{
'start': 0,
'end': 2,
'entity': 'karan singla',
'tag': 'PERSON'
}, {
'start': 2,
'end': 6,
'entity': 'is famous in Delhi',
'tag': 'LOCATION'
}]
assert resp == expected
def test_tag_with_serviceid(mocker):
model = mocker.patch.object(ModelStore,
'get_model',
return_value=SpacyDefaultNER())
store = ModelStore()
mocker.patch('ice_commons.store.models.get_entities_for_default_model',
return_value=[])
mocker.patch.object(model.return_value, 'predict', return_value=([], []))
mocker.patch.object(store, 'filter_prediction_results', return_value=[])
mocker.patch.object(IceEntitiesModelManager,
'get_ice_entities',
return_value=None)
mocker.patch.object(store, 'get_resolved_mappings', return_value=[])
mocker.patch.object(store, 'handle_resolution_overlap', return_value=[])
mocker.patch.object(store, 'get_entities_for_ds', return_value=[])
mocker.patch.object(store, 'filter_tag_results', return_value=([], []))
resp = store.tag('abc', 'Hello How are you ?', 'hello how are you?', None,
None, None, {}, {})
expected = {
"utterance": 'hello how are you?',
"case_converted_utterance": 'Hello How are you ?',
"tokens": ['Hello', 'How', 'are', 'you', '?'],
"custom_tags": [],
"default_tags": []
}
assert resp == expected
def test_handle_resolution_overlap_with_overlap():
store = ModelStore()
resp = store.handle_resolution_overlap([{
'start': 0,
'end': 2,
'entity': 'karan singla'
}, {
'start': 1,
'end': 3,
'entity': 'singla is'
}, {
'start': 2,
'end': 6,
'entity': 'is famous in Delhi'
}])
expected = [{
'start': 0,
'end': 2,
'entity': 'karan singla'
}, {
'start': 2,
'end': 6,
'entity': 'is famous in Delhi'
}]
assert resp == expected
def test_filter_tag_results_null(mocker):
store = ModelStore()
m1 = mocker.patch.object(store,
'remove_overlaps_and_duplicates',
return_value=([], []))
assert m1.return_value == ([], [])
m2 = mocker.patch.object(store,
'remove_overlaps_and_duplicates',
return_value=([], []))
assert m2.return_value == ([], [])
m3 = mocker.patch.object(store,
'remove_overlaps_and_duplicates',
return_value=([], []))
assert m3.return_value == ([], [])
mocker.patch.object(store, 'filter_model_tags', return_value=([], []))
m4 = mocker.patch.object(store,
'remove_overlaps_and_duplicates',
return_value=([], []))
assert m4.return_value == ([], [])
m5 = mocker.patch.object(store,
'remove_overlaps_and_duplicates',
return_value=([], []))
assert m5.return_value == ([], [])
resp1, resp2 = store.filter_tag_results([], [], [], [], [], [], [])
assert resp1 == []
assert resp2 == []
def test_filter_tag_results_data1(mocker):
store = ModelStore()
m = mocker.patch.object(store, 'remove_overlaps_and_duplicates')
m.side_effect = [([], []), ([], []), ([], []), ([], []), ([], [])]
resp1, resp2 = store.filter_tag_results([], [], [], [], [], [], [])
assert resp1 == concat([], [])
assert resp2 == concat([], [], [])
def test_get_entities_for_ds_with_data2(mocker):
store = ModelStore()
mocker.patch('ice_commons.store.models.regex_checker',
return_value=[{
"tag": 'email',
"start": 0,
"entity": '*****@*****.**',
"end": 1
}])
mocker.patch('ice_commons.store.models.phrase_checker', return_value=[])
pattern_response, phrase_response, predefined_tags = store.get_entities_for_ds(
None, 'Karan singla', {
'patterns':
[{
"entity": "email",
"pattern": "[a-z0-9A-Z_.-]+@[da-zA-Z.-]+.[a-z.A-Z]{2,6}"
}],
'predefined_entities': ['a', 'b']
})
assert pattern_response == [{
"tag": 'email',
"start": 0,
"entity": '*****@*****.**',
"end": 1
}]
assert phrase_response == []
assert predefined_tags == ['a', 'b']
def test_get_entities_data(mocker):
store = ModelStore()
serviceid = "MedicalAssistant-test"
model_type = "ner"
datasources_map = dict(
predefined_entities=[
"PERSON", "NORP", "FACILITY", "ORG", "GPE", "LOC", "PRODUCT",
"EVENT", "WORK_OF_ART", "LANGUAGE", "ORDINAL"
],
entities=["CATEGORY", "DISEASE", "DOCTOR", "PATIENT"],
patterns=[
dict(
entity="custom_date",
pattern=
"\\d{4}\\s/\\s\\d{2}\\s/\\s\\d{2}|\\d{2}\\s/\\s\\d{2}\\s/\\s\\d{4}"
),
dict(entity="custom_email",
pattern="[a-z0-9A-Z_.-]+@[da-zA-Z.-]+.[a-zA-Z.]{2,6}")
],
distinct_token_list=[],
intents=[],
synonymMappings=[])
default_engine = "ice_commons.er.engines.spacy_ner.SpacyDefaultNER"
custom_engine = "ice_commons.er.engines.mitie_ner.MitieCustomNER"
pos = True
projects_map = dict(
custom_entity_model="ice_commons.er.engines.mitie_ner.MitieCustomNER",
ner_status="trained",
language="EN",
predefined_entity_model=
"ice_commons.er.engines.spacy_ner.SpacyDefaultNER")
original_text = "Is Dr . Manu Nair , Cardiologist available for a consultation on 21 / 10 / 2018 at 2 PM "
text = "Is Dr . Manu Nair , cardiologist available for a consultation on 21 / 10 / 2018 at 2 PM"
model = mocker.patch('ice_commons.store.models.ModelStore.get_model')
model.return_value.predict.return_value = ([], [])
mocker.patch(
'ice_commons.store.models.ModelStore.filter_prediction_results',
return_value=[
dict(start=12, tag='custom_date', end=17, entity='21 / 10 / 2018')
])
ice_ent = mocker.patch('ice_commons.store.models.IceEntitiesModelManager')
ice_ent.return_value.get_ice_entities.return_value = []
mocker.patch('ice_commons.store.models.ModelStore.get_resolved_mappings',
return_value=[])
mocker.patch('ice_commons.store.models.ModelStore.get_entities_for_ds',
return_value=[[], [], []])
resp = (store.get_entities(serviceid, model_type, custom_engine, text,
original_text, pos, default_engine,
datasources_map, projects_map))
assert resp == ([{
'start': 12,
'tag': 'custom_date',
'end': 17,
'entity': '21 / 10 / 2018'
}], [])
def test_get_intent_data(mocker):
store = ModelStore()
serviceid = "MedicalAssistant-test"
model_type = "ir"
engine = "ice_commons.er.engines.mitie_ner.MitieCustomNER"
text = "I would like to have an appointment of Dr Merin on 15 / 10 / 2018 for Eliza"
mocker.patch.object(store, 'get_model', None)
assert store.get_intent(serviceid, model_type, engine, text) is None
def test_remove_overlaps_and_duplicates1():
store = ModelStore()
predictions, tagged_indexes = store.remove_overlaps_and_duplicates(
[{
'start': 8,
'end': 10
}], [9])
assert predictions == []
assert tagged_indexes == [9]
def test_get_entities_for_ds_with_null(mocker):
store = ModelStore()
mocker.patch('ice_commons.store.models.regex_checker', return_value=[])
mocker.patch('ice_commons.store.models.phrase_checker', return_value=[])
pattern_response, phrase_response, predefined_tags = store.get_entities_for_ds(
None, None, {})
assert pattern_response == []
assert phrase_response == []
assert predefined_tags == []
def test_get_resolved_mappings_emptylist(mocker):
store = ModelStore()
obj = mocker.patch('ice_commons.store.models.create_instance',
return_value=MeasurementsResolutionNER())
expected = mocker.patch.object(obj.return_value,
'resolve',
return_value=[])
resp = store.get_resolved_mappings([{
'resolutionClass': 'i',
'name': 'Weight'
}], 'karan weighs around 68 kgs')
assert resp == expected.return_value
def test_change_case_with_null(mocker):
tokens = mocker.patch('ice_commons.store.models.tokenize_utterance',
return_value=[])
store = ModelStore()
mocker.patch.object(store, 'get_model', None)
case_conve = mocker.patch('ice_commons.store.models.convert_case',
return_value=[])
store = ModelStore()
text = ''
text, original_text = store.change_case(text)
assert text == " ".join(case_conve.return_value)
assert original_text == " ".join(tokens.return_value)
def test_change_case_with_text(mocker):
tokens = mocker.patch('ice_commons.store.models.tokenize_utterance',
return_value=['hello', 'how', 'are', 'you', '?'])
store = ModelStore()
mocker.patch.object(store, 'get_model', None)
case_conve = mocker.patch('ice_commons.store.models.convert_case',
return_value=['Hello', 'How', 'Are', 'You', '?'])
store = ModelStore()
text = 'hello how are you?'
text, original_text = store.change_case(text)
assert text == " ".join(case_conve.return_value)
assert original_text == " ".join(tokens.return_value)
def test_tag_for_intent_null(mocker):
store = ModelStore()
service_id = text = original_text = engine = default_engine = default_model_class = ""
datasources_map = projects_map = {}
mocker.patch.object(store,
'tag',
return_value={
"custom_tags": [],
"default_tags": []
})
resp = store.tag_for_intent(service_id, text, original_text, engine,
default_engine, default_model_class,
datasources_map, projects_map)
assert resp == []
def test_tag_for_intent_data(mocker):
store = ModelStore()
service_id = "MedicalAssistant-test"
text = original_text = "Is Dr . Manu Nair , Cardiologist available for a consultation on 21 / 10 / 2018 at 2 PM"
engine = "ice_commons.er.engines.mitie_ner.MitieCustomNER"
default_engine = default_model_class = "ice_commons.er.engines.spacy_ner.SpacyDefaultNER"
datasources_map = projects_map = {}
mocker.patch.object(store,
'tag',
return_value={
"custom_tags": [],
"default_tags": []
})
resp = store.tag_for_intent(service_id, text, original_text, engine,
default_engine, default_model_class,
datasources_map, projects_map)
assert resp == []
def test_get_entities_null(mocker):
store = ModelStore()
serviceid = model_type = custom_engine = text = original_text = pos = default_engine = ""
datasources_map = projects_map = {}
model = mocker.patch('ice_commons.store.models.ModelStore.get_model')
model.return_value.predict.return_value = ([], [])
mocker.patch(
'ice_commons.store.models.ModelStore.filter_prediction_results')
ice_ent = mocker.patch('ice_commons.store.models.IceEntitiesModelManager')
ice_ent.return_value.get_ice_entities.return_value = []
mocker.patch('ice_commons.store.models.ModelStore.get_resolved_mappings',
return_value=[])
mocker.patch('ice_commons.store.models.ModelStore.get_entities_for_ds',
return_value=[[], [], []])
resp = (store.get_entities(serviceid, model_type, custom_engine, text,
original_text, pos, default_engine,
datasources_map, projects_map))
assert resp == ([], None)
def test_utterance_recase_with_text(mocker):
store = mocker.patch(
'ice_rest.rest.services.parse.predefined_components.get_model_store',
return_value=ModelStore())
mock_remove_overlap_duplicate = mocker.patch.object(store, 'change_case')
mock_remove_overlap_duplicate.side_effects = ["hi", "Hi", "hello", "Hello"]
resp = utterance_recase(['Hi', "Hello", "HoW ArE YoU"])
print(resp)
ex = ['hi', 'hello', 'how are you']
assert ex == resp
def test_handle_resolution_overlap_without_overlap():
store = ModelStore()
resp = store.handle_resolution_overlap([{
'start': 4,
'end': 6
}, {
'start': 6,
'end': 7
}, {
'start': 8,
'end': 10
}])
expected = [{
'start': 4,
'end': 6
}, {
'start': 6,
'end': 7
}, {
'start': 8,
'end': 10
}]
assert resp == expected
def test_get_entities_for_ds_with_data1(mocker):
store = ModelStore()
mocker.patch('ice_commons.store.models.regex_checker', return_value=[])
mocker.patch('ice_commons.store.models.phrase_checker',
return_value=[{
"tag": 'name',
"start": 0,
"entity": 'Karan',
"end": 1
}])
pattern_response, phrase_response, predefined_tags = store.get_entities_for_ds(
None, 'Karan singla',
{'phrases': [{
"phrase": ["Karan"],
"entity": "name"
}]})
assert pattern_response == []
assert phrase_response == [{
"tag": 'name',
"start": 0,
"entity": 'Karan',
"end": 1
}]
assert predefined_tags == []
def test_tag_with_exception(mocker):
store = mocker.patch.object(ModelStore, 'get_model', ModelStore())
mocker.patch('ice_commons.store.models.get_entities_for_default_model',
return_value=None)
mocker.patch.object(SpacyDefaultNER, 'predict', return_value=(None))
mocker.patch.object(store, 'filter_prediction_results', return_value=None)
mocker.patch.object(IceEntitiesModelManager,
'get_ice_entities',
return_value=None)
mocker.patch.object(store, 'get_resolved_mappings', return_value=None)
mocker.patch.object(store, 'handle_resolution_overlap', return_value=[])
mocker.patch.object(store, 'get_entities_for_ds', return_value=None)
mocker.patch.object(store, 'filter_tag_results', return_value=([], []))
resp = store.tag(None, '', '', None, None, None, {}, {})
expected = {
"utterance": '',
"case_converted_utterance": '',
"tokens": ''.split(),
"custom_tags": [],
"default_tags": []
}
assert resp == expected
def test_tag_with_null(mocker):
store = mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_model_store', return_value=ModelStore())
mocker.patch.object(store.return_value , 'change_case', return_value= (None, None))
mocker.patch.object(ProjectManager, 'find_model', return_value= None)
mocker.patch.object(DatasourceManager, 'find_datasource_by_service_id', return_value= None)
mocker.patch.object(store.return_value,'tag', return_value=None)
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_engine', return_value=None)
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_model_name', return_value=None)
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_all_corenlp_engines', return_value=[])
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.remove_resolved_to', return_value=[])
doc = {'serviceid':'abc', 'text': ''}
resp = tag(doc)
assert resp == []
from ice_rest.rest.appengine import AppEngine from ice_commons.store.models import ModelStore model_store = ModelStore() model_store.load_default_models() model_store.store.load_ngrams() app = AppEngine().get_app()
def test_handle_resolution_overlap_with_empty_list():
store = ModelStore()
resp = store.handle_resolution_overlap([])
expected = []
assert resp == expected
import re
import logging
from ice_commons.store.models import ModelStore
model_store = ModelStore()
logger = logging.getLogger(__name__)
def pos_tags_predict(text, original_text, language):
tokens = model_store.get_pos_tags(text, original_text, language)
text_tokens = text.split()
logger.info("pos_tags_predict %s" % text)
logger.info("pos_tags_predict %s" % original_text)
tags = []
for ind, tok in enumerate(tokens):
tags.append((text_tokens[ind], tok.get("pos"), tok.get("lemma")))
return tags
def pos_tags_train(text, original_text, language):
tokens = model_store.get_pos_tags(text, original_text, language)
tags = []
for tok in tokens:
tags.append((tok.get("text"), tok.get("pos"), tok.get("lemma")))
return tags
def get_type(token):
T = (
'AllUpper',
def test_get_model_data():
store = ModelStore()
serviceid = "MedicalAssistant-test"
model_type = "ner"
engine = "SPACY"
assert store.get_model(serviceid, model_type, engine) is None
def test_get_intent_null():
store = ModelStore()
serviceid = model_type = engine = text = ""
assert store.get_intent(serviceid, model_type, engine, text) == ([], [])
def test_tag_with_data2(mocker):
store = mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_model_store', return_value=ModelStore())
mocker.patch.object(store.return_value , 'change_case', return_value= ('Hello How are you ?', 'hello how are you?'))
mocker.patch.object(ProjectManager, 'find_model', return_value= None)
mocker.patch.object(DatasourceManager, 'find_datasource_by_service_id', return_value= None)
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_engine', return_value='CoreNLP')
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_model_name', return_value=None)
mocker.patch('ice_rest.rest.services.parse.impl.tag_impl.get_all_corenlp_engines', return_value=['CoreNLP','CoreNLP-es'])
expected = mocker.patch.object(store.return_value, 'tag', return_value= None)
expected.return_value = {
"utterance": 'hello how are you?',
"case_converted_utterance": 'Hello How are you ?',
"tokens": ['hello', 'how', 'are', 'you', '?'],
"custom_tags": [],
"default_tags": []
}
doc = {'serviceid': 'abc', 'text': 'hello how are you?'}
resp = tag(doc)
assert resp == expected.return_value
def test_get_pos_tags_data(mocker):
store = ModelStore()
text = original_text = "Is Dr . Manu Nair , Cardiologist available for a consultation on 21 / 10 / 2018 at 2 PM"
mocker.patch.object(store, 'get_model', return_value=None)
assert store.get_pos_tags(text, original_text) == []
from ice_commons.store.models import ModelStore
from ice_commons.celery_jobs.config import get_app
model_store = ModelStore()
model_store.load_default_models_celery()
app = get_app()
if __name__ == '__main__':
app.start()
def test_get_model_null():
store = ModelStore()
serviceid = model_type = engine = ""
assert store.get_model(serviceid, model_type, engine) is None