def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs):
if entity_extractor_file:
self.extractor = named_entity_extractor(entity_extractor_file) # ,metadata["feature_extractor"])
with open(intent_classifier_file, 'rb') as f:
self.classifier = cloudpickle.load(f)
self.featurizer = MITIEFeaturizer(feature_extractor_file)
self.tokenizer = MITIETokenizer()
def test_mitie():
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
filename = os.environ.get('MITIE_FILE')
if (filename and os.path.isfile(filename)):
ftr = MITIEFeaturizer(os.environ.get('MITIE_FILE'))
sentence = "Hey how are you today"
vecs = ftr.create_bow_vecs([sentence])
assert np.allclose(vecs[0][:5], np.array([0., -4.4551446, 0.26073121, -1.46632245, -1.84205751]), atol=1e-5)
def test_mitie_featurizer():
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
filename = os.environ.get('MITIE_FILE')
if not filename or not os.path.isfile(filename):
filename = "data/total_word_feature_extractor.dat"
ftr = MITIEFeaturizer(filename)
sentence = "Hey how are you today"
tokens = MITIETokenizer().tokenize(sentence)
vecs = ftr.features_for_tokens(tokens)
assert np.allclose(
vecs[:5],
np.array([0., -4.4551446, 0.26073121, -1.46632245, -1.84205751]),
atol=1e-5)
def load(meta, featurizer=None):
"""
:type meta: rasa_nlu.model.Metadata
:rtype: MITIESklearnInterpreter
"""
if meta.entity_extractor_path:
extractor = named_entity_extractor(meta.entity_extractor_path)
else:
extractor = None
if featurizer is None:
featurizer = MITIEFeaturizer(meta.feature_extractor_path)
if meta.intent_classifier_path:
with open(meta.intent_classifier_path, 'rb') as f:
classifier = cloudpickle.load(f)
else:
classifier = None
if meta.entity_synonyms_path:
entity_synonyms = Interpreter.load_synonyms(
meta.entity_synonyms_path)
else:
entity_synonyms = None
return MITIESklearnInterpreter(classifier, extractor, featurizer,
entity_synonyms)
def load(meta, featurizer=None):
"""
:type meta: rasa_nlu.model.Metadata
:rtype: MITIEInterpreter
"""
if meta.entity_extractor_path:
extractor = named_entity_extractor(meta.entity_extractor_path)
else:
extractor = None
if meta.intent_classifier_path:
classifier = text_categorizer(meta.intent_classifier_path)
else:
classifier = None
if featurizer is None:
featurizer = MITIEFeaturizer(meta.feature_extractor_path)
if meta.entity_synonyms_path:
entity_synonyms = Interpreter.load_synonyms(
meta.entity_synonyms_path)
else:
entity_synonyms = None
return MITIEInterpreter(classifier, extractor, featurizer,
entity_synonyms)
def __init__(self,
intent_classifier=None,
entity_extractor=None,
feature_extractor=None,
entity_synonyms=None,
**kwargs):
self.extractor = None
self.classifier = None
if entity_extractor:
self.extractor = named_entity_extractor(entity_extractor,
feature_extractor)
if intent_classifier:
with open(intent_classifier, 'rb') as f:
self.classifier = cloudpickle.load(f)
self.featurizer = MITIEFeaturizer(feature_extractor)
self.tokenizer = MITIETokenizer()
self.ent_synonyms = None
if entity_synonyms:
self.ent_synonyms = Interpreter.load_synonyms(entity_synonyms)
def __init__(self, config):
self.server = None
self.config = config
self.logfile = config.write
self.emulator = self.__create_emulator()
self.interpreter = self.__create_interpreter()
self.data_router = DataRouter(config, self.interpreter, self.emulator)
if 'DYNO' in os.environ and config.backend == 'mitie': # running on Heroku
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
MITIEFeaturizer(config.mitie_file)
def featurizer_for_model(model, nlp):
"""Initialize featurizer for backends. If it can NOT be shared between models, `None` should be returned."""
if model.backend_name() == spacy.SPACY_BACKEND_NAME:
from featurizers.spacy_featurizer import SpacyFeaturizer
return SpacyFeaturizer(nlp)
elif model.backend_name() == mitie.MITIE_BACKEND_NAME or \
model.backend_name() == mitie.MITIE_SKLEARN_BACKEND_NAME:
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
return MITIEFeaturizer(model.feature_extractor_path)
else:
return None
class MITIESklearnInterpreter(Interpreter):
def __init__(self,
intent_classifier=None,
entity_extractor=None,
feature_extractor=None,
entity_synonyms=None,
**kwargs):
self.extractor = None
self.classifier = None
if entity_extractor:
self.extractor = named_entity_extractor(entity_extractor,
feature_extractor)
if intent_classifier:
with open(intent_classifier, 'rb') as f:
self.classifier = cloudpickle.load(f)
self.featurizer = MITIEFeaturizer(feature_extractor)
self.tokenizer = MITIETokenizer()
self.ent_synonyms = None
if entity_synonyms:
self.ent_synonyms = Interpreter.load_synonyms(entity_synonyms)
def get_intent(self, sentence_tokens):
"""Returns the most likely intent and its probability for the input text.
:param sentence_tokens: text to classify
:return: tuple of most likely intent name and its probability"""
if self.classifier:
X = self.featurizer.features_for_tokens(sentence_tokens).reshape(
1, -1)
intent_ids, probabilities = self.classifier.predict(X)
intents = self.classifier.transform_labels_num2str(intent_ids)
intent, score = intents[0], probabilities[0]
else:
intent, score = "None", 0.0
return intent, score
def parse(self, text):
tokens = self.tokenizer.tokenize(text)
intent, probability = self.get_intent(tokens)
entities = get_entities(text, tokens, self.extractor)
if self.ent_synonyms:
Interpreter.replace_synonyms(entities, self.ent_synonyms)
return {
'text': text,
'intent': intent,
'entities': entities,
'confidence': probability
}
class MITIESklearnInterpreter(Interpreter):
def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs):
if entity_extractor_file:
self.extractor = named_entity_extractor(entity_extractor_file) # ,metadata["feature_extractor"])
with open(intent_classifier_file, 'rb') as f:
self.classifier = cloudpickle.load(f)
self.featurizer = MITIEFeaturizer(feature_extractor_file)
self.tokenizer = MITIETokenizer()
def get_entities(self, tokens):
d = {}
entities = self.extractor.extract_entities(tokens)
for e in entities:
_range = e[0]
d[e[1]] = " ".join(tokens[i] for i in _range)
return d
def get_intent(self, text):
"""Returns the most likely intent and its probability for the input text.
:param text: text to classify
:return: tuple of most likely intent name and its probability"""
if self.classifier:
X = self.featurizer.create_bow_vecs([text])
intent_ids, probabilities = self.classifier.predict(X)
intents = self.classifier.transform_labels_num2str(intent_ids)
intent, score = intents[0], probabilities[0]
else:
intent, score = "None", 0.0
return intent, score
def parse(self, text):
tokens = self.tokenizer.tokenize(text)
intent, probability = self.get_intent(tokens)
entities = self.get_entities(tokens)
return {'text': text, 'intent': intent, 'entities': entities, 'confidence': probability}
def download(config, pkg="mitie"):
if pkg == "mitie":
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
MITIEFeaturizer(config.mitie_file)
else:
print("Error. Package {0} not available for download.".format(pkg))
def __init__(self, fe_file, language_name, max_num_threads=1):
super(self.__class__, self).__init__(language_name, max_num_threads)
self.fe_file = fe_file
self.featurizer = MITIEFeaturizer(self.fe_file)