def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs): if entity_extractor_file: self.extractor = named_entity_extractor(entity_extractor_file) # ,metadata["feature_extractor"]) with open(intent_classifier_file, 'rb') as f: self.classifier = cloudpickle.load(f) self.featurizer = MITIEFeaturizer(feature_extractor_file) self.tokenizer = MITIETokenizer()
def test_mitie(): from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer filename = os.environ.get('MITIE_FILE') if (filename and os.path.isfile(filename)): ftr = MITIEFeaturizer(os.environ.get('MITIE_FILE')) sentence = "Hey how are you today" vecs = ftr.create_bow_vecs([sentence]) assert np.allclose(vecs[0][:5], np.array([0., -4.4551446, 0.26073121, -1.46632245, -1.84205751]), atol=1e-5)
def test_mitie_featurizer(): from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer filename = os.environ.get('MITIE_FILE') if not filename or not os.path.isfile(filename): filename = "data/total_word_feature_extractor.dat" ftr = MITIEFeaturizer(filename) sentence = "Hey how are you today" tokens = MITIETokenizer().tokenize(sentence) vecs = ftr.features_for_tokens(tokens) assert np.allclose( vecs[:5], np.array([0., -4.4551446, 0.26073121, -1.46632245, -1.84205751]), atol=1e-5)
def load(meta, featurizer=None): """ :type meta: rasa_nlu.model.Metadata :rtype: MITIESklearnInterpreter """ if meta.entity_extractor_path: extractor = named_entity_extractor(meta.entity_extractor_path) else: extractor = None if featurizer is None: featurizer = MITIEFeaturizer(meta.feature_extractor_path) if meta.intent_classifier_path: with open(meta.intent_classifier_path, 'rb') as f: classifier = cloudpickle.load(f) else: classifier = None if meta.entity_synonyms_path: entity_synonyms = Interpreter.load_synonyms( meta.entity_synonyms_path) else: entity_synonyms = None return MITIESklearnInterpreter(classifier, extractor, featurizer, entity_synonyms)
def load(meta, featurizer=None): """ :type meta: rasa_nlu.model.Metadata :rtype: MITIEInterpreter """ if meta.entity_extractor_path: extractor = named_entity_extractor(meta.entity_extractor_path) else: extractor = None if meta.intent_classifier_path: classifier = text_categorizer(meta.intent_classifier_path) else: classifier = None if featurizer is None: featurizer = MITIEFeaturizer(meta.feature_extractor_path) if meta.entity_synonyms_path: entity_synonyms = Interpreter.load_synonyms( meta.entity_synonyms_path) else: entity_synonyms = None return MITIEInterpreter(classifier, extractor, featurizer, entity_synonyms)
def __init__(self, intent_classifier=None, entity_extractor=None, feature_extractor=None, entity_synonyms=None, **kwargs): self.extractor = None self.classifier = None if entity_extractor: self.extractor = named_entity_extractor(entity_extractor, feature_extractor) if intent_classifier: with open(intent_classifier, 'rb') as f: self.classifier = cloudpickle.load(f) self.featurizer = MITIEFeaturizer(feature_extractor) self.tokenizer = MITIETokenizer() self.ent_synonyms = None if entity_synonyms: self.ent_synonyms = Interpreter.load_synonyms(entity_synonyms)
def __init__(self, config): self.server = None self.config = config self.logfile = config.write self.emulator = self.__create_emulator() self.interpreter = self.__create_interpreter() self.data_router = DataRouter(config, self.interpreter, self.emulator) if 'DYNO' in os.environ and config.backend == 'mitie': # running on Heroku from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer MITIEFeaturizer(config.mitie_file)
def featurizer_for_model(model, nlp): """Initialize featurizer for backends. If it can NOT be shared between models, `None` should be returned.""" if model.backend_name() == spacy.SPACY_BACKEND_NAME: from featurizers.spacy_featurizer import SpacyFeaturizer return SpacyFeaturizer(nlp) elif model.backend_name() == mitie.MITIE_BACKEND_NAME or \ model.backend_name() == mitie.MITIE_SKLEARN_BACKEND_NAME: from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer return MITIEFeaturizer(model.feature_extractor_path) else: return None
class MITIESklearnInterpreter(Interpreter): def __init__(self, intent_classifier=None, entity_extractor=None, feature_extractor=None, entity_synonyms=None, **kwargs): self.extractor = None self.classifier = None if entity_extractor: self.extractor = named_entity_extractor(entity_extractor, feature_extractor) if intent_classifier: with open(intent_classifier, 'rb') as f: self.classifier = cloudpickle.load(f) self.featurizer = MITIEFeaturizer(feature_extractor) self.tokenizer = MITIETokenizer() self.ent_synonyms = None if entity_synonyms: self.ent_synonyms = Interpreter.load_synonyms(entity_synonyms) def get_intent(self, sentence_tokens): """Returns the most likely intent and its probability for the input text. :param sentence_tokens: text to classify :return: tuple of most likely intent name and its probability""" if self.classifier: X = self.featurizer.features_for_tokens(sentence_tokens).reshape( 1, -1) intent_ids, probabilities = self.classifier.predict(X) intents = self.classifier.transform_labels_num2str(intent_ids) intent, score = intents[0], probabilities[0] else: intent, score = "None", 0.0 return intent, score def parse(self, text): tokens = self.tokenizer.tokenize(text) intent, probability = self.get_intent(tokens) entities = get_entities(text, tokens, self.extractor) if self.ent_synonyms: Interpreter.replace_synonyms(entities, self.ent_synonyms) return { 'text': text, 'intent': intent, 'entities': entities, 'confidence': probability }
class MITIESklearnInterpreter(Interpreter): def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs): if entity_extractor_file: self.extractor = named_entity_extractor(entity_extractor_file) # ,metadata["feature_extractor"]) with open(intent_classifier_file, 'rb') as f: self.classifier = cloudpickle.load(f) self.featurizer = MITIEFeaturizer(feature_extractor_file) self.tokenizer = MITIETokenizer() def get_entities(self, tokens): d = {} entities = self.extractor.extract_entities(tokens) for e in entities: _range = e[0] d[e[1]] = " ".join(tokens[i] for i in _range) return d def get_intent(self, text): """Returns the most likely intent and its probability for the input text. :param text: text to classify :return: tuple of most likely intent name and its probability""" if self.classifier: X = self.featurizer.create_bow_vecs([text]) intent_ids, probabilities = self.classifier.predict(X) intents = self.classifier.transform_labels_num2str(intent_ids) intent, score = intents[0], probabilities[0] else: intent, score = "None", 0.0 return intent, score def parse(self, text): tokens = self.tokenizer.tokenize(text) intent, probability = self.get_intent(tokens) entities = self.get_entities(tokens) return {'text': text, 'intent': intent, 'entities': entities, 'confidence': probability}
def download(config, pkg="mitie"): if pkg == "mitie": from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer MITIEFeaturizer(config.mitie_file) else: print("Error. Package {0} not available for download.".format(pkg))
def __init__(self, fe_file, language_name, max_num_threads=1): super(self.__class__, self).__init__(language_name, max_num_threads) self.fe_file = fe_file self.featurizer = MITIEFeaturizer(self.fe_file)