def load(meta, nlp, featurizer=None): """ :type meta: rasa_nlu.model.Metadata :type nlp: spacy.language.Language :type featurizer: None or rasa_nlu.featurizers.spacy_featurizer.SpacyFeaturizer :rtype: MITIEInterpreter """ if meta.entity_extractor_path: extractor = SpacyEntityExtractor(nlp, meta.entity_extractor_path, meta.metadata.get("should_fine_tune_spacy_ner")) else: extractor = None if meta.intent_classifier_path: with open(meta.intent_classifier_path, 'rb') as f: classifier = cloudpickle.load(f) else: classifier = None if meta.entity_synonyms_path: entity_synonyms = Interpreter.load_synonyms(meta.entity_synonyms_path) else: entity_synonyms = None if featurizer is None: featurizer = SpacyFeaturizer(nlp) return SpacySklearnInterpreter( classifier, extractor, entity_synonyms, featurizer, nlp)
def test_spacy_featurizer(spacy_nlp_en, sentence, expected): from rasa_nlu.featurizers.spacy_featurizer import SpacyFeaturizer ftr = SpacyFeaturizer(spacy_nlp_en) doc = spacy_nlp_en(sentence) vecs = ftr.features_for_doc(doc) assert np.allclose(doc.vector[:5], expected, atol=1e-5) assert np.allclose(vecs, doc.vector, atol=1e-5)
def __init__(self, entity_extractor=None, intent_classifier=None, language_name='en', **kwargs): self.nlp = spacy.load(language_name, parser=False, entity=False, matcher=False) self.featurizer = SpacyFeaturizer(self.nlp) with open(intent_classifier, 'rb') as f: self.classifier = cloudpickle.load(f) self.extractor = SpacyEntityExtractor(self.nlp, entity_extractor)
def __init__(self, config, language_name): self.ensure_language_support(language_name) self.name = "spacy_sklearn" self.language_name = language_name self.training_data = None self.nlp = spacy.load(self.language_name, parser=False, entity=False) self.featurizer = SpacyFeaturizer(self.nlp) self.intent_classifier = None self.entity_extractor = None
def test_spacy_featurizer(sentence, language, expected): import spacy from rasa_nlu.featurizers.spacy_featurizer import SpacyFeaturizer nlp = spacy.load(language, tagger=False, parser=False) ftr = SpacyFeaturizer(nlp) doc = nlp(sentence) vecs = ftr.features_for_doc(doc) assert np.allclose(doc.vector[:5], expected, atol=1e-5) assert np.allclose(vecs, doc.vector, atol=1e-5)
def test_sentence(sentence, language, _ref): import spacy from rasa_nlu.featurizers.spacy_featurizer import SpacyFeaturizer nlp = spacy.load(language, tagger=False, parser=False) doc = nlp(sentence) ftr = SpacyFeaturizer(nlp) vecs = ftr.create_bow_vecs([sentence]) assert np.allclose(doc.vector[:5], _ref, atol=1e-5) assert np.allclose(vecs[0], doc.vector, atol=1e-5)
def __init__(self, language_name, max_num_threads=1, should_fine_tune_spacy_ner=False): super(self.__class__, self).__init__(language_name, max_num_threads) self.should_fine_tune_spacy_ner = should_fine_tune_spacy_ner self.nlp = self._load_nlp_model(language_name, should_fine_tune_spacy_ner) self.featurizer = SpacyFeaturizer(self.nlp) ensure_proper_language_model(self.nlp)
def __init__(self, intent_classifier=None, entity_extractor=None, entity_synonyms=None, nlp=None): self.extractor = entity_extractor self.classifier = intent_classifier self.ent_synonyms = entity_synonyms self.nlp = nlp self.featurizer = SpacyFeaturizer(nlp) ensure_proper_language_model(nlp)
def __init__(self, entity_extractor=None, entity_synonyms=None, intent_classifier=None, language_name='en', **kwargs): self.extractor = None self.classifier = None self.ent_synonyms = None self.nlp = spacy.load(language_name, parser=False, entity=False, matcher=False) self.featurizer = SpacyFeaturizer(self.nlp) ensure_proper_language_model(self.nlp) if intent_classifier: with open(intent_classifier, 'rb') as f: self.classifier = cloudpickle.load(f) if entity_extractor: self.extractor = SpacyEntityExtractor(self.nlp, entity_extractor) self.ent_synonyms = Interpreter.load_synonyms(entity_synonyms)
def __init__(self, language_name, max_num_threads=1): super(self.__class__, self).__init__("spacy_sklearn", language_name, max_num_threads) self.nlp = spacy.load(self.language_name, parser=False, entity=False) self.featurizer = SpacyFeaturizer(self.nlp) ensure_proper_language_model(self.nlp)
from rasa_nlu.train import load_data from rasa_nlu.config import RasaNLUModelConfig from rasa_nlu.utils.spacy_utils import SpacyNLP from rasa_nlu.tokenizers.spacy_tokenizer import SpacyTokenizer from rasa_nlu.featurizers.spacy_featurizer import SpacyFeaturizer import numpy as np, spacy training_data = load_data("data/examples/rasa/demo-rasa.json") config = RasaNLUModelConfig() SpacyNLP(nlp=spacy.load("en")).train(training_data, config) SpacyTokenizer().train(training_data, config) SpacyFeaturizer().train(training_data, config) from sklearn.preprocessing import LabelEncoder from sklearn.model_selection import GridSearchCV from sklearn.svm import SVC labels = [e.get("intent") for e in training_data.intent_examples] le = LabelEncoder() y = le.fit_transform(labels) X = np.stack([ example.get("text_features") for example in training_data.intent_examples ]) defaults = { # C parameter of the svm - cross validation will select the best value "C": [1, 2, 5, 10, 20, 100], # the kernels to use for the svm training - cross validation will # decide which one of them performs best