def load(cls, model_dir: Text = None, model_metadata: Metadata = None, cached_component: Optional['LanguageSetter'] = None, **kwargs: Any) -> 'LanguageSetter': component_config = model_metadata.for_component(cls.name) return cls(component_config, model_metadata.get("language"))
def test_loading_no_warning(): syn = EntitySynonymMapper(synonyms=None) syn.persist("test", "test") meta = Metadata({"test": 1}, "test") with pytest.warns(None) as warn: syn.load(meta.for_component(0), "test", meta) assert len(warn) == 0
def read_model_metadata(model_dir, config): if model_dir is None: data = DataRouter.default_model_metadata() return Metadata(data, model_dir) else: if not os.path.isabs(model_dir): model_dir = os.path.join(config['path'], model_dir) # download model from S3 if needed if not os.path.isdir(model_dir): DataRouter.load_model_from_cloud(model_dir, config) return Metadata.load(model_dir)
def _read_model_metadata(self, model_dir): if model_dir is None: data = Project._default_model_metadata() return Metadata(data, model_dir) else: if not os.path.isabs(model_dir): model_dir = os.path.join(self._path, model_dir) # download model from S3 if needed if not os.path.isdir(model_dir): Project._load_model_from_cloud(model_dir, self._config) return Metadata.load(model_dir)
def _read_model_metadata(self, model_name): if model_name is None: data = Project._default_model_metadata() return Metadata(data, model_name) else: if not os.path.isabs(model_name) and self._path: path = os.path.join(self._path, model_name) else: path = model_name # download model from cloud storage if needed and possible if not os.path.isdir(path): self._load_model_from_cloud(model_name, path) return Metadata.load(path)
def run_intent_evaluation(config, model_path, component_builder=None): from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels # get the metadata config from the package data test_data = load_data(config['data']) metadata = Metadata.load(model_path) interpreter = Interpreter.load(metadata, config, component_builder) test_y = [e.get("intent") for e in test_data.training_examples] preds = [] for e in test_data.training_examples: res = interpreter.parse(e.text) if res.get('intent'): preds.append(res['intent'].get('name')) else: preds.append(None) log_evaluation_table(test_y, preds) cnf_matrix = confusion_matrix(test_y, preds) plot_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds), title='Intent Confusion matrix') plt.show() return
def __init__(self): self.id_obra = "0" # CONFIGURA METAFILE logger.info("[*] CONFIGURANDO CONFIG FILE INTENCAO") _data_j = json.loads(codecs.open("models/intencao_config.json").read()) _data_j["path"] = real_path("models") _data_j["mitie_file"] = real_path( "utils/total_word_feature_extractor.dat") _data_j["data"] = real_path("models/intencao_data.json") with codecs.open("models/intencao_config.json", "w") as f: json.dump(_data_j, f, indent=4) # CONFIGURA METAFILE logger.info("[*] CONFIGURANDO METAFILE INTENCAO") _meta_j = json.loads(codecs.open("models/metadata.json").read()) _meta_j["mitie_file"] = real_path( "utils/total_word_feature_extractor.dat") with codecs.open("models/metadata.json", "w") as f: json.dump(_meta_j, f, indent=4) # CARREGA MODELO logger.info("[*] CARREGANDO MODELO -> INTENCOES") meta_data = Metadata.load(real_path("models")) self.inteligencia = Interpreter.load( meta_data, RasaNLUConfig(real_path("models/intencao_config.json")))
def run_intent_evaluation(config, model_path, component_builder=None): from sklearn.metrics import accuracy_score from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from sklearn.metrics import f1_score from sklearn.metrics import precision_score from sklearn.utils.multiclass import unique_labels # get the metadata config from the package data test_data = load_data(config['data']) metadata = Metadata.load(model_path) interpreter = Interpreter.load(metadata, config, component_builder) test_y = [e.get("intent") for e in test_data.training_examples] preds = [] for e in test_data.training_examples: res = interpreter.parse(e.text) if res.get('intent'): preds.append(res['intent'].get('name')) else: preds.append(None) logger.info("Intent Evaluation Results") logger.info("F1-Score: {}".format(f1_score(test_y, preds, average='weighted'))) logger.info("Precision: {}".format(precision_score(test_y, preds, average='weighted'))) logger.info("Accuracy: {}".format(accuracy_score(test_y, preds))) logger.info("Classification report: \n{}".format(classification_report(test_y, preds))) cnf_matrix = confusion_matrix(test_y, preds) plot_intent_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds), title='Intent Confusion matrix') plt.show() return
def start_bot(config): rasa_section = config['rasa'] telegram_section = config['telegram'] logger.info('Loading model') model = get_model(rasa_section) metadata = Metadata.load(model) logger.info('Loading interpreter') rasa_config = rasa_section['config'] interpreter = Interpreter.load(metadata, RasaNLUConfig(rasa_config)) logger.info('Preparing Bot') token = telegram_section['token'] bot = telepot.DelegatorBot(token, [ pave_event_space()(per_chat_id(), create_open, KiririnHandler, interpreter, config, timeout=20) ]) MessageLoop(bot).run_as_thread() logger.info('Listening') while 1: time.sleep(10)
def __create_model_store(self): # Fallback for users that specified the model path as a string and hence only want a single default model. if type(self.config.server_model_dirs) is Text: model_dict = { self.DEFAULT_MODEL_NAME: self.config.server_model_dirs } elif self.config.server_model_dirs is None: model_dict = self.__search_for_models() else: model_dict = self.config.server_model_dirs model_store = {} for alias, model_path in list(model_dict.items()): try: logger.info("Loading model '{}'...".format(model_path)) model_store[alias] = self.__interpreter_for_model(model_path) except Exception as e: logger.exception("Failed to load model '{}'. Error: {}".format( model_path, e)) if not model_store: meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "") interpreter = Interpreter.load(meta, self.config, self.component_builder) model_store[self.DEFAULT_MODEL_NAME] = interpreter return model_store
def interpret_query_for(text_to_be_parsed, user_name="User"): # Example scenario: # text_to_be_parsed: 'what is the total population?' metadata = Metadata.load(model_directory) interpreter = Interpreter.load(model_directory) interpreted_data = interpreter.parse(text_to_be_parsed.decode('utf-8')) return get_structured_query(interpreted_data, user_name)
def load(cls, meta: Dict[Text, Any], model_dir: Text = None, model_metadata: Metadata = None, cached_component: Optional['DucklingHTTPExtractor'] = None, **kwargs: Any) -> 'DucklingHTTPExtractor': return cls(meta, model_metadata.get("language"))
def cache_key(cls, model_metadata: Metadata) -> Optional[Text]: component_meta = model_metadata.for_component(cls.name) mitie_file = component_meta.get("model", None) if mitie_file is not None: return cls.name + "-" + str(os.path.abspath(mitie_file)) else: return None
def __init__(self, model, config, builder=None, percentage=0.65): self.model = model self.metadata = Metadata.load(self.model) self.config = config self.percentage = percentage if builder: self.interpreter = Interpreter.load(self.metadata, RasaNLUConfig(config), builder) else: self.interpreter = Interpreter.load(self.metadata, RasaNLUConfig(config))
def _fallback_model(self): meta = Metadata( { "pipeline": [{ "name": "intent_classifier_keyword", "class": utils.module_path_from_object(KeywordIntentClassifier()) }] }, "") return Interpreter.create(meta, self._component_builder)
def _search_for_models(self): prefix = 'model_' if not self._path or not os.path.isdir(self._path): meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "") interpreter = Interpreter.create(meta, self._config, self._component_builder) models = {'fallback': interpreter} else: models = {model: None for model in os.listdir(self._path) if model.startswith(prefix)} models.update(self._models) self._models = models
def __init__(self, languages): utils.check_languages(languages) self.languages = languages self.config = RasaNLUConfig(cmdline_args=utils.load_config()) self.query_logger = DataRouter._create_query_logger(self.config['response_log']) self._metadata = {} self._interpreter = {} for lang in languages: self._metadata[lang] = Metadata.load(utils.get_model_dir(lang)) self._interpreter[lang] = Interpreter.load(self._metadata[lang], self.config) self.registry = IntentRegistry(self.config['active_apps'])
def __init__(self, model_directory, config_file=None, lazy_init=False): from rasa_nlu.model import Interpreter from rasa_nlu.model import Metadata from rasa_nlu.config import RasaNLUConfig self.metadata = Metadata.load(model_directory) self.lazy_init = lazy_init self.config_file = config_file if not lazy_init: self.interpreter = Interpreter.load(self.metadata, RasaNLUConfig(config_file)) else: self.interpreter = None
def create_component(self, component_name, config): # type: (Text, RasaNLUConfig) -> Component """Tries to retrieve a component from the cache, calls `create` to create a new component.""" from rasa_nlu import registry from rasa_nlu.model import Metadata try: component, cache_key = self.__get_cached_component(component_name, Metadata(config.as_dict(), None)) if component is None: component = registry.create_component_by_name(component_name, config) self.__add_to_cache(component, cache_key) return component except MissingArgumentError as e: # pragma: no cover raise Exception("Failed to create component '{}'. {}".format(component_name, e))
def load(cls, model_dir: str, model_metadata: Metadata, cached_component: Component, **kwargs: Any) -> 'Word2vecKerasIntentClassifier': meta: Dict[str, Any] = model_metadata.for_component(cls.name) clf_config_file_path: str = meta['clf_config_file_path'] with open(clf_config_file_path) as f: clf_config: Dict[str, Any] = json.load(f) clf_file_path: str = meta['clf_file_path'] clf_model: Model = load_model(clf_file_path) clf: KerasBaseModel = KerasBaseModel(clf_config, clf_model) labels_file_path: str = meta['labels_file_path'] with open(labels_file_path) as g: labels: List[str] = json.load(g) return cls(meta, clf_config, clf, labels)
def _read_model_metadata(self, model_name): if model_name is None: data = Project._default_model_metadata() return Metadata(data, model_name) else: if not os.path.isabs(model_name): path = os.path.join(self._path, model_name) else: path = model_name # download model from cloud storage if needed and possible if not os.path.isdir(path): self._load_model_from_cloud(model_name, path, self._config) return Metadata.load(path)
def load(cls, model_dir: Text = None, model_metadata: Metadata = None, cached_component: Optional['CRFEntityExtractor'] = None, **kwargs: Any) -> 'CRFEntityExtractor': from sklearn.externals import joblib meta = model_metadata.for_component(cls.name) file_name = meta.get("classifier_file", CRF_MODEL_FILE_NAME) model_file = os.path.join(model_dir, file_name) if os.path.exists(model_file): ent_tagger = joblib.load(model_file) return cls(meta, ent_tagger) else: return cls(meta)
def load(cls, model_dir: Text = None, model_metadata: Metadata = None, cached_component: Optional['CountVectorsFeaturizer'] = None, **kwargs: Any) -> 'CountVectorsFeaturizer': meta = model_metadata.for_component(cls.name) if model_dir and meta.get("featurizer_file"): file_name = meta.get("featurizer_file") featurizer_file = os.path.join(model_dir, file_name) return utils.pycloud_unpickle(featurizer_file) else: logger.warning("Failed to load featurizer. Maybe path {} " "doesn't exist".format(os.path.abspath(model_dir))) return CountVectorsFeaturizer(meta)
def create_component(self, component_config: Dict[Text, Any], cfg: RasaNLUModelConfig) -> Component: """Tries to retrieve a component from the cache, calls `create` to create a new component.""" from rasa_nlu import registry from rasa_nlu.model import Metadata try: component, cache_key = self.__get_cached_component( component_config, Metadata(cfg.as_dict(), None)) if component is None: component = registry.create_component_by_config( component_config, cfg) self.__add_to_cache(component, cache_key) return component except MissingArgumentError as e: # pragma: no cover raise Exception("Failed to create component `{}`. " "{}".format(component_config['name'], e))
def __init__(self, model_dir=MODEL_DIR, conf_file=CONFIG_DIR): # Takes some time,to initialize from data.intents import INTENTS self.intents_info = INTENTS from data.contexts import CONTEXTS self.contexts_info = CONTEXTS from data.fallback import RESPONSES self.fallback_responses = RESPONSES print("Initializing the model...") metadata = Metadata.load(model_dir) interpreter = Interpreter.load(metadata, RasaNLUConfig(conf_file)) print("Ready") print("") self.modelInterpreter = interpreter
def load(cls, model_dir: Text = None, model_metadata: Metadata = None, cached_component: Optional['MitieEntityExtractor'] = None, **kwargs: Any) -> 'MitieEntityExtractor': import mitie meta = model_metadata.for_component(cls.name) file_name = meta.get("classifier_file", MITIE_ENTITY_MODEL_FILE_NAME) if not file_name: return cls(meta) classifier_file = os.path.join(model_dir, file_name) if os.path.exists(classifier_file): extractor = mitie.named_entity_extractor(classifier_file) return cls(meta, extractor) else: return cls(meta)
def run_intent_evaluation(config, model_path, component_builder=None): from sklearn.metrics import accuracy_score from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from sklearn.metrics import f1_score from sklearn.metrics import precision_score from sklearn.utils.multiclass import unique_labels # get the metadata config from the package data test_data = load_data(config['data']) metadata = Metadata.load(model_path) interpreter = Interpreter.load(metadata, config, component_builder) test_y = [e.get("intent") for e in test_data.training_examples] preds = [] for e in test_data.training_examples: res = interpreter.parse(e.text) if res.get('intent'): preds.append(res['intent'].get('name')) else: preds.append(None) logger.info("Intent Evaluation Results") logger.info("F1-Score: {}".format( f1_score(test_y, preds, average='weighted'))) logger.info("Precision: {}".format( precision_score(test_y, preds, average='weighted'))) logger.info("Accuracy: {}".format(accuracy_score(test_y, preds))) logger.info("Classification report: \n{}".format( classification_report(test_y, preds))) cnf_matrix = confusion_matrix(test_y, preds) plot_intent_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds), title='Intent Confusion matrix') plt.show() return
def classify(self, sent, user): if self.interpreter is None: print("train if you got new.. using old one ") try: model_directory = self.trainer.persist('./models/') metadata = Metadata.load(model_directory) self.interpreter = Interpreter.load(metadata, config) except: raise ("Need to train model. No pre-trained model found.") result = self.interpreter.parse(sent) probable_module = result["intent"]['name'].split("-") is_calc = len(probable_module) - 1 print(is_calc) if is_calc: print('here') return eval(probable_module[0])(sent, user) return best_match(sent, probable_module[0])
def __create_model_store(self): # Fallback for users that specified the model path as a string and hence only want a single default model. if type(self.config.server_model_dirs) is unicode or type( self.config.server_model_dirs) is str: model_dict = { self.DEFAULT_MODEL_NAME: self.config.server_model_dirs } elif self.config.server_model_dirs is None: model_dict = self.__search_for_models() else: model_dict = self.config.server_model_dirs # Reuse nlp and featurizers where possible to save memory cache = {} model_store = {} for alias, model_path in model_dict.items(): metadata = DataRouter.read_model_metadata(model_path, self.config) cache_key = metadata.model_group() if cache_key in cache: nlp = cache[cache_key]['nlp'] featurizer = cache[cache_key]['featurizer'] else: nlp = DataRouter.nlp_for_backend(metadata.backend_name(), metadata.language_name()) featurizer = DataRouter.featurizer_for_model(metadata, nlp) cache[cache_key] = {'nlp': nlp, 'featurizer': featurizer} interpreter = DataRouter.create_interpreter( metadata, nlp, featurizer) model_store[alias] = Model(metadata, model_path, interpreter, nlp, featurizer) if not model_store: meta = Metadata({}, "") interpreter = DataRouter.create_interpreter(meta) model_store[self.DEFAULT_MODEL_NAME] = Model(meta, "", interpreter) return model_store
from rasa_nlu.model import Interpreter, Metadata from rasa_nlu.config import RasaNLUConfig config = RasaNLUConfig('E:/chat_demo_updated/Data/config_spacy.json') metadata = Metadata.load('./models/trained') interpreter = Interpreter.load(metadata, config) # #for testing purposes # result='' # while(result!='exit'): # user_says=raw_input("User_says> ") # result1=interpreter.parse(unicode(user_says)) # # print(result1['entities'][0]['entity']) # print(result1) # result=user_says
if detected_lang == 'en': trans_input = out_list else: mid_trans = translator.translate(out_list, src="hi", dest = "ur").text trans_input = translator.translate(mid_trans, src = "ur", dest = "en").text print("Translated Input: ", trans_input) #----- ASSIGN TO RASA_NLU -----# #Training Time from rasa_nlu.converters import load_data from rasa_nlu.config import RasaNLUConfig from rasa_nlu.components import ComponentBuilder from rasa_nlu.model import Trainer builder = ComponentBuilder(use_cache=True) #will cache components between pipelines (where possible) training_data = load_data('testData.json') trainer = Trainer(RasaNLUConfig("config_spacy.json"), builder) trainer.train(training_data) model_directory = trainer.persist('./models/') #returns the directory the model is stored in #Prediction Time from rasa_nlu.model import Metadata, Interpreter config = RasaNLUConfig("config_spacy.json") metad = Metadata.load(model_directory) #loads metadata.json interpreter = Interpreter.load(metad, config, builder) interpreter.parse(trans_input) #output
def test_builder_load_unknown(component_builder): with pytest.raises(Exception) as excinfo: component_builder.load_component("my_made_up_componment", "", Metadata({}, None)) assert "Unknown component name" in str(excinfo.value)
from rasa_nlu.model import Metadata, Interpreter from rasa_nlu.config import RasaNLUConfig import json from flask import Flask, jsonify, render_template app = Flask(__name__) metadata = Metadata.load('/home/sikim/RNLU/models/model_20170403-084112') interpreter = Interpreter.load(metadata, RasaNLUConfig('/home/sikim/RNLU/config.json')) @app.route("/index", methods =['GET']) def index(): return render_template("index.html") @app.route("/fetchentityintent/<text>", methods = ['GET']) def fetchentityintent(text): resp = interpreter.parse(unicode(text)) return jsonify(resp) if __name__ == "__main__": app.run(host='0.0.0.0')