def load(cls,
model_dir: Text = None,
model_metadata: Metadata = None,
cached_component: Optional['LanguageSetter'] = None,
**kwargs: Any) -> 'LanguageSetter':
component_config = model_metadata.for_component(cls.name)
return cls(component_config, model_metadata.get("language"))
def test_loading_no_warning():
syn = EntitySynonymMapper(synonyms=None)
syn.persist("test", "test")
meta = Metadata({"test": 1}, "test")
with pytest.warns(None) as warn:
syn.load(meta.for_component(0), "test", meta)
assert len(warn) == 0
def read_model_metadata(model_dir, config):
if model_dir is None:
data = DataRouter.default_model_metadata()
return Metadata(data, model_dir)
else:
if not os.path.isabs(model_dir):
model_dir = os.path.join(config['path'], model_dir)
# download model from S3 if needed
if not os.path.isdir(model_dir):
DataRouter.load_model_from_cloud(model_dir, config)
return Metadata.load(model_dir)
def _read_model_metadata(self, model_dir):
if model_dir is None:
data = Project._default_model_metadata()
return Metadata(data, model_dir)
else:
if not os.path.isabs(model_dir):
model_dir = os.path.join(self._path, model_dir)
# download model from S3 if needed
if not os.path.isdir(model_dir):
Project._load_model_from_cloud(model_dir, self._config)
return Metadata.load(model_dir)
def _read_model_metadata(self, model_name):
if model_name is None:
data = Project._default_model_metadata()
return Metadata(data, model_name)
else:
if not os.path.isabs(model_name) and self._path:
path = os.path.join(self._path, model_name)
else:
path = model_name
# download model from cloud storage if needed and possible
if not os.path.isdir(path):
self._load_model_from_cloud(model_name, path)
return Metadata.load(path)
def run_intent_evaluation(config, model_path, component_builder=None):
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
# get the metadata config from the package data
test_data = load_data(config['data'])
metadata = Metadata.load(model_path)
interpreter = Interpreter.load(metadata, config, component_builder)
test_y = [e.get("intent") for e in test_data.training_examples]
preds = []
for e in test_data.training_examples:
res = interpreter.parse(e.text)
if res.get('intent'):
preds.append(res['intent'].get('name'))
else:
preds.append(None)
log_evaluation_table(test_y, preds)
cnf_matrix = confusion_matrix(test_y, preds)
plot_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds),
title='Intent Confusion matrix')
plt.show()
return
def __init__(self):
self.id_obra = "0"
# CONFIGURA METAFILE
logger.info("[*] CONFIGURANDO CONFIG FILE INTENCAO")
_data_j = json.loads(codecs.open("models/intencao_config.json").read())
_data_j["path"] = real_path("models")
_data_j["mitie_file"] = real_path(
"utils/total_word_feature_extractor.dat")
_data_j["data"] = real_path("models/intencao_data.json")
with codecs.open("models/intencao_config.json", "w") as f:
json.dump(_data_j, f, indent=4)
# CONFIGURA METAFILE
logger.info("[*] CONFIGURANDO METAFILE INTENCAO")
_meta_j = json.loads(codecs.open("models/metadata.json").read())
_meta_j["mitie_file"] = real_path(
"utils/total_word_feature_extractor.dat")
with codecs.open("models/metadata.json", "w") as f:
json.dump(_meta_j, f, indent=4)
# CARREGA MODELO
logger.info("[*] CARREGANDO MODELO -> INTENCOES")
meta_data = Metadata.load(real_path("models"))
self.inteligencia = Interpreter.load(
meta_data, RasaNLUConfig(real_path("models/intencao_config.json")))
def run_intent_evaluation(config, model_path, component_builder=None):
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score
from sklearn.utils.multiclass import unique_labels
# get the metadata config from the package data
test_data = load_data(config['data'])
metadata = Metadata.load(model_path)
interpreter = Interpreter.load(metadata, config, component_builder)
test_y = [e.get("intent") for e in test_data.training_examples]
preds = []
for e in test_data.training_examples:
res = interpreter.parse(e.text)
if res.get('intent'):
preds.append(res['intent'].get('name'))
else:
preds.append(None)
logger.info("Intent Evaluation Results")
logger.info("F1-Score: {}".format(f1_score(test_y, preds, average='weighted')))
logger.info("Precision: {}".format(precision_score(test_y, preds, average='weighted')))
logger.info("Accuracy: {}".format(accuracy_score(test_y, preds)))
logger.info("Classification report: \n{}".format(classification_report(test_y, preds)))
cnf_matrix = confusion_matrix(test_y, preds)
plot_intent_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds),
title='Intent Confusion matrix')
plt.show()
return
def start_bot(config):
rasa_section = config['rasa']
telegram_section = config['telegram']
logger.info('Loading model')
model = get_model(rasa_section)
metadata = Metadata.load(model)
logger.info('Loading interpreter')
rasa_config = rasa_section['config']
interpreter = Interpreter.load(metadata, RasaNLUConfig(rasa_config))
logger.info('Preparing Bot')
token = telegram_section['token']
bot = telepot.DelegatorBot(token, [
pave_event_space()(per_chat_id(),
create_open,
KiririnHandler,
interpreter,
config,
timeout=20)
])
MessageLoop(bot).run_as_thread()
logger.info('Listening')
while 1:
time.sleep(10)
def __create_model_store(self):
# Fallback for users that specified the model path as a string and hence only want a single default model.
if type(self.config.server_model_dirs) is Text:
model_dict = {
self.DEFAULT_MODEL_NAME: self.config.server_model_dirs
}
elif self.config.server_model_dirs is None:
model_dict = self.__search_for_models()
else:
model_dict = self.config.server_model_dirs
model_store = {}
for alias, model_path in list(model_dict.items()):
try:
logger.info("Loading model '{}'...".format(model_path))
model_store[alias] = self.__interpreter_for_model(model_path)
except Exception as e:
logger.exception("Failed to load model '{}'. Error: {}".format(
model_path, e))
if not model_store:
meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "")
interpreter = Interpreter.load(meta, self.config,
self.component_builder)
model_store[self.DEFAULT_MODEL_NAME] = interpreter
return model_store
def interpret_query_for(text_to_be_parsed, user_name="User"):
# Example scenario:
# text_to_be_parsed: 'what is the total population?'
metadata = Metadata.load(model_directory)
interpreter = Interpreter.load(model_directory)
interpreted_data = interpreter.parse(text_to_be_parsed.decode('utf-8'))
return get_structured_query(interpreted_data, user_name)
def load(cls,
meta: Dict[Text, Any],
model_dir: Text = None,
model_metadata: Metadata = None,
cached_component: Optional['DucklingHTTPExtractor'] = None,
**kwargs: Any) -> 'DucklingHTTPExtractor':
return cls(meta, model_metadata.get("language"))
def cache_key(cls, model_metadata: Metadata) -> Optional[Text]:
component_meta = model_metadata.for_component(cls.name)
mitie_file = component_meta.get("model", None)
if mitie_file is not None:
return cls.name + "-" + str(os.path.abspath(mitie_file))
else:
return None
def __init__(self, model, config, builder=None, percentage=0.65):
self.model = model
self.metadata = Metadata.load(self.model)
self.config = config
self.percentage = percentage
if builder:
self.interpreter = Interpreter.load(self.metadata,
RasaNLUConfig(config), builder)
else:
self.interpreter = Interpreter.load(self.metadata,
RasaNLUConfig(config))
def _fallback_model(self):
meta = Metadata(
{
"pipeline": [{
"name":
"intent_classifier_keyword",
"class":
utils.module_path_from_object(KeywordIntentClassifier())
}]
}, "")
return Interpreter.create(meta, self._component_builder)
def _search_for_models(self):
prefix = 'model_'
if not self._path or not os.path.isdir(self._path):
meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "")
interpreter = Interpreter.create(meta, self._config, self._component_builder)
models = {'fallback': interpreter}
else:
models = {model: None for model in os.listdir(self._path) if model.startswith(prefix)}
models.update(self._models)
self._models = models
def __init__(self, languages):
utils.check_languages(languages)
self.languages = languages
self.config = RasaNLUConfig(cmdline_args=utils.load_config())
self.query_logger = DataRouter._create_query_logger(self.config['response_log'])
self._metadata = {}
self._interpreter = {}
for lang in languages:
self._metadata[lang] = Metadata.load(utils.get_model_dir(lang))
self._interpreter[lang] = Interpreter.load(self._metadata[lang], self.config)
self.registry = IntentRegistry(self.config['active_apps'])
def read_model_metadata(model_dir, config):
if model_dir is None:
data = DataRouter.default_model_metadata()
return Metadata(data, model_dir)
else:
if not os.path.isabs(model_dir):
model_dir = os.path.join(config['path'], model_dir)
# download model from S3 if needed
if not os.path.isdir(model_dir):
DataRouter.load_model_from_cloud(model_dir, config)
return Metadata.load(model_dir)
def __init__(self, model_directory, config_file=None, lazy_init=False):
from rasa_nlu.model import Interpreter
from rasa_nlu.model import Metadata
from rasa_nlu.config import RasaNLUConfig
self.metadata = Metadata.load(model_directory)
self.lazy_init = lazy_init
self.config_file = config_file
if not lazy_init:
self.interpreter = Interpreter.load(self.metadata,
RasaNLUConfig(config_file))
else:
self.interpreter = None
def create_component(self, component_name, config):
# type: (Text, RasaNLUConfig) -> Component
"""Tries to retrieve a component from the cache, calls `create` to create a new component."""
from rasa_nlu import registry
from rasa_nlu.model import Metadata
try:
component, cache_key = self.__get_cached_component(component_name, Metadata(config.as_dict(), None))
if component is None:
component = registry.create_component_by_name(component_name, config)
self.__add_to_cache(component, cache_key)
return component
except MissingArgumentError as e: # pragma: no cover
raise Exception("Failed to create component '{}'. {}".format(component_name, e))
def load(cls, model_dir: str, model_metadata: Metadata,
cached_component: Component,
**kwargs: Any) -> 'Word2vecKerasIntentClassifier':
meta: Dict[str, Any] = model_metadata.for_component(cls.name)
clf_config_file_path: str = meta['clf_config_file_path']
with open(clf_config_file_path) as f:
clf_config: Dict[str, Any] = json.load(f)
clf_file_path: str = meta['clf_file_path']
clf_model: Model = load_model(clf_file_path)
clf: KerasBaseModel = KerasBaseModel(clf_config, clf_model)
labels_file_path: str = meta['labels_file_path']
with open(labels_file_path) as g:
labels: List[str] = json.load(g)
return cls(meta, clf_config, clf, labels)
def _read_model_metadata(self, model_name):
if model_name is None:
data = Project._default_model_metadata()
return Metadata(data, model_name)
else:
if not os.path.isabs(model_name):
path = os.path.join(self._path, model_name)
else:
path = model_name
# download model from cloud storage if needed and possible
if not os.path.isdir(path):
self._load_model_from_cloud(model_name, path, self._config)
return Metadata.load(path)
def load(cls,
model_dir: Text = None,
model_metadata: Metadata = None,
cached_component: Optional['CRFEntityExtractor'] = None,
**kwargs: Any) -> 'CRFEntityExtractor':
from sklearn.externals import joblib
meta = model_metadata.for_component(cls.name)
file_name = meta.get("classifier_file", CRF_MODEL_FILE_NAME)
model_file = os.path.join(model_dir, file_name)
if os.path.exists(model_file):
ent_tagger = joblib.load(model_file)
return cls(meta, ent_tagger)
else:
return cls(meta)
def load(cls,
model_dir: Text = None,
model_metadata: Metadata = None,
cached_component: Optional['CountVectorsFeaturizer'] = None,
**kwargs: Any) -> 'CountVectorsFeaturizer':
meta = model_metadata.for_component(cls.name)
if model_dir and meta.get("featurizer_file"):
file_name = meta.get("featurizer_file")
featurizer_file = os.path.join(model_dir, file_name)
return utils.pycloud_unpickle(featurizer_file)
else:
logger.warning("Failed to load featurizer. Maybe path {} "
"doesn't exist".format(os.path.abspath(model_dir)))
return CountVectorsFeaturizer(meta)
def create_component(self, component_config: Dict[Text, Any],
cfg: RasaNLUModelConfig) -> Component:
"""Tries to retrieve a component from the cache,
calls `create` to create a new component."""
from rasa_nlu import registry
from rasa_nlu.model import Metadata
try:
component, cache_key = self.__get_cached_component(
component_config, Metadata(cfg.as_dict(), None))
if component is None:
component = registry.create_component_by_config(
component_config, cfg)
self.__add_to_cache(component, cache_key)
return component
except MissingArgumentError as e: # pragma: no cover
raise Exception("Failed to create component `{}`. "
"{}".format(component_config['name'], e))
def __init__(self, model_dir=MODEL_DIR, conf_file=CONFIG_DIR):
# Takes some time,to initialize
from data.intents import INTENTS
self.intents_info = INTENTS
from data.contexts import CONTEXTS
self.contexts_info = CONTEXTS
from data.fallback import RESPONSES
self.fallback_responses = RESPONSES
print("Initializing the model...")
metadata = Metadata.load(model_dir)
interpreter = Interpreter.load(metadata, RasaNLUConfig(conf_file))
print("Ready")
print("")
self.modelInterpreter = interpreter
def load(cls,
model_dir: Text = None,
model_metadata: Metadata = None,
cached_component: Optional['MitieEntityExtractor'] = None,
**kwargs: Any) -> 'MitieEntityExtractor':
import mitie
meta = model_metadata.for_component(cls.name)
file_name = meta.get("classifier_file", MITIE_ENTITY_MODEL_FILE_NAME)
if not file_name:
return cls(meta)
classifier_file = os.path.join(model_dir, file_name)
if os.path.exists(classifier_file):
extractor = mitie.named_entity_extractor(classifier_file)
return cls(meta, extractor)
else:
return cls(meta)
def run_intent_evaluation(config, model_path, component_builder=None):
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import precision_score
from sklearn.utils.multiclass import unique_labels
# get the metadata config from the package data
test_data = load_data(config['data'])
metadata = Metadata.load(model_path)
interpreter = Interpreter.load(metadata, config, component_builder)
test_y = [e.get("intent") for e in test_data.training_examples]
preds = []
for e in test_data.training_examples:
res = interpreter.parse(e.text)
if res.get('intent'):
preds.append(res['intent'].get('name'))
else:
preds.append(None)
logger.info("Intent Evaluation Results")
logger.info("F1-Score: {}".format(
f1_score(test_y, preds, average='weighted')))
logger.info("Precision: {}".format(
precision_score(test_y, preds, average='weighted')))
logger.info("Accuracy: {}".format(accuracy_score(test_y, preds)))
logger.info("Classification report: \n{}".format(
classification_report(test_y, preds)))
cnf_matrix = confusion_matrix(test_y, preds)
plot_intent_confusion_matrix(cnf_matrix,
classes=unique_labels(test_y, preds),
title='Intent Confusion matrix')
plt.show()
return
def classify(self, sent, user):
if self.interpreter is None:
print("train if you got new.. using old one ")
try:
model_directory = self.trainer.persist('./models/')
metadata = Metadata.load(model_directory)
self.interpreter = Interpreter.load(metadata, config)
except:
raise ("Need to train model. No pre-trained model found.")
result = self.interpreter.parse(sent)
probable_module = result["intent"]['name'].split("-")
is_calc = len(probable_module) - 1
print(is_calc)
if is_calc:
print('here')
return eval(probable_module[0])(sent, user)
return best_match(sent, probable_module[0])
def __create_model_store(self):
# Fallback for users that specified the model path as a string and hence only want a single default model.
if type(self.config.server_model_dirs) is unicode or type(
self.config.server_model_dirs) is str:
model_dict = {
self.DEFAULT_MODEL_NAME: self.config.server_model_dirs
}
elif self.config.server_model_dirs is None:
model_dict = self.__search_for_models()
else:
model_dict = self.config.server_model_dirs
# Reuse nlp and featurizers where possible to save memory
cache = {}
model_store = {}
for alias, model_path in model_dict.items():
metadata = DataRouter.read_model_metadata(model_path, self.config)
cache_key = metadata.model_group()
if cache_key in cache:
nlp = cache[cache_key]['nlp']
featurizer = cache[cache_key]['featurizer']
else:
nlp = DataRouter.nlp_for_backend(metadata.backend_name(),
metadata.language_name())
featurizer = DataRouter.featurizer_for_model(metadata, nlp)
cache[cache_key] = {'nlp': nlp, 'featurizer': featurizer}
interpreter = DataRouter.create_interpreter(
metadata, nlp, featurizer)
model_store[alias] = Model(metadata, model_path, interpreter, nlp,
featurizer)
if not model_store:
meta = Metadata({}, "")
interpreter = DataRouter.create_interpreter(meta)
model_store[self.DEFAULT_MODEL_NAME] = Model(meta, "", interpreter)
return model_store
from rasa_nlu.model import Interpreter, Metadata
from rasa_nlu.config import RasaNLUConfig
config = RasaNLUConfig('E:/chat_demo_updated/Data/config_spacy.json')
metadata = Metadata.load('./models/trained')
interpreter = Interpreter.load(metadata, config)
#
#for testing purposes
# result=''
# while(result!='exit'):
# user_says=raw_input("User_says> ")
# result1=interpreter.parse(unicode(user_says))
# # print(result1['entities'][0]['entity'])
# print(result1)
# result=user_says
if detected_lang == 'en':
trans_input = out_list
else:
mid_trans = translator.translate(out_list, src="hi", dest = "ur").text
trans_input = translator.translate(mid_trans, src = "ur", dest = "en").text
print("Translated Input: ", trans_input)
#----- ASSIGN TO RASA_NLU -----#
#Training Time
from rasa_nlu.converters import load_data
from rasa_nlu.config import RasaNLUConfig
from rasa_nlu.components import ComponentBuilder
from rasa_nlu.model import Trainer
builder = ComponentBuilder(use_cache=True) #will cache components between pipelines (where possible)
training_data = load_data('testData.json')
trainer = Trainer(RasaNLUConfig("config_spacy.json"), builder)
trainer.train(training_data)
model_directory = trainer.persist('./models/') #returns the directory the model is stored in
#Prediction Time
from rasa_nlu.model import Metadata, Interpreter
config = RasaNLUConfig("config_spacy.json")
metad = Metadata.load(model_directory) #loads metadata.json
interpreter = Interpreter.load(metad, config, builder)
interpreter.parse(trans_input) #output
def test_builder_load_unknown(component_builder):
with pytest.raises(Exception) as excinfo:
component_builder.load_component("my_made_up_componment", "",
Metadata({}, None))
assert "Unknown component name" in str(excinfo.value)
from rasa_nlu.model import Metadata, Interpreter
from rasa_nlu.config import RasaNLUConfig
import json
from flask import Flask, jsonify, render_template
app = Flask(__name__)
metadata = Metadata.load('/home/sikim/RNLU/models/model_20170403-084112')
interpreter = Interpreter.load(metadata, RasaNLUConfig('/home/sikim/RNLU/config.json'))
@app.route("/index", methods =['GET'])
def index():
return render_template("index.html")
@app.route("/fetchentityintent/<text>", methods = ['GET'])
def fetchentityintent(text):
resp = interpreter.parse(unicode(text))
return jsonify(resp)
if __name__ == "__main__":
app.run(host='0.0.0.0')