def run_evaluation(data_path, model_path, component_builder=None): # pragma: no cover """Evaluate intent classification and entity extraction.""" # get the metadata config from the package data interpreter = Interpreter.load(model_path, component_builder) test_data = training_data.load_data(data_path, interpreter.model_metadata.language) extractors = get_entity_extractors(interpreter) entity_predictions, tokens = get_entity_predictions(interpreter, test_data) if duckling_extractors.intersection(extractors): entity_predictions = remove_duckling_entities(entity_predictions) extractors = remove_duckling_extractors(extractors) if is_intent_classifier_present(interpreter): intent_targets = get_intent_targets(test_data) intent_predictions = get_intent_predictions(interpreter, test_data) logger.info("Intent evaluation results:") evaluate_intents(intent_targets, intent_predictions) if extractors: entity_targets = get_entity_targets(test_data) logger.info("Entity evaluation results:") evaluate_entities(entity_targets, entity_predictions, tokens, extractors)
def run_intent_evaluation(config, model_path, component_builder=None): from sklearn.metrics import accuracy_score from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from sklearn.metrics import f1_score from sklearn.metrics import precision_score from sklearn.utils.multiclass import unique_labels # get the metadata config from the package data test_data = load_data(config['data']) metadata = Metadata.load(model_path) interpreter = Interpreter.load(metadata, config, component_builder) test_y = [e.get("intent") for e in test_data.training_examples] preds = [] for e in test_data.training_examples: res = interpreter.parse(e.text) if res.get('intent'): preds.append(res['intent'].get('name')) else: preds.append(None) logger.info("Intent Evaluation Results") logger.info("F1-Score: {}".format(f1_score(test_y, preds, average='weighted'))) logger.info("Precision: {}".format(precision_score(test_y, preds, average='weighted'))) logger.info("Accuracy: {}".format(accuracy_score(test_y, preds))) logger.info("Classification report: \n{}".format(classification_report(test_y, preds))) cnf_matrix = confusion_matrix(test_y, preds) plot_intent_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds), title='Intent Confusion matrix') plt.show() return
def run_intent_evaluation(config, model_path, component_builder=None): from sklearn.metrics import confusion_matrix from sklearn.utils.multiclass import unique_labels # get the metadata config from the package data test_data = load_data(config['data']) interpreter = Interpreter.load(model_path, config, component_builder) test_y = [e.get("intent") for e in test_data.training_examples] preds = [] for e in test_data.training_examples: res = interpreter.parse(e.text) if res.get('intent'): preds.append(res['intent'].get('name')) else: preds.append(None) log_evaluation_table(test_y, preds) cnf_matrix = confusion_matrix(test_y, preds) plot_confusion_matrix(cnf_matrix, classes=unique_labels(test_y, preds), title='Intent Confusion matrix') plt.show() return
def run_evaluation(data_path, model, report_folder=None, successes_filename=None, errors_filename='errors.json', confmat_filename=None, intent_hist_filename=None, component_builder=None): # pragma: no cover """Evaluate intent classification and entity extraction.""" # get the metadata config from the package data if isinstance(model, Interpreter): interpreter = model else: interpreter = Interpreter.load(model, component_builder) test_data = training_data.load_data(data_path, interpreter.model_metadata.language) extractors = get_entity_extractors(interpreter) entity_predictions, tokens = get_entity_predictions(interpreter, test_data) if duckling_extractors.intersection(extractors): entity_predictions = remove_duckling_entities(entity_predictions) extractors = remove_duckling_extractors(extractors) result = { "intent_evaluation": None, "entity_evaluation": None } if report_folder: utils.create_dir(report_folder) if is_intent_classifier_present(interpreter): intent_targets = get_intent_targets(test_data) intent_results = get_intent_predictions( intent_targets, interpreter, test_data) logger.info("Intent evaluation results:") result['intent_evaluation'] = evaluate_intents(intent_results, report_folder, successes_filename, errors_filename, confmat_filename, intent_hist_filename) if extractors: entity_targets = get_entity_targets(test_data) logger.info("Entity evaluation results:") result['entity_evaluation'] = evaluate_entities(entity_targets, entity_predictions, tokens, extractors, report_folder) return result
def run_cmdline(model_path, component_builder=None): interpreter = Interpreter.load(model_path, component_builder) logger.info("NLU model loaded. Type a message and " "press enter to parse it.") while True: text = input().strip() r = interpreter.parse(text) print(json.dumps(r, indent=2)) logger.info("Next message:")
def test_train_with_empty_data(language, pipeline, component_builder, tmpdir): _config = RasaNLUModelConfig({"pipeline": pipeline, "language": language}) trainer = Trainer(_config, component_builder) trainer.train(TrainingData()) persistor = create_persistor(_config) persisted_path = trainer.persist(tmpdir.strpath, persistor, project_name="my_project") loaded = Interpreter.load(persisted_path, component_builder) assert loaded.pipeline assert loaded.parse("hello") is not None assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model_noents(language, pipeline, component_builder, tmpdir): _config = RasaNLUModelConfig({"pipeline": pipeline, "language": language}) (trained, _, persisted_path) = train.do_train( _config, path=tmpdir.strpath, data="./data/test/demo-rasa-noents.json", component_builder=component_builder) assert trained.pipeline loaded = Interpreter.load(persisted_path, component_builder) assert loaded.pipeline assert loaded.parse("hello") is not None assert loaded.parse("Hello today is Monday, again!") is not None
def test_train_model(pipeline_template, component_builder, tmpdir): _config = utilities.base_test_conf(pipeline_template) (trained, _, persisted_path) = train.do_train( _config, path=tmpdir.strpath, data=DEFAULT_DATA_PATH, component_builder=component_builder) assert trained.pipeline loaded = Interpreter.load(persisted_path, component_builder) assert loaded.pipeline assert loaded.parse("hello") is not None assert loaded.parse("Hello today is Monday, again!") is not None
def __init__(self, languages): utils.check_languages(languages) self.languages = languages self.config = RasaNLUConfig(cmdline_args=utils.load_config()) self.query_logger = DataRouter._create_query_logger(self.config['response_log']) self._metadata = {} self._interpreter = {} for lang in languages: self._metadata[lang] = Metadata.load(utils.get_model_dir(lang)) self._interpreter[lang] = Interpreter.load(self._metadata[lang], self.config) self.registry = IntentRegistry(self.config['active_apps'])
def run_cv_evaluation(data, n_folds, nlu_config): from sklearn import metrics from sklearn.model_selection import StratifiedKFold from collections import defaultdict # type: (List[rasa_nlu.training_data.Message], int, RasaNLUConfig) -> Dict[Text, List[float]] """Stratified cross validation on data :param data: list of rasa_nlu.training_data.Message objects :param n_folds: integer, number of cv folds :param nlu_config: nlu config file :return: dictionary with key, list structure, where each entry in list corresponds to the relevant result for one fold """ trainer = Trainer(nlu_config) results = defaultdict(list) y_true = [e.get("intent") for e in data] skf = StratifiedKFold(n_splits=n_folds, random_state=11, shuffle=True) counter = 1 logger.info("Evaluation started") for train_index, test_index in skf.split(data, y_true): train = [data[i] for i in train_index] test = [data[i] for i in test_index] logger.debug("Fold: {}".format(counter)) logger.debug("Training ...") trainer.train(TrainingData(training_examples=train)) model_directory = trainer.persist("projects/") # Returns the directory the model is stored in logger.debug("Evaluation ...") interpreter = Interpreter.load(model_directory, nlu_config) test_y = [e.get("intent") for e in test] preds = [] for e in test: res = interpreter.parse(e.text) if res.get('intent'): preds.append(res['intent'].get('name')) else: preds.append(None) # compute fold metrics results["Accuracy"].append(metrics.accuracy_score(test_y, preds)) results["F1-score"].append(metrics.f1_score(test_y, preds, average='weighted')) results["Precision"] = metrics.precision_score(test_y, preds, average='weighted') # increase fold counter counter += 1 return dict(results)
def test_train_model_on_test_pipelines(language, pipeline, component_builder, tmpdir): _config = RasaNLUModelConfig({"pipeline": pipeline, "language": language}) (trained, _, persisted_path) = train.do_train( _config, path=tmpdir.strpath, data=DEFAULT_DATA_PATH, component_builder=component_builder) assert trained.pipeline loaded = Interpreter.load(persisted_path, component_builder) assert loaded.pipeline assert loaded.parse("hello") is not None assert loaded.parse("Hello today is Monday, again!") is not None
def run_cmdline(model_path, component_builder=None): interpreter = Interpreter.load(model_path, component_builder) logger.info("NLU model loaded. Type a message and " "press enter to parse it.") while True: text = input().strip() if six.PY2: # in python 2 input doesn't return unicode values text = text.decode("utf-8") r = interpreter.parse(text) print(json.dumps(r, indent=2)) logger.info("Next message:")
def test_random_seed(component_builder, tmpdir): """test if train result is the same for two runs of tf embedding""" _config = utilities.base_test_conf("supervised_embeddings") # set fixed random seed to 1 _config.set_component_attr(5, random_seed=1) # first run (trained_a, _, persisted_path_a) = train( _config, path=tmpdir.strpath + "_a", data=DEFAULT_DATA_PATH, component_builder=component_builder) # second run (trained_b, _, persisted_path_b) = train( _config, path=tmpdir.strpath + "_b", data=DEFAULT_DATA_PATH, component_builder=component_builder) loaded_a = Interpreter.load(persisted_path_a, component_builder) loaded_b = Interpreter.load(persisted_path_b, component_builder) result_a = loaded_a.parse("hello")["intent"]["confidence"] result_b = loaded_b.parse("hello")["intent"]["confidence"] assert result_a == result_b
def run_evaluation(config, model_path, component_builder=None): # pragma: no cover """Evaluate intent classification and entity extraction.""" # get the metadata config from the package data test_data = training_data.load_data(config['data'], config['language']) interpreter = Interpreter.load(model_path, config, component_builder) intent_targets, entity_targets = get_targets(test_data) intent_predictions, entity_predictions, tokens = get_predictions(interpreter, test_data) extractors = get_entity_extractors(interpreter) if extractors.intersection(duckling_extractors): entity_predictions = patch_duckling_entities(entity_predictions) extractors = patch_duckling_extractors(interpreter, extractors) evaluate_intents(intent_targets, intent_predictions) evaluate_entities(entity_targets, entity_predictions, tokens, extractors)
def __create_model_store(self): # Fallback for users that specified the model path as a string and hence only want a single default model. if type(self.config.server_model_dirs) is Text: model_dict = {self.DEFAULT_MODEL_NAME: self.config.server_model_dirs} elif self.config.server_model_dirs is None: model_dict = self.__search_for_models() else: model_dict = self.config.server_model_dirs model_store = {} for alias, model_path in list(model_dict.items()): try: logger.info("Loading model '{}'...".format(model_path)) model_store[alias] = self.__interpreter_for_model(model_path) except Exception as e: logger.exception("Failed to load model '{}'. Error: {}".format(model_path, e)) if not model_store: meta = Metadata({"pipeline": ["intent_classifier_keyword"]}, "") interpreter = Interpreter.load(meta, self.config, self.component_builder) model_store[self.DEFAULT_MODEL_NAME] = interpreter return model_store
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/current') print(interpreter.parse(u"I am sad, plased send me a picture of a dog"))
def interpreter_for(component_builder, data, path, config): (trained, _, path) = train(config, data, path, component_builder=component_builder) interpreter = Interpreter.load(path, component_builder) return interpreter
def _load_interpreter(self): from rasa_nlu.model import Interpreter self.interpreter = Interpreter.load(self.model_directory)
def load_interpreter_for_model(config, persisted_path, component_builder): metadata = DataRouter.read_model_metadata(persisted_path, config) return Interpreter.load(metadata, config, component_builder)
def __interpreter_for_model(self, model_path): metadata = DataRouter.read_model_metadata(model_path, self.config) return Interpreter.load(metadata, self.config, self.component_builder)
import sys from rasa_nlu.components import ComponentBuilder from rasa_nlu.model import Metadata, Interpreter builder = ComponentBuilder(use_cache = True) #laedt NLU-Interpreter und wartet auf Input if __name__ == '__main__': print(sys.argv) interpreter = Interpreter.load(sys.argv[1],builder); print("Model geladen") while(True): sentence = input() print(interpreter.parse(sentence))
from rasa_nlu.model import Interpreter import os rasa_model_path = os.path.abspath("rasa/models/20200321-174355.tar.gz") print(rasa_model_path) interpreter = Interpreter.load(rasa_model_path) def rasa_output(text): message = str(text).strip() result = interpreter.parse(message) return result rasa_output("hello")
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/weathernlu', RasaNLUConfig('config_spacy.json')) print(interpreter.parse("I am planning my holiday to Lithuania. I wonder what is the weather out there."))
from rasa_nlu.config import RasaNLUModelConfig from rasa_nlu.model import Trainer from rasa_nlu.model import Metadata, Interpreter import json app = Flask(__name__) import warnings warnings.filterwarnings('ignore') @app.route('/') def index(): return render_template('index.html') interpreter = Interpreter.load('./models/nlu/default/restaurantnlu') @app.route('/nlu_parsing', methods=['POST']) def transform(): if request.headers['Content-Type'] == 'application/json': query = request.json.get("utterance") results = interpreter.parse(query) js = json.dumps(results) resp = Response(js, status=200, mimetype='application/json') return resp if __name__ == '__main__': app.run(debug=True)
#!/usr/bin/env python3 import argparse import pathlib import re from typing import List, Dict, Union from flask import Flask, jsonify, request, wrappers from flask_cors import CORS from rasa_nlu.model import Interpreter MODEL_DIR = (pathlib.Path(__file__).parent / 'projects' / 'default' / 'model_20180613-130746').absolute() INTERPRETER = Interpreter.load(str(MODEL_DIR)) # Initialize the server app = Flask(__name__) CORS(app) def get_whatsapp_messages(content: str) -> List[Dict[str, str]]: """ Args: content: All messages in a string Returns: [{'name': 'matched_string', ...}] """ pattern = r'(?P<time>[0-9\/, :AMP]+) \- (?P<name>[a-zA-Z0-9+ ]+): (?P<text>.+)' regex_ = re.compile(pattern, re.DOTALL)
if __name__ == '__main__': args = docopt(__doc__) source_file_path = args['<source>'] target_language = args['--target-language'] output_file_path = args['--output'] # import code dictionary for target language if target_language == 'python': from languages.python import code_dict elif target_language == 'c++': from languages.c_plus_plus import code_dict elif target_language == 'java': from languages.java import code_dict # load NLU model interpreter = Interpreter.load(MODEL_DIR + listdir(MODEL_DIR)[0]) # parse each line from the input file and store its transpiled code in the # `code` list code = [code_dict['default_code']] current_indent_level = 0 prev_indent_level = 0 with open(source_file_path, 'r') as source: source_lines = source.readlines() for line in source_lines: print(line, end='') current_indent_level = counts_tabs(line) if current_indent_level < prev_indent_level: code.append('\t' * current_indent_level + code_dict['end_block']) if not line.isspace(): # line is not "blank"
def run_nlu(): interpreter = Interpreter.load('./models/default/current/') print(interpreter.parse("Can I book an appointment please?"))
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/robosoc') print(interpreter.parse("what is the reputation of 10.54.1.3"))
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/chatter') pprint.pprint(interpreter.parse("CCU"))
def run_nlu(sentence, nlu_model_path='./models/nlu/default/current'): interpreter = Interpreter.load(nlu_model_path) pprint(interpreter.parse(sentence))
from rasa_nlu.model import Interpreter # where model_directory points to the model folder interpreter = Interpreter.load("../projects/default/model_20180812-135546") output = interpreter.parse(u"looking for chinese food") print(output)
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/restaurantnlu', builder) print( interpreter.parse( "south indian restaurants in agra costs less than 500"))
from rasa_nlu.model import Interpreter CONFIDENCE_THRESHOLD = 0.5 FALLBACK_INTENT = 'N/A' interpreter = Interpreter.load('./models/nlu/default/test') def extract_structured_data(query): result = interpreter.parse(query) data = {'input': query, 'intent': FALLBACK_INTENT, 'entities': []} if result['intent']['confidence'] > CONFIDENCE_THRESHOLD: data['intent'] = result['intent']['name'] for entity in result['entities']: if entity['confidence'] > CONFIDENCE_THRESHOLD: data['entities'].append({ 'name': entity['entity'], 'value': entity['value'] }) return data if '__main__' == __name__: queries = [ { 'input': 'roll a die', 'intent': 'dice', 'entities': [] }, { 'input': 'songs by linkin park',
from flask import Flask, render_template, request from config import DevConfig import pickle import jieba import numpy import json import os import shutil import pymysql import datetime import xmljson import xml.etree.ElementTree as ET import requests import qa_inference2 as qa_inference interpreter_proj = Interpreter.load("./intent_models/proj2") interpreter_bill = Interpreter.load("./intent_models/bill2") interpreter_mb1 = Interpreter.load("./intent_models/mb1") interpreter_mb2 = Interpreter.load("./intent_models/mb2") interpreter_mba = Interpreter.load("./intent_models/mba3v1") sess = tf.Session() set_session(sess) graph = tf.get_default_graph() w2vmodel = Word2Vec.load('./LSTM_model4/aptg_20180309wiki_model.bin') lstmmodel = load_model('./LSTM_model4/sentiment_test_aptg.h5') # zzz = np.ones((1, 200)) # lstmmodel.predict(zzz) # label_dic = {'合約查詢': 0, '帳務查詢': 1, '魔速方塊1.0': 2, '魔速方塊2.0': 3} label_dic = {'合約查詢': 0, '帳務查詢': 1, '魔速方塊': 2} gensim_dict = Dictionary()
def find_dataset(self, threshold=0.70): """ runs the model for entity extraction :return: citation and mention lists """ model_directory = "project/models/model_20190116-214229" all_interpreter = Interpreter.load(model_directory) dataset_citations = [] dataset_mentions = [] count_jel_methods = 0 for _, row in self.pub_df.iterrows(): if len(row['jel_method']) > 0: count_jel_methods += 1 try: print(row['pdf_file_name']) distinct_datasets = { } # { id1 : [('data1', score1), ('data2', score2)] , id2 : ..} entities, jel_field = self.find_entities_jel_field( all_interpreter, threshold, row['text_file_name']) with open('project/additional_files/pub_field.json', 'r+') as file: pub_field_dict = json.load(file) for ent in entities: belongsToField = True if len(entities) == 1 and ent['data_set'].lower( ) == 'sage': continue for _, rowd in self.dataset_vocab.iterrows(): if ent['data_set'] in ( mention for mention in rowd['mention_list'] ) or ent['data_set'] == rowd['name']: if not self.dataset_belongsTo_field( pub_field_dict[ row['pdf_file_name'].replace( '.pdf', '').strip()], jel_field, rowd["subjects"], rowd["description"]): belongsToField = False break if rowd['data_set_id'] not in distinct_datasets: distinct_datasets[rowd['data_set_id']] = [ (ent['data_set'], ent['confidence']) ] else: distinct_datasets[rowd['data_set_id']].append( (ent['data_set'], ent['confidence'])) break if not belongsToField: print(f"not field: {ent['data_set']}") continue result_dict = {} result_dict["publication_id"] = row["publication_id"] result_dict["score"] = ent['confidence'] result_dict["mention"] = ent['data_set'] dataset_mentions.append(result_dict) for id in distinct_datasets: result_dict = {} result_dict["publication_id"] = row["publication_id"] result_dict["data_set_id"] = id result_dict["score"] = max(distinct_datasets[id], key=itemgetter(1))[1] result_dict["mention_list"] = [ i[0] for i in distinct_datasets[id] ] dataset_citations.append(result_dict) except Exception as e: logging.exception(e) continue # write results from rasa to intermediate files with open("project/additional_files/data_set_citations_rasa.json", "w+") as write_file: json.dump(dataset_citations, write_file, indent=4, ensure_ascii=False) with open("project/additional_files/data_set_mentions_rasa.json", "w+") as write_file: json.dump(dataset_mentions, write_file, indent=4, ensure_ascii=False) return dataset_citations, dataset_mentions
from rasa_nlu.model import Interpreter import json interpreter = Interpreter.load("./models/current/nlu") messages = [ "the password could be bananas", "username foo, password: Zzzsa23", "the weather is just fine" ] for message in messages: print("Evaluating: %s" % message) result = interpreter.parse(message) print("Most likely intent: %s" % result['intent']) print("all data: %s" % json.dumps(result, indent=2))
def run_nlu(model_directory): #interpreter = Interpreter.load('./models/nlu/default/customernlu', RasaNLUModelConfig('config_spacy.yml')) interpreter = Interpreter.load(model_directory) print(interpreter.parse(u"I am planning my to order an 829 router. How much does it cost?"))
#!/usr/bin/env python from rasa_nlu.model import Interpreter import json interpreter = Interpreter.load("./models/current/nlu") def print_intent(msg): result = interpreter.parse(msg) print(json.dumps(result, indent=2)) print_intent("Can you book me a dental checkup appointment for 10'o clock") print_intent("I need an appointment for eye checkup at 10'o clock")
#!/usr/bin/env python3 import os import rospy from std_msgs.msg import String from roboga_nlu.srv import Nlu, NluResponse from rasa_nlu.model import Interpreter if __name__ == "__main__": directory = os.path.dirname(os.path.realpath(__file__)) rospy.init_node('rasa_nlu', anonymous=True) interpreter = Interpreter.load(directory + "/rasa/models/nlu") def handler(req): result = interpreter.parse(req.data) message = str(result['text']) intent = str(result["intent"].get("name")) entities = [ "{{\"entity\":\"{0}\", \"value\":\"{1}\"}}".format( x.get("entity"), x.get("value")) for x in result.get("entities") ] return NluResponse(message=message, intent=intent, entities=entities) pub = rospy.Service("zordon/nlu", Nlu, handler) rospy.spin()
def __init__(self, model_dir): from rasa_nlu.model import Metadata, Interpreter self.interpreter = Interpreter.load(model_dir, RasaNLUConfig("mobile_nlu_model_config.json")) self._items = {}
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/weathernlu',) print(interpreter.parse("I am planning my holiday to Lithuania. I wonder what is the weather out there."))
def run_nlu(): query = request.args['q'] print(query) interpreter = Interpreter.load('./models/nlu/default/stagebot', RasaNLUConfig('config_sapcy.json')) response = interpreter.parse(query) return jsonify({"Engine Status" : response}), 200
def run_nlu(): interpreter = Interpreter.load('./models/nlu/default/Restaurant_NPSR', builder) print(interpreter.parse("show some chinese restaurant in Delhi"))