def test_nlu(model: Text, nlu_data: Text, **kwargs: Dict): from rasa_nlu.test import run_evaluation unpacked_model = get_model(model) nlu_model = os.path.join(unpacked_model, "nlu") kwargs = minimal_kwargs(kwargs, run_evaluation) run_evaluation(nlu_data, nlu_model, **kwargs)
def train_nlu(self, data_path, configs, model_path): training_data = load_data(data_path) trainer = Trainer(config.load(configs)) trainer.train(training_data) model_directory = trainer.persist(model_path, project_name='rasaproject', fixed_model_name='SmallTalk') run_evaluation(data_path, model_directory)
def train_nlu(data_path, configs, model_path): logging.basicConfig(filename=logfile, level=logging.DEBUG) training_data = load_data(data_path) trainer = Trainer(config.load(configs)) trainer.train(training_data) model_directory = trainer.persist(model_path, project_name='current', fixed_model_name='nlu') run_evaluation(data_path, model_directory)
def evaluateModel(pathToData, model_dir): path_to_data = "data/test_data.json" # Create a directory if not exist. if not os.path.exists(model_dir +"/evaluation"): os.mkdir(model_dir +"/evaluation") print("Directory ", "evaluation", " Created ") # save the error file, Confusion matrix image, and histogram file errors_path = model_dir + "/evaluation/errors.json" confmat_path = model_dir + "/evaluation/confmat" intent_hist_path = model_dir + "/evaluation/hist" run_evaluation(path_to_data, model_dir, errors_filename=errors_path, confmat_filename=confmat_path, intent_hist_filename=intent_hist_path)
def evaluate(self, data: Text, project: Optional[Text] = None, model: Optional[Text] = None) -> Dict[Text, Any]: """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) model_name = self.project_store[project]._dynamic_load_model(model) self.project_store[project]._loader_lock.acquire() try: if not self.project_store[project]._models.get(model_name): interpreter = self.project_store[project]. \ _interpreter_for_model(model_name) self.project_store[project]._models[model_name] = interpreter finally: self.project_store[project]._loader_lock.release() return run_evaluation( data_path=file_name, model=self.project_store[project]._models[model_name], errors_filename=None)
def evaluate(self, data: Text, project: Optional[Text] = None, model: Optional[Text] = None) -> Dict[Text, Any]: """Perform a model evaluation.""" project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME model = model or None file_name = utils.create_temporary_file(data, "_training_data") if project not in self.project_store: raise InvalidProjectError("Project {} could not " "be found".format(project)) model_name = self.project_store[project]._dynamic_load_model(model) self.project_store[project]._loader_lock.acquire() try: if not self.project_store[project]._models.get(model_name): interpreter = self.project_store[project]. \ _interpreter_for_model(model_name) self.project_store[project]._models[model_name] = interpreter finally: self.project_store[project]._loader_lock.release() return run_evaluation( data_path=file_name, model=self.project_store[project]._models[model_name], errors_filename=None )
async def evaluate_intents(request: Request): """Evaluate intents against a Rasa NLU model.""" # create `tmpdir` and cast as str for py3.5 compatibility tmpdir = str(tempfile.mkdtemp()) zipped_model_path = os.path.join(tmpdir, 'model.tar.gz') write_request_body_to_file(request, zipped_model_path) model_path, nlu_files = \ await nlu_model_and_evaluation_files_from_archive( zipped_model_path, tmpdir) if len(nlu_files) == 1: data_path = os.path.abspath(nlu_files[0]) try: evaluation = run_evaluation(data_path, model_path) return response.json(evaluation) except ValueError as e: return ErrorResponse( 400, "FailedIntentEvaluation", "Evaluation could not be created. " "Error: {}".format(e)) else: return ErrorResponse( 400, "FailedIntentEvaluation", "NLU evaluation file could not be found. " "This endpoint requires a single file ending " "on `.md` or `.json`.")
def train_nlu(train_path, test_path, configs, model_path): logging.basicConfig(filename=logfile, level=logging.DEBUG) training_data = load_data(train_path) trainer = Trainer(config.load(configs)) trainer.train(training_data) model_directory = trainer.persist(model_path, project_name='current', fixed_model_name='nlu') result = run_evaluation(test_path, model_directory) predictions = result['intent_evaluation']['predictions'] for predict in predictions: print('{}:{}-{}'.format(predict['text'], predict['intent'], predict['confidence'])) print('Acc: {}'.format(result['intent_evaluation']['accuracy'])) print('F1 : {}'.format(result['intent_evaluation']['f1_score'])) print('Pre: {}'.format(result['intent_evaluation']['precision']))
async def test_stack_model_intent_evaluation(tmpdir, trained_stack_model, default_nlu_data): with open(default_nlu_data, 'r') as f: nlu_data = f.read() # add evaluation data to model archive new_model_path = add_evaluation_file_to_model(trained_stack_model, nlu_data, data_format='md') nlu_model_path, nlu_files = \ await nlu_model_and_evaluation_files_from_archive( new_model_path, tmpdir) assert len(nlu_files) == 1 evaluation = run_evaluation(nlu_files[0], nlu_model_path) assert set(evaluation.keys()) == {"intent_evaluation", "entity_evaluation"}
def train_nlu(data_path, configs, model_path): training_data = load_data(data_path) trainer = Trainer(config.load(configs)) trainer.train(training_data) model_directory = trainer.persist(model_path, project_name='current', fixed_model_name='nlu') run_evaluation(data_path, model_directory)
def evaluateNLU(): run_evaluation('./data/data.json', './models/nlu/default/restaurantnlu')
#def run_nlu(): #interpreter = Interpreter.load('./models/nlu/default/mentalnlu',config.load('config.yml')) #print(interpreter.parse(u"I am depressed today.")) #if __name__ == "__main__": #train_nlu('./data/nlu.md','config.yml','./models/nlu') #run_nlu() from rasa_nlu.training_data import load_data from rasa_nlu.config import RasaNLUModelConfig from rasa_nlu.model import Trainer from rasa_nlu import config # loading the nlu training samples training_data = load_data("./data/nlu.md") # trainer to educate our pipeline trainer = Trainer(config.load("config.yml")) # train the model! interpreter = trainer.train(training_data) # store it for future use model_directory = trainer.persist("./models/nlu", fixed_model_name="mentalnlu") #interpreter = Interpreter.load('./models/nlu/default/mentalnlu',config.load('config.yml')) #print(interpreter.parse(u"I am depressed today.")) from rasa_nlu.test import run_evaluation run_evaluation("./data/nlu.md", model_directory)
from rasa_nlu import config import json from rasa_nlu.test import run_evaluation def train(data, config_file, model_dir): training_data = load_data(data) trainer = Trainer(config.load(config_file)) trainer.train(training_data) model_directory = trainer.persist(model_dir, fixed_model_name='nlu_model') return model_directory model_directory = train("data/nlu.md", "config.yml", 'models/nlu') # # # def ask_question(text): # print(text) # interpreter = Interpreter.load(model_directory) # t = interpreter.parse(text) # print(json.dumps(t, indent=2)) # # # ask_question("How do I get to Douglass") result = run_evaluation("./data/nlu.md", model_directory, confmat_filename="trainresult.png") # print(result["intent_evaluation"]['report'])