def persist(self, path, persistor=None, model_name=None):
# type: (Text, Optional[Persistor], Text) -> Text
"""Persist all components of the pipeline to the passed path. Returns the directory of the persited model."""
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
metadata = {
"language": self.config["language"],
"pipeline": [component.name for component in self.pipeline],
}
if model_name is None:
dir_name = os.path.join(path, "model_" + timestamp)
else:
dir_name = os.path.join(path, model_name)
create_dir(dir_name)
if self.training_data:
metadata.update(self.training_data.persist(dir_name))
for component in self.pipeline:
update = component.persist(dir_name)
if update:
metadata.update(update)
Metadata(metadata, dir_name).persist(dir_name)
if persistor is not None:
persistor.save_tar(dir_name)
logger.info("Successfully saved model into '{}'".format(
os.path.abspath(dir_name)))
return dir_name
def persist(self, path, persistor=None, model_name=None):
# type: (Text, Optional[Persistor], Text) -> Text
"""Persist all components of the pipeline to the passed path. Returns the directory of the persited model."""
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
metadata = {
"language": self.config["language"],
"pipeline": [component.name for component in self.pipeline],
}
if model_name is None:
dir_name = os.path.join(path, "model_" + timestamp)
else:
dir_name = os.path.join(path, model_name)
create_dir(dir_name)
if self.training_data:
metadata.update(self.training_data.persist(dir_name))
for component in self.pipeline:
update = component.persist(dir_name)
if update:
metadata.update(update)
Metadata(metadata, dir_name).persist(dir_name)
if persistor is not None:
persistor.save_tar(dir_name)
logger.info("Successfully saved model into '{}'".format(os.path.abspath(dir_name)))
return dir_name
def run_evaluation(data_path, model,
report_folder=None,
successes_filename=None,
errors_filename='errors.json',
confmat_filename=None,
intent_hist_filename=None,
component_builder=None): # pragma: no cover
"""Evaluate intent classification and entity extraction."""
# get the metadata config from the package data
if isinstance(model, Interpreter):
interpreter = model
else:
interpreter = Interpreter.load(model, component_builder)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
extractors = get_entity_extractors(interpreter)
entity_predictions, tokens = get_entity_predictions(interpreter,
test_data)
if duckling_extractors.intersection(extractors):
entity_predictions = remove_duckling_entities(entity_predictions)
extractors = remove_duckling_extractors(extractors)
result = {
"intent_evaluation": None,
"entity_evaluation": None
}
if report_folder:
utils.create_dir(report_folder)
if extractors:
print(extractors)
entity_targets = get_entity_targets(test_data)
logger.info("Entity evaluation results:")
result['entity_evaluation'] = evaluate_entities(entity_targets,
entity_predictions,
tokens,
extractors,
report_folder)
if is_intent_classifier_present(interpreter):
intent_targets = get_intent_targets(test_data)
intent_results = get_intent_predictions(
intent_targets, interpreter, test_data)
logger.info("Intent evaluation results:")
result['intent_evaluation'] = evaluate_intents(intent_results,
report_folder,
successes_filename,
errors_filename,
confmat_filename,
intent_hist_filename)
return result
def test_intent_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
utils.create_dir(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search",
"I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet",
"hello", 0.98765)]
result = evaluate_intents(intent_results,
report_folder,
successes_filename=None,
errors_filename=None,
confmat_filename=None,
intent_hist_filename=None)
report = json.loads(utils.read_file(report_filename))
greet_results = {"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1}
prediction = {'text': 'hello',
'intent': 'greet',
'predicted': 'greet',
'confidence': 0.98765}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
def test_intent_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
report_filename = os.path.join(report_folder, "intent_report.json")
utils.create_dir(report_folder)
intent_results = [
IntentEvaluationResult("", "restaurant_search",
"I am hungry", 0.12345),
IntentEvaluationResult("greet", "greet",
"hello", 0.98765)]
result = evaluate_intents(intent_results,
report_folder,
successes_filename=None,
errors_filename=None,
confmat_filename=None,
intent_hist_filename=None)
report = json.loads(utils.read_file(report_filename))
greet_results = {"precision": 1.0,
"recall": 1.0,
"f1-score": 1.0,
"support": 1}
prediction = {'text': 'hello',
'intent': 'greet',
'predicted': 'greet',
'confidence': 0.98765}
assert len(report.keys()) == 4
assert report["greet"] == greet_results
assert result["predictions"][0] == prediction
def run_evaluation(data_path, model,
report_folder=None,
successes_filename=None,
errors_filename='errors.json',
confmat_filename=None,
intent_hist_filename=None,
component_builder=None): # pragma: no cover
"""Evaluate intent classification and entity extraction."""
# get the metadata config from the package data
if isinstance(model, Interpreter):
interpreter = model
else:
interpreter = Interpreter.load(model, component_builder)
test_data = training_data.load_data(data_path,
interpreter.model_metadata.language)
extractors = get_entity_extractors(interpreter)
entity_predictions, tokens = get_entity_predictions(interpreter,
test_data)
if duckling_extractors.intersection(extractors):
entity_predictions = remove_duckling_entities(entity_predictions)
extractors = remove_duckling_extractors(extractors)
result = {
"intent_evaluation": None,
"entity_evaluation": None
}
if report_folder:
utils.create_dir(report_folder)
if is_intent_classifier_present(interpreter):
intent_targets = get_intent_targets(test_data)
intent_results = get_intent_predictions(
intent_targets, interpreter, test_data)
logger.info("Intent evaluation results:")
result['intent_evaluation'] = evaluate_intents(intent_results,
report_folder,
successes_filename,
errors_filename,
confmat_filename,
intent_hist_filename)
if extractors:
entity_targets = get_entity_targets(test_data)
logger.info("Entity evaluation results:")
result['entity_evaluation'] = evaluate_entities(entity_targets,
entity_predictions,
tokens,
extractors,
report_folder)
return result
def main():
from rasa.core.agent import Agent
from rasa.core.interpreter import NaturalLanguageInterpreter
from rasa.core.utils import (AvailableEndpoints, set_default_subparser)
import rasa_nlu.utils as nlu_utils
import rasa.core.cli
from rasa.core import utils
loop = asyncio.get_event_loop()
# Running as standalone python application
arg_parser = create_argument_parser()
set_default_subparser(arg_parser, 'default')
cmdline_arguments = arg_parser.parse_args()
logging.basicConfig(level=cmdline_arguments.loglevel)
_endpoints = AvailableEndpoints.read_endpoints(cmdline_arguments.endpoints)
if cmdline_arguments.output:
nlu_utils.create_dir(cmdline_arguments.output)
if not cmdline_arguments.core:
raise ValueError("you must provide a core model directory to evaluate "
"using -d / --core")
if cmdline_arguments.mode == 'default':
_interpreter = NaturalLanguageInterpreter.create(
cmdline_arguments.nlu, _endpoints.nlu)
_agent = Agent.load(cmdline_arguments.core, interpreter=_interpreter)
stories = loop.run_until_complete(
rasa.core.cli.train.stories_from_cli_args(cmdline_arguments))
loop.run_until_complete(
test(stories, _agent, cmdline_arguments.max_stories,
cmdline_arguments.output,
cmdline_arguments.fail_on_prediction_errors,
cmdline_arguments.e2e))
elif cmdline_arguments.mode == 'compare':
compare(cmdline_arguments.core, cmdline_arguments.stories,
cmdline_arguments.output)
story_n_path = os.path.join(cmdline_arguments.core, 'num_stories.json')
number_of_stories = utils.read_json_file(story_n_path)
plot_curve(cmdline_arguments.output, number_of_stories)
logger.info("Finished evaluation")
def persist(self,
path,
persistor=None,
project_name=None,
fixed_model_name=None):
# type: (Text, Optional[Persistor], Text) -> Text
"""Persist all components of the pipeline to the passed path.
Returns the directory of the persisted model."""
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
metadata = {
"language":
self.config["language"],
"pipeline": [
utils.module_path_from_object(component)
for component in self.pipeline
],
"model_fasttext":
self.config["model_fasttext"],
}
if project_name is None:
project_name = "default"
if fixed_model_name:
model_name = fixed_model_name
else:
model_name = "model_" + timestamp
dir_name = os.path.join(path, project_name, model_name)
create_dir(dir_name)
if self.training_data:
metadata.update(self.training_data.persist(dir_name))
for component in self.pipeline:
update = component.persist(dir_name)
if update:
metadata.update(update)
Metadata(metadata, dir_name).persist(dir_name)
if persistor is not None:
persistor.persist(dir_name, model_name, project_name)
logger.info("Successfully saved model into "
"'{}'".format(os.path.abspath(dir_name)))
return dir_name
def persist(self,
path: Text,
persistor: Optional[Persistor] = None,
project_name: Text = None,
fixed_model_name: Text = None) -> Text:
"""Persist all components of the pipeline to the passed path.
Returns the directory of the persisted model."""
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
metadata = {
"language": self.config["language"],
"pipeline": [],
}
if project_name is None:
project_name = "default"
if fixed_model_name:
model_name = fixed_model_name
else:
model_name = "model_" + timestamp
path = make_path_absolute(path)
dir_name = os.path.join(path, project_name, model_name)
create_dir(dir_name)
if self.training_data:
metadata.update(self.training_data.persist(dir_name))
for i, component in enumerate(self.pipeline):
file_name = self._file_name(i, component.name)
update = component.persist(file_name, dir_name)
component_meta = component.component_config
if update:
component_meta.update(update)
component_meta["class"] = utils.module_path_from_object(component)
metadata["pipeline"].append(component_meta)
Metadata(metadata, dir_name).persist(dir_name)
if persistor is not None:
persistor.persist(dir_name, model_name, project_name)
logger.info("Successfully saved model into "
"'{}'".format(os.path.abspath(dir_name)))
return dir_name
def test_core(model: Text,
stories: Text,
endpoints: Text = None,
output: Text = DEFAULT_RESULTS_PATH,
model_path: Text = None,
**kwargs: Dict):
import rasa.core.test
import rasa.core.utils as core_utils
from rasa_nlu import utils as nlu_utils
from rasa.model import get_model
from rasa.core.interpreter import NaturalLanguageInterpreter
from rasa.core.agent import Agent
_endpoints = core_utils.AvailableEndpoints.read_endpoints(endpoints)
if output:
nlu_utils.create_dir(output)
if os.path.isfile(model):
model_path = get_model(model)
if model_path:
# Single model: Normal evaluation
loop = asyncio.get_event_loop()
model_path = get_model(model)
core_path, nlu_path = get_model_subdirectories(model_path)
_interpreter = NaturalLanguageInterpreter.create(
nlu_path, _endpoints.nlu)
_agent = Agent.load(core_path, interpreter=_interpreter)
kwargs = minimal_kwargs(kwargs, rasa.core.test)
loop.run_until_complete(
rasa.core.test(stories, _agent, out_directory=output, **kwargs))
else:
from rasa.core.test import compare, plot_curve
compare(model, stories, output)
story_n_path = os.path.join(model, 'num_stories.json')
number_of_stories = core_utils.read_json_file(story_n_path)
plot_curve(output, number_of_stories)
def persist(self, path, persistor=None, project_name=None,
fixed_model_name=None):
# type: (Text, Optional[Persistor], Text) -> Text
"""Persist all components of the pipeline to the passed path.
Returns the directory of the persisted model."""
timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
metadata = {
"language": self.config["language"],
"pipeline": [],
}
if project_name is None:
project_name = "default"
if fixed_model_name:
model_name = fixed_model_name
else:
model_name = "model_" + timestamp
path = config.make_path_absolute(path)
dir_name = os.path.join(path, project_name, model_name)
create_dir(dir_name)
if self.training_data:
metadata.update(self.training_data.persist(dir_name))
for component in self.pipeline:
update = component.persist(dir_name)
component_meta = component.component_config
if update:
component_meta.update(update)
component_meta["class"] = utils.module_path_from_object(component)
metadata["pipeline"].append(component_meta)
Metadata(metadata, dir_name).persist(dir_name)
if persistor is not None:
persistor.persist(dir_name, model_name, project_name)
logger.info("Successfully saved model into "
"'{}'".format(os.path.abspath(dir_name)))
return dir_name
def test_entity_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
mock_extractors = ["A", "B"]
report_filename_a = os.path.join(report_folder, "A_report.json")
report_filename_b = os.path.join(report_folder, "B_report.json")
utils.create_dir(report_folder)
result = evaluate_entities([EN_targets], [EN_predicted], [EN_tokens],
mock_extractors, report_folder)
report_a = json.loads(utils.read_file(report_filename_a))
report_b = json.loads(utils.read_file(report_filename_b))
assert len(report_a) == 8
assert report_a["datetime"]["support"] == 1.0
assert report_b["macro avg"]["recall"] == 0.2
assert result["A"]["accuracy"] == 0.75
def test_entity_evaluation_report(tmpdir_factory):
path = tmpdir_factory.mktemp("evaluation").strpath
report_folder = os.path.join(path, "reports")
mock_extractors = ["A", "B"]
report_filename_a = os.path.join(report_folder, "A_report.json")
report_filename_b = os.path.join(report_folder, "B_report.json")
utils.create_dir(report_folder)
result = evaluate_entities([EN_targets],
[EN_predicted],
[EN_tokens],
mock_extractors,
report_folder)
report_a = json.loads(utils.read_file(report_filename_a))
report_b = json.loads(utils.read_file(report_filename_b))
assert len(report_a) == 8
assert report_a["datetime"]["support"] == 1.0
assert report_b["macro avg"]["recall"] == 0.2
assert result["A"]["accuracy"] == 0.75
def test_creation_of_existing_dir(tmpdir):
assert create_dir(
tmpdir.strpath) is None # makes sure there is no exception
plt.savefig(os.path.join(output, 'model_comparison_graph.pdf'),
format='pdf')
plt.show()
if __name__ == '__main__':
# Running as standalone python application
arg_parser = create_argument_parser()
set_default_subparser(arg_parser, 'default')
cmdline_arguments = arg_parser.parse_args()
logging.basicConfig(level=cmdline_arguments.loglevel)
_endpoints = AvailableEndpoints.read_endpoints(cmdline_arguments.endpoints)
if cmdline_arguments.output:
nlu_utils.create_dir(cmdline_arguments.output)
if not cmdline_arguments.core:
raise ValueError("you must provide a core model directory to evaluate "
"using -d / --core")
if cmdline_arguments.mode == 'default':
_interpreter = NaturalLanguageInterpreter.create(
cmdline_arguments.nlu, _endpoints.nlu)
_agent = Agent.load(cmdline_arguments.core, interpreter=_interpreter)
stories = cli.stories_from_cli_args(cmdline_arguments)
run_story_evaluation(stories, _agent, cmdline_arguments.max_stories,
cmdline_arguments.output,
def test_creation_of_existing_dir(tmpdir):
# makes sure there is no exception
assert create_dir(tmpdir.strpath) is None
