def evaluate(self,
data: Text,
project: Optional[Text] = None,
model: Optional[Text] = None) -> Dict[Text, Any]:
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
if project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
model_name = self.project_store[project]._dynamic_load_model(model)
self.project_store[project]._loader_lock.acquire()
try:
if not self.project_store[project]._models.get(model_name):
interpreter = self.project_store[project]. \
_interpreter_for_model(model_name)
self.project_store[project]._models[model_name] = interpreter
finally:
self.project_store[project]._loader_lock.release()
return run_evaluation(
data_path=file_name,
model=self.project_store[project]._models[model_name],
errors_filename=None
)
def test_url_data_format():
data = u"""
{
"rasa_nlu_data": {
"entity_synonyms": [
{
"value": "nyc",
"synonyms": ["New York City", "nyc", "the big apple"]
}
],
"common_examples" : [
{
"text": "show me flights to New York City",
"intent": "unk",
"entities": [
{
"entity": "destination",
"start": 19,
"end": 32,
"value": "NYC"
}
]
}
]
}
}"""
fname = utils.create_temporary_file(data.encode("utf-8"),
suffix="_tmp_training_data.json",
mode="w+b")
data = utils.read_json_file(fname)
assert data is not None
validate_rasa_nlu_data(data)
def dump_to_data_file(data):
if isinstance(data, six.string_types):
data_string = data
else:
data_string = utils.json_to_string(data)
return utils.create_temporary_file(data_string, "_training_data")
def dump_to_data_file(data):
if isinstance(data, six.string_types):
data_string = data
else:
data_string = utils.json_to_string(data)
return utils.create_temporary_file(data_string, "_training_data")
def test_url_data_format():
data = u"""
{
"rasa_nlu_data": {
"entity_synonyms": [
{
"value": "nyc",
"synonyms": ["New York City", "nyc", "the big apple"]
}
],
"common_examples" : [
{
"text": "show me flights to New York City",
"intent": "unk",
"entities": [
{
"entity": "destination",
"start": 19,
"end": 32,
"value": "NYC"
}
]
}
]
}
}"""
fname = utils.create_temporary_file(data.encode("utf-8"),
suffix="_tmp_training_data.json",
mode="w+b")
data = utils.read_json_file(fname)
assert data is not None
validate_rasa_nlu_data(data)
def evaluate(self, data, project=None, model=None):
# type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any]
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
if project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
model_name = self.project_store[project]._dynamic_load_model(model)
self.project_store[project]._loader_lock.acquire()
try:
if not self.project_store[project]._models.get(model_name):
interpreter = self.project_store[project]. \
_interpreter_for_model(model_name)
self.project_store[project]._models[model_name] = interpreter
finally:
self.project_store[project]._loader_lock.release()
return run_evaluation(
data_path=file_name,
model=self.project_store[project]._models[model_name],
errors_filename=None)
def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁💯 👩🏿💻👨🏿💻
- two £ (?u)\\b\\w+\\b f\u00fcr
"""
test_file = utils.create_temporary_file(test_data)
with io.open(test_file, mode='r', encoding="utf-8") as f:
content = f.read()
actual = utils.read_yaml(content)
assert actual["data"][0] == "one 😁💯 👩🏿💻👨🏿💻"
assert actual["data"][1] == "two £ (?u)\\b\\w+\\b für"
def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁💯 👩🏿💻👨🏿💻
- two £ (?u)\\b\\w+\\b f\u00fcr
"""
test_file = utils.create_temporary_file(test_data)
with io.open(test_file, mode='r', encoding="utf-8") as f:
content = f.read()
actual = utils.read_yaml(content)
assert actual["data"][0] == "one 😁💯 👩🏿💻👨🏿💻"
assert actual["data"][1] == "two £ (?u)\\b\\w+\\b für"
def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁
- two £
"""
test_file = utils.create_temporary_file(test_data)
with io.open(test_file, mode='r', encoding="utf-8") as f:
content = f.read()
actual = utils.read_yaml(content)
assert actual["data"][0] == "one 😁"
assert actual["data"][1] == "two £"
def load_data_from_url(url, language='en'):
# type: (Text, Optional[Text]) -> TrainingData
"""Load training data from a URL."""
if not utils.is_url(url):
raise requests.exceptions.InvalidURL(url)
try:
response = requests.get(url)
response.raise_for_status()
temp_data_file = utils.create_temporary_file(response.content)
return _load(temp_data_file, language)
except Exception as e:
logger.warning("Could not retrieve training data "
"from URL:\n{}".format(e))
def download_file_from_url(url: Text) -> Text:
"""Download a story file from a url and persists it into a temp file.
Returns the file path of the temp file that contains the
downloaded content."""
if not nlu_utils.is_url(url):
raise InvalidURL(url)
response = requests.get(url)
response.raise_for_status()
filename = nlu_utils.create_temporary_file(response.content, mode="w+b")
return filename
def load_data_from_url(url, language='en'):
# type: (Text, Optional[Text]) -> TrainingData
"""Load training data from a URL."""
if not utils.is_url(url):
raise requests.exceptions.InvalidURL(url)
try:
response = requests.get(url)
response.raise_for_status()
temp_data_file = utils.create_temporary_file(response.content)
return _load(temp_data_file, language)
except Exception as e:
logger.warning("Could not retrieve training data "
"from URL:\n{}".format(e))
def load_data_from_endpoint(data_endpoint, language='en'):
# type: (EndpointConfig, Optional[Text]) -> TrainingData
"""Load training data from a URL."""
if not utils.is_url(data_endpoint.url):
raise requests.exceptions.InvalidURL(data_endpoint.url)
try:
response = data_endpoint.request("get")
response.raise_for_status()
temp_data_file = utils.create_temporary_file(response.content,
mode="w+b")
return _load(temp_data_file, language)
except Exception as e:
logger.warning("Could not retrieve training data "
"from URL:\n{}".format(e))
async def download_file_from_url(url: Text) -> Text:
"""Download a story file from a url and persists it into a temp file.
Returns the file path of the temp file that contains the
downloaded content."""
from rasa_nlu import utils as nlu_utils
if not nlu_utils.is_url(url):
raise InvalidURL(url)
async with aiohttp.ClientSession() as session:
async with session.get(url, raise_for_status=True) as resp:
filename = nlu_utils.create_temporary_file(await resp.read(),
mode="w+b")
return filename
def download_file_from_url(url):
# type: (Text) -> Text
"""Download a story file from a url and persists it into a temp file.
Returns the file path of the temp file that contains the
downloaded content."""
if not nlu_utils.is_url(url):
raise InvalidURL(url)
response = requests.get(url)
response.raise_for_status()
filename = nlu_utils.create_temporary_file(response.content,
mode="w+b")
return filename
def load_data_from_endpoint(data_endpoint: EndpointConfig,
language: Optional[Text] = 'en') -> 'TrainingData':
"""Load training data from a URL."""
if not utils.is_url(data_endpoint.url):
raise requests.exceptions.InvalidURL(data_endpoint.url)
try:
response = data_endpoint.request("get")
response.raise_for_status()
temp_data_file = utils.create_temporary_file(response.content,
mode="w+b")
training_data = _load(temp_data_file, language)
return training_data
except Exception as e:
logger.warning("Could not retrieve training data "
"from URL:\n{}".format(e))
def evaluate(self, data, project=None, model=None):
# type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any]
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
test_data = load_data(file_name)
if project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
preds_json = self.parse_training_examples(test_data.intent_examples,
project,
model)
predictions = [
{"text": e.text,
"intent": e.data.get("intent"),
"predicted": p.get("intent", {}).get("name"),
"confidence": p.get("intent", {}).get("confidence")}
for e, p in zip(test_data.intent_examples, preds_json)
]
y_true = [e.data.get("intent") for e in test_data.intent_examples]
y_true = clean_intent_labels(y_true)
y_pred = [p.get("intent", {}).get("name") for p in preds_json]
y_pred = clean_intent_labels(y_pred)
report, precision, f1, accuracy = get_evaluation_metrics(y_true,
y_pred)
return {
"intent_evaluation": {
"report": report,
"predictions": predictions,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy}
}
def evaluate(self, data, project=None, model=None):
# type: (Text, Optional[Text], Optional[Text]) -> Dict[Text, Any]
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
test_data = load_data(file_name)
if project not in self.project_store:
raise InvalidProjectError("Project {} could not "
"be found".format(project))
preds_json = self.parse_training_examples(test_data.intent_examples,
project,
model)
predictions = [
{"text": e.text,
"intent": e.data.get("intent"),
"predicted": p.get("intent", {}).get("name"),
"confidence": p.get("intent", {}).get("confidence")}
for e, p in zip(test_data.intent_examples, preds_json)
]
y_true = [e.data.get("intent") for e in test_data.intent_examples]
y_true = clean_intent_labels(y_true)
y_pred = [p.get("intent", {}).get("name") for p in preds_json]
y_pred = clean_intent_labels(y_pred)
report, precision, f1, accuracy = get_evaluation_metrics(y_true,
y_pred)
return {
"intent_evaluation": {
"report": report,
"predictions": predictions,
"precision": precision,
"f1_score": f1,
"accuracy": accuracy}
}
