def dump_to_data_file(data):
if isinstance(data, str):
data_string = data
else:
data_string = utils.json_to_string(data)
return utils.create_temporary_file(data_string, "_training_data")
def evaluate(
self, data: Text, project: Optional[Text] = None, model: Optional[Text] = None
) -> Dict[Text, Any]:
"""Perform a model evaluation."""
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
model = model or None
file_name = utils.create_temporary_file(data, "_training_data")
if project not in self.project_store:
raise InvalidProjectError("Project {} could not be found".format(project))
model_name = self.project_store[project]._dynamic_load_model(model)
self.project_store[project]._loader_lock.acquire()
try:
if not self.project_store[project]._models.get(model_name):
interpreter = self.project_store[project]._interpreter_for_model(
model_name
)
self.project_store[project]._models[model_name] = interpreter
finally:
self.project_store[project]._loader_lock.release()
return run_evaluation(
data_path=file_name,
model=self.project_store[project]._models[model_name],
errors_filename=None,
)
def test_url_data_format():
data = """
{
"rasa_nlu_data": {
"entity_synonyms": [
{
"value": "nyc",
"synonyms": ["New York City", "nyc", "the big apple"]
}
],
"common_examples" : [
{
"text": "show me flights to New York City",
"intent": "unk",
"entities": [
{
"entity": "destination",
"start": 19,
"end": 32,
"value": "NYC"
}
]
}
]
}
}"""
fname = utils.create_temporary_file(data.encode("utf-8"),
suffix="_tmp_training_data.json",
mode="w+b")
data = utils.read_json_file(fname)
assert data is not None
validate_rasa_nlu_data(data)
def evaluate(self,
data: Text,
project: Optional[Text] = None,
model: Optional[Text] = None) -> Deferred:
"""Perform a model evaluation."""
logger.debug("Evaluation request received for "
"project '{}' and model '{}'.".format(project, model))
if self._worker_processes <= self._current_worker_processes:
raise MaxWorkerProcessError
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
data_path = utils.create_temporary_file(data, "_training_data")
if project not in self.project_store:
raise InvalidProjectError("Project '{}' could not "
"be found.".format(project))
model = model or self.project_store[project]._dynamic_load_model(model)
if model == FALLBACK_MODEL_NAME:
raise UnsupportedModelError("No model in project '{}' to "
"evaluate.".format(project))
model_path = os.path.join(self.project_store[project]._path, model)
def evaluation_callback(result):
logger.debug("Evaluation was successful")
self._current_worker_processes -= 1
self.project_store[project].current_worker_processes -= 1
return result
def evaluation_errback(failure):
logger.warning(failure)
self._current_worker_processes -= 1
self.project_store[project].current_worker_processes -= 1
return failure
logger.debug("New evaluation queued.")
self._current_worker_processes += 1
self.project_store[project].current_worker_processes += 1
result = self.pool.submit(run_evaluation,
data_path,
model_path,
errors_filename=None)
result = deferred_from_future(result)
result.addCallback(evaluation_callback)
result.addErrback(evaluation_errback)
return result
def test_emojis_in_tmp_file():
test_data = """
data:
- one 😁💯 👩🏿💻👨🏿💻
- two £ (?u)\\b\\w+\\b f\u00fcr
"""
test_file = utils.create_temporary_file(test_data)
with io.open(test_file, mode="r", encoding="utf-8") as f:
content = f.read()
actual = rasa.utils.io.read_yaml(content)
assert actual["data"][0] == "one 😁💯 👩🏿💻👨🏿💻"
assert actual["data"][1] == "two £ (?u)\\b\\w+\\b für"
async def evaluate(
self, data: Text, project: Optional[Text] = None, model: Optional[Text] = None
) -> Dict:
"""Perform a model evaluation."""
logger.debug(
"Evaluation request received for "
"project '{}' and model '{}'.".format(project, model)
)
if self._worker_processes <= self._current_worker_processes:
raise MaxWorkerProcessError
project = project or RasaNLUModelConfig.DEFAULT_PROJECT_NAME
data_path = utils.create_temporary_file(data, "_training_data")
if project not in self.project_store:
raise InvalidProjectError(
"Project '{}' could not be found.".format(project)
)
model = model or self.project_store[project]._dynamic_load_model(model)
if model == FALLBACK_MODEL_NAME:
raise UnsupportedModelError(
"No model in project '{}' to evaluate.".format(project)
)
model_path = os.path.join(self.project_store[project]._path, model)
logger.debug("New evaluation queued.")
self._current_worker_processes += 1
self.project_store[project].current_worker_processes += 1
loop = asyncio.get_event_loop()
task = loop.run_in_executor(self.pool, run_evaluation, data_path, model_path)
try:
return await task
except Exception as e:
logger.warning(e)
self.project_store[project].status = STATUS_FAILED
self.project_store[project].error_message = str(e)
raise
finally:
self._current_worker_processes -= 1
self.project_store[project].current_worker_processes -= 1
async def download_file_from_url(url: Text) -> Text:
"""Download a story file from a url and persists it into a temp file.
Returns the file path of the temp file that contains the
downloaded content."""
from rasa.nlu import utils as nlu_utils
if not nlu_utils.is_url(url):
raise InvalidURL(url)
async with aiohttp.ClientSession() as session:
async with session.get(url, raise_for_status=True) as resp:
filename = nlu_utils.create_temporary_file(await resp.read(), mode="w+b")
return filename
async def load_data_from_endpoint(
data_endpoint: EndpointConfig, language: Optional[Text] = "en"
) -> "TrainingData":
"""Load training data from a URL."""
if not utils.is_url(data_endpoint.url):
raise requests.exceptions.InvalidURL(data_endpoint.url)
try:
response = await data_endpoint.request("get")
response.raise_for_status()
temp_data_file = utils.create_temporary_file(response.content, mode="w+b")
training_data = _load(temp_data_file, language)
return training_data
except Exception as e:
logger.warning("Could not retrieve training data from URL:\n{}".format(e))
